diff --git a/.claude/skills/setup-perftrace/SKILL.md b/.claude/skills/setup-perftrace/SKILL.md index 3ff573ccf1..546a7e660e 100644 --- a/.claude/skills/setup-perftrace/SKILL.md +++ b/.claude/skills/setup-perftrace/SKILL.md @@ -387,7 +387,7 @@ file they live in. Verify against the branch rather than trusting this table if | `PATCH /v6/groupmembership` | `experimentUserService.updateGroupMembership` | controller | | `PATCH /v6/workinggroup` | `experimentUserService.updateWorkingGroup` | controller | | `PATCH /v6/useraliases` | `experimentUserService.setAliasesForUser(aliases=N)` | controller | -| `POST /v6/reward` | `moocletRewardsService.sendReward` | controller | +| `POST /v6/reward` | `thompsonSamplingRewardService.acceptReward` | controller | | `POST /v6/mark` | `experimentAssignmentService.markExperimentPoint` | controller | | " | 8 spans inside `markExperimentPoint` — `previewUserService.findOneFromCache`, `getCachedExperiments`, `checkUserOrGroupIsGloballyExcluded`, `experimentLevelExclusionInclusion`, `monitoredDecisionPointRepository.findOne`, `saveGroupExclusionDoc`, `updateEnrollmentExclusionDocumentsAndCheckEndingCriteria`, `monitoredDecisionPointRepository.saveRawJson` | `services/ExperimentAssignmentService.ts` | | `POST /v6/assign` | `formatAssignments` (sync) | controller | diff --git a/CLAUDE.md b/CLAUDE.md index b9efa9cda6..185ca222d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,3 +65,130 @@ Environment files: copy `packages/backend/.env.example` to `packages/backend/.en ## Package Versions All packages are versioned together (currently v6.5.0). Keep versions in sync when bumping. + +--- + +## Native Thompson Sampling Migration + +**Goal:** Remove the third-party MoocLet integration and replace it with a native TypeScript Thompson Sampling implementation. Retain UI visually as-is. No use of the term "mooclet" anywhere in the new codebase. + +**Reference:** MoocLet engine source at `~/Code/mooclet-engine` (read-only reference — do NOT copy logic from it). + +### Algorithm parameters carried over (renamed) + +| Old (MoocLet) | New | Purpose | +|---|---|---| +| `prior` | `prior` | Beta(α, β) priors per condition | +| `uniform_threshold` | `warmupThreshold` | Uniform random during cold-start | +| `batch_size` | `batchSize` | Batch posterior updates | +| `tspostdiff_thresh` | `minimumDrawDifference` | Fall back to uniform when arms converge | + +Binary rewards only (SUCCESS=1, FAILURE=0). No `max_rating`/`min_rating`. + +### Progress + +**Phase 1 — Core algorithm (backend, pure functions)** +- [x] `ThompsonSamplingService` with `selectCondition()` — no DB or HTTP dependencies +- [x] Unit tests (14 passing) — warmup, priors, multi-arm, thresholds + +**Phase 2 — Data model** +- [x] New entity: `ThompsonSamplingExperimentConfig` (replaces `MoocletExperimentRef`) +- [x] New entity: `ConditionPosteriorState` per condition (stores current α/β) +- [x] New entity: `ThompsonSamplingReward` (raw reward events — audit trail + recalculation) +- [x] Repositories: `ThompsonSamplingExperimentConfigRepository`, `ConditionPosteriorStateRepository` +- [x] Add `ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING = 'thompson_sampling'` to `upgrade_types` +- [x] DB migration: consolidated into `1788362726319-nativeThompsonSampling` (see Phase 6 note) + +**Phase 3 — Assignment integration** +- [x] Wire `ThompsonSamplingService` into `ExperimentAssignmentService` — `assignThompsonSampling()` method handles THOMPSON_SAMPLING algorithm +- [x] `ThompsonSamplingRewardService` replaces `MoocletRewardsService` — stores `ThompsonSamplingReward` + increments `ConditionPosteriorState` counts +- [x] `POST /v6/reward` rewired to `ThompsonSamplingRewardService.recordReward()`; mooclet guard removed + +**Phase 4 — Experiment CRUD** +- [x] `ThompsonSamplingExperimentCrudService` — `createConfig`, `updateConfig`, `syncConditions` +- [x] `ExperimentController` create/update/delete handlers use native TS flow; mooclet routing removed +- [x] `ExperimentAssignmentService` mooclet branch removed; `handleEnrollCondition` mooclet path removed +- [x] `ImportExportService` mooclet routing removed from bulk create and export +- [x] Deleted: `MoocletExperimentService`, `MoocletDataService`, `MoocletRewardsService` + their tests/mocks + +**Phase 5 — Frontend** +- [x] `ThompsonSamplingHelperService` replaces `MoocletExperimentHelperService` across all components +- [x] NgRx state: `moocletPolicyParameters` → `thompsonSamplingConfig`; `ThompsonSamplingConfigDTO` added to model +- [x] Form fields renamed: `batch_size`→`batchSize`, `uniform_threshold`→`warmupThreshold`, `tspostdiff_thresh`→`minimumDrawDifference` +- [x] Algorithm check: `MOOCLET_TS_CONFIGURABLE` → `THOMPSON_SAMPLING` throughout +- [x] Dead rewards effect + data service method removed; `mooclet-rewards` API endpoint removed +- [x] `moocletToggle` feature flag removed from all env files +- [x] Deleted: `mooclet-helper.service.ts`, `mooclet-helper.service.spec.ts`; rewards effect tests removed + +**Phase 6 — Cleanup** +- [x] DB migration `1788362726319-nativeThompsonSampling`: data-migrates `ts_configurable` → `thompson_sampling`, drops mooclet tables, removes `ts_configurable` from enum, creates the three Thompson Sampling tables (including `pendingSuccessCount`/`pendingFailureCount`/`pendingTotalCount` for batched posterior updates) and bootstraps config/posterior rows. This single migration replaces what were originally four separate ones (`thompsonSamplingEntities`, `cleanupMoocletEntities`, `bootstrapThompsonSamplingConfigs`, and a later `addPendingRewardCountsToConditionPosteriorState`) — none had shipped to dev or been applied to any real database, so they were squashed rather than kept as sequential steps. Its timestamp is deliberately ordered after `1783627365221-experimentPrecomputedSegment`, the newest migration actually merged to dev as of this branch's last rebase — check `git log origin/dev -- packages/backend/src/database/migrations` before merging in case dev has moved further; this migration's timestamp must stay the newest of the two. +- [x] Removed `MOOCLETS_*` env vars from `env.ts` and `.env.example` +- [x] `upgrade_types`: removed `Mooclet/` directory, `MOOCLET_TS_CONFIGURABLE` enum value, `SUPPORTED_MOOCLET_ALGORITHMS`; `Prior`, `BinaryRewardAllowedValue`, `ExperimentRewardsSummary` moved to `Experiment/interfaces.ts` +- [x] Deleted: mooclet models, repository, error class, types file +- [x] Removed mooclet validation from `ExperimentDTO`, mooclet inject from `ExperimentService`, mooclet error cases from `ErrorHandlerMiddleware` + +### Post-migration fixes + +- **Reward error type**: `ThompsonSamplingRewardService.throwConflictError` sets `(error as any).type = SERVER_ERROR.ASSIGNMENT_ERROR` explicitly. Without this, a plain `HttpError(409)` falls through to the middleware's httpCode switch where 409 → `SERVER_ERROR.DUPLICATE_KEY` (whose string is the unrelated feature flag message). Added `case SERVER_ERROR.ASSIGNMENT_ERROR` to `ErrorHandlerMiddleware` outer switch. + +- **thompsonSamplingConfig missing in API responses**: `getSingleExperiment`, `create`, and `update` all return `ExperimentDTO` without loading `thompsonSamplingConfig` (it's in a separate table). Added `attachThompsonSamplingConfig()` private method to `ExperimentController` that queries the config for THOMPSON_SAMPLING experiments and attaches it; called on all three response paths. Also added `getConfigForExperiment()` to `ThompsonSamplingExperimentCrudService`. (This private method was later moved onto `ThompsonSamplingExperimentCrudService` itself as the public `attachConfigToExperiment()` — see "Bulk import/batch-create never created a Thompson Sampling config" below — so it could be reused outside the controller.) + +- **Form/overview labels still showing old terms**: Updated `packages/frontend/projects/upgrade/src/assets/i18n/en.json` — `"uniform-threshold.label.text"` → "Warmup Threshold", `"tspostdiff-thresh.label.text"` → "Minimum Draw Difference" (with updated hints). The translation keys themselves are unchanged; only the values were updated. + +- **`thompsonSamplingConfig` request validation was missing**: `ExperimentDTO.thompsonSamplingConfig` was an unvalidated inline object type, so negative/NaN priors or thresholds could reach `ThompsonSamplingService`'s Beta/Gamma sampling math. Replaced with a `ThompsonSamplingConfigValidator` class (`@ValidateNested()` + `@Type()`), bounds mirrored from `ThompsonSamplingHelperService.getFieldValidators()`/`getPriorFieldValidators()` on the frontend: `warmupThreshold`/`batchSize` are integers in `1`–`1,000,000` (`warmupThreshold` allows `0`), `minimumDrawDifference` is `0`–`1`, and `priors` (a `Record`) is validated via a custom `IsThompsonSamplingPriorsRecord` decorator requiring integer `success`/`failure` in `1`–`1,000,000` — strictly positive, since they feed `alpha`/`beta` into `sampleGamma()`. + +- **`warmupThreshold` was gated on the wrong count**: `assignThompsonSampling()` computed `totalEnrollments` as `sum(ConditionPosteriorState.totalCount)`, but `totalCount` only increments on reward (`ThompsonSamplingRewardService`), never on assignment — so the variable name lied about what it measured. Renamed to `totalRewardCount` throughout (`ThompsonSamplingService.ts`, `ExperimentAssignmentService.ts`, `ThompsonSamplingExperimentConfig.ts`). This is also the semantically correct thing to gate on: the posteriors (`alpha`/`beta`) only move when rewards arrive, so warmup should measure how much reward evidence has accumulated, not how many users were assigned — gating on assignment count would exit warmup while posteriors are still sitting at the prior, or (in the low-latency-reward case) too early relative to actual evidence. The MoocLet reference engine gated on assignment count (`variable__name="version"` in `policies.py`), but that behavior was never a requirement here — CLAUDE.md already says not to copy logic from that reference — and gating on rewards is the better design regardless. + +- **`batchSize` wired in**: was stored on `ThompsonSamplingExperimentConfig` and exposed in the UI form but never consumed anywhere. `ConditionPosteriorState` gained `pendingSuccessCount`/`pendingFailureCount`/`pendingTotalCount` columns; `ThompsonSamplingRewardService.applyOrBufferReward()` now buffers rewards there and only folds them into `successCount`/`failureCount`/`totalCount` (the values that drive the posteriors used for sampling) once `batchSize` observations have accumulated. `batchSize` unset or `≤1` applies immediately (matches pre-batching behavior). The raw `ThompsonSamplingReward` audit row is always written regardless of batching — nothing is lost, batching only delays when a reward affects assignment. + +- **`failureCount`/`pendingFailureCount` are stored, not derived**: `ConditionPosteriorState` now stores both counts explicitly rather than computing failures as `totalCount - successCount` (and `pendingFailureCount` as `pendingTotalCount - pendingSuccessCount`) at every call site. All three prior read sites were updated to read the column directly: `ThompsonSamplingExperimentCrudService.getRewardsSummary()` (`failures`/`beta`), `ThompsonSamplingService.selectCondition()` (`beta`, via a new `failureCount` field on `ConditionRewardSummary`), and `ExperimentAssignmentService.assignThompsonSampling()` (populates `ConditionRewardSummary.failureCount` from `state.failureCount` when building reward summaries for assignment). + +- **`batchSize` was gated per-condition instead of per-experiment**: `applyOrBufferReward()` originally checked `pendingTotalCount` on only the one `ConditionPosteriorState` row the incoming reward belonged to, so `batchSize` behaved as a per-condition threshold — e.g. with `batchSize=5`, 4 rewards on condition A and 1 on condition B would never flush, since neither condition individually reached 5. `batchSize` is meant to pace how often posteriors move for the experiment as a whole, and a reward for any condition is evidence toward that same shared cadence. Fixed by summing `pendingTotalCount` across every `ConditionPosteriorState` row for the experiment (`ConditionPosteriorStateRepository.findByConfigId(state.configId)`) after buffering the incoming reward, and — once that sum reaches `batchSize` — flushing every condition's pending buffer (not just the one that tipped it over), so a low-volume condition still gets its pending counts folded in as soon as the shared batch closes. + +- **`warmupThreshold` undercounted rewards sitting in an unflushed batch**: `ExperimentAssignmentService.assignThompsonSampling()` computed `totalRewardCount` as `sum(ConditionPosteriorState.totalCount)` across conditions, which is correct only when `batchSize` is unset/≤1 (immediate apply). Once batching is active, a reward is "collected" (persisted to `ThompsonSamplingReward`, buffered in `pendingTotalCount`) before it's folded into `totalCount` — so with a large `batchSize`, warmup could stay active far longer than the actual evidence collected would justify, since pending rewards across all conditions weren't counted at all. Fixed by summing `state.totalCount + state.pendingTotalCount` across conditions, consistent with the reward-evidence semantics the earlier `warmupThreshold` fix (above) already established: warmup should track how much reward evidence has actually arrived, not how much of it has been flushed into the posteriors yet. + +- **`POST /v6/reward` is now fire-and-forget, with the config lookup cached**: this endpoint sat on the client's response path doing 8-10 sequential, uncached DB round trips (config lookup, enrollment lookup, audit insert, posterior increments, and — when a batch closes — a flush per condition) even though nothing in the client SDK waits on the result to make a UI decision. `ThompsonSamplingRewardService.recordReward()` was split into `acceptReward()` (public, synchronous — returns a "received and is being processed" receipt immediately) and `processReward()` (private, does the actual work in the background via `acceptReward().catch(...)`, never awaited by the controller). A failure that used to become a `409`/`500` HTTP response is now only logged: `logAndAbort()` (renamed from `throwConflictError()`) logs and throws an internal `RewardProcessingAborted` sentinel purely for control flow, which `acceptReward()`'s catch recognizes and does not re-log (an error of any other type gets one generic "Unexpected error processing..." log line). The `/v6/reward` swagger doc's `200` response and description were updated to describe the receipt/async-logging contract, and its `409`/`500` response entries were removed since nothing throws to the HTTP layer here anymore. Separately, `findConfigById`/`findConfigByDecisionPoint` now go through `CacheService.wrap()` under a new `CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX` (bucketed under the existing `experiments` TTL/refresh-threshold config in `CacheService`'s `PREFIX_CATEGORY` map) — the same `wrap()`-based pattern `ExperimentService.getCachedValidExperiments()` already uses, since `warmupThreshold`/`minimumDrawDifference`/`batchSize` and the joined `experiment.state` only change on an admin edit, far less often than rewards arrive. `ThompsonSamplingExperimentCrudService.createConfig()`/`updateConfig()` call a new `invalidateConfigCache()` (`cacheService.resetPrefixCache(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX)`) after writing — a full-prefix reset rather than a single-key delete, because the decision-point-keyed cache entries embed the same config fields and there's no cheap way to know which `context:site:target` keys reference a given experiment (same blunt-but-safe approach `ExperimentService.updateList()` already takes with `EXPERIMENT_KEY_PREFIX` for the analogous problem). `syncConditions()` does not invalidate, since it only touches `conditionPosteriorStates`, which these cached config objects don't include. One accepted gap: `ExperimentService.updateState()` (experiment start/stop) does not invalidate this cache, so a just-stopped experiment can still accept a reward for up to the `experiments` bucket's TTL — the same TTL-bounded staleness `getCachedValidExperiments()` already tolerates for the assignment path. + +- **`minimumDrawDifference` verified against MoocLet's `tspostdiff_thresh`**: confirmed the full plumbing is live (frontend form → `ThompsonSamplingConfigValidator` (`0`–`1`) → `ThompsonSamplingExperimentConfig.minimumDrawDifference` → `ExperimentAssignmentService.assignThompsonSampling()` → `ThompsonSamplingService.selectCondition()`'s fallback check) and matches the reference policy's intent. `ts_postdiff_sample`/`thompson_sampling_postdiff` in `policies.py` draw one Beta sample per arm, and if `abs(draw_1 - draw_2) < tspostdiff_thresh`, fall back to uniform random across all arms instead of picking the highest draw — a hedge against picking a "winner" whose lead over its closest rival is really just sampling noise. But that reference implementation is hardcoded for exactly 2 arms (`thompson_sampling_postdiff`'s own docstring: "Assumes only 2 versions"; `ts_postdiff_sample` reuses the same logic unchanged, indexing `list(versions_dict.values())[0]`/`[1]` regardless of how many conditions exist) — with 3+ arms it silently compares whichever two conditions happen to be first in dict order, not the two that are actually closest, and its `else` branch re-draws every arm from scratch for the real pick rather than reusing the draws the diff check already made. `selectCondition()` doesn't have either problem: it draws once per condition, sorts, and compares `draws[0]` vs `draws[1]` (the actual top two by sampled value, however many conditions there are) before reusing that same top draw as the pick — the correct N-arm generalization of the same idea, not a port of the 2-arm-specific code. Added a test (`ThompsonSamplingService.test.ts` → "compares the top two draws by value, not the first two conditions by list order") that would fail under a MoocLet-style hardcoded-first-two comparison but passes under the current top-two-by-value implementation. + +- **Bulk import/batch-create never created a Thompson Sampling config**: `ImportExportService.addBulkExperiments()` (the shared path behind both `/experiments/import` and `/experiments/batch`) called `experimentService.create()` directly with no reference to `ThompsonSamplingExperimentCrudService` — config/posterior-state creation existed only in `ExperimentController.create()`/`update()`. An imported or batch-created `THOMPSON_SAMPLING` experiment got an `Experiment` row but no `ThompsonSamplingExperimentConfig`/`ConditionPosteriorState` rows, so `assignThompsonSampling()` would log an error and return `undefined` for every user, forever. Fixed by moving the per-call-site `if (assignmentAlgorithm === THOMPSON_SAMPLING) {...}` gating (previously duplicated across the controller's `create()`/`update()`/`one()`) into three orchestration methods on `ThompsonSamplingExperimentCrudService`: `createConfigIfApplicable()`, `syncConfigIfApplicable()`, and `attachConfigToExperiment()` (which replaced the controller's private `attachThompsonSamplingConfig()`). `ImportExportService.addBulkExperiments()` and `exportExperiment()` now call these too, so import, batch-create, and export get the same config handling as the single-experiment `POST`/`PUT`/`GET` endpoints. Reward data can't leak through this path regardless of caller: `ThompsonSamplingConfigParams`/`createConfig()` have no field for `successCount`/`failureCount`, so posterior state always starts at Beta(priorSuccess, priorFailure) with zero counts, whatever the source experiment (e.g. one round-tripped through export) had accumulated. + +- **`ConditionRewardSummary.conditionCode` actually held a `conditionId`**: every caller populated it with a condition UUID, never an actual (nullable) `conditionCode`. Renamed the field to `conditionId`, and renamed `ThompsonSamplingService.selectCondition()`'s `conditionCodes` parameter (and its private `uniformRandom()` helper) to `conditionIds`/`conditionId` throughout, so the algorithm-facing types now say what they actually hold. `ThompsonSamplingExperimentCrudService.getRewardsSummary()`'s own `conditionCode`/`code` fields are unrelated and untouched — those hold (or fall back to) a real display-facing condition code, a different concept from the algorithm's internal `conditionId` key. + +- **Beta posterior formula and priors-record construction were duplicated**: `alpha = prior.success + successCount` / `beta = prior.failure + failureCount` was written independently in both `ThompsonSamplingService.selectCondition()` and `ThompsonSamplingExperimentCrudService.getRewardsSummary()`; extracted to `ThompsonSamplingService.computePosterior()`, used by both. The `{ [conditionId]: {success, failure} }` priors-record construction was likewise duplicated between `ExperimentAssignmentService.assignThompsonSampling()` and `ThompsonSamplingExperimentCrudService.attachConfigToExperiment()`; extracted to `ThompsonSamplingService.buildPriorsRecord()`. + +- **`getRewardsSummary()` built a query directly, violating this package's own layering rule** (`packages/backend/CLAUDE.md`: "repositories own ALL query-building; services never build queries directly"): moved the `createQueryBuilder` call (joining `conditionPosteriorStates` and `condition`) into a new `ThompsonSamplingExperimentConfigRepository.findByExperimentIdWithConditions()` method. + +- **Misplaced JSDoc**: the doc comment describing `syncConditions()`'s add/remove-rows behavior was sitting above `getRewardsSummary()` (which is a read-only aggregation with no side effects). Moved to `syncConditions()`; `getRewardsSummary()` got its own accurate doc. + +- **`ThompsonSamplingRewardService.applyOrBufferReward()` mixed concerns and duplicated increment logic**: split into `applyOrBufferReward()` (buffers or immediately applies one reward) and `flushIfBatchReady()` (the experiment-wide pending-count check and flush across every condition). The immediate-apply path (`batchSize` unset/≤1) now routes through the same `flushPendingRewards()` a real batch flush uses, instead of a second hand-rolled copy of the `totalCount`/`successCount`/`failureCount` increment sequence. + +- **Frontend: duplicate `isThompsonSamplingExperiment()` reimplementation**: `experiment-conditions-section-card.component.ts` and `enrollment-condition-expandable-row.component.ts` each defined a local `isThompsonSamplingExperiment(experiment)` that reimplemented the check via `thompsonSamplingHelperService.isThompsonSamplingAlgorithm(experiment?.assignmentAlgorithm)` instead of calling `ThompsonSamplingHelperService.isThompsonSamplingExperiment(experiment)` — an existing method of the same name/purpose. Both local methods now delegate to it directly (kept as thin per-component wrappers only because their templates already call them by that name in several places). + +- **Frontend: duplicated adaptive-weight tooltip translation key**: the `'experiments.details.conditions.weight-adaptive-tooltip.text'` literal was duplicated 4 times across `experiment-conditions-table.component.html` and `enrollment-condition-expandable-row.component.html`. Extracted to `THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY` in `experiments.model.ts`, referenced from both components' `.ts` files and bound in the templates instead of the literal string. The two Angular Material table structures themselves stay separate (one is a static `matColumnDef` table, the other a dynamic per-key loop over `displayedColumns`), so this only removed the duplicated literal, not the surrounding markup — the `matTooltip`/`matTooltipDisabled`/`matTooltipPosition` attribute repetition is this codebase's established idiom for every tooltip, adaptive-weight or not. + +- **Extensibility analysis, and the low-risk pieces of it that were implemented**: a follow-up analysis (`adaptive-algorithm-extensibility-analysis.md`) scoped what to change now vs. wait on, given the expectation of at most 2-3 adaptive algorithms ever, not a general plugin framework. Two "do now" items: + - **Backend config CRUD dispatch (done)**: `ThompsonSamplingExperimentCrudService`'s three self-gated methods (`createConfigIfApplicable`, `syncConfigIfApplicable`, `attachConfigToExperiment`) were extracted into an `AdaptiveExperimentConfigService` interface (`src/api/services/AdaptiveExperimentConfigService.ts`). A new `AdaptiveExperimentConfigDispatcherService` holds an array of implementations (today just Thompson Sampling's) and loops over it for each of the three methods; `ExperimentController` (`one()`, `create()`, `update()`) and `ImportExportService` (`addBulkExperiments()`, `exportExperiment()`) now call the dispatcher instead of `ThompsonSamplingExperimentCrudService` directly. `ExperimentController.thompsonSamplingCrudService` is still injected and used directly only for `GET /experiments/rewards/:id` (`getRewardsSummary()`), which isn't part of the three-method interface and stays Thompson-Sampling-specific. Adding algorithm #2's config service means adding it to the array in `AdaptiveExperimentConfigDispatcherService`'s constructor — no call-site changes. + - **Frontend swappable config form (already satisfied, no change needed)**: the analysis recommended extracting a standalone Thompson-Sampling-specific form component with a small `[existingConfig]`/`(configChange)`/`(validityChange)`-style contract, swapped in by a `@switch` on `assignmentAlgorithm`. Checked `upsert-experiment-modal.component.ts`/`.html` and found this already exists — `TsConfigurablePolicyParametersFormComponent` (`.../upsert-experiment-modal/ts-configurable-policy-parameters-form/`) is a standalone component with exactly that contract (`@Input existingPolicyParams`/`disabled`, `@Output parametersChange`/`validationChange`/`formChanged`), hosted behind a single `@if (assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING)` in the parent modal. A single `@if` is simpler than `@switch` for one case and converts trivially to `@switch`/`@else if` when algorithm #2 needs its own form component — not worth doing preemptively. + - **Left alone, per the analysis**: the reward pipeline split (waits for a second reward-consuming algorithm to know its real shape), the conditions-table column-swapping, renaming the Thompson-Sampling-named tables/summary shape to something generic, and any dynamic component/plugin registry. Same reasoning as the original "no shared interface" note below still applies to the *assignment/reward* seam — only the config-CRUD seam was formalized. +- **Deliberately not fixed — no shared interface for the assignment/reward seam**: the only seam for a *second* adaptive algorithm's assignment and reward handling is still a hardcoded `if (assignmentAlgorithm === THOMPSON_SAMPLING)` check in `ExperimentAssignmentService`, `ExperimentDTO`, and the reward endpoint (the config-CRUD seam described above is now the exception). Left unresolved on purpose — it's a larger architectural change, out of scope for this cleanup pass, and worth designing deliberately once a second algorithm is actually on the table rather than guessing its shape now. + +- **`/mark` trusts the client-reported condition for Thompson Sampling, instead of replaying the algorithm**: `updateEnrollmentExclusion()` used to route `THOMPSON_SAMPLING` (like `RANDOM`) through `assignExperiment()` → `assignThompsonSampling()` for any not-yet-enrolled user — a fresh, non-deterministic Beta-sampling draw (unseeded `Math.random()` in `sampleGamma()`/`sampleBeta()`, plus posteriors that shift as other users' rewards arrive between calls). That's safe for `RANDOM`/`STRATIFIED_RANDOM_SAMPLING`, which are pure functions of `(experimentId, userId, weights)` and always reproduce the same answer — but for Thompson Sampling it meant the condition persisted at `/mark` could differ from the condition `/assign` had already returned to the client, silently mis-recording which condition a user actually saw (and therefore mis-attributing whatever reward they later generate). Fixed by adding `THOMPSON_SAMPLING` alongside the existing `STRATIFIED_RANDOM_SAMPLING` special case in both the group and individual branches: look up the client-reported `condition` string against `experiment.conditions` by `conditionCode` and trust it directly, the same way stratified random and within-subjects already do, rather than calling `assignExperiment()`. The "already enrolled" case is unaffected either way — `conditionAssigned` is only ever persisted when `!individualEnrollment`/`!groupEnrollment`, so this only changes behavior for the first mark that creates the enrollment. + +- **Adaptive algorithms excluded from `/batch-assign`**: `BatchAssignController` → `ExperimentAssignmentService.getBatchExperimentConditions()` filters out `THOMPSON_SAMPLING` experiments before assigning (a one-line `.filter()` on `assignmentAlgorithm`, with a comment at the filter site). Reasoning: unlike `/assign`/`/mark`, this endpoint neither persists an enrollment nor is paired with any follow-up call, so an adaptive draw here would be a live, non-deterministic bandit sample disconnected from reward learning and from whatever `/assign` or `/mark` would separately say for the same user at another time — there's no coherent product meaning for it. `/batch-assign` also has no current callers and a known-separate code path from `/assign` (per this repo's own note that only `/assign` produces the `experimentSegmentInclusion` SQL — `/batch-assign` isn't exercised by the load test), so rather than design the right seam now (e.g. trusting a client-reported condition the way `/mark` now does, when there's no real `/mark` pairing to trust it against), adaptive experiments are just filtered out until a real use case forces the question. Covered by a unit test (`ExperimentAssignmentService.test.ts` → "should exclude Thompson Sampling (adaptive) experiments from batch assignment"). + +- **Copilot code-review fixes**: a Copilot review of this branch surfaced 24 comments; after checking each against the current code (several were already stale, having been posted against earlier commits), 8 were live, ranked, and fixed: + - **Algorithm-transition asymmetry, both directions**: `ThompsonSamplingExperimentCrudService.syncConfigIfApplicable()` previously assumed a config already existed on update. Switching an experiment *to* Thompson Sampling via edit never created one (assignment silently broke forever); switching *away from* it never deleted the old one (the reward endpoint could keep treating a now-non-adaptive experiment as Thompson Sampling). Fixed both: creates the config if missing when switching in, deletes it via a new `deleteConfigIfExists()` when switching out. `ThompsonSamplingExperimentConfigRepository.findByDecisionPoint()`/`findConfigsForActivelyEnrollingExperiments()` also gained an `assignmentAlgorithm` filter as defense-in-depth against any config row that outlives the delete. + - **Concurrent reward race**: `applyOrBufferReward()`/`flushPendingRewards()` used separate increment/read/reset calls with no locking, so two rewards for the same experiment arriving close together could double-apply or lose pending counts. Rewritten to run inside one transaction taking a `pessimistic_write` lock (ordered by id) across every `ConditionPosteriorState` row for the config before mutating any of them. + - **Thompson Sampling + Within-Subjects**: Within-Subjects assignment never stores a condition on the individual enrollment (tracked per-repeat via `RepeatedEnrollment` instead), so Thompson Sampling's reward path — which reads that enrollment's `conditionId` — could never succeed for that combination, and nothing rejected it. Added `IsAssignmentAlgorithmCompatibleWithUnit()` to `ExperimentDTO` (backend) and disabled/auto-reset the Thompson Sampling option when Within-Subjects is selected in `upsert-experiment-modal.component.ts` (frontend). + - **No rollback on partial create failure**: `ExperimentController.create()` and `ImportExportService.addBulkExperiments()` committed the `Experiment` row before creating its adaptive config; a config-creation failure left an orphaned, permanently-broken experiment behind. Both now delete the just-created experiment if `createConfigIfApplicable()` throws. + - **`getRewardsSummary()` weight-map collision**: was keyed by `conditionCode`, which has no uniqueness constraint (only `ExperimentCondition.twoCharacterId` is unique) — two conditions sharing a code would collide and one would silently show the other's `estimatedWeight`. Now keyed by `conditionId`, matching the pattern the algorithm-facing code already uses for the same reason. + - **`updateConfig()` nulling out omitted fields**: `params.field ?? null` cleared any field a partial payload omitted. Now only fields actually present in `params` are included in the update. + - **`isTSFormValid$` stuck after leaving Thompson Sampling**: if the TS sub-form was invalid at the moment the algorithm was switched away, the child component was destroyed without emitting a final validity event, permanently disabling Save. `checkForAlgorithmChange()`'s else-branch now resets `isTSFormValid$`/`isTSFormChanged$`. + - Not fixed (explicitly out of scope, per this branch's data): the migration's default/zero-seeded posterior state for remapped `ts_configurable` experiments — no such experiments exist in the environments this migrates to. + +### Architecture notes + +- **conditionId as algorithm key**: `ThompsonSamplingService.selectCondition()` uses condition UUIDs (not `conditionCode`) as identifiers, since `conditionCode` is nullable. `ConditionPosteriorState` rows, `ConditionRewardSummary.conditionId`, and `selectCondition()`'s own `conditionIds` parameter are all named and keyed accordingly. The `priors` field in `ThompsonSamplingConfigDTO` is therefore also keyed by conditionId. + +- **Reward summary endpoint**: `GET /experiments/rewards/:id` delegates to `ThompsonSamplingExperimentCrudService.getRewardsSummary()`, which calls `ThompsonSamplingExperimentConfigRepository.findByExperimentIdWithConditions()` to load `ConditionPosteriorState` rows joined to their conditions, computes `successes`, `failures`, `successRate`, `priorSuccess`, `priorFailure` per condition (via the shared `ThompsonSamplingService.computePosterior()`), and sorts by condition order. Fully implemented. diff --git a/clientlibs/js/package.json b/clientlibs/js/package.json index d2c781be38..e76e9936ee 100644 --- a/clientlibs/js/package.json +++ b/clientlibs/js/package.json @@ -50,7 +50,8 @@ "docs:markdown": "typedoc --options typedoc.json", "test": "jest", "test:coverage": "jest --coverage", - "quicktest": "([ -d dist ] || yarn build) && ts-node quickTest.ts" + "quicktest": "([ -d dist ] || yarn build) && ts-node quickTest.ts", + "quicktest:adaptive": "([ -d dist ] || yarn build) && ts-node quickTestAdaptive.ts" }, "keywords": [], "author": "", diff --git a/clientlibs/js/quickTestAdaptive.ts b/clientlibs/js/quickTestAdaptive.ts new file mode 100644 index 0000000000..bf2d54e3a4 --- /dev/null +++ b/clientlibs/js/quickTestAdaptive.ts @@ -0,0 +1,218 @@ +// to run: npx ts-node clientlibs/js/quickTestAdaptive.ts +// +// Manual smoke test for Thompson Sampling (adaptive) experiments end-to-end: +// 1. Creates a real experiment via the admin API (POST /experiments) and starts enrollment. +// 2. Simulates a batch of synthetic users each calling /v6/init -> /v6/assign -> /v6/mark -> +// /v6/reward, the same sequence a real client would. +// 3. Prints the reward summary (GET /experiments/rewards/:id) -- the same data the frontend's +// Reward Feedback card reads -- so you can watch "Pending rewards" cycle with batchSize and +// "Algorithm in Effect" flip from Random Assignment to Thompson Sampling as warmupThreshold +// is crossed, without opening a browser. +// 4. Deletes the experiment when done (see CLEANUP_AFTER_RUN below). +// +// Local dev only. See ADMIN_TOKEN below for why. + +import axios, { AxiosError } from 'axios'; +import UpgradeClient from './dist/node'; + +const URL = { + // 3030 is the standard docker-compose port (see root CLAUDE.md); a git worktree set up via + // /new-worktree auto-assigns its own port instead (check packages/backend/.env's APP_PORT) -- + // update this if you're running in a worktree. + LOCAL: 'http://localhost:3030', + ECS_QA: 'https://apps.qa-cli.net/upgrade-service', + ECS_STAGING: 'https://apps.qa-cli.com/upgrade-service', +}; + +// ------------------------------------------------------------------------------------------- +// Admin auth +// ------------------------------------------------------------------------------------------- +// authorizationChecker.ts (packages/backend/src/auth/) bypasses real Google token validation +// for this exact string, attaching a dev admin user instead -- but only when the target +// server's GOOGLE_AUTH_TOKEN_REQUIRED env var is false (check packages/backend/.env; this is +// this worktree's current local setting, not a given for every environment). This will NOT +// work against a real deployed server. Value must match FAKE_DEV_CREDENTIAL in +// packages/types/src/User/index.ts -- hardcoded rather than imported from upgrade_types +// because this file runs directly under ts-node (see quickTest.ts), which doesn't resolve the +// upgrade_types path alias at runtime the way a webpack-built consumer does. +const ADMIN_TOKEN = 'fake-dev-user-google-credential'; + +// ------------------------------------------------------------------------------------------- +// Config -- edit these to change what gets created/simulated +// ------------------------------------------------------------------------------------------- +const hostUrl = URL.LOCAL; +const adminApiUrl = hostUrl + '/api'; +const context = 'upgrade-internal'; +const site = 'quicktest-adaptive-site'; +const target = 'quicktest-adaptive-target'; + +// Adaptive algorithm parameters -- see packages/frontend .../thompson-sampling-helper.service.ts +// and the Reward Feedback card for how these show up in the UI. +const BATCH_SIZE = 3; // rewards buffered before posteriors update; watch "Pending rewards" cycle 0..batchSize-1 +const WARMUP_THRESHOLD = 4; // reward-count gate before real TS sampling kicks in; watch "Algorithm in Effect" flip +const MINIMUM_DRAW_DIFFERENCE = 0; + +const CONDITIONS = [ + { tempId: 'quicktest-cond-control', conditionCode: 'control', priorSuccess: 1, priorFailure: 1 }, + { tempId: 'quicktest-cond-variant', conditionCode: 'variant', priorSuccess: 1, priorFailure: 1 }, +]; + +const NUM_SIMULATED_USERS = 10; +// Each simulated user's reward outcome. A fixed pattern by default so runs are reproducible -- +// swap in `Math.random() < 0.7 ? 'SUCCESS' : 'FAILURE'` if you want noisy data instead. +function rewardForUser(index: number): 'SUCCESS' | 'FAILURE' { + return index % 3 === 0 ? 'FAILURE' : 'SUCCESS'; +} + +const CLEANUP_AFTER_RUN = false; // delete the created experiment when the script finishes + +// ------------------------------------------------------------------------------------------- + +const adminClient = axios.create({ + baseURL: adminApiUrl, + headers: { Authorization: `Bearer ${ADMIN_TOKEN}` }, +}); + +quickTestAdaptive(); + +/** main test *******************************************************************************/ +async function quickTestAdaptive() { + const experiment = await createAdaptiveExperiment(); + if (!experiment) return; + + console.log(`\n[Created experiment]: ${experiment.id} (${experiment.name})`); + console.log( + '[Conditions]:', + experiment.conditions.map((c: { conditionCode: string; id: string }) => `${c.conditionCode}=${c.id}`).join(', ') + ); + + await setExperimentState(experiment.id, 'enrolling'); + console.log('[Experiment state]: enrolling'); + + for (let i = 0; i < NUM_SIMULATED_USERS; i++) { + await simulateUser(i, experiment.id); + } + + // /v6/reward is fire-and-forget (POST /v6/reward acknowledges before the DB write happens -- + // see ThompsonSamplingRewardService.acceptReward()), so give the background processing a beat + // to finish before reading the summary back, or the last few rewards may not show up yet. + await sleep(1000); + + await printRewardsSummary(experiment.id); + + if (CLEANUP_AFTER_RUN) { + await deleteExperiment(experiment.id); + console.log(`\n[Cleaned up]: deleted experiment ${experiment.id}`); + } else { + console.log(`\n[Left in place]: experiment ${experiment.id} -- delete manually when done.`); + } +} + +/** admin API calls (experiment CRUD) *******************************************************/ + +async function createAdaptiveExperiment(): Promise<{ + id: string; + name: string; + conditions: { id: string; conditionCode: string }[]; +} | null> { + const payload = { + name: `quicktest-adaptive-${Date.now()}`, + description: 'Created by clientlibs/js/quickTestAdaptive.ts -- safe to delete.', + context: [context], + state: 'inactive', + consistencyRule: 'individual', + assignmentUnit: 'individual', + postExperimentRule: 'continue', + tags: ['quicktest'], + filterMode: 'includeAll', // excludeAll would exclude every user unless individually/group included via a segment + type: 'Simple', + assignmentAlgorithm: 'thompson_sampling', + // Conditions/partitions need a client-supplied id even though the server regenerates its + // own -- ExperimentService.create() remaps thompsonSamplingConfig.priors from these ids onto + // the server-generated ones automatically (see ThompsonSamplingExperimentCrudService). + conditions: CONDITIONS.map((c, index) => ({ + id: c.tempId, + name: c.conditionCode, + description: '', + conditionCode: c.conditionCode, + assignmentWeight: 100 / CONDITIONS.length, // ignored for Thompson Sampling, but required by the DTO + order: index + 1, + })), + partitions: [{ id: 'quicktest-adaptive-dp-1', site, target, description: '', order: 1, excludeIfReached: false }], + thompsonSamplingConfig: { + warmupThreshold: WARMUP_THRESHOLD, + batchSize: BATCH_SIZE, + minimumDrawDifference: MINIMUM_DRAW_DIFFERENCE, + priors: Object.fromEntries( + CONDITIONS.map((c) => [c.tempId, { success: c.priorSuccess, failure: c.priorFailure }]) + ), + }, + }; + + try { + const response = await adminClient.post('/experiments', payload); + return response.data; + } catch (error) { + logAxiosError('Create experiment', error); + return null; + } +} + +async function setExperimentState(experimentId: string, state: string): Promise { + try { + await adminClient.post('/experiments/state', { experimentId, state }); + } catch (error) { + logAxiosError('Set experiment state', error); + } +} + +async function printRewardsSummary(experimentId: string): Promise { + try { + const response = await adminClient.get(`/experiments/rewards/${experimentId}`); + console.log('\n[Rewards summary]:', JSON.stringify(response.data, null, 2)); + } catch (error) { + logAxiosError('Rewards summary', error); + } +} + +async function deleteExperiment(experimentId: string): Promise { + try { + await adminClient.delete(`/experiments/${experimentId}`); + } catch (error) { + logAxiosError('Delete experiment', error); + } +} + +/** simulated user flow (client SDK, same as a real client would call) **********************/ + +async function simulateUser(index: number, experimentId: string): Promise { + const userId = `quicktest_adaptive_user_${Date.now()}_${index}`; + const client = new UpgradeClient(userId, hostUrl, context); + + try { + await client.init(); + + const assignment = await client.getDecisionPointAssignment(site, target); + const condition = assignment.getCondition(); + await assignment.markDecisionPoint(UpgradeClient.MARKED_DECISION_POINT_STATUS.CONDITION_APPLIED); + + const rewardValue = rewardForUser(index); + await client.sendReward({ rewardValue, experimentId }); + + console.log(`[User ${index}]: condition=${condition} reward=${rewardValue}`); + } catch (error) { + logAxiosError(`User ${index}`, error); + } +} + +/** utility functions *************************************************************************/ + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function logAxiosError(functionContext: string, error: unknown): void { + const axiosError = error as AxiosError; + const data = axiosError?.response?.data; + console.error(`\n[${functionContext} error]:`, data ?? axiosError?.message ?? error); +} diff --git a/clientlibs/js/src/UpGradeClient/UpgradeClient.ts b/clientlibs/js/src/UpGradeClient/UpgradeClient.ts index 2bffb52fea..6740d58c49 100644 --- a/clientlibs/js/src/UpGradeClient/UpgradeClient.ts +++ b/clientlibs/js/src/UpGradeClient/UpgradeClient.ts @@ -630,7 +630,7 @@ export default class UpgradeClient { } /** - * Sends a binary reward signal for an adaptive experiment (Mooclet). + * Sends a binary reward signal for an adaptive experiment. * * This method allows sending reward feedback (SUCCESS or FAILURE) for adaptive experiments. * The reward is used by the adaptive algorithm to update its learning model and improve future assignments. diff --git a/clientlibs/js/src/types/Interfaces.ts b/clientlibs/js/src/types/Interfaces.ts index 28a2df222b..f6c83340b4 100644 --- a/clientlibs/js/src/types/Interfaces.ts +++ b/clientlibs/js/src/types/Interfaces.ts @@ -109,13 +109,6 @@ export namespace UpGradeClientInterfaces { context?: string; decisionPoint?: IDecisionPoint; }; - reward: { - variable: string; - value: number; - mooclet: number; - version: number; - learner: string; - }; } export interface IHttpClientWrapperRequestConfig { diff --git a/clientlibs/python/BUILD_PLAN.md b/clientlibs/python/BUILD_PLAN.md index c87951bbe7..32fdd26ea9 100644 --- a/clientlibs/python/BUILD_PLAN.md +++ b/clientlibs/python/BUILD_PLAN.md @@ -103,7 +103,7 @@ class BinaryRewardValue(str, Enum): - `MarkDecisionPointResponse` — userId, site, target, experimentId - `LogEventResponse` — id, uniquifier, timeStamp, data - `UserAliasResponse` — userId, aliases -- `SendRewardResponse` — message, request (rewardValue, experimentId, context, decisionPoint), reward (variable, value, mooclet, version, learner) +- `SendRewardResponse` — message, request (rewardValue, experimentId, context, decisionPoint) - `ErrorResponse` — message, httpStatusCode, type ### Deliverable diff --git a/clientlibs/python/src/upgrade_client_lib/client.py b/clientlibs/python/src/upgrade_client_lib/client.py index b7598b418a..c1bb28484c 100644 --- a/clientlibs/python/src/upgrade_client_lib/client.py +++ b/clientlibs/python/src/upgrade_client_lib/client.py @@ -330,7 +330,7 @@ async def send_reward( context: str | None = None, decision_point: dict[str, str] | None = None, ) -> SendRewardResponse: - """Send a binary reward signal for an adaptive (Mooclet) experiment.""" + """Send a binary reward signal for an adaptive experiment.""" return await self._api_service.send_reward( reward_value=reward_value, experiment_id=experiment_id, diff --git a/clientlibs/python/src/upgrade_client_lib/types/__init__.py b/clientlibs/python/src/upgrade_client_lib/types/__init__.py index ca47c22eb3..54f42dfa5e 100644 --- a/clientlibs/python/src/upgrade_client_lib/types/__init__.py +++ b/clientlibs/python/src/upgrade_client_lib/types/__init__.py @@ -33,7 +33,6 @@ LogEventResponse, MarkDecisionPointResponse, Payload, - RewardDetails, RewardRequest, SendRewardResponse, UserAliasResponse, @@ -71,7 +70,6 @@ "LogEventResponse", "MarkDecisionPointResponse", "Payload", - "RewardDetails", "RewardRequest", "SendRewardResponse", "UserAliasResponse", diff --git a/clientlibs/python/src/upgrade_client_lib/types/responses.py b/clientlibs/python/src/upgrade_client_lib/types/responses.py index 1771196a7a..174f864d61 100644 --- a/clientlibs/python/src/upgrade_client_lib/types/responses.py +++ b/clientlibs/python/src/upgrade_client_lib/types/responses.py @@ -74,18 +74,9 @@ class RewardRequest(BaseModel): decisionPoint: dict[str, str] | None = None -class RewardDetails(BaseModel): - variable: str - value: float - mooclet: int - version: int - learner: str - - class SendRewardResponse(BaseModel): message: str request: RewardRequest - reward: RewardDetails class ErrorResponse(BaseModel): diff --git a/clientlibs/python/tests/test_api_service.py b/clientlibs/python/tests/test_api_service.py index 35cf3d7e4c..77fabab29b 100644 --- a/clientlibs/python/tests/test_api_service.py +++ b/clientlibs/python/tests/test_api_service.py @@ -406,7 +406,6 @@ async def test_sends_aliases(self) -> None: REWARD_RESPONSE = { "message": "Reward sent", "request": {"rewardValue": "SUCCESS", "experimentId": "exp-1"}, - "reward": {"variable": "score", "value": 1.0, "mooclet": 42, "version": 3, "learner": USER_ID}, } @@ -416,13 +415,13 @@ async def test_async_minimal(self) -> None: respx.post(f"{BASE}/reward").mock(return_value=Response(200, json=REWARD_RESPONSE)) result = await make_service().send_reward(BinaryRewardValue.SUCCESS) assert result.message == "Reward sent" - assert result.reward.variable == "score" + assert result.request.experimentId == "exp-1" @respx.mock def test_sync(self) -> None: respx.post(f"{BASE}/reward").mock(return_value=Response(200, json=REWARD_RESPONSE)) result = make_service().send_reward_sync(BinaryRewardValue.SUCCESS) - assert result.reward.mooclet == 42 + assert result.message == "Reward sent" @respx.mock async def test_full_params(self) -> None: diff --git a/clientlibs/python/tests/test_client.py b/clientlibs/python/tests/test_client.py index 4d878172ac..40eb978509 100644 --- a/clientlibs/python/tests/test_client.py +++ b/clientlibs/python/tests/test_client.py @@ -68,7 +68,6 @@ REWARD_PAYLOAD = { "message": "ok", "request": {"rewardValue": "SUCCESS"}, - "reward": {"variable": "score", "value": 1.0, "mooclet": 1, "version": 1, "learner": USER}, } @@ -596,7 +595,7 @@ async def test_async_success(self) -> None: respx.post(f"{BASE}/reward").mock(return_value=Response(200, json=REWARD_PAYLOAD)) result = await make_client().send_reward(BinaryRewardValue.SUCCESS) assert result.message == "ok" - assert result.reward.variable == "score" + assert result.request.rewardValue == BinaryRewardValue.SUCCESS @respx.mock async def test_passes_all_params(self) -> None: @@ -617,4 +616,4 @@ async def test_passes_all_params(self) -> None: def test_sync(self) -> None: respx.post(f"{BASE}/reward").mock(return_value=Response(200, json=REWARD_PAYLOAD)) result = make_client().send_reward_sync(BinaryRewardValue.SUCCESS) - assert result.reward.mooclet == 1 + assert result.message == "ok" diff --git a/packages/backend/.env.docker.local.example b/packages/backend/.env.docker.local.example index 4ed43d5a38..ad3122a644 100644 --- a/packages/backend/.env.docker.local.example +++ b/packages/backend/.env.docker.local.example @@ -88,15 +88,6 @@ EMAIL_EXPIRE_AFTER_SECONDS=36000 EMAIL_BUCKET="s3_bucket" EMAIL_TMP_WRITEABLE_FILE_PATH="src/api/assets/files/" -# -# Mooclets -# - -MOOCLETS_ENABLED = false -MOOCLETS_HOST_URL = mooclet_host_url -MOOCLETS_API_ROUTE = /engine/api/v1 -MOOCLETS_API_TOKEN = some_token - # # Initialization # diff --git a/packages/backend/.env.example b/packages/backend/.env.example index f00e631b02..5d7e529ceb 100644 --- a/packages/backend/.env.example +++ b/packages/backend/.env.example @@ -90,15 +90,6 @@ EMAIL_EXPIRE_AFTER_SECONDS=36000 EMAIL_BUCKET="s3_bucket" EMAIL_TMP_WRITEABLE_FILE_PATH="src/api/assets/files/" -# -# Mooclets -# - -MOOCLETS_ENABLED=false -MOOCLETS_HOST_URL=mooclet_host_url -MOOCLETS_API_ROUTE=/engine/api/v1 -MOOCLETS_API_TOKEN=some_token - # # Initialization # diff --git a/packages/backend/CLAUDE.md b/packages/backend/CLAUDE.md index 3621c8c259..a4337cebe5 100644 --- a/packages/backend/CLAUDE.md +++ b/packages/backend/CLAUDE.md @@ -140,14 +140,13 @@ Experiment join tables (`ExperimentSegmentInclusion` / `ExperimentSegmentExclusi | Segment list members updated | `ExperimentService.updateList` → `withRecompute` (passes `skipScheduleRecompute=true` to the segment upsert) | | Segment list removed | `ExperimentService.deleteList` → delegates to `SegmentService.deleteSegment` (which owns the recompute) | | Experiment created | `ExperimentService.create` → `await recomputeForExperiment` after lists attach (only when it owns the commit; recompute yields empty arrays for list-less experiments, so no separate empty-seed) | -| Experiment created inside a Mooclet transaction | `MoocletExperimentService.syncCreate` → `await recomputeForExperiment` after the transaction commits (create deferred it because it ran inside the transaction) | | Experiment lists imported | `ExperimentService.importExperimentLists` → `await recomputeForExperiment` after the import transaction commits | -| Experiment context changed (deletes all its lists) | `ExperimentService.updateExperimentInDB` → `scheduleRecomputeForExperiments` after commit (recomputes to empty; `deleteAllListsFromExperiment` deletes the private segments directly, so the precomputed row would otherwise keep stale IDs). When a caller owns the transaction (`MoocletExperimentService.syncUpdate` / `syncUpdateWithMoocletAlgorithmTransition`), those methods recompute after their own commit — mirrors the flag side's `updateFeatureFlagInDB` → `withRecompute` | +| Experiment context changed (deletes all its lists) | `ExperimentService.updateExperimentInDB` → `scheduleRecomputeForExperiments` after commit (recomputes to empty; `deleteAllListsFromExperiment` deletes the private segments directly, so the precomputed row would otherwise keep stale IDs) | | Shared segment members/structure changed | `SegmentService.addList` / `deleteList` / `addSegmentDataWithPipeline` → `scheduleRecomputeForSegment` for **both** the flag and experiment services | | Segment deleted entirely | `SegmentService.deleteSegment` → collects affected experiment IDs **before** the delete, recomputes **after** commit (flags use `withRecompute` in the same method) | | Server startup | `app.ts` → `backfillExperimentPrecomputedSegments` (guarded by `.catch` — a missing table never crashes startup) | -Same invariant as feature flags: recompute **after** the change commits; for deletes, collect affected experiment IDs **before**. All write-path recomputes are fire-and-forget except the `create` / import / Mooclet paths, which `await` so "done" means the row is ready. +Same invariant as feature flags: recompute **after** the change commits; for deletes, collect affected experiment IDs **before**. All write-path recomputes are fire-and-forget except the `create` / import paths, which `await` so "done" means the row is ready. ### INCLUDE_ALL semantics (resolved) diff --git a/packages/backend/rest-client-vscode/MoocletAPI.http b/packages/backend/rest-client-vscode/MoocletAPI.http deleted file mode 100644 index 3f2207cf38..0000000000 --- a/packages/backend/rest-client-vscode/MoocletAPI.http +++ /dev/null @@ -1,204 +0,0 @@ -# Mooclet -# To be used with the Rest Client extension for Visual Studio Code -# https://marketplace.visualstudio.com/items?itemName=humao.rest-client - -# This is not an exhaustive list, but running down the list emulates the process of -# of creating and running mooclet policies like UpGrade does - -# Other CRUD operations like GET / DELETE / PUT will work as expected -# https://docs.google.com/spreadsheets/d/1eL3z5zJfpraqEONYMVhiWgGYj6PSm8T8KFKFyeOMnHU/edit?gid=0#gid=0 - -### Creating a Mooclet -# 1. Get policy id by assignment algorithm name -# 2. Create mooclet -# 3. Create policyparameters -# 4. Create version 1 -# 5. Create version 2 -# 6. Create outcome variable - -#### Assigning and sending rewards -# 7. Create a learner (optional) -# 8. Get a new assignment -# 9. Check the policy parameters (optional to view changes to policy parameters) -# 10. Send reward -# 11. Repeat steps 8-10 as needed - -############ env variables -@host = https://apps.qa-cli.net/mooclet-service - -# Replace with your token, i.e. -@token = Token abc123 -# @token = - -@apiEndpoint = /engine/api/v1 - -############ request variables (change as needed) -@moocletId = 196 -@moocletName = newmooc4 -@policyId = 17 -@policyParametersId = 2 -@version1Name = controlz -@version1Id = -@version2Name = variant -@version2Id = 6 - -@outcomeVariableName = mooc_4_variable -@learnerName = dave - -########### Get policy id by assignment algorithm name -GET {{host}}{{apiEndpoint}}/policy HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -########### create mooclet -POST {{host}}{{apiEndpoint}}/mooclet HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "name": "{{moocletName}}", - "policy": {{policyId}} -} - -########### create policyparameters -POST {{host}}{{apiEndpoint}}/policyparameters HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "policy": {{policyId}}, - "parameters": { - "prior": { - "failure": 1, - "success": 1 - }, - "batch_size": 1, - "max_rating": 1, - "min_rating": 0, - "uniform_threshold": 0, - "tspostdiff_thresh": 0, - "outcome_variable_name": "{{outcomeVariableName}}" - } -} - -########### create version 1 -POST {{host}}{{apiEndpoint}}/version HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "name": "{{version1Name}}" -} - -########### create version 2 -POST {{host}}{{apiEndpoint}}/version HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "name": "{{version2Name}}" -} - -########### create outcome variable -POST {{host}}{{apiEndpoint}}/variable HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "name": "{{outcomeVariableName}}", - "value": 0 -} - -########### create a learner? -POST {{host}}{{apiEndpoint}}/learner HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "name": "{{ learnerName }}" -} - -########### get a new assignment -GET {{host}}{{apiEndpoint}}/mooclet/{{moocletId}}/run HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -########### check the policy parameters -GET {{host}}{{apiEndpoint}}/policyparameters/{{policyParametersId}} HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -########### send reward -POST {{host}}{{apiEndpoint}}/value HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "variable": "{{outcomeVariableName}}", - "value": 0, - "mooclet": {{moocletId}}, - "version": {{version1Id}}, - "policy": {{policyId}} -} - -########## query rewards by mooclet id: -GET {{host}}{{apiEndpoint}}/value?mooclet={{moocletId}}&variable__name={{outcomeVariableName}} -Authorization: {{token}} -Content-type: application/json - -##### EDITS: - -########### create policyparameters -PUT {{host}}{{apiEndpoint}}/policyparameters/{{policyParametersId}} HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "policy": {{policyId}}, - "parameters": { - "prior": { - "failure": 1, - "success": 1 - }, - "batch_size": 1, - "max_rating": 1, - "min_rating": 0, - "uniform_threshold": 0, - "tspostdiff_thresh": 0, - "outcome_variable_name": "{{outcomeVariableName}}" - } -} - -########### update version 1 -PUT {{host}}{{apiEndpoint}}/version/{{version1Id}} HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "name": "qwerty" -} - -########### update version 1 -GET {{host}}{{apiEndpoint}}/version/{{version1Id}} HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "mooclet": {{moocletId}}, - "name": "qwerty" -} - -########### create outcome variable -PUT {{host}}{{apiEndpoint}}/variable HTTP/1.1 -Authorization: {{token}} -Content-type: application/json - -{ - "name": "{{outcomeVariableName}}", -} - diff --git a/packages/backend/src/api/DTO/ExperimentDTO.ts b/packages/backend/src/api/DTO/ExperimentDTO.ts index 9cacd480dd..513718a480 100644 --- a/packages/backend/src/api/DTO/ExperimentDTO.ts +++ b/packages/backend/src/api/DTO/ExperimentDTO.ts @@ -3,7 +3,6 @@ import { IsArray, IsBoolean, IsDateString, - IsDefined, IsEnum, IsInt, IsNotEmpty, @@ -12,6 +11,8 @@ import { IsOptional, IsString, IsUUID, + Max, + Min, ValidateIf, ValidateNested, ValidationArguments, @@ -37,9 +38,6 @@ import { REPEATED_MEASURE, EXPERIMENT_TYPE, ASSIGNMENT_ALGORITHM, - MoocletTSConfigurablePolicyParametersDTO, - MoocletPolicyParametersDTO, - SUPPORTED_MOOCLET_ALGORITHMS, } from 'upgrade_types'; import { Type, Transform } from 'class-transformer'; @@ -383,6 +381,7 @@ abstract class BaseExperimentWithoutPayload { @IsOptional() @IsEnum(ASSIGNMENT_ALGORITHM) + @IsAssignmentAlgorithmCompatibleWithUnit() public assignmentAlgorithm?: ASSIGNMENT_ALGORITHM; // TODO add conditional validity here ie endOn is null @@ -471,9 +470,6 @@ abstract class BaseExperimentWithoutPayload { public type: EXPERIMENT_TYPE; } -const isMoocletAssignmentAlgorithm = (experiment: ExperimentDTO) => - experiment.assignmentAlgorithm && SUPPORTED_MOOCLET_ALGORITHMS.includes(experiment.assignmentAlgorithm); - function IsAssignmentUnitGroupConsistent(validationOptions?: ValidationOptions) { return function (object: any, propertyName: string) { registerDecorator({ @@ -502,6 +498,105 @@ function IsAssignmentUnitGroupConsistent(validationOptions?: ValidationOptions) }; } +/** + * Within-Subjects assignment never runs through assignExperiment()/assignThompsonSampling() -- + * the individual enrollment's condition is always stored as null, with the per-repeat condition + * tracked separately via RepeatedEnrollment instead. Thompson Sampling's reward path then reads + * that null conditionId when trying to record a reward, which can never succeed. Reject the + * combination outright rather than let it silently produce an experiment whose rewards can never + * be recorded. + */ +function IsAssignmentAlgorithmCompatibleWithUnit(validationOptions?: ValidationOptions) { + return function (object: any, propertyName: string) { + registerDecorator({ + name: 'isAssignmentAlgorithmCompatibleWithUnit', + target: object.constructor, + propertyName: propertyName, + options: validationOptions, + validator: { + validate(assignmentAlgorithmValue: any, args: ValidationArguments) { + const experiment = args.object as any; + return !( + assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING && + experiment.assignmentUnit === ASSIGNMENT_UNIT.WITHIN_SUBJECTS + ); + }, + defaultMessage() { + return 'Thompson Sampling cannot be used with Within-Subjects assignment: rewards cannot be attributed to a condition under that assignment unit.'; + }, + }, + }); + }; +} + +const MAX_NUMBER_INPUT = 1_000_000; +const MIN_PRIOR_VALUE = 1; + +function IsThompsonSamplingPriorsRecord(validationOptions?: ValidationOptions) { + return function (object: any, propertyName: string) { + registerDecorator({ + name: 'isThompsonSamplingPriorsRecord', + target: object.constructor, + propertyName: propertyName, + options: validationOptions, + validator: { + validate(value: any) { + if (value === undefined || value === null) { + return true; + } + if (typeof value !== 'object' || Array.isArray(value)) { + return false; + } + return Object.values(value).every((prior: any) => { + if (typeof prior !== 'object' || prior === null) { + return false; + } + const { success, failure } = prior; + return ( + Number.isInteger(success) && + success >= MIN_PRIOR_VALUE && + success <= MAX_NUMBER_INPUT && + Number.isInteger(failure) && + failure >= MIN_PRIOR_VALUE && + failure <= MAX_NUMBER_INPUT + ); + }); + }, + defaultMessage() { + return ( + 'Each entry in priors must have integer success/failure values between ' + + `${MIN_PRIOR_VALUE} and ${MAX_NUMBER_INPUT}.` + ); + }, + }, + }); + }; +} + +class ThompsonSamplingConfigValidator { + @IsOptional() + @IsInt() + @Min(0) + @Max(MAX_NUMBER_INPUT) + public warmupThreshold?: number; + + @IsOptional() + @IsNumber() + @Min(0) + @Max(1) + public minimumDrawDifference?: number; + + @IsOptional() + @IsInt() + @Min(1) + @Max(MAX_NUMBER_INPUT) + public batchSize?: number; + + @IsOptional() + @IsThompsonSamplingPriorsRecord() + public priors?: Record; +} + export class ExperimentDTO extends BaseExperimentWithoutPayload { @IsOptional() @IsArray() @@ -509,21 +604,10 @@ export class ExperimentDTO extends BaseExperimentWithoutPayload { @Type(() => ConditionPayloadValidator) public conditionPayloads?: ConditionPayloadValidator[]; - // This should be validated when assignmentAlgorithm is not RANDOM or STRATIFIED_RANDOM_SAMPLING - @ValidateIf(isMoocletAssignmentAlgorithm) - @IsDefined() + @IsOptional() @ValidateNested() - @Type(() => MoocletPolicyParametersDTO, { - discriminator: { - property: 'assignmentAlgorithm', - subTypes: [ - { value: MoocletTSConfigurablePolicyParametersDTO, name: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE }, - // Other policy types can be added here - ], - }, - keepDiscriminatorProperty: true, - }) - public moocletPolicyParameters?: MoocletPolicyParametersDTO; + @Type(() => ThompsonSamplingConfigValidator) + public thompsonSamplingConfig?: ThompsonSamplingConfigValidator; } export class OldExperimentDTO extends BaseExperimentWithoutPayload { diff --git a/packages/backend/src/api/controllers/ExperimentClientController.v6.ts b/packages/backend/src/api/controllers/ExperimentClientController.v6.ts index 814b5bcbe6..af2d26118f 100644 --- a/packages/backend/src/api/controllers/ExperimentClientController.v6.ts +++ b/packages/backend/src/api/controllers/ExperimentClientController.v6.ts @@ -8,7 +8,6 @@ import { Delete, Patch, Authorized, - BadRequestError, } from 'routing-controllers'; import { ExperimentService } from '../services/ExperimentService'; import { ExperimentAssignmentService } from '../services/ExperimentAssignmentService'; @@ -30,8 +29,10 @@ import { Log } from '../models/Log'; import { ExperimentUserValidatorv6 } from './validators/ExperimentUserValidator'; import { UserCheckMiddleware } from '../middlewares/UserCheckMiddleware'; import { RewardValidator } from './validators/RewardValidator'; -import { IRewardResponse, MoocletRewardsService } from '../services/MoocletRewardsService'; -import { env } from '../../env'; +import { + IThompsonSamplingRewardResponse, + ThompsonSamplingRewardService, +} from '../services/ThompsonSamplingRewardService'; interface IMonitoredDecisionPoint { id: string; @@ -100,7 +101,7 @@ export class ExperimentClientController { public experimentUserService: ExperimentUserService, public featureFlagService: FeatureFlagService, public metricService: MetricService, - public moocletRewardsService: MoocletRewardsService + public thompsonSamplingRewardService: ThompsonSamplingRewardService ) {} /** @@ -831,7 +832,7 @@ export class ExperimentClientController { * /v6/reward: * post: * description: | - * Send a reward signal for an adaptive experiment (Mooclet). + * Send a reward signal for an adaptive (Thompson Sampling) experiment. * * This endpoint allows sending binary reward feedback (SUCCESS or FAILURE) for adaptive experiments. * The reward is used by the adaptive algorithm to update its learning model and improve future assignments. @@ -844,6 +845,11 @@ export class ExperimentClientController { * 2. **Decision Point Lookup** - Provide `context` and `decisionPoint` (site and target) to look up the experiment * * At least one of these methods must be provided. + * + * **Asynchronous processing:** This endpoint acknowledges receipt immediately and records the reward + * (config/enrollment lookup, posterior update) in the background — the response does not wait on it. + * A problem with the reward itself (unknown experiment, no matching enrollment, experiment no longer + * enrolling, etc.) is therefore not returned to the caller; it is only visible in server-side logs. * consumes: * - application/json * parameters: @@ -910,14 +916,16 @@ export class ExperimentClientController { * - application/json * responses: * '200': - * description: Reward successfully sent to the adaptive experiment + * description: | + * Reward received and queued for processing. This does not guarantee the reward was recorded - + * see "Asynchronous processing" above. * schema: * type: object * properties: * message: * type: string - * example: Reward sent successfully - * description: Success message + * example: Reward received and is being processed. + * description: Receipt message * request: * type: object * description: Echo of the original request data @@ -936,17 +944,10 @@ export class ExperimentClientController { * type: string * target: * type: string - * reward: - * type: object - * description: The reward data that was sent to the Mooclet API * '400': * description: BadRequestError - Invalid parameters (e.g., missing required fields, invalid rewardValue) * '401': * description: AuthorizationRequiredError - * '409': - * description: Conflict - Data conflict (e.g., site or target not found, enrollment data not found, etc) - * '500': - * description: Internal Server Error */ @Post('reward') public async sendReward( @@ -954,14 +955,9 @@ export class ExperimentClientController { request: AppRequest, @Body({ validate: true }) rewardData: RewardValidator - ): Promise { + ): Promise { request.logger.info({ message: 'Starting the sendReward call for user' }); - if (!env.mooclets?.enabled) { - throw new BadRequestError('Failed to send reward: mooclet is not currently enabled on backend.'); - } - - const experimentUserDoc = request.userDoc; - return this.moocletRewardsService.sendReward(experimentUserDoc, rewardData, request.logger); + return this.thompsonSamplingRewardService.acceptReward(request.userDoc, rewardData, request.logger); } /** diff --git a/packages/backend/src/api/controllers/ExperimentController.ts b/packages/backend/src/api/controllers/ExperimentController.ts index a337c0edf3..b3ce91cedb 100644 --- a/packages/backend/src/api/controllers/ExperimentController.ts +++ b/packages/backend/src/api/controllers/ExperimentController.ts @@ -27,17 +27,19 @@ import { AssignmentStateUpdateValidator } from './validators/AssignmentStateUpda import { AppRequest, PaginationResponse } from '../../types'; import { ExperimentDTO, ExperimentFile, ValidatedExperimentError } from '../DTO/ExperimentDTO'; import { ExperimentIds } from './validators/ExperimentIdsValidator'; -import { MoocletExperimentService } from '../services/MoocletExperimentService'; -import { env } from '../../env'; +import { ThompsonSamplingExperimentCrudService } from '../services/ThompsonSamplingExperimentCrudService'; +import { AdaptiveExperimentConfigDispatcherService } from '../services/AdaptiveExperimentConfigDispatcherService'; import { Response } from 'express'; import { NotFoundException } from '@nestjs/common/exceptions'; import { ExperimentIdValidator } from '../DTO/ExperimentDTO'; import { + ASSIGNMENT_ALGORITHM, CACHE_PREFIX, + EXPERIMENT_STATE, IImportError, LIST_FILTER_MODE, SERVER_ERROR, - SUPPORTED_MOOCLET_ALGORITHMS, + ExperimentRewardsSummary, } from 'upgrade_types'; import { ImportExportService } from '../services/ImportExportService'; import { getInstanceId } from '../../lib/instanceIdentity'; @@ -46,8 +48,6 @@ import { SegmentInputValidator } from './validators/SegmentInputValidator'; import { ExperimentSegmentExclusion } from '../models/ExperimentSegmentExclusion'; import { IdValidator } from './validators/ExperimentUserValidator'; import { Segment } from '../models/Segment'; -import { MoocletRewardsService } from '../services/MoocletRewardsService'; -import { ExperimentRewardsSummary } from 'upgrade_types'; import { CacheService } from '../services/CacheService'; interface ExperimentPaginationInfo extends PaginationResponse { @@ -661,10 +661,10 @@ export class ExperimentController { constructor( public experimentService: ExperimentService, public experimentAssignmentService: ExperimentAssignmentService, - public moocletExperimentService: MoocletExperimentService, - public moocletRewardService: MoocletRewardsService, public importExportService: ImportExportService, - public cacheService: CacheService + public cacheService: CacheService, + public thompsonSamplingCrudService: ThompsonSamplingExperimentCrudService, + public adaptiveExperimentConfigDispatcher: AdaptiveExperimentConfigDispatcherService ) {} /** @@ -919,19 +919,16 @@ export class ExperimentController { @Params({ validate: true }) { id }: ExperimentIdValidator, @Req() request: AppRequest ): Promise { - let experiment = await this.experimentService.getSingleExperiment(id, request.logger); - - if (SUPPORTED_MOOCLET_ALGORITHMS.includes(experiment?.assignmentAlgorithm)) { - if (!env.mooclets?.enabled) { - throw new BadRequestError( - 'MoocletPolicyParameters are present in the experiment but Mooclet is not enabled in the environment' - ); - } else { - experiment = await this.moocletExperimentService.attachPolicyParamsToExperimentDTO(experiment, request.logger); - } - } + const experiment = await this.experimentService.getSingleExperiment(id, request.logger); + return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(experiment); + } - return experiment; + @Get('/rewards/:id') + public async getRewardsSummary( + @Params({ validate: true }) { id }: ExperimentIdValidator, + @Req() request: AppRequest + ): Promise { + return this.thompsonSamplingCrudService.getRewardsSummary(id); } /** @@ -1046,7 +1043,7 @@ export class ExperimentController { */ @Post() - public create( + public async create( @Body({ validate: true }) experiment: ExperimentDTO, @CurrentUser() currentUser: UserDTO, @Req() request: AppRequest @@ -1058,21 +1055,27 @@ export class ExperimentController { throw new BadRequestError(contextValidationError); } - if ('moocletPolicyParameters' in experiment) { - if (!env.mooclets?.enabled) { - throw new BadRequestError( - 'Failed to create Experiment: moocletPolicyParameters was provided but mooclets are not enabled on backend.' - ); - } else { - return this.moocletExperimentService.syncCreate({ - experimentDTO: experiment, - currentUser, - logger: request.logger, - }); - } + // Captured before create() runs: ExperimentService.create() replaces every condition's id with + // a freshly generated one in place, so `experiment.conditions[].id` no longer matches whatever + // ids the client used to key thompsonSamplingConfig.priors by the time create() returns. + const originalConditionIds = experiment.conditions?.map((condition) => condition.id); + const createdExperiment = await this.experimentService.create(experiment, currentUser, request.logger); + + try { + await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable( + experiment, + createdExperiment, + originalConditionIds + ); + } catch (error) { + // The experiment row already committed above. Without this, a failed adaptive-config write + // would leave a Thompson Sampling experiment with no config/posterior rows behind -- invisible + // and permanently unable to assign a condition. Remove it rather than leave it orphaned. + await this.experimentService.delete(createdExperiment.id, currentUser, { logger: request.logger }); + throw error; } - return this.experimentService.create(experiment, currentUser, request.logger); + return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(createdExperiment); } /** @@ -1155,20 +1158,6 @@ export class ExperimentController { ): Promise { request.logger.child({ user: currentUser }); - // Manually check if the experiment has a mooclet ref - if (env.mooclets.enabled) { - const moocletExperimentRef = await this.moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId(id); - - if (moocletExperimentRef) { - return await this.moocletExperimentService.syncDelete({ - moocletExperimentRef, - experimentId: id, - currentUser, - logger: request.logger, - }); - } - } - const experiment = await this.experimentService.delete(id, currentUser, { logger: request.logger }); if (!experiment) { @@ -1216,13 +1205,14 @@ export class ExperimentController { @CurrentUser() currentUser: UserDTO, @Req() request: AppRequest ): Promise { - return this.experimentService.updateState( + const updatedExperiment = await this.experimentService.updateState( experiment.experimentId, experiment.state, currentUser, request.logger, experiment.scheduleDate ); + return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(updatedExperiment); } /** @@ -1274,29 +1264,140 @@ export class ExperimentController { throw new BadRequestError(contextValidationError); } - if (env.mooclets.enabled) { - // if mooclet is enabled, we must check for potential assignment algorithm changes in all experiments - const updatedMoocletExperiment = - await this.moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - { ...experiment, id }, - currentUser, - request.logger - ); + const previousExperiment = await this.experimentService.getSingleExperiment(id, request.logger); + if (previousExperiment) { + await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(previousExperiment); + this.assertAssignmentAlgorithmNotChangedToOrFromThompsonSampling(previousExperiment, experiment); + this.assertConditionsNotModifiedAfterStart(previousExperiment, experiment); + } - if (updatedMoocletExperiment) { - return updatedMoocletExperiment; - } - } else { - // if mooclet is not enabled, but experiment has mooclet params, throw error - if ('moocletPolicyParameters' in experiment) { - throw new BadRequestError( - 'Failed to update Experiment: moocletPolicyParameters was provided but mooclets are not enabled on backend.' - ); + const updatedExperiment = await this.experimentService.update({ ...experiment, id }, currentUser, request.logger); + + try { + await this.adaptiveExperimentConfigDispatcher.syncConfigIfApplicable(experiment, updatedExperiment); + } catch (error) { + // The base experiment update above already committed (e.g. assignmentAlgorithm switched to + // THOMPSON_SAMPLING). Without reverting, a failed config sync would leave that change in place + // with no config/posterior rows -- invisible and permanently unable to assign a condition, the + // same failure mode create() already guards against by deleting the just-created experiment. + // Restore the pre-update experiment, then re-run the config sync against the reverted state so + // any config/posterior rows the failed attempt did manage to write get cleaned up too. Best + // effort: a failure here is logged rather than allowed to replace/mask the original error. + if (previousExperiment) { + try { + const revertedExperiment = await this.experimentService.update( + previousExperiment, + currentUser, + request.logger + ); + await this.adaptiveExperimentConfigDispatcher.syncConfigIfApplicable(previousExperiment, revertedExperiment); + } catch (revertError) { + request.logger.error({ + message: `Failed to fully revert experiment ${id} after adaptive config sync failure`, + error: revertError, + }); + } } + throw error; + } + + return this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(updatedExperiment); + } + + /** + * Switching an experiment to or from Thompson Sampling on an update is not supported at all -- + * regardless of experiment state, even INACTIVE/DRAFT -- because there is no sane way to backfill + * or discard the config/posterior rows a switch implies. The intended workflow is to delete the + * experiment and create a new one with the desired algorithm instead. Unlike + * assertConditionsNotModifiedAfterStart below, this has no "not started yet" exception. + */ + private assertAssignmentAlgorithmNotChangedToOrFromThompsonSampling( + previousExperiment: ExperimentDTO, + incomingExperiment: ExperimentDTO + ): void { + const wasThompsonSampling = previousExperiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + const isThompsonSampling = incomingExperiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + + if (wasThompsonSampling !== isThompsonSampling) { + throw new BadRequestError( + 'The assignment algorithm cannot be changed to or from Thompson Sampling. Create a new experiment instead.' + ); } + } + + /** + * Mirrors the frontend's own restriction (selectSectionCardRestriction / + * selectDisabledExperimentFields in experiments.selectors.ts): conditions -- including Thompson + * Sampling priors -- are only editable while an experiment is still in one of these states. The + * frontend disables the relevant fields/section card once an experiment has left this set, but + * nothing previously stopped the same change from being sent directly to this endpoint. + */ + private static readonly CONDITIONS_EDITABLE_STATES = new Set([ + EXPERIMENT_STATE.INACTIVE, + EXPERIMENT_STATE.SCHEDULED, + EXPERIMENT_STATE.PREVIEW, + EXPERIMENT_STATE.DRAFT, + ]); + + private assertConditionsNotModifiedAfterStart( + previousExperiment: ExperimentDTO, + incomingExperiment: ExperimentDTO + ): void { + // Mirrors selectDisabledExperimentFields/selectSectionCardRestriction's own `!state || ...` + // guard: a missing state means there's nothing to compare against yet, not that editing should + // be blocked. + if (!previousExperiment.state || ExperimentController.CONDITIONS_EDITABLE_STATES.has(previousExperiment.state)) { + return; + } + + const previousConditions = previousExperiment.conditions ?? []; + const incomingConditions = incomingExperiment.conditions ?? []; + + const previousIds = new Set(previousConditions.map((condition) => condition.id)); + const incomingIds = new Set(incomingConditions.map((condition) => condition.id)); + const conditionSetChanged = + previousIds.size !== incomingIds.size || [...previousIds].some((id) => !incomingIds.has(id)); + + const conditionFieldsChanged = previousConditions.some((previousCondition) => { + const incomingCondition = incomingConditions.find((condition) => condition.id === previousCondition.id); + return ( + incomingCondition && + (previousCondition.conditionCode !== incomingCondition.conditionCode || + previousCondition.name !== incomingCondition.name || + previousCondition.description !== incomingCondition.description || + previousCondition.assignmentWeight !== incomingCondition.assignmentWeight) + ); + }); + + const priorsChanged = this.havePriorsChanged( + previousExperiment.thompsonSamplingConfig?.priors, + incomingExperiment.thompsonSamplingConfig?.priors + ); - // else, if mooclet is not involved, we can do a normal update - return this.experimentService.update({ ...experiment, id }, currentUser, request.logger); + if (conditionSetChanged || conditionFieldsChanged || priorsChanged) { + throw new BadRequestError( + `Conditions (including Thompson Sampling priors) cannot be modified once an experiment has started. Current state: ${previousExperiment.state}.` + ); + } + } + + private havePriorsChanged( + previousPriors: Record | undefined, + incomingPriors: Record | undefined + ): boolean { + // No priors submitted at all (e.g. a non-Thompson-Sampling experiment, or a caller that only + // sends the fields it means to change) -- nothing to compare or block. + if (!incomingPriors) { + return false; + } + + const previous = previousPriors ?? {}; + const keys = new Set([...Object.keys(previous), ...Object.keys(incomingPriors)]); + return [...keys].some((conditionId) => { + const previousPrior = previous[conditionId]; + const incomingPrior = incomingPriors[conditionId]; + return previousPrior?.success !== incomingPrior?.success || previousPrior?.failure !== incomingPrior?.failure; + }); } /** @@ -1937,20 +2038,6 @@ export class ExperimentController { return lists; } - /** - * Get Mooclet Rewards Feedback data - */ - @Get('/mooclet-rewards/:id') - public getMoocletRewards( - @Params({ validate: true }) { id }: IdValidator, - @Req() request: AppRequest - ): Promise { - if (!env.mooclets?.enabled) { - throw new BadRequestError('Mooclet is not enabled in the environment'); - } - return this.moocletRewardService.getRewardsSummaryForExperiment(id, request.logger); - } - /** * Debugging endpoint: a cache report for THIS instance. * diff --git a/packages/backend/src/api/errors/MoocletError.ts b/packages/backend/src/api/errors/MoocletError.ts deleted file mode 100644 index a503939764..0000000000 --- a/packages/backend/src/api/errors/MoocletError.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { SERVER_ERROR } from 'upgrade_types'; -import { ErrorWithType } from './ErrorWithType'; - -export class MoocletError extends ErrorWithType { - public httpCode: number; - - constructor(message: string, httpCode = 500) { - super(); - this.message = message; - this.type = SERVER_ERROR.MOOCLET_ERROR; - this.httpCode = httpCode; - this.name = 'MoocletError'; - } -} diff --git a/packages/backend/src/api/middlewares/ErrorHandlerMiddleware.ts b/packages/backend/src/api/middlewares/ErrorHandlerMiddleware.ts index 673fd0866d..c1668fc078 100644 --- a/packages/backend/src/api/middlewares/ErrorHandlerMiddleware.ts +++ b/packages/backend/src/api/middlewares/ErrorHandlerMiddleware.ts @@ -79,14 +79,6 @@ export class ErrorHandlerMiddleware implements ExpressErrorMiddlewareInterface { type = SERVER_ERROR.EMAIL_SEND_ERROR; message = errorMessage; break; - case SERVER_ERROR.MOOCLET_REWARD_ERROR: - type = SERVER_ERROR.MOOCLET_REWARD_ERROR; - message = errorMessage; - break; - case SERVER_ERROR.MOOCLET_ERROR: - type = SERVER_ERROR.MOOCLET_ERROR; - message = errorMessage; - break; default: switch (error.httpCode) { case 400: diff --git a/packages/backend/src/api/models/ConditionPosteriorState.ts b/packages/backend/src/api/models/ConditionPosteriorState.ts new file mode 100644 index 0000000000..a24bd79424 --- /dev/null +++ b/packages/backend/src/api/models/ConditionPosteriorState.ts @@ -0,0 +1,69 @@ +import { Entity, Column, JoinColumn, ManyToOne, PrimaryGeneratedColumn, Unique } from 'typeorm'; +import { ThompsonSamplingExperimentConfig } from './ThompsonSamplingExperimentConfig'; +import { ExperimentCondition } from './ExperimentCondition'; +import { Experiment } from './Experiment'; +import { BaseModel } from './base/BaseModel'; + +@Entity() +@Unique(['configId', 'conditionId']) +export class ConditionPosteriorState extends BaseModel { + @PrimaryGeneratedColumn('uuid') + public id: string; + + @ManyToOne(() => Experiment, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'experimentId' }) + experiment: Experiment; + + @Column() + experimentId: string; + + @ManyToOne(() => ThompsonSamplingExperimentConfig, (config) => config.conditionPosteriorStates, { + onDelete: 'CASCADE', + }) + @JoinColumn({ name: 'configId' }) + config: ThompsonSamplingExperimentConfig; + + @Column() + configId: string; + + @ManyToOne(() => ExperimentCondition, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'conditionId' }) + condition: ExperimentCondition; + + @Column() + conditionId: string; + + /** α₀ — Beta prior success count for this condition. */ + @Column({ type: 'float', default: 1 }) + priorSuccess: number; + + /** β₀ — Beta prior failure count for this condition. */ + @Column({ type: 'float', default: 1 }) + priorFailure: number; + + /** Accumulated successes since the experiment started. */ + @Column({ type: 'int', default: 0 }) + successCount: number; + + /** Accumulated failures since the experiment started. Always equal to totalCount - successCount; + * stored (not derived) so callers can read it directly rather than recomputing it everywhere. */ + @Column({ type: 'int', default: 0 }) + failureCount: number; + + /** Total rewards received for this condition (successes + failures). */ + @Column({ type: 'int', default: 0 }) + totalCount: number; + + /** Successes recorded since the last batch flush; folded into successCount once batchSize is reached. */ + @Column({ type: 'int', default: 0 }) + pendingSuccessCount: number; + + /** Failures recorded since the last batch flush; folded into failureCount once batchSize is reached. + * Always equal to pendingTotalCount - pendingSuccessCount; stored for the same reason as failureCount. */ + @Column({ type: 'int', default: 0 }) + pendingFailureCount: number; + + /** Rewards recorded since the last batch flush; folded into totalCount once batchSize is reached. */ + @Column({ type: 'int', default: 0 }) + pendingTotalCount: number; +} diff --git a/packages/backend/src/api/models/MoocletExperimentRef.ts b/packages/backend/src/api/models/MoocletExperimentRef.ts deleted file mode 100644 index 64e75762a1..0000000000 --- a/packages/backend/src/api/models/MoocletExperimentRef.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Entity, Column, OneToMany, PrimaryColumn, JoinColumn, OneToOne } from 'typeorm'; -import { MoocletVersionConditionMap } from './MoocletVersionConditionMap'; -import { Experiment } from './Experiment'; - -@Entity() -export class MoocletExperimentRef { - @PrimaryColumn('uuid') - public id?: string; - - @OneToOne(() => Experiment, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'experimentId' }) - experiment: Experiment; - - @Column({ nullable: true }) - experimentId?: string; - - @Column() - moocletId?: number; - - @OneToMany(() => MoocletVersionConditionMap, (versionConditionMap) => versionConditionMap.moocletExperimentRef, { - cascade: true, - onDelete: 'CASCADE', - }) - versionConditionMaps: MoocletVersionConditionMap[]; - - @Column({ nullable: true }) - policyId?: number; - - @Column({ nullable: true }) - policyParametersId?: number; - - @Column({ nullable: true }) - variableId?: number; - - @Column({ nullable: true }) - outcomeVariableName?: string; -} diff --git a/packages/backend/src/api/models/MoocletVersionConditionMap.ts b/packages/backend/src/api/models/MoocletVersionConditionMap.ts deleted file mode 100644 index 8eda1e474f..0000000000 --- a/packages/backend/src/api/models/MoocletVersionConditionMap.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Entity, Column, JoinColumn, ManyToOne, PrimaryGeneratedColumn } from 'typeorm'; -import { MoocletExperimentRef } from './MoocletExperimentRef'; -import { ExperimentCondition } from './ExperimentCondition'; - -@Entity() -export class MoocletVersionConditionMap { - @PrimaryGeneratedColumn() - public id?: number; - - @ManyToOne(() => MoocletExperimentRef, (moocletExperimentRef) => moocletExperimentRef.versionConditionMaps, { - onDelete: 'CASCADE', - }) - @JoinColumn({ name: 'moocletExperimentRefId' }) - moocletExperimentRef: MoocletExperimentRef; - - @Column() - moocletExperimentRefId?: string; - - @ManyToOne(() => ExperimentCondition, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'experimentConditionId' }) - experimentCondition: ExperimentCondition; - - @Column() - experimentConditionId?: string; - - @Column() - moocletVersionId?: number; -} diff --git a/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts b/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts new file mode 100644 index 0000000000..8b7fe87e43 --- /dev/null +++ b/packages/backend/src/api/models/ThompsonSamplingExperimentConfig.ts @@ -0,0 +1,34 @@ +import { Entity, Column, OneToMany, PrimaryGeneratedColumn, JoinColumn, OneToOne } from 'typeorm'; +import { ConditionPosteriorState } from './ConditionPosteriorState'; +import { Experiment } from './Experiment'; +import { BaseModel } from './base/BaseModel'; + +@Entity() +export class ThompsonSamplingExperimentConfig extends BaseModel { + @PrimaryGeneratedColumn('uuid') + public id: string; + + @OneToOne(() => Experiment, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'experimentId' }) + experiment: Experiment; + + @Column({ nullable: true }) + experimentId?: string; + + /** Use uniform random selection until total reward observations exceed this count. */ + @Column({ default: 0 }) + warmupThreshold: number; + + /** Fall back to uniform when the top two sampled draws differ by less than this value. */ + @Column({ type: 'float', default: 0 }) + minimumDrawDifference: number; + + /** Update posteriors every N reward events rather than on every reward. */ + @Column({ default: 1 }) + batchSize: number; + + @OneToMany(() => ConditionPosteriorState, (state) => state.config, { + cascade: true, + }) + conditionPosteriorStates: ConditionPosteriorState[]; +} diff --git a/packages/backend/src/api/models/ThompsonSamplingReward.ts b/packages/backend/src/api/models/ThompsonSamplingReward.ts new file mode 100644 index 0000000000..46d75ad237 --- /dev/null +++ b/packages/backend/src/api/models/ThompsonSamplingReward.ts @@ -0,0 +1,31 @@ +import { Entity, Column, ManyToOne, PrimaryGeneratedColumn, JoinColumn, Index } from 'typeorm'; +import { Experiment } from './Experiment'; +import { ExperimentCondition } from './ExperimentCondition'; +import { BaseModel } from './base/BaseModel'; + +@Entity() +@Index(['experimentId', 'conditionId']) +export class ThompsonSamplingReward extends BaseModel { + @PrimaryGeneratedColumn('uuid') + public id: string; + + @ManyToOne(() => Experiment, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'experimentId' }) + experiment: Experiment; + + @Column() + experimentId: string; + + @ManyToOne(() => ExperimentCondition, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'conditionId' }) + condition: ExperimentCondition; + + @Column() + conditionId: string; + + @Column() + userId: string; + + @Column({ type: 'boolean' }) + success: boolean; +} diff --git a/packages/backend/src/api/repositories/ConditionPosteriorStateRepository.ts b/packages/backend/src/api/repositories/ConditionPosteriorStateRepository.ts new file mode 100644 index 0000000000..373a5f33b0 --- /dev/null +++ b/packages/backend/src/api/repositories/ConditionPosteriorStateRepository.ts @@ -0,0 +1,17 @@ +import { Repository } from 'typeorm'; +import { EntityRepository } from '../../typeorm-typedi-extensions'; +import { ConditionPosteriorState } from '../models/ConditionPosteriorState'; + +@EntityRepository(ConditionPosteriorState) +export class ConditionPosteriorStateRepository extends Repository { + public async findByConfigId(configId: string): Promise { + return this.createQueryBuilder('state') + .leftJoinAndSelect('state.condition', 'condition') + .where('state.configId = :configId', { configId }) + .getMany(); + } + + public async findByConditionId(conditionId: string): Promise { + return this.createQueryBuilder('state').where('state.conditionId = :conditionId', { conditionId }).getOne(); + } +} diff --git a/packages/backend/src/api/repositories/MoocletExperimentRefRepository.ts b/packages/backend/src/api/repositories/MoocletExperimentRefRepository.ts deleted file mode 100644 index 7f911162a0..0000000000 --- a/packages/backend/src/api/repositories/MoocletExperimentRefRepository.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Repository } from 'typeorm'; -import { EntityRepository } from '../../typeorm-typedi-extensions'; -import { MoocletExperimentRef } from '../models/MoocletExperimentRef'; -import { EXPERIMENT_STATE } from 'upgrade_types'; - -@EntityRepository(MoocletExperimentRef) -export class MoocletExperimentRefRepository extends Repository { - public async getRefsForActivelyEnrollingExperiments(): Promise { - return this.createQueryBuilder('moocletExperimentRef') - .leftJoinAndSelect('moocletExperimentRef.versionConditionMaps', 'versionConditionMaps') - .leftJoinAndSelect('moocletExperimentRef.experiment', 'experiment') - .where('experiment.state = :status', { status: EXPERIMENT_STATE.ENROLLING }) - .getMany(); - } - - public async findActivelyEnrollingMoocletExperimentsByContextSiteTarget( - context: string, - site: string, - target: string - ): Promise { - return this.createQueryBuilder('moocletExperimentRef') - .leftJoinAndSelect('moocletExperimentRef.versionConditionMaps', 'versionConditionMaps') - .leftJoinAndSelect('moocletExperimentRef.experiment', 'experiment') - .leftJoinAndSelect('experiment.partitions', 'decisionPoint') - .where('experiment.state = :status', { status: EXPERIMENT_STATE.ENROLLING }) - .andWhere(':context = ANY(experiment.context)', { context }) - .andWhere('decisionPoint.site = :site', { site }) - .andWhere('decisionPoint.target = :target', { target }) - .getMany(); - } -} diff --git a/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts b/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts new file mode 100644 index 0000000000..5575051cf2 --- /dev/null +++ b/packages/backend/src/api/repositories/ThompsonSamplingExperimentConfigRepository.ts @@ -0,0 +1,57 @@ +import { Repository } from 'typeorm'; +import { EntityRepository } from '../../typeorm-typedi-extensions'; +import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; +import { ASSIGNMENT_ALGORITHM, EXPERIMENT_STATE } from 'upgrade_types'; + +@EntityRepository(ThompsonSamplingExperimentConfig) +export class ThompsonSamplingExperimentConfigRepository extends Repository { + public async findByExperimentId(experimentId: string): Promise { + return this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .where('config.experimentId = :experimentId', { experimentId }) + .getOne(); + } + + /** + * Same as findByExperimentId, but also joins each posterior state's condition — needed to display + * the condition's code/order alongside its reward counts (e.g. the rewards summary endpoint). + */ + public async findByExperimentIdWithConditions(experimentId: string): Promise { + return this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .leftJoinAndSelect('conditionPosteriorStates.condition', 'condition') + .where('config.experimentId = :experimentId', { experimentId }) + .getOne(); + } + + public async findByDecisionPoint( + context: string, + site: string, + target: string + ): Promise { + return ( + this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .leftJoinAndSelect('config.experiment', 'experiment') + .leftJoinAndSelect('experiment.partitions', 'decisionPoint') + .where('experiment.state = :state', { state: EXPERIMENT_STATE.ENROLLING }) + // Defends against a stale config row surviving an algorithm change away from Thompson + // Sampling (deleteConfigIfExists() is the primary fix -- this guards against any config row + // that outlives it, e.g. one created before that fix shipped). + .andWhere('experiment.assignmentAlgorithm = :algorithm', { algorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }) + .andWhere(':context = ANY(experiment.context)', { context }) + .andWhere('decisionPoint.site = :site', { site }) + .andWhere('decisionPoint.target = :target', { target }) + .getMany() + ); + } + + public async findConfigsForActivelyEnrollingExperiments(): Promise { + return this.createQueryBuilder('config') + .leftJoinAndSelect('config.conditionPosteriorStates', 'conditionPosteriorStates') + .leftJoinAndSelect('config.experiment', 'experiment') + .where('experiment.state = :state', { state: EXPERIMENT_STATE.ENROLLING }) + .andWhere('experiment.assignmentAlgorithm = :algorithm', { algorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }) + .getMany(); + } +} diff --git a/packages/backend/src/api/repositories/ThompsonSamplingRewardRepository.ts b/packages/backend/src/api/repositories/ThompsonSamplingRewardRepository.ts new file mode 100644 index 0000000000..12dc2ea21e --- /dev/null +++ b/packages/backend/src/api/repositories/ThompsonSamplingRewardRepository.ts @@ -0,0 +1,17 @@ +import { Repository } from 'typeorm'; +import { EntityRepository } from '../../typeorm-typedi-extensions'; +import { ThompsonSamplingReward } from '../models/ThompsonSamplingReward'; + +@EntityRepository(ThompsonSamplingReward) +export class ThompsonSamplingRewardRepository extends Repository { + public async findByExperimentAndCondition( + experimentId: string, + conditionId: string + ): Promise { + return this.createQueryBuilder('reward') + .where('reward.experimentId = :experimentId', { experimentId }) + .andWhere('reward.conditionId = :conditionId', { conditionId }) + .orderBy('reward.createdAt', 'ASC') + .getMany(); + } +} diff --git a/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts b/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts new file mode 100644 index 0000000000..5d2f551332 --- /dev/null +++ b/packages/backend/src/api/services/AdaptiveExperimentConfigDispatcherService.ts @@ -0,0 +1,43 @@ +import { Service } from 'typedi'; +import { ExperimentDTO } from '../DTO/ExperimentDTO'; +import { AdaptiveExperimentConfigService } from './AdaptiveExperimentConfigService'; +import { ThompsonSamplingExperimentCrudService } from './ThompsonSamplingExperimentCrudService'; + +/** + * Fans the config lifecycle for an experiment out to every registered adaptive + * algorithm's config service. Each service self-gates on assignmentAlgorithm, so + * adding a second adaptive algorithm means adding its service to `services` here -- + * no branching needed at the call sites in ExperimentController/ImportExportService. + */ +@Service() +export class AdaptiveExperimentConfigDispatcherService implements AdaptiveExperimentConfigService { + private readonly services: AdaptiveExperimentConfigService[]; + + constructor(thompsonSamplingCrudService: ThompsonSamplingExperimentCrudService) { + this.services = [thompsonSamplingCrudService]; + } + + public async createConfigIfApplicable( + experiment: ExperimentDTO, + createdExperiment: ExperimentDTO, + originalConditionIds?: string[] + ): Promise { + for (const service of this.services) { + await service.createConfigIfApplicable(experiment, createdExperiment, originalConditionIds); + } + } + + public async syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise { + for (const service of this.services) { + await service.syncConfigIfApplicable(experiment, updatedExperiment); + } + } + + public async attachConfigToExperiment(experiment: T): Promise { + let result = experiment; + for (const service of this.services) { + result = await service.attachConfigToExperiment(result); + } + return result; + } +} diff --git a/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts b/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts new file mode 100644 index 0000000000..7c2792d804 --- /dev/null +++ b/packages/backend/src/api/services/AdaptiveExperimentConfigService.ts @@ -0,0 +1,23 @@ +import { ExperimentDTO } from '../DTO/ExperimentDTO'; + +/** + * Contract for an adaptive assignment algorithm's per-experiment config lifecycle + * (Thompson Sampling today). Each implementation checks experiment.assignmentAlgorithm + * itself and no-ops when it doesn't apply, so callers can dispatch to every registered + * implementation without branching on algorithm. + */ +export interface AdaptiveExperimentConfigService { + /** + * `originalConditionIds` is the condition ID list exactly as submitted by the caller, captured + * before ExperimentService.create()/deduceConditions() replace every condition ID with a freshly + * generated one. Implementations that key caller-supplied data (e.g. priors) by condition ID need + * this to remap those keys onto `createdExperiment`'s actual (post-creation) condition IDs. + */ + createConfigIfApplicable( + experiment: ExperimentDTO, + createdExperiment: ExperimentDTO, + originalConditionIds?: string[] + ): Promise; + syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise; + attachConfigToExperiment(experiment: T): Promise; +} diff --git a/packages/backend/src/api/services/CacheService.ts b/packages/backend/src/api/services/CacheService.ts index d73c0af493..cf88873dc4 100644 --- a/packages/backend/src/api/services/CacheService.ts +++ b/packages/backend/src/api/services/CacheService.ts @@ -14,6 +14,7 @@ const PREFIX_CATEGORY: Record = { [CACHE_PREFIX.FEATURE_FLAG_PRECOMPUTED_SEGMENT_KEY_PREFIX]: 'featureFlags', [CACHE_PREFIX.EXPERIMENT_PRECOMPUTED_SEGMENT_KEY_PREFIX]: 'experiments', [CACHE_PREFIX.SETTING_KEY_PREFIX]: 'settings', + [CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX]: 'experiments', }; // this module will get swapped in if caching is enabled but the cache manager fails to initialize as a dummy default deliverer diff --git a/packages/backend/src/api/services/ExperimentAssignmentService.ts b/packages/backend/src/api/services/ExperimentAssignmentService.ts index 4c5fae8ae5..d928888d34 100644 --- a/packages/backend/src/api/services/ExperimentAssignmentService.ts +++ b/packages/backend/src/api/services/ExperimentAssignmentService.ts @@ -69,12 +69,12 @@ import { UserStratificationFactorRepository } from '../repositories/UserStratifi import { UserStratificationFactor } from '../models/UserStratificationFactor'; import { RequestedExperimentUser } from '../controllers/validators/ExperimentUserValidator'; import { Brackets, In } from 'typeorm'; -import { env } from '../../env'; -import { MoocletExperimentService } from './MoocletExperimentService'; import { ExperimentPrecomputedSegmentService } from './ExperimentPrecomputedSegmentService'; import { ExperimentPrecomputedSegment } from '../models/ExperimentPrecomputedSegment'; import { precomputedGroupKey } from './precomputedSegmentHelpers'; import { EntitySegmentMembers, EntitySegmentResolutionInput, SegmentGroupMember } from '../../types'; +import { ThompsonSamplingService, ThompsonSamplingConfig, ConditionRewardSummary } from './ThompsonSamplingService'; +import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; export interface FactorialConditionResult { factorialCondition: Omit; @@ -117,6 +117,9 @@ export class ExperimentAssignmentService { @InjectRepository() private userStratificationFactorRepository: UserStratificationFactorRepository, + @InjectRepository() + private thompsonSamplingConfigRepository: ThompsonSamplingExperimentConfigRepository, + public previewUserService: PreviewUserService, public experimentUserService: ExperimentUserService, public errorService: ErrorService, @@ -124,8 +127,8 @@ export class ExperimentAssignmentService { public segmentService: SegmentService, public experimentService: ExperimentService, public cacheService: CacheService, - public moocletExperimentService: MoocletExperimentService, - public experimentPrecomputedSegmentService: ExperimentPrecomputedSegmentService + public experimentPrecomputedSegmentService: ExperimentPrecomputedSegmentService, + public thompsonSamplingService: ThompsonSamplingService ) {} /** @@ -569,7 +572,15 @@ export class ExperimentAssignmentService { logger.info({ message: `getAllExperimentConditions: User: ${experimentUserDocs.map((doc) => doc.id).join(', ')}`, }); - const experiments: Experiment[] = await this.getExperimentsForContextAndDecisionPoint(context, site, target); + // Adaptive algorithms (currently just Thompson Sampling) are deliberately excluded from batch-assign. + // This endpoint neither persists an enrollment nor is paired with a /mark call, so an adaptive draw here + // is a live, non-deterministic bandit sample disconnected from reward learning and from whatever /assign + // or /mark would separately say for the same user -- there's no coherent product meaning for it. This + // endpoint has no current callers, so rather than design that seam now, adaptive experiments are just + // filtered out until a real use case forces the question. + const experiments: Experiment[] = ( + await this.getExperimentsForContextAndDecisionPoint(context, site, target) + ).filter((experiment) => experiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING); if (experiments.length === 0) { return {}; @@ -1844,7 +1855,10 @@ export class ExperimentAssignmentService { const promiseArray = []; let conditionAssigned; if (!noGroupSpecified && !invalidGroup) { - if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING) { + if ( + experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING || + experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING + ) { conditionAssigned = experiment.conditions.find((expCondition) => expCondition.conditionCode === condition); } else { conditionAssigned = await this.assignExperiment( @@ -1941,26 +1955,17 @@ export class ExperimentAssignmentService { }; await this.repeatedEnrollmentRepository.save(RepeatedEnrollmentDocument); } else { - let conditionAssigned: ExperimentCondition | void; - - const isMoocletExperiment = this.moocletExperimentService.isMoocletExperiment(experiment.assignmentAlgorithm); - - if (isMoocletExperiment) { - conditionAssigned = await this.moocletExperimentService.handleEnrollCondition( - experiment.id, - condition, - logger - ); - } else { - conditionAssigned = await this.assignExperiment( - user, - experiment, - individualEnrollment, - groupEnrollment, - individualExclusion, - groupExclusion - ); - } + const conditionAssigned = + experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING + ? experiment.conditions.find((expCondition) => expCondition.conditionCode === condition) + : await this.assignExperiment( + user, + experiment, + individualEnrollment, + groupEnrollment, + individualExclusion, + groupExclusion + ); if (!individualEnrollment && !individualExclusion && conditionAssigned) { const individualEnrollmentDocument: Omit = { @@ -2068,34 +2073,69 @@ export class ExperimentAssignmentService { logger: UpgradeLogger, enrollmentCount?: { conditionId: string; userCount: number }[] ): Promise { - const isMoocletExperiment = this.moocletExperimentService.isMoocletExperiment(experiment.assignmentAlgorithm); - - if (isMoocletExperiment && !env.mooclets.enabled) { - logger.error({ - message: 'Mooclet experiment algorithm is indicated but mooclets are not enabled', - experiment, - user, - }); - return undefined; + if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return this.assignThompsonSampling(experiment, user, logger); } - if (isMoocletExperiment && env.mooclets.enabled) { - return this.getConditionFromMoocletProxy(experiment, user, logger); - } else { - return this.assignRandom(experiment, user, enrollmentCount); - } + return this.assignRandom(experiment, user, enrollmentCount); } - private async getConditionFromMoocletProxy(experiment: Experiment, user: ExperimentUser, logger: UpgradeLogger) { - const userId = user.id; - + private async assignThompsonSampling( + experiment: Experiment, + user: ExperimentUser, + logger: UpgradeLogger + ): Promise { try { - return await this.moocletExperimentService.getConditionFromMoocletProxy(experiment, userId, logger); + const config = await this.thompsonSamplingConfigRepository.findByExperimentId(experiment.id); + + if (!config) { + logger.error({ + message: 'Thompson Sampling config not found for experiment; no condition assigned', + experimentId: experiment.id, + userId: user.id, + }); + return undefined; + } + + // Use conditionId as the identifier throughout — conditionCode is nullable + const conditionIds = experiment.conditions.map((c) => c.id); + + const rewardSummaries: ConditionRewardSummary[] = config.conditionPosteriorStates.map((state) => ({ + conditionId: state.conditionId, + successCount: state.successCount, + failureCount: state.failureCount, + totalCount: state.totalCount, + })); + + const priors = this.thompsonSamplingService.buildPriorsRecord(config.conditionPosteriorStates); + + // Include pendingTotalCount so a batchSize buffer awaiting flush still counts toward + // warmup evidence — rewards are "collected" as soon as recordReward() persists them, + // even if batching hasn't folded them into totalCount/successCount/failureCount yet. + const totalRewardCount = config.conditionPosteriorStates.reduce( + (sum, s) => sum + s.totalCount + s.pendingTotalCount, + 0 + ); + + const tsConfig: ThompsonSamplingConfig = { + priors, + warmupThreshold: config.warmupThreshold, + minimumDrawDifference: config.minimumDrawDifference, + }; + + const selectedConditionId = this.thompsonSamplingService.selectCondition( + conditionIds, + rewardSummaries, + totalRewardCount, + tsConfig + ); + + return experiment.conditions.find((c) => c.id === selectedConditionId); } catch (err) { logger.error({ - message: 'Error getting condition from Mooclet proxy; experiment will return no condition for this user', + message: 'Error in Thompson Sampling assignment; no condition assigned', experimentId: experiment.id, - userId, + userId: user.id, error: err, }); return undefined; diff --git a/packages/backend/src/api/services/ExperimentService.ts b/packages/backend/src/api/services/ExperimentService.ts index 640070ee0c..e17b7c0d56 100644 --- a/packages/backend/src/api/services/ExperimentService.ts +++ b/packages/backend/src/api/services/ExperimentService.ts @@ -103,7 +103,6 @@ import { plainToClass } from 'class-transformer'; import { StratificationFactorRepository } from '../repositories/StratificationFactorRepository'; import { ExperimentDetailsForCSVData } from '../repositories/AnalyticsRepository'; import { MetricService } from './MetricService'; -import { MoocletExperimentRefRepository } from '../repositories/MoocletExperimentRefRepository'; import { ExperimentAuditLog } from '../models/ExperimentAuditLog'; import { SegmentRepository } from '../repositories/SegmentRepository'; import { NotFoundException } from '@nestjs/common/exceptions'; @@ -141,7 +140,6 @@ export class ExperimentService { @InjectRepository() protected levelCombinationElementsRepository: LevelCombinationElementRepository, @InjectRepository() protected archivedStatsRepository: ArchivedStatsRepository, @InjectRepository() protected stratificationRepository: StratificationFactorRepository, - @InjectRepository() protected moocletExperimentRefRepository: MoocletExperimentRefRepository, @InjectDataSource() protected dataSource: DataSource, protected previewUserService: PreviewUserService, protected segmentService: SegmentService, @@ -414,7 +412,7 @@ export class ExperimentService { // Populate the experiment_precomputed_segment row from the just-attached lists. When an // existingEntityManager was provided the caller owns the (still-open) transaction, so it — not - // create — must recompute after commit (see MoocletExperimentService); recomputing here would + // create — must recompute after commit; recomputing here would // read uncommitted writes. recomputeForExperiment yields empty arrays when the experiment has no // lists, so this also seeds list-less experiments (no separate empty-seed needed). if (!existingEntityManager) { @@ -1289,7 +1287,7 @@ export class ExperimentService { // filterMode was forced to EXCLUDE_ALL, but the experiment_precomputed_segment row still holds the // old member IDs. Recompute it (to empty) after commit so the assignment read path never serves // stale inclusion/exclusion data. Mirrors FeatureFlagService.updateFeatureFlagInDB's withRecompute. - // When a caller owns the transaction (existingEntityManager, e.g. MoocletExperimentService), the + // When a caller owns the transaction (existingEntityManager), the // writes are not committed yet, so the caller recomputes after its own commit (see syncUpdate) — // the same deferral create() uses. if (isChangingContext && !existingEntityManager) { @@ -1604,14 +1602,6 @@ export class ExperimentService { const experimentJSONValidationError = await this.validateExperimentJSON(newExperiment); const fileName = experimentFile.fileName; - if ('moocletPolicyParameters' in newExperiment && !env.mooclets?.enabled) { - return { - fileName, - error: 'moocletPolicyParameters was provided but mooclets are not enabled on backend.', - compatibilityType: IMPORT_COMPATIBILITY_TYPE.INCOMPATIBLE, - }; - } - try { experiment = this.autoFillSomeMissingProperties(experiment); experiment = this.deduceExperimentDetails(experiment); @@ -1755,6 +1745,16 @@ export class ExperimentService { if (result.revertTo && this.allIdMap[result.revertTo]) { result.revertTo = this.allIdMap[result.revertTo]; } + if (result.thompsonSamplingConfig?.priors) { + const remappedPriors = {}; + Object.entries(result.thompsonSamplingConfig.priors).forEach(([oldConditionId, prior]) => { + const newConditionId = this.allIdMap[oldConditionId]; + if (newConditionId) { + remappedPriors[newConditionId] = prior; + } + }); + result.thompsonSamplingConfig.priors = remappedPriors; + } } deduceConditionPayload(result) { @@ -2119,7 +2119,7 @@ export class ExperimentService { // The caller owns the outer transaction. We must NOT recompute here: recomputeForExperiment // reads through its own repositories and cannot see this transaction's uncommitted writes, so // it would persist a stale experiment_precomputed_segment row that never self-heals. The caller - // (create / importExperimentLists / MoocletExperimentService) recomputes after it commits. + // (create / importExperimentLists) recomputes after it commits. return await executeTransaction(transactionalEntityManager); } else { // withRecompute runs the mutation in its own transaction, then fires a fire-and-forget diff --git a/packages/backend/src/api/services/ImportExportService.ts b/packages/backend/src/api/services/ImportExportService.ts index 5d9079757e..a09d8e02dd 100644 --- a/packages/backend/src/api/services/ImportExportService.ts +++ b/packages/backend/src/api/services/ImportExportService.ts @@ -3,13 +3,13 @@ import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; import { ExperimentService } from './ExperimentService'; import { ExperimentDTO, ExperimentFile } from '../DTO/ExperimentDTO'; import { env } from '../../env'; -import { MoocletExperimentService } from './MoocletExperimentService'; import { UserDTO } from '../DTO/UserDTO'; -import { LOG_TYPE, SUPPORTED_MOOCLET_ALGORITHMS } from 'upgrade_types'; +import { LOG_TYPE } from 'upgrade_types'; import { In } from 'typeorm'; import { InjectRepository } from '../../typeorm-typedi-extensions'; import { ExperimentRepository } from '../repositories/ExperimentRepository'; import { ExperimentAuditLogRepository } from '../repositories/ExperimentAuditLogRepository'; +import { AdaptiveExperimentConfigDispatcherService } from './AdaptiveExperimentConfigDispatcherService'; @Service() export class ImportExportService { @@ -17,7 +17,7 @@ export class ImportExportService { @InjectRepository() protected experimentRepository: ExperimentRepository, @InjectRepository() protected experimentAuditLogRepository: ExperimentAuditLogRepository, protected experimentService: ExperimentService, - protected moocletExperimentService: MoocletExperimentService + protected adaptiveExperimentConfigDispatcher: AdaptiveExperimentConfigDispatcherService ) {} public async importExperiments(experiments: ExperimentFile[], user: UserDTO, logger: UpgradeLogger) { @@ -41,19 +41,23 @@ export class ImportExportService { await Promise.all( experiments.map(async (experiment) => { try { - if (this.moocletExperimentService.isMoocletExperiment(experiment.assignmentAlgorithm)) { - if (!env.mooclets.enabled) { - throw new Error('Attempting to import a moclet experiment, but mooclets are not enabled'); - } - await this.moocletExperimentService.syncCreate({ - experimentDTO: experiment, - currentUser, - logger, - }); - } else { - const result = await this.experimentService.create(experiment, currentUser, logger); - createdExperiments.push(result); + // Captured before create() runs -- see ExperimentController.create() for why: create() + // mutates condition ids in place, so this is the last point they still match whatever ids + const originalConditionIds = experiment.conditions?.map((condition) => condition.id); + const result = await this.experimentService.create(experiment, currentUser, logger); + try { + await this.adaptiveExperimentConfigDispatcher.createConfigIfApplicable( + experiment, + result, + originalConditionIds + ); + } catch (configError) { + // Same reasoning as the single-experiment POST /experiments path: don't leave an + // orphaned, config-less Thompson Sampling experiment behind when this step fails. + await this.experimentService.delete(result.id, currentUser, { logger }); + throw configError; } + createdExperiments.push(await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment(result)); } catch (error) { logger.error({ message: 'Failed to create experiment during import', @@ -133,32 +137,12 @@ export class ImportExportService { return a.order - b.order; }); - let experimentRecord = this.experimentService.reducedConditionPayload( - this.experimentService.formattingPayload(this.experimentService.formattingConditionPayload(experiment)) + const experimentRecord = await this.adaptiveExperimentConfigDispatcher.attachConfigToExperiment( + this.experimentService.reducedConditionPayload( + this.experimentService.formattingPayload(this.experimentService.formattingConditionPayload(experiment)) + ) ); - // If it's a mooclet experiment, policy parameters - if (SUPPORTED_MOOCLET_ALGORITHMS.includes(experiment.assignmentAlgorithm)) { - try { - experimentRecord = await this.moocletExperimentService.attachPolicyParamsToExperimentDTO( - experimentRecord, - logger - ); - } catch (error) { - logger.error({ - message: 'Failed to get mooclet data for experiment', - error: error, - experiment: experiment, - user: user, - }); - throw error; - } - // remove currentPosteriors from moocletPolicyParameters for export - const { current_posteriors: _, ...filteredPolictParameters } = experimentRecord.moocletPolicyParameters; - - experimentRecord.moocletPolicyParameters = filteredPolictParameters; - } - this.experimentAuditLogRepository.saveRawJson( LOG_TYPE.EXPERIMENT_DESIGN_EXPORTED, { experimentId: experiment.id, experimentName: experiment.name }, diff --git a/packages/backend/src/api/services/MoocletDataService.ts b/packages/backend/src/api/services/MoocletDataService.ts deleted file mode 100644 index 5cf609a606..0000000000 --- a/packages/backend/src/api/services/MoocletDataService.ts +++ /dev/null @@ -1,394 +0,0 @@ -import { Service } from 'typedi'; -import { env } from '../../env'; - -import axios, { AxiosRequestConfig } from 'axios'; -import { - MoocletBatchResponse, - MoocletPolicyResponseDetails, - MoocletProxyRequestParams, - MoocletRequestBody, - MoocletResponseDetails, - MoocletVersionRequestBody, - MoocletVersionResponseDetails, - MoocletPolicyParametersRequestBody, - MoocletPolicyParametersResponseDetails, - MoocletVariableRequestBody, - MoocletVariableResponseDetails, - MoocletValueRequestBody, - MoocletValueResponseDetails, - MoocletRewardCountRequestBody, - MoocletPaginatedResponse, -} from '../../types/Mooclet'; -import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; -import { MoocletError } from '../errors/MoocletError'; - -@Service() -export class MoocletDataService { - private apiUrl = env.mooclets.hostUrl + env.mooclets.apiRoute; - private apiToken = 'Token ' + env.mooclets.apiToken; - - /************************************************************************************************* - * EXTERNAL DATA FETCHING METHODS - */ - - public async getMoocletIdByName(policyName: string, logger: UpgradeLogger): Promise { - const response: MoocletBatchResponse = await this.getPoliciesList(logger); - let matchedPolicy: MoocletPolicyResponseDetails = null; - - if (response?.results.length) { - matchedPolicy = response.results.find((policy) => policy.name === policyName); - } - - if (matchedPolicy) { - return matchedPolicy.id; - } - - return null; - } - - public async getPoliciesList(logger: UpgradeLogger): Promise> { - const endpoint = '/policy'; - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response: MoocletBatchResponse = await this.fetchExternalMoocletsData( - requestParams, - logger - ); - - return response; - } - - public async postNewMooclet(requestBody: MoocletRequestBody, logger: UpgradeLogger): Promise { - const endpoint = '/mooclet'; - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async deleteMooclet(moocletId: number, logger: UpgradeLogger): Promise { - const endpoint = `/mooclet/${moocletId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'DELETE', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async postNewVersion( - requestBody: MoocletVersionRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = '/version'; - - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getVersion(versionId: number, logger: UpgradeLogger): Promise { - const endpoint = `/version/${versionId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async updateVersion( - versionId: number, - requestBody: MoocletVersionRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = `/version/${versionId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'PUT', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async deleteVersion(versionId: number, logger: UpgradeLogger): Promise { - const endpoint = `/version/${versionId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'DELETE', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async postNewPolicyParameters( - requestBody: MoocletPolicyParametersRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = '/policyparameters'; - - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getPolicyParameters( - policyParametersId: number, - logger: UpgradeLogger - ): Promise { - const endpoint = `/policyparameters/${policyParametersId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async updatePolicyParameters( - policyParametersId: number, - policyParameters: MoocletPolicyParametersRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = `/policyparameters/${policyParametersId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'PUT', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: policyParameters, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async deletePolicyParameters(policyParametersId: number, logger: UpgradeLogger): Promise { - const endpoint = `/policyparameters/${policyParametersId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'DELETE', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async postNewReward( - requestBody: MoocletValueRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = `/value?learner=${requestBody.learner}`; - - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getRewardsForExperiment( - requestBody: MoocletRewardCountRequestBody, - logger: UpgradeLogger, - nextPageUrl?: string - ): Promise> { - // this endpoint serves a paginated response - // if there are more results "pages" mooclet api sends the exact url to use for "next" page - // else it is nul/undefined and we'll fetch from the beginning - const url = - nextPageUrl || `${this.apiUrl}/value?mooclet=${requestBody.moocletId}&variable__name=${requestBody.variableName}`; - - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async postNewVariable( - requestBody: MoocletVariableRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = '/variable'; - - const requestParams: MoocletProxyRequestParams = { - method: 'POST', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getVariable(variableId: number, logger: UpgradeLogger): Promise { - const endpoint = `/variable/${variableId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async updateVariable( - variableId: number, - requestBody: MoocletVariableRequestBody, - logger: UpgradeLogger - ): Promise { - const endpoint = `/variable/${variableId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'PUT', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - body: requestBody, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async deleteVariable(variableId: number, logger: UpgradeLogger): Promise { - const endpoint = `/variable/${variableId}`; - const requestParams: MoocletProxyRequestParams = { - method: 'DELETE', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - public async getVersionForNewLearner( - moocletId: number, - userId: string, - logger: UpgradeLogger - ): Promise { - const endpoint = `/mooclet/${moocletId}/run?learner=${userId}`; - - const requestParams: MoocletProxyRequestParams = { - method: 'GET', - url: this.apiUrl + endpoint, - apiToken: this.apiToken, - }; - - const response = await this.fetchExternalMoocletsData(requestParams, logger); - - return response; - } - - /** - * Generic Requests to Mooclets API - */ - - public async fetchExternalMoocletsData( - requestParams: MoocletProxyRequestParams, - logger: UpgradeLogger - ): Promise { - const { method, url, body } = requestParams; - if (method && url) { - const headers: HeadersInit = { - Authorization: this.apiToken, - 'Content-Type': 'application/json', - }; - - const JSONbody = JSON.stringify(body); - - logger.info({ message: `Fetching data from Mooclets API: ${url}`, body: JSONbody }); - - try { - const options: AxiosRequestConfig = { - method, - data: JSONbody, - headers, - url, - }; - - const res = await axios.request(options); - return res.data; - } catch (err) { - if (err instanceof MoocletError) { - throw err; - } - if (axios.isAxiosError(err)) { - logger.error({ - message: 'Error fetching data from Mooclets API', - url, - method, - status: err.response?.status, - responseBody: err.response?.data, - error: err, - }); - throw new MoocletError(`Mooclet server returned non-2xx status: ${err.response?.status}`); - } else { - logger.error({ - message: 'Error fetching data from Mooclets API', - url, - method, - error: err, - }); - throw new MoocletError('Failed to communicate with Mooclet server'); - } - } - } - } -} diff --git a/packages/backend/src/api/services/MoocletExperimentService.ts b/packages/backend/src/api/services/MoocletExperimentService.ts deleted file mode 100644 index 3133a51d3e..0000000000 --- a/packages/backend/src/api/services/MoocletExperimentService.ts +++ /dev/null @@ -1,1630 +0,0 @@ -import { Service } from 'typedi'; -import { MoocletDataService } from './MoocletDataService'; -import { - MoocletPolicyParametersRequestBody, - MoocletPolicyParametersResponseDetails, - MoocletRequestBody, - MoocletResponseDetails, - MoocletVariableRequestBody, - MoocletVariableResponseDetails, - MoocletVersionRequestBody, - MoocletVersionResponseDetails, -} from '../../types/Mooclet'; -import { ExperimentService } from './ExperimentService'; -import { ExperimentPrecomputedSegmentService } from './ExperimentPrecomputedSegmentService'; -import { MoocletError } from '../errors/MoocletError'; -import { ExperimentRepository } from '../repositories/ExperimentRepository'; -import { ExperimentConditionRepository } from '../repositories/ExperimentConditionRepository'; -import { InjectRepository, InjectDataSource } from '../../typeorm-typedi-extensions'; -import { DataSource, EntityManager } from 'typeorm'; -import { ArchivedStatsRepository } from '../repositories/ArchivedStatsRepository'; -import { ConditionPayloadRepository } from '../repositories/ConditionPayloadRepository'; -import { SegmentRepository } from '../repositories/SegmentRepository'; -import { DecisionPointRepository } from '../repositories/DecisionPointRepository'; -import { ExperimentAuditLogRepository } from '../repositories/ExperimentAuditLogRepository'; -import { ExperimentSegmentExclusionRepository } from '../repositories/ExperimentSegmentExclusionRepository'; -import { ExperimentSegmentInclusionRepository } from '../repositories/ExperimentSegmentInclusionRepository'; -import { ExperimentUserRepository } from '../repositories/ExperimentUserRepository'; -import { FactorRepository } from '../repositories/FactorRepository'; -import { GroupExclusionRepository } from '../repositories/GroupExclusionRepository'; -import { IndividualExclusionRepository } from '../repositories/IndividualExclusionRepository'; -import { LevelCombinationElementRepository } from '../repositories/LevelCombinationElements'; -import { LevelRepository } from '../repositories/LevelRepository'; -import { MetricRepository } from '../repositories/MetricRepository'; -import { MonitoredDecisionPointRepository } from '../repositories/MonitoredDecisionPointRepository'; -import { QueryRepository } from '../repositories/QueryRepository'; -import { StateTimeLogsRepository } from '../repositories/StateTimeLogsRepository'; -import { StratificationFactorRepository } from '../repositories/StratificationFactorRepository'; -import { CacheService } from './CacheService'; -import { ErrorService } from './ErrorService'; -import { PreviewUserService } from './PreviewUserService'; -import { QueryService } from './QueryService'; -import { SegmentService } from './SegmentService'; -import { MoocletExperimentRef } from '../models/MoocletExperimentRef'; -import { MoocletVersionConditionMap } from '../models/MoocletVersionConditionMap'; - -import { MoocletExperimentRefRepository } from '../repositories/MoocletExperimentRefRepository'; -import { ConditionValidator, ExperimentDTO } from '../DTO/ExperimentDTO'; -import { UserDTO } from '../DTO/UserDTO'; -import { Experiment } from '../models/Experiment'; -import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; -import { - ASSIGNMENT_ALGORITHM, - EXPERIMENT_STATE, - MoocletPolicyParametersDTO, - MoocletTSConfigurablePolicyParametersDTO, - SUPPORTED_MOOCLET_ALGORITHMS, -} from 'upgrade_types'; -import { ExperimentCondition } from '../models/ExperimentCondition'; -import { MetricService } from './MetricService'; -import { env } from '../../env'; -import { ExperimentSchedulerService } from './ExperimentSchedulerService'; - -export interface SyncCreateParams { - experimentDTO: ExperimentDTO; - currentUser: UserDTO; - createType?: string; - logger: UpgradeLogger; -} - -export interface SyncEditParams { - experimentDTO: ExperimentDTO; - currentUser: UserDTO; - moocletRefToDelete?: MoocletExperimentRef; - logger: UpgradeLogger; -} - -export interface SyncDeleteParams { - moocletExperimentRef: MoocletExperimentRef; - experimentId: string; - currentUser: UserDTO; - logger: UpgradeLogger; -} - -export interface AllowedInactiveStateChanges { - addedConditions: ConditionValidator[] | false; - removedConditions: MoocletVersionConditionMap[] | false; - modifiedConditions: MoocletVersionConditionMap[] | false; -} - -export interface EditRollbackRef { - revertPolicyParameters: MoocletPolicyParametersDTO; - restoreVersions: ExperimentCondition[]; - revertVersionModifications: MoocletVersionConditionMap[]; - removeVersions: MoocletVersionConditionMap[]; - currentMoocletExperimentRef: MoocletExperimentRef; - currentExperiment: Experiment; -} - -@Service() -export class MoocletExperimentService extends ExperimentService { - constructor( - private moocletDataService: MoocletDataService, - @InjectRepository() experimentRepository: ExperimentRepository, - @InjectRepository() experimentConditionRepository: ExperimentConditionRepository, - @InjectRepository() decisionPointRepository: DecisionPointRepository, - @InjectRepository() experimentAuditLogRepository: ExperimentAuditLogRepository, - @InjectRepository() individualExclusionRepository: IndividualExclusionRepository, - @InjectRepository() groupExclusionRepository: GroupExclusionRepository, - @InjectRepository() monitoredDecisionPointRepository: MonitoredDecisionPointRepository, - @InjectRepository() userRepository: ExperimentUserRepository, - @InjectRepository() metricRepository: MetricRepository, - @InjectRepository() queryRepository: QueryRepository, - @InjectRepository() stateTimeLogsRepository: StateTimeLogsRepository, - @InjectRepository() experimentSegmentInclusionRepository: ExperimentSegmentInclusionRepository, - @InjectRepository() experimentSegmentExclusionRepository: ExperimentSegmentExclusionRepository, - @InjectRepository() conditionPayloadRepository: ConditionPayloadRepository, - @InjectRepository() factorRepository: FactorRepository, - @InjectRepository() levelRepository: LevelRepository, - @InjectRepository() levelCombinationElementsRepository: LevelCombinationElementRepository, - @InjectRepository() archivedStatsRepository: ArchivedStatsRepository, - @InjectRepository() stratificationRepository: StratificationFactorRepository, - @InjectRepository() segmentRepository: SegmentRepository, - @InjectRepository() - moocletExperimentRefRepository: MoocletExperimentRefRepository, - @InjectDataSource() dataSource: DataSource, - previewUserService: PreviewUserService, - segmentService: SegmentService, - experimentSchedulerService: ExperimentSchedulerService, - errorService: ErrorService, - cacheService: CacheService, - queryService: QueryService, - metricService: MetricService, - experimentPrecomputedSegmentService: ExperimentPrecomputedSegmentService - ) { - super( - experimentRepository, - experimentConditionRepository, - decisionPointRepository, - experimentAuditLogRepository, - individualExclusionRepository, - groupExclusionRepository, - monitoredDecisionPointRepository, - userRepository, - metricRepository, - queryRepository, - stateTimeLogsRepository, - experimentSegmentInclusionRepository, - experimentSegmentExclusionRepository, - segmentRepository, - conditionPayloadRepository, - factorRepository, - levelRepository, - levelCombinationElementsRepository, - archivedStatsRepository, - stratificationRepository, - moocletExperimentRefRepository, - dataSource, - previewUserService, - segmentService, - experimentSchedulerService, - errorService, - cacheService, - queryService, - metricService, - experimentPrecomputedSegmentService - ); - } - - public async syncCreate(params: SyncCreateParams): Promise { - const experiment = await this.dataSource.transaction(async (manager) => { - const experimentResponse = await this.createUpgradeExperiment(manager, params); - params.experimentDTO = experimentResponse; - return this.handleCreateMoocletTransaction(manager, params); - }); - - // create() ran inside the transaction above (via existingEntityManager), so it deferred the - // precomputed recompute to us. Now that the transaction has committed, populate the - // experiment_precomputed_segment row (awaited, mirroring create()). - await this.experimentPrecomputedSegmentService.recomputeForExperiment(experiment.id, params.logger); - return experiment; - } - - public async syncUpdate(params: SyncEditParams): Promise { - const experiment = await this.dataSource.transaction((manager) => - this.handleEditMoocletTransaction(manager, params) - ); - - // super.update() ran inside the transaction above (caller-owned entityManager), so it deferred the - // precomputed recompute to us. Recompute after commit so a context change — which deletes all - // segment lists — can't leave a stale experiment_precomputed_segment row. Fire-and-forget. - this.experimentPrecomputedSegmentService.scheduleRecomputeForExperiments([experiment.id], params.logger); - return experiment; - } - - public async syncUpdateWithMoocletAlgorithmTransition(params: SyncEditParams): Promise { - const experiment = await this.dataSource.transaction(async (manager) => { - const updatedExperiment = await this.updateUpgradeExperiment(manager, params); - - // when transitioning away from Mooclet, delete the old refs after successful experiment update - if (params.moocletRefToDelete) { - await this.moocletExperimentRefRepository.delete(params.moocletRefToDelete.id); - } - - // when transitioning to Mooclet, create the Mooclet resources - if (this.isMoocletExperiment(updatedExperiment.assignmentAlgorithm)) { - params.experimentDTO = { ...updatedExperiment }; - return await this.handleCreateMoocletTransaction(manager, params); - } - - return updatedExperiment; - }); - - // Same deferral as syncUpdate: super.update() ran inside the transaction, so recompute after commit. - this.experimentPrecomputedSegmentService.scheduleRecomputeForExperiments([experiment.id], params.logger); - return experiment; - } - - public async syncDelete(params: SyncDeleteParams): Promise { - return this.dataSource.transaction((manager) => this.handleDeleteMoocletTransaction(manager, params)); - } - - /** - * handleCreateMoocletTransaction - * - * 1. Save the upgrade experiment - * 2. Create and save the Mooclet experiment resources (outputs MoocletExperimentRef) - * 3. Save the MoocletExperimentRef and VersionConditionMaps - * - * On any error, rollback the Mooclet resources and abort the transaction - */ - - private async handleCreateMoocletTransaction( - manager: EntityManager, - params: SyncCreateParams - ): Promise { - const logger = params.logger; - const { moocletPolicyParameters } = params.experimentDTO; - const experiment = params.experimentDTO; - - // Auto-generate outcome variable name for ts_configurable experiments (always generate new) - if (experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE) { - const generatedName = this.generateUniqueOutcomeVariableName(experiment.name); - (moocletPolicyParameters as MoocletTSConfigurablePolicyParametersDTO).outcome_variable_name = generatedName; - logger.info({ - message: 'Generated outcome variable name for new TS Configurable Mooclet experiment', - experimentId: experiment.id, - generatedName, - }); - } - - // create Mooclet resources. If this fails, it will internally rollback any mooclet resources created, and the UpGrade experiment transaction will abort - const moocletExperimentRefResponse = await this.orchestrateMoocletCreation( - experiment, - moocletPolicyParameters, - logger - ); - - logger.info({ - message: 'Mooclet experiment created successfully:', - moocletExperimentRef: JSON.stringify(moocletExperimentRefResponse), - }); - - // additionally, create MoocletExperimentRef and VersionConditionMaps. - // If either of THESE fail, we will need to rollback the mooclet resources here also before aborting the UpGrade experiment transaction - try { - await this.saveMoocletExperimentRef(manager, moocletExperimentRefResponse); - await this.createAndSaveVersionConditionMapEntities( - manager, - moocletExperimentRefResponse.id, - moocletExperimentRefResponse.versionConditionMaps, - logger - ); - - experiment.moocletPolicyParameters = moocletPolicyParameters; - - return experiment; - } catch (error) { - await this.orchestrateDeleteMoocletResources(moocletExperimentRefResponse, logger); - throw error; - } - } - - private async createUpgradeExperiment(manager: EntityManager, params: SyncCreateParams): Promise { - const { experimentDTO, currentUser, createType, logger } = params; - return await super.create(experimentDTO, currentUser, logger, { - existingEntityManager: manager, - createType, - }); - } - - private async updateUpgradeExperiment(manager: EntityManager, params: SyncEditParams): Promise { - const { experimentDTO, currentUser, logger } = params; - return await super.update(experimentDTO, currentUser, logger, manager); - } - - private async saveMoocletExperimentRef( - manager: EntityManager, - moocletExperimentRefResponse: MoocletExperimentRef - ): Promise { - await manager.save(MoocletExperimentRef, moocletExperimentRefResponse); - } - - private async createAndSaveVersionConditionMapEntities( - manager: EntityManager, - moocletExperimentRefId: string, - versionConditionMaps: MoocletVersionConditionMap[], - logger: UpgradeLogger - ): Promise { - for (const versionConditionMap of versionConditionMaps) { - const versionConditionMapEntity = manager.create(MoocletVersionConditionMap, { - ...versionConditionMap, - moocletExperimentRefId, - experimentConditionId: versionConditionMap.experimentConditionId, - }); - const map = await manager.save(MoocletVersionConditionMap, versionConditionMapEntity); - logger.info({ - message: 'MoocletVersionConditionMap created successfully:', - versionConditionMap: JSON.stringify(map), - }); - } - } - - /** - * Handles a transaction to edit Mooclet experiment resources with rollback capability. - * - * The function performs the following sequential operations: - * 1. Validates experiment state eligibility (INACTIVE, PREVIEW, SCHEDULED, or RUNNING) - * 2. Fetches current resources for comparison with incoming changes - * 3. Detects experiment design changes that affect Mooclet resources and validates if they're allowed in current state - * 4. Updates the base experiment via parent class - * 5. Updates policy parameters (allowed in all eligible states) - * 6. If version edits exist, applies them with rollback capability - * - * If any step fails, the transaction will abort and we will automatically attempt to roll back any Mooclet resources to previous state - * - * @param manager - Entity manager passed for transaction handling - * @param params - Object containing experimentDTO, currentUser, and logger - * @returns Promise resolving to updated ExperimentDTO - * @throws Error if experiment state is ineligible or if disallowed changes are detected - */ - private async handleEditMoocletTransaction(manager: EntityManager, params: SyncEditParams): Promise { - const { experimentDTO: incomingExperiment, currentUser, logger } = params; - const rollbackRef: EditRollbackRef = { - revertPolicyParameters: null, - restoreVersions: null, - revertVersionModifications: null, - removeVersions: null, - currentMoocletExperimentRef: null, - currentExperiment: null, - }; - - // bail if the experiment is in not in an eligible edit state for mooclet resources - if ( - ![ - EXPERIMENT_STATE.INACTIVE, - EXPERIMENT_STATE.PREVIEW, - EXPERIMENT_STATE.SCHEDULED, - EXPERIMENT_STATE.RUNNING, - EXPERIMENT_STATE.PAUSED, - ].includes(incomingExperiment.state) - ) { - logger.error({ - message: '[Mooclet Edit] Ineligible experiment state for Mooclet edits', - state: incomingExperiment.state, - }); - throw new MoocletError('[Mooclet Edit] Ineligible experiment state for Mooclet edits', 400); - } - - // NOTE: Currently allowed states for updating mooclet resources: - // conditions (versions): PREVIEW, SCHEDULED, INACTIVE, PAUSED - // outcome variable name: IMMUTABLE after creation (no changes allowed) - // policy parameters: RUNNING, PREVIEW, SCHEDULED, INACTIVE, PAUSED - - try { - // ---------- fetch current resources for comparison to incoming ------------------------------- - const { currentMoocletExperimentRef, currentPolicyParametersResponse, currentExperiment } = - await this.fetchCurrentResources(incomingExperiment, logger); - - const versionEdits = this.detectExperimentDesignChanges(incomingExperiment, currentMoocletExperimentRef); - - // bail if disallowed changes are detected for running experiments - if (versionEdits && EXPERIMENT_STATE.RUNNING === incomingExperiment.state) { - logger.error({ - message: '[Mooclet Edit] Ineligible version edits detected for an active Mooclet experiment', - changes: versionEdits, - }); - throw new MoocletError( - '[Mooclet Edit] Ineligible version edits detected for an active Mooclet experiment', - 400 - ); - } - - logger.debug({ - message: '[Mooclet Edit] Experiment changes affect Mooclet resources, begin sync.', - changes: versionEdits, - }); - - // ---------- update the experiment first --------------------------------------- - const updatedExperiment = await super.update(incomingExperiment, currentUser, logger, manager); - - rollbackRef.currentExperiment = currentExperiment; - rollbackRef.currentMoocletExperimentRef = currentMoocletExperimentRef; - - // ---------- update policy parameters ------------------------------------------ - // NOTE: the rollbackRef is mutated within these methods and will be used to revert changes if any step fails - - // go ahead and PUT policy parameters (these are allowed when running and not really worth checking for changes) - const policyParameterResponse = await this.doRevertablePolicyParameterChange({ - incomingExperiment, - currentMoocletExperimentRef, - currentPolicyParametersResponse, - rollbackRef, - logger, - }); - - updatedExperiment.moocletPolicyParameters = policyParameterResponse.parameters; - - // Transform prior keys from Mooclet version IDs back to UpGrade condition codes, - // so the PUT response is consistent with the GET response from attachPolicyParamsToExperimentDTO. - const updatedTsParams = updatedExperiment.moocletPolicyParameters as MoocletTSConfigurablePolicyParametersDTO; - if (updatedTsParams?.prior) { - updatedTsParams.prior = this.translateVersionIdsToConditionCodes( - updatedTsParams.prior, - currentMoocletExperimentRef.versionConditionMaps - ); - } - - // --------- update versions ---------------------- - - if (!versionEdits) { - return updatedExperiment; - } - - await this.doRevertableVersionEdits({ - versionEdits, - currentMoocletExperimentRef, - incomingExperiment, - rollbackRef, - manager, - logger, - }); - - return updatedExperiment; - } catch (error) { - logger.error({ message: '[Mooclet Edits] Error updating experiment', error }); - await this.rollbackMoocletEdits(rollbackRef, logger); - throw error; - } - } - - /** - * Identifies conditions that exist in the incoming experiment but not in the current Mooclet experiment. - * - * @param incomingExperimentDTO - The updated experiment data being submitted - * @param currentMoocletExperimentRef - The current Mooclet experiment reference - * @returns Array of new conditions if any are found, or null if no new conditions - */ - private detectNewConditions( - incomingExperimentDTO: ExperimentDTO, - currentMoocletExperimentRef: MoocletExperimentRef - ): ConditionValidator[] | null { - const newConditions = incomingExperimentDTO.conditions.filter( - (condition) => - !currentMoocletExperimentRef.versionConditionMaps.find((map) => map.experimentCondition.id === condition.id) - ); - - return newConditions.length ? newConditions : null; - } - - /** - * Identifies conditions that exist in the current Mooclet experiment but not in the incoming experiment. - * - * @param incomingExperimentDTO - The updated experiment data being submitted - * @param currentMoocletExperimentRef - The current Mooclet experiment reference - * @returns Array of version-condition maps for removed conditions if any are found, or null if no conditions were removed - */ - private detectRemovedConditions( - incomingExperimentDTO: ExperimentDTO, - currentMoocletExperimentRef: MoocletExperimentRef - ): MoocletVersionConditionMap[] | null { - const mapsForRemovedConditions = currentMoocletExperimentRef.versionConditionMaps.filter( - (map) => !incomingExperimentDTO.conditions.find((condition) => condition.id === map.experimentCondition.id) - ); - - return mapsForRemovedConditions.length ? mapsForRemovedConditions : null; - } - - /** - * Identifies conditions whose code has been modified between the incoming experiment and the current Mooclet experiment. - * - * @param incomingExperimentDTO - The updated experiment data being submitted - * @param currentMoocletExperimentRef - The current Mooclet experiment reference - * @returns Array of version-condition maps for modified conditions if any are found, or null if no conditions were modified - */ - private detectModifiedConditions( - incomingExperimentDTO: ExperimentDTO, - currentMoocletExperimentRef: MoocletExperimentRef - ): MoocletVersionConditionMap[] | null { - const versionsToUpdate = currentMoocletExperimentRef.versionConditionMaps.filter((map) => { - const condition = incomingExperimentDTO.conditions.find( - (condition) => condition.id === map.experimentCondition.id - ); - - if (!condition) { - return null; - } - - return condition && condition.conditionCode !== map.experimentCondition.conditionCode; - }); - - return versionsToUpdate.length ? versionsToUpdate : null; - } - - /** - * Aggregates all experiment design changes by calling individual detection methods. - * - * Checks for added conditions, removed conditions, and modified conditions. - * Returns an object containing all detected changes, or null if no changes were detected. - * - * @param incomingExperimentDTO - The updated experiment data being submitted - * @param currentMoocletExperimentRef - The current Mooclet experiment reference - * @returns Object with all detected changes if any exist, or null if no changes were detected - */ - private detectExperimentDesignChanges( - incomingExperimentDTO: ExperimentDTO, - currentMoocletExperimentRef: MoocletExperimentRef - ): AllowedInactiveStateChanges | null { - const changes = { - addedConditions: this.detectNewConditions(incomingExperimentDTO, currentMoocletExperimentRef), - removedConditions: this.detectRemovedConditions(incomingExperimentDTO, currentMoocletExperimentRef), - modifiedConditions: this.detectModifiedConditions(incomingExperimentDTO, currentMoocletExperimentRef), - }; - - const hasChanges = Object.values(changes).some((change) => !!change); - - return hasChanges ? changes : null; - } - - /** - * Updates policy parameters for a Mooclet experiment. - * - * This method updates the policy parameters of a Mooclet experiment in response to - * experiment design changes. It forwards the request to the - * Mooclet data service. - * - * @param newPolicyParameters - The updated policy parameters to apply - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @param logger - Logger instance for recording operation details - * @returns Promise resolving to the updated policy parameters response from Mooclet - */ - private async handleUpdatePolicyParameters( - newPolicyParameters: MoocletPolicyParametersDTO, - currentMoocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): Promise { - const tsParams = newPolicyParameters as MoocletTSConfigurablePolicyParametersDTO; - if (tsParams.prior) { - // Translate conditionCode keys to Mooclet version IDs before sending to the Mooclet API - newPolicyParameters = { - ...tsParams, - prior: this.translateConditionCodesToVersionIds( - tsParams.prior, - currentMoocletExperimentRef.versionConditionMaps - ), - } as MoocletPolicyParametersDTO; - } - - return this.moocletDataService.updatePolicyParameters( - currentMoocletExperimentRef.policyParametersId, - { - mooclet: currentMoocletExperimentRef.moocletId, - policy: currentMoocletExperimentRef.policyId, - parameters: newPolicyParameters, - }, - logger - ); - } - - /** - * Removes conditions from a Mooclet experiment. - * - * This method deletes versions from Mooclet that correspond to experiment conditions - * that have been removed from the experiment design. It - * processes each removal in parallel. - * - * @param removedConditions - Array of version-condition maps for conditions to remove - * @param logger - Logger instance for recording operation details - * @returns Promise resolving to an array of void results, one for each deleted version - */ - private async handleRemoveConditions( - removedConditions: MoocletVersionConditionMap[], - logger: UpgradeLogger - ): Promise { - return Promise.all( - removedConditions.map(async (map) => { - return await this.moocletDataService.deleteVersion(map.moocletVersionId, logger); - }) - ); - } - - /** - * Modifies conditions in a Mooclet experiment. - * - * This method updates versions in Mooclet to reflect changes to their corresponding - * experiment conditions. It retrieves each version, updates its name and text fields - * if they differ from the incoming condition, and updates the version in Mooclet. - * - * @param versionMapsToUpdate - Array of version-condition maps for conditions to modify - * @param incomingExperiment - The updated experiment data containing modified conditions - * @param logger - Logger instance for recording operation details - * @returns Promise resolving to the array of updated version-condition maps - */ - private async handleModifyConditions( - versionMapsToUpdate: MoocletVersionConditionMap[], - incomingExperiment: ExperimentDTO | Experiment, - logger: UpgradeLogger - ): Promise { - return Promise.all( - versionMapsToUpdate.map(async (versionMap) => { - const version = await this.moocletDataService.getVersion(versionMap.moocletVersionId, logger); - const condition = incomingExperiment.conditions.find( - (changedCondition) => changedCondition.id === versionMap.experimentConditionId - ); - - if (version && condition && version.name !== condition.conditionCode) { - version.name = condition.conditionCode; - version.text = condition.conditionCode; // this could be mapped to payload - await this.moocletDataService.updateVersion(versionMap.moocletVersionId, version, logger); - } - - return versionMap; - }) - ); - } - - /** - * Adds new conditions to a Mooclet experiment. - * - * This method creates new versions in Mooclet for conditions that have been added to - * the experiment design. It creates each version in parallel, and - * then creates mappings between the new versions and their corresponding conditions. - * - * @param addedConditions - Array of conditions to add to the Mooclet experiment - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @returns Promise resolving to an array of newly created version-condition maps - */ - private async handleAddConditions( - addedConditions: ExperimentCondition[] | ConditionValidator[], - currentMoocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): Promise { - const newVersions: MoocletVersionResponseDetails[] = []; - - await Promise.all( - addedConditions.map(async (condition) => { - const newVersionRequest: MoocletVersionRequestBody = { - mooclet: currentMoocletExperimentRef.moocletId, - name: condition.conditionCode, - text: condition.conditionCode, - }; - - const version = await this.moocletDataService.postNewVersion(newVersionRequest, logger); - newVersions.push(version); - }) - ); - - return this.createMoocletVersionConditionMaps(newVersions, addedConditions); - } - - /** - * Updates policy parameters with rollback capability. - * - * @param options - Object containing: - * @param incomingExperiment - The updated experiment data containing new policy parameters - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @param currentPolicyParametersResponse - Current policy parameters for rollback - * @param rollbackRef - Reference object to store rollback information - * @param logger - Logger instance - * @returns Promise that resolves to the updated policy parameters response - */ - async doRevertablePolicyParameterChange({ - incomingExperiment, - currentMoocletExperimentRef, - currentPolicyParametersResponse, - rollbackRef, - logger, - }) { - logger.debug({ - message: '[Mooclet Edit] Upserting policy parameters due to experiment design change', - incomingPolicyParameters: incomingExperiment.moocletPolicyParameters, - }); - const policyParametersResponse = await this.handleUpdatePolicyParameters( - incomingExperiment.moocletPolicyParameters, - currentMoocletExperimentRef, - logger - ); - - rollbackRef.revertPolicyParameters = currentPolicyParametersResponse.parameters; - return policyParametersResponse; - } - - /** - * Executes revertable edits to versions within a Mooclet experiment. - * Handles condition modifications, additions, and removals - * with built-in rollback capability if any step fails. - * - * @param options - Object containing: - * @param versionEdits - Changes to make to versions - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @param incomingExperiment - New experiment configuration - * @param manager - Entity manager for database operations - * @param logger - Logger instance - * @param rollbackRef - Reference object to store rollback information - * @returns Promise that resolves when all edits are complete - * @throws Will throw the first encountered error if any task fails - */ - async doRevertableVersionEdits({ - versionEdits, - currentMoocletExperimentRef, - incomingExperiment, - manager, - logger, - rollbackRef, - }) { - const { addedConditions, removedConditions, modifiedConditions } = versionEdits; - - const taskPromises = []; - - if (removedConditions) { - taskPromises.push(this.doRevertableRemovedConditions({ removedConditions, rollbackRef, logger })); - } - - if (modifiedConditions) { - taskPromises.push( - this.doRevertableModifiedConditions({ - modifiedConditions, - incomingExperiment, - experimentRefId: currentMoocletExperimentRef.id, - manager, - rollbackRef, - logger, - }) - ); - } - - if (addedConditions) { - taskPromises.push( - this.doRevertableAddConditions({ addedConditions, currentMoocletExperimentRef, manager, rollbackRef, logger }) - ); - } - - // Use Promise.allSettled to let all tasks complete or fail independently - const results = await Promise.allSettled(taskPromises); - - // Check if any tasks failed - const failures = results.filter((result): result is PromiseRejectedResult => result.status === 'rejected'); - - if (failures.length > 0) { - // Log all the errors that occurred - failures.forEach((failure, index) => { - logger.error({ - message: `[Mooclet Edit] Task ${index} failed during version edits.`, - error: failure.reason, - }); - }); - - // Throw the first error to signal failure to the caller - // The rollbackRef will already be populated with all successful operations - throw failures[0].reason; - } - } - - /** - * Removes conditions from a Mooclet experiment with rollback capability. - * - * @param options - Object containing: - * @param removedConditions - Conditions to be removed - * @param rollbackRef - Reference object to store rollback information - * @param logger - Logger instance - * @returns Promise that resolves when condition removal is complete - */ - async doRevertableRemovedConditions({ removedConditions, rollbackRef, logger }) { - logger.debug({ - message: '[Mooclet Edit] Removing versions from Mooclet due to experiment design change.', - removedConditions, - }); - - await this.handleRemoveConditions(removedConditions, logger); - rollbackRef.restoreVersions = removedConditions.map((map: MoocletVersionConditionMap) => map.experimentCondition); - } - - /** - * Modifies conditions in a Mooclet experiment with rollback capability. - * - * @param options - Object containing: - * @param modifiedConditions - Conditions to be modified - * @param incomingExperiment - New experiment configuration - * @param experimentRefId - ID of the experiment reference - * @param manager - Entity manager for database operations - * @param rollbackRef - Reference object to store rollback information - * @param logger - Logger instance - * @returns Promise that resolves when condition modifications are complete - */ - async doRevertableModifiedConditions({ - modifiedConditions, - incomingExperiment, - experimentRefId, - manager, - rollbackRef, - logger, - }) { - logger.debug({ - message: '[Mooclet Edit] Modifying versions from Mooclet due to experiment design change.', - modifiedConditions, - }); - - const modifiedVersionConditionMaps = await this.handleModifyConditions( - modifiedConditions, - incomingExperiment, - logger - ); - - await this.createAndSaveVersionConditionMapEntities(manager, experimentRefId, modifiedVersionConditionMaps, logger); - - rollbackRef.revertVersionModifications = modifiedConditions; - } - - /** - * Adds conditions to a Mooclet experiment with rollback capability. - * - * @param options - Object containing: - * @param addedConditions - Conditions to be added - * @param currentMoocletExperimentRef - Reference to the current Mooclet experiment - * @param manager - Entity manager for database operations - * @param rollbackRef - Reference object to store rollback information - * @param logger - Logger instance - * @returns Promise that resolves when condition additions are complete - */ - async doRevertableAddConditions({ addedConditions, currentMoocletExperimentRef, manager, rollbackRef, logger }) { - logger.debug({ - message: '[Mooclet Edit] Adding conditions to Mooclet due to experiment design change.', - addedConditions, - }); - - const addedVersionConditionMaps = await this.handleAddConditions( - addedConditions, - currentMoocletExperimentRef, - logger - ); - - await this.createAndSaveVersionConditionMapEntities( - manager, - currentMoocletExperimentRef.id, - addedVersionConditionMaps, - logger - ); - - rollbackRef.removeVersions = addedVersionConditionMaps; - } - - /** - * Fetches all current resources related to a Mooclet experiment. - * - * @param incomingExperiment - The experiment configuration to retrieve resources for - * @param logger - Logger instance for operation logging - * @returns Promise that resolves to an object containing the current Mooclet experiment reference, - * policy parameters, and current experiment - */ - async fetchCurrentResources(incomingExperiment: ExperimentDTO, logger: UpgradeLogger) { - const currentMoocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(incomingExperiment.id); - - const [currentPolicyParametersResponse, currentExperiment] = await Promise.all([ - this.moocletDataService.getPolicyParameters(currentMoocletExperimentRef.policyParametersId, logger), - this.experimentRepository.findOne({ - where: { id: incomingExperiment.id }, - relations: { - conditions: true, - }, - }), - ]); - - return { - currentMoocletExperimentRef, - currentPolicyParametersResponse, - currentExperiment, - }; - } - - /** - * Rolls back changes made to Mooclet experiment resources in case of transaction failure. - * - * This method attempts to restore the Mooclet experiment to its previous state when - * an error occurs during the edit process. It systematically reverses each type of - * change that may have been made: - * - * 1. Reverts policy parameters to their original values - * 2. Recreates versions that were deleted - * 3. Reverts modifications to existing versions - * 4. Removes versions that were newly added - * - * If an error occurs during the rollback process itself, a critical error is logged - * and thrown, indicating that the experiment resources may be in an inconsistent state. - * - * @param rollbackRef - Reference object containing all the data needed for rollback operations - * @param logger - Logger instance for recording rollback operation details - * @returns Promise that resolves when rollback is complete - * @throws Error if the rollback itself fails, indicating resources are likely out of sync - */ - private async rollbackMoocletEdits(rollbackRef: EditRollbackRef, logger: UpgradeLogger): Promise { - logger.error({ - message: '[Mooclet Edit] Rolling back Mooclet edits', - rollbackRef, - }); - - const { - revertPolicyParameters, - restoreVersions, - revertVersionModifications, - removeVersions, - currentMoocletExperimentRef, - currentExperiment, - } = rollbackRef; - - try { - if (revertPolicyParameters) { - logger.debug({ - message: '[Mooclet Edit] Will attempt to reverting policy parameters to previous state', - revertPolicyParameters, - currentMoocletExperimentRef, - }); - await this.handleUpdatePolicyParameters(revertPolicyParameters, currentMoocletExperimentRef, logger); - } - - if (restoreVersions) { - logger.debug({ - message: '[Mooclet Edit] Will attempt to restore deleted versions to previous state', - restoreVersions, - }); - await this.handleAddConditions(restoreVersions, currentMoocletExperimentRef, logger); - } - - if (revertVersionModifications) { - logger.debug({ - message: '[Mooclet Edit] Will attempt to revert modified versions to previous state', - revertVersionModifications, - }); - await this.handleModifyConditions(revertVersionModifications, currentExperiment, logger); - } - - if (removeVersions) { - logger.debug({ - message: '[Mooclet Edit] Will attempt to remove added versions to previous state', - removeVersions, - }); - await this.handleRemoveConditions(removeVersions, logger); - } - } catch (rollbackError) { - logger.error({ - message: - '[Mooclet Edit] Error during rollback, Mooclet resources are likely out of sync for this experiment! Check the resources noted in the rollback ref', - error: rollbackError, - rollbackRef, - }); - throw new MoocletError( - '[Mooclet Edit] Error during rollback, Mooclet resources are likely out of sync for this experiment' - ); - } - } - - private async handleDeleteMoocletTransaction(manager: EntityManager, params: SyncDeleteParams): Promise { - const { moocletExperimentRef, currentUser, logger } = params; - let deleteResponse: Experiment | undefined; - - try { - // delete the upgrade experiment. If this fails, the Mooclet resources will not be deleted, and the transaction will abort - deleteResponse = await super.delete(params.experimentId, currentUser, { - existingEntityManager: manager, - }); - - // delete the mooclet resources. If this fails, the transaction will abort and the upgrade experiment will not be deleted, - // but the Mooclet resources may not be deleted either - const removedResources = await this.orchestrateDeleteMoocletResources(moocletExperimentRef, logger); - if (!removedResources) { - logger.error({ - message: 'Failed to delete Mooclet resources, aborting transaction to preserve upgrade experiment', - experimentId: params.experimentId, - moocletExperimentRef, - }); - throw new MoocletError( - 'Failed to delete Mooclet resources, aborting transaction to preserve upgrade experiment' - ); - } - - return deleteResponse; - } catch (error) { - logger.error({ - message: 'Failed to delete experiment with Mooclet', - error: error, - experimentId: params.experimentId, - user: currentUser, - }); - throw error; - } - } - - async handleRemoveAnyMoocletResources(experimentId: string, logger: UpgradeLogger): Promise { - const moocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(experimentId); - if (moocletExperimentRef) { - this.orchestrateDeleteMoocletResources(moocletExperimentRef, logger); - } else { - logger.info({ - message: `No MoocletExperimentRef found for experiment ${experimentId}, skipping Mooclet resource deletion.`, - }); - } - } - - /** - * orchestrateMoocletCreation: Sequential calls needed to create of a new Mooclet experiment from experiment data and supplied policy parameters - * - * 1. Retrieves the Mooclet policy ID based on the assignment algorithm. - * 2. Creates a new Mooclet experiment with name an policy ID. - * 3. Creates Mooclet versions for each Experiment condition - * 4. Creates Policy Parameters - * 5. Creates a variable if needed based on the assignment algorithm and policy parameters. - * 6. Constructs and returns the Mooclet experiment reference if all steps were successful. - */ - - public async orchestrateMoocletCreation( - upgradeExperiment: ExperimentDTO, - moocletPolicyParameters: MoocletPolicyParametersDTO, - logger: UpgradeLogger - ): Promise { - const newMoocletRequest: MoocletRequestBody = { - name: upgradeExperiment.name + '-' + upgradeExperiment.id.slice(0, 8), //using part of exp uuid so this will make sure mooclet name is unique - policy: null, - }; - const moocletExperimentRef = new MoocletExperimentRef(); - moocletExperimentRef.experimentId = upgradeExperiment.id; - - logger.debug({ - message: '[Mooclet Creation] 0. Starting Mooclet creation process', - upgradeExperiment: JSON.stringify(upgradeExperiment), - }); - - try { - newMoocletRequest.policy = await this.getMoocletPolicy(upgradeExperiment.assignmentAlgorithm, logger); - logger.debug({ - message: `[Mooclet Creation] 1. Policy id fetched:`, - newMoocletRequest: JSON.stringify(newMoocletRequest), - }); - - const moocletResponse = await this.createMooclet(newMoocletRequest, logger); - moocletExperimentRef.moocletId = moocletResponse.id; - - logger.debug({ - message: `[Mooclet Creation] 2. Mooclet created:`, - moocletResponse: JSON.stringify(moocletResponse), - }); - - const moocletVersionsResponse = await this.createMoocletVersions(upgradeExperiment, moocletResponse, logger); - moocletExperimentRef.versionConditionMaps = this.createMoocletVersionConditionMaps( - moocletVersionsResponse, - upgradeExperiment.conditions - ); - logger.debug({ - message: `[Mooclet Creation] 3. Mooclet versions created:`, - moocletVersionsResponse: JSON.stringify(moocletVersionsResponse), - }); - - const moocletPolicyParametersResponse = await this.createPolicyParameters( - moocletResponse, - moocletPolicyParameters, - logger - ); - moocletExperimentRef.policyParametersId = moocletPolicyParametersResponse.id; - logger.debug({ - message: `[Mooclet Creation] 4. Policy parameters created:`, - moocletPolicyParametersResponse: JSON.stringify(moocletPolicyParametersResponse), - }); - - const moocletVariableResponse = await this.createVariableIfNeeded( - moocletPolicyParameters, - upgradeExperiment.assignmentAlgorithm, - logger - ); - logger.debug({ - message: `[Mooclet Creation] 5. Variable created (if needed):`, - moocletVariableResponse: JSON.stringify(moocletVariableResponse), - }); - - moocletExperimentRef.variableId = moocletVariableResponse?.id; - moocletExperimentRef.policyId = newMoocletRequest.policy; - moocletExperimentRef.outcomeVariableName = ( - moocletPolicyParameters as MoocletTSConfigurablePolicyParametersDTO - ).outcome_variable_name; - } catch (err) { - await this.orchestrateDeleteMoocletResources(moocletExperimentRef, logger); - throw err; - } - - moocletExperimentRef.id = crypto.randomUUID(); - - return moocletExperimentRef; - } - - private createMoocletVersionConditionMaps( - versions: MoocletVersionResponseDetails[], - conditions: ConditionValidator[] | ExperimentCondition[] - ): MoocletVersionConditionMap[] { - const versionConditionMaps: MoocletVersionConditionMap[] = conditions.map((condition) => { - const versionConditionMap = new MoocletVersionConditionMap(); - versionConditionMap.moocletVersionId = versions.find((version) => version.name === condition.conditionCode)?.id; - versionConditionMap.experimentConditionId = condition.id; - - return versionConditionMap; - }); - return versionConditionMaps; - } - - public async orchestrateDeleteMoocletResources( - moocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): Promise { - try { - if (!moocletExperimentRef) { - logger.error({ message: 'MoocletExperimentRef not defined for deletion' }); - throw new MoocletError('MoocletExperimentRef not defined'); - } - - logger.debug({ message: '[Mooclet Deletion]: Starting deletion of Mooclet resources', moocletExperimentRef }); - - // Delete Mooclet resources if they exist - logger.debug({ message: '[Mooclet Deletion]: Deleting Mooclet', moocletId: moocletExperimentRef.moocletId }); - if (moocletExperimentRef.moocletId) { - await this.moocletDataService.deleteMooclet(moocletExperimentRef.moocletId, logger); - logger.debug({ message: '[Mooclet Deletion]: Deleted Mooclet', moocletId: moocletExperimentRef.moocletId }); - } - - await this.deleteMoocletVersions(moocletExperimentRef, logger); - - logger.debug({ - message: '[Mooclet Deletion]: Deleting policy parameters', - policyParametersId: moocletExperimentRef.policyParametersId, - }); - if (moocletExperimentRef.policyParametersId) { - await this.moocletDataService.deletePolicyParameters(moocletExperimentRef.policyParametersId, logger); - logger.debug({ - message: '[Mooclet Deletion]: Deleted policy parameters', - policyParametersId: moocletExperimentRef.policyParametersId, - }); - } - - logger.debug({ message: '[Mooclet Deletion]: Deleting variable', variableId: moocletExperimentRef.variableId }); - if (moocletExperimentRef.variableId) { - await this.moocletDataService.deleteVariable(moocletExperimentRef.variableId, logger); - logger.debug({ message: '[Mooclet Deletion]: Deleted variable', variableId: moocletExperimentRef.variableId }); - } - - logger.info({ message: '[Mooclet Deletion]: Completed deletion of Mooclet resources', moocletExperimentRef }); - return true; // Return true to indicate successful deletion of all resources - } catch (err) { - const error = { - message: - '[Mooclet Deletion]: Failed to delete Mooclet resources, please check manually for out of sync resources', - error: err, - moocletExperimentRef: moocletExperimentRef, - }; - - logger.error(error); - return false; // Return false to indicate failure, but do not throw an error to allow the transaction to complete - } - } - - private async deleteMoocletVersions( - moocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): Promise { - if (moocletExperimentRef.versionConditionMaps) { - for (const versionConditionMap of moocletExperimentRef.versionConditionMaps) { - logger.debug({ - message: '[Mooclet Deletion]: Deleting Mooclet version', - moocletVersionId: versionConditionMap.moocletVersionId, - }); - if (versionConditionMap.moocletVersionId) { - await this.moocletDataService.deleteVersion(versionConditionMap.moocletVersionId, logger); - logger.debug({ - message: '[Mooclet Deletion]: Deleted Mooclet version', - moocletVersionId: versionConditionMap.moocletVersionId, - }); - } - } - } - } - - private async getMoocletPolicy(assignmentAlgorithm: string, logger: UpgradeLogger): Promise { - try { - return await this.moocletDataService.getMoocletIdByName(assignmentAlgorithm, logger); - } catch (err) { - logger.error({ message: 'Failed to get Mooclet policy', error: err, assignmentAlgorithm }); - throw new MoocletError('Failed to get Mooclet policy'); - } - } - - public async attachPolicyParamsToExperimentDTO( - experiment: ExperimentDTO, - logger: UpgradeLogger - ): Promise { - try { - const moocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(experiment.id); - const policyParameters = await this.moocletDataService.getPolicyParameters( - moocletExperimentRef.policyParametersId, - logger - ); - - // Transform current_posteriors and prior keys from Mooclet version IDs to UpGrade condition codes - const tsConfigurableParams = policyParameters.parameters as MoocletTSConfigurablePolicyParametersDTO; - if (tsConfigurableParams.current_posteriors) { - tsConfigurableParams.current_posteriors = this.translateVersionIdsToConditionCodes( - tsConfigurableParams.current_posteriors, - moocletExperimentRef.versionConditionMaps - ); - } - - if (tsConfigurableParams.prior) { - tsConfigurableParams.prior = this.translateVersionIdsToConditionCodes( - tsConfigurableParams.prior, - moocletExperimentRef.versionConditionMaps - ); - } - - experiment.moocletPolicyParameters = policyParameters.parameters; - - return experiment; - } catch (err) { - logger.error({ message: 'Failed to get Mooclet policy parameters', error: err, experimentId: experiment.id }); - throw new MoocletError('Failed to get Mooclet policy parameters'); - } - } - - private async createMooclet( - newMoocletRequest: MoocletRequestBody, - logger: UpgradeLogger - ): Promise { - try { - return await this.moocletDataService.postNewMooclet(newMoocletRequest, logger); - } catch (err) { - logger.error({ message: 'Failed to create Mooclet', error: err, newMoocletRequest }); - throw new MoocletError('Failed to create Mooclet'); - } - } - - private async createMoocletVersions( - experiment: ExperimentDTO, - moocletResponse: MoocletResponseDetails, - logger: UpgradeLogger - ): Promise { - if (!moocletResponse?.id || !experiment.conditions) return null; - - try { - return await Promise.all( - experiment.conditions.map(async (condition) => this.createNewVersion(condition, moocletResponse.id, logger)) - ); - } catch (err) { - logger.error({ message: 'Failed to create Mooclet versions', error: err, experimentId: experiment.id }); - throw new MoocletError('Failed to create Mooclet versions'); - } - } - - private async createNewVersion( - upgradeCondition: ConditionValidator | ExperimentCondition, - moocletId: number, - logger: UpgradeLogger - ): Promise { - const newVersionRequest: MoocletVersionRequestBody = { - mooclet: moocletId, - name: upgradeCondition.conditionCode, - text: upgradeCondition.conditionCode, - }; - - try { - return await this.moocletDataService.postNewVersion(newVersionRequest, logger); - } catch (err) { - logger.error({ message: 'Failed to create new version for Mooclet', error: err, newVersionRequest }); - throw new MoocletError('Failed to create new version for Mooclet'); - } - } - - private async createPolicyParameters( - moocletResponse: MoocletResponseDetails, - moocletPolicyParameters: MoocletPolicyParametersDTO, - logger: UpgradeLogger - ): Promise { - if (!moocletResponse) return null; - - const policyParametersRequest: MoocletPolicyParametersRequestBody = { - mooclet: moocletResponse.id, - policy: moocletResponse.policy, - parameters: moocletPolicyParameters, - }; - - try { - return await this.moocletDataService.postNewPolicyParameters(policyParametersRequest, logger); - } catch (err) { - logger.error({ message: 'Failed to create Mooclet policy parameters', error: err, policyParametersRequest }); - throw new MoocletError('Failed to create Mooclet policy parameters'); - } - } - - private async createVariableIfNeeded( - moocletPolicyParametersResponse: MoocletPolicyParametersDTO, - assignmentAlgorithm: string, - logger: UpgradeLogger - ): Promise { - if (!moocletPolicyParametersResponse || assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE) { - return null; - } - - const variableRequest: MoocletVariableRequestBody = { - name: (moocletPolicyParametersResponse as MoocletTSConfigurablePolicyParametersDTO)?.outcome_variable_name, - }; - - try { - return await this.moocletDataService.postNewVariable(variableRequest, logger); - } catch (err) { - logger.error({ message: 'Failed to create Mooclet variable', error: err, variableRequest }); - throw new MoocletError('Failed to create variable'); - } - } - - public async getMoocletExperimentRefByUpgradeExperimentId( - upgradeExperimentId: string - ): Promise { - const moocletExperimentRef = await this.moocletExperimentRefRepository.findOne({ - where: { experimentId: upgradeExperimentId }, - relations: { - versionConditionMaps: { - experimentCondition: true, - }, - }, - }); - return moocletExperimentRef; - } - - public async getConditionFromMoocletProxy( - experiment: Experiment, - userId: string, - logger: UpgradeLogger - ): Promise { - const moocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(experiment.id); - - if (!moocletExperimentRef) { - throw new MoocletError(`MoocletExperimentRef not found for experiment id ${experiment.id}`); - } - - const versionResponse = await this.moocletDataService.getVersionForNewLearner( - moocletExperimentRef.moocletId, - userId, - logger - ); - const experimentCondition = this.mapMoocletVersionToUpgradeCondition(versionResponse, moocletExperimentRef, logger); - return experimentCondition; - } - - private mapMoocletVersionToUpgradeCondition( - versionResponse: MoocletVersionResponseDetails, - moocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger - ): ExperimentCondition { - // Find the corresponding versionConditionMap - const versionConditionMap = moocletExperimentRef.versionConditionMaps.find( - (map) => map.moocletVersionId === versionResponse.id - ); - - if (!versionConditionMap) { - logger.error({ - message: 'Version ID not found in version condition maps', - version: versionResponse, - versionConditionMaps: moocletExperimentRef.versionConditionMaps, - }); - throw new MoocletError('Version ID not found in version condition maps'); - } - - // Get the experiment condition from the versionConditionMap - const experimentCondition = versionConditionMap.experimentCondition; - - if (!experimentCondition) { - logger.error({ - message: 'Experiment condition not found in version condition map', - version: versionResponse, - versionConditionMap, - }); - throw new MoocletError('Experiment condition not found in version condition map'); - } - - return experimentCondition; - } - - public isMoocletExperiment(assignmentAlgorithm: ASSIGNMENT_ALGORITHM): boolean { - return SUPPORTED_MOOCLET_ALGORITHMS.includes(assignmentAlgorithm); - } - - private translateConditionCodesToVersionIds( - record: Record, - versionConditionMaps: MoocletVersionConditionMap[] - ): Record { - const result: Record = {}; - for (const [conditionCode, value] of Object.entries(record)) { - const map = versionConditionMaps?.find((m) => m.experimentCondition?.conditionCode === conditionCode); - if (!map?.moocletVersionId) { - throw new MoocletError(`No version mapping found for condition code ${conditionCode}`); - } - if (map?.moocletVersionId) { - result[String(map.moocletVersionId)] = value; - } - } - return result; - } - - private translateVersionIdsToConditionCodes( - record: Record, - versionConditionMaps: MoocletVersionConditionMap[] - ): Record { - const result: Record = {}; - for (const [versionId, value] of Object.entries(record)) { - const map = versionConditionMaps?.find((m) => String(m.moocletVersionId) === versionId); - if (!map?.experimentCondition?.conditionCode) { - throw new MoocletError(`No condition mapping found for Mooclet version ${versionId}`); - } - result[map.experimentCondition.conditionCode] = value; - } - return result; - } - - /** - * Generate a unique outcome variable name based on experiment name and timestamp. - * Returns a string in this format: - * "[first 10 chars of sanitized experiment name]_[ISO_timestamp]_REWARD_VARIABLE" - * - * This method mirrors the frontend logic but ensures outcome variable names - * are generated server-side and remain immutable after creation. - */ - private generateUniqueOutcomeVariableName(experimentName: string): string { - // first 10 chars of experiment name, sanitized - const baseName = experimentName - .toLowerCase() - .replace(/[^a-z0-9]+/g, '_') - .replace(/^_+|_+$/g, '') - .substring(0, 10); - - // append ISO date timestamp suffix to ensure uniqueness - const timestamp = new Date().toISOString(); - return `${baseName}_${timestamp}_REWARD_VARIABLE`; - } - - public async checkForMoocletAssignmentAlgorithmChange( - experiment: ExperimentDTO, - logger: UpgradeLogger - ): Promise<{ - hasChanged: boolean; - wasMooclet: boolean; - isNowMooclet: boolean; - oldAlgorithm: ASSIGNMENT_ALGORITHM; - }> { - logger.info({ message: `Check for assignment algorithm change for experiment id => ${experiment.id}` }); - const oldExperiment = await this.findOne(experiment.id, logger); - - if (!oldExperiment) { - logger.error({ message: 'Experiment unexpectedly not found', experimentId: experiment.id }); - throw new MoocletError(`Experiment unexpectedly not found for id ${experiment.id}`); - } - - const hasChanged = oldExperiment.assignmentAlgorithm !== experiment.assignmentAlgorithm; - const wasMooclet = this.isMoocletExperiment(oldExperiment.assignmentAlgorithm); - const isNowMooclet = this.isMoocletExperiment(experiment.assignmentAlgorithm); - return { - hasChanged, - wasMooclet, - isNowMooclet, - oldAlgorithm: oldExperiment.assignmentAlgorithm, - }; - } - - public async handlePotentialMoocletAssignmentAlgorithmChange( - experiment: ExperimentDTO, - currentUser: UserDTO, - logger: UpgradeLogger - ): Promise { - const { hasChanged, wasMooclet, isNowMooclet, oldAlgorithm } = await this.checkForMoocletAssignmentAlgorithmChange( - experiment, - logger - ); - let moocletRefToDelete: MoocletExperimentRef | undefined; - - if (hasChanged) { - if (experiment.state !== EXPERIMENT_STATE.INACTIVE) { - logger.error({ - message: 'Cannot edit assignment algorithm for non-INACTIVE experiment', - experimentId: experiment.id, - state: experiment.state, - }); - throw new MoocletError( - `Experiment state is ${experiment.state}. Can only edit assignment algorithm for experiments yet to begin (INACTIVE state)`, - 400 - ); - } - - // 1. Cache old mooclet ref BEFORE update, but don't delete stuff from mooclet db yet until after successful update in upgrade db - if (wasMooclet) { - moocletRefToDelete = await this.getMoocletExperimentRefByUpgradeExperimentId(experiment.id); - logger.debug({ - message: '[Algorithm Change] Fetched mooclet resources for deletion after update', - moocletRefId: moocletRefToDelete?.id, - experimentId: experiment.id, - }); - } - - // 2. Update experiment with new assignment algorithm, creating new mooclet resources when "isNowMooclet" - let message = `[Algorithm Change] Assignment algorithm changed from ${oldAlgorithm} to ${experiment.assignmentAlgorithm}.`; - if (wasMooclet) message += ' Will delete old Mooclet resources after successful update.'; - if (isNowMooclet) message += ' Will create new Mooclet resources as part of update.'; - - logger.info({ - message, - experimentId: experiment.id, - newAlgorithm: experiment.assignmentAlgorithm, - moocletRefToDelete, - }); - const updatedExperiment = await this.syncUpdateWithMoocletAlgorithmTransition({ - experimentDTO: experiment, - currentUser, - logger, - moocletRefToDelete, - }); - - // 3. AFTER successful update, delete old mooclet resources when "wasMooclet" - if (wasMooclet && moocletRefToDelete) { - try { - logger.info({ - message: '[Algorithm Change] Deleting old mooclet resources after successful update', - moocletRefId: moocletRefToDelete.id, - experimentId: experiment.id, - }); - await this.orchestrateDeleteMoocletResources(moocletRefToDelete, logger); - logger.info({ - message: '[Algorithm Change] Successfully deleted old mooclet resources', - experimentId: experiment.id, - }); - } catch (error) { - // Log error but don't throw - the experiment update was successful - // Orphaned mooclet resources are less critical than rolling back the experiment - logger.error({ - message: - '[Algorithm Change] Failed to delete old mooclet resources after algorithm change. Resources may be orphaned.', - error, - moocletRefToDelete: { - id: moocletRefToDelete.id, - moocletId: moocletRefToDelete.moocletId, - experimentId: moocletRefToDelete.experimentId, - }, - experimentId: experiment.id, - }); - } - } - - return updatedExperiment; - } - - // Handle regular mooclet experiment updates (no algorithm change) - if (!hasChanged && isNowMooclet) { - return await this.syncUpdate({ - experimentDTO: experiment, - currentUser, - logger, - }); - } - - // No mooclet involvement, return null to let controller handle via normal path - return Promise.resolve(null); - } - - public async handleEnrollCondition( - experimentId: string, - condition: string, - logger: UpgradeLogger - ): Promise { - if (!env.mooclets.enabled) { - logger.error({ - message: 'Mooclet experiment algorithm is indicated but mooclets are not enabled', - }); - return undefined; - } - - // Note: Unlike regular experiments where we infer the condition the user "should" have gotten, - // we have to enroll the condition the client has marked, because the Mooclet assignment won't persist on a user basis. - // This means that outside factors can potentially influence the intended balance. - - try { - const moocletExperimentRef = await this.getMoocletExperimentRefByUpgradeExperimentId(experimentId); - - if (!moocletExperimentRef) { - logger.error({ - message: '[Mooclet Mark Condition] No MoocletExperimentRef found for experiment', - experimentId, - }); - throw new MoocletError( - `[Mooclet Mark Condition] No MoocletExperimentRef found for experiment id ${experimentId}` - ); - } - const versionConditionMap = moocletExperimentRef.versionConditionMaps.find( - (expCondition) => expCondition.experimentCondition.conditionCode === condition - ); - - if (!versionConditionMap) { - logger.error({ - message: '[Mooclet Mark Condition] No version found for condition', - condition, - moocletExperimentRef, - }); - throw new MoocletError(`[Mooclet Mark Condition] No version found for condition ${condition}`); - } - return versionConditionMap.experimentCondition; - } catch (err) { - logger.error({ - message: '[Mooclet Mark Condition] There was an error processing marked condition.', - err, - }); - throw err; - } - } -} diff --git a/packages/backend/src/api/services/MoocletRewardsService.ts b/packages/backend/src/api/services/MoocletRewardsService.ts deleted file mode 100644 index cdde84d90a..0000000000 --- a/packages/backend/src/api/services/MoocletRewardsService.ts +++ /dev/null @@ -1,353 +0,0 @@ -import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; -import { - EXPERIMENT_STATE, - SERVER_ERROR, - BinaryRewardValueMap, - MoocletTSConfigurablePolicyParametersDTO, - Prior, -} from 'upgrade_types'; -import { RequestedExperimentUser } from '../controllers/validators/ExperimentUserValidator'; -import { MoocletExperimentRef } from '../models/MoocletExperimentRef'; -import { MoocletDataService } from './MoocletDataService'; -import { MoocletExperimentRefRepository } from '../repositories/MoocletExperimentRefRepository'; -import { IndividualEnrollment } from '../models/IndividualEnrollment'; -import { IndividualEnrollmentRepository } from '../repositories/IndividualEnrollmentRepository'; -import { Service } from 'typedi'; -import { InjectRepository } from '../../typeorm-typedi-extensions'; -import { HttpError } from 'routing-controllers'; -import { - MoocletPaginatedResponse, - MoocletRewardCountRequestBody, - MoocletValueRequestBody, - MoocletValueResponseDetails, -} from '../../types/Mooclet'; -import { RewardValidator } from '../controllers/validators/RewardValidator'; -import { ExperimentRewardsByCondition, ExperimentRewardsSummary } from 'upgrade_types'; -import { MoocletExperimentService } from './MoocletExperimentService'; - -export interface IRewardResponse { - message: string; - request: RewardValidator; - reward: MoocletValueRequestBody; -} - -@Service() -export class MoocletRewardsService { - constructor( - @InjectRepository() - private moocletExperimentRefRepository: MoocletExperimentRefRepository, - @InjectRepository() - private individualEnrollmentRepository: IndividualEnrollmentRepository, - private moocletDataService: MoocletDataService, - private moocletExperimentService: MoocletExperimentService - ) {} - - /** - * Attempt to send a reward to the external mooclet API. - * This is intended to be a "fire-and-forget" operation; we do not wait for - * confirmation that the reward was received successfully. - * - * Several criteria must be met to validate that a reward can be sent: - * - * 1. Mooclets feature must be currently enabled - * - * 2. The unique experiment must be ascertained before sending a reward. - * `experimentId` is preferred, but if not provided, `context`, `site`, and `target` can - * be used to identify the experiment. - * - * 3. A complete synced Mooclet experiment reference must exist. - * - * 4. The user must have marked and have a unique enrollment. - * - * 5. The condition enrolled must match a `version` in the mooclet experiment ref. - * - * - If any of these criteria are not met, a 409 data-conflict error is thrown. - */ - public async sendReward( - user: RequestedExperimentUser, - request: RewardValidator, - logger: UpgradeLogger - ): Promise { - const { experimentId, context, decisionPoint, rewardValue } = request; - - try { - // Find the mooclet experiment ref by ID or decision point - const moocletExperimentRef = experimentId - ? await this.findMoocletExperimentRefById(experimentId, request, logger) - : await this.findMoocletExperimentRefByDecisionPoint(context, decisionPoint, request, logger); - - // Find user's enrollment - const enrollments = await this.individualEnrollmentRepository.findEnrollments(user.id, [ - moocletExperimentRef.experimentId, - ]); - - if (!enrollments.length || enrollments.length > 1) { - this.throwConflictError( - `Could not find unique user enrollment for experiment (userId: ${user.id}, enrollments: ${enrollments}), no reward sent.`, - request, - logger - ); - } - - // Get version ID for the user's condition - const enrollment = enrollments[0]; - const versionId = this.getVersionIdByConditionId(enrollment, moocletExperimentRef, request, logger); - - if (!versionId) { - this.throwConflictError( - `Could not find version id for user enrollment (userId: ${user.id}, experimentId: ${moocletExperimentRef.experimentId}, conditionId: ${enrollment.conditionId}).`, - request, - logger - ); - } - - // Prepare and send reward - const reward: MoocletValueRequestBody = { - variable: moocletExperimentRef.outcomeVariableName, - value: BinaryRewardValueMap[rewardValue], - mooclet: moocletExperimentRef.moocletId, - version: versionId, - learner: user.id, - }; - - logger.info({ message: 'Sending reward to mooclet', reward, user }); - - // Fire-and-forget operation - // NOTE: in the future we may want to batch these, the mooclet API technically supports it by adding "/create_many" to an endpoint but it's not documented - this.moocletDataService.postNewReward(reward, logger); - - return { message: `Reward sent to mooclet successfully.`, request, reward }; - } catch (error) { - if (error instanceof HttpError) { - throw error; - } - - // Log and wrap unexpected errors - this.throwConflictError( - `Failed to process reward request due to unexpected error (userId: ${user.id}, experimentId: ${ - experimentId || 'not provided' - }, rewardValue: ${rewardValue}).`, - request, - logger - ); - } - } - - /** - * Finds mooclet experiment ref by experiment ID - */ - private async findMoocletExperimentRefById( - experimentId: string, - request: RewardValidator, - logger: UpgradeLogger - ): Promise { - const moocletExperimentRef = await this.moocletExperimentRefRepository.findOne({ - where: { experimentId }, - relations: { - versionConditionMaps: { - experimentCondition: true, - }, - - experiment: true, - }, - }); - - if (!moocletExperimentRef) { - this.throwConflictError( - `No active mooclet experiment ref found for experiment id: ${experimentId}, could not send reward.`, - request, - logger - ); - } - - if (moocletExperimentRef.experiment.state !== EXPERIMENT_STATE.ENROLLING) { - this.throwConflictError( - `Experiment with id: ${experimentId} is not actively enrolling (current state: ${moocletExperimentRef.experiment.state}), could not send reward.`, - request, - logger - ); - } - - return moocletExperimentRef; - } - - /** - * Finds mooclet experiment ref by decision point - */ - private async findMoocletExperimentRefByDecisionPoint( - context: string, - decisionPoint: { site: string; target: string }, - request: RewardValidator, - logger: UpgradeLogger - ): Promise { - const { site, target } = decisionPoint; - const moocletExperimentRefs = - await this.moocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget( - context, - site, - target - ); - - if (moocletExperimentRefs.length === 0) { - this.throwConflictError( - `No active experiment found for decision point (context: ${context}, site: ${site}, target: ${target}), could not send reward.`, - request, - logger - ); - } - - // TODO: this is a spot where we want to use shared-decision-point pooling logic, - // but for now if there are competing experiments, it will be required to use experimentId - if (moocletExperimentRefs.length > 1) { - this.throwConflictError( - `Multiple active experiments found for decision point (context: ${context}, site: ${site}, target: ${target}), cannot determine which to send reward to.`, - request, - logger - ); - } - - return moocletExperimentRefs[0]; - } - - private getVersionIdByConditionId( - enrollment: IndividualEnrollment, - moocletExperimentRef: MoocletExperimentRef, - request: RewardValidator, - logger: UpgradeLogger - ): number | null { - const map = moocletExperimentRef.versionConditionMaps.find( - (map) => enrollment.conditionId === map.experimentConditionId - ); - if (!map) { - this.throwConflictError(`Version-condition mapping not found, no reward sent.`, request, logger); - } - return map.moocletVersionId; - } - - public async getRewardsSummaryForExperiment( - experimentId: string, - logger: UpgradeLogger - ): Promise { - try { - const moocletExperimentRef = await this.moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId( - experimentId - ); - const rewards: MoocletValueResponseDetails[] = []; - logger.info({ - message: `Fetching Rewards data from mooclet server.`, - experimentId, - }); - let response = await this.fetchRewardsForExperiment(moocletExperimentRef, logger); - if (Array.isArray(response.results)) { - rewards.push(...response.results); - } - - while (response.next) { - logger.info({ - message: `But wait there's more (Fetching more Rewards data from Mooclet server for experiment...)`, - totalFound: response.count, - totalFetched: response.results.length, - next: response.next, - }); - response = await this.fetchRewardsForExperiment(moocletExperimentRef, logger, response.next); - if (Array.isArray(response.results)) { - rewards.push(...response.results); - } - } - - let tsConfigurableParams: MoocletTSConfigurablePolicyParametersDTO | undefined; - if (moocletExperimentRef.policyParametersId) { - try { - const policyParametersResponse = await this.moocletDataService.getPolicyParameters( - moocletExperimentRef.policyParametersId, - logger - ); - tsConfigurableParams = policyParametersResponse.parameters as MoocletTSConfigurablePolicyParametersDTO; - } catch (policyError) { - logger.warn({ - message: 'Could not fetch policy parameters for Thompson sampling estimate', - experimentId, - error: policyError, - }); - } - } - - return this.createExperimentRewardsSummary(moocletExperimentRef, rewards, logger, tsConfigurableParams); - } catch (error) { - logger.error({ message: 'Error fetching rewards summary for experiment', experimentId, error }); - throw error; - } - } - - public async fetchRewardsForExperiment( - moocletExperimentRef: MoocletExperimentRef, - logger: UpgradeLogger, - nextPageUrl?: string - ): Promise> { - const requestBody: MoocletRewardCountRequestBody = { - moocletId: moocletExperimentRef.moocletId, - variableName: moocletExperimentRef.outcomeVariableName, - }; - - return await this.moocletDataService.getRewardsForExperiment(requestBody, logger, nextPageUrl); - } - - public async createExperimentRewardsSummary( - moocletExperimentRef: MoocletExperimentRef, - rewardsData: MoocletValueResponseDetails[], - logger: UpgradeLogger, - policyParameters?: MoocletTSConfigurablePolicyParametersDTO - ): Promise { - const rewards: MoocletValueResponseDetails[] = rewardsData; - - if (!rewardsData) { - logger.warn({ - message: 'No rewards data returned from Mooclet API', - experimentId: moocletExperimentRef.experimentId, - }); - return []; - } - - const DEFAULT_PRIOR: Prior = { success: 1, failure: 1 }; - - const rewardsSummaries = moocletExperimentRef.versionConditionMaps.map( - ({ experimentCondition, moocletVersionId }) => { - const conditionCode = experimentCondition.conditionCode; - const versionIdKey = String(moocletVersionId); - const versionRewards = rewards.filter((reward) => reward.version === moocletVersionId); - const successes = versionRewards.filter((reward) => reward.value === 1.0).length; - const failures = versionRewards.filter((reward) => reward.value === 0.0).length; - const total = successes + failures; - const percentSuccess = total > 0 ? (successes / total) * 100 : 0.0; - const successRate = percentSuccess.toFixed(1) + '%'; - - const conditionPrior: Prior = policyParameters?.prior?.[versionIdKey] ?? DEFAULT_PRIOR; - - const rewardsForCondition: ExperimentRewardsByCondition = { - conditionCode, - successes, - failures, - successRate, - order: experimentCondition.order, - priorSuccess: conditionPrior.success, - priorFailure: conditionPrior.failure, - }; - return rewardsForCondition; - } - ); - - const orderedRewardsSummary = rewardsSummaries.sort((a, b) => a.order - b.order); - return orderedRewardsSummary; - } - - /** - * Throws a 409 data-conflict error for most unexpected cases - */ - private throwConflictError(message: string, request: RewardValidator, logger: UpgradeLogger): never { - logger.error({ message, request }); - - const error = new HttpError(409, message); - (error as any).type = SERVER_ERROR.MOOCLET_REWARD_ERROR; - throw error; - } -} diff --git a/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts new file mode 100644 index 0000000000..2ae0c0b466 --- /dev/null +++ b/packages/backend/src/api/services/ThompsonSamplingExperimentCrudService.ts @@ -0,0 +1,354 @@ +import { Service } from 'typedi'; +import { InjectRepository } from '../../typeorm-typedi-extensions'; +import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; +import { ConditionPosteriorStateRepository } from '../repositories/ConditionPosteriorStateRepository'; +import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; +import { ThompsonSamplingService } from './ThompsonSamplingService'; +import { CacheService } from './CacheService'; +import { ASSIGNMENT_ALGORITHM, CACHE_PREFIX, ExperimentRewardsSummary } from 'upgrade_types'; +import { ExperimentDTO } from '../DTO/ExperimentDTO'; +import { AdaptiveExperimentConfigService } from './AdaptiveExperimentConfigService'; + +type ConditionRef = { id: string }; + +export interface ThompsonSamplingConfigParams { + warmupThreshold?: number; + minimumDrawDifference?: number; + batchSize?: number; + /** Beta priors per condition, keyed by conditionId. Defaults to Beta(1,1) for missing entries. */ + priors?: Record; +} + +@Service() +export class ThompsonSamplingExperimentCrudService implements AdaptiveExperimentConfigService { + constructor( + @InjectRepository() private configRepository: ThompsonSamplingExperimentConfigRepository, + @InjectRepository() private posteriorStateRepository: ConditionPosteriorStateRepository, + private thompsonSamplingService: ThompsonSamplingService, + private cacheService: CacheService + ) {} + + public async getConfigForExperiment(experimentId: string): Promise { + return this.configRepository.findByExperimentId(experimentId); + } + + /** + * Single gate for "does this experiment need a Thompson Sampling config", so every experiment + * creation path (single create, bulk import, batch create) gets config/posterior rows the same + * way instead of each caller re-checking assignmentAlgorithm itself. Only `priors`/`warmupThreshold`/ + * `batchSize`/`minimumDrawDifference` from `experiment.thompsonSamplingConfig` are ever read here — + * there is no field for success/failure counts, so posterior state always starts at zero regardless + * of what the caller's source experiment (e.g. an imported/exported one) previously accumulated. + * + * `priors` is keyed by whatever condition IDs the caller submitted (client-generated temp IDs on + * create, or the previously-exported IDs on import), but ExperimentService.create()/deduceConditions() + * replace every condition ID with a freshly generated one before/while persisting — so those keys + * never match `createdExperiment.conditions[].id` on their own. `originalConditionIds` (captured by + * the caller before create() ran) lets remapPriorsToNewConditionIds() translate them. + */ + public async createConfigIfApplicable( + experiment: ExperimentDTO, + createdExperiment: ExperimentDTO, + originalConditionIds?: string[] + ): Promise { + if (experiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return; + } + const remappedConfig = this.remapPriorsToNewConditionIds( + experiment.thompsonSamplingConfig, + originalConditionIds, + createdExperiment.conditions + ); + await this.createConfig(createdExperiment.id, createdExperiment.conditions, remappedConfig ?? {}); + } + + /** + * Translates a priors record keyed by pre-creation condition IDs onto the condition IDs the + * experiment actually ended up with. Both `originalConditionIds` and `newConditions` are produced + * by order-preserving map/forEach transforms all the way through ExperimentService's create/import + * pipeline (conditions are never reordered, only replaced in place), so corresponding entries at the + * same array index refer to the same condition — there is no other stable, unique-per-condition key + * available to correlate on (ExperimentCondition.twoCharacterId was removed; conditionCode is not + * guaranteed unique). Without this, every condition would silently fall back to the default + * Beta(1,1) prior whenever the caller's condition IDs get regenerated. + */ + private remapPriorsToNewConditionIds( + config: ThompsonSamplingConfigParams | undefined, + originalConditionIds: string[] | undefined, + newConditions: ConditionRef[] + ): ThompsonSamplingConfigParams | undefined { + if (!config?.priors || !originalConditionIds) { + return config; + } + + const remappedPriors: Record = {}; + originalConditionIds.forEach((oldId, index) => { + const prior = config.priors?.[oldId]; + const newId = newConditions[index]?.id; + if (prior && newId) { + remappedPriors[newId] = prior; + } + }); + + return { ...config, priors: remappedPriors }; + } + + /** + * Update-path counterpart to createConfigIfApplicable: keeps posterior rows in sync with the + * current condition list and applies any prior/threshold changes, only for Thompson Sampling + * experiments. Also handles both directions of an algorithm change on an existing experiment: + * switching TO Thompson Sampling creates the config that createConfigIfApplicable never got a + * chance to (this is an update, not the original create), and switching AWAY FROM it deletes any + * config left over from before, so the reward path can't keep treating a now-non-adaptive + * experiment as Thompson Sampling. + */ + public async syncConfigIfApplicable(experiment: ExperimentDTO, updatedExperiment: ExperimentDTO): Promise { + if (updatedExperiment.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + await this.deleteConfigIfExists(updatedExperiment.id); + return; + } + + const existingConfig = await this.getConfigForExperiment(updatedExperiment.id); + if (!existingConfig) { + await this.createConfig( + updatedExperiment.id, + updatedExperiment.conditions, + experiment.thompsonSamplingConfig ?? {} + ); + return; + } + + await this.syncConditions(updatedExperiment.id, updatedExperiment.conditions); + if (experiment.thompsonSamplingConfig) { + await this.updateConfig(updatedExperiment.id, experiment.thompsonSamplingConfig); + } + } + + /** + * Populates `experiment.thompsonSamplingConfig` from the stored config/posterior rows for API + * responses and experiment export. Only ever reads `priorSuccess`/`priorFailure` (the Beta seed) — + * never `successCount`/`failureCount` — so exporting an experiment and re-importing it carries the + * configured priors forward without also carrying forward accumulated reward evidence. + */ + public async attachConfigToExperiment(experiment: T): Promise { + if (experiment?.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return experiment; + } + const config = await this.getConfigForExperiment(experiment.id); + if (!config) { + return experiment; + } + experiment.thompsonSamplingConfig = { + warmupThreshold: config.warmupThreshold, + minimumDrawDifference: config.minimumDrawDifference, + batchSize: config.batchSize, + priors: this.thompsonSamplingService.buildPriorsRecord(config.conditionPosteriorStates ?? []), + }; + return experiment; + } + + public async createConfig( + experimentId: string, + conditions: ConditionRef[], + params: ThompsonSamplingConfigParams = {} + ): Promise { + const config = await this.configRepository.save({ + experimentId, + warmupThreshold: params.warmupThreshold ?? 0, + minimumDrawDifference: params.minimumDrawDifference ?? 0, + batchSize: params.batchSize ?? 1, + }); + + await Promise.all( + conditions.map((condition) => + this.posteriorStateRepository.save({ + experimentId, + configId: config.id, + conditionId: condition.id, + priorSuccess: params.priors?.[condition.id]?.success ?? 1, + priorFailure: params.priors?.[condition.id]?.failure ?? 1, + successCount: 0, + totalCount: 0, + }) + ) + ); + + await this.invalidateConfigCache(); + + return config; + } + + public async updateConfig(experimentId: string, params: ThompsonSamplingConfigParams): Promise { + // Only touch fields the caller actually provided -- `params.field ?? null` would otherwise + // silently clear any field a partial payload omits, since each is independently optional. + const fieldsToUpdate: Partial< + Pick + > = {}; + if (params.warmupThreshold !== undefined) { + fieldsToUpdate.warmupThreshold = params.warmupThreshold; + } + if (params.minimumDrawDifference !== undefined) { + fieldsToUpdate.minimumDrawDifference = params.minimumDrawDifference; + } + if (params.batchSize !== undefined) { + fieldsToUpdate.batchSize = params.batchSize; + } + + if (Object.keys(fieldsToUpdate).length > 0) { + await this.configRepository.update({ experimentId }, fieldsToUpdate); + } + + await this.invalidateConfigCache(); + + if (!params.priors) { + return; + } + + const config = await this.configRepository.findByExperimentId(experimentId); + if (!config) { + return; + } + + await Promise.all( + Object.entries(params.priors).map(([conditionId, prior]) => + this.posteriorStateRepository.update( + { configId: config.id, conditionId }, + { + priorSuccess: prior?.success ?? 1, + priorFailure: prior?.failure ?? 1, + } + ) + ) + ); + } + + /** + * Per-condition reward totals and estimated win-rate weight, plus experiment-wide batch/warmup + * progress, for the reward feedback card display. Read-only aggregation — does not touch + * ConditionPosteriorState rows (see syncConditions() for that). + */ + public async getRewardsSummary(experimentId: string): Promise { + const config = await this.configRepository.findByExperimentIdWithConditions(experimentId); + + if (!config) { + return { conditions: [], pendingRewardsCount: 0, totalRewardCount: 0, warmupThreshold: 0, batchSize: 1 }; + } + + // A reward is only ever buffered (pendingTotalCount > 0) when batchSize > 1 -- unset/<=1 applies + // immediately, so there's nothing to sum and no batch to cycle through. + const pendingRewardsCount = + config.batchSize > 1 ? config.conditionPosteriorStates.reduce((sum, s) => sum + s.pendingTotalCount, 0) : 0; + // Same measure warmupThreshold gates on in ThompsonSamplingService/ExperimentAssignmentService: + // flushed evidence plus whatever's still sitting in a pending batch. + const totalRewardCount = config.conditionPosteriorStates.reduce( + (sum, s) => sum + s.totalCount + s.pendingTotalCount, + 0 + ); + + const rows = config.conditionPosteriorStates.map((state) => { + const successes = state.successCount; + const failures = state.failureCount; + const successRate = state.totalCount > 0 ? ((successes / state.totalCount) * 100).toFixed(1) + '%' : '0.0%'; + const { alpha, beta } = this.thompsonSamplingService.computePosterior( + state.priorSuccess, + state.priorFailure, + state.successCount, + state.failureCount + ); + return { + conditionId: state.conditionId, + alpha, + beta, + conditionCode: state.condition?.conditionCode ?? state.conditionId, + successes, + failures, + successRate, + order: state.condition?.order ?? 0, + priorSuccess: state.priorSuccess, + priorFailure: state.priorFailure, + }; + }); + + // Keyed by conditionId, not conditionCode: conditionCode has no uniqueness constraint (only + // ExperimentCondition.twoCharacterId is unique), so two conditions sharing a code would + // otherwise collide in the weight map and silently swap estimatedWeight values. + const weightMap = this.thompsonSamplingService.estimateConditionWeights( + rows.map((r) => ({ code: r.conditionId, alpha: r.alpha, beta: r.beta })) + ); + + const conditions = rows + .map(({ conditionId, alpha: _alpha, beta: _beta, ...rest }) => ({ + ...rest, + estimatedWeight: weightMap[conditionId], + })) + .sort((a, b) => a.order - b.order); + + return { + conditions, + pendingRewardsCount, + totalRewardCount, + warmupThreshold: config.warmupThreshold, + batchSize: config.batchSize, + }; + } + + /** + * Keeps ConditionPosteriorState rows in sync with the experiment's current conditions. + * Adds rows for new conditions (using default priors) and removes rows for deleted conditions. + */ + public async syncConditions(experimentId: string, currentConditions: ConditionRef[]): Promise { + const config = await this.configRepository.findByExperimentId(experimentId); + if (!config) return; + + const existingIds = new Set(config.conditionPosteriorStates.map((s) => s.conditionId)); + const currentIds = new Set(currentConditions.map((c) => c.id)); + + const toAdd = currentConditions.filter((c) => !existingIds.has(c.id)); + await Promise.all( + toAdd.map((condition) => + this.posteriorStateRepository.save({ + experimentId, + configId: config.id, + conditionId: condition.id, + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + totalCount: 0, + }) + ) + ); + + const toRemove = config.conditionPosteriorStates.filter((s) => !currentIds.has(s.conditionId)); + if (toRemove.length > 0) { + await this.posteriorStateRepository.remove(toRemove); + } + } + + /** + * Deletes the config (and, via ON DELETE CASCADE, its posterior-state rows) when an experiment + * that used to be Thompson Sampling is switched to a different algorithm. Without this, the + * config row would linger and ThompsonSamplingExperimentConfigRepository's enrolling-state-only + * queries would keep finding it, letting the reward path treat a now-non-adaptive experiment as + * if it were still Thompson Sampling. + */ + private async deleteConfigIfExists(experimentId: string): Promise { + const config = await this.configRepository.findByExperimentId(experimentId); + if (!config) { + return; + } + await this.configRepository.remove(config); + await this.invalidateConfigCache(); + } + + /** + * Clears every cached config lookup ThompsonSamplingRewardService may have made — both the + * by-experimentId and by-decision-point keys share this prefix. A targeted delete of just the + * affected experimentId key isn't enough on its own: the decision-point-keyed entries embed the + * same config fields and there's no cheap way to know which context/site/target keys reference + * this experiment, so the whole prefix is reset instead (same approach ExperimentService.updateList + * uses for validExperiments- when a similar can't-cheaply-target-one-key situation comes up). + */ + private async invalidateConfigCache(): Promise { + await this.cacheService.resetPrefixCache(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + } +} diff --git a/packages/backend/src/api/services/ThompsonSamplingRewardService.ts b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts new file mode 100644 index 0000000000..cf271e231b --- /dev/null +++ b/packages/backend/src/api/services/ThompsonSamplingRewardService.ts @@ -0,0 +1,279 @@ +import { Service } from 'typedi'; +import { EntityManager } from 'typeorm'; +import { InjectRepository } from '../../typeorm-typedi-extensions'; +import { UpgradeLogger } from '../../lib/logger/UpgradeLogger'; +import { BinaryRewardAllowedValue, CACHE_PREFIX, EXPERIMENT_STATE } from 'upgrade_types'; +import { ConditionPosteriorStateRepository } from '../repositories/ConditionPosteriorStateRepository'; +import { ThompsonSamplingExperimentConfigRepository } from '../repositories/ThompsonSamplingExperimentConfigRepository'; +import { IndividualEnrollmentRepository } from '../repositories/IndividualEnrollmentRepository'; +import { ThompsonSamplingExperimentConfig } from '../models/ThompsonSamplingExperimentConfig'; +import { ConditionPosteriorState } from '../models/ConditionPosteriorState'; +import { ThompsonSamplingReward } from '../models/ThompsonSamplingReward'; +import { CacheService } from './CacheService'; +import { RewardValidator } from '../controllers/validators/RewardValidator'; +import { RequestedExperimentUser } from '../controllers/validators/ExperimentUserValidator'; + +export interface IThompsonSamplingRewardResponse { + message: string; + request: RewardValidator; +} + +/** + * Thrown internally to unwind out of processReward() once a failure has already been logged via + * logAndAbort() — nothing downstream is waiting on this rejection (see acceptReward()), so it + * exists purely for control flow, not to be reported anywhere else. + */ +class RewardProcessingAborted extends Error {} + +@Service() +export class ThompsonSamplingRewardService { + constructor( + @InjectRepository() + private posteriorStateRepository: ConditionPosteriorStateRepository, + @InjectRepository() + private tsConfigRepository: ThompsonSamplingExperimentConfigRepository, + @InjectRepository() + private individualEnrollmentRepository: IndividualEnrollmentRepository, + private cacheService: CacheService + ) {} + + /** + * Acknowledges the reward immediately and does the actual work (config/enrollment lookups, the + * audit write, posterior updates) in the background. Nothing on the client side is waiting on + * this to make a UI decision, so there's no reason to hold the connection — and make the caller + * pay for however many DB round trips recording and batching take — before responding. A failure + * only surfaces in the server logs; see processReward()/logAndAbort(). + */ + public acceptReward( + user: RequestedExperimentUser, + request: RewardValidator, + logger: UpgradeLogger + ): IThompsonSamplingRewardResponse { + this.processReward(user, request, logger).catch((error) => { + if (!(error instanceof RewardProcessingAborted)) { + logger.error({ + message: `Unexpected error processing Thompson Sampling reward (userId: ${user.id}, experimentId: ${ + request.experimentId ?? 'not provided' + }).`, + error, + request, + }); + } + }); + + return { message: 'Reward received and is being processed.', request }; + } + + private async processReward( + user: RequestedExperimentUser, + request: RewardValidator, + logger: UpgradeLogger + ): Promise { + const { experimentId, context, decisionPoint, rewardValue } = request; + const success = rewardValue === BinaryRewardAllowedValue.SUCCESS; + + const config = experimentId + ? await this.findConfigById(experimentId, request, logger) + : await this.findConfigByDecisionPoint(context, decisionPoint, request, logger); + + if (config.experiment.state !== EXPERIMENT_STATE.ENROLLING) { + this.logAndAbort( + `Experiment ${config.experimentId} is not actively enrolling (state: ${config.experiment.state}), reward not recorded.`, + request, + logger + ); + } + + const enrollments = await this.individualEnrollmentRepository.findEnrollments(user.id, [config.experimentId]); + + if (!enrollments.length || enrollments.length > 1) { + this.logAndAbort( + `Could not find unique enrollment for user ${user.id} in experiment ${config.experimentId}, reward not recorded.`, + request, + logger + ); + } + + const { conditionId } = enrollments[0]; + + const state = await this.posteriorStateRepository.findByConditionId(conditionId); + + if (!state) { + this.logAndAbort( + `No posterior state found for condition ${conditionId} in experiment ${config.experimentId}, reward not recorded.`, + request, + logger + ); + } + + await this.recordRewardAtomically(config.experimentId, conditionId, user.id, success, state, config.batchSize); + + logger.info({ + message: 'Thompson Sampling reward recorded', + experimentId: config.experimentId, + conditionId, + userId: user.id, + success, + }); + } + + /** + * Writes the ThompsonSamplingReward audit row and folds the reward into the posterior + * (successCount/totalCount) -- or buffers it as pending until batchSize reward observations have + * accumulated across the whole experiment -- inside one transaction. Committing both together + * means a failure partway through (the locked query, a save) rolls back the audit row too, so + * there's no window where a reward is durably recorded but never reflected in the posteriors. + * An unset/≤1 batchSize applies the reward immediately. + * + * The posterior update takes a pessimistic write lock on every ConditionPosteriorState row for + * this config up front (ordered by id, to avoid deadlocking against a concurrent reward that + * locks the same rows). batchSize paces how often posteriors move for the experiment as a whole, + * and a reward for any condition is evidence toward that same shared cadence, so the "is the + * batch ready" check has to see a consistent snapshot across every condition, not just the one + * that just received a reward — without the lock, two rewards arriving close together could both + * read the same pending totals and double-apply them, or one could have its just-buffered + * increment silently overwritten by the other's flush-reset. Once the shared total reaches + * batchSize, every condition's pending buffer is flushed, not just the one that tipped it over, + * so a low-volume condition still gets its pending counts folded in as soon as the batch closes. + */ + private async recordRewardAtomically( + experimentId: string, + conditionId: string, + userId: string, + success: boolean, + state: Pick, + batchSize?: number + ): Promise { + const effectiveBatchSize = batchSize && batchSize > 1 ? batchSize : 1; + + await this.posteriorStateRepository.manager.transaction(async (manager) => { + await manager.save(ThompsonSamplingReward, { + experimentId, + conditionId, + userId, + success, + }); + + const experimentStates = await manager + .createQueryBuilder(ConditionPosteriorState, 'state') + .where('state.configId = :configId', { configId: state.configId }) + .orderBy('state.id', 'ASC') + .setLock('pessimistic_write') + .getMany(); + + const current = experimentStates.find((s) => s.id === state.id); + if (!current) { + throw new Error(`Posterior state ${state.id} no longer exists`); + } + + current.pendingTotalCount += 1; + if (success) { + current.pendingSuccessCount += 1; + } else { + current.pendingFailureCount += 1; + } + + if (effectiveBatchSize <= 1) { + await Promise.all( + experimentStates.filter((s) => s.pendingTotalCount > 0).map((s) => this.flushPendingRewards(manager, s)) + ); + return; + } + + await manager.save(current); + + const totalPending = experimentStates.reduce((sum, s) => sum + s.pendingTotalCount, 0); + if (totalPending < effectiveBatchSize) { + return; + } + + await Promise.all( + experimentStates.filter((s) => s.pendingTotalCount > 0).map((s) => this.flushPendingRewards(manager, s)) + ); + }); + } + + private async flushPendingRewards(manager: EntityManager, state: ConditionPosteriorState): Promise { + state.totalCount += state.pendingTotalCount; + state.successCount += state.pendingSuccessCount; + state.failureCount += state.pendingFailureCount; + state.pendingTotalCount = 0; + state.pendingSuccessCount = 0; + state.pendingFailureCount = 0; + await manager.save(state); + } + + /** + * Config lookups are cached — warmupThreshold/minimumDrawDifference/batchSize and the + * experiment's enrolling state change only on an admin edit (ThompsonSamplingExperimentCrudService + * invalidates on write), while a reward can arrive for the same experiment far more often. This + * keeps a burst of rewards from re-querying the DB for data that hasn't moved. Same read-only + * contract as ExperimentService.getCachedValidExperiments(): the returned object is shared across + * callers, never mutate it. + * + * Note: an experiment's enrolling state can also change via ExperimentService.updateState() + * (start/stop), which this cache does not invalidate — that state change is tolerated as + * TTL-bounded staleness, the same tradeoff already accepted by getCachedValidExperiments for the + * assignment path. + */ + private async findConfigById( + experimentId: string, + request: RewardValidator, + logger: UpgradeLogger + ): Promise { + const config = await this.cacheService.wrap( + CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX + 'id:' + experimentId, + () => + this.tsConfigRepository.findOne({ + where: { experimentId }, + relations: { experiment: true }, + }) + ); + + if (!config) { + this.logAndAbort( + `No Thompson Sampling config found for experiment ${experimentId}, reward not recorded.`, + request, + logger + ); + } + + return config; + } + + private async findConfigByDecisionPoint( + context: string, + decisionPoint: { site: string; target: string }, + request: RewardValidator, + logger: UpgradeLogger + ): Promise { + const { site, target } = decisionPoint; + const configs = await this.cacheService.wrap( + CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX + `dp:${context}:${site}:${target}`, + () => this.tsConfigRepository.findByDecisionPoint(context, site, target) + ); + + if (configs.length === 0) { + this.logAndAbort( + `No active Thompson Sampling experiment found for decision point (context: ${context}, site: ${site}, target: ${target}).`, + request, + logger + ); + } + + if (configs.length > 1) { + this.logAndAbort( + `Multiple active Thompson Sampling experiments found for decision point (context: ${context}, site: ${site}, target: ${target}); use experimentId to disambiguate.`, + request, + logger + ); + } + + return configs[0]; + } + + private logAndAbort(message: string, request: RewardValidator, logger: UpgradeLogger): never { + logger.error({ message, request }); + throw new RewardProcessingAborted(message); + } +} diff --git a/packages/backend/src/api/services/ThompsonSamplingService.ts b/packages/backend/src/api/services/ThompsonSamplingService.ts new file mode 100644 index 0000000000..304a5b6330 --- /dev/null +++ b/packages/backend/src/api/services/ThompsonSamplingService.ts @@ -0,0 +1,247 @@ +import { Service } from 'typedi'; + +export interface ConditionPrior { + success: number; + failure: number; +} + +export interface ConditionRewardSummary { + conditionId: string; + successCount: number; + failureCount: number; + totalCount: number; +} + +export interface ThompsonSamplingConfig { + /** Per-condition Beta distribution priors. Conditions without an entry use DEFAULT_PRIOR. */ + priors?: Record; + /** Use uniform random selection until total reward observations exceed this count. */ + warmupThreshold?: number; + /** Fall back to uniform when the top two sampled draws differ by less than this value. */ + minimumDrawDifference?: number; +} + +export const DEFAULT_PRIOR: ConditionPrior = { success: 1, failure: 1 }; + +@Service() +export class ThompsonSamplingService { + /** + * Estimate how often each condition would "win" a Thompson Sampling draw given current posteriors. + * + * Runs `numDraws` simulated rounds. Each round samples Beta(alpha, beta) for every condition and + * awards the round to the highest draw. Win counts are converted to integer percentages via the + * Largest Remainder Method so the result always sums to exactly 100. + * + * When every condition shares the same alpha/beta (e.g. equal priors and no reward data yet), + * the true win rate is exactly uniform by symmetry — skip the simulation and its sampling noise. + * + * @param conditions - Each condition with its current posterior alpha and beta parameters + * @param numDraws - Number of simulated draws (default 10 000) + * @returns Map of conditionCode → integer percentage in [0, 100]; sums to 100 + */ + estimateConditionWeights( + conditions: Array<{ code: string; alpha: number; beta: number }>, + numDraws = 10_000 + ): Record { + if (conditions.length === 0) return {}; + if (conditions.length === 1) return { [conditions[0].code]: 100 }; + + if (conditions.every((c) => c.alpha === conditions[0].alpha && c.beta === conditions[0].beta)) { + return this.distributeEvenly(conditions.map((c) => c.code)); + } + + const wins = new Map(conditions.map((c) => [c.code, 0])); + + for (let i = 0; i < numDraws; i++) { + let bestDraw = -1; + let winner = conditions[0].code; + for (const { code, alpha, beta } of conditions) { + const draw = this.sampleBeta(alpha, beta); + if (draw > bestDraw) { + bestDraw = draw; + winner = code; + } + } + wins.set(winner, (wins.get(winner) ?? 0) + 1); + } + + return this.toIntegerPercentages( + conditions.map((c) => ({ code: c.code, raw: ((wins.get(c.code) ?? 0) / numDraws) * 100 })) + ); + } + + // Splits 100 points evenly across codes (Largest Remainder Method handles the non-divisible case). + private distributeEvenly(codes: string[]): Record { + return this.toIntegerPercentages(codes.map((code) => ({ code, raw: 100 / codes.length }))); + } + + // Largest Remainder Method: floor each raw percentage then distribute the + // remaining integer points to the entries with the largest fractional parts. + private toIntegerPercentages(raws: Array<{ code: string; raw: number }>): Record { + const withFloors = raws.map(({ code, raw }) => ({ + code, + floor: Math.floor(raw), + remainder: raw - Math.floor(raw), + })); + const pointsLeft = 100 - withFloors.reduce((sum, r) => sum + r.floor, 0); + withFloors.sort((a, b) => b.remainder - a.remainder); + + const result: Record = {}; + withFloors.forEach((r, i) => { + result[r.code] = r.floor + (i < pointsLeft ? 1 : 0); + }); + return result; + } + + /** + * Select a condition using Thompson Sampling. + * + * @param conditionIds - All eligible condition IDs for this experiment + * @param rewardSummaries - Accumulated reward counts per condition + * @param totalRewardCount - Total number of reward observations collected across all conditions, + * including any not yet folded into the posteriors by a pending batch flush + * @param config - Optional algorithm parameters (priors, warmup, thresholds) + */ + selectCondition( + conditionIds: string[], + rewardSummaries: ConditionRewardSummary[], + totalRewardCount: number, + config: ThompsonSamplingConfig = {} + ): string { + if (conditionIds.length === 0) { + throw new Error('Cannot select from an empty condition list'); + } + if (conditionIds.length === 1) { + return conditionIds[0]; + } + + // Warmup phase: use uniform random until sufficient reward evidence has been collected. + // Gated on reward observations (not assignments) — the posteriors only move when rewards + // arrive, so that's the right measure of "how much evidence do we actually have." + // warmupThreshold === 0 means warmup is disabled (matches the frontend's isWarmupConfigured + // check) — without the > 0 guard, totalRewardCount <= 0 would still force one uniform draw + // before any reward ever arrives, contradicting a UI that already reports TS as active at 0/0. + if ( + config.warmupThreshold !== undefined && + config.warmupThreshold > 0 && + totalRewardCount <= config.warmupThreshold + ) { + return this.uniformRandom(conditionIds); + } + + const summaryMap = new Map(rewardSummaries.map((s) => [s.conditionId, s])); + + const draws = conditionIds.map((conditionId) => { + const summary = summaryMap.get(conditionId); + const prior = config.priors?.[conditionId] ?? DEFAULT_PRIOR; + const { alpha, beta } = this.computePosterior( + prior.success, + prior.failure, + summary?.successCount ?? 0, + summary?.failureCount ?? 0 + ); + return { conditionId, draw: this.sampleBeta(alpha, beta) }; + }); + + draws.sort((a, b) => b.draw - a.draw); + + // Fall back to uniform when the top two draws are too close to distinguish + if ( + config.minimumDrawDifference && + draws.length >= 2 && + draws[0].draw - draws[1].draw < config.minimumDrawDifference + ) { + return this.uniformRandom(conditionIds); + } + + return draws[0].conditionId; + } + + /** + * Beta posterior parameters for a condition, given its Beta(priorSuccess, priorFailure) prior and + * its accumulated success/failure counts. Shared by selectCondition() (which condition to draw) + * and ThompsonSamplingExperimentCrudService.getRewardsSummary() (the same math, for display) so + * a future correction to this formula can't be applied to one and missed in the other. + */ + computePosterior( + priorSuccess: number, + priorFailure: number, + successCount: number, + failureCount: number + ): { alpha: number; beta: number } { + return { alpha: priorSuccess + successCount, beta: priorFailure + failureCount }; + } + + /** + * Builds a `{ [conditionId]: { success, failure } }` prior record from posterior state rows. + * Shared by ExperimentAssignmentService (feeding the algorithm's config) and + * ThompsonSamplingExperimentCrudService (attaching config to API responses/export). + */ + buildPriorsRecord( + states: Array<{ conditionId: string; priorSuccess: number; priorFailure: number }> + ): Record { + const priors: Record = {}; + states.forEach((state) => { + priors[state.conditionId] = { success: state.priorSuccess, failure: state.priorFailure }; + }); + return priors; + } + + private uniformRandom(conditionIds: string[]): string { + return conditionIds[Math.floor(Math.random() * conditionIds.length)]; + } + + // Beta(α, β) sampled as the ratio of two independent Gamma samples + private sampleBeta(alpha: number, beta: number): number { + const x = this.sampleGamma(alpha); + const y = this.sampleGamma(beta); + return x / (x + y); + } + + // Gamma(α, 1) via Marsaglia–Tsang's squeeze method. + // Source: Marsaglia, G. & Tsang, W.W. (2000). "A Simple Method for Generating Gamma Variables." + // ACM Transactions on Mathematical Software, 26(3), pp. 363–372. DOI: 10.1145/358407.358414 + // The same algorithm is used by d3-random (https://github.com/d3/d3-random/blob/main/src/gamma.js) + // and jStat (https://github.com/jstat/jstat). This implementation follows the paper directly. + private sampleGamma(alpha: number): number { + if (alpha < 1) { + // Reduction: Gamma(α) = Gamma(α+1) · U^(1/α) where U ~ Uniform(0,1) + return this.sampleGamma(alpha + 1) * Math.pow(Math.random(), 1 / alpha); + } + const d = alpha - 1 / 3; + const c = 1 / Math.sqrt(9 * d); + for (;;) { + let x: number; + let v: number; + do { + x = this.sampleNormal(); + v = 1 + c * x; + } while (v <= 0); + v = v * v * v; + const u = Math.random(); + // Fast accept path (avoids log when safe) + if (u < 1 - 0.0331 * x * x * x * x) { + return d * v; + } + if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) { + return d * v; + } + } + } + + // N(0,1) via the Marsaglia polar method (polar form of Box–Muller). + // Source: Knuth, D.E. (1997). The Art of Computer Programming, Vol. 2, §3.4.1, Algorithm P. + // Original derivation: Marsaglia, G. & Bray, T.A. (1964). "A Convenient Method for Generating + // Normal Variables." SIAM Review, 6(3), pp. 260–264. DOI: 10.1137/1006063 + // Preferred over the basic Box–Muller form because it avoids evaluating trig functions and + // handles the degenerate case where Math.random() returns exactly 0 (log(0) = -Infinity). + private sampleNormal(): number { + let u: number, v: number, s: number; + do { + u = Math.random() * 2 - 1; + v = Math.random() * 2 - 1; + s = u * u + v * v; + } while (s >= 1 || s === 0); + return u * Math.sqrt((-2 * Math.log(s)) / s); + } +} diff --git a/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts b/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts new file mode 100644 index 0000000000..0eebc91664 --- /dev/null +++ b/packages/backend/src/database/migrations/1788362726319-nativeThompsonSampling.ts @@ -0,0 +1,202 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +/** + * Consolidates what was originally three (then four) separate migrations for the native Thompson + * Sampling feature (thompsonSamplingEntities, cleanupMoocletEntities, bootstrapThompsonSamplingConfigs, + * addPendingRewardCountsToConditionPosteriorState) into one. None of those had shipped to dev or been + * applied to any real database — they only ever existed on this unmerged branch — so there was no + * reason to keep them as separate steps; consolidating avoids interleaving with migrations that *have* + * since been merged to dev (remove-twoCharacterId, featureFlagPrecomputedSegment, + * experimentPrecomputedSegment), which this migration's timestamp is intentionally ordered after. + */ +export class NativeThompsonSampling1788362726319 implements MigrationInterface { + name = 'NativeThompsonSampling1788362726319'; + + public async up(queryRunner: QueryRunner): Promise { + // Swap 'ts_configurable' for 'thompson_sampling' in the assignment algorithm enum. Existing rows + // are remapped in the same USING expression that performs the type conversion, so there's no + // intermediate enum state where both values coexist. + await queryRunner.query( + `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` + ); + await queryRunner.query( + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'thompson_sampling')` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); + await queryRunner.query(` + ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" + USING ( + CASE "assignmentAlgorithm"::text + WHEN 'ts_configurable' THEN 'thompson_sampling' + ELSE "assignmentAlgorithm"::text + END + )::"public"."experiment_assignmentalgorithm_enum" + `); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); + await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); + + // Drop mooclet tables (cascade handles FK references) + await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_version_condition_map"`); + await queryRunner.query(`DROP TABLE IF EXISTS "mooclet_experiment_ref"`); + + // experiment_condition pre-exists this migration (base schema), so its composite unique + // constraint is added here via ALTER rather than in a CREATE TABLE. It's the target of the + // composite FKs below: pairing the already-unique "id" with "experimentId" lets a child row + // pin down *which experiment's* condition it's referencing, not just that the id exists + // somewhere, without changing the real uniqueness semantics (a condition's id is still + // globally unique on its own). + await queryRunner.query( + `ALTER TABLE "experiment_condition" ADD CONSTRAINT "UQ_experiment_condition_experimentId_id" UNIQUE ("experimentId", "id")` + ); + + // thompson_sampling_experiment_config: one-to-one with experiment + await queryRunner.query( + `CREATE TABLE "thompson_sampling_experiment_config" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "experimentId" uuid, + "warmupThreshold" integer NOT NULL DEFAULT 0, + "minimumDrawDifference" double precision NOT NULL DEFAULT 0, + "batchSize" integer NOT NULL DEFAULT 1, + "createdAt" TIMESTAMP NOT NULL DEFAULT now(), + "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), + "versionNumber" integer NOT NULL, + CONSTRAINT "UQ_ts_config_experimentId" UNIQUE ("experimentId"), + CONSTRAINT "UQ_ts_config_experimentId_id" UNIQUE ("experimentId", "id"), + CONSTRAINT "PK_ts_config" PRIMARY KEY ("id") + )` + ); + + // condition_posterior_state: per-condition Beta distribution state. pendingSuccessCount/ + // pendingTotalCount buffer rewards between batch flushes (see ThompsonSamplingRewardService) — + // included from the start since nothing has been applied anywhere yet. "experimentId" is + // denormalized down from the config so configId/conditionId can be tied together with composite + // FKs below, rather than two independent simple FKs that could each point at a *different* + // experiment's config/condition with nothing to catch it. + await queryRunner.query( + `CREATE TABLE "condition_posterior_state" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "experimentId" uuid NOT NULL, + "configId" uuid NOT NULL, + "conditionId" uuid NOT NULL, + "priorSuccess" double precision NOT NULL DEFAULT 1, + "priorFailure" double precision NOT NULL DEFAULT 1, + "successCount" integer NOT NULL DEFAULT 0, + "failureCount" integer NOT NULL DEFAULT 0, + "totalCount" integer NOT NULL DEFAULT 0, + "pendingSuccessCount" integer NOT NULL DEFAULT 0, + "pendingFailureCount" integer NOT NULL DEFAULT 0, + "pendingTotalCount" integer NOT NULL DEFAULT 0, + "createdAt" TIMESTAMP NOT NULL DEFAULT now(), + "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), + "versionNumber" integer NOT NULL, + CONSTRAINT "UQ_posterior_config_condition" UNIQUE ("configId", "conditionId"), + CONSTRAINT "PK_condition_posterior_state" PRIMARY KEY ("id") + )` + ); + + // thompson_sampling_reward: raw reward events (audit trail + posterior recalculation) + await queryRunner.query( + `CREATE TABLE "thompson_sampling_reward" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "experimentId" uuid NOT NULL, + "conditionId" uuid NOT NULL, + "userId" character varying NOT NULL, + "success" boolean NOT NULL, + "createdAt" TIMESTAMP NOT NULL DEFAULT now(), + "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), + "versionNumber" integer NOT NULL, + CONSTRAINT "PK_ts_reward" PRIMARY KEY ("id") + )` + ); + await queryRunner.query( + `CREATE INDEX "IDX_ts_reward_experiment_condition" ON "thompson_sampling_reward" ("experimentId", "conditionId")` + ); + + await queryRunner.query( + `ALTER TABLE "thompson_sampling_experiment_config" ADD CONSTRAINT "FK_ts_config_experiment" FOREIGN KEY ("experimentId") REFERENCES "experiment"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + // condition_posterior_state gets a direct FK on experimentId (for a clean cascade if the + // experiment itself is deleted) plus two *composite* FKs -- each requiring experimentId to + // agree with the config's/condition's own experimentId, so a row can no longer pair a config + // and a condition that belong to different experiments. + await queryRunner.query( + `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_experiment" FOREIGN KEY ("experimentId") REFERENCES "experiment"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_config" FOREIGN KEY ("experimentId", "configId") REFERENCES "thompson_sampling_experiment_config"("experimentId", "id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "condition_posterior_state" ADD CONSTRAINT "FK_posterior_state_condition" FOREIGN KEY ("experimentId", "conditionId") REFERENCES "experiment_condition"("experimentId", "id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "thompson_sampling_reward" ADD CONSTRAINT "FK_ts_reward_experiment" FOREIGN KEY ("experimentId") REFERENCES "experiment"("id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + // Same fix as condition_posterior_state above: thompson_sampling_reward already denormalizes + // experimentId (originally just for the composite index below), so pairing it with conditionId + // in this FK is a small extension that closes the identical gap here. + await queryRunner.query( + `ALTER TABLE "thompson_sampling_reward" ADD CONSTRAINT "FK_ts_reward_condition" FOREIGN KEY ("experimentId", "conditionId") REFERENCES "experiment_condition"("experimentId", "id") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + + // Bootstrap config + posterior state rows for any experiment already flagged thompson_sampling + // (including ones just remapped from ts_configurable above). Both tables are brand new in this + // migration, so every such experiment necessarily lacks rows — no NOT EXISTS guard needed. + await queryRunner.query(` + INSERT INTO "thompson_sampling_experiment_config" ("experimentId", "versionNumber") + SELECT e.id, 1 + FROM "experiment" e + WHERE e."assignmentAlgorithm" = 'thompson_sampling' + `); + + await queryRunner.query(` + INSERT INTO "condition_posterior_state" + ("experimentId", "configId", "conditionId", "priorSuccess", "priorFailure", "successCount", "failureCount", "totalCount", "pendingSuccessCount", "pendingFailureCount", "pendingTotalCount", "versionNumber") + SELECT c."experimentId", c.id, ec.id, 1, 1, 0, 0, 0, 0, 0, 0, 1 + FROM "thompson_sampling_experiment_config" c + JOIN "experiment_condition" ec ON ec."experimentId" = c."experimentId" + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "thompson_sampling_reward" DROP CONSTRAINT "FK_ts_reward_condition"`); + await queryRunner.query(`ALTER TABLE "thompson_sampling_reward" DROP CONSTRAINT "FK_ts_reward_experiment"`); + await queryRunner.query(`ALTER TABLE "condition_posterior_state" DROP CONSTRAINT "FK_posterior_state_condition"`); + await queryRunner.query(`ALTER TABLE "condition_posterior_state" DROP CONSTRAINT "FK_posterior_state_config"`); + await queryRunner.query(`ALTER TABLE "condition_posterior_state" DROP CONSTRAINT "FK_posterior_state_experiment"`); + await queryRunner.query( + `ALTER TABLE "thompson_sampling_experiment_config" DROP CONSTRAINT "FK_ts_config_experiment"` + ); + + await queryRunner.query(`DROP INDEX "IDX_ts_reward_experiment_condition"`); + await queryRunner.query(`DROP TABLE "thompson_sampling_reward"`); + await queryRunner.query(`DROP TABLE "condition_posterior_state"`); + await queryRunner.query(`DROP TABLE "thompson_sampling_experiment_config"`); + + // experiment_condition pre-exists this migration and isn't dropped above, so its composite + // unique constraint (added in up(), once the referencing composite FKs above are gone) needs + // an explicit drop rather than going away with the table. + await queryRunner.query( + `ALTER TABLE "experiment_condition" DROP CONSTRAINT "UQ_experiment_condition_experimentId_id"` + ); + + // Best-effort: the pre-migration enum has no 'thompson_sampling' value, so any experiment left in + // that state would fail the column cast below. Fall back to 'random' — this is a rollback of a + // feature that was never live, not a data-preserving downgrade. + await queryRunner.query( + `UPDATE "experiment" SET "assignmentAlgorithm" = 'random' WHERE "assignmentAlgorithm" = 'thompson_sampling'` + ); + + await queryRunner.query( + `ALTER TYPE "public"."experiment_assignmentalgorithm_enum" RENAME TO "experiment_assignmentalgorithm_enum_old"` + ); + await queryRunner.query( + `CREATE TYPE "public"."experiment_assignmentalgorithm_enum" AS ENUM('random', 'stratified random sampling', 'ts_configurable')` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" DROP DEFAULT`); + await queryRunner.query( + `ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" TYPE "public"."experiment_assignmentalgorithm_enum" USING "assignmentAlgorithm"::"text"::"public"."experiment_assignmentalgorithm_enum"` + ); + await queryRunner.query(`ALTER TABLE "experiment" ALTER COLUMN "assignmentAlgorithm" SET DEFAULT 'random'`); + await queryRunner.query(`DROP TYPE "public"."experiment_assignmentalgorithm_enum_old"`); + } +} diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 06889c67a4..089e47c49b 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -122,10 +122,4 @@ export const env = { secret: getOsEnv('CLIENT_API_SECRET'), key: getOsEnv('CLIENT_API_KEY'), }, - mooclets: { - enabled: toBool(getOsEnvOptional('MOOCLETS_ENABLED')) || false, - hostUrl: getOsEnvOptional('MOOCLETS_HOST_URL') || '', - apiRoute: getOsEnvOptional('MOOCLETS_API_ROUTE') || '', - apiToken: getOsEnvOptional('MOOCLETS_API_TOKEN') || '', - }, }; diff --git a/packages/backend/src/types/Mooclet.ts b/packages/backend/src/types/Mooclet.ts deleted file mode 100644 index 6ac16b1186..0000000000 --- a/packages/backend/src/types/Mooclet.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { MoocletPolicyParametersDTO } from 'upgrade_types'; - -export interface MoocletProxyRequestParams { - method: string; - url: string; - apiToken: string; - body?: MoocletRequestBody | MoocletPolicyParametersRequestBody | MoocletValueRequestBody | MoocletVariableRequestBody; -} - -export interface MoocletRequestBody { - name: string; - policy: number; -} - -export interface MoocletPaginatedResponse { - count: number; - next: string | null; - previous: string | null; - results: T[]; -} - -export interface MoocletResponseDetails { - id: number; - name: string; - policy: number; - mooclet_id?: string; - environment?: string; -} - -export interface MoocletVersionRequestBody { - mooclet: number; - name: string; - text?: string; - version_json?: Record; -} - -export interface MoocletVersionResponseDetails { - id: number; - name: string; - mooclet: number; - version_id?: number; - text?: string; - version_json?: Record; -} - -export interface MoocletPolicyParametersRequestBody { - mooclet: number; - policy: number; - parameters: MoocletPolicyParametersDTO; -} - -export interface MoocletPolicyParametersResponseDetails { - id: number; - mooclet: number; - policy: number; - parameters: MoocletPolicyParametersDTO; -} - -export interface MoocletVariableRequestBody { - name: string; -} - -export interface MoocletVariableResponseDetails { - id: number; - environment?: null; - variable_id?: null; - name: string; - min_value: number; - max_value: number; - value_type: 'BIN'; // binary is only supported type, must use 0 or 1 - sample_thres: number; -} - -export interface MoocletValueRequestBody { - variable: string; - value: number; - mooclet: number; - version: number; - learner?: number | string; - policy?: number; -} - -export interface MoocletRewardCountRequestBody { - moocletId: number; - variableName: string; -} - -export interface MoocletValueResponseDetails { - id: string; - variable: string; - learner: string; - mooclet: number; - version: number; - policy: number; - value: number; - text: string; - timestamp: string; -} - -export interface MoocletPolicyResponseDetails { - id: number; - name: string; - environment?: string; -} - -export interface MoocletBatchResponse { - count: number; - next: string; - previous: string; - results: T[]; -} diff --git a/packages/backend/test/unit/DTO/ExperimentDTO.test.ts b/packages/backend/test/unit/DTO/ExperimentDTO.test.ts new file mode 100644 index 0000000000..4262c712c0 --- /dev/null +++ b/packages/backend/test/unit/DTO/ExperimentDTO.test.ts @@ -0,0 +1,46 @@ +import 'reflect-metadata'; +import { validate } from 'class-validator'; +import { plainToInstance } from 'class-transformer'; +import { ExperimentDTO } from '../../../src/api/DTO/ExperimentDTO'; +import { ASSIGNMENT_UNIT, ASSIGNMENT_ALGORITHM } from 'upgrade_types'; + +describe('ExperimentDTO', () => { + describe('assignmentAlgorithm / assignmentUnit compatibility', () => { + it('rejects Thompson Sampling combined with Within-Subjects assignment', async () => { + // Within-Subjects assignment never stores a condition on the individual enrollment (it's + // tracked per-repeat via RepeatedEnrollment instead), which is what Thompson Sampling's + // reward path reads to attribute a reward to a condition -- so this combination can never + // record a reward and must be rejected up front. + const dto = plainToInstance(ExperimentDTO, { + assignmentUnit: ASSIGNMENT_UNIT.WITHIN_SUBJECTS, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + }); + + const errors = await validate(dto, { skipMissingProperties: true }); + + expect(errors.some((e) => e.property === 'assignmentAlgorithm')).toBe(true); + }); + + it('allows Thompson Sampling with Individual assignment', async () => { + const dto = plainToInstance(ExperimentDTO, { + assignmentUnit: ASSIGNMENT_UNIT.INDIVIDUAL, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + }); + + const errors = await validate(dto, { skipMissingProperties: true }); + + expect(errors.some((e) => e.property === 'assignmentAlgorithm')).toBe(false); + }); + + it('allows Within-Subjects assignment with a non-Thompson-Sampling algorithm', async () => { + const dto = plainToInstance(ExperimentDTO, { + assignmentUnit: ASSIGNMENT_UNIT.WITHIN_SUBJECTS, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, + }); + + const errors = await validate(dto, { skipMissingProperties: true }); + + expect(errors.some((e) => e.property === 'assignmentAlgorithm')).toBe(false); + }); + }); +}); diff --git a/packages/backend/test/unit/controllers/ExperimentController.test.ts b/packages/backend/test/unit/controllers/ExperimentController.test.ts index f5990855f0..6ca6e745a3 100644 --- a/packages/backend/test/unit/controllers/ExperimentController.test.ts +++ b/packages/backend/test/unit/controllers/ExperimentController.test.ts @@ -9,13 +9,9 @@ import { ExperimentService } from '../../../src/api/services/ExperimentService'; import { useContainer as classValidatorUseContainer } from 'class-validator'; import { ExperimentAssignmentService } from '../../../src/api/services/ExperimentAssignmentService'; import ExperimentAssignmentServiceMock from './mocks/ExperimentAssignmentServiceMock'; -import { MoocletExperimentService } from '../../../src/api/services/MoocletExperimentService'; -import MoocletExperimentServiceMock from './mocks/MoocletExperimentServiceMock'; -import { MoocletRewardsService } from '../../../src/api/services/MoocletRewardsService'; -import MoocletRewardsServiceMock from './mocks/MoocletRewardsServiceMock'; import { ImportExportService } from '../../../src/api/services/ImportExportService'; +import { ThompsonSamplingExperimentCrudService } from '../../../src/api/services/ThompsonSamplingExperimentCrudService'; import ImportExportServiceMock from './mocks/ImportExportServiceMock'; -import { env } from './../../../src/env'; import { ASSIGNMENT_ALGORITHM, ASSIGNMENT_UNIT, @@ -23,7 +19,6 @@ import { EXPERIMENT_STATE, EXPERIMENT_TYPE, FILTER_MODE, - MoocletTSConfigurablePolicyParametersDTO, POST_EXPERIMENT_RULE, SEGMENT_TYPE, } from 'upgrade_types'; @@ -38,9 +33,12 @@ describe('Experiment Controller Testing', () => { // set mock container Container.set(ExperimentService, new ExperimentServiceMock()); Container.set(ExperimentAssignmentService, new ExperimentAssignmentServiceMock()); - Container.set(MoocletExperimentService, new MoocletExperimentServiceMock()); - Container.set(MoocletRewardsService, new MoocletRewardsServiceMock()); Container.set(ImportExportService, new ImportExportServiceMock()); + Container.set(ThompsonSamplingExperimentCrudService, { + createConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + syncConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + attachConfigToExperiment: jest.fn().mockImplementation((experiment) => Promise.resolve(experiment)), + } as any); }); afterAll(() => { @@ -113,19 +111,6 @@ describe('Experiment Controller Testing', () => { ], }; - const tsConfigurablePolicyParameters = new MoocletTSConfigurablePolicyParametersDTO(); - tsConfigurablePolicyParameters.assignmentAlgorithm = ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE; - tsConfigurablePolicyParameters.outcome_variable_name = 'test_outcome'; - - const moocletExperimentData: ExperimentDTO = { - ...experimentData, - id: crypto.randomUUID(), - moocletPolicyParameters: tsConfigurablePolicyParameters, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - - console.log({ moocletExperimentData }); - //for future use where user will be mocked for all testcases // const mockUser: User = { @@ -147,16 +132,6 @@ describe('Experiment Controller Testing', () => { return request(app).post('/api/experiments').send(experimentData).expect('Content-Type', /json/).expect(200); }); - test('Post request for /api/experiments with moocletPolicyParameters and mooclets enabled', () => { - env.mooclets.enabled = true; - return request(app).post('/api/experiments').send(moocletExperimentData).expect('Content-Type', /json/).expect(200); - }); - - test('Post request for /api/experiments with moocletPolicyParameters and mooclets disabled', () => { - env.mooclets.enabled = false; - return request(app).post('/api/experiments').send(moocletExperimentData).expect(500); - }); - test('Get request for /api/experiments/names', () => { return request(app).get('/api/experiments/names').expect('Content-Type', /json/).expect(200); }); diff --git a/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts new file mode 100644 index 0000000000..b13f1acc8e --- /dev/null +++ b/packages/backend/test/unit/controllers/ExperimentControllerAdaptiveConfig.test.ts @@ -0,0 +1,383 @@ +import { ExperimentController } from '../../../src/api/controllers/ExperimentController'; +import { ASSIGNMENT_ALGORITHM, EXPERIMENT_STATE } from 'upgrade_types'; + +describe('ExperimentController adaptive config wiring', () => { + let experimentService: any; + let adaptiveExperimentConfigDispatcher: any; + let controller: ExperimentController; + let request: any; + + beforeEach(() => { + experimentService = { + validateExperimentContext: jest.fn().mockReturnValue(undefined), + create: jest.fn(), + update: jest.fn(), + updateState: jest.fn(), + delete: jest.fn().mockResolvedValue(undefined), + getSingleExperiment: jest.fn(), + }; + adaptiveExperimentConfigDispatcher = { + createConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + syncConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + attachConfigToExperiment: jest.fn().mockImplementation((experiment) => Promise.resolve(experiment)), + }; + request = { logger: { child: jest.fn(), error: jest.fn(), info: jest.fn() } }; + + controller = new ExperimentController( + experimentService, + {} as any, + {} as any, + {} as any, + {} as any, + adaptiveExperimentConfigDispatcher + ); + }); + + describe('create()', () => { + it('captures condition ids before create() runs and forwards them to createConfigIfApplicable', async () => { + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'client-temp-id-1' }, { id: 'client-temp-id-2' }], + thompsonSamplingConfig: { priors: { 'client-temp-id-1': { success: 2, failure: 3 } } }, + } as any; + // Mirrors what ExperimentService.create() actually does: mutate condition ids in place. + experimentService.create.mockImplementation((exp: any) => { + exp.conditions.forEach((condition: any, index: number) => { + condition.id = `server-id-${index + 1}`; + }); + return Promise.resolve({ ...exp, id: 'experiment-1' }); + }); + + await controller.create(experiment, {} as any, request); + + expect(adaptiveExperimentConfigDispatcher.createConfigIfApplicable).toHaveBeenCalledWith( + expect.objectContaining({ conditions: [{ id: 'server-id-1' }, { id: 'server-id-2' }] }), + expect.objectContaining({ id: 'experiment-1' }), + ['client-temp-id-1', 'client-temp-id-2'] + ); + }); + + it('deletes the just-created experiment and rethrows when config creation fails', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + experimentService.create.mockResolvedValue({ id: 'experiment-1', conditions: [] }); + const configError = new Error('config create failed'); + adaptiveExperimentConfigDispatcher.createConfigIfApplicable.mockRejectedValue(configError); + + await expect(controller.create(experiment, {} as any, request)).rejects.toThrow(configError); + + expect(experimentService.delete).toHaveBeenCalledWith('experiment-1', {}, { logger: request.logger }); + }); + }); + + describe('update()', () => { + it('does not touch previous state on the happy path', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: [] }); + + await controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenCalledTimes(1); + }); + + it('reverts the experiment and re-syncs the config when syncConfigIfApplicable fails', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const updatedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const revertedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const syncError = new Error('config sync failed'); + + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValueOnce(updatedExperiment).mockResolvedValueOnce(revertedExperiment); + adaptiveExperimentConfigDispatcher.syncConfigIfApplicable + .mockRejectedValueOnce(syncError) + .mockResolvedValueOnce(undefined); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + syncError + ); + + // First update is the caller's requested change; second is the revert back to the pre-update state. + expect(experimentService.update).toHaveBeenNthCalledWith( + 1, + { ...experiment, id: 'experiment-1' }, + {}, + request.logger + ); + expect(experimentService.update).toHaveBeenNthCalledWith(2, previousExperiment, {}, request.logger); + // First sync is the caller's requested change (which failed); second cleans up against the reverted state. + expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenNthCalledWith( + 1, + experiment, + updatedExperiment + ); + expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenNthCalledWith( + 2, + previousExperiment, + revertedExperiment + ); + }); + + it('snapshots thompsonSamplingConfig onto the previous experiment before the forward update runs, so a revert restores it', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const updatedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const revertedExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const syncError = new Error('config sync failed'); + const originalConfig = { warmupThreshold: 42, batchSize: 3, minimumDrawDifference: 0.1, priors: {} }; + const callOrder: string[] = []; + + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockImplementation(() => { + callOrder.push('update'); + return Promise.resolve( + callOrder.filter((c) => c === 'update').length === 1 ? updatedExperiment : revertedExperiment + ); + }); + // Mimics attachConfigToExperiment()'s real behavior: mutate the passed-in object in place. + // If this ran fresh inside the catch block instead of up front, it would see whatever a + // partially-failed sync had already committed to the DB -- not the true pre-update values. + adaptiveExperimentConfigDispatcher.attachConfigToExperiment.mockImplementation((exp: any) => { + callOrder.push('attachConfig'); + exp.thompsonSamplingConfig = originalConfig; + return Promise.resolve(exp); + }); + adaptiveExperimentConfigDispatcher.syncConfigIfApplicable + .mockRejectedValueOnce(syncError) + .mockResolvedValueOnce(undefined); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + syncError + ); + + expect(callOrder).toEqual(['attachConfig', 'update', 'update']); + expect(adaptiveExperimentConfigDispatcher.syncConfigIfApplicable).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ thompsonSamplingConfig: originalConfig }), + revertedExperiment + ); + }); + + it('still throws the original error, logged rather than masked, when the revert attempt itself fails', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const previousExperiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING }; + const syncError = new Error('config sync failed'); + const revertError = new Error('revert update failed'); + + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValueOnce({ id: 'experiment-1' }).mockRejectedValueOnce(revertError); + adaptiveExperimentConfigDispatcher.syncConfigIfApplicable.mockRejectedValue(syncError); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + syncError + ); + + expect(request.logger.error).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('Failed to fully revert'), error: revertError }) + ); + }); + + it('skips the revert (but still throws) when no previous experiment can be found', async () => { + const experiment = { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, conditions: [] } as any; + const syncError = new Error('config sync failed'); + + experimentService.getSingleExperiment.mockResolvedValue(undefined); + experimentService.update.mockResolvedValue({ id: 'experiment-1' }); + adaptiveExperimentConfigDispatcher.syncConfigIfApplicable.mockRejectedValue(syncError); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + syncError + ); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + }); + + it('rejects a condition field change once the experiment has started', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.RUNNING, + conditions: [{ id: 'condition-1', conditionCode: 'A', name: 'A', description: '', assignmentWeight: 50 }], + }; + const experiment = { + id: 'experiment-1', + conditions: [ + { id: 'condition-1', conditionCode: 'A-renamed', name: 'A', description: '', assignmentWeight: 50 }, + ], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be modified/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('rejects an added/removed condition once the experiment has started', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.PAUSED, + conditions: [{ id: 'condition-1', conditionCode: 'A', assignmentWeight: 100 }], + }; + const experiment = { + id: 'experiment-1', + conditions: [ + { id: 'condition-1', conditionCode: 'A', assignmentWeight: 50 }, + { id: 'condition-2', conditionCode: 'B', assignmentWeight: 50 }, + ], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be modified/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('rejects a Thompson Sampling prior change once the experiment has started', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.RUNNING, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } }, + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 5, failure: 1 } } }, + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be modified/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('allows an unrelated field change once the experiment has started, when conditions/priors are unchanged', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.RUNNING, + name: 'old name', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A', assignmentWeight: 100 }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } }, + }; + const experiment = { + id: 'experiment-1', + name: 'new name', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A', assignmentWeight: 100 }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } }, + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: experiment.conditions }); + + await controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + }); + + it('rejects switching the assignment algorithm from Thompson Sampling to something else, even on an inactive experiment', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.INACTIVE, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [], + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, + conditions: [], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be changed to or from Thompson Sampling/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('rejects switching the assignment algorithm to Thompson Sampling from something else, even on an inactive experiment', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.INACTIVE, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, + conditions: [], + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + + await expect(controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request)).rejects.toThrow( + /cannot be changed to or from Thompson Sampling/ + ); + expect(experimentService.update).not.toHaveBeenCalled(); + }); + + it('allows switching between two non-Thompson-Sampling algorithms', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.INACTIVE, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, + conditions: [], + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING, + conditions: [], + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: [] }); + + await controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + }); + + it('allows a condition/prior change while the experiment has not started yet', async () => { + const previousExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.INACTIVE, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } }, + }; + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1', conditionCode: 'A-renamed' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 5, failure: 1 } } }, + } as any; + experimentService.getSingleExperiment.mockResolvedValue(previousExperiment); + experimentService.update.mockResolvedValue({ id: 'experiment-1', conditions: experiment.conditions }); + + await controller.update({ id: 'experiment-1' } as any, experiment, {} as any, request); + + expect(experimentService.update).toHaveBeenCalledTimes(1); + }); + }); + + describe('updateState()', () => { + it('attaches the Thompson Sampling config onto the state-change response, like the other endpoints', async () => { + const updatedExperiment = { + id: 'experiment-1', + state: EXPERIMENT_STATE.PAUSED, + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + }; + experimentService.updateState.mockResolvedValue(updatedExperiment); + + const stateUpdate = { experimentId: 'experiment-1', state: EXPERIMENT_STATE.PAUSED } as any; + const result = await controller.updateState(stateUpdate, {} as any, request); + + expect(adaptiveExperimentConfigDispatcher.attachConfigToExperiment).toHaveBeenCalledWith(updatedExperiment); + expect(result).toBe(updatedExperiment); + }); + }); +}); diff --git a/packages/backend/test/unit/controllers/mocks/MoocletExperimentServiceMock.ts b/packages/backend/test/unit/controllers/mocks/MoocletExperimentServiceMock.ts deleted file mode 100644 index 62860a66fb..0000000000 --- a/packages/backend/test/unit/controllers/mocks/MoocletExperimentServiceMock.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { SyncCreateParams } from '../../../../src/api/services/MoocletExperimentService'; -import { Service } from 'typedi'; - -@Service() -export default class MoocletExperimentServiceMock { - public async syncCreate(params: SyncCreateParams): Promise<[]> { - return Promise.resolve([]); - } -} diff --git a/packages/backend/test/unit/controllers/mocks/MoocletRewardsServiceMock.ts b/packages/backend/test/unit/controllers/mocks/MoocletRewardsServiceMock.ts deleted file mode 100644 index 39796fefd2..0000000000 --- a/packages/backend/test/unit/controllers/mocks/MoocletRewardsServiceMock.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Service } from 'typedi'; - -@Service() -export default class MoocletRewardsServiceMock { - public async getRewardsSummaryForExperiment(experimentId: string, logger: any): Promise { - return [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; - } -} diff --git a/packages/backend/test/unit/services/CacheService.test.ts b/packages/backend/test/unit/services/CacheService.test.ts index cd8f108c82..1983ae5497 100644 --- a/packages/backend/test/unit/services/CacheService.test.ts +++ b/packages/backend/test/unit/services/CacheService.test.ts @@ -270,6 +270,8 @@ describe('CacheService', () => { // owner's TTL rather than the segments TTL [CACHE_PREFIX.FEATURE_FLAG_PRECOMPUTED_SEGMENT_KEY_PREFIX, 45000], [CACHE_PREFIX.EXPERIMENT_PRECOMPUTED_SEGMENT_KEY_PREFIX, 30000], + // Thompson Sampling config lookups share the experiments bucket/TTL + [CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX, 30000], ])('wraps %s with its category TTL (ms)', async (prefix, expectedTtl) => { const fn = jest.fn().mockResolvedValue('v'); await service.wrap(prefix + 'context', fn); diff --git a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts index 79eb590c44..2f5664fc29 100644 --- a/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts +++ b/packages/backend/test/unit/services/ExperimentAssignmentService.test.ts @@ -31,11 +31,16 @@ import { withinSubjectDPExperiment, } from '../mockdata'; import { GroupEnrollment } from '../../../src/api/models/GroupEnrollment'; -import { ENROLLMENT_CODE, EXPERIMENT_STATE, FILTER_MODE, MARKED_DECISION_POINT_STATUS } from 'upgrade_types'; +import { + ASSIGNMENT_ALGORITHM, + ENROLLMENT_CODE, + EXPERIMENT_STATE, + FILTER_MODE, + MARKED_DECISION_POINT_STATUS, +} from 'upgrade_types'; import { CacheService } from '../../../src/api/services/CacheService'; import { UserStratificationFactorRepository } from '../../../src/api/repositories/UserStratificationRepository'; import { configureLogger } from '../../utils/logger'; -import { MoocletExperimentService } from '../../../src/api/services/MoocletExperimentService'; import { ExperimentPrecomputedSegmentService } from '../../../src/api/services/ExperimentPrecomputedSegmentService'; import { factorialGroupExperiment, factorialIndividualExperiment } from '../mockdata/raw'; import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; @@ -72,7 +77,6 @@ describe('Experiment Assignment Service Test', () => { const segmentServiceMock = sinon.createStubInstance(SegmentService); const experimentServiceMock = sinon.createStubInstance(ExperimentService); const cacheServiceMock = sinon.createStubInstance(CacheService); - const moocletExperimentServiceMock = sinon.createStubInstance(MoocletExperimentService); const experimentPrecomputedSegmentServiceMock = sinon.createStubInstance(ExperimentPrecomputedSegmentService); // Default to "no precomputed rows" so the assignment read path exercises the on-the-fly fallback // (recursive segment resolution) these tests were written against. @@ -156,6 +160,7 @@ describe('Experiment Assignment Service Test', () => { stateTimeLogsRepositoryMock, analyticsRepositoryMock, userStratificationFactorRepositoryMock, + {} as any, // thompsonSamplingConfigRepository — not used in existing tests previewUserServiceMock, experimentUserServiceMock, errorServiceMock, @@ -163,8 +168,8 @@ describe('Experiment Assignment Service Test', () => { segmentServiceMock, experimentServiceMock, cacheServiceMock, - moocletExperimentServiceMock, - experimentPrecomputedSegmentServiceMock + experimentPrecomputedSegmentServiceMock, + {} as any // thompsonSamplingService — not used in existing tests ); testedModule.cacheService.wrap.resolves([]); @@ -2059,6 +2064,21 @@ describe('Experiment Assignment Service Test', () => { expect(result).toEqual({}); }); + it('should exclude Thompson Sampling (adaptive) experiments from batch assignment', async () => { + const context = 'home'; + const site = 'CurriculumSequence'; + const target = 'W1'; + const userDocs = [{ id: 'user1', group: { schoolId: ['school1'] }, workingGroup: {} }]; + const exp = structuredClone(simpleIndividualAssignmentExperiment) as any; + exp.assignmentAlgorithm = ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + + testedModule.experimentRepository.getValidExperimentsForContextAndDecisionPoint = sandbox.stub().resolves([exp]); + + const result = await testedModule.getBatchExperimentConditions(userDocs, context, site, target, loggerMock); + + expect(result).toEqual({}); + }); + it('should return batch experiment conditions for multiple users with simple individual experiment', async () => { const context = 'home'; const site = 'CurriculumSequence'; @@ -2273,16 +2293,99 @@ describe('Experiment Assignment Service Test', () => { }); }); - it('[getConditionFromMoocletProxy] should return undefined and log error when mooclet proxy throws', async () => { - const userDoc = { id: 'user123', group: {}, workingGroup: {} }; - const exp = structuredClone(simpleIndividualAssignmentExperiment); - const mockError = new Error('Mooclet proxy error'); + describe('[assignThompsonSampling] warmup evidence', () => { + const thompsonExperiment: any = { + id: 'ts-experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-a' }, { id: 'condition-b' }], + }; + const thompsonUser: any = { id: 'user-1' }; + + function makePosteriorState(conditionId: string, totalCount: number, pendingTotalCount: number) { + return { + conditionId, + successCount: 0, + failureCount: 0, + totalCount, + pendingTotalCount, + priorSuccess: 1, + priorFailure: 1, + }; + } + + it('counts rewards still sitting in a pending batch toward totalRewardCount, not just flushed ones', async () => { + testedModule.thompsonSamplingConfigRepository = { + findByExperimentId: sandbox.stub().resolves({ + warmupThreshold: 9, + minimumDrawDifference: undefined, + conditionPosteriorStates: [ + makePosteriorState('condition-a', 2, 3), // 2 flushed + 3 pending + makePosteriorState('condition-b', 1, 4), // 1 flushed + 4 pending + ], + }), + }; + testedModule.thompsonSamplingService = { + selectCondition: sandbox.stub().returns('condition-a'), + buildPriorsRecord: sandbox.stub().returns({}), + }; + + await (testedModule as any).assignThompsonSampling(thompsonExperiment, thompsonUser, loggerMock); - moocletExperimentServiceMock.getConditionFromMoocletProxy.rejects(mockError); + // Flushed-only totals (2 + 1 = 3) would wrongly stay inside a warmupThreshold of 9. + // The real reward evidence collected so far also includes the pending buffers + // (3 + 4 = 7), for a true total of 10 — past warmup. + const totalRewardCountArg = testedModule.thompsonSamplingService.selectCondition.getCall(0).args[2]; + expect(totalRewardCountArg).toBe(10); + }); + }); - const result = await (testedModule as any).getConditionFromMoocletProxy(exp, userDoc, loggerMock); + describe('[updateEnrollmentExclusion] Thompson Sampling trusts the client-reported condition', () => { + const experiment: any = { + id: 'ts-experiment-2', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + assignmentUnit: 'individual', + consistencyRule: 'individual', + state: 'enrolling', + conditions: [ + { id: 'condition-a', conditionCode: 'ConditionA' }, + { id: 'condition-b', conditionCode: 'ConditionB' }, + ], + }; + const user: any = { id: 'user-1', workingGroup: {} }; + const decisionPoint: any = { id: 'dp-1', site: 'site1', target: 'target1' }; + + it('persists the client-reported condition on first mark without re-running assignThompsonSampling', async () => { + testedModule.individualEnrollmentRepository.save = sandbox.stub().resolves(undefined); + // If mark ever falls back to assignExperiment() -> assignThompsonSampling() for this algorithm, + // this stub throws -- proving the fix (trusting the client-reported condition, like stratified + // random and within-subjects) rather than silently passing on an incidental TypeError. + testedModule.thompsonSamplingConfigRepository = { + findByExperimentId: sandbox + .stub() + .rejects(new Error('assignThompsonSampling should not run when mark trusts the client condition')), + }; - expect(result).toBeUndefined(); - sinon.assert.calledOnce(loggerMock.error); + await (testedModule as any).updateEnrollmentExclusion( + user, + experiment, + decisionPoint, + { + individualEnrollment: undefined, + individualExclusion: undefined, + groupEnrollment: undefined, + groupExclusion: undefined, + }, + { isUserExcluded: false, isGroupExcluded: false }, + [], + MARKED_DECISION_POINT_STATUS.CONDITION_APPLIED, + 'ConditionB', + undefined, + loggerMock + ); + + sinon.assert.calledOnce(testedModule.individualEnrollmentRepository.save); + const savedDoc = testedModule.individualEnrollmentRepository.save.getCall(0).args[0]; + expect(savedDoc.condition).toEqual(experiment.conditions[1]); + }); }); }); diff --git a/packages/backend/test/unit/services/ExperimentService.test.ts b/packages/backend/test/unit/services/ExperimentService.test.ts index dad3915099..0a9e833269 100644 --- a/packages/backend/test/unit/services/ExperimentService.test.ts +++ b/packages/backend/test/unit/services/ExperimentService.test.ts @@ -23,7 +23,6 @@ import { LevelRepository } from '../../../src/api/repositories/LevelRepository'; import { LevelCombinationElementRepository } from '../../../src/api/repositories/LevelCombinationElements'; import { ArchivedStatsRepository } from '../../../src/api/repositories/ArchivedStatsRepository'; import { StratificationFactorRepository } from '../../../src/api/repositories/StratificationFactorRepository'; -import { MoocletExperimentRefRepository } from '../../../src/api/repositories/MoocletExperimentRefRepository'; import { PreviewUserService } from '../../../src/api/services/PreviewUserService'; import { SegmentService } from '../../../src/api/services/SegmentService'; import { ExperimentPrecomputedSegmentService } from '../../../src/api/services/ExperimentPrecomputedSegmentService'; @@ -32,7 +31,6 @@ import { ErrorService } from '../../../src/api/services/ErrorService'; import { CacheService } from '../../../src/api/services/CacheService'; import { QueryService } from '../../../src/api/services/QueryService'; import { MetricService } from '../../../src/api/services/MetricService'; -import { MoocletRewardsService } from '../../../src/api/services/MoocletRewardsService'; import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; import { Experiment } from '../../../src/api/models/Experiment'; import { ExperimentCondition } from '../../../src/api/models/ExperimentCondition'; @@ -399,10 +397,6 @@ describe('ExperimentService Testing', () => { provide: getRepositoryToken(StratificationFactorRepository), useValue: {}, }, - { - provide: getRepositoryToken(MoocletExperimentRefRepository), - useValue: {}, - }, { provide: CacheService, useValue: { @@ -449,10 +443,6 @@ describe('ExperimentService Testing', () => { withRecompute: jest.fn(async (_logger: any, _resolve: any, work: any) => work()), }, }, - { - provide: MoocletRewardsService, - useValue: {}, - }, ], }).compile(); @@ -501,6 +491,43 @@ describe('ExperimentService Testing', () => { }); }); + describe('deduceConditions()', () => { + it('remaps thompsonSamplingConfig.priors onto the newly generated condition ids', () => { + const result: any = { + conditions: [ + { id: 'old-condition-1', levelCombinationElements: [] }, + { id: 'old-condition-2', levelCombinationElements: [] }, + ], + thompsonSamplingConfig: { + priors: { + 'old-condition-1': { success: 3, failure: 1 }, + 'old-condition-2': { success: 1, failure: 5 }, + }, + }, + }; + + service.deduceConditions(result); + + const [newCondition1, newCondition2] = result.conditions; + expect(newCondition1.id).not.toBe('old-condition-1'); + expect(newCondition2.id).not.toBe('old-condition-2'); + expect(result.thompsonSamplingConfig.priors).toEqual({ + [newCondition1.id]: { success: 3, failure: 1 }, + [newCondition2.id]: { success: 1, failure: 5 }, + }); + expect(result.thompsonSamplingConfig.priors['old-condition-1']).toBeUndefined(); + }); + + it('leaves experiments without a thompsonSamplingConfig unaffected', () => { + const result: any = { + conditions: [{ id: 'old-condition-1', levelCombinationElements: [] }], + }; + + expect(() => service.deduceConditions(result)).not.toThrow(); + expect(result.thompsonSamplingConfig).toBeUndefined(); + }); + }); + describe('update()', () => { it('should successfully update an experiment with basic changes', async () => { const result = await service.update(mockExperimentDTO, mockUser, logger); @@ -965,7 +992,7 @@ describe('ExperimentService Testing', () => { it('awaits exclusion-list attachment before returning, even with no inclusion lists, under a caller-owned transaction', async () => { // Regression: the only `await Promise.all(addListPromises)` used to sit inside the inclusion-list // branch, so an exclusion-only experiment created within a caller-owned transaction - // (existingEntityManager, e.g. MoocletExperimentService) returned with its addList insert still in + // (existingEntityManager) returned with its addList insert still in // flight — racing the caller's commit. create() must now await it unconditionally. let resolveAddList: () => void; const addListPending = new Promise((res) => { diff --git a/packages/backend/test/unit/services/ImportExportService.test.ts b/packages/backend/test/unit/services/ImportExportService.test.ts new file mode 100644 index 0000000000..ce9339360a --- /dev/null +++ b/packages/backend/test/unit/services/ImportExportService.test.ts @@ -0,0 +1,84 @@ +import { ImportExportService } from '../../../src/api/services/ImportExportService'; +import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; + +describe('ImportExportService', () => { + let experimentService: any; + let adaptiveExperimentConfigDispatcher: any; + let service: ImportExportService; + let logger: any; + + beforeEach(() => { + experimentService = { + create: jest.fn().mockImplementation((experiment) => Promise.resolve({ ...experiment })), + }; + adaptiveExperimentConfigDispatcher = { + createConfigIfApplicable: jest.fn().mockResolvedValue(undefined), + attachConfigToExperiment: jest.fn().mockImplementation((experiment) => Promise.resolve(experiment)), + }; + logger = { info: jest.fn(), error: jest.fn() }; + service = new ImportExportService({} as any, {} as any, experimentService, adaptiveExperimentConfigDispatcher); + }); + + describe('addBulkExperiments', () => { + it('creates a Thompson Sampling config for an imported/batch-created THOMPSON_SAMPLING experiment', async () => { + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 2, failure: 3 } } }, + } as any; + + await service.addBulkExperiments([experiment], {} as any, logger); + + // Without this call, an imported/batch-created THOMPSON_SAMPLING experiment previously got no + // ThompsonSamplingExperimentConfig/ConditionPosteriorState rows at all (only the single-experiment + // POST /experiments controller path wired this up), so assignment could never select a condition. + expect(adaptiveExperimentConfigDispatcher.createConfigIfApplicable).toHaveBeenCalledWith( + experiment, + expect.objectContaining({ id: 'experiment-1' }), + ['condition-1'] + ); + }); + + it('captures condition ids before create() runs, so a caller that regenerates them can still be remapped', async () => { + // ExperimentService.create() regenerates condition ids in place; a caller whose mock (or the + // real service) does this must not lose the pre-creation ids the imported/batch-created + // experiment's priors are keyed by -- addBulkExperiments has to snapshot them beforehand. + experimentService.create.mockImplementation((experiment: any) => + Promise.resolve({ + ...experiment, + conditions: experiment.conditions.map((condition: any) => ({ ...condition, id: `new-${condition.id}` })), + }) + ); + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1' }], + thompsonSamplingConfig: { priors: { 'condition-1': { success: 2, failure: 3 } } }, + } as any; + + await service.addBulkExperiments([experiment], {} as any, logger); + + expect(adaptiveExperimentConfigDispatcher.createConfigIfApplicable).toHaveBeenCalledWith( + experiment, + expect.objectContaining({ conditions: [{ id: 'new-condition-1' }] }), + ['condition-1'] + ); + }); + + it('attaches the created config onto the returned experiment', async () => { + const experiment = { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1' }], + } as any; + adaptiveExperimentConfigDispatcher.attachConfigToExperiment.mockImplementation((exp: any) => + Promise.resolve({ ...exp, thompsonSamplingConfig: { priors: { 'condition-1': { success: 1, failure: 1 } } } }) + ); + + const [result] = await service.addBulkExperiments([experiment], {} as any, logger); + + expect(result.thompsonSamplingConfig).toEqual({ priors: { 'condition-1': { success: 1, failure: 1 } } }); + }); + }); +}); diff --git a/packages/backend/test/unit/services/MoocletDataService.test.ts b/packages/backend/test/unit/services/MoocletDataService.test.ts deleted file mode 100644 index 83055080d0..0000000000 --- a/packages/backend/test/unit/services/MoocletDataService.test.ts +++ /dev/null @@ -1,571 +0,0 @@ -import { - MoocletBatchResponse, - MoocletPolicyParametersRequestBody, - MoocletPolicyParametersResponseDetails, - MoocletPolicyResponseDetails, - MoocletProxyRequestParams, - MoocletRequestBody, - MoocletResponseDetails, - MoocletVariableRequestBody, - MoocletVariableResponseDetails, - MoocletVersionRequestBody, - MoocletVersionResponseDetails, -} from '../../../src/types/Mooclet'; -import { MoocletDataService } from '../../../src/api/services/MoocletDataService'; -import { Container } from 'typedi'; -import axios from 'axios'; -import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; -import { MoocletTSConfigurablePolicyParametersDTO, ASSIGNMENT_ALGORITHM } from 'upgrade_types'; - -jest.mock('axios'); - -describe('#MoocletDataService', () => { - let moocletDataService: MoocletDataService; - const logger = { - error: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - } as unknown as UpgradeLogger; - - beforeAll(() => { - moocletDataService = Container.get(MoocletDataService); - }); - - afterAll(() => { - Container.reset(); - }); - - describe('#getMoocletIdByName', () => { - it('should return the correct Mooclet ID when the policy name matches', async () => { - const mockPoliciesList: MoocletBatchResponse = { - count: 2, - next: null, - previous: null, - results: [ - { id: 1, name: 'Policy1' }, - { id: 2, name: 'Policy2' }, - ], - }; - - jest.spyOn(moocletDataService, 'getPoliciesList').mockResolvedValue(mockPoliciesList); - - const moocletId = await moocletDataService.getMoocletIdByName('Policy2', logger); - expect(moocletId).toBe(2); - }); - - it('should return null when the policy name does not match', async () => { - const mockPoliciesList: MoocletBatchResponse = { - count: 2, - next: null, - previous: null, - results: [ - { id: 1, name: 'Policy1' }, - { id: 2, name: 'Policy2' }, - ], - }; - - jest.spyOn(moocletDataService, 'getPoliciesList').mockResolvedValue(mockPoliciesList); - - const moocletId = await moocletDataService.getMoocletIdByName('Policy3', logger); - expect(moocletId).toBeNull(); - }); - - it('should return null when the policies list is empty', async () => { - const mockPoliciesList: MoocletBatchResponse = { - count: 0, - next: null, - previous: null, - results: [], - }; - - jest.spyOn(moocletDataService, 'getPoliciesList').mockResolvedValue(mockPoliciesList); - - const moocletId = await moocletDataService.getMoocletIdByName('Policy1', logger); - expect(moocletId).toBeNull(); - }); - - it('should throw an error when getPoliciesList fails', async () => { - jest.spyOn(moocletDataService, 'getPoliciesList').mockRejectedValue(new Error('Failed to fetch policies')); - - await expect(moocletDataService.getMoocletIdByName('Policy1', logger)).rejects.toThrow( - 'Failed to fetch policies' - ); - }); - - afterAll(() => { - jest.restoreAllMocks(); - }); - }); - - describe('#getPoliciesList', () => { - it('should return the list of policies', async () => { - const mockPoliciesList: MoocletBatchResponse = { - count: 2, - next: null, - previous: null, - results: [ - { id: 1, name: 'Policy1' }, - { id: 2, name: 'Policy2' }, - ], - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockPoliciesList); - - const policiesList = await moocletDataService.getPoliciesList(logger); - expect(policiesList).toEqual(mockPoliciesList); - }); - - it('should throw an error when fetchExternalMoocletsData fails', async () => { - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to fetch policies')); - - await expect(moocletDataService.getPoliciesList(logger)).rejects.toThrow('Failed to fetch policies'); - }); - }); - - describe('#postNewMooclet', () => { - it('should return the new Mooclet details when the request is successful', async () => { - const mockRequestBody: MoocletRequestBody = { - name: 'New Mooclet', - policy: 3, - }; - - const mockResponseDetails: MoocletResponseDetails = { - id: 1, - name: 'New Mooclet', - policy: 3, - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponseDetails); - - const response = await moocletDataService.postNewMooclet(mockRequestBody, logger); - expect(response).toEqual(mockResponseDetails); - }); - - it('should throw an error when the request fails', async () => { - const mockRequestBody: MoocletRequestBody = { - name: 'New Mooclet', - policy: 3, - }; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to create mooclet')); - - await expect(moocletDataService.postNewMooclet(mockRequestBody, logger)).rejects.toThrow( - 'Failed to create mooclet' - ); - }); - }); - - describe('#deleteMooclet', () => { - it('should return the response when the delete request is successful', async () => { - const moocletId = 1; - const mockResponse = { success: true }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const response = await moocletDataService.deleteMooclet(moocletId, logger); - expect(response).toEqual(mockResponse); - }); - - it('should throw an error when the delete request fails', async () => { - const moocletId = 1; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to delete mooclet')); - - await expect(moocletDataService.deleteMooclet(moocletId, logger)).rejects.toThrow('Failed to delete mooclet'); - }); - }); - - describe('#postNewVersion', () => { - it('should return the new version details when the request is successful', async () => { - const mockRequestBody: MoocletVersionRequestBody = { - mooclet: 1, - name: 'Version 1', - text: 'This is version 1', - version_json: { key: 1 }, - }; - - const mockResponseDetails: MoocletVersionResponseDetails = { - id: 1, - mooclet: 1, - name: 'Version 1', - text: 'This is version 1', - version_json: { key: 1 }, - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponseDetails); - - const response = await moocletDataService.postNewVersion(mockRequestBody, logger); - expect(response).toEqual(mockResponseDetails); - }); - - it('should throw an error when the request fails', async () => { - const mockRequestBody: MoocletVersionRequestBody = { - mooclet: 1, - name: 'Version 1', - text: 'This is version 1', - version_json: { key: 1 }, - }; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to create version')); - - await expect(moocletDataService.postNewVersion(mockRequestBody, logger)).rejects.toThrow( - 'Failed to create version' - ); - }); - }); - - describe('#deleteVersion', () => { - it('should return the response when the delete request is successful', async () => { - const versionId = 1; - const mockResponse = { success: true }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const response = await moocletDataService.deleteVersion(versionId, logger); - expect(response).toEqual(mockResponse); - }); - - it('should throw an error when the delete request fails', async () => { - const versionId = 1; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to delete version')); - - await expect(moocletDataService.deleteVersion(versionId, logger)).rejects.toThrow('Failed to delete version'); - }); - }); - - describe('#postNewPolicyParameters', () => { - const mockPolicyParameters: MoocletTSConfigurablePolicyParametersDTO = { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - prior: { - 1: { failure: 1, success: 1 }, - 2: { failure: 1, success: 1 }, - }, - batch_size: 1, - max_rating: 1, - min_rating: 0, - uniform_threshold: 0, - tspostdiff_thresh: 0, - outcome_variable_name: 'example_reward_var', - }; - it('should return the new policy parameters details when the request is successful', async () => { - const mockRequestBody: MoocletPolicyParametersRequestBody = { - mooclet: 1, - policy: 2, - parameters: { - ...mockPolicyParameters, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }, - }; - - const mockResponseDetails: MoocletPolicyParametersResponseDetails = { - id: 1, - mooclet: 1, - policy: 2, - parameters: { - ...mockPolicyParameters, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }, - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponseDetails); - - const response = await moocletDataService.postNewPolicyParameters(mockRequestBody, logger); - expect(response).toEqual(mockResponseDetails); - }); - - it('should throw an error when the request fails', async () => { - const mockRequestBody: MoocletPolicyParametersRequestBody = { - mooclet: 1, - policy: 2, - parameters: { - ...mockPolicyParameters, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }, - }; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to create policy parameters')); - - await expect(moocletDataService.postNewPolicyParameters(mockRequestBody, logger)).rejects.toThrow( - 'Failed to create policy parameters' - ); - }); - }); - - describe('#deletePolicyParameters', () => { - it('should return the response when the delete request is successful', async () => { - const policyParametersId = 1; - const mockResponse = { success: true }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const response = await moocletDataService.deletePolicyParameters(policyParametersId, logger); - expect(response).toEqual(mockResponse); - }); - - it('should throw an error when the delete request fails', async () => { - const policyParametersId = 1; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to delete policy parameters')); - - await expect(moocletDataService.deletePolicyParameters(policyParametersId, logger)).rejects.toThrow( - 'Failed to delete policy parameters' - ); - }); - }); - - describe('#postNewVariable', () => { - it('should return the new variable details when the request is successful', async () => { - const mockRequestBody: MoocletVariableRequestBody = { - name: 'New Variable', - }; - - const mockResponseDetails: MoocletVariableResponseDetails = { - id: 1, - environment: null, - variable_id: null, - name: 'New Variable', - min_value: 0, - max_value: 1, - value_type: 'BIN', - sample_thres: 1, - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponseDetails); - - const response = await moocletDataService.postNewVariable(mockRequestBody, logger); - expect(response).toEqual(mockResponseDetails); - }); - - it('should throw an error when the request fails', async () => { - const mockRequestBody: MoocletVariableRequestBody = { - name: 'New Variable', - }; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to create variable')); - - await expect(moocletDataService.postNewVariable(mockRequestBody, logger)).rejects.toThrow( - 'Failed to create variable' - ); - }); - }); - - describe('#deleteVariable', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should return the response when the delete request is successful', async () => { - const variableId = 1; - const mockResponse = { success: true }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const response = await moocletDataService.deleteVariable(variableId, logger); - expect(response).toEqual(mockResponse); - }); - - it('should throw an error when the delete request fails', async () => { - const variableId = 1; - - jest - .spyOn(moocletDataService, 'fetchExternalMoocletsData') - .mockRejectedValue(new Error('Failed to delete variable')); - - await expect(moocletDataService.deleteVariable(variableId, logger)).rejects.toThrow('Failed to delete variable'); - }); - - afterAll(() => { - jest.restoreAllMocks(); - }); - }); - - describe('#fetchExternalMoocletsData', () => { - const mockRequest: MoocletRequestBody = { - name: 'New Mooclet', - policy: 3, - }; - it('should return the response data when the request is successful', async () => { - const mockRequestParams: MoocletProxyRequestParams = { - method: 'POST', - url: 'https://api.example.com/mooclet', - apiToken: 'test-token', - body: { ...mockRequest }, - }; - - const mockResponse = { - status: 200, - data: { success: true }, - }; - - (axios.request as jest.Mock).mockResolvedValue(mockResponse); - - const response = await moocletDataService.fetchExternalMoocletsData(mockRequestParams, logger); - expect(response).toEqual(mockResponse.data); - }); - - it('should throw a MoocletError when the request fails with a non-2xx status', async () => { - const mockRequestParams: MoocletProxyRequestParams = { - method: 'POST', - url: 'https://api.example.com/mooclet', - apiToken: 'test-token', - body: { ...mockRequest }, - }; - - const mockAxiosError = Object.assign(new Error('Request failed with status code 400'), { - response: { - status: 400, - data: { error: 'Bad Request' }, - }, - isAxiosError: true, - }); - - (axios.request as jest.Mock).mockRejectedValue(mockAxiosError); - (axios as any).isAxiosError.mockReturnValueOnce(true); - - await expect(moocletDataService.fetchExternalMoocletsData(mockRequestParams, logger)).rejects.toThrow( - 'Mooclet server returned non-2xx status: 400' - ); - }); - - it('should handle and log errors when the request fails', async () => { - const mockRequestParams: MoocletProxyRequestParams = { - method: 'POST', - url: 'https://api.example.com/mooclet', - apiToken: 'test-token', - body: { ...mockRequest }, - }; - - const mockError = new Error('Mock Network Error'); - const mockErrorMessage = { message: `Failed to communicate with Mooclet server` }; - - (axios.request as jest.Mock).mockRejectedValue(mockError); - logger.error = jest.fn().mockReturnValue(mockErrorMessage); - await expect(moocletDataService.fetchExternalMoocletsData(mockRequestParams, logger)).rejects.toThrow( - mockErrorMessage.message - ); - }); - }); - - describe('#getRewardsForExperiment', () => { - it('should successfully fetch rewards for an experiment', async () => { - const requestBody = { - moocletId: 456, - variableName: 'outcome_var', - }; - - const mockResponse = { - count: 5, - next: null, - previous: null, - results: [ - { id: '1', version: 100, value: 1.0, variable: 'outcome_var', learner: 'user-1', mooclet: 456 }, - { id: '2', version: 100, value: 0.0, variable: 'outcome_var', learner: 'user-2', mooclet: 456 }, - { id: '3', version: 200, value: 1.0, variable: 'outcome_var', learner: 'user-3', mooclet: 456 }, - ], - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const result = await moocletDataService.getRewardsForExperiment(requestBody, logger); - - expect(result).toEqual(mockResponse); - expect(moocletDataService.fetchExternalMoocletsData).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: expect.stringContaining('/value?mooclet=456&variable__name=outcome_var'), - }), - logger - ); - }); - - it('should construct correct query parameters with moocletId and variableName', async () => { - const requestBody = { - moocletId: 789, - variableName: 'test_variable', - }; - - const mockResponse = { - count: 0, - next: null, - previous: null, - results: [], - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - await moocletDataService.getRewardsForExperiment(requestBody, logger); - - expect(moocletDataService.fetchExternalMoocletsData).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: expect.stringContaining('mooclet=789'), - }), - logger - ); - expect(moocletDataService.fetchExternalMoocletsData).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: expect.stringContaining('variable__name=test_variable'), - }), - logger - ); - }); - - it('should handle paginated responses with next page', async () => { - const requestBody = { - moocletId: 456, - variableName: 'outcome_var', - }; - - const mockResponse = { - count: 100, - next: 'http://api.mooclet.com/next-page', - previous: null, - results: [{ id: '1', version: 100, value: 1.0 }], - }; - - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockResolvedValue(mockResponse); - - const result = await moocletDataService.getRewardsForExperiment(requestBody, logger); - - expect(result.next).toBe('http://api.mooclet.com/next-page'); - expect(result.count).toBe(100); - }); - - it('should throw error when fetchExternalMoocletsData fails', async () => { - const requestBody = { - moocletId: 456, - variableName: 'outcome_var', - }; - - const mockError = new Error('API connection failed'); - jest.spyOn(moocletDataService, 'fetchExternalMoocletsData').mockRejectedValue(mockError); - - await expect(moocletDataService.getRewardsForExperiment(requestBody, logger)).rejects.toThrow( - 'API connection failed' - ); - }); - - afterAll(() => { - jest.restoreAllMocks(); - }); - }); -}); diff --git a/packages/backend/test/unit/services/MoocletExperimentService.test.ts b/packages/backend/test/unit/services/MoocletExperimentService.test.ts deleted file mode 100644 index 1eb3a05de3..0000000000 --- a/packages/backend/test/unit/services/MoocletExperimentService.test.ts +++ /dev/null @@ -1,1855 +0,0 @@ -import 'reflect-metadata'; -import { ExperimentDTO } from '../../../src/api/DTO/ExperimentDTO'; -import { UserDTO } from '../../../src/api/DTO/UserDTO'; -import { MoocletExperimentRef } from '../../../src/api/models/MoocletExperimentRef'; -import { MoocletVersionConditionMap } from '../../../src/api/models/MoocletVersionConditionMap'; -import { Experiment } from '../../../src/api/models/Experiment'; -import { ExperimentCondition } from '../../../src/api/models/ExperimentCondition'; -import { ArchivedStatsRepository } from '../../../src/api/repositories/ArchivedStatsRepository'; -import { ConditionPayloadRepository } from '../../../src/api/repositories/ConditionPayloadRepository'; -import { DecisionPointRepository } from '../../../src/api/repositories/DecisionPointRepository'; -import { ExperimentAuditLogRepository } from '../../../src/api/repositories/ExperimentAuditLogRepository'; -import { SegmentRepository } from '../../../src/api/repositories/SegmentRepository'; -import { ExperimentConditionRepository } from '../../../src/api/repositories/ExperimentConditionRepository'; -import { ExperimentRepository } from '../../../src/api/repositories/ExperimentRepository'; -import { ExperimentSegmentExclusionRepository } from '../../../src/api/repositories/ExperimentSegmentExclusionRepository'; -import { ExperimentSegmentInclusionRepository } from '../../../src/api/repositories/ExperimentSegmentInclusionRepository'; -import { ExperimentUserRepository } from '../../../src/api/repositories/ExperimentUserRepository'; -import { FactorRepository } from '../../../src/api/repositories/FactorRepository'; -import { GroupExclusionRepository } from '../../../src/api/repositories/GroupExclusionRepository'; -import { IndividualExclusionRepository } from '../../../src/api/repositories/IndividualExclusionRepository'; -import { LevelCombinationElementRepository } from '../../../src/api/repositories/LevelCombinationElements'; -import { LevelRepository } from '../../../src/api/repositories/LevelRepository'; -import { MetricRepository } from '../../../src/api/repositories/MetricRepository'; -import { MonitoredDecisionPointRepository } from '../../../src/api/repositories/MonitoredDecisionPointRepository'; -import { MoocletExperimentRefRepository } from '../../../src/api/repositories/MoocletExperimentRefRepository'; -import { QueryRepository } from '../../../src/api/repositories/QueryRepository'; -import { StateTimeLogsRepository } from '../../../src/api/repositories/StateTimeLogsRepository'; -import { StratificationFactorRepository } from '../../../src/api/repositories/StratificationFactorRepository'; -import { CacheService } from '../../../src/api/services/CacheService'; -import { ErrorService } from '../../../src/api/services/ErrorService'; -import { MoocletDataService } from '../../../src/api/services/MoocletDataService'; -import { - MoocletExperimentService, - SyncCreateParams, - SyncEditParams, - SyncDeleteParams, -} from '../../../src/api/services/MoocletExperimentService'; -import { ExperimentPrecomputedSegmentService } from '../../../src/api/services/ExperimentPrecomputedSegmentService'; -import { PreviewUserService } from '../../../src/api/services/PreviewUserService'; -import { QueryService } from '../../../src/api/services/QueryService'; -import { SegmentService } from '../../../src/api/services/SegmentService'; -import { DataSource, EntityManager } from 'typeorm'; -import { - ASSIGNMENT_ALGORITHM, - ASSIGNMENT_UNIT, - CONSISTENCY_RULE, - EXPERIMENT_STATE, - EXPERIMENT_TYPE, - FILTER_MODE, - PAYLOAD_TYPE, - POST_EXPERIMENT_RULE, - SEGMENT_TYPE, -} from 'upgrade_types'; -import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; -import { MetricService } from '../../../src/api/services/MetricService'; -import { ExperimentSchedulerService } from '../../../src/api/services/ExperimentSchedulerService'; - -const mockDataSource = { - initialize: jest.fn(), - destroy: jest.fn(), - transaction: jest.fn((callback) => callback(mockDataSource.manager)), - manager: { - transaction: jest.fn(), - save: jest.fn(), - create: jest.fn(), - findOne: jest.fn(), - find: jest.fn(), - delete: jest.fn(), - }, - // Add other properties that your code might use -} as unknown as DataSource; - -jest.mock('typeorm', () => { - const originalModule = jest.requireActual('typeorm'); - return { - ...originalModule, - DataSource: jest.fn().mockImplementation(() => mockDataSource), - }; -}); - -jest.mock('../../../src/api/services/MoocletDataService'); -jest.mock('../../../src/api/repositories/ExperimentRepository'); -jest.mock('../../../src/api/repositories/ExperimentConditionRepository'); -jest.mock('../../../src/api/repositories/DecisionPointRepository'); -jest.mock('../../../src/api/repositories/ExperimentAuditLogRepository'); -jest.mock('../../../src/api/repositories/SegmentRepository'); -jest.mock('../../../src/api/repositories/IndividualExclusionRepository'); -jest.mock('../../../src/api/repositories/GroupExclusionRepository'); -jest.mock('../../../src/api/repositories/MonitoredDecisionPointRepository'); -jest.mock('../../../src/api/repositories/ExperimentUserRepository'); -jest.mock('../../../src/api/repositories/MetricRepository'); -jest.mock('../../../src/api/repositories/QueryRepository'); -jest.mock('../../../src/api/repositories/StateTimeLogsRepository'); -jest.mock('../../../src/api/repositories/ExperimentSegmentInclusionRepository'); -jest.mock('../../../src/api/repositories/ExperimentSegmentExclusionRepository'); -jest.mock('../../../src/api/repositories/ConditionPayloadRepository'); -jest.mock('../../../src/api/repositories/FactorRepository'); -jest.mock('../../../src/api/repositories/LevelRepository'); -jest.mock('../../../src/api/repositories/ArchivedStatsRepository'); -jest.mock('../../../src/api/repositories/StratificationFactorRepository'); -jest.mock('../../../src/api/repositories/MoocletExperimentRefRepository'); -jest.mock('../../../src/api/services/PreviewUserService'); -jest.mock('../../../src/api/services/SegmentService'); -jest.mock('../../../src/api/services/ScheduledJobService'); -jest.mock('../../../src/api/services/ErrorService'); -jest.mock('../../../src/api/services/CacheService'); -jest.mock('../../../src/api/services/QueryService'); -jest.mock('../../../src/api/services/MetricService'); - -jest.mock('../../../src/env', () => ({ - env: { - mooclets: { - enabled: true, - }, - aws: { - region: 'us-east-1', - accessKeyId: 'test', - secretAccessKey: 'test', - s3BucketName: 'test', - }, - app: { - version: '1.0.0-test', - }, - }, -})); - -const mockTSConfigMoocletPolicyParameters = { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - prior: { - success: 1, - failure: 1, - }, - batch_size: 1, - max_rating: 1, - min_rating: 0, - uniform_threshold: 0, - tspostdiff_thresh: 0, - outcome_variable_name: 'TS_CONFIG_TEST', -}; - -const moocletExperimentDataTSConfigurable = { - id: 'test-exp-123', - name: 'test', - description: '', - consistencyRule: CONSISTENCY_RULE.INDIVIDUAL, - assignmentUnit: ASSIGNMENT_UNIT.INDIVIDUAL, - type: EXPERIMENT_TYPE.SIMPLE, - context: ['mathstream'], - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - stratificationFactor: null, - tags: [], - conditions: [ - { - id: 'A', - conditionCode: 'question-hint-default', - assignmentWeight: 50, - description: null, - order: 1, - name: '', - }, - { - id: 'B', - conditionCode: 'question-hint-tutorbot', - assignmentWeight: 50, - description: null, - order: 2, - name: '', - }, - ], - conditionPayloads: [ - { - id: 'E', - payload: { - type: PAYLOAD_TYPE.STRING, - value: 'question-hint-default', - }, - parentCondition: 'A', - decisionPoint: 'C', - }, - { - id: 'F', - payload: { - type: PAYLOAD_TYPE.STRING, - value: 'question-hint-tutorbot', - }, - parentCondition: 'B', - decisionPoint: 'C', - }, - ], - partitions: [ - { - id: 'C', - site: 'lesson-stream', - target: 'question-hint', - description: '', - order: 1, - excludeIfReached: false, - }, - ], - experimentSegmentInclusion: [ - { - segment: { - individualForSegment: [], - groupForSegment: [ - { - type: 'All', - groupId: 'All', - }, - ], - subSegments: [], - type: SEGMENT_TYPE.PRIVATE, - }, - }, - ], - experimentSegmentExclusion: [ - { - segment: { - individualForSegment: [], - groupForSegment: [], - subSegments: [], - type: SEGMENT_TYPE.PRIVATE, - }, - }, - ], - filterMode: FILTER_MODE.EXCLUDE_ALL, - queries: [], - endOn: null, - enrollmentCompleteCondition: null, - startOn: null, - state: EXPERIMENT_STATE.ENROLLING, - postExperimentRule: POST_EXPERIMENT_RULE.CONTINUE, - revertTo: null, - moocletPolicyParameters: mockTSConfigMoocletPolicyParameters, -}; - -describe('#MoocletExperimentService', () => { - let moocletExperimentService: MoocletExperimentService; - let moocletDataService: MoocletDataService; - let experimentRepository: ExperimentRepository; - let experimentConditionRepository: ExperimentConditionRepository; - let decisionPointRepository: DecisionPointRepository; - let experimentAuditLogRepository: ExperimentAuditLogRepository; - let segmentRepository: SegmentRepository; - let individualExclusionRepository: IndividualExclusionRepository; - let groupExclusionRepository: GroupExclusionRepository; - let monitoredDecisionPointRepository: MonitoredDecisionPointRepository; - let experimentUserRepository: ExperimentUserRepository; - let metricRepository: MetricRepository; - let queryRepository: QueryRepository; - let stateTimeLogsRepository: StateTimeLogsRepository; - let experimentSegmentInclusionRepository: ExperimentSegmentInclusionRepository; - let experimentSegmentExclusionRepository: ExperimentSegmentExclusionRepository; - let conditionPayloadRepository: ConditionPayloadRepository; - let factorRepository: FactorRepository; - let levelRepository: LevelRepository; - let levelCombinationElementsRepository: LevelCombinationElementRepository; - let archivedStatsRepository: ArchivedStatsRepository; - let stratificationRepository: StratificationFactorRepository; - let moocletExperimentRefRepository: MoocletExperimentRefRepository; - let previewUserService: PreviewUserService; - let segmentService: SegmentService; - let experimentSchedulerService: ExperimentSchedulerService; - let errorService: ErrorService; - let cacheService: CacheService; - let queryService: QueryService; - let metricService: MetricService; - let experimentPrecomputedSegmentService: ExperimentPrecomputedSegmentService; - - beforeEach(() => { - moocletDataService = { - deleteMooclet: jest.fn(), - deletePolicyParameters: jest.fn(), - deleteVariable: jest.fn(), - getPolicyParameters: jest.fn(), - getVersionForNewLearner: jest.fn(), - getVariable: jest.fn(), - } as unknown as MoocletDataService; - - metricService = { - saveAllMetrics: jest.fn(), - delete: jest.fn(), - } as unknown as MetricService; - experimentPrecomputedSegmentService = { - recomputeForExperiment: jest.fn().mockResolvedValue(undefined), - scheduleRecomputeForExperiments: jest.fn(), - scheduleRecomputeForSegment: jest.fn(), - getAffectedExperimentIds: jest.fn().mockResolvedValue([]), - getPrecomputedSets: jest.fn().mockResolvedValue(new Map()), - withRecompute: jest.fn(async (_logger, _resolve, work) => work()), - } as unknown as ExperimentPrecomputedSegmentService; - - cacheService = { - delCache: jest.fn().mockResolvedValue(undefined), - } as unknown as CacheService; - - experimentRepository = { - findOneExperiment: jest.fn(), - findOne: jest.fn(), - save: jest.fn(), - } as unknown as ExperimentRepository; - - moocletExperimentRefRepository = { - findOne: jest.fn(), - delete: jest.fn().mockResolvedValue(undefined), - } as unknown as MoocletExperimentRefRepository; - - // Create service with mocked dependencies - moocletExperimentService = new MoocletExperimentService( - moocletDataService, - experimentRepository, - experimentConditionRepository, - decisionPointRepository, - experimentAuditLogRepository, - individualExclusionRepository, - groupExclusionRepository, - monitoredDecisionPointRepository, - experimentUserRepository, - metricRepository, - queryRepository, - stateTimeLogsRepository, - experimentSegmentInclusionRepository, - experimentSegmentExclusionRepository, - conditionPayloadRepository, - factorRepository, - levelRepository, - levelCombinationElementsRepository, - archivedStatsRepository, - stratificationRepository, - segmentRepository, - moocletExperimentRefRepository, - mockDataSource, - previewUserService, - segmentService, - experimentSchedulerService, - errorService, - cacheService, - queryService, - metricService, - experimentPrecomputedSegmentService - ); - }); - - const logger = { - error: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - } as unknown as UpgradeLogger; - - describe('#handleCreateMoocletTransaction', () => { - let manager: EntityManager; - let params: SyncCreateParams; - let mockExperimentResponse: ExperimentDTO; - let mockMoocletExperimentRefResponse: MoocletExperimentRef; - - beforeEach(() => { - mockExperimentResponse = { - id: 'exp-123', - moocletPolicyParameters: mockTSConfigMoocletPolicyParameters, - } as any as ExperimentDTO; - - mockMoocletExperimentRefResponse = { - id: 'moocletRef-123', - versionConditionMaps: [], - } as MoocletExperimentRef; - - manager = mockDataSource.manager as EntityManager; - params = { - // Note: experimentDTO should already be the created experiment - // since handleCreateMoocletTransaction is now called AFTER createUpgradeExperiment - experimentDTO: mockExperimentResponse, - currentUser: {} as UserDTO, - logger, - }; - - // Spy on class methods - jest - .spyOn(moocletExperimentService as any, 'orchestrateMoocletCreation') - .mockResolvedValue(mockMoocletExperimentRefResponse); - jest.spyOn(moocletExperimentService as any, 'saveMoocletExperimentRef').mockResolvedValue(undefined); - jest - .spyOn(moocletExperimentService as any, 'createAndSaveVersionConditionMapEntities') - .mockResolvedValue(undefined); - jest.spyOn(moocletExperimentService as any, 'orchestrateDeleteMoocletResources').mockResolvedValue(undefined); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should successfully create a mooclet experiment with all required resources', async () => { - const result = await moocletExperimentService['handleCreateMoocletTransaction'](manager, params); - - // Verify orchestrateMoocletCreation was called with the experiment returned from createUpgradeExperiment - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - mockExperimentResponse, - params.experimentDTO.moocletPolicyParameters, - logger - ); - - expect(moocletExperimentService['saveMoocletExperimentRef']).toHaveBeenCalledWith( - manager, - mockMoocletExperimentRefResponse - ); - - expect(moocletExperimentService['createAndSaveVersionConditionMapEntities']).toHaveBeenCalledWith( - manager, - mockMoocletExperimentRefResponse.id, - mockMoocletExperimentRefResponse.versionConditionMaps, - logger - ); - - expect(moocletExperimentService.orchestrateDeleteMoocletResources).not.toHaveBeenCalled(); - - // Verify the result - expect(result).toEqual({ - ...mockExperimentResponse, - moocletPolicyParameters: params.experimentDTO.moocletPolicyParameters, - }); - - // Verify orchestrateDeleteMoocletResources was not called - expect(moocletExperimentService['orchestrateDeleteMoocletResources']).not.toHaveBeenCalled(); - }); - - it('should use experimentDTO from params (not create new)', async () => { - // Verify that handleCreateMoocletTransaction uses the experimentDTO passed in params - // (it doesn't create a new experiment - that's done in syncCreate) - const result = await moocletExperimentService['handleCreateMoocletTransaction'](manager, params); - - // orchestrateMoocletCreation should be called with params.experimentDTO - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - params.experimentDTO, - params.experimentDTO.moocletPolicyParameters, - logger - ); - - expect(result.id).toBe(params.experimentDTO.id); - }); - - it('should handle failure during orchestrateMoocletCreation and throw error', async () => { - const error = new Error('Failed to create mooclet resources'); - jest.spyOn(moocletExperimentService as any, 'orchestrateMoocletCreation').mockRejectedValue(error); - - await expect(moocletExperimentService['handleCreateMoocletTransaction'](manager, params)).rejects.toThrow(error); - - // Verify subsequent methods were not called - expect(moocletExperimentService['saveMoocletExperimentRef']).not.toHaveBeenCalled(); - expect(moocletExperimentService['createAndSaveVersionConditionMapEntities']).not.toHaveBeenCalled(); - }); - - it('should handle failure during saveMoocletExperimentRef and cleanup resources', async () => { - const error = new Error('Failed to save mooclet ref'); - jest.spyOn(moocletExperimentService as any, 'saveMoocletExperimentRef').mockRejectedValue(error); - - await expect(moocletExperimentService['handleCreateMoocletTransaction'](manager, params)).rejects.toThrow(error); - - // Verify cleanup was called - expect(moocletExperimentService['orchestrateDeleteMoocletResources']).toHaveBeenCalledWith( - mockMoocletExperimentRefResponse, - logger - ); - }); - - it('should handle failure during createAndSaveVersionConditionMapEntities and cleanup resources', async () => { - const error = new Error('Failed to save version condition maps'); - jest.spyOn(moocletExperimentService as any, 'createAndSaveVersionConditionMapEntities').mockRejectedValue(error); - - await expect(moocletExperimentService['handleCreateMoocletTransaction'](manager, params)).rejects.toThrow(error); - - // Verify cleanup was called - expect(moocletExperimentService['orchestrateDeleteMoocletResources']).toHaveBeenCalledWith( - mockMoocletExperimentRefResponse, - logger - ); - }); - - it('should auto-generate outcome variable name for ts_configurable experiments', async () => { - // Create params for ts_configurable experiment with empty outcome_variable_name - const paramsWithTSConfigurable = { - ...params, - experimentDTO: { - ...params.experimentDTO, - name: 'Test Experiment', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: { - ...mockTSConfigMoocletPolicyParameters, - outcome_variable_name: '', // Empty outcome variable name - }, - }, - }; - - // Spy on the private generateUniqueOutcomeVariableName method - const generateNameSpy = jest - .spyOn(moocletExperimentService as any, 'generateUniqueOutcomeVariableName') - .mockReturnValue('test_experi_2026-03-13T00:00:00.000Z_REWARD_VARIABLE'); - - await moocletExperimentService['handleCreateMoocletTransaction'](manager, paramsWithTSConfigurable); - - // Verify that generateUniqueOutcomeVariableName was called - expect(generateNameSpy).toHaveBeenCalledWith('Test Experiment'); - - // Verify that orchestrateMoocletCreation was called with the generated name - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - paramsWithTSConfigurable.experimentDTO, - expect.objectContaining({ - outcome_variable_name: 'test_experi_2026-03-13T00:00:00.000Z_REWARD_VARIABLE', - }), - logger - ); - }); - - it('should always generate new outcome variable name even when one exists for ts_configurable', async () => { - // Create params for ts_configurable experiment with existing outcome_variable_name - const paramsWithExistingOutcomeName = { - ...params, - experimentDTO: { - ...params.experimentDTO, - name: 'Test Experiment', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: { - ...mockTSConfigMoocletPolicyParameters, - outcome_variable_name: 'existing_outcome_variable', - }, - }, - }; - - // Spy on the private generateUniqueOutcomeVariableName method - const generateNameSpy = jest - .spyOn(moocletExperimentService as any, 'generateUniqueOutcomeVariableName') - .mockReturnValue('test_experi_2026-03-13T00:00:00.000Z_REWARD_VARIABLE'); - - await moocletExperimentService['handleCreateMoocletTransaction'](manager, paramsWithExistingOutcomeName); - - // Verify that generateUniqueOutcomeVariableName was called (should always generate) - expect(generateNameSpy).toHaveBeenCalledWith('Test Experiment'); - - // Verify that orchestrateMoocletCreation was called with the NEW generated name (not existing) - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - paramsWithExistingOutcomeName.experimentDTO, - expect.objectContaining({ - outcome_variable_name: 'test_experi_2026-03-13T00:00:00.000Z_REWARD_VARIABLE', - }), - logger - ); - }); - - it('should not generate outcome variable name for non-ts_configurable experiments', async () => { - // Create params for random algorithm (non-ts_configurable) - const paramsWithRandomAlgorithm = { - ...params, - experimentDTO: { - ...params.experimentDTO, - name: 'Test Experiment', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - moocletPolicyParameters: { - // Generic policy parameters without outcome_variable_name - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }, - }, - }; - - // Spy on the private generateUniqueOutcomeVariableName method - const generateNameSpy = jest.spyOn(moocletExperimentService as any, 'generateUniqueOutcomeVariableName'); - - await moocletExperimentService['handleCreateMoocletTransaction'](manager, paramsWithRandomAlgorithm); - - // Verify that generateUniqueOutcomeVariableName was NOT called - expect(generateNameSpy).not.toHaveBeenCalled(); - - // Verify that orchestrateMoocletCreation was called with original parameters - expect(moocletExperimentService['orchestrateMoocletCreation']).toHaveBeenCalledWith( - paramsWithRandomAlgorithm.experimentDTO, - paramsWithRandomAlgorithm.experimentDTO.moocletPolicyParameters, - logger - ); - }); - }); - - describe('#orchestrateMoocletCreation', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should orchestrate mooclet creation successfully', async () => { - const mockMoocletPolicy = { id: 'mockMoocletPolicy123' }; - const mockMoocletResponse = { id: 'mockMoocletResponse' }; - const mockMoocletVersionsResponse = [{ id: 'mockMoocletVersionsResponse' }]; - const mockMoocletPolicyParametersResponse = { id: 'mockMoocletPolicyParametersResponse' }; - const mockMoocletVariableResponse = { id: 'mockMoocletVariableResponse' }; - - jest.spyOn(moocletExperimentService as any, 'getMoocletPolicy').mockResolvedValue(mockMoocletPolicy); - jest.spyOn(moocletExperimentService as any, 'createMooclet').mockResolvedValue(mockMoocletResponse); - jest - .spyOn(moocletExperimentService as any, 'createMoocletVersions') - .mockResolvedValue(mockMoocletVersionsResponse); - jest.spyOn(moocletExperimentService as any, 'createMoocletVersionConditionMaps').mockReturnValue([]); - jest - .spyOn(moocletExperimentService as any, 'createPolicyParameters') - .mockResolvedValue(mockMoocletPolicyParametersResponse); - jest - .spyOn(moocletExperimentService as any, 'createVariableIfNeeded') - .mockResolvedValue(mockMoocletVariableResponse); - jest.spyOn(moocletExperimentService as any, 'orchestrateDeleteMoocletResources').mockResolvedValue(undefined); - - const result = await moocletExperimentService.orchestrateMoocletCreation( - moocletExperimentDataTSConfigurable, - mockTSConfigMoocletPolicyParameters, - logger - ); - - expect(result).toBeDefined(); - expect(result?.experimentId).toBe(moocletExperimentDataTSConfigurable.id); - expect(result?.moocletId).toBe(mockMoocletResponse.id); - expect(result?.policyParametersId).toBe(mockMoocletPolicyParametersResponse.id); - expect(result?.variableId).toBe(mockMoocletVariableResponse.id); - expect(moocletExperimentService.orchestrateDeleteMoocletResources).not.toHaveBeenCalled(); - }); - - it('should handle errors and orchestrate deletion of mooclet resources', async () => { - const mockError = new Error('Test Error'); - - jest.spyOn(moocletExperimentService as any, 'getMoocletPolicy').mockRejectedValue(mockError); - jest.spyOn(moocletExperimentService as any, 'orchestrateDeleteMoocletResources').mockRejectedValue(mockError); - - await expect( - moocletExperimentService.orchestrateMoocletCreation( - moocletExperimentDataTSConfigurable, - mockTSConfigMoocletPolicyParameters, - logger - ) - ).rejects.toThrow(mockError); - expect(moocletExperimentService.orchestrateDeleteMoocletResources).toHaveBeenCalled(); - }); - }); - - describe('#orchestrateDeleteMoocletResources', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.moocletId = 1; - mockMoocletExperimentRef.policyParametersId = 2; - mockMoocletExperimentRef.variableId = 3; - mockMoocletExperimentRef.id = 'mockMoocletExperimentRef123'; - mockMoocletExperimentRef.versionConditionMaps = []; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should delete all mooclet resources successfully', async () => { - jest.spyOn(moocletDataService, 'deleteMooclet').mockResolvedValue(undefined); - jest.spyOn(moocletDataService, 'deletePolicyParameters').mockResolvedValue(undefined); - jest.spyOn(moocletDataService, 'deleteVariable').mockResolvedValue(undefined); - - await moocletExperimentService.orchestrateDeleteMoocletResources(mockMoocletExperimentRef, logger); - - expect(moocletDataService.deleteMooclet).toHaveBeenCalledWith(mockMoocletExperimentRef.moocletId, logger); - expect(moocletDataService.deletePolicyParameters).toHaveBeenCalledWith( - mockMoocletExperimentRef.policyParametersId, - logger - ); - expect(moocletDataService.deleteVariable).toHaveBeenCalledWith(mockMoocletExperimentRef.variableId, logger); - }); - it('should handle errors, log them, and return false', async () => { - const mockError = new Error('Failed to delete mooclet'); - jest.spyOn(moocletDataService, 'deleteMooclet').mockRejectedValue(mockError); - jest.spyOn(moocletExperimentService as any, 'deleteMoocletVersions').mockResolvedValue(undefined); - jest.spyOn(moocletDataService, 'deletePolicyParameters').mockResolvedValue(undefined); - jest.spyOn(moocletDataService, 'deleteVariable').mockResolvedValue(undefined); - - const result = await moocletExperimentService.orchestrateDeleteMoocletResources(mockMoocletExperimentRef, logger); - - expect(result).toBe(false); - expect(moocletDataService.deleteMooclet).toHaveBeenCalledWith(mockMoocletExperimentRef.moocletId, logger); - expect(logger.error).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Failed to delete Mooclet resources'), - error: mockError, - moocletExperimentRef: mockMoocletExperimentRef, - }) - ); - }); - }); - - describe('#handleDeleteMoocletTransaction', () => { - const mockExperiment = { - id: 'test-exp-123', - name: 'Test Experiment', - conditions: [], - }; - - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.moocletId = 1; - mockMoocletExperimentRef.policyParametersId = 2; - mockMoocletExperimentRef.variableId = 3; - mockMoocletExperimentRef.id = 'mockMoocletExperimentRef123'; - mockMoocletExperimentRef.experimentId = 'test-exp-123'; - mockMoocletExperimentRef.versionConditionMaps = []; - - const mockUser = { - id: 'user-123', - firstName: 'Test', - lastName: 'User', - email: 'test@test.com', - } as UserDTO; - - const mockDeleteParams = { - moocletExperimentRef: mockMoocletExperimentRef, - experimentId: 'test-exp-123', - currentUser: mockUser, - logger, - }; - - const mockManager = mockDataSource.manager as EntityManager; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should rollback upgrade experiment deletion when mooclet deletion fails', async () => { - // Mock the parent class delete method - const deleteSpy = jest - .spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'delete') - .mockResolvedValue(mockExperiment); - - // Mock orchestrateDeleteMoocletResources to fail (return false) - const orchestrateSpy = jest - .spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources') - .mockResolvedValue(false); - - // Call the private method directly - await expect( - (moocletExperimentService as any).handleDeleteMoocletTransaction(mockManager, mockDeleteParams) - ).rejects.toThrow(/Failed to delete Mooclet resources, aborting transaction to preserve upgrade experiment/); - - // Verify that upgrade deletion was attempted first - expect(deleteSpy).toHaveBeenCalledWith('test-exp-123', mockUser, { - existingEntityManager: mockManager, - }); - - // Verify that mooclet deletion was attempted - expect(orchestrateSpy).toHaveBeenCalledWith(mockMoocletExperimentRef, logger); - }); - - it('should successfully delete both resources when mooclet deletion succeeds', async () => { - // Mock the parent class delete method - const deleteSpy = jest - .spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'delete') - .mockResolvedValue(mockExperiment); - - // Mock orchestrateDeleteMoocletResources to succeed (return true) - const orchestrateSpy = jest - .spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources') - .mockResolvedValue(true); - - // Call the private method directly - const result = await (moocletExperimentService as any).handleDeleteMoocletTransaction( - mockManager, - mockDeleteParams - ); - - expect(result).toEqual(mockExperiment); - expect(deleteSpy).toHaveBeenCalledWith('test-exp-123', mockUser, { - existingEntityManager: mockManager, - }); - expect(orchestrateSpy).toHaveBeenCalledWith(mockMoocletExperimentRef, logger); - }); - - it('should throw error if upgrade experiment deletion fails (before mooclet deletion)', async () => { - const deleteError = new Error('Database error: cannot delete experiment'); - - // Mock the parent class delete method to fail - const deleteSpy = jest - .spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'delete') - .mockRejectedValue(deleteError); - - // Mock orchestrateDeleteMoocletResources (should not be called) - const orchestrateSpy = jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources'); - - // Call the private method directly - await expect( - (moocletExperimentService as any).handleDeleteMoocletTransaction(mockManager, mockDeleteParams) - ).rejects.toThrow(deleteError); - - // Verify upgrade deletion was attempted - expect(deleteSpy).toHaveBeenCalledWith('test-exp-123', mockUser, { - existingEntityManager: mockManager, - }); - - // Mooclet deletion should not be attempted if upgrade deletion fails - expect(orchestrateSpy).not.toHaveBeenCalled(); - }); - }); - - describe('#generateUniqueOutcomeVariableName', () => { - it('should generate a unique outcome variable name with correct format', () => { - // Mock Date to have predictable timestamp - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName('Test Experiment Name'); - - expect(result).toBe('test_exper_2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - - // Restore original Date - jest.restoreAllMocks(); - }); - - it('should handle long experiment names by truncating to 10 characters', () => { - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName( - 'This is a very long experiment name that should be truncated' - ); - - expect(result).toBe('this_is_a__2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - expect(result.substring(0, result.indexOf('_2026'))).toHaveLength(10); - - jest.restoreAllMocks(); - }); - - it('should sanitize special characters in experiment names', () => { - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName('Test@#$%Exp!'); - - expect(result).toBe('test_exp_2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - - jest.restoreAllMocks(); - }); - - it('should handle experiment names with leading and trailing underscores', () => { - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName('___Test___'); - - expect(result).toBe('test_2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - - jest.restoreAllMocks(); - }); - - it('should convert uppercase names to lowercase', () => { - const mockDate = new Date('2026-03-13T10:30:45.123Z'); - jest.spyOn(global, 'Date').mockImplementation(() => mockDate as any); - jest.spyOn(mockDate, 'toISOString').mockReturnValue('2026-03-13T10:30:45.123Z'); - - const result = (moocletExperimentService as any).generateUniqueOutcomeVariableName('UPPERCASE'); - - expect(result).toBe('uppercase_2026-03-13T10:30:45.123Z_REWARD_VARIABLE'); - - jest.restoreAllMocks(); - }); - }); - - describe('#handlePotentialMoocletAssignmentAlgorithmChange', () => { - const currentUser = { - id: 'user-123', - firstName: 'Test', - lastName: 'User', - email: 'test@example.com', - } as UserDTO; - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.id = 'mooclet-ref-123'; - mockMoocletExperimentRef.moocletId = 1; - mockMoocletExperimentRef.experimentId = 'test-exp-123'; - - beforeEach(() => { - jest.clearAllMocks(); - }); - - describe('Algorithm changes involving mooclets', () => { - it('should handle Mooclet A → Mooclet B (fetch, update+create, delete) in INACTIVE state', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - const updatedExperiment = { ...experiment, name: 'Updated' }; - - // Mock checkForMoocletAssignmentAlgorithmChange - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: true, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - // Mock getMoocletExperimentRefByUpgradeExperimentId (should be called to fetch old ref) - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - // Mock syncUpdateWithNewMoocletResources (should be called to update + create new) - jest - .spyOn(moocletExperimentService, 'syncUpdateWithMoocletAlgorithmTransition') - .mockResolvedValue(updatedExperiment); - - // Mock orchestrateDeleteMoocletResources (should be called AFTER update) - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources').mockResolvedValue(true); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - // Verify order of operations - expect(moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId).toHaveBeenCalledWith( - experiment.id - ); - expect(moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition).toHaveBeenCalledWith({ - experimentDTO: experiment, - currentUser, - logger, - moocletRefToDelete: mockMoocletExperimentRef, - }); - expect(moocletExperimentService.orchestrateDeleteMoocletResources).toHaveBeenCalledWith( - mockMoocletExperimentRef, - logger - ); - - // Verify order: fetch called before update, update called before delete - const fetchOrder = (moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId as jest.Mock).mock - .invocationCallOrder[0]; - const updateOrder = (moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition as jest.Mock).mock - .invocationCallOrder[0]; - const deleteOrder = (moocletExperimentService.orchestrateDeleteMoocletResources as jest.Mock).mock - .invocationCallOrder[0]; - expect(fetchOrder).toBeLessThan(updateOrder); - expect(updateOrder).toBeLessThan(deleteOrder); - - expect(result).toEqual(updatedExperiment); - }); - - it('should handle Mooclet → Non-mooclet (fetch, update, delete) in INACTIVE state', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - const updatedExperiment = { ...experiment }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - // Directly mock the update method (simpler than mocking all internal dependencies) - const updateSpy = jest.spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'update'); - updateSpy.mockResolvedValue(updatedExperiment); - - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources').mockResolvedValue(true); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - expect(moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId).toHaveBeenCalledWith( - experiment.id - ); - expect(updateSpy).toHaveBeenCalledWith(experiment, currentUser, logger, mockDataSource.manager); - expect(moocletExperimentService.orchestrateDeleteMoocletResources).toHaveBeenCalledWith( - mockMoocletExperimentRef, - logger - ); - - expect(result).toEqual(updatedExperiment); - - updateSpy.mockRestore(); - }); - - it('should handle Non-mooclet → Mooclet (no fetch, update+create, no delete) in INACTIVE state', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - const updatedExperiment = { ...experiment }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: false, - isNowMooclet: true, - oldAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - - jest - .spyOn(moocletExperimentService, 'syncUpdateWithMoocletAlgorithmTransition') - .mockResolvedValue(updatedExperiment); - - jest.spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId'); - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources'); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - // Verify no fetch since wasMooclet is false - expect(moocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId).not.toHaveBeenCalled(); - - expect(moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition).toHaveBeenCalledWith({ - experimentDTO: experiment, - currentUser, - logger, - moocletRefToDelete: undefined, - }); - - // Verify no delete since nothing to delete - expect(moocletExperimentService.orchestrateDeleteMoocletResources).not.toHaveBeenCalled(); - - expect(result).toEqual(updatedExperiment); - }); - - it('should throw error for algorithm change in non-INACTIVE state (ENROLLING)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.ENROLLING, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - await expect( - moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange(experiment, currentUser, logger) - ).rejects.toThrow(/INACTIVE state/); - }); - - it('should gracefully handle deletion failure (log but not throw)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - const updatedExperiment = { ...experiment }; - const deletionError = new Error('Mooclet API unavailable'); - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - // Directly mock the update method - const updateSpy = jest.spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'update'); - updateSpy.mockResolvedValue(updatedExperiment); - - // Mock deletion failure - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources').mockRejectedValue(deletionError); - - // Should not throw - deletion failure should be caught and logged - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - expect(result).toEqual(updatedExperiment); - expect(logger.error).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining('Failed to delete old mooclet resources'), - }) - ); - - updateSpy.mockRestore(); - }); - - it('should not delete if update fails (transaction safety)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.INACTIVE, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - const updateError = new Error('Database error'); - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - // Mock the update method to fail - const updateSpy = jest.spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'update'); - updateSpy.mockRejectedValue(updateError); - - jest.spyOn(moocletExperimentService, 'orchestrateDeleteMoocletResources'); - - await expect( - moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange(experiment, currentUser, logger) - ).rejects.toThrow(updateError); - - // Verify deletion was NOT called since update failed - expect(moocletExperimentService.orchestrateDeleteMoocletResources).not.toHaveBeenCalled(); - - updateSpy.mockRestore(); - }); - }); - - describe('Regular mooclet updates (no algorithm change)', () => { - it('should handle regular mooclet update in ENROLLING state (allowed)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.ENROLLING, - }; - const updatedExperiment = { ...experiment, name: 'Updated name' }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: false, - wasMooclet: true, - isNowMooclet: true, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - - jest.spyOn(moocletExperimentService, 'syncUpdate').mockResolvedValue(updatedExperiment); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - expect(moocletExperimentService.syncUpdate).toHaveBeenCalledWith({ - experimentDTO: experiment, - currentUser, - logger, - }); - expect(result).toEqual(updatedExperiment); - }); - }); - - describe('Non-mooclet updates', () => { - it('should throw error for non-mooclet algorithm changes in non-INACTIVE state', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.ENROLLING, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING, - }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: true, - wasMooclet: false, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - - await expect( - moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange(experiment, currentUser, logger) - ).rejects.toThrow(/INACTIVE state/); - }); - - it('should return null for non-mooclet experiments (let controller handle)', async () => { - const experiment = { - ...moocletExperimentDataTSConfigurable, - state: EXPERIMENT_STATE.ENROLLING, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - - jest.spyOn(moocletExperimentService, 'checkForMoocletAssignmentAlgorithmChange').mockResolvedValue({ - hasChanged: false, - wasMooclet: false, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - - const result = await moocletExperimentService.handlePotentialMoocletAssignmentAlgorithmChange( - experiment, - currentUser, - logger - ); - - expect(result).toBeNull(); - }); - }); - }); - - describe('#syncCreate', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should create experiment and mooclet resources in a transaction', async () => { - const experimentDTO = { ...moocletExperimentDataTSConfigurable } as ExperimentDTO; - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncCreateParams = { experimentDTO, currentUser, logger }; - - const createdExperiment = { ...experimentDTO, id: 'new-exp-id' }; - jest.spyOn(moocletExperimentService as any, 'createUpgradeExperiment').mockResolvedValue(createdExperiment); - jest - .spyOn(moocletExperimentService as any, 'handleCreateMoocletTransaction') - .mockResolvedValue(createdExperiment); - - const result = await moocletExperimentService.syncCreate(params); - - expect(result).toEqual(createdExperiment); - }); - }); - - describe('#syncUpdate', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should update experiment and mooclet resources in a transaction', async () => { - const experimentDTO = { ...moocletExperimentDataTSConfigurable } as ExperimentDTO; - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncEditParams = { experimentDTO, currentUser, logger }; - - const updatedExperiment = { ...experimentDTO, name: 'Updated' }; - jest.spyOn(moocletExperimentService as any, 'handleEditMoocletTransaction').mockResolvedValue(updatedExperiment); - - const result = await moocletExperimentService.syncUpdate(params); - - expect(result).toEqual(updatedExperiment); - }); - }); - - describe('#syncUpdateWithMoocletAlgorithmTransition', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should update experiment and create new mooclet resources when transitioning to mooclet', async () => { - const experimentDTO = { - ...moocletExperimentDataTSConfigurable, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - } as ExperimentDTO; - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncEditParams = { experimentDTO, currentUser, logger }; - - const updatedExperiment = { ...experimentDTO }; - jest.spyOn(moocletExperimentService as any, 'updateUpgradeExperiment').mockResolvedValue(updatedExperiment); - jest - .spyOn(moocletExperimentService as any, 'handleCreateMoocletTransaction') - .mockResolvedValue(updatedExperiment); - jest.spyOn(moocletExperimentService, 'isMoocletExperiment').mockReturnValue(true); - - const result = await moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition(params); - - expect(result).toEqual(updatedExperiment); - expect(moocletExperimentService['updateUpgradeExperiment']).toHaveBeenCalled(); - expect(moocletExperimentService['handleCreateMoocletTransaction']).toHaveBeenCalled(); - }); - - it('should update experiment and delete old mooclet ref when transitioning away from mooclet', async () => { - const mockMoocletRef = new MoocletExperimentRef(); - mockMoocletRef.id = 'ref-123'; - - const experimentDTO = { - ...moocletExperimentDataTSConfigurable, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - } as ExperimentDTO; - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncEditParams = { experimentDTO, currentUser, logger, moocletRefToDelete: mockMoocletRef }; - - const updatedExperiment = { ...experimentDTO }; - jest.spyOn(moocletExperimentService as any, 'updateUpgradeExperiment').mockResolvedValue(updatedExperiment); - jest.spyOn(moocletExperimentService, 'isMoocletExperiment').mockReturnValue(false); - jest.spyOn(moocletExperimentRefRepository, 'delete').mockResolvedValue(undefined); - - const result = await moocletExperimentService.syncUpdateWithMoocletAlgorithmTransition(params); - - expect(result).toEqual(updatedExperiment); - expect(moocletExperimentRefRepository.delete).toHaveBeenCalledWith(mockMoocletRef.id); - }); - }); - - describe('#syncDelete', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should delete experiment and mooclet resources in a transaction', async () => { - const mockMoocletRef = new MoocletExperimentRef(); - mockMoocletRef.id = 'ref-123'; - - const currentUser = { id: 'user-123' } as any as UserDTO; - const params: SyncDeleteParams = { - moocletExperimentRef: mockMoocletRef, - experimentId: 'exp-123', - currentUser, - logger, - }; - - const deletedExperiment = { id: 'exp-123' } as any as Experiment; - jest - .spyOn(moocletExperimentService as any, 'handleDeleteMoocletTransaction') - .mockResolvedValue(deletedExperiment); - - const result = await moocletExperimentService.syncDelete(params); - - expect(result).toEqual(deletedExperiment); - }); - }); - - describe('#isMoocletExperiment', () => { - it('should return true for mooclet algorithms', () => { - expect(moocletExperimentService.isMoocletExperiment(ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE)).toBe(true); - }); - - it('should return false for non-mooclet algorithms', () => { - expect(moocletExperimentService.isMoocletExperiment(ASSIGNMENT_ALGORITHM.RANDOM)).toBe(false); - expect(moocletExperimentService.isMoocletExperiment(ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING)).toBe(false); - }); - }); - - describe('#checkForMoocletAssignmentAlgorithmChange', () => { - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should detect algorithm change from mooclet to non-mooclet', async () => { - const oldExperiment = { - id: 'exp-123', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }; - const newExperiment = { - id: 'exp-123', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - } as ExperimentDTO; - - jest.spyOn(moocletExperimentService, 'findOne').mockResolvedValue(oldExperiment as any); - - const result = await moocletExperimentService.checkForMoocletAssignmentAlgorithmChange(newExperiment, logger); - - expect(result).toEqual({ - hasChanged: true, - wasMooclet: true, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - }); - }); - - it('should detect no change when algorithm stays the same', async () => { - const oldExperiment = { - id: 'exp-123', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }; - const newExperiment = { - id: 'exp-123', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - } as ExperimentDTO; - - jest.spyOn(moocletExperimentService, 'findOne').mockResolvedValue(oldExperiment as any); - - const result = await moocletExperimentService.checkForMoocletAssignmentAlgorithmChange(newExperiment, logger); - - expect(result).toEqual({ - hasChanged: false, - wasMooclet: false, - isNowMooclet: false, - oldAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - }); - - it('should throw error when experiment not found', async () => { - const newExperiment = { id: 'exp-123', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as ExperimentDTO; - - jest.spyOn(moocletExperimentService, 'findOne').mockResolvedValue(null); - - await expect( - moocletExperimentService.checkForMoocletAssignmentAlgorithmChange(newExperiment, logger) - ).rejects.toThrow(/Experiment unexpectedly not found/); - }); - }); - - describe('#attachPolicyParamsToExperimentDTO', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.policyParametersId = 123; - mockMoocletExperimentRef.versionConditionMaps = [ - { - moocletVersionId: 1, - experimentCondition: { conditionCode: 'control' } as any, - } as MoocletVersionConditionMap, - { - moocletVersionId: 2, - experimentCondition: { conditionCode: 'treatment' } as any, - } as MoocletVersionConditionMap, - ]; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should attach policy parameters to experiment DTO', async () => { - const experiment = { id: 'exp-123' } as ExperimentDTO; - const mockPolicyParams = { - parameters: { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - prior: { - '1': { success: 1, failure: 1 }, - '2': { success: 1, failure: 1 }, - }, - }, - }; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getPolicyParameters').mockResolvedValue(mockPolicyParams as any); - - const result = await moocletExperimentService.attachPolicyParamsToExperimentDTO(experiment, logger); - - expect(result.moocletPolicyParameters).toEqual(mockPolicyParams.parameters); - }); - - it('should transform current_posteriors from version IDs to condition codes', async () => { - const experiment = { id: 'exp-123' } as ExperimentDTO; - const mockPolicyParams = { - parameters: { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - current_posteriors: { - '1': { success: 5, failure: 2 }, - '2': { success: 3, failure: 4 }, - }, - }, - }; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getPolicyParameters').mockResolvedValue(mockPolicyParams as any); - - const result = await moocletExperimentService.attachPolicyParamsToExperimentDTO(experiment, logger); - - expect(result.moocletPolicyParameters['current_posteriors']).toEqual({ - control: { success: 5, failure: 2 }, - treatment: { success: 3, failure: 4 }, - }); - }); - - it('should throw error if policy parameters cannot be fetched', async () => { - const experiment = { id: 'exp-123' } as ExperimentDTO; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getPolicyParameters').mockRejectedValue(new Error('API error')); - - await expect(moocletExperimentService.attachPolicyParamsToExperimentDTO(experiment, logger)).rejects.toThrow( - /Failed to get Mooclet policy parameters/ - ); - }); - }); - - describe('#getConditionFromMoocletProxy', () => { - const mockExperiment = { id: 'exp-123' } as Experiment; - const mockUserId = 'user-456'; - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.moocletId = 1; - mockMoocletExperimentRef.versionConditionMaps = [ - { - moocletVersionId: 10, - experimentCondition: { id: 'cond-1', conditionCode: 'control' } as ExperimentCondition, - } as MoocletVersionConditionMap, - ]; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should get condition from mooclet proxy', async () => { - const mockVersionResponse = { id: 10, name: 'control' }; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getVersionForNewLearner').mockResolvedValue(mockVersionResponse as any); - - const result = await moocletExperimentService.getConditionFromMoocletProxy(mockExperiment, mockUserId, logger); - - expect(result).toEqual(mockMoocletExperimentRef.versionConditionMaps[0].experimentCondition); - expect(moocletDataService.getVersionForNewLearner).toHaveBeenCalledWith( - mockMoocletExperimentRef.moocletId, - mockUserId, - logger - ); - }); - - it('should throw MoocletError if moocletExperimentRef is not found', async () => { - jest.spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId').mockResolvedValue(undefined); - - await expect( - moocletExperimentService.getConditionFromMoocletProxy(mockExperiment, mockUserId, logger) - ).rejects.toThrow(`MoocletExperimentRef not found for experiment id ${mockExperiment.id}`); - }); - - it('should throw error if version not found in maps', async () => { - const mockVersionResponse = { id: 999, name: 'unknown' }; - - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - jest.spyOn(moocletDataService, 'getVersionForNewLearner').mockResolvedValue(mockVersionResponse as any); - - await expect( - moocletExperimentService.getConditionFromMoocletProxy(mockExperiment, mockUserId, logger) - ).rejects.toThrow(/Version ID not found in version condition maps/); - }); - }); - - describe('#handleEnrollCondition', () => { - const mockExperimentId = 'exp-123'; - const mockConditionCode = 'control'; - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.versionConditionMaps = [ - { - experimentCondition: { id: 'cond-1', conditionCode: 'control' } as ExperimentCondition, - } as MoocletVersionConditionMap, - ]; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should return experiment condition for valid condition code', async () => { - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - const result = await moocletExperimentService.handleEnrollCondition(mockExperimentId, mockConditionCode, logger); - - expect(result).toEqual(mockMoocletExperimentRef.versionConditionMaps[0].experimentCondition); - }); - - it('should throw error if mooclet experiment ref not found', async () => { - jest.spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId').mockResolvedValue(null); - - await expect( - moocletExperimentService.handleEnrollCondition(mockExperimentId, mockConditionCode, logger) - ).rejects.toThrow(/No MoocletExperimentRef found/); - }); - - it('should throw error if condition code not found in version maps', async () => { - jest - .spyOn(moocletExperimentService, 'getMoocletExperimentRefByUpgradeExperimentId') - .mockResolvedValue(mockMoocletExperimentRef); - - await expect( - moocletExperimentService.handleEnrollCondition(mockExperimentId, 'unknown-condition', logger) - ).rejects.toThrow(/No version found for condition/); - }); - }); - - describe('Detection methods', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.versionConditionMaps = [ - { - experimentConditionId: 'cond-1', - experimentCondition: { id: 'cond-1', conditionCode: 'control' } as any as ExperimentCondition, - } as any as MoocletVersionConditionMap, - ]; - - describe('#detectNewConditions', () => { - it('should detect new conditions', () => { - const experimentDTO = { - conditions: [ - { id: 'cond-1', conditionCode: 'control' }, - { id: 'cond-2', conditionCode: 'treatment' }, - ], - } as ExperimentDTO; - - const result = moocletExperimentService['detectNewConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toHaveLength(1); - expect(result[0].id).toBe('cond-2'); - }); - - it('should return null if no new conditions', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'control' }], - } as ExperimentDTO; - - const result = moocletExperimentService['detectNewConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toBeNull(); - }); - }); - - describe('#detectRemovedConditions', () => { - it('should detect removed conditions', () => { - const experimentDTO = { - conditions: [], - } as ExperimentDTO; - - const result = moocletExperimentService['detectRemovedConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toHaveLength(1); - expect(result[0].experimentConditionId).toBe('cond-1'); - }); - - it('should return null if no conditions removed', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'control' }], - } as ExperimentDTO; - - const result = moocletExperimentService['detectRemovedConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toBeNull(); - }); - }); - - describe('#detectModifiedConditions', () => { - it('should detect modified condition codes', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'modified_control' }], - } as ExperimentDTO; - - const result = moocletExperimentService['detectModifiedConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toHaveLength(1); - expect(result[0].experimentConditionId).toBe('cond-1'); - }); - - it('should return null if no conditions modified', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'control' }], - } as ExperimentDTO; - - const result = moocletExperimentService['detectModifiedConditions'](experimentDTO, mockMoocletExperimentRef); - - expect(result).toBeNull(); - }); - }); - - describe('#detectExperimentDesignChanges', () => { - it('should detect multiple changes', () => { - const experimentDTO = { - conditions: [ - { id: 'cond-1', conditionCode: 'modified_control' }, - { id: 'cond-2', conditionCode: 'treatment' }, - ], - } as any as ExperimentDTO; - - const result = moocletExperimentService['detectExperimentDesignChanges']( - experimentDTO, - mockMoocletExperimentRef - ); - - expect(result.addedConditions).toHaveLength(1); - expect(result.modifiedConditions).toHaveLength(1); - }); - - it('should return null if no changes detected', () => { - const experimentDTO = { - conditions: [{ id: 'cond-1', conditionCode: 'control' }], - } as any as ExperimentDTO; - - const result = moocletExperimentService['detectExperimentDesignChanges']( - experimentDTO, - mockMoocletExperimentRef - ); - - expect(result).toBeNull(); - }); - }); - }); - - describe('#handleEditMoocletTransaction', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.id = 'ref-123'; - mockMoocletExperimentRef.policyParametersId = 1; - mockMoocletExperimentRef.variableId = 2; - mockMoocletExperimentRef.versionConditionMaps = [ - { - moocletVersionId: 10, - experimentCondition: { conditionCode: 'control' } as any, - } as MoocletVersionConditionMap, - { - moocletVersionId: 20, - experimentCondition: { conditionCode: 'treatment' } as any, - } as MoocletVersionConditionMap, - ]; - - const mockExperiment = { - id: 'exp-123', - state: EXPERIMENT_STATE.INACTIVE, - conditions: [], - moocletPolicyParameters: mockTSConfigMoocletPolicyParameters, - } as any as ExperimentDTO; - - const mockCurrentExperiment = { - id: 'exp-123', - conditions: [], - } as any as Experiment; - - const mockCurrentUser = { id: 'user-123' } as any as UserDTO; - const manager = mockDataSource.manager as EntityManager; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should throw error for ineligible experiment state', async () => { - const ineligibleExperiment = { ...mockExperiment, state: EXPERIMENT_STATE.CANCELLED }; - const params = { experimentDTO: ineligibleExperiment, currentUser: mockCurrentUser, logger }; - - await expect((moocletExperimentService as any).handleEditMoocletTransaction(manager, params)).rejects.toThrow( - /Ineligible experiment state/ - ); - }); - - it('should update experiment without version/variable edits when no changes detected', async () => { - const params = { experimentDTO: mockExperiment, currentUser: mockCurrentUser, logger }; - const updatedExperiment = { ...mockExperiment }; - - jest.spyOn(moocletExperimentService as any, 'fetchCurrentResources').mockResolvedValue({ - currentMoocletExperimentRef: mockMoocletExperimentRef, - currentPolicyParametersResponse: { parameters: mockExperiment.moocletPolicyParameters }, - currentExperiment: mockCurrentExperiment, - }); - - jest.spyOn(moocletExperimentService as any, 'detectExperimentDesignChanges').mockReturnValue(null); - - const updateSpy = jest.spyOn(Object.getPrototypeOf(Object.getPrototypeOf(moocletExperimentService)), 'update'); - updateSpy.mockResolvedValue(updatedExperiment); - - jest.spyOn(moocletExperimentService as any, 'doRevertablePolicyParameterChange').mockResolvedValue({ - parameters: { - ...mockExperiment.moocletPolicyParameters, - prior: { - '10': { success: 1, failure: 1 }, - '20': { success: 1, failure: 1 }, - }, - }, - }); - - const result = await (moocletExperimentService as any).handleEditMoocletTransaction(manager, params); - - expect(result).toEqual(updatedExperiment); - expect(updateSpy).toHaveBeenCalled(); - - updateSpy.mockRestore(); - }); - - it('should throw error when version/variable edits attempted on enrolling experiment', async () => { - const enrollingExperiment = { - ...mockExperiment, - state: EXPERIMENT_STATE.RUNNING, - }; - const params = { experimentDTO: enrollingExperiment, currentUser: mockCurrentUser, logger }; - - jest.spyOn(moocletExperimentService as any, 'fetchCurrentResources').mockResolvedValue({ - currentMoocletExperimentRef: mockMoocletExperimentRef, - currentPolicyParametersResponse: { parameters: mockExperiment.moocletPolicyParameters }, - currentExperiment: mockCurrentExperiment, - }); - - jest.spyOn(moocletExperimentService as any, 'detectExperimentDesignChanges').mockReturnValue({ - addedConditions: [{ id: 'new-cond', conditionCode: 'new-condition' }], - removedConditions: null, - modifiedConditions: null, - }); - - await expect((moocletExperimentService as any).handleEditMoocletTransaction(manager, params)).rejects.toThrow( - /Ineligible version edits detected for an active Mooclet experiment/ - ); - }); - - it('should rollback on error', async () => { - const params = { experimentDTO: mockExperiment, currentUser: mockCurrentUser, logger }; - const mockError = new Error('Update failed'); - - jest.spyOn(moocletExperimentService as any, 'fetchCurrentResources').mockRejectedValue(mockError); - jest.spyOn(moocletExperimentService as any, 'rollbackMoocletEdits').mockResolvedValue(undefined); - - await expect((moocletExperimentService as any).handleEditMoocletTransaction(manager, params)).rejects.toThrow( - mockError - ); - - expect(moocletExperimentService['rollbackMoocletEdits']).toHaveBeenCalled(); - }); - }); - - describe('#rollbackMoocletEdits', () => { - const mockMoocletExperimentRef = new MoocletExperimentRef(); - mockMoocletExperimentRef.id = 'ref-123'; - - const mockCurrentExperiment = { - id: 'exp-123', - conditions: [{ id: 'cond-1', conditionCode: 'old_code' }], - } as any as Experiment; - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should rollback policy parameters', async () => { - const rollbackRef = { - revertPolicyParameters: mockTSConfigMoocletPolicyParameters, - restoreVersions: null, - revertVersionModifications: null, - removeVersions: null, - currentMoocletExperimentRef: mockMoocletExperimentRef, - currentExperiment: mockCurrentExperiment, - }; - - jest.spyOn(moocletExperimentService as any, 'handleUpdatePolicyParameters').mockResolvedValue({}); - - await (moocletExperimentService as any).rollbackMoocletEdits(rollbackRef, logger); - - expect(moocletExperimentService['handleUpdatePolicyParameters']).toHaveBeenCalledWith( - mockTSConfigMoocletPolicyParameters, - mockMoocletExperimentRef, - logger - ); - }); - - it('should throw error if rollback itself fails', async () => { - const rollbackRef = { - revertPolicyParameters: mockTSConfigMoocletPolicyParameters, - restoreVersions: null, - revertVersionModifications: null, - removeVersions: null, - currentMoocletExperimentRef: mockMoocletExperimentRef, - currentExperiment: mockCurrentExperiment, - }; - - const rollbackError = new Error('Rollback failed'); - jest.spyOn(moocletExperimentService as any, 'handleUpdatePolicyParameters').mockRejectedValue(rollbackError); - - await expect((moocletExperimentService as any).rollbackMoocletEdits(rollbackRef, logger)).rejects.toThrow( - /Error during rollback/ - ); - }); - }); -}); diff --git a/packages/backend/test/unit/services/MoocletRewardsService.test.ts b/packages/backend/test/unit/services/MoocletRewardsService.test.ts deleted file mode 100644 index 0fb7106934..0000000000 --- a/packages/backend/test/unit/services/MoocletRewardsService.test.ts +++ /dev/null @@ -1,1345 +0,0 @@ -import { MoocletRewardsService } from '../../../src/api/services/MoocletRewardsService'; -import { MoocletExperimentRefRepository } from '../../../src/api/repositories/MoocletExperimentRefRepository'; -import { IndividualEnrollmentRepository } from '../../../src/api/repositories/IndividualEnrollmentRepository'; -import { MoocletDataService } from '../../../src/api/services/MoocletDataService'; -import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; -import { MoocletExperimentRef } from '../../../src/api/models/MoocletExperimentRef'; -import { IndividualEnrollment } from '../../../src/api/models/IndividualEnrollment'; -import { Experiment } from '../../../src/api/models/Experiment'; -import { EXPERIMENT_STATE } from 'upgrade_types'; -import { BinaryRewardAllowedValue } from 'upgrade_types'; -import { RewardValidator } from '../../../src/api/controllers/validators/RewardValidator'; -import { RequestedExperimentUser } from '../../../src/api/controllers/validators/ExperimentUserValidator'; -import { HttpError } from 'routing-controllers'; -import { configureLogger } from '../../utils/logger'; -import { MoocletExperimentService } from 'src/api/services/MoocletExperimentService'; - -describe('MoocletRewardsService', () => { - let service: MoocletRewardsService; - let mockLogger: jest.Mocked; - let mockMoocletDataService: jest.Mocked; - let mockMoocletExperimentRefRepository: jest.Mocked; - let mockIndividualEnrollmentRepository: jest.Mocked; - let mockMoocletExperimentService: jest.Mocked; - - const mockUser = { - id: 'user-123', - group: {}, - workingGroup: {}, - } as RequestedExperimentUser; - - const mockExperiment: Partial = { - id: 'experiment-123', - state: EXPERIMENT_STATE.ENROLLING, - }; - - const mockMoocletExperimentRef: Partial = { - id: 'ref-123', - experimentId: 'experiment-123', - moocletId: 456, - outcomeVariableName: 'reward_variable', - experiment: mockExperiment as Experiment, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - } as any, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - } as any, - ], - }; - - const mockEnrollment: Partial = { - id: 'enrollment-123', - userId: 'user-123', - experimentId: 'experiment-123', - conditionId: 'condition-1', - }; - - beforeAll(() => { - configureLogger(); - }); - - beforeEach(() => { - mockLogger = { - info: jest.fn(), - error: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - } as any; - - mockMoocletDataService = { - postNewReward: jest.fn(), - } as any; - - mockMoocletExperimentRefRepository = { - findOne: jest.fn(), - findActivelyEnrollingMoocletExperimentsByContextSiteTarget: jest.fn(), - } as any; - - mockIndividualEnrollmentRepository = { - findEnrollments: jest.fn(), - } as any; - - mockMoocletExperimentService = { - getMoocletExperimentRefByUpgradeExperimentId: jest.fn(), - } as any; - - service = new MoocletRewardsService( - mockMoocletExperimentRefRepository as any, - mockIndividualEnrollmentRepository as any, - mockMoocletDataService as any, - mockMoocletExperimentService as any - ); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('sendReward', () => { - describe('successful reward sending', () => { - it('should successfully send reward when all criteria are met using experimentId', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.message).toBe('Reward sent to mooclet successfully.'); - expect(result.reward).toEqual({ - variable: 'reward_variable', - value: 1, - mooclet: 456, - version: 100, - learner: 'user-123', - }); - expect(mockMoocletDataService.postNewReward).toHaveBeenCalledTimes(1); - expect(mockLogger.info).toHaveBeenCalledTimes(1); - }); - - it('should successfully send reward using decision point when experimentId not provided', async () => { - const request: RewardValidator = { - experimentId: undefined, - rewardValue: BinaryRewardAllowedValue.FAILURE, - context: 'home', - decisionPoint: { - site: 'site-1', - target: 'target-1', - }, - }; - - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget.mockResolvedValue( - [mockMoocletExperimentRef as MoocletExperimentRef] - ); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.message).toBe('Reward sent to mooclet successfully.'); - expect(result.reward.value).toBe(0); // FAILURE maps to 0 - expect(mockMoocletDataService.postNewReward).toHaveBeenCalledTimes(1); - }); - - it('should call postNewReward without awaiting (fire-and-forget)', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - await service.sendReward(mockUser, request, mockLogger as any); - - // Verify postNewReward was called but not awaited - expect(mockMoocletDataService.postNewReward).toHaveBeenCalledTimes(1); - }); - - it('should map SUCCESS reward value to 1', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.value).toBe(1); - }); - - it('should map FAILURE reward value to 0', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.FAILURE, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.value).toBe(0); - }); - }); - - describe('mooclet experiment ref validation', () => { - it('should throw 409 when no mooclet ref found by experimentId', async () => { - const request: RewardValidator = { - experimentId: 'nonexistent-experiment', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(null); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('No active mooclet experiment ref found'); - } - }); - - it('should throw 409 when no mooclet ref found by decision point', async () => { - const request: RewardValidator = { - experimentId: undefined, - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: 'home', - decisionPoint: { - site: 'site-1', - target: 'target-1', - }, - }; - - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget.mockResolvedValue( - [] - ); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('No active experiment found for decision point'); - } - }); - - it('should throw 409 when multiple mooclet refs found by decision point', async () => { - const request: RewardValidator = { - experimentId: undefined, - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: 'home', - decisionPoint: { - site: 'site-1', - target: 'target-1', - }, - }; - - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget.mockResolvedValue( - [ - mockMoocletExperimentRef as MoocletExperimentRef, - { ...mockMoocletExperimentRef, id: 'ref-456' } as MoocletExperimentRef, - ] - ); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('Multiple active experiments found for decision point'); - } - }); - - it('should throw 409 when experiment is not in ENROLLING state', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const nonEnrollingRef = { - ...mockMoocletExperimentRef, - experiment: { - ...mockExperiment, - state: EXPERIMENT_STATE.ENROLLMENT_COMPLETE, - }, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(nonEnrollingRef as MoocletExperimentRef); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('not actively enrolling'); - } - }); - }); - - describe('user enrollment validation', () => { - it('should throw 409 when no enrollment found for user', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as Error).message).toContain('Could not find unique user enrollment'); - } - }); - - it('should throw 409 when multiple enrollments found for user', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - mockEnrollment as IndividualEnrollment, - { ...mockEnrollment, id: 'enrollment-456' } as IndividualEnrollment, - ]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('Could not find unique user enrollment'); - } - }); - - it('should succeed when exactly one enrollment found', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.message).toBe('Reward sent to mooclet successfully.'); - }); - }); - - describe('version mapping validation', () => { - it('should throw 409 when no version mapping found for enrolled condition', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const enrollmentWithUnmappedCondition = { - ...mockEnrollment, - conditionId: 'unmapped-condition', - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - enrollmentWithUnmappedCondition as IndividualEnrollment, - ]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('Version-condition mapping not found'); - } - }); - - it('should find correct versionId for enrolled condition', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const enrollmentCondition2 = { - ...mockEnrollment, - conditionId: 'condition-2', - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - enrollmentCondition2 as IndividualEnrollment, - ]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.version).toBe(200); // version for condition-2 - }); - }); - - describe('error handling', () => { - it('should wrap unexpected errors as 409 HttpError', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockRejectedValue(new Error('Database connection failed')); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - expect((error as HttpError).message).toContain('Failed to process reward request due to unexpected error'); - } - }); - - it('should re-throw HttpErrors without wrapping', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const originalError = new HttpError(404, 'Experiment not found'); - mockMoocletExperimentRefRepository.findOne.mockRejectedValue(originalError); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(originalError); - }); - - it('should log errors with appropriate context', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(null); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect(mockLogger.error).toHaveBeenCalledTimes(1); - expect(mockLogger.error).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.any(String), - request: expect.any(Object), - }) - ); - } - }); - }); - - describe('reward payload construction', () => { - it('should construct reward with all required fields', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward).toHaveProperty('variable'); - expect(result.reward).toHaveProperty('value'); - expect(result.reward).toHaveProperty('mooclet'); - expect(result.reward).toHaveProperty('version'); - expect(result.reward).toHaveProperty('learner'); - }); - - it('should use outcomeVariableName from mooclet ref', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.variable).toBe('reward_variable'); - }); - - it('should use moocletId from mooclet ref', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.mooclet).toBe(456); - }); - - it('should use user id as learner', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.learner).toBe('user-123'); - }); - }); - }); - - describe('getRewardsSummaryForExperiment', () => { - const mockMoocletRewardsResponse = { - count: 10, - next: null, - previous: null, - results: [ - { id: '1', version: 100, value: 1.0, variable: 'reward_variable', learner: 'user-1', mooclet: 456 }, - { id: '2', version: 100, value: 0.0, variable: 'reward_variable', learner: 'user-2', mooclet: 456 }, - { id: '3', version: 100, value: 1.0, variable: 'reward_variable', learner: 'user-3', mooclet: 456 }, - { id: '4', version: 200, value: 1.0, variable: 'reward_variable', learner: 'user-4', mooclet: 456 }, - { id: '5', version: 200, value: 0.0, variable: 'reward_variable', learner: 'user-5', mooclet: 456 }, - ], - }; - - beforeEach(() => { - mockMoocletDataService.getRewardsForExperiment = jest.fn(); - }); - - it('should successfully fetch and return rewards summary', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue(mockMoocletRewardsResponse as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(result).toHaveLength(2); - expect(result[0]).toEqual({ - conditionCode: 'Control', - successes: 2, - failures: 1, - successRate: '66.7%', - order: 0, - priorSuccess: 1, - priorFailure: 1, - }); - expect(result[1]).toEqual({ - conditionCode: 'Treatment', - successes: 1, - failures: 1, - successRate: '50.0%', - order: 1, - priorSuccess: 1, - priorFailure: 1, - }); - }); - - it('should handle experiment with no rewards', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue({ - count: 0, - next: null, - previous: null, - results: [], - } as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - conditionCode: 'Control', - successes: 0, - failures: 0, - successRate: '0.0%', - order: 0, - priorSuccess: 1, - priorFailure: 1, - }); - }); - - it('should fetch all pages and accumulate results when response has multiple pages', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - const page1Response = { - count: 6, - next: 'http://mooclet-api/value?page=2', - previous: null, - results: [ - { id: '1', version: 100, value: 1.0, variable: 'reward_variable', learner: 'user-1', mooclet: 456 }, - { id: '2', version: 100, value: 0.0, variable: 'reward_variable', learner: 'user-2', mooclet: 456 }, - { id: '3', version: 200, value: 1.0, variable: 'reward_variable', learner: 'user-3', mooclet: 456 }, - ], - }; - - const page2Response = { - count: 6, - next: null, - previous: 'http://mooclet-api/value?page=1', - results: [ - { id: '4', version: 100, value: 1.0, variable: 'reward_variable', learner: 'user-4', mooclet: 456 }, - { id: '5', version: 200, value: 0.0, variable: 'reward_variable', learner: 'user-5', mooclet: 456 }, - { id: '6', version: 200, value: 1.0, variable: 'reward_variable', learner: 'user-6', mooclet: 456 }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment - .mockResolvedValueOnce(page1Response as any) - .mockResolvedValueOnce(page2Response as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenCalledTimes(2); - // Control (version 100): page1 has id1 (success) + id2 (failure), page2 has id4 (success) => 2 successes, 1 failure - expect(result[0]).toEqual({ - conditionCode: 'Control', - successes: 2, - failures: 1, - successRate: '66.7%', - order: 0, - priorSuccess: 1, - priorFailure: 1, - }); - // Treatment (version 200): page1 has id3 (success), page2 has id5 (failure) + id6 (success) => 2 successes, 1 failure - expect(result[1]).toEqual({ - conditionCode: 'Treatment', - successes: 2, - failures: 1, - successRate: '66.7%', - order: 1, - priorSuccess: 1, - priorFailure: 1, - }); - }); - - it('should pass next page URL to subsequent fetches', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - const nextPageUrl = 'http://mooclet-api/value?page=2'; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment - .mockResolvedValueOnce({ count: 2, next: nextPageUrl, previous: null, results: [] } as any) - .mockResolvedValueOnce({ count: 2, next: null, previous: null, results: [] } as any); - - await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenNthCalledWith( - 2, - expect.anything(), - expect.anything(), - nextPageUrl - ); - }); - - it('should log info for each additional page fetched', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment - .mockResolvedValueOnce({ - count: 4, - next: 'http://mooclet-api/value?page=2', - previous: null, - results: [{ id: '1', version: 100, value: 1.0 }], - } as any) - .mockResolvedValueOnce({ count: 4, next: null, previous: null, results: [] } as any); - - await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - // Once for initial fetch, once for the paginated fetch - expect(mockLogger.info).toHaveBeenCalledTimes(2); - expect(mockLogger.info).toHaveBeenCalledWith( - expect.objectContaining({ - message: expect.stringContaining("But wait there's more"), - }) - ); - }); - - it('should handle three or more pages of results', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment - .mockResolvedValueOnce({ - count: 3, - next: 'http://mooclet-api/value?page=2', - previous: null, - results: [{ id: '1', version: 100, value: 1.0 }], - } as any) - .mockResolvedValueOnce({ - count: 3, - next: 'http://mooclet-api/value?page=3', - previous: null, - results: [{ id: '2', version: 100, value: 0.0 }], - } as any) - .mockResolvedValueOnce({ - count: 3, - next: null, - previous: null, - results: [{ id: '3', version: 100, value: 1.0 }], - } as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenCalledTimes(3); - expect(result[0].successes).toBe(2); - expect(result[0].failures).toBe(1); - expect(result[0].successes + result[0].failures).toBe(3); - }); - - it('should log error and re-throw when mooclet service fails', async () => { - const error = new Error('Mooclet API error'); - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockRejectedValue(error); - - await expect(service.getRewardsSummaryForExperiment('experiment-123', mockLogger)).rejects.toThrow(error); - - expect(mockLogger.error).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Error fetching rewards summary for experiment', - experimentId: 'experiment-123', - error, - }) - ); - }); - - it('should sort results by condition order', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - mockMoocletExperimentService.getMoocletExperimentRefByUpgradeExperimentId.mockResolvedValue( - refWithConditions as any - ); - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue(mockMoocletRewardsResponse as any); - - const result = await service.getRewardsSummaryForExperiment('experiment-123', mockLogger); - - expect(result[0].conditionCode).toBe('Control'); - expect(result[0].order).toBe(0); - expect(result[1].conditionCode).toBe('Treatment'); - expect(result[1].order).toBe(1); - }); - }); - - describe('fetchRewardsForExperiment', () => { - beforeEach(() => { - mockMoocletDataService.getRewardsForExperiment = jest.fn(); - }); - - it('should call mooclet data service with correct parameters', async () => { - const moocletRef = { - moocletId: 456, - outcomeVariableName: 'test_variable', - } as MoocletExperimentRef; - - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue({ - count: 5, - next: null, - previous: null, - results: [], - } as any); - - await service.fetchRewardsForExperiment(moocletRef, mockLogger as any); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenCalledWith( - { - moocletId: 456, - variableName: 'test_variable', - }, - mockLogger, - undefined - ); - }); - - it('should return paginated response from mooclet data service', async () => { - const moocletRef = { - moocletId: 456, - outcomeVariableName: 'test_variable', - } as MoocletExperimentRef; - - const expectedResponse = { - count: 3, - next: 'http://next-page', - previous: null, - results: [{ id: '1', value: 1.0 }], - }; - - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue(expectedResponse as any); - - const result = await service.fetchRewardsForExperiment(moocletRef, mockLogger as any); - - expect(result).toEqual(expectedResponse); - }); - - it('should pass nextPageUrl to mooclet data service when provided', async () => { - const moocletRef = { - moocletId: 456, - outcomeVariableName: 'test_variable', - } as MoocletExperimentRef; - - const nextPageUrl = 'http://mooclet-api/value?page=2'; - - mockMoocletDataService.getRewardsForExperiment.mockResolvedValue({ - count: 5, - next: null, - previous: 'http://mooclet-api/value?page=1', - results: [{ id: '2', value: 1.0 }], - } as any); - - await service.fetchRewardsForExperiment(moocletRef, mockLogger as any, nextPageUrl); - - expect(mockMoocletDataService.getRewardsForExperiment).toHaveBeenCalledWith( - { - moocletId: 456, - variableName: 'test_variable', - }, - mockLogger, - nextPageUrl - ); - }); - }); - - describe('createExperimentRewardsSummary', () => { - it('should calculate success rate correctly for each condition', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 1.0 }, - { id: '2', version: 100, value: 1.0 }, - { id: '3', version: 100, value: 0.0 }, - { id: '4', version: 200, value: 1.0 }, - { id: '5', version: 200, value: 0.0 }, - { id: '6', version: 200, value: 0.0 }, - { id: '7', version: 200, value: 0.0 }, - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - expect(result[0]).toEqual({ - conditionCode: 'Control', - successes: 2, - failures: 1, - successRate: '66.7%', - order: 0, - priorSuccess: 1, - priorFailure: 1, - }); - expect(result[1]).toEqual({ - conditionCode: 'Treatment', - successes: 1, - failures: 3, - successRate: '25.0%', - order: 1, - priorSuccess: 1, - priorFailure: 1, - }); - }); - - it('should handle 100% success rate', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 1.0 }, - { id: '2', version: 100, value: 1.0 }, - { id: '3', version: 100, value: 1.0 }, - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - expect(result[0].successRate).toBe('100.0%'); - }); - - it('should handle 0% success rate', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 0.0 }, - { id: '2', version: 100, value: 0.0 }, - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - expect(result[0].successRate).toBe('0.0%'); - }); - - it('should return empty array when results is undefined', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - ], - }; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - undefined as any, - mockLogger as any - ); - - expect(result).toEqual([]); - expect(mockLogger.warn).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'No rewards data returned from Mooclet API', - experimentId: 'exp-123', - }) - ); - }); - - it('should filter rewards by version ID correctly', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 1.0 }, - { id: '2', version: 999, value: 1.0 }, // Different version, should be ignored - { id: '3', version: 100, value: 0.0 }, - { id: '4', version: 200, value: 1.0 }, - { id: '5', version: 200, value: 0.0 }, - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - // Version 999 should not be counted - expect(result[0].successes + result[0].failures).toBe(2); // Only version 100 - expect(result[1].successes + result[1].failures).toBe(2); // Only version 200 - }); - - it('should handle condition with no rewards', async () => { - const refWithConditions = { - ...mockMoocletExperimentRef, - experimentId: 'exp-123', - versionConditionMaps: [ - { - experimentConditionId: 'condition-1', - moocletVersionId: 100, - experimentCondition: { conditionCode: 'Control', order: 0 }, - }, - { - experimentConditionId: 'condition-2', - moocletVersionId: 200, - experimentCondition: { conditionCode: 'Treatment', order: 1 }, - }, - ], - }; - - const rewardsData = [ - { id: '1', version: 100, value: 1.0 }, - { id: '2', version: 100, value: 0.0 }, - // No rewards for version 200 - ]; - - const result = await service.createExperimentRewardsSummary( - refWithConditions as any, - rewardsData as any, - mockLogger as any - ); - - expect(result[1]).toEqual({ - conditionCode: 'Treatment', - successes: 0, - failures: 0, - successRate: '0.0%', - order: 1, - priorSuccess: 1, - priorFailure: 1, - }); - }); - }); - - describe('private helper methods', () => { - describe('findMoocletExperimentRefById', () => { - it('should return mooclet ref when found', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(mockMoocletExperimentRefRepository.findOne).toHaveBeenCalledTimes(1); - expect(result.message).toBe('Reward sent to mooclet successfully.'); - }); - - it('should query with correct relations', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - await service.sendReward(mockUser, request, mockLogger as any); - - expect(mockMoocletExperimentRefRepository.findOne).toHaveBeenCalledWith( - expect.objectContaining({ - where: expect.objectContaining({ experimentId: 'experiment-123' }), - relations: expect.objectContaining({ - versionConditionMaps: expect.anything(), - experiment: expect.anything(), - }), - }) - ); - }); - }); - - describe('findMoocletExperimentRefByDecisionPoint', () => { - it('should use context, site, and target to find experiment', async () => { - const request: RewardValidator = { - experimentId: undefined, - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: 'home', - decisionPoint: { - site: 'site-1', - target: 'target-1', - }, - }; - - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget.mockResolvedValue( - [mockMoocletExperimentRef as MoocletExperimentRef] - ); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - await service.sendReward(mockUser, request, mockLogger as any); - - expect( - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget - ).toHaveBeenCalledTimes(1); - expect( - mockMoocletExperimentRefRepository.findActivelyEnrollingMoocletExperimentsByContextSiteTarget - ).toHaveBeenCalledWith('home', 'site-1', 'target-1'); - }); - }); - - describe('getVersionIdByConditionId', () => { - it('should return mooclet version ID when matching condition is found', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.version).toBe(100); // version for condition-1 - }); - - it('should throw 409 when no matching condition found', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const unmappedEnrollment = { - ...mockEnrollment, - conditionId: 'unmapped-condition', - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(mockMoocletExperimentRef as MoocletExperimentRef); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - unmappedEnrollment as IndividualEnrollment, - ]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - } - }); - - it('should correctly match when multiple maps exist', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const refWithMultipleMaps = { - ...mockMoocletExperimentRef, - versionConditionMaps: [ - { experimentConditionId: 'condition-1', moocletVersionId: 100 }, - { experimentConditionId: 'condition-2', moocletVersionId: 200 }, - { experimentConditionId: 'condition-3', moocletVersionId: 300 }, - ], - }; - - const enrollmentCondition3 = { - ...mockEnrollment, - conditionId: 'condition-3', - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(refWithMultipleMaps as any); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([ - enrollmentCondition3 as IndividualEnrollment, - ]); - - const result = await service.sendReward(mockUser, request, mockLogger as any); - - expect(result.reward.version).toBe(300); - }); - - it('should throw 409 when versionConditionMaps is empty', async () => { - const request: RewardValidator = { - experimentId: 'experiment-123', - rewardValue: BinaryRewardAllowedValue.SUCCESS, - context: undefined, - decisionPoint: undefined, - }; - - const refWithEmptyMaps = { - ...mockMoocletExperimentRef, - versionConditionMaps: [], - }; - - mockMoocletExperimentRefRepository.findOne.mockResolvedValue(refWithEmptyMaps as any); - mockIndividualEnrollmentRepository.findEnrollments.mockResolvedValue([mockEnrollment as IndividualEnrollment]); - - await expect(service.sendReward(mockUser, request, mockLogger as any)).rejects.toThrow(HttpError); - - try { - await service.sendReward(mockUser, request, mockLogger as any); - } catch (error) { - expect((error as HttpError).httpCode).toBe(409); - } - }); - }); - }); -}); diff --git a/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts new file mode 100644 index 0000000000..6d37a5b434 --- /dev/null +++ b/packages/backend/test/unit/services/ThompsonSamplingExperimentCrudService.test.ts @@ -0,0 +1,466 @@ +import { ThompsonSamplingExperimentCrudService } from '../../../src/api/services/ThompsonSamplingExperimentCrudService'; +import { ThompsonSamplingService } from '../../../src/api/services/ThompsonSamplingService'; +import { ASSIGNMENT_ALGORITHM, CACHE_PREFIX } from 'upgrade_types'; + +describe('ThompsonSamplingExperimentCrudService', () => { + let configRepository: any; + let posteriorStateRepository: any; + let cacheService: any; + let service: ThompsonSamplingExperimentCrudService; + + beforeEach(() => { + configRepository = { + save: jest.fn().mockResolvedValue({ id: 'config-1', experimentId: 'experiment-1' }), + update: jest.fn().mockResolvedValue(undefined), + remove: jest.fn().mockResolvedValue(undefined), + findByExperimentId: jest.fn().mockResolvedValue(undefined), + findByExperimentIdWithConditions: jest.fn().mockResolvedValue(undefined), + }; + posteriorStateRepository = { + save: jest.fn().mockResolvedValue(undefined), + update: jest.fn().mockResolvedValue(undefined), + remove: jest.fn().mockResolvedValue(undefined), + }; + cacheService = { resetPrefixCache: jest.fn().mockResolvedValue(undefined) }; + + // Real ThompsonSamplingService — it's a pure, dependency-free service, so using the genuine + // computePosterior/buildPriorsRecord/estimateConditionWeights implementations here (instead of + // re-mocking them) keeps this suite honest about what getRewardsSummary/attachConfigToExperiment + // actually compute. + service = new ThompsonSamplingExperimentCrudService( + configRepository, + posteriorStateRepository, + new ThompsonSamplingService(), + cacheService + ); + }); + + describe('config cache invalidation', () => { + it('resets the Thompson Sampling config cache prefix after creating a config', async () => { + await service.createConfig('experiment-1', [{ id: 'condition-1' }]); + + expect(cacheService.resetPrefixCache).toHaveBeenCalledWith(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + }); + + it('resets the Thompson Sampling config cache prefix after updating a config', async () => { + await service.updateConfig('experiment-1', { batchSize: 5 }); + + expect(cacheService.resetPrefixCache).toHaveBeenCalledWith(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + }); + + it('resets the cache even when updating priors (a second, later write in updateConfig)', async () => { + configRepository.findByExperimentId.mockResolvedValue({ + id: 'config-1', + conditionPosteriorStates: [{ conditionId: 'condition-1' }], + }); + + await service.updateConfig('experiment-1', { priors: { 'condition-1': { success: 3, failure: 2 } } }); + + expect(cacheService.resetPrefixCache).toHaveBeenCalledWith(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + expect(posteriorStateRepository.update).toHaveBeenCalledWith( + { configId: 'config-1', conditionId: 'condition-1' }, + { priorSuccess: 3, priorFailure: 2 } + ); + }); + }); + + describe('updateConfig', () => { + it('only writes fields the caller actually provided, leaving omitted fields untouched', async () => { + await service.updateConfig('experiment-1', { batchSize: 5 }); + + expect(configRepository.update).toHaveBeenCalledWith({ experimentId: 'experiment-1' }, { batchSize: 5 }); + }); + + it('does not issue an update call at all when no threshold/batchSize fields are provided', async () => { + await service.updateConfig('experiment-1', { priors: { 'condition-1': { success: 1, failure: 1 } } }); + + expect(configRepository.update).not.toHaveBeenCalled(); + }); + }); + + describe('createConfigIfApplicable', () => { + it('does nothing for a non-Thompson-Sampling experiment', async () => { + await service.createConfigIfApplicable( + { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any, + { + id: 'experiment-1', + conditions: [{ id: 'condition-1' }], + } as any + ); + + expect(configRepository.save).not.toHaveBeenCalled(); + expect(posteriorStateRepository.save).not.toHaveBeenCalled(); + }); + + it('creates a config seeded only from priors/thresholds, never from reward counts, for a newly created/imported experiment', async () => { + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { + warmupThreshold: 10, + batchSize: 5, + priors: { 'condition-1': { success: 7, failure: 4 } }, + // A source experiment (e.g. one carried over via export/import) might still have these + // fields on it, but ThompsonSamplingConfigParams has no slot for them and createConfig + // never reads anything but priors/thresholds — so posterior state must still start at 0. + successCount: 999, + totalCount: 999, + }, + } as any; + + await service.createConfigIfApplicable(experiment, { + id: 'experiment-1', + conditions: [{ id: 'condition-1' }], + } as any); + + expect(configRepository.save).toHaveBeenCalledWith( + expect.objectContaining({ experimentId: 'experiment-1', warmupThreshold: 10, batchSize: 5 }) + ); + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', + configId: 'config-1', + conditionId: 'condition-1', + priorSuccess: 7, + priorFailure: 4, + successCount: 0, + totalCount: 0, + }); + }); + + it('remaps priors keyed by pre-creation condition ids onto the actual created condition ids', async () => { + // ExperimentService.create()/deduceConditions() regenerate every condition id in place, so the + // client-submitted (or previously-exported) ids the priors record is keyed by never match + // createdExperiment.conditions[].id on their own -- without originalConditionIds to remap + // through, this would silently fall back to the default Beta(1,1) prior for every condition. + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { + priors: { + 'client-temp-id-1': { success: 7, failure: 4 }, + 'client-temp-id-2': { success: 3, failure: 9 }, + }, + }, + } as any; + + await service.createConfigIfApplicable( + experiment, + { + id: 'experiment-1', + conditions: [{ id: 'server-id-1' }, { id: 'server-id-2' }], + } as any, + ['client-temp-id-1', 'client-temp-id-2'] + ); + + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', + configId: 'config-1', + conditionId: 'server-id-1', + priorSuccess: 7, + priorFailure: 4, + successCount: 0, + totalCount: 0, + }); + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', + configId: 'config-1', + conditionId: 'server-id-2', + priorSuccess: 3, + priorFailure: 9, + successCount: 0, + totalCount: 0, + }); + }); + + it('falls back to default priors when originalConditionIds is not provided', async () => { + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { priors: { 'client-temp-id-1': { success: 7, failure: 4 } } }, + } as any; + + await service.createConfigIfApplicable(experiment, { + id: 'experiment-1', + conditions: [{ id: 'server-id-1' }], + } as any); + + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', + configId: 'config-1', + conditionId: 'server-id-1', + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + totalCount: 0, + }); + }); + }); + + describe('syncConfigIfApplicable', () => { + it('does nothing for a non-Thompson-Sampling experiment with no existing config', async () => { + await service.syncConfigIfApplicable( + { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any, + { + id: 'experiment-1', + conditions: [], + } as any + ); + + expect(configRepository.remove).not.toHaveBeenCalled(); + expect(configRepository.update).not.toHaveBeenCalled(); + }); + + it('deletes a stale config left over from before the experiment switched away from Thompson Sampling', async () => { + const staleConfig = { id: 'config-1', conditionPosteriorStates: [{ conditionId: 'condition-1' }] }; + configRepository.findByExperimentId.mockResolvedValue(staleConfig); + + await service.syncConfigIfApplicable( + { assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any, + { + id: 'experiment-1', + conditions: [], + } as any + ); + + expect(configRepository.remove).toHaveBeenCalledWith(staleConfig); + expect(cacheService.resetPrefixCache).toHaveBeenCalledWith(CACHE_PREFIX.THOMPSON_SAMPLING_CONFIG_KEY_PREFIX); + }); + + it('creates a config when an experiment is switched to Thompson Sampling on an update rather than at create time', async () => { + configRepository.findByExperimentId.mockResolvedValue(undefined); + + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { batchSize: 5, priors: { 'condition-1': { success: 2, failure: 1 } } }, + } as any; + + await service.syncConfigIfApplicable(experiment, { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1' }], + } as any); + + expect(configRepository.save).toHaveBeenCalledWith( + expect.objectContaining({ experimentId: 'experiment-1', batchSize: 5 }) + ); + expect(posteriorStateRepository.save).toHaveBeenCalledWith({ + experimentId: 'experiment-1', + configId: 'config-1', + conditionId: 'condition-1', + priorSuccess: 2, + priorFailure: 1, + successCount: 0, + totalCount: 0, + }); + }); + + it('syncs conditions and applies prior updates for a Thompson Sampling experiment', async () => { + configRepository.findByExperimentId.mockResolvedValue({ + id: 'config-1', + conditionPosteriorStates: [{ conditionId: 'condition-1' }], + }); + + const experiment = { + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + thompsonSamplingConfig: { priors: { 'condition-1': { success: 2, failure: 1 } } }, + } as any; + + await service.syncConfigIfApplicable(experiment, { + id: 'experiment-1', + assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + conditions: [{ id: 'condition-1' }], + } as any); + + expect(posteriorStateRepository.update).toHaveBeenCalledWith( + { configId: 'config-1', conditionId: 'condition-1' }, + { priorSuccess: 2, priorFailure: 1 } + ); + }); + }); + + describe('attachConfigToExperiment', () => { + it('leaves a non-Thompson-Sampling experiment untouched', async () => { + const experiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM } as any; + + const result = await service.attachConfigToExperiment(experiment); + + expect(result).toBe(experiment); + expect(configRepository.findByExperimentId).not.toHaveBeenCalled(); + }); + + it('attaches only priors (never accumulated reward counts) for a Thompson Sampling experiment', async () => { + configRepository.findByExperimentId.mockResolvedValue({ + warmupThreshold: 10, + minimumDrawDifference: 0.05, + batchSize: 5, + conditionPosteriorStates: [ + { conditionId: 'condition-1', priorSuccess: 3, priorFailure: 2, successCount: 40, failureCount: 12 }, + ], + }); + + const experiment = { id: 'experiment-1', assignmentAlgorithm: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING } as any; + + const result = await service.attachConfigToExperiment(experiment); + + expect(result.thompsonSamplingConfig).toEqual({ + warmupThreshold: 10, + minimumDrawDifference: 0.05, + batchSize: 5, + priors: { 'condition-1': { success: 3, failure: 2 } }, + }); + }); + }); + + describe('getRewardsSummary', () => { + it('returns an empty summary when no config exists', async () => { + const result = await service.getRewardsSummary('experiment-1'); + + expect(result).toEqual({ + conditions: [], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }); + expect(configRepository.findByExperimentIdWithConditions).toHaveBeenCalledWith('experiment-1'); + }); + + it('computes alpha/beta from priors + counts and sorts by condition order', async () => { + configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + warmupThreshold: 100, + batchSize: 1, + conditionPosteriorStates: [ + { + conditionId: 'condition-2', + priorSuccess: 1, + priorFailure: 1, + successCount: 5, + failureCount: 5, + totalCount: 10, + pendingTotalCount: 0, + condition: { conditionCode: 'B', order: 1 }, + }, + { + conditionId: 'condition-1', + priorSuccess: 2, + priorFailure: 3, + successCount: 8, + failureCount: 2, + totalCount: 10, + pendingTotalCount: 0, + condition: { conditionCode: 'A', order: 0 }, + }, + ], + }); + + const result = await service.getRewardsSummary('experiment-1'); + + expect(result.conditions.map((r) => r.conditionCode)).toEqual(['A', 'B']); + const [conditionA] = result.conditions; + expect(conditionA).toMatchObject({ + conditionCode: 'A', + successes: 8, + failures: 2, + successRate: '80.0%', + priorSuccess: 2, + priorFailure: 3, + }); + expect(result.totalRewardCount).toBe(20); + expect(result.warmupThreshold).toBe(100); + expect(result.batchSize).toBe(1); + }); + + it('keys weight estimation by conditionId, not conditionCode, so two conditions sharing a code do not collide', async () => { + // conditionCode has no uniqueness constraint -- give both conditions the same one, but with + // vastly different posteriors, so a code-keyed weight map (the bug) would collapse them into + // a single shared value instead of each reflecting its own evidence. + configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + warmupThreshold: 0, + batchSize: 1, + conditionPosteriorStates: [ + { + conditionId: 'condition-1', + priorSuccess: 1000, + priorFailure: 1, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingTotalCount: 0, + condition: { conditionCode: 'DUPLICATE', order: 0 }, + }, + { + conditionId: 'condition-2', + priorSuccess: 1, + priorFailure: 1000, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingTotalCount: 0, + condition: { conditionCode: 'DUPLICATE', order: 1 }, + }, + ], + }); + + const { + conditions: [strong, weak], + } = await service.getRewardsSummary('experiment-1'); + + expect(strong.estimatedWeight).toBeGreaterThan(90); + expect(weak.estimatedWeight).toBeLessThan(10); + }); + + it('sums pendingTotalCount across conditions when batchSize > 1', async () => { + configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + warmupThreshold: 0, + batchSize: 5, + conditionPosteriorStates: [ + { + conditionId: 'condition-1', + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingTotalCount: 2, + condition: { conditionCode: 'A', order: 0 }, + }, + { + conditionId: 'condition-2', + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingTotalCount: 1, + condition: { conditionCode: 'B', order: 1 }, + }, + ], + }); + + const result = await service.getRewardsSummary('experiment-1'); + + expect(result.pendingRewardsCount).toBe(3); + expect(result.totalRewardCount).toBe(3); + }); + + it('always reports 0 pending rewards when batchSize is 1, even if a row has a stale pendingTotalCount', async () => { + configRepository.findByExperimentIdWithConditions.mockResolvedValue({ + warmupThreshold: 0, + batchSize: 1, + conditionPosteriorStates: [ + { + conditionId: 'condition-1', + priorSuccess: 1, + priorFailure: 1, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingTotalCount: 3, + condition: { conditionCode: 'A', order: 0 }, + }, + ], + }); + + const result = await service.getRewardsSummary('experiment-1'); + + expect(result.pendingRewardsCount).toBe(0); + }); + }); +}); diff --git a/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts new file mode 100644 index 0000000000..f87c32ca54 --- /dev/null +++ b/packages/backend/test/unit/services/ThompsonSamplingRewardService.test.ts @@ -0,0 +1,539 @@ +import { ThompsonSamplingRewardService } from '../../../src/api/services/ThompsonSamplingRewardService'; +import { UpgradeLogger } from '../../../src/lib/logger/UpgradeLogger'; +import { BinaryRewardAllowedValue, EXPERIMENT_STATE } from 'upgrade_types'; +import { RewardValidator } from '../../../src/api/controllers/validators/RewardValidator'; +import { RequestedExperimentUser } from '../../../src/api/controllers/validators/ExperimentUserValidator'; +import { configureLogger } from '../../utils/logger'; + +const logger = new UpgradeLogger(); + +const EXPERIMENT_ID = 'experiment-1'; +const CONFIG_ID = 'config-1'; +const CONDITION_ID = 'condition-1'; +const CONDITION_A_ID = 'condition-a'; +const CONDITION_B_ID = 'condition-b'; +const USER_ID = 'user-1'; + +interface PosteriorStateRow { + id: string; + configId: string; + conditionId: string; + successCount: number; + failureCount: number; + totalCount: number; + pendingSuccessCount: number; + pendingFailureCount: number; + pendingTotalCount: number; +} + +function makeUser(): RequestedExperimentUser { + return { id: USER_ID, requestedUserId: USER_ID } as RequestedExperimentUser; +} + +function makeRequest(rewardValue: BinaryRewardAllowedValue = BinaryRewardAllowedValue.SUCCESS): RewardValidator { + return { experimentId: EXPERIMENT_ID, rewardValue } as RewardValidator; +} + +function makeStateRow(id: string, conditionId: string): PosteriorStateRow { + return { + id, + configId: CONFIG_ID, + conditionId, + successCount: 0, + failureCount: 0, + totalCount: 0, + pendingSuccessCount: 0, + pendingFailureCount: 0, + pendingTotalCount: 0, + }; +} + +// acceptReward() fires processReward() without awaiting it, so tests that exercise the +// background path need to let its promise chain drain before asserting on side effects. +function flushPromises(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +// The default test double: caching is transparent (always misses through to the source), so +// most tests can ignore caching entirely and assert on repository calls as before. +function makePassthroughCacheService() { + return { + wrap: jest.fn((_key: string, fn: () => Promise) => fn()), + resetPrefixCache: jest.fn().mockResolvedValue(undefined), + }; +} + +// A real (in-memory) cache double, for the tests that specifically exercise caching behavior. +function makeMemoizingCacheService() { + const store = new Map(); + return { + wrap: jest.fn(async (key: string, fn: () => Promise) => { + if (store.has(key)) return store.get(key); + const value = await fn(); + store.set(key, value); + return value; + }), + resetPrefixCache: jest.fn(async (prefix: string) => { + for (const key of Array.from(store.keys())) { + if (key.startsWith(prefix)) store.delete(key); + } + }), + }; +} + +describe('ThompsonSamplingRewardService', () => { + beforeAll(() => { + configureLogger(); + }); + + let posteriorStateRepository: any; + let tsConfigRepository: any; + let individualEnrollmentRepository: any; + let cacheService: ReturnType; + let service: ThompsonSamplingRewardService; + + // In-memory posterior state rows, keyed by conditionId, mutated by the mocked + // increment/update calls so assertions can inspect the final counts after one + // or more processReward() calls. Reward flow always resolves the row through + // findByConditionId(conditionId), so the enrollment mock's conditionId is what + // selects which row a given call mutates. + let statesByCondition: Record; + + // ThompsonSamplingReward audit rows saved via manager.save(ThompsonSamplingReward, plainObject) -- + // the two-arg form, distinct from the single-arg save(entityInstance) used for + // ConditionPosteriorState updates. Tests assert against this instead of a repository mock. + let savedRewards: Array<{ experimentId: string; conditionId: string; userId: string; success: boolean }>; + + function makeConfig(batchSize?: number) { + return { + experimentId: EXPERIMENT_ID, + batchSize, + experiment: { state: EXPERIMENT_STATE.ENROLLING }, + }; + } + + function allStates(): PosteriorStateRow[] { + return Object.values(statesByCondition); + } + + function findRowById(id: string): PosteriorStateRow { + return allStates().find((row) => row.id === id); + } + + // Fakes just enough of TypeORM's EntityManager for recordRewardAtomically()'s transaction: a + // transaction() that runs the callback inline (no real DB transaction/lock semantics -- those + // aren't meaningfully unit-testable without a real Postgres instance), a createQueryBuilder() + // that filters the in-memory rows by configId (the only clause the service issues), and a + // save() that handles both call shapes the service uses: the single-arg entity-instance form + // for ConditionPosteriorState updates (persisted in-memory, since getMany() already hands back + // references into statesByCondition, not copies), and the two-arg (EntityClass, plainObject) + // form for the ThompsonSamplingReward audit insert (recorded into savedRewards). + function makeFakeManager() { + const manager: any = { + transaction: (work: (m: any) => Promise) => work(manager), + createQueryBuilder: () => { + let configIdFilter: string | undefined; + const builder: any = { + where: (_cond: string, params: { configId: string }) => { + configIdFilter = params.configId; + return builder; + }, + orderBy: () => builder, + setLock: () => builder, + getMany: () => Promise.resolve(allStates().filter((row) => row.configId === configIdFilter)), + }; + return builder; + }, + save: jest.fn((entityOrClass: any, maybeEntity?: any) => { + if (maybeEntity !== undefined) { + savedRewards.push(maybeEntity); + return Promise.resolve(maybeEntity); + } + Object.assign(findRowById(entityOrClass.id), entityOrClass); + return Promise.resolve(entityOrClass); + }), + }; + return manager; + } + + beforeEach(() => { + statesByCondition = { + [CONDITION_ID]: makeStateRow('state-1', CONDITION_ID), + }; + + savedRewards = []; + + posteriorStateRepository = { + findByConditionId: jest.fn((conditionId: string) => Promise.resolve(statesByCondition[conditionId])), + manager: makeFakeManager(), + }; + + tsConfigRepository = { + findOne: jest.fn().mockResolvedValue(makeConfig()), + findByDecisionPoint: jest.fn().mockResolvedValue([]), + }; + + individualEnrollmentRepository = { + findEnrollments: jest.fn().mockResolvedValue([{ conditionId: CONDITION_ID }]), + }; + + cacheService = makePassthroughCacheService(); + + service = new ThompsonSamplingRewardService( + posteriorStateRepository, + tsConfigRepository, + individualEnrollmentRepository, + cacheService as any + ); + }); + + describe('recordRewardAtomically (audit row + posterior update as one unit)', () => { + it('propagates a posterior-update failure instead of leaving the audit row committed on its own', async () => { + const dbError = new Error('connection reset mid-transaction'); + // Fail only the ConditionPosteriorState save (single-arg form); the ThompsonSamplingReward + // audit save (two-arg form) still succeeds first, same as the real save order in + // recordRewardAtomically(). + posteriorStateRepository.manager.save = jest.fn((entityOrClass: any, maybeEntity?: any) => { + if (maybeEntity !== undefined) { + savedRewards.push(maybeEntity); + return Promise.resolve(maybeEntity); + } + return Promise.reject(dbError); + }); + + await expect( + (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger) + ).rejects.toThrow(dbError); + + // Both saves ran through the one manager passed into posteriorStateRepository.manager.transaction() + // -- in a real transaction, this failure rolls the audit insert back with it, rather than + // leaving an "acknowledged" reward whose audit row is durable but never reached the posteriors. + // makeFakeManager() has no real rollback semantics (see its own comment), so this proves the two + // writes are coupled in one atomic unit and that a failure surfaces instead of being swallowed -- + // not that the in-memory rollback itself occurs. + expect(savedRewards).toHaveLength(1); + }); + }); + + describe('warmup threshold (reward count)', () => { + it('always persists the raw reward event regardless of batching', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(5)); + + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + + expect(savedRewards).toContainEqual({ + experimentId: EXPERIMENT_ID, + conditionId: CONDITION_ID, + userId: USER_ID, + success: true, + }); + }); + + it('increments totalCount immediately when batchSize is unset (default behavior)', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(undefined)); + + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + + const state = statesByCondition[CONDITION_ID]; + expect(state.totalCount).toBe(1); + expect(state.successCount).toBe(1); + expect(state.pendingTotalCount).toBe(0); + }); + + it('increments totalCount immediately when batchSize is 1', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(1)); + + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + + const state = statesByCondition[CONDITION_ID]; + expect(state.totalCount).toBe(1); + expect(state.successCount).toBe(0); + expect(state.failureCount).toBe(1); + expect(state.pendingTotalCount).toBe(0); + }); + }); + + describe('batchSize (single condition)', () => { + it('buffers rewards as pending until batchSize is reached', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(3)); + + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + let state = statesByCondition[CONDITION_ID]; + expect(state.pendingTotalCount).toBe(1); + expect(state.pendingSuccessCount).toBe(1); + expect(state.pendingFailureCount).toBe(0); + expect(state.totalCount).toBe(0); + expect(state.successCount).toBe(0); + + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + state = statesByCondition[CONDITION_ID]; + expect(state.pendingTotalCount).toBe(2); + expect(state.pendingSuccessCount).toBe(1); + expect(state.pendingFailureCount).toBe(1); + expect(state.totalCount).toBe(0); + expect(state.successCount).toBe(0); + }); + + it('flushes pending counts into successCount/totalCount once batchSize is reached', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(3)); + + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + + const state = statesByCondition[CONDITION_ID]; + expect(state.totalCount).toBe(3); + expect(state.successCount).toBe(2); + expect(state.failureCount).toBe(1); + expect(state.pendingTotalCount).toBe(0); + expect(state.pendingSuccessCount).toBe(0); + expect(state.pendingFailureCount).toBe(0); + }); + + it('resets the pending buffer after a flush so the next batch starts fresh', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(2)); + + // First batch of 2 flushes... + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + let state = statesByCondition[CONDITION_ID]; + expect(state.totalCount).toBe(2); + expect(state.successCount).toBe(2); + + // ...a single reward into the next batch should only be pending, not yet applied. + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.FAILURE), logger); + state = statesByCondition[CONDITION_ID]; + expect(state.totalCount).toBe(2); + expect(state.successCount).toBe(2); + expect(state.pendingTotalCount).toBe(1); + expect(state.pendingSuccessCount).toBe(0); + expect(state.pendingFailureCount).toBe(1); + }); + + it('keeps pendingFailureCount readable without deriving it — always equals pendingTotalCount - pendingSuccessCount', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(10)); + + const values = [ + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.SUCCESS, + ]; + for (const value of values) { + await (service as any).processReward(makeUser(), makeRequest(value), logger); + } + + const state = statesByCondition[CONDITION_ID]; + expect(state.pendingSuccessCount).toBe(2); + expect(state.pendingFailureCount).toBe(2); + expect(state.pendingFailureCount).toBe(state.pendingTotalCount - state.pendingSuccessCount); + }); + + it('keeps failureCount readable without deriving it — always equals totalCount - successCount', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(4)); + + const values = [ + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.SUCCESS, + ]; + for (const value of values) { + await (service as any).processReward(makeUser(), makeRequest(value), logger); + } + + const state = statesByCondition[CONDITION_ID]; + expect(state.successCount).toBe(2); + expect(state.failureCount).toBe(2); + expect(state.failureCount).toBe(state.totalCount - state.successCount); + }); + + it('never drops rewards — total applied + pending always equals rewards recorded', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(4)); + + const values = [ + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.FAILURE, + BinaryRewardAllowedValue.SUCCESS, + BinaryRewardAllowedValue.FAILURE, + ]; + for (const value of values) { + await (service as any).processReward(makeUser(), makeRequest(value), logger); + } + + const state = statesByCondition[CONDITION_ID]; + expect(state.totalCount + state.pendingTotalCount).toBe(values.length); + expect(savedRewards).toHaveLength(values.length); + }); + }); + + describe('batchSize (across conditions of the same experiment)', () => { + beforeEach(() => { + statesByCondition = { + [CONDITION_A_ID]: makeStateRow('state-a', CONDITION_A_ID), + [CONDITION_B_ID]: makeStateRow('state-b', CONDITION_B_ID), + }; + }); + + function rewardCondition(conditionId: string, value: BinaryRewardAllowedValue) { + individualEnrollmentRepository.findEnrollments = jest.fn().mockResolvedValue([{ conditionId }]); + return (service as any).processReward(makeUser(), makeRequest(value), logger); + } + + it('counts pending rewards for batchSize against the whole experiment, not per condition', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(5)); + + // 4 rewards land on condition A, 1 on condition B — 5 total, so batchSize=5 + // should flush even though neither condition alone reached 5. + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.FAILURE); + + const stateA = statesByCondition[CONDITION_A_ID]; + const stateB = statesByCondition[CONDITION_B_ID]; + expect(stateA.pendingTotalCount).toBe(4); + expect(stateA.totalCount).toBe(0); + expect(stateB.pendingTotalCount).toBe(0); + + await rewardCondition(CONDITION_B_ID, BinaryRewardAllowedValue.SUCCESS); + + // The 5th reward (on B) tips the shared batch over the threshold, so both + // conditions' pending buffers flush together. + expect(stateA.totalCount).toBe(4); + expect(stateA.successCount).toBe(3); + expect(stateA.failureCount).toBe(1); + expect(stateA.pendingTotalCount).toBe(0); + + expect(stateB.totalCount).toBe(1); + expect(stateB.successCount).toBe(1); + expect(stateB.pendingTotalCount).toBe(0); + }); + + it('does not flush a condition with no pending rewards of its own', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig(2)); + + // Condition B never receives a reward in this batch; only A's two rewards + // trip the threshold. B's row should be left untouched (still all zeros). + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + await rewardCondition(CONDITION_A_ID, BinaryRewardAllowedValue.SUCCESS); + + const stateB = statesByCondition[CONDITION_B_ID]; + expect(stateB.totalCount).toBe(0); + expect(stateB.pendingTotalCount).toBe(0); + expect(posteriorStateRepository.manager.save).not.toHaveBeenCalledWith( + expect.objectContaining({ id: stateB.id }) + ); + }); + }); + + describe('acceptReward (quick receipt, background processing)', () => { + it('returns a receipt synchronously, without waiting on the DB', () => { + // Never resolves — if acceptReward awaited this, the test would hang instead of returning. + tsConfigRepository.findOne = jest.fn().mockReturnValue(new Promise(() => undefined)); + + const request = makeRequest(BinaryRewardAllowedValue.SUCCESS); + const result = service.acceptReward(makeUser(), request, logger); + + expect(result).toEqual({ message: 'Reward received and is being processed.', request }); + }); + + it('logs the specific reason (once) when the background reward cannot be recorded', async () => { + individualEnrollmentRepository.findEnrollments = jest.fn().mockResolvedValue([]); // no enrollment found + const loggerMock: any = { info: jest.fn(), error: jest.fn(), warn: jest.fn() }; + + const result = service.acceptReward(makeUser(), makeRequest(), loggerMock); + expect(result.message).toBe('Reward received and is being processed.'); + + await flushPromises(); + + expect(loggerMock.error).toHaveBeenCalledTimes(1); + expect(loggerMock.error).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('Could not find unique enrollment') }) + ); + }); + + it('logs a single generic failure message for an unexpected (non-abort) error', async () => { + tsConfigRepository.findOne = jest.fn().mockRejectedValue(new Error('connection reset')); + const loggerMock: any = { info: jest.fn(), error: jest.fn(), warn: jest.fn() }; + + service.acceptReward(makeUser(), makeRequest(), loggerMock); + await flushPromises(); + + expect(loggerMock.error).toHaveBeenCalledTimes(1); + expect(loggerMock.error).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Unexpected error processing Thompson Sampling reward'), + error: expect.any(Error), + }) + ); + }); + + it('still records the reward in the background after returning the receipt', async () => { + const result = service.acceptReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + expect(result.message).toBe('Reward received and is being processed.'); + expect(savedRewards).toHaveLength(0); + + await flushPromises(); + + expect(savedRewards).toContainEqual({ + experimentId: EXPERIMENT_ID, + conditionId: CONDITION_ID, + userId: USER_ID, + success: true, + }); + }); + }); + + describe('config lookup caching', () => { + beforeEach(() => { + cacheService = makeMemoizingCacheService(); + service = new ThompsonSamplingRewardService( + posteriorStateRepository, + tsConfigRepository, + individualEnrollmentRepository, + cacheService as any + ); + }); + + it('reuses a cached experimentId lookup across rewards instead of re-querying the DB', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig()); + + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + await (service as any).processReward(makeUser(), makeRequest(BinaryRewardAllowedValue.SUCCESS), logger); + + expect(tsConfigRepository.findOne).toHaveBeenCalledTimes(1); + // Only the config lookup is cached — the reward itself is still recorded every time. + expect(savedRewards).toHaveLength(2); + }); + + it('reuses a cached decision-point lookup across rewards instead of re-querying the DB', async () => { + tsConfigRepository.findByDecisionPoint = jest.fn().mockResolvedValue([makeConfig()]); + const dpRequest = (): RewardValidator => + ({ + rewardValue: BinaryRewardAllowedValue.SUCCESS, + context: 'context-1', + decisionPoint: { site: 'site-1', target: 'target-1' }, + } as RewardValidator); + + await (service as any).processReward(makeUser(), dpRequest(), logger); + await (service as any).processReward(makeUser(), dpRequest(), logger); + + expect(tsConfigRepository.findByDecisionPoint).toHaveBeenCalledTimes(1); + }); + + it('does not share cache entries between different experiments', async () => { + tsConfigRepository.findOne = jest.fn().mockResolvedValue(makeConfig()); + + await (service as any).processReward(makeUser(), makeRequest(), logger); + await (service as any).processReward( + makeUser(), + { experimentId: 'experiment-2', rewardValue: BinaryRewardAllowedValue.SUCCESS } as RewardValidator, + logger + ); + + expect(tsConfigRepository.findOne).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/packages/backend/test/unit/services/ThompsonSamplingService.test.ts b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts new file mode 100644 index 0000000000..78479c2c1b --- /dev/null +++ b/packages/backend/test/unit/services/ThompsonSamplingService.test.ts @@ -0,0 +1,361 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { + ThompsonSamplingService, + ConditionRewardSummary, + ThompsonSamplingConfig, + DEFAULT_PRIOR, +} from '../../../src/api/services/ThompsonSamplingService'; +import { configureLogger } from '../../utils/logger'; + +describe('ThompsonSamplingService', () => { + let service: ThompsonSamplingService; + + beforeAll(() => { + configureLogger(); + }); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ThompsonSamplingService], + }).compile(); + + service = module.get(ThompsonSamplingService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('selectCondition', () => { + it('throws when given an empty condition list', () => { + expect(() => service.selectCondition([], [], 0)).toThrow(); + }); + + it('returns the only condition when list has one entry', () => { + expect(service.selectCondition(['A'], [], 10)).toBe('A'); + }); + + it('always returns a valid condition code', () => { + const conditions = ['A', 'B', 'C']; + for (let i = 0; i < 30; i++) { + expect(conditions).toContain(service.selectCondition(conditions, [], 100)); + } + }); + + describe('warmup phase', () => { + it('uses uniform random during warmup — both conditions appear across 50 draws', () => { + const conditions = ['A', 'B']; + // A has an overwhelmingly dominant posterior; without warmup it would always win + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, + { conditionId: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, + ]; + const config: ThompsonSamplingConfig = { warmupThreshold: 50 }; + + const results = new Set(); + for (let i = 0; i < 50; i++) { + results.add(service.selectCondition(conditions, rewardSummaries, 5, config)); + } + expect(results.has('A')).toBe(true); + expect(results.has('B')).toBe(true); + }); + + it('exits warmup once reward count exceeds threshold', () => { + const conditions = ['A', 'B']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, + { conditionId: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, + ]; + const config: ThompsonSamplingConfig = { warmupThreshold: 10 }; + + let aCount = 0; + const runs = 100; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, 100, config) === 'A') { + aCount++; + } + } + // After warmup, A's dominant posterior should win nearly every draw + expect(aCount / runs).toBeGreaterThan(0.95); + }); + + it('treats warmupThreshold of 0 as disabled, not a one-draw warmup, even with no rewards yet', () => { + const conditions = ['A', 'B']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 1000, failureCount: 0, totalCount: 1000 }, + { conditionId: 'B', successCount: 0, failureCount: 1000, totalCount: 1000 }, + ]; + const config: ThompsonSamplingConfig = { warmupThreshold: 0 }; + + let aCount = 0; + const runs = 100; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, 0, config) === 'A') { + aCount++; + } + } + // Matches the frontend's isWarmupConfigured (warmupThreshold > 0): at totalRewardCount 0 + // with warmupThreshold 0, real Thompson Sampling should already be in effect, not one + // forced uniform draw. + expect(aCount / runs).toBeGreaterThan(0.95); + }); + }); + + describe('Thompson Sampling selection', () => { + it('selects the condition with a better reward history reliably', () => { + const conditions = ['good', 'bad']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'good', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionId: 'bad', successCount: 10, failureCount: 90, totalCount: 100 }, + ]; + + let goodCount = 0; + const runs = 500; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, runs) === 'good') { + goodCount++; + } + } + expect(goodCount / runs).toBeGreaterThan(0.9); + }); + + it('handles three or more conditions — clear winner dominates', () => { + const conditions = ['A', 'B', 'C', 'D']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 5, failureCount: 95, totalCount: 100 }, + { conditionId: 'B', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionId: 'C', successCount: 10, failureCount: 90, totalCount: 100 }, + { conditionId: 'D', successCount: 5, failureCount: 95, totalCount: 100 }, + ]; + + let bCount = 0; + const runs = 500; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, runs) === 'B') { + bCount++; + } + } + expect(bCount / runs).toBeGreaterThan(0.8); + }); + + it('respects strong priors when no rewards have been collected', () => { + const conditions = ['A', 'B']; + const config: ThompsonSamplingConfig = { + priors: { + A: { success: 100, failure: 1 }, + B: { success: 1, failure: 100 }, + }, + }; + + let aCount = 0; + const runs = 100; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, [], 0, config) === 'A') { + aCount++; + } + } + expect(aCount / runs).toBeGreaterThan(0.95); + }); + + it('uses DEFAULT_PRIOR for conditions missing a prior entry', () => { + const conditions = ['A', 'B']; + const config: ThompsonSamplingConfig = { + priors: { + A: { success: 100, failure: 1 }, // strong prior for A + // B intentionally absent — should use DEFAULT_PRIOR Beta(1,1) + }, + }; + + let aCount = 0; + const runs = 100; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, [], 0, config) === 'A') { + aCount++; + } + } + expect(aCount / runs).toBeGreaterThan(0.9); + }); + + it('handles a condition with no reward summary entry (defaults to prior)', () => { + const conditions = ['A', 'B']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 50, failureCount: 50, totalCount: 100 }, + // B has no entry — treated as zero rewards, uses prior only + ]; + + for (let i = 0; i < 20; i++) { + expect(conditions).toContain(service.selectCondition(conditions, rewardSummaries, 200)); + } + }); + }); + + describe('minimumDrawDifference', () => { + it('falls back to uniform when threshold is larger than any possible draw difference', () => { + const conditions = ['A', 'B']; + // Threshold of 2 always triggers since Beta draws are in [0,1] + const config: ThompsonSamplingConfig = { minimumDrawDifference: 2 }; + + const results = new Set(); + for (let i = 0; i < 50; i++) { + results.add(service.selectCondition(conditions, [], 100, config)); + } + expect(results.has('A')).toBe(true); + expect(results.has('B')).toBe(true); + }); + + it('does not interfere when threshold is zero', () => { + const conditions = ['A', 'B']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 90, failureCount: 10, totalCount: 100 }, + { conditionId: 'B', successCount: 10, failureCount: 90, totalCount: 100 }, + ]; + const config: ThompsonSamplingConfig = { minimumDrawDifference: 0 }; + + let aCount = 0; + const runs = 200; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, runs, config) === 'A') { + aCount++; + } + } + expect(aCount / runs).toBeGreaterThan(0.85); + }); + + it('compares the top two draws by value, not the first two conditions by list order', () => { + // The MoocLet reference this threshold is based on (tspostdiff_thresh) hardcodes "the + // first two versions" for its diff check, which only happens to work because it assumes + // exactly 2 arms. With 3+ conditions that would mean comparing whichever two conditions + // come first in the list, not the two that are actually closest. Here B's posterior is far + // below A/C's, so a "first two by list order" comparison (B vs A) would see a huge gap and + // almost never fall back to uniform — even though A and C (identically strong) draw values + // within minimumDrawDifference of each other very often and should trigger the fallback. + const conditions = ['B', 'A', 'C']; + const rewardSummaries: ConditionRewardSummary[] = [ + { conditionId: 'A', successCount: 99, failureCount: 0, totalCount: 99 }, + { conditionId: 'B', successCount: 0, failureCount: 99, totalCount: 99 }, + { conditionId: 'C', successCount: 99, failureCount: 0, totalCount: 99 }, + ]; + const config: ThompsonSamplingConfig = { minimumDrawDifference: 0.3 }; + + let bCount = 0; + const runs = 500; + for (let i = 0; i < runs; i++) { + if (service.selectCondition(conditions, rewardSummaries, runs, config) === 'B') { + bCount++; + } + } + + // B only ever wins via the uniform fallback, so a meaningful rate here proves the + // fallback is firing off of A/C's close draws — a first-two-by-list-order comparison + // would keep this near zero instead. + expect(bCount / runs).toBeGreaterThan(0.05); + }); + }); + }); + + describe('estimateConditionWeights', () => { + it('returns empty object for empty conditions', () => { + expect(service.estimateConditionWeights([])).toEqual({}); + }); + + it('returns 100 for a single condition', () => { + expect(service.estimateConditionWeights([{ code: 'A', alpha: 1, beta: 1 }])).toEqual({ A: 100 }); + }); + + it('weights always sum to exactly 100', () => { + const conditions = [ + { code: 'A', alpha: 5, beta: 3 }, + { code: 'B', alpha: 2, beta: 8 }, + { code: 'C', alpha: 10, beta: 1 }, + ]; + const weights = service.estimateConditionWeights(conditions); + const total = Object.values(weights).reduce((sum, w) => sum + w, 0); + expect(total).toBe(100); + }); + + it('all weights are non-negative integers', () => { + const conditions = [ + { code: 'A', alpha: 3, beta: 3 }, + { code: 'B', alpha: 3, beta: 3 }, + { code: 'C', alpha: 3, beta: 3 }, + { code: 'D', alpha: 3, beta: 3 }, + ]; + const weights = service.estimateConditionWeights(conditions); + for (const w of Object.values(weights)) { + expect(w).toBeGreaterThanOrEqual(0); + expect(Number.isInteger(w)).toBe(true); + } + }); + + it('two equal Beta(1,1) arms split exactly 50/50 (skips simulation by symmetry)', () => { + const conditions = [ + { code: 'A', alpha: 1, beta: 1 }, + { code: 'B', alpha: 1, beta: 1 }, + ]; + const weights = service.estimateConditionWeights(conditions); + expect(weights).toEqual({ A: 50, B: 50 }); + }); + + it('four equal arms each get exactly 25% regardless of the shared prior strength', () => { + const conditions = ['A', 'B', 'C', 'D'].map((code) => ({ code, alpha: 2, beta: 2 })); + const weights = service.estimateConditionWeights(conditions); + expect(weights).toEqual({ A: 25, B: 25, C: 25, D: 25 }); + }); + + it('three equal arms split evenly and still sum to 100', () => { + const conditions = ['A', 'B', 'C'].map((code) => ({ code, alpha: 4, beta: 6 })); + const weights = service.estimateConditionWeights(conditions); + const total = Object.values(weights).reduce((sum, w) => sum + w, 0); + expect(total).toBe(100); + for (const w of Object.values(weights)) { + expect(w).toBeGreaterThanOrEqual(33); + expect(w).toBeLessThanOrEqual(34); + } + }); + + it('unequal alpha/beta still runs the simulation (not the equal-posterior shortcut)', () => { + const conditions = [ + { code: 'A', alpha: 5, beta: 3 }, + { code: 'B', alpha: 3, beta: 5 }, + ]; + const weights = service.estimateConditionWeights(conditions); + expect(weights['A']).toBeGreaterThan(50); + expect(weights['B']).toBeLessThan(50); + }); + + it('dominant condition (90 successes / 10 failures) captures most weight', () => { + const conditions = [ + { code: 'winner', alpha: 1 + 90, beta: 1 + 10 }, + { code: 'loser', alpha: 1 + 10, beta: 1 + 90 }, + ]; + const weights = service.estimateConditionWeights(conditions); + expect(weights['winner']).toBeGreaterThanOrEqual(90); + expect(weights['loser']).toBeLessThanOrEqual(10); + }); + + it('strong prior with no reward data drives weight toward the favored arm', () => { + const conditions = [ + { code: 'favored', alpha: 100, beta: 1 }, + { code: 'weak', alpha: 1, beta: 100 }, + ]; + const weights = service.estimateConditionWeights(conditions); + expect(weights['favored']).toBeGreaterThanOrEqual(90); + }); + + it('respects numDraws parameter — custom draw count still sums to 100', () => { + const conditions = [ + { code: 'A', alpha: 3, beta: 2 }, + { code: 'B', alpha: 2, beta: 3 }, + ]; + const weights = service.estimateConditionWeights(conditions, 500); + const total = Object.values(weights).reduce((sum, w) => sum + w, 0); + expect(total).toBe(100); + }); + }); + + describe('DEFAULT_PRIOR', () => { + it('is Beta(1,1) — the uninformative prior', () => { + expect(DEFAULT_PRIOR).toEqual({ success: 1, failure: 1 }); + }); + }); +}); diff --git a/packages/frontend/projects/upgrade/src/app/core/api-endpoints.constants.ts b/packages/frontend/projects/upgrade/src/app/core/api-endpoints.constants.ts index 06113fe0d0..ce9a4efd48 100644 --- a/packages/frontend/projects/upgrade/src/app/core/api-endpoints.constants.ts +++ b/packages/frontend/projects/upgrade/src/app/core/api-endpoints.constants.ts @@ -69,6 +69,6 @@ export const API_ENDPOINTS: APIEndpoints = { exportAllExperimentIncludeLists: '/experiments/export/includeLists', exportAllExperimentExcludeLists: '/experiments/export/excludeLists', importExperimentList: '/experiments/lists/import', - getMoocletRewardsData: '/experiments/mooclet-rewards', featureFlagGraphInfo: '/flags/date', + experimentsRewardsSummary: '/experiments/rewards', } as const; diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.data.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.data.service.ts index f601fb2acc..e471ec11e9 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.data.service.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.data.service.ts @@ -129,6 +129,11 @@ export class ExperimentDataService { return this.http.post(url, params); } + fetchRewardsDataForExperiment(experimentId: string): Observable { + const url = `${API_ENDPOINTS.experimentsRewardsSummary}/${experimentId}`; + return this.http.get(url); + } + addInclusionList(list: ExperimentSegmentListRequest): Observable { const url = API_ENDPOINTS.addExperimentInclusionList; return this.http.post(url, list); @@ -221,9 +226,4 @@ export class ExperimentDataService { }; return this.updateExperiment(updatedExperiment); } - - fetchMoocletRewardsDataForExperiment(experimentId: string): Observable { - const url = `${API_ENDPOINTS.getMoocletRewardsData}/${experimentId}`; - return this.http.get(url); - } } diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.service.ts index 073a05c199..0f77bd74a9 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.service.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/experiments.service.ts @@ -63,7 +63,7 @@ import { map, take, tap } from 'rxjs/operators'; import { LocalStorageService } from '../local-storage/local-storage.service'; import { ExperimentSegmentListRequest } from '../segments/store/segments.model'; import { ConditionWeightUpdate } from '../../features/dashboard/experiments/modals/edit-condition-weights-modal/edit-condition-weights-modal.component'; -import { MoocletTSConfigurablePolicyParametersDTO, Prior } from 'upgrade_types'; +import { Prior } from 'upgrade_types'; import { selectCurrentUserEmail } from '../auth/store/auth.selectors'; @Injectable() @@ -304,10 +304,10 @@ export class ExperimentService { updateExperimentConditionPrior(experiment: ExperimentVM, prior: Record): void { const updatedExperiment: ExperimentVM = { ...experiment, - moocletPolicyParameters: { - ...experiment.moocletPolicyParameters, - prior, - } as MoocletTSConfigurablePolicyParametersDTO, + thompsonSamplingConfig: { + ...experiment.thompsonSamplingConfig, + priors: prior, + }, }; this.store$.dispatch( experimentAction.actionUpsertExperiment({ diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.spec.ts deleted file mode 100644 index dacc5c746c..0000000000 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.spec.ts +++ /dev/null @@ -1,483 +0,0 @@ -import { Validators } from '@angular/forms'; -import { - MoocletExperimentHelperService, - EditableTSConfigurablePolicyParameters, - formatTSConfigurablePolicyParamDetails, -} from './mooclet-helper.service'; -import { ASSIGNMENT_ALGORITHM, MoocletTSConfigurablePolicyParametersDTO } from 'upgrade_types'; -import { ExperimentVM } from './store/experiments.model'; -import { environment } from '../../../environments/environment'; - -describe('MoocletAlgorithmHelperService', () => { - let service: MoocletExperimentHelperService; - - beforeEach(() => { - service = new MoocletExperimentHelperService(); - }); - - // ============================================================================ - // Environment and Feature Flag Methods - // ============================================================================ - - describe('isMoocletEnabled', () => { - it('should return true when mooclet toggle is enabled', () => { - environment.moocletToggle = true; - expect(service.isMoocletEnabled()).toBe(true); - }); - - it('should return false when mooclet toggle is disabled', () => { - environment.moocletToggle = false; - expect(service.isMoocletEnabled()).toBe(false); - }); - }); - - describe('isMoocletAlgorithm', () => { - it('should return true for TS_CONFIGURABLE algorithm', () => { - expect(service.isMoocletAlgorithm(ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE)).toBe(true); - }); - - it('should return false for RANDOM algorithm', () => { - expect(service.isMoocletAlgorithm(ASSIGNMENT_ALGORITHM.RANDOM)).toBe(false); - }); - - it('should return false for STRATIFIED_RANDOM_SAMPLING algorithm', () => { - expect(service.isMoocletAlgorithm(ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING)).toBe(false); - }); - }); - - // ============================================================================ - // TS Configurable Default Parameters - // ============================================================================ - - describe('getTSConfigurableDefaults', () => { - it('should return a MoocletTSConfigurablePolicyParametersDTO instance', () => { - const result = service.getTSConfigurableDefaults(); - expect(result).toBeInstanceOf(MoocletTSConfigurablePolicyParametersDTO); - }); - - it('should return default values matching DTO defaults', () => { - const result = service.getTSConfigurableDefaults(); - const expected = new MoocletTSConfigurablePolicyParametersDTO(); - - expect(result.batch_size).toBe(expected.batch_size); - expect(result.uniform_threshold).toBe(expected.uniform_threshold); - expect(result.tspostdiff_thresh).toBe(expected.tspostdiff_thresh); - expect(result.max_rating).toBe(expected.max_rating); - expect(result.min_rating).toBe(expected.min_rating); - }); - }); - - // ============================================================================ - // Derive Editable Parameters - // ============================================================================ - - describe('deriveEditableParametersForTSConfigurable', () => { - it('should return default values when no existing params provided', () => { - const result = service.deriveEditableParametersForTSConfigurable(); - const defaults = new MoocletTSConfigurablePolicyParametersDTO(); - - expect(result).toEqual({ - batch_size: defaults.batch_size, - uniform_threshold: defaults.uniform_threshold, - tspostdiff_thresh: defaults.tspostdiff_thresh, - }); - }); - - it('should return default values when existing params is undefined', () => { - const result = service.deriveEditableParametersForTSConfigurable(undefined); - const defaults = new MoocletTSConfigurablePolicyParametersDTO(); - - expect(result.batch_size).toBe(defaults.batch_size); - }); - - it('should extract editable fields from existing params', () => { - const existingParams = createMockTSConfigurableParams({ - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }); - - const result = service.deriveEditableParametersForTSConfigurable(existingParams); - - expect(result).toEqual({ - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }); - }); - - it('should not include non-editable fields in result', () => { - const existingParams = createMockTSConfigurableParams({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - outcome_variable_name: 'test_outcome', - max_rating: 5, - min_rating: 1, - }); - - const result = service.deriveEditableParametersForTSConfigurable(existingParams); - - expect((result as any).assignmentAlgorithm).toBeUndefined(); - expect((result as any).outcome_variable_name).toBeUndefined(); - expect((result as any).max_rating).toBeUndefined(); - expect((result as any).min_rating).toBeUndefined(); - }); - }); - - // ============================================================================ - // Build Complete DTO - // ============================================================================ - - describe('buildTSConfigurablePolicyParametersDTO', () => { - beforeEach(() => { - jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-01-15T10:30:00.000Z'); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should build complete DTO from editable parameters', () => { - const editableParams: EditableTSConfigurablePolicyParameters = { - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }; - - const result = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result.batch_size).toBe(50); - expect(result.uniform_threshold).toBe(100); - expect(result.tspostdiff_thresh).toBe(5); - }); - - it('should set assignmentAlgorithm to MOOCLET_TS_CONFIGURABLE', () => { - const editableParams = createMockEditableParams(); - const result = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result.assignmentAlgorithm).toBe(ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE); - }); - - it('should not set outcome_variable_name since it is generated server-side', () => { - const editableParams = createMockEditableParams(); - const result = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result.outcome_variable_name).toBeUndefined(); - }); - - it('should set max_rating and min_rating from defaults', () => { - const editableParams = createMockEditableParams(); - const defaults = service.getTSConfigurableDefaults(); - - const result = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result.max_rating).toBe(defaults.max_rating); - expect(result.min_rating).toBe(defaults.min_rating); - }); - - it('should not include outcome_variable_name for any experiment name', () => { - const editableParams = createMockEditableParams(); - - const result1 = service.buildTSConfigurablePolicyParametersDTO(editableParams); - const result2 = service.buildTSConfigurablePolicyParametersDTO(editableParams); - - expect(result1.outcome_variable_name).toBeUndefined(); - expect(result2.outcome_variable_name).toBeUndefined(); - }); - }); - - // ============================================================================ - // Field Validators - // ============================================================================ - - describe('getTSConfigurableFieldValidators', () => { - it('should return validators for all editable fields', () => { - const validators = service.getTSConfigurableFieldValidators(); - - expect(validators.batch_size).toBeDefined(); - expect(validators.uniform_threshold).toBeDefined(); - expect(validators.tspostdiff_thresh).toBeDefined(); - }); - - it('should include required validator for all fields', () => { - const validators = service.getTSConfigurableFieldValidators(); - - Object.values(validators).forEach((fieldValidators) => { - expect(fieldValidators).toContain(Validators.required); - }); - }); - - it('should include min validators based on defaults', () => { - const validators = service.getTSConfigurableFieldValidators(); - const defaults = service.getTSConfigurableDefaults(); - - expect(validators.batch_size.length).toBe(4); - expect(validators.uniform_threshold.length).toBe(4); - expect(validators.tspostdiff_thresh.length).toBe(3); - - // Test that min validator works correctly for batch_size - const batchSizeMinValidator = validators.batch_size[1]; - // Use a value below the minimum - should fail (return error object) - expect(batchSizeMinValidator({ value: defaults.batch_size - 1 } as any)).toBeTruthy(); - // Use the minimum value - should pass (return null) - expect(batchSizeMinValidator({ value: defaults.batch_size } as any)).toBeNull(); - // Use a value above the minimum - should pass (return null) - expect(batchSizeMinValidator({ value: defaults.batch_size + 1 } as any)).toBeNull(); - }); - - it('should return exactly 3 validators per field (required + min)', () => { - const validators = service.getTSConfigurableFieldValidators(); - - Object.values(validators).forEach((fieldValidators) => { - if (fieldValidators === validators.tspostdiff_thresh) { - expect(fieldValidators.length).toBe(3); - return; - } - expect(fieldValidators.length).toBe(4); - }); - }); - }); - - // ============================================================================ - // Supported Mooclet Algorithms - // ============================================================================ - - describe('getSupportedMoocletAlgorithmOptions', () => { - it('should return empty array when mooclet is disabled', () => { - environment.moocletToggle = false; - const result = service.getSupportedMoocletAlgorithmOptions(); - - expect(result).toEqual([]); - }); - - it('should return algorithm options when mooclet is enabled', () => { - environment.moocletToggle = true; - const result = service.getSupportedMoocletAlgorithmOptions(); - - expect(result.length).toBeGreaterThan(0); - expect(Array.isArray(result)).toBe(true); - }); - - it('should return options with value and description properties', () => { - environment.moocletToggle = true; - const result = service.getSupportedMoocletAlgorithmOptions(); - - result.forEach((option) => { - expect(option).toHaveProperty('value'); - expect(option).toHaveProperty('description'); - expect(typeof option.value).toBe('string'); - expect(typeof option.description).toBe('string'); - }); - }); - - it('should include MOOCLET_TS_CONFIGURABLE in options', () => { - environment.moocletToggle = true; - const result = service.getSupportedMoocletAlgorithmOptions(); - - const tsConfigOption = result.find((opt) => opt.value === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE); - expect(tsConfigOption).toBeDefined(); - expect(tsConfigOption.description).toContain('Adaptive Experiment Algorithm'); - }); - - it('should format description correctly', () => { - environment.moocletToggle = true; - const result = service.getSupportedMoocletAlgorithmOptions(); - - result.forEach((option) => { - expect(option.description).toMatch(/^Adaptive Experiment Algorithm: /); - expect(option.description).toContain(option.value); - }); - }); - }); - - // ============================================================================ - // Validation - // ============================================================================ - - describe('validateTSConfigurablePolicyParameters', () => { - it('should return observable of validation errors', (done) => { - const params = createMockTSConfigurableParams(); - - service.validateTSConfigurablePolicyParameters(params).subscribe((errors) => { - expect(Array.isArray(errors)).toBe(true); - done(); - }); - }); - - it('should validate valid parameters successfully', (done) => { - const validParams = { - batch_size: 30, - uniform_threshold: 50, - tspostdiff_thresh: 3, - outcome_variable_name: 'test_outcome', - max_rating: 5, - min_rating: 1, - }; - - service.validateTSConfigurablePolicyParameters(validParams).subscribe((errors) => { - expect(errors.length).toBe(0); - done(); - }); - }); - - it('should add assignmentAlgorithm automatically', (done) => { - const params = { - batch_size: 30, - uniform_threshold: 50, - tspostdiff_thresh: 3, - outcome_variable_name: 'test', - max_rating: 5, - min_rating: 1, - }; - - service.validateTSConfigurablePolicyParameters(params).subscribe((errors) => { - expect(errors.length).toBe(0); - done(); - }); - }); - - it('should validate successfully even with empty outcome_variable_name', (done) => { - const params = { - batch_size: 30, - uniform_threshold: 50, - tspostdiff_thresh: 3, - outcome_variable_name: '', // Empty string should not cause validation errors - max_rating: 5, - min_rating: 1, - }; - - service.validateTSConfigurablePolicyParameters(params).subscribe((errors) => { - expect(errors.length).toBe(0); - done(); - }); - }); - - it('should validate successfully without outcome_variable_name property', (done) => { - const params = { - batch_size: 30, - uniform_threshold: 50, - tspostdiff_thresh: 3, - // outcome_variable_name property completely missing - max_rating: 5, - min_rating: 1, - }; - - service.validateTSConfigurablePolicyParameters(params).subscribe((errors) => { - expect(errors.length).toBe(0); - done(); - }); - }); - }); -}); - -// ============================================================================ -// Pure Function Tests -// ============================================================================ - -describe('formatTSConfigurablePolicyParamDetails (pure function)', () => { - it('should return undefined when mooclet toggle is disabled', () => { - environment.moocletToggle = false; - const experiment = createMockExperimentVM(); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - - expect(result).toBeUndefined(); - }); - - it('should return undefined when algorithm is not TS_CONFIGURABLE', () => { - environment.moocletToggle = true; - const experiment = createMockExperimentVM({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - }); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - - expect(result).toBeUndefined(); - }); - - it('should return undefined when moocletPolicyParameters is missing', () => { - environment.moocletToggle = true; - const experiment = createMockExperimentVM({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: undefined, - }); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - - expect(result).toBeUndefined(); - }); - - it('should format parameters correctly', () => { - environment.moocletToggle = true; - const experiment = createMockExperimentVM({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: createMockTSConfigurableParams({ - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }), - }); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - - expect(result).toBeDefined(); - expect(Array.isArray(result)).toBe(true); - expect(result.length).toBe(3); - }); - - it('should include all configurable parameters', () => { - environment.moocletToggle = true; - const experiment = createMockExperimentVM({ - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - moocletPolicyParameters: createMockTSConfigurableParams({ - batch_size: 50, - uniform_threshold: 100, - tspostdiff_thresh: 5, - }), - }); - - const result = formatTSConfigurablePolicyParamDetails(experiment); - const values = result.map((param) => param.value); - - expect(values).toContain(50); - expect(values).toContain(100); - expect(values).toContain(5); - }); -}); - -// ============================================================================ -// Helper Functions -// ============================================================================ - -function createMockEditableParams( - overrides?: Partial -): EditableTSConfigurablePolicyParameters { - const defaults = new MoocletTSConfigurablePolicyParametersDTO(); - return { - batch_size: defaults.batch_size, - uniform_threshold: defaults.uniform_threshold, - tspostdiff_thresh: defaults.tspostdiff_thresh, - ...overrides, - }; -} - -function createMockTSConfigurableParams( - overrides?: Partial -): MoocletTSConfigurablePolicyParametersDTO { - const defaults = new MoocletTSConfigurablePolicyParametersDTO(); - return { - ...defaults, - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - outcome_variable_name: 'test_outcome', - ...overrides, - } as MoocletTSConfigurablePolicyParametersDTO; -} - -function createMockExperimentVM(overrides?: Partial): ExperimentVM { - return { - id: 'exp-1', - name: 'Test Experiment', - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.RANDOM, - moocletPolicyParameters: undefined, - ...overrides, - } as ExperimentVM; -} diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.ts deleted file mode 100644 index 1769318e94..0000000000 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/mooclet-helper.service.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { Injectable } from '@angular/core'; -import { Observable, from } from 'rxjs'; -import { validate, ValidationError } from 'class-validator'; -import { plainToInstance } from 'class-transformer'; -import { ValidatorFn, Validators } from '@angular/forms'; -import { - ASSIGNMENT_ALGORITHM, - MOOCLET_POLICY_SCHEMA_MAP, - MoocletTSConfigurablePolicyParametersDTO, - SUPPORTED_MOOCLET_ALGORITHMS, -} from 'upgrade_types'; -import { ExperimentVM, TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS } from './store/experiments.model'; -import { environment } from '../../../environments/environment'; -import { BullettedListKeyValueFormat } from '@shared-component-lib/common-section-card-overview-details/common-section-card-overview-details.component'; -import { CommonFormHelpersService } from '../../shared/services/common-form-helpers.service'; - -// ============================================================================ -// Pure Functions (exported for use in selectors and other pure contexts) -// ============================================================================ - -const DEFAULT_MAX_NUMBER_INPUT = 1000000; - -/** - * Format TS Configurable policy parameters for display in the overview section. - * Returns an array of objects with translation keys and values for proper i18n support. - */ -export function formatTSConfigurablePolicyParamDetails( - experiment: ExperimentVM -): BullettedListKeyValueFormat[] | undefined { - if ( - !isMoocletEnabled() || - !isMoocletExperiment(experiment) || - experiment?.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE - ) { - console.warn('mooclet is disabled or experiment is not using TS Configurable algorithm'); - return undefined; - } - - const params = experiment.moocletPolicyParameters; - const formattedParams: BullettedListKeyValueFormat[] = []; - - formattedParams.push({ - labelKey: TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS.BATCH_SIZE, - value: params.batch_size, - }); - formattedParams.push({ - labelKey: TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS.UNIFORM_THRESHOLD, - value: params.uniform_threshold, - }); - formattedParams.push({ - labelKey: TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS.TSPOSTDIFF_THRESH, - value: params.tspostdiff_thresh, - }); - return formattedParams; -} - -export function isMoocletEnabled(): boolean { - return environment.moocletToggle; -} - -export function isMoocletAlgorithm(algorithm: ASSIGNMENT_ALGORITHM): boolean { - return !!MOOCLET_POLICY_SCHEMA_MAP[algorithm]; -} - -export function isMoocletExperiment(experiment: ExperimentVM): boolean { - return isMoocletAlgorithm(experiment.assignmentAlgorithm) && !!experiment.moocletPolicyParameters; -} - -// ============================================================================ -// Type Definitions -// ============================================================================ - -export interface EditableTSConfigurablePolicyParameters { - batch_size: number; - uniform_threshold: number; - tspostdiff_thresh: number; -} - -/** - * Service providing helper methods for mooclet adaptive algorithms - */ -@Injectable({ - providedIn: 'root', -}) -export class MoocletExperimentHelperService { - // ============================================================================ - // General Mooclet Algorithm Helpers - // ============================================================================ - - isMoocletEnabled(): boolean { - return isMoocletEnabled(); - } - - isMoocletAlgorithm(algorithm: ASSIGNMENT_ALGORITHM): boolean { - return isMoocletAlgorithm(algorithm); - } - - /** - * Get supported mooclet algorithm options for UI display. - * Returns algorithm options with descriptions, or empty array if mooclet is disabled. - */ - getSupportedMoocletAlgorithmOptions(): Array<{ - value: ASSIGNMENT_ALGORITHM; - description: string; - }> { - if (!this.isMoocletEnabled()) { - console.warn('Mooclet API is disabled in the environment configuration.'); - return []; - } - - const supportedMoocletAlgorithms = SUPPORTED_MOOCLET_ALGORITHMS as ASSIGNMENT_ALGORITHM[]; - return supportedMoocletAlgorithms.map((algorithmName) => ({ - value: algorithmName, - description: `Adaptive Experiment Algorithm: ${algorithmName}`, - })); - } - - // ============================================================================ - // ts_configurable Policy Parameters Helpers - // ============================================================================ - - isTSConfigurable(algorithm: ASSIGNMENT_ALGORITHM): boolean { - return algorithm === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE; - } - - /** - * Get default TS Configurable parameters instance. - * Provides a single source of truth for default parameter values. - */ - getTSConfigurableDefaults(): MoocletTSConfigurablePolicyParametersDTO { - return new MoocletTSConfigurablePolicyParametersDTO(); - } - - /** - * Validate mooclet policy parameters against the TS Configurable schema. - * Returns an Observable of validation errors (empty array if valid). - */ - validateTSConfigurablePolicyParameters(jsonValue: any): Observable { - const ValidatorClass = MOOCLET_POLICY_SCHEMA_MAP[ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE]; - - const plainDTO = { - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - ...jsonValue, - }; - const DTOInstance = plainToInstance(ValidatorClass, plainDTO); - // allowing unknown values until we can use nested validation properly - return from(validate(DTOInstance, { forbidUnknownValues: false })); - } - - /** - * Derive editable form values from existing or default parameters. - * Extracts only the user-configurable fields for form initialization. - */ - deriveEditableParametersForTSConfigurable( - existingParams?: MoocletTSConfigurablePolicyParametersDTO - ): EditableTSConfigurablePolicyParameters { - const defaults = this.getTSConfigurableDefaults(); - const source = existingParams || defaults; - - return { - batch_size: source.batch_size, - uniform_threshold: source.uniform_threshold, - tspostdiff_thresh: source.tspostdiff_thresh, - }; - } - - /** - * Build complete DTO from editable parameters. - * Combines user-provided values with system-generated and default values. - * Note: outcome_variable_name is now generated server-side during experiment creation. - */ - buildTSConfigurablePolicyParametersDTO( - editableParams: EditableTSConfigurablePolicyParameters - ): MoocletTSConfigurablePolicyParametersDTO { - const defaults = this.getTSConfigurableDefaults(); - return { - // User-configurable fields - batch_size: editableParams.batch_size, - uniform_threshold: editableParams.uniform_threshold, - tspostdiff_thresh: editableParams.tspostdiff_thresh, - // System-managed fields - assignmentAlgorithm: ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE, - max_rating: defaults.max_rating, - min_rating: defaults.min_rating, - } as MoocletTSConfigurablePolicyParametersDTO; - } - - /** - * Get field validators configuration for TS Configurable parameters. - * Provides validation rules based on default minimum values. - */ - getTSConfigurableFieldValidators(): Record { - const defaults = this.getTSConfigurableDefaults(); - - return { - batch_size: [ - Validators.required, - Validators.min(defaults.batch_size), - Validators.max(DEFAULT_MAX_NUMBER_INPUT), - CommonFormHelpersService.integerValidator(), - ], - uniform_threshold: [ - Validators.required, - Validators.min(defaults.uniform_threshold), - Validators.max(DEFAULT_MAX_NUMBER_INPUT), - CommonFormHelpersService.integerValidator(), - ], - tspostdiff_thresh: [Validators.required, Validators.min(defaults.tspostdiff_thresh), Validators.max(1.0)], - }; - } - - /** - * Get field validators for per-condition prior success/failure inputs used in the prior editor. - */ - getPriorFieldValidators(): Record { - const priorDefault = 1; - return { - successes: [ - Validators.required, - Validators.min(priorDefault), - Validators.max(DEFAULT_MAX_NUMBER_INPUT), - CommonFormHelpersService.integerValidator(), - ], - failures: [ - Validators.required, - Validators.min(priorDefault), - Validators.max(DEFAULT_MAX_NUMBER_INPUT), - CommonFormHelpersService.integerValidator(), - ], - }; - } -} diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts index b8117f1295..0f82d15cbf 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.spec.ts @@ -1649,26 +1649,32 @@ describe('ExperimentEffects', () => { describe('fetchRewardsDataForExperiment$', () => { it('should dispatch actionFetchRewardsDataForExperimentSuccess on successful fetch', fakeAsync(() => { const experimentId = 'test-experiment-123'; - const mockRewardsSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; + const mockRewardsSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + { + conditionCode: 'Treatment', + successes: 8, + failures: 7, + total: 15, + successRate: '53.3%', + order: 1, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 30, + warmupThreshold: 0, + batchSize: 1, + }; - experimentDataService.fetchMoocletRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); + experimentDataService.fetchRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); const expectedAction = actionFetchRewardsDataForExperimentSuccess({ experimentId, @@ -1688,7 +1694,7 @@ describe('ExperimentEffects', () => { const experimentId = 'test-experiment-123'; const error = new Error('API error'); - experimentDataService.fetchMoocletRewardsDataForExperiment = jest.fn().mockReturnValue(throwError(error)); + experimentDataService.fetchRewardsDataForExperiment = jest.fn().mockReturnValue(throwError(error)); const expectedAction = actionFetchRewardsDataForExperimentFailure({ error }); @@ -1705,10 +1711,10 @@ describe('ExperimentEffects', () => { const experimentId = 'test-experiment-456'; const mockRewardsSummary = []; - experimentDataService.fetchMoocletRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); + experimentDataService.fetchRewardsDataForExperiment = jest.fn().mockReturnValue(of(mockRewardsSummary)); service.fetchRewardsDataForExperiment$.subscribe(() => { - expect(experimentDataService.fetchMoocletRewardsDataForExperiment).toHaveBeenCalledWith(experimentId); + expect(experimentDataService.fetchRewardsDataForExperiment).toHaveBeenCalledWith(experimentId); }); actions$.next(actionFetchRewardsDataForExperiment({ experimentId })); diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.ts index a881f06bd1..02fb6e26e8 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.effects.ts @@ -779,14 +779,10 @@ export class ExperimentEffects { fetchRewardsDataForExperiment$ = createEffect(() => this.actions$.pipe( ofType(experimentAction.actionFetchRewardsDataForExperiment), - map((action) => action.experimentId), - switchMap((experimentId) => - this.experimentDataService.fetchMoocletRewardsDataForExperiment(experimentId).pipe( + switchMap(({ experimentId }) => + this.experimentDataService.fetchRewardsDataForExperiment(experimentId).pipe( map((rewardsSummary) => - experimentAction.actionFetchRewardsDataForExperimentSuccess({ - experimentId, - rewardsSummary, - }) + experimentAction.actionFetchRewardsDataForExperimentSuccess({ experimentId, rewardsSummary }) ), catchError((error) => of(experimentAction.actionFetchRewardsDataForExperimentFailure({ error }))) ) diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts index 7337200bb3..edd521c118 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.model.ts @@ -18,8 +18,6 @@ import { PAYLOAD_TYPE, CONDITION_ORDER, ASSIGNMENT_ALGORITHM, - MoocletTSConfigurablePolicyParametersDTO, - MoocletPolicyParametersDTO, REPEATED_MEASURE, SEGMENT_TYPE, IEnrollmentCompleteCondition, @@ -29,6 +27,13 @@ import { ExperimentRewardsSummary, } from 'upgrade_types'; import { Segment } from '../../segments/store/segments.model'; + +export interface ThompsonSamplingConfigDTO { + warmupThreshold?: number; + minimumDrawDifference?: number; + batchSize?: number; + priors?: Record; +} export { CONSISTENCY_RULE, ASSIGNMENT_UNIT, @@ -285,7 +290,7 @@ export interface Experiment { experimentSegmentExclusion: SegmentNew[]; groupSatisfied?: number; backendVersion: string; - moocletPolicyParameters?: MoocletTSConfigurablePolicyParametersDTO; + thompsonSamplingConfig?: ThompsonSamplingConfigDTO; } export interface ParticipantsMember { @@ -511,7 +516,7 @@ export interface DraftExperimentRequest { endOn?: string; revertTo?: string; backendVersion?: string; - moocletPolicyParameters?: MoocletPolicyParametersDTO; + thompsonSamplingConfig?: ThompsonSamplingConfigDTO; rewardMetricKey?: string; // Arrays that can be empty for drafts @@ -594,12 +599,16 @@ export const EXPERIMENT_OVERVIEW_LABELS = { TAGS: 'Tags', } as const; -export const TS_CONFIGURABLE_OVERVIEW_PARAM_LABELS = { +export const THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS = { BATCH_SIZE: 'home.new-experiment.design.ts-configurable-policy.batch-size.label.text', - UNIFORM_THRESHOLD: 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text', - TSPOSTDIFF_THRESH: 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text', + WARMUP_THRESHOLD: 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text', + MINIMUM_DRAW_DIFFERENCE: 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text', }; +// Shared by the conditions table and the enrollment expandable row — both hide/tooltip the weight +// column the same way for adaptive experiments, and must reference the same translation key. +export const THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY = 'experiments.details.conditions.weight-adaptive-tooltip.text'; + export const EXPERIMENT_ROOT_DISPLAYED_COLUMNS = Object.values(EXPERIMENT_ROOT_COLUMN_NAMES); export interface ExperimentState { diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.reducer.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.reducer.spec.ts index cd1014dbca..14a81f2c21 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.reducer.spec.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.reducer.spec.ts @@ -786,24 +786,30 @@ describe('ExperimentsReducer', () => { previousState.isLoadingRewardsSummary = true; previousState.rewardsSummaries = {}; - const mockRewardsSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; + const mockRewardsSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + { + conditionCode: 'Treatment', + successes: 8, + failures: 7, + total: 15, + successRate: '53.3%', + order: 1, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 30, + warmupThreshold: 0, + batchSize: 1, + }; const testAction: Action = actionFetchRewardsDataForExperimentSuccess({ experimentId: 'exp-123', @@ -818,32 +824,44 @@ describe('ExperimentsReducer', () => { }); it('action "actionFetchRewardsDataForExperimentSuccess" should update existing rewards summary', () => { - const oldSummary = [ - { - conditionCode: 'Control', - successes: 5, - failures: 5, - total: 10, - successRate: '50.0%', - order: 0, - }, - ]; + const oldSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 5, + failures: 5, + total: 10, + successRate: '50.0%', + order: 0, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 10, + warmupThreshold: 0, + batchSize: 1, + }; const previousState = { ...initialState }; previousState.rewardsSummaries = { 'exp-123': oldSummary, }; - const newSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - ]; + const newSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 15, + warmupThreshold: 0, + batchSize: 1, + }; const testAction: Action = actionFetchRewardsDataForExperimentSuccess({ experimentId: 'exp-123', @@ -857,8 +875,20 @@ describe('ExperimentsReducer', () => { }); it('action "actionFetchRewardsDataForExperimentSuccess" should preserve other experiment summaries', () => { - const summary1 = [{ conditionCode: 'A', successes: 1, failures: 0, total: 1, successRate: '100%', order: 0 }]; - const summary2 = [{ conditionCode: 'B', successes: 2, failures: 0, total: 2, successRate: '100%', order: 0 }]; + const summary1 = { + conditions: [{ conditionCode: 'A', successes: 1, failures: 0, total: 1, successRate: '100%', order: 0 }], + pendingRewardsCount: 0, + totalRewardCount: 1, + warmupThreshold: 0, + batchSize: 1, + }; + const summary2 = { + conditions: [{ conditionCode: 'B', successes: 2, failures: 0, total: 2, successRate: '100%', order: 0 }], + pendingRewardsCount: 0, + totalRewardCount: 2, + warmupThreshold: 0, + batchSize: 1, + }; const previousState = { ...initialState }; previousState.rewardsSummaries = { @@ -891,9 +921,13 @@ describe('ExperimentsReducer', () => { }); it('action "actionFetchRewardsDataForExperimentFailure" should not modify rewardsSummaries', () => { - const existingSummary = [ - { conditionCode: 'A', successes: 1, failures: 0, total: 1, successRate: '100%', order: 0 }, - ]; + const existingSummary = { + conditions: [{ conditionCode: 'A', successes: 1, failures: 0, total: 1, successRate: '100%', order: 0 }], + pendingRewardsCount: 0, + totalRewardCount: 1, + warmupThreshold: 0, + batchSize: 1, + }; const previousState = { ...initialState }; previousState.rewardsSummaries = { diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selector.spec.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selector.spec.ts index cbe514a3a5..970b89251c 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selector.spec.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selector.spec.ts @@ -823,24 +823,30 @@ describe('Experiments Selectors', () => { describe('#selectRewardsDataForSelectedExperiment', () => { it('should return rewards summary for selected experiment', () => { - const mockRewardsSummary = [ - { - conditionCode: 'Control', - successes: 10, - failures: 5, - total: 15, - successRate: '66.7%', - order: 0, - }, - { - conditionCode: 'Treatment', - successes: 8, - failures: 7, - total: 15, - successRate: '53.3%', - order: 1, - }, - ]; + const mockRewardsSummary = { + conditions: [ + { + conditionCode: 'Control', + successes: 10, + failures: 5, + total: 15, + successRate: '66.7%', + order: 0, + }, + { + conditionCode: 'Treatment', + successes: 8, + failures: 7, + total: 15, + successRate: '53.3%', + order: 1, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 30, + warmupThreshold: 0, + batchSize: 1, + }; const state = { ...mockState, @@ -862,24 +868,30 @@ describe('Experiments Selectors', () => { rewardsSummaries: {}, }; - const expectedDefault = [ - { - conditionCode: 'control', - failures: 0, - order: 1, - successRate: 'n/a', - successes: 0, - total: 0, - }, - { - conditionCode: 'variant', - failures: 0, - order: 2, - successRate: 'n/a', - successes: 0, - total: 0, - }, - ]; + const expectedDefault = { + conditions: [ + { + conditionCode: 'control', + failures: 0, + order: 1, + successRate: 'n/a', + successes: 0, + total: 0, + }, + { + conditionCode: 'variant', + failures: 0, + order: 2, + successRate: 'n/a', + successes: 0, + total: 0, + }, + ], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }; const selectedExperiment = state.experiments.find((exp) => exp.id === '1f12cd8f-7ff9-4731-a4eb-7104918ed252'); @@ -888,11 +900,17 @@ describe('Experiments Selectors', () => { expect(result).toEqual(expectedDefault); }); - it('should return empty array when experiment is null', () => { + it('should return an empty summary when experiment is null', () => { const state = { ...mockState, rewardsSummaries: { - 'some-id': [], + 'some-id': { + conditions: [], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }, }, }; @@ -900,7 +918,13 @@ describe('Experiments Selectors', () => { const result = selectRewardsDataForSelectedExperiment.projector(selectedExperiment, state); - expect(result).toEqual([]); + expect(result).toEqual({ + conditions: [], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }); }); }); diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts index 49ccbe1f1b..7ef30cfe10 100644 --- a/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/store/experiments.selectors.ts @@ -26,7 +26,7 @@ import { ExperimentRewardsSummary, } from 'upgrade_types'; import { determineWeightingMethod, isWeightSumValid } from '../condition-helper.service'; -import { formatTSConfigurablePolicyParamDetails } from '../mooclet-helper.service'; +import { formatThompsonSamplingConfigDetails } from '../thompson-sampling-helper.service'; import { KeyValueFormat } from '@shared-component-lib/common-section-card-overview-details/common-section-card-overview-details.component'; import { DetailsPageError } from '@shared-component-lib/common-page-error/common-page-error.model'; @@ -210,9 +210,8 @@ export const selectExperimentOverviewDetails = createSelector(selectSelectedExpe }; // Add policy parameters if they exist - if (experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE) { - details[EXPERIMENT_OVERVIEW_LABELS.ADAPTIVE_ALGORITHM_PARAMETERS] = - formatTSConfigurablePolicyParamDetails(experiment); + if (experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + details[EXPERIMENT_OVERVIEW_LABELS.ADAPTIVE_ALGORITHM_PARAMETERS] = formatThompsonSamplingConfigDetails(experiment); } // Always add tags at the end @@ -361,23 +360,33 @@ export const selectRewardsDataForSelectedExperiment = createSelector( selectSelectedExperiment, selectExperimentState, (experiment: ExperimentVM, state: ExperimentState): ExperimentRewardsSummary => { + const emptySummary: ExperimentRewardsSummary = { + conditions: [], + pendingRewardsCount: 0, + totalRewardCount: 0, + warmupThreshold: 0, + batchSize: 1, + }; + if (!experiment || !experiment.id) { - return []; + return emptySummary; } const rewardsSummary = state.rewardsSummaries[experiment.id]; if (!rewardsSummary) { - const defaultRewardsSummary: ExperimentRewardsSummary = experiment.conditions.map((condition) => { - return { + return { + ...emptySummary, + conditions: experiment.conditions.map((condition) => ({ conditionCode: condition.conditionCode, successes: 0, failures: 0, total: 0, successRate: 'n/a', order: condition.order, - }; - }); - return defaultRewardsSummary; + })), + warmupThreshold: experiment.thompsonSamplingConfig?.warmupThreshold ?? 0, + batchSize: experiment.thompsonSamplingConfig?.batchSize ?? 1, + }; } return rewardsSummary; @@ -466,7 +475,7 @@ export const selectDisabledExperimentFields = createSelector(selectSelectedExper } if ([EXPERIMENT_STATE.COMPLETED, EXPERIMENT_STATE.ARCHIVED].includes(state)) { - return [...baseRestrictedFields, 'moocletPolicyParameters']; + return [...baseRestrictedFields, 'thompsonSamplingConfig']; } return baseRestrictedFields; diff --git a/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts b/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts new file mode 100644 index 0000000000..46e9a16c18 --- /dev/null +++ b/packages/frontend/projects/upgrade/src/app/core/experiments/thompson-sampling-helper.service.ts @@ -0,0 +1,110 @@ +import { Injectable } from '@angular/core'; +import { ValidatorFn, Validators } from '@angular/forms'; +import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; +import { + ExperimentVM, + THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS, + ThompsonSamplingConfigDTO, +} from './store/experiments.model'; +import { BullettedListKeyValueFormat } from '@shared-component-lib/common-section-card-overview-details/common-section-card-overview-details.component'; +import { CommonFormHelpersService } from '../../shared/services/common-form-helpers.service'; + +const DEFAULT_MAX_NUMBER_INPUT = 1000000; + +export interface EditableThompsonSamplingConfig { + batchSize: number; + warmupThreshold: number; + minimumDrawDifference: number; +} + +export function isThompsonSamplingExperiment(experiment: ExperimentVM): boolean { + return experiment?.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; +} + +export function formatThompsonSamplingConfigDetails( + experiment: ExperimentVM +): BullettedListKeyValueFormat[] | undefined { + if (experiment?.assignmentAlgorithm !== ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { + return undefined; + } + + const config = experiment.thompsonSamplingConfig; + return [ + { labelKey: THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS.BATCH_SIZE, value: config?.batchSize }, + { labelKey: THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS.WARMUP_THRESHOLD, value: config?.warmupThreshold }, + { + labelKey: THOMPSON_SAMPLING_OVERVIEW_PARAM_LABELS.MINIMUM_DRAW_DIFFERENCE, + value: config?.minimumDrawDifference?.toFixed(1), + }, + ]; +} + +@Injectable({ + providedIn: 'root', +}) +export class ThompsonSamplingHelperService { + isThompsonSamplingAlgorithm(algorithm: ASSIGNMENT_ALGORITHM): boolean { + return algorithm === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING; + } + + isThompsonSamplingExperiment(experiment: ExperimentVM): boolean { + return isThompsonSamplingExperiment(experiment); + } + + getDefaults(): EditableThompsonSamplingConfig { + return { batchSize: 1, warmupThreshold: 0, minimumDrawDifference: 0 }; + } + + deriveEditableParameters(existing?: ThompsonSamplingConfigDTO): EditableThompsonSamplingConfig { + const defaults = this.getDefaults(); + return { + batchSize: existing?.batchSize ?? defaults.batchSize, + warmupThreshold: existing?.warmupThreshold ?? defaults.warmupThreshold, + minimumDrawDifference: existing?.minimumDrawDifference ?? defaults.minimumDrawDifference, + }; + } + + buildConfig(editableParams: EditableThompsonSamplingConfig): ThompsonSamplingConfigDTO { + return { + batchSize: editableParams.batchSize, + warmupThreshold: editableParams.warmupThreshold, + minimumDrawDifference: editableParams.minimumDrawDifference, + }; + } + + getFieldValidators(): Record { + return { + batchSize: [ + Validators.required, + Validators.min(1), + Validators.max(DEFAULT_MAX_NUMBER_INPUT), + CommonFormHelpersService.integerValidator(), + ], + warmupThreshold: [ + Validators.required, + Validators.min(0), + Validators.max(DEFAULT_MAX_NUMBER_INPUT), + CommonFormHelpersService.integerValidator(), + ], + minimumDrawDifference: [Validators.required, Validators.min(0), Validators.max(1.0)], + }; + } + + getPriorFieldValidators(): Record { + const minValue = 1; + return { + successes: [ + Validators.required, + Validators.min(minValue), + Validators.max(DEFAULT_MAX_NUMBER_INPUT), + CommonFormHelpersService.integerValidator(), + ], + failures: [ + Validators.required, + Validators.min(minValue), + Validators.max(DEFAULT_MAX_NUMBER_INPUT), + CommonFormHelpersService.integerValidator(), + ], + }; + } +} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts index 1e7a22dc5c..d8627614bd 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/edit-condition-prior-modal/edit-condition-prior-modal.component.ts @@ -11,11 +11,12 @@ import { Observable, combineLatest, map, startWith } from 'rxjs'; import { CommonLearnMoreLinkComponent, CommonModalComponent } from '@shared-component-lib'; import { CommonFormHelpersService } from '../../../../../shared/services/common-form-helpers.service'; import { CommonModalConfig } from '@shared-component-lib/common-modal/common-modal.types'; -import { MoocletExperimentHelperService } from '../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../core/experiments/thompson-sampling-helper.service'; import { Prior } from 'upgrade_types'; import { SharedModule } from '../../../../../shared/shared.module'; export interface ConditionPriorUpdate { + conditionId: string; conditionCode: string; successes: number; failures: number; @@ -51,7 +52,7 @@ export class EditConditionPriorModalComponent implements OnInit { public config: CommonModalConfig<{ conditions: ConditionPriorUpdate[] }>, public dialogRef: MatDialogRef, private readonly formBuilder: FormBuilder, - private readonly moocletHelperService: MoocletExperimentHelperService + private readonly thompsonSamplingHelperService: ThompsonSamplingHelperService ) {} ngOnInit(): void { @@ -60,12 +61,12 @@ export class EditConditionPriorModalComponent implements OnInit { } createPriorForm(): void { - const validators = this.moocletHelperService.getPriorFieldValidators(); + const validators = this.thompsonSamplingHelperService.getPriorFieldValidators(); const conditionsFormArray = this.formBuilder.array( this.conditions.map((condition) => this.formBuilder.group({ - conditionCode: [condition.conditionCode], + conditionId: [condition.conditionId], successes: [condition.successes, validators.successes], failures: [condition.failures, validators.failures], }) @@ -96,8 +97,8 @@ export class EditConditionPriorModalComponent implements OnInit { if (this.priorForm.valid) { const result: Record = {}; this.conditionsFormArray.controls.forEach((control) => { - const conditionCode = control.get('conditionCode')?.value; - result[conditionCode] = { + const conditionId = control.get('conditionId')?.value; + result[conditionId] = { success: control.get('successes')?.value, failure: control.get('failures')?.value, }; diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.html index 85acb7ff5a..c18d95e2d1 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.html @@ -14,19 +14,19 @@ {{ 'home.new-experiment.design.ts-configurable-policy.batch-size.label.text' | translate }} - + {{ 'home.new-experiment.design.ts-configurable-policy.batch-size.hint.text' | translate }} - @if (policyForm.get('batch_size')?.hasError('min')) { + @if (policyForm.get('batchSize')?.hasError('min')) { {{ 'home.new-experiment.design.ts-configurable-policy.batch-size.min-error.text' | translate }} - } @if (policyForm.get('batch_size')?.hasError('max')) { + } @if (policyForm.get('batchSize')?.hasError('max')) { {{ 'home.new-experiment.design.ts-configurable-policy.general.max-error.text' | translate }} - } @if (policyForm.get('batch_size')?.hasError('integer')) { + } @if (policyForm.get('batchSize')?.hasError('integer')) { {{ 'home.new-experiment.design.ts-configurable-policy.general.integer-error.text' | translate }} @@ -37,19 +37,19 @@ {{ 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text' | translate }} - + {{ 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.hint.text' | translate }} - @if (policyForm.get('uniform_threshold')?.hasError('min')) { + @if (policyForm.get('warmupThreshold')?.hasError('min')) { {{ 'home.new-experiment.design.ts-configurable-policy.uniform-threshold.min-error.text' | translate }} - } @if (policyForm.get('uniform_threshold')?.hasError('max')) { + } @if (policyForm.get('warmupThreshold')?.hasError('max')) { {{ 'home.new-experiment.design.ts-configurable-policy.general.max-error.text' | translate }} - } @if (policyForm.get('uniform_threshold')?.hasError('integer')) { + } @if (policyForm.get('warmupThreshold')?.hasError('integer')) { {{ 'home.new-experiment.design.ts-configurable-policy.general.integer-error.text' | translate }} @@ -63,7 +63,7 @@ {{ 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.hint.text' | translate }} - @if (policyForm.get('tspostdiff_thresh')?.hasError('min')) { + @if (policyForm.get('minimumDrawDifference')?.hasError('min')) { {{ 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.min-error.text' | translate }} - } @if (policyForm.get('tspostdiff_thresh')?.hasError('max')) { + } @if (policyForm.get('minimumDrawDifference')?.hasError('max')) { {{ 'home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.max-error.text' | translate }} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts index cc28d0b6c7..0217b52ee8 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component.ts @@ -1,26 +1,14 @@ import { Component, EventEmitter, inject, Input, OnDestroy, OnInit, Output } from '@angular/core'; import { FormBuilder, FormGroup, ReactiveFormsModule } from '@angular/forms'; - import { MatFormFieldModule } from '@angular/material/form-field'; import { MatInputModule } from '@angular/material/input'; import { TranslateModule } from '@ngx-translate/core'; +import { BehaviorSubject, map, Observable, startWith, Subject, Subscription } from 'rxjs'; +import { ThompsonSamplingConfigDTO } from '../../../../../../core/experiments/store/experiments.model'; import { - BehaviorSubject, - debounceTime, - from, - map, - Observable, - startWith, - Subject, - Subscription, - switchMap, -} from 'rxjs'; -import { ValidationError } from 'class-validator'; -import { MoocletTSConfigurablePolicyParametersDTO } from 'upgrade_types'; -import { - EditableTSConfigurablePolicyParameters, - MoocletExperimentHelperService, -} from '../../../../../../core/experiments/mooclet-helper.service'; + EditableThompsonSamplingConfig, + ThompsonSamplingHelperService, +} from '../../../../../../core/experiments/thompson-sampling-helper.service'; import isEqual from 'lodash.isequal'; @Component({ @@ -31,27 +19,26 @@ import isEqual from 'lodash.isequal'; styleUrl: './ts-configurable-policy-parameters-form.component.scss', }) export class TsConfigurablePolicyParametersFormComponent implements OnInit, OnDestroy { - @Input() existingPolicyParams?: MoocletTSConfigurablePolicyParametersDTO; - @Input() disabled = false; // Disable all form fields when true - @Output() parametersChange = new EventEmitter(); + @Input() existingPolicyParams?: ThompsonSamplingConfigDTO; + @Input() disabled = false; + @Output() parametersChange = new EventEmitter(); @Output() validationChange = new EventEmitter(); @Output() formChanged = new EventEmitter(); private readonly formBuilder = inject(FormBuilder); - private readonly moocletExperimentHelperService = inject(MoocletExperimentHelperService); + private readonly thompsonSamplingHelperService = inject(ThompsonSamplingHelperService); policyForm: FormGroup; - validationErrors$ = new BehaviorSubject([]); + validationErrors$ = new BehaviorSubject([]); isInitialFormValueChanged$: Observable; - initialFormValue: EditableTSConfigurablePolicyParameters; - formValueChanges$ = new Subject(); + initialFormValue: EditableThompsonSamplingConfig; + formValueChanges$ = new Subject(); subscriptions = new Subscription(); ngOnInit(): void { this.initializeFormValues(); this.createForm(); - // Disable form if disabled input is true if (this.disabled) { this.policyForm.disable(); } @@ -66,73 +53,48 @@ export class TsConfigurablePolicyParametersFormComponent implements OnInit, OnDe } private initializeFormValues(): void { - // Delegate to service to derive initial form values from existing or default parameters - this.initialFormValue = this.moocletExperimentHelperService.deriveEditableParametersForTSConfigurable( - this.existingPolicyParams - ); + this.initialFormValue = this.thompsonSamplingHelperService.deriveEditableParameters(this.existingPolicyParams); } private createForm(): void { const params = this.initialFormValue; - const validators = this.moocletExperimentHelperService.getTSConfigurableFieldValidators(); + const validators = this.thompsonSamplingHelperService.getFieldValidators(); this.policyForm = this.formBuilder.group({ - batch_size: [params.batch_size, validators.batch_size], - uniform_threshold: [params.uniform_threshold, validators.uniform_threshold], - tspostdiff_thresh: [params.tspostdiff_thresh, validators.tspostdiff_thresh], + batchSize: [params.batchSize, validators.batchSize], + warmupThreshold: [params.warmupThreshold, validators.warmupThreshold], + minimumDrawDifference: [params.minimumDrawDifference, validators.minimumDrawDifference], }); } private setupValidation(): void { - // Set up validation pipeline with debounce - this.subscriptions.add( - this.formValueChanges$ - .pipe( - debounceTime(300), - switchMap((formValue) => this.validateParameters(formValue)) - ) - .subscribe((errors) => { - this.validationErrors$.next(errors); - this.emitValidationState(errors); - }) - ); - - // Emit validation state immediately when Angular form validity changes this.subscriptions.add( this.policyForm.statusChanges.subscribe(() => { - this.emitValidationState(this.validationErrors$.value); + this.emitValidationState(); }) ); } - private emitValidationState(backendErrors: ValidationError[]): void { - // Treat disabled form as valid; otherwise require Angular + backend validation to pass - const formDisabled = this.policyForm.disabled; - const isValid = formDisabled || (this.policyForm.valid && backendErrors.length === 0); + private emitValidationState(): void { + const isValid = this.policyForm.disabled || this.policyForm.valid; this.validationChange.emit(isValid); } private listenToFormChanges(): void { this.subscriptions.add( - this.policyForm.valueChanges.subscribe((formValue: EditableTSConfigurablePolicyParameters) => { + this.policyForm.valueChanges.subscribe((formValue: EditableThompsonSamplingConfig) => { this.emitFormValueChanges(formValue); }) ); - // Trigger initial validation and emit initial form state this.emitFormValueChanges(this.policyForm.value); - // Emit initial validation state considering Angular form validity - this.emitValidationState(this.validationErrors$.value); + this.emitValidationState(); } private listenForIsInitialFormValueChanged() { this.isInitialFormValueChanged$ = this.policyForm.valueChanges.pipe( startWith(this.policyForm.value), - map(() => { - // Compare form values with initial parameters - const currentValues = this.policyForm.value; - return !isEqual(currentValues, this.initialFormValue); - }) + map(() => !isEqual(this.policyForm.value, this.initialFormValue)) ); this.subscriptions.add( this.isInitialFormValueChanged$.subscribe((hasChanged) => { @@ -141,20 +103,8 @@ export class TsConfigurablePolicyParametersFormComponent implements OnInit, OnDe ); } - private validateParameters(formValue: EditableTSConfigurablePolicyParameters): Observable { - const completeParams = this.buildCompletePolicyParametersDTO(formValue); - return from(this.moocletExperimentHelperService.validateTSConfigurablePolicyParameters(completeParams)); - } - - private emitFormValueChanges(formValue: EditableTSConfigurablePolicyParameters): void { + private emitFormValueChanges(formValue: EditableThompsonSamplingConfig): void { this.formValueChanges$.next(formValue); - this.parametersChange.emit(this.buildCompletePolicyParametersDTO(formValue)); - } - - private buildCompletePolicyParametersDTO( - formValue: EditableTSConfigurablePolicyParameters - ): MoocletTSConfigurablePolicyParametersDTO { - // Delegate DTO assembly to service - return this.moocletExperimentHelperService.buildTSConfigurablePolicyParametersDTO(formValue); + this.parametersChange.emit(this.thompsonSamplingHelperService.buildConfig(formValue)); } } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.html index 30c92b84f4..22946fa96d 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.html @@ -159,16 +159,13 @@ {{ 'experiments.upsert-experiment-modal.stratification-factor-hint.text' | translate }} - } - - - @if (assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE) { + } @if (assignmentAlgorithmValue === ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING) { } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts index eefc94f756..d0b18d77ab 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/modals/upsert-experiment-modal/upsert-experiment-modal.component.ts @@ -36,14 +36,14 @@ import { POST_EXPERIMENT_RULE, ASSIGNMENT_ALGORITHM_DISPLAY_MAP, EXPERIMENT_TYPE, - MoocletPolicyParametersDTO, } from 'upgrade_types'; import { CommonModalConfig } from '@shared-component-lib/common-modal/common-modal.types'; import { StratificationFactorsService } from '../../../../../core/stratification-factors/stratification-factors.service'; import { ENV, Environment } from '../../../../../../environments/environment-types'; import { SharedModule } from '../../../../../shared/shared.module'; import { TsConfigurablePolicyParametersFormComponent } from './ts-configurable-policy-parameters-form/ts-configurable-policy-parameters-form.component'; -import { MoocletExperimentHelperService } from '../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../core/experiments/thompson-sampling-helper.service'; +import { ThompsonSamplingConfigDTO } from '../../../../../core/experiments/store/experiments.model'; @Component({ selector: 'upsert-experiment-modal', @@ -94,20 +94,20 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { CommonTagInputType = CommonTagInputType; /** - * Mooclet policy parameters state management + * Thompson Sampling config state management * - * This property holds the current mooclet policy parameters and is updated in several scenarios: - * 1. EDIT mode initialization: Set from sourceExperiment.moocletPolicyParameters if present - * 2. Algorithm changes: Set to defaults when switching TO a mooclet algorithm (if not already set) - * 3. Algorithm changes: Cleared when switching FROM mooclet to non-mooclet algorithm + * This property holds the current Thompson Sampling config and is updated in several scenarios: + * 1. EDIT mode initialization: Set from sourceExperiment.thompsonSamplingConfig if present + * 2. Algorithm changes: Set to defaults when switching TO Thompson Sampling (if not already set) + * 3. Algorithm changes: Cleared when switching FROM Thompson Sampling to another algorithm * 4. Child form events: Updated when user modifies parameters in the child form * * The child form receives existing params via async pipe but emits updates through event handlers. */ - moocletPolicyParametersFormValue: MoocletPolicyParametersDTO; - isMoocletFormValid$ = new BehaviorSubject(true); - isMoocletFormChanged$ = new BehaviorSubject(false); - isMoocletFormDisabled = false; // Set to true when experiment is in CANCELLED state + thompsonSamplingConfigFormValue: ThompsonSamplingConfigDTO; + isTSFormValid$ = new BehaviorSubject(true); + isTSFormChanged$ = new BehaviorSubject(false); + isTSFormDisabled = false; // Set to true when experiment is in CANCELLED state // Enum references for template UPSERT_EXPERIMENT_ACTION = UPSERT_EXPERIMENT_ACTION; @@ -180,6 +180,10 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { description: 'experiments.upsert-experiment-modal.assignment-algorithm-stratified-random-sampling-description.text', }, + { + value: ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + description: 'experiments.upsert-experiment-modal.assignment-algorithm-thompson-sampling-description.text', + }, ]; constructor( @@ -189,7 +193,7 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { private readonly formBuilder: FormBuilder, private readonly experimentService: ExperimentService, private readonly stratificationFactorsService: StratificationFactorsService, - private readonly moocletExperimentHelperService: MoocletExperimentHelperService, + private readonly thompsonSamplingHelperService: ThompsonSamplingHelperService, public dialogRef: MatDialogRef, @Inject(ENV) private readonly environment: Environment ) { @@ -199,15 +203,13 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { description: 'Condition will be assigned within subjects (e.g., participant sees multiple conditions).', }); } - // Delegate to service to get supported mooclet algorithm options - const moocletAlgorithmOptions = this.moocletExperimentHelperService.getSupportedMoocletAlgorithmOptions(); - this.assignmentAlgorithms.push(...moocletAlgorithmOptions); } ngOnInit(): void { this.experimentService.fetchContextMetaData(); this.stratificationFactorsService.fetchStratificationFactors(true); this.createExperimentForm(); + this.updateAssignmentAlgorithms(); // Set up subscriptions this.listenForContextMetaData(); @@ -235,9 +237,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { this.experimentForm.get(fieldName)?.disable(); }); - // Handle moocletPolicyParameters separately (it's a child form component) - if (fieldsToDisable.includes('moocletPolicyParameters')) { - this.isMoocletFormDisabled = true; + if (fieldsToDisable.includes('thompsonSamplingConfig')) { + this.isTSFormDisabled = true; } }) ); @@ -268,9 +269,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { // Initialize consistency rules state based on initial unit of assignment this.initializeConsistencyRules(initialValues.unitOfAssignment); - // Initialize moocletPolicyParameters in EDIT mode to preserve existing values - if (action === UPSERT_EXPERIMENT_ACTION.EDIT && sourceExperiment?.moocletPolicyParameters) { - this.moocletPolicyParametersFormValue = sourceExperiment.moocletPolicyParameters; + if (action === UPSERT_EXPERIMENT_ACTION.EDIT && sourceExperiment?.thompsonSamplingConfig) { + this.thompsonSamplingConfigFormValue = sourceExperiment.thompsonSamplingConfig; } this.initialFormValues$.next(this.experimentForm.getRawValue() as ExperimentFormData); @@ -365,15 +365,15 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { // Button is disabled if: // 1. Currently loading, OR - // 2. Changes are required and none detected in primary or mooclet form, OR - // 3. Either primary or mooclet form is invalid + // 2. Changes are required and none detected in primary or Thompson Sampling form, OR + // 3. Either primary or Thompson Sampling form is invalid listenForPrimaryButtonDisabled() { this.isPrimaryButtonDisabled$ = this.isLoadingUpsertExperiment$.pipe( - combineLatestWith(this.isInitialFormValueChanged$, this.isMoocletFormValid$, this.isMoocletFormChanged$), - map(([isLoading, isInitialFormValueChanged, isMoocletFormValid, isMoocletFormChanged]) => { + combineLatestWith(this.isInitialFormValueChanged$, this.isTSFormValid$, this.isTSFormChanged$), + map(([isLoading, isInitialFormValueChanged, isTSFormValid, isTSFormChanged]) => { const changesRequiredToAllowSubmit = this.config.params.action !== UPSERT_EXPERIMENT_ACTION.DUPLICATE; - const hasChanges = isInitialFormValueChanged || isMoocletFormChanged; - const allFormsValid = this.experimentForm.valid && isMoocletFormValid; + const hasChanges = isInitialFormValueChanged || isTSFormChanged; + const allFormsValid = this.experimentForm.valid && isTSFormValid; return isLoading || (changesRequiredToAllowSubmit && !hasChanges) || !allFormsValid; }) @@ -398,8 +398,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { this.subscriptions.add( this.experimentForm.get('assignmentAlgorithm')?.valueChanges.subscribe((algorithm) => { this.validateStratificationFactorSelection(algorithm); - // Always check for mooclet algorithm changes to handle both TO and FROM mooclet transitions - this.checkForMoocletAlgorithmChange(); + // Check for Thompson Sampling changes to handle both TO and FROM transitions + this.checkForAlgorithmChange(); }) ); } @@ -481,36 +481,69 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { groupTypeControl?.updateValueAndValidity(); consistencyRuleControl?.updateValueAndValidity(); conditionOrderControl?.updateValueAndValidity(); + + // Thompson Sampling can't attribute rewards under Within-Subjects assignment (see + // updateAssignmentAlgorithms()) -- disable it going forward, and bump an already-selected + // Thompson Sampling algorithm back to Random rather than leave an invalid combination + // sitting in the form. + this.updateAssignmentAlgorithms(); + if (assignmentUnit === ASSIGNMENT_UNIT.WITHIN_SUBJECTS && this.isCurrentAlgorithmThompsonSampling) { + this.experimentForm.get('assignmentAlgorithm')?.setValue(ASSIGNMENT_ALGORITHM.RANDOM); + } }) ); } - checkForMoocletAlgorithmChange(): void { - const algorithm = this.assignmentAlgorithmValue; - - if (!this.moocletExperimentHelperService.isMoocletAlgorithm(algorithm)) { - // Clear params when switching to non-mooclet algorithm - this.moocletPolicyParametersFormValue = undefined; - } else if (this.moocletExperimentHelperService.isTSConfigurable(algorithm)) { - // Only set defaults if we don't already have params - // This preserves existing values in EDIT mode or after user has made changes - if (!this.moocletPolicyParametersFormValue) { - this.moocletPolicyParametersFormValue = this.moocletExperimentHelperService.getTSConfigurableDefaults(); + checkForAlgorithmChange(): void { + if (this.isCurrentAlgorithmThompsonSampling) { + if (!this.thompsonSamplingConfigFormValue) { + this.thompsonSamplingConfigFormValue = this.thompsonSamplingHelperService.buildConfig( + this.thompsonSamplingHelperService.getDefaults() + ); } } else { - throw new Error(`Unsupported mooclet algorithm selected: ${algorithm}`); + this.thompsonSamplingConfigFormValue = undefined; + this.isTSFormValid$.next(true); + this.isTSFormChanged$.next(false); + } + } + + setAlgorithmDisabled(algorithm: ASSIGNMENT_ALGORITHM, disabled: boolean): void { + const option = this.assignmentAlgorithms.find((alg) => alg.value === algorithm); + if (option) { + (option as any).disabled = disabled; } } updateAssignmentAlgorithms(): void { - // Disable stratified random sampling if no stratification factors are available - const stratifiedAlgorithm = this.assignmentAlgorithms.find( - (alg) => alg.value === ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING + // Switching an experiment to or from Thompson Sampling on edit isn't supported + const isEditingExistingExperiment = this.config.params.action === UPSERT_EXPERIMENT_ACTION.EDIT; + const wasThompsonSampling = + isEditingExistingExperiment && + this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm( + this.config.params.sourceExperiment?.assignmentAlgorithm + ); + + // Disable stratified random sampling if no stratification factors are available, or if editing + // an experiment that's already Thompson Sampling (switching away isn't allowed). + this.setAlgorithmDisabled( + ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING, + this.allStratificationFactors.length === 0 || wasThompsonSampling ); - if (stratifiedAlgorithm) { - (stratifiedAlgorithm as any).disabled = this.allStratificationFactors.length === 0; - } + // Disable Random for the same "already Thompson Sampling" reason as above. + this.setAlgorithmDisabled(ASSIGNMENT_ALGORITHM.RANDOM, wasThompsonSampling); + + // Thompson Sampling can't be used with Within-Subjects assignment: that assignment unit never + // stores a condition on the individual enrollment (it's tracked per-repeat instead), which is + // what Thompson Sampling's reward path reads to attribute a reward to a condition. It's also + // disabled when editing an experiment that *wasn't* created as Thompson Sampling, for the + // switch-lock reason above. + this.setAlgorithmDisabled( + ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING, + this.unitOfAssignmentValue === ASSIGNMENT_UNIT.WITHIN_SUBJECTS || + (isEditingExistingExperiment && !wasThompsonSampling) + ); } validateStratificationFactorSelection(algorithm: ASSIGNMENT_ALGORITHM): void { @@ -536,24 +569,28 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { return this.experimentForm.get('unitOfAssignment')?.value; } + get isCurrentAlgorithmThompsonSampling(): boolean { + return this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(this.assignmentAlgorithmValue); + } + /** - * Event handlers for mooclet policy parameters child form + * Event handlers for Thompson Sampling config child form * * These handlers receive events from the child form and update parent state: - * - onMoocletParametersChange: Updates moocletPolicyParameters with user changes - * - onMoocletFormValidityChange: Updates validation state for button disable logic - * - onMoocletFormChanged: Tracks if user has modified the form (for save button enable) + * - onTSConfigChange: Updates thompsonSamplingConfig with user changes + * - onTSFormValidityChange: Updates validation state for button disable logic + * - onTSFormChanged: Tracks if user has modified the form (for save button enable) */ - onMoocletParametersChange(params: MoocletPolicyParametersDTO): void { - this.moocletPolicyParametersFormValue = { ...params, assignmentAlgorithm: this.assignmentAlgorithmValue }; + onTSConfigChange(params: ThompsonSamplingConfigDTO): void { + this.thompsonSamplingConfigFormValue = params; } - onMoocletFormValidityChange(isValid: boolean): void { - this.isMoocletFormValid$.next(isValid); + onTSFormValidityChange(isValid: boolean): void { + this.isTSFormValid$.next(isValid); } - onMoocletFormChanged(hasChanged: boolean): void { - this.isMoocletFormChanged$.next(hasChanged); + onTSFormChanged(hasChanged: boolean): void { + this.isTSFormChanged$.next(hasChanged); } // --------------------------------------------------------------------------- @@ -638,8 +675,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { backendVersion: undefined, // @IsOptional - can be undefined }; - if (this.moocletPolicyParametersFormValue) { - experimentRequest.moocletPolicyParameters = this.moocletPolicyParametersFormValue; + if (this.thompsonSamplingConfigFormValue) { + experimentRequest.thompsonSamplingConfig = this.thompsonSamplingConfigFormValue; } this.experimentService.createNewExperiment(experimentRequest); @@ -690,8 +727,8 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { backendVersion: sourceExperiment.backendVersion, }; - if (this.moocletPolicyParametersFormValue) { - experimentRequest.moocletPolicyParameters = this.moocletPolicyParametersFormValue; + if (this.thompsonSamplingConfigFormValue) { + experimentRequest.thompsonSamplingConfig = this.thompsonSamplingConfigFormValue; } this.experimentService.createNewExperiment(experimentRequest); @@ -757,10 +794,10 @@ export class UpsertExperimentModalComponent implements OnInit, OnDestroy { revertTo: sourceExperiment.revertTo, }; - if (this.moocletExperimentHelperService.isMoocletAlgorithm(assignmentAlgorithm)) { - experimentRequest.moocletPolicyParameters = this.moocletPolicyParametersFormValue; + if (this.isCurrentAlgorithmThompsonSampling) { + experimentRequest.thompsonSamplingConfig = this.thompsonSamplingConfigFormValue; } else { - experimentRequest.moocletPolicyParameters = undefined; + experimentRequest.thompsonSamplingConfig = undefined; } this.experimentService.updateExperiment(experimentRequest as unknown as ExperimentVM); diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.html index 0bdadd2140..4ce16bf521 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.html @@ -34,8 +34,8 @@ [showActions]="canShowActions" [actionsDisabled]="vm.restriction.isDisabled" [actionsTooltip]="restrictionTooltip" - [isMoocletExperiment]="isMoocletExperiment(vm.experiment)" - [prior]="vm.experiment.moocletPolicyParameters?.prior" + [isThompsonSamplingExperiment]="isThompsonSamplingExperiment(vm.experiment)" + [prior]="vm.experiment.thompsonSamplingConfig?.priors" (rowAction)="onRowAction($event, vm.experiment.id, appContext)" (editWeights)="onEditWeights($event, vm.experiment)" (editPrior)="onEditPrior($event, vm.experiment)" diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts index edd563220b..af64e3c65c 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-section-card.component.ts @@ -26,7 +26,7 @@ import { Prior } from 'upgrade_types'; import { ConditionHelperService } from '../../../../../../../core/experiments/condition-helper.service'; import { selectConditionWeightsValid } from '../../../../../../../core/experiments/store/experiments.selectors'; import { Store } from '@ngrx/store'; -import { MoocletExperimentHelperService } from '../../../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../../../core/experiments/thompson-sampling-helper.service'; @Component({ selector: 'app-experiment-conditions-section-card', @@ -62,7 +62,7 @@ export class ExperimentConditionsSectionCardComponent implements OnInit { private readonly dialogService: DialogService, private readonly conditionHelperService: ConditionHelperService, private readonly store: Store, - private readonly moocletHelperService: MoocletExperimentHelperService + private readonly thompsonSamplingHelperService: ThompsonSamplingHelperService ) {} ngOnInit() { @@ -79,8 +79,8 @@ export class ExperimentConditionsSectionCardComponent implements OnInit { ); } - isMoocletExperiment(experiment: Experiment) { - return this.moocletHelperService.isMoocletAlgorithm(experiment?.assignmentAlgorithm); + isThompsonSamplingExperiment(experiment: Experiment) { + return this.thompsonSamplingHelperService.isThompsonSamplingExperiment(experiment); } onAddConditionClick(appContext: string, experimentId: string): void { @@ -135,7 +135,7 @@ export class ExperimentConditionsSectionCardComponent implements OnInit { } onEditPrior(conditions: ExperimentCondition[], experiment: ExperimentVM): void { - const existingPrior = experiment.moocletPolicyParameters?.prior; + const existingPrior = experiment.thompsonSamplingConfig?.priors; this.dialogService .openEditConditionPriorModal(conditions, existingPrior) .subscribe((result: Record | undefined) => { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html index a65b9e8c3c..2635ff8848 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-conditions-section-card/experiment-conditions-table/experiment-conditions-table.component.html @@ -24,8 +24,8 @@ mat-header-cell *matHeaderCellDef class="weight-column ft-14-600" - [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" - [matTooltipDisabled]="!isMoocletExperiment" + [matTooltip]="WEIGHT_ADAPTIVE_TOOLTIP_KEY | translate" + [matTooltipDisabled]="!isThompsonSamplingExperiment" matTooltipPosition="above" > {{ CONDITION_TRANSLATION_KEYS.WEIGHT | translate }} @@ -34,18 +34,18 @@ mat-cell *matCellDef="let condition" class="weight-column ft-14-400" - [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" - [matTooltipDisabled]="!isMoocletExperiment" + [matTooltip]="WEIGHT_ADAPTIVE_TOOLTIP_KEY | translate" + [matTooltipDisabled]="!isThompsonSamplingExperiment" matTooltipPosition="above" > - {{ isMoocletExperiment ? 'N/A' : condition.assignmentWeight }} + {{ isThompsonSamplingExperiment ? 'N/A' : condition.assignmentWeight }} - @if (showActions && !isMoocletExperiment) { + @if (showActions && !isThompsonSamplingExperiment) {
- @if (showActions && isMoocletExperiment) { + @if (showActions && isThompsonSamplingExperiment) {
; @Output() rowAction = new EventEmitter(); @Output() editWeights = new EventEmitter(); @Output() editPrior = new EventEmitter(); get displayedColumns(): string[] { - if (this.isMoocletExperiment) { + if (this.isThompsonSamplingExperiment) { return ['condition', 'priorSuccesses', 'priorFailures', 'priorEdit', 'description', 'actions']; } return ['condition', 'weight', 'weightEdit', 'description', 'actions']; } + readonly WEIGHT_ADAPTIVE_TOOLTIP_KEY = THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY; + CONDITION_TRANSLATION_KEYS = { CONDITION: 'experiments.details.conditions.condition.text', DESCRIPTION: 'experiments.details.conditions.description.text', @@ -60,11 +63,11 @@ export class ExperimentConditionsTableComponent { }; getPriorSuccesses(condition: ExperimentCondition): number { - return this.prior?.[condition.conditionCode]?.success ?? 1; + return this.prior?.[condition.id]?.success ?? 1; } getPriorFailures(condition: ExperimentCondition): number { - return this.prior?.[condition.conditionCode]?.failure ?? 1; + return this.prior?.[condition.id]?.failure ?? 1; } onEditButtonClick(condition: ExperimentCondition): void { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts index 2c88690af8..f0e010e80f 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-details-page-content.component.ts @@ -23,7 +23,7 @@ import { Observable, Subscription, combineLatest } from 'rxjs'; import { map, filter, startWith } from 'rxjs/operators'; import { Experiment } from '../../../../../../core/experiments/store/experiments.model'; import { SegmentsService } from '../../../../../../core/segments/segments.service'; -import { MoocletExperimentHelperService } from '../../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../../core/experiments/thompson-sampling-helper.service'; import { ASSIGNMENT_ALGORITHM } from 'upgrade_types'; @Component({ @@ -71,7 +71,7 @@ export class ExperimentDetailsPageContentComponent implements OnInit, OnDestroy private readonly router: Router, private readonly route: ActivatedRoute, private readonly segmentService: SegmentsService, - private readonly moocletHelperService: MoocletExperimentHelperService + private readonly thompsonSamplingHelperService: ThompsonSamplingHelperService ) {} ngOnInit() { @@ -107,10 +107,7 @@ export class ExperimentDetailsPageContentComponent implements OnInit, OnDestroy if (!experiment) { return false; } - const isMoocletEnabled = this.moocletHelperService.isMoocletEnabled(); - const hasMoocletPolicyParameters = !!experiment.moocletPolicyParameters; - const isTSConfigurable = experiment.assignmentAlgorithm === ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE; - return isMoocletEnabled && hasMoocletPolicyParameters && isTSConfigurable; + return this.thompsonSamplingHelperService.isThompsonSamplingAlgorithm(experiment.assignmentAlgorithm); }) ); } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html index d09cc9b7c0..bdad133da2 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.html @@ -10,26 +10,30 @@ style="justify-content: left; padding-left: 16px" class="ft-14-600" *matHeaderCellDef - [matTooltip]="'experiments.details.conditions.weight-adaptive-tooltip.text' | translate" - [matTooltipDisabled]="!isMoocletExperiment(experiment)" + [matTooltip]="WEIGHT_ADAPTIVE_TOOLTIP_KEY | translate" + [matTooltipDisabled]="!isThompsonSamplingExperiment(experiment)" matTooltipPosition="above" > - {{ key.includes('Icon') || (key === 'weight' && isMoocletExperiment(experiment)) ? '' : columnHeaders[key] }} + {{ + key.includes('Icon') || (key === 'weight' && isThompsonSamplingExperiment(experiment)) + ? '' + : columnHeaders[key] + }} - - @if (key === 'weight' && isMoocletExperiment(experiment)) { + + @if (key === 'weight' && isThompsonSamplingExperiment(experiment)) { N/A } @else { @if (!key.includes('Icon')) { {{ element.data[key] }} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts index a6a5f36345..caa86c65d1 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-enrollment-data-section-card/enrollment-condition-table/enrollment-condition-expandable-row/enrollment-condition-expandable-row.component.ts @@ -1,12 +1,15 @@ import { Component, ChangeDetectionStrategy, Input, OnDestroy, forwardRef } from '@angular/core'; -import { ExperimentVM } from '../../../../../../../../../core/experiments/store/experiments.model'; +import { + ExperimentVM, + THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY, +} from '../../../../../../../../../core/experiments/store/experiments.model'; import { TranslateModule, TranslateService } from '@ngx-translate/core'; import { Subscription } from 'rxjs'; import { MatIconModule } from '@angular/material/icon'; import { MatTableModule } from '@angular/material/table'; import { CommonModule } from '@angular/common'; import { EnrollmentPointPartitionTableComponent } from '../enrollment-point-partition-table/enrollment-point-partition-table.component'; -import { MoocletExperimentHelperService } from '../../../../../../../../../core/experiments/mooclet-helper.service'; +import { ThompsonSamplingHelperService } from '../../../../../../../../../core/experiments/thompson-sampling-helper.service'; import { MatTooltipModule } from '@angular/material/tooltip'; @Component({ selector: 'app-enrollment-condition-expandable-row', @@ -28,11 +31,16 @@ export class EnrollmentConditionExpandableRowComponent implements OnDestroy { @Input() referenceId: string; @Input() experiment: ExperimentVM; + readonly WEIGHT_ADAPTIVE_TOOLTIP_KEY = THOMPSON_SAMPLING_WEIGHT_TOOLTIP_KEY; + expandedId = ''; columnHeaders = {}; translateSub: Subscription; - constructor(private translate: TranslateService, private moocletHelperService: MoocletExperimentHelperService) { + constructor( + private translate: TranslateService, + private thompsonSamplingHelperService: ThompsonSamplingHelperService + ) { this.translateSub = this.translate .get([ 'global.condition.text', @@ -52,8 +60,8 @@ export class EnrollmentConditionExpandableRowComponent implements OnDestroy { }); } - isMoocletExperiment(experiment: ExperimentVM): boolean { - return this.moocletHelperService.isMoocletAlgorithm(experiment?.assignmentAlgorithm); + isThompsonSamplingExperiment(experiment: ExperimentVM): boolean { + return this.thompsonSamplingHelperService.isThompsonSamplingExperiment(experiment); } toggleExpandableSymbol(id: string): void { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.html index ce1daaffc2..fc04334b8f 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.html @@ -1,3 +1,4 @@ +@if (rewardsSummary$ | async; as summary) { @if (isSectionCardExpanded) { - + + + @let isBatchingActive = summary.batchSize > 1; @let isWarmupConfigured = summary.warmupThreshold > 0; @let + isWarmupActive = isWarmupConfigured && summary.totalRewardCount <= summary.warmupThreshold; +
+
+ {{ 'experiments.details.posteriors.pending-rewards.text' | translate }}: + {{ summary.pendingRewardsCount }}/{{ summary.batchSize }} +
+
+ {{ 'experiments.details.posteriors.algorithm-in-effect.text' | translate }}: + + @if (isWarmupActive) { + {{ + 'experiments.details.posteriors.algorithm-warmup.text' + | translate : { rewardCount: summary.totalRewardCount, warmupThreshold: summary.warmupThreshold } + }} + } @else { + {{ 'experiments.details.posteriors.algorithm-active.text' | translate }} + } + +
+
+ + } +} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.scss b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.scss new file mode 100644 index 0000000000..755207a37e --- /dev/null +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.scss @@ -0,0 +1,27 @@ +// Mirrors CommonSectionCardOverviewDetailsComponent's key-value look (bold label, normal-weight +// dark-grey value, stacked rows) -- not reused directly since these two rows need tooltips and +// conditional/parameterized content that component doesn't support. +.reward-status-section { + padding: 22px 32px 0; + + .item-key-value { + min-height: 32px; + display: flex; + align-items: flex-start; + column-gap: 8px; + cursor: default; + + .item-value { + color: var(--dark-grey); + } + + // Default/not-in-play configuration (batchSize <= 1, warmupThreshold 0) -- shown, not hidden, + // but muted to signal there's nothing meaningful to track here. + &.inactive { + .ft-14-600, + .item-value { + color: var(--light-grey); + } + } + } +} diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.ts index d049fb4bdc..69476f84a2 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/experiment-reward-feedback-section-card.component.ts @@ -1,6 +1,7 @@ import { ChangeDetectionStrategy, Component, inject, Input } from '@angular/core'; import { CommonModule } from '@angular/common'; import { TranslateModule } from '@ngx-translate/core'; +import { MatTooltipModule } from '@angular/material/tooltip'; import { CommonSectionCardComponent } from '@shared-component-lib/common-section-card/common-section-card.component'; import { CommonSectionCardTitleHeaderComponent } from '@shared-component-lib/common-section-card-title-header/common-section-card-title-header.component'; import { CommonSectionCardActionButtonsComponent } from '@shared-component-lib/common-section-card-action-buttons/common-section-card-action-buttons.component'; @@ -18,9 +19,11 @@ import { ExperimentRewardsSummary } from 'upgrade_types'; CommonSectionCardTitleHeaderComponent, CommonSectionCardActionButtonsComponent, TranslateModule, + MatTooltipModule, TSConfigurableRewardCountTableComponent, ], templateUrl: './experiment-reward-feedback-section-card.component.html', + styleUrl: './experiment-reward-feedback-section-card.component.scss', changeDetection: ChangeDetectionStrategy.OnPush, }) export class ExperimentRewardFeedbackSectionCardComponent { diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html index 814e5b3633..6986fcd58f 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.html @@ -6,6 +6,13 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss index 2de24b6719..4294a1f475 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.scss @@ -28,7 +28,10 @@ th { padding-left: 0; color: var(--darker-grey); + vertical-align: middle; + } + &:first-of-type th { &:first-child { border-top-left-radius: 4px; } @@ -37,6 +40,14 @@ border-top-right-radius: 4px; } } + + &:last-of-type th { + border-top: 1px solid var(--light-grey-2); + } + } + + .group-header { + text-align: center; } } @@ -96,6 +107,10 @@ width: 8%; } + .estimated-weight-column { + width: 10%; + } + .data-column { text-align: center; } diff --git a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts index 00edc1d6a8..455927ff06 100644 --- a/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts +++ b/packages/frontend/projects/upgrade/src/app/features/dashboard/experiments/pages/experiment-details-page/experiment-details-page-content/experiment-reward-feedback-section-card/ts-configurable-reward-count-table/ts-configurable-reward-count-table.component.ts @@ -2,21 +2,48 @@ import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; import { CommonModule } from '@angular/common'; import { MatTableModule } from '@angular/material/table'; import { MatProgressBarModule } from '@angular/material/progress-bar'; +import { MatTooltipModule } from '@angular/material/tooltip'; import { TranslateModule } from '@ngx-translate/core'; -import { ExperimentRewardsSummary } from 'upgrade_types'; +import { ExperimentRewardsByCondition } from 'upgrade_types'; @Component({ selector: 'app-ts-configurable-reward-count-table', standalone: true, - imports: [CommonModule, MatTableModule, MatProgressBarModule, TranslateModule], + imports: [CommonModule, MatTableModule, MatProgressBarModule, MatTooltipModule, TranslateModule], templateUrl: './ts-configurable-reward-count-table.component.html', styleUrl: './ts-configurable-reward-count-table.component.scss', changeDetection: ChangeDetectionStrategy.OnPush, }) export class TSConfigurableRewardCountTableComponent { - @Input() dataSource: ExperimentRewardsSummary = []; + @Input() dataSource: ExperimentRewardsByCondition[] = []; @Input() isLoading = false; + groupHeaderColumns = [ + 'conditionGroup', + 'successesGroup', + 'spacerGroup', + 'failuresGroup', + 'spacer2Group', + 'estimatedWeightGroup', + ]; + + // Condition/spacer/spacer2/estimatedWeight get real (if blank-topped) cells in both header rows + // rather than a rowspan, so their label sits in the same single-row cell as Count/Prior/Posterior + // -- that's what makes a shared `vertical-align: middle` center all of them on the same line, and + // lets the row1/row2 divider border-top carry all the way across instead of stopping at a rowspan. + subHeaderColumns = [ + 'conditionCode', + 'successes', + 'successPrior', + 'successPosterior', + 'spacer', + 'failures', + 'failurePrior', + 'failurePosterior', + 'spacer2', + 'estimatedWeight', + ]; + displayedColumns = [ 'conditionCode', 'successes', @@ -26,5 +53,7 @@ export class TSConfigurableRewardCountTableComponent { 'failures', 'failurePrior', 'failurePosterior', + 'spacer2', + 'estimatedWeight', ]; } diff --git a/packages/frontend/projects/upgrade/src/app/shared-standalone-component-lib/components/common-section-card-overview-details/common-section-card-overview-details.component.ts b/packages/frontend/projects/upgrade/src/app/shared-standalone-component-lib/components/common-section-card-overview-details/common-section-card-overview-details.component.ts index 044f81fb69..0609c270bd 100644 --- a/packages/frontend/projects/upgrade/src/app/shared-standalone-component-lib/components/common-section-card-overview-details/common-section-card-overview-details.component.ts +++ b/packages/frontend/projects/upgrade/src/app/shared-standalone-component-lib/components/common-section-card-overview-details/common-section-card-overview-details.component.ts @@ -11,7 +11,7 @@ export interface KeyValueFormat { export interface BullettedListKeyValueFormat { labelKey: string; - value: number; + value: number | string; } /** diff --git a/packages/frontend/projects/upgrade/src/app/shared/services/common-dialog.service.ts b/packages/frontend/projects/upgrade/src/app/shared/services/common-dialog.service.ts index cd9596a972..31cf18d105 100644 --- a/packages/frontend/projects/upgrade/src/app/shared/services/common-dialog.service.ts +++ b/packages/frontend/projects/upgrade/src/app/shared/services/common-dialog.service.ts @@ -623,9 +623,10 @@ export class DialogService { existingPrior?: Record ): Observable> { const conditionPriorUpdates: ConditionPriorUpdate[] = conditions.map((condition) => ({ + conditionId: condition.id, conditionCode: condition.conditionCode, - successes: existingPrior?.[condition.conditionCode]?.success ?? 1, - failures: existingPrior?.[condition.conditionCode]?.failure ?? 1, + successes: existingPrior?.[condition.id]?.success ?? 1, + failures: existingPrior?.[condition.id]?.failure ?? 1, })); const dialogRef = this.dialog.open(EditConditionPriorModalComponent, { diff --git a/packages/frontend/projects/upgrade/src/assets/i18n/en.json b/packages/frontend/projects/upgrade/src/assets/i18n/en.json index f2bba4c7f4..50b54acdbc 100644 --- a/packages/frontend/projects/upgrade/src/assets/i18n/en.json +++ b/packages/frontend/projects/upgrade/src/assets/i18n/en.json @@ -185,7 +185,6 @@ "home.new-experiment.design.condition.column-header.alias.text": "ALIAS", "home.new-experiment.design.condition.column-header.weight.text": "Weight(%)", "home.new-experiment.design.condition.column-header.include.text": "INCLUDE", - "home.new-experiment.design.mooclet-policy-parameters.header.text": "Mooclet Policy Parameters", "home.new-experiment.design.ts-configurable-policy.section-label.text": "Thompson Sampling (Configurable) Policy Parameters", "home.new-experiment.design.ts-configurable-policy.prior-success.label.text": "Prior Success", "home.new-experiment.design.ts-configurable-policy.prior-success.hint.text": "Prior success count for Beta distribution (default: 1)", @@ -197,13 +196,13 @@ "home.new-experiment.design.ts-configurable-policy.batch-size.label.text": "Batch Size", "home.new-experiment.design.ts-configurable-policy.batch-size.hint.text": "Number of rewards to collect before updating the model (default: 1)", "home.new-experiment.design.ts-configurable-policy.batch-size.min-error.text": "Batch size must be at least 1", - "home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text": "Uniform Threshold", - "home.new-experiment.design.ts-configurable-policy.uniform-threshold.hint.text": "Start experiment with uniform random distribution until this number of rewards collected (default: 0)", - "home.new-experiment.design.ts-configurable-policy.uniform-threshold.min-error.text": "Uniform threshold must be 0 or higher", - "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text": "TS Post Diff Threshold", - "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.hint.text": "Thompson Sampling posterior difference threshold", - "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.min-error.text": "TS Post Diff threshold must be 0.0 or higher.", - "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.max-error.text": "TS Post Diff threshold must be 1.0 or lower.", + "home.new-experiment.design.ts-configurable-policy.uniform-threshold.label.text": "Warmup Threshold", + "home.new-experiment.design.ts-configurable-policy.uniform-threshold.hint.text": "Use uniform random until the total reward count exceeds this threshold (default: 0)", + "home.new-experiment.design.ts-configurable-policy.uniform-threshold.min-error.text": "Warmup threshold must be 0 or higher", + "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.label.text": "Minimum Draw Difference", + "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.hint.text": "Fall back to uniform selection when the top two sampled draws differ by less than this value (0.0–1.0, default: 0)", + "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.min-error.text": "Minimum draw difference must be 0.0 or higher.", + "home.new-experiment.design.ts-configurable-policy.tspostdiff-thresh.max-error.text": "Minimum draw difference must be 1.0 or lower.", "home.new-experiment.design.ts-configurable-policy.general.max-error.text": "Please enter a value less than 1,000,000.", "home.new-experiment.design.ts-configurable-policy.general.integer-error.text": "Please enter a whole number (integer) value.", "home.new-experiment.metrics.text": "Metrics", @@ -275,7 +274,6 @@ "home.view-experiment.experiment-payloads.column-label.target": "TARGET", "home.view-experiment.experiment-payloads.column-label.condition": "CONDITION", "home.view-experiment.experiment-payloads.column-label.payload": "PAYLOAD", - "home.view-experiment.experiment-mooclet-policy-parameters-title.text": "Mooclet Policy Parameters", "home.view-experiment.metrics-analysis-unavailable.text": "Experiment metrics analysis has been temporarily disabled in this view. This information is still available when exporting the experiment data.", "home.view-experiment.graph-type.text": "Type", "home.view-experiment.graph-conditions.text": "Conditions", @@ -422,6 +420,7 @@ "experiments.upsert-experiment-modal.condition-order-ordered-round-robin-description.text": "Conditions are presented in the same fixed sequence each round.", "experiments.upsert-experiment-modal.assignment-algorithm-random-description.text": "Random assignment according to the configured condition weights.", "experiments.upsert-experiment-modal.assignment-algorithm-stratified-random-sampling-description.text": "Random assignment within balanced subgroups to ensure representation.", + "experiments.upsert-experiment-modal.assignment-algorithm-thompson-sampling-description.text": "Adaptive assignment based on Thompson Sampling.", "experiments.upsert-experiment-modal.stratification-factor-hint.text": "Select a stratification factor for balanced random assignment.", "experiments.upsert-experiment-modal.tags-label.text": "Tags (optional)", "experiments.upsert-experiment-modal.tags-placeholder.text": "Tags separated by commas", @@ -581,11 +580,20 @@ "experiments.details.export-metrics-data.menu-item.text": "Export Metrics Data", "experiments.details.posteriors.condition.text": "Condition", "experiments.details.posteriors.successes.text": "Successes", + "experiments.details.posteriors.count.text": "Count", "experiments.details.posteriors.prior.text": "Prior", "experiments.details.posteriors.posterior.text": "Posterior", "experiments.details.posteriors.failures.text": "Failures", "experiments.details.posteriors.percentOfTotalRewards.text": "% of All Rewards Received", + "experiments.details.posteriors.estimated-weight.text": "Est. Weight", + "experiments.details.posteriors.estimated-weight-tooltip.text": "Estimated % of Thompson Sampling draws this condition would win based on current posteriors (10,000 simulated draws).", "experiments.details.posteriors.no-data-row.text": "Experiment has not begun to collect feedback.", + "experiments.details.posteriors.pending-rewards.text": "Pending rewards", + "experiments.details.posteriors.pending-rewards-tooltip.text": "Batch-Size: {{batchSize}}", + "experiments.details.posteriors.algorithm-in-effect.text": "Algorithm in Effect", + "experiments.details.posteriors.algorithm-warmup.text": "Random Assignment ({{rewardCount}}/{{warmupThreshold}})", + "experiments.details.posteriors.algorithm-active.text": "Thompson Sampling", + "experiments.details.posteriors.warmup-threshold-tooltip.text": "Warm-up Threshold: {{warmupThreshold}}", "experiments.upsert-include-list-modal.name-hint.text": "The name for this include list.", "experiments.upsert-exclude-list-modal.name-hint.text": "The name for this exclude list.", "experiments.upsert-list-modal.values-label.text": "Values", diff --git a/packages/frontend/projects/upgrade/src/environments/environment-types.ts b/packages/frontend/projects/upgrade/src/environments/environment-types.ts index f4f5964f5a..75ab1ecf33 100644 --- a/packages/frontend/projects/upgrade/src/environments/environment-types.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment-types.ts @@ -67,8 +67,8 @@ export interface APIEndpoints { addSegmentList: string; getGroupAssignmentStatus: string; stratification: string; - getMoocletRewardsData: string; featureFlagGraphInfo: string; + experimentsRewardsSummary: string; } export interface Environment { @@ -84,7 +84,6 @@ export interface Environment { errorLogsToggle: boolean; withinSubjectExperimentSupportToggle: boolean; metricAnalyticsExperimentDisplayToggle: boolean; - moocletToggle: boolean; // these have been removed but optional to prevent annoyance switching between branches pollingEnabled?: boolean; pollingInterval?: number; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.bsnl.ts b/packages/frontend/projects/upgrade/src/environments/environment.bsnl.ts index 1f3da7bdcf..fbfed33a10 100644 --- a/packages/frontend/projects/upgrade/src/environments/environment.bsnl.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.bsnl.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: false, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.demo.prod.ts b/packages/frontend/projects/upgrade/src/environments/environment.demo.prod.ts index b5c82d145f..2db41ea2c5 100755 --- a/packages/frontend/projects/upgrade/src/environments/environment.demo.prod.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.demo.prod.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: false, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.local.example.ts b/packages/frontend/projects/upgrade/src/environments/environment.local.example.ts index f36fe056a5..b0f70ccc87 100755 --- a/packages/frontend/projects/upgrade/src/environments/environment.local.example.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.local.example.ts @@ -18,5 +18,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: false, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.prod.ts b/packages/frontend/projects/upgrade/src/environments/environment.prod.ts index 7aedb5d89b..68c48e299b 100755 --- a/packages/frontend/projects/upgrade/src/environments/environment.prod.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.prod.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: true, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.qa.ts b/packages/frontend/projects/upgrade/src/environments/environment.qa.ts index d8af43b3bd..0b92b5eb13 100644 --- a/packages/frontend/projects/upgrade/src/environments/environment.qa.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.qa.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: true, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: true, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.staging.ts b/packages/frontend/projects/upgrade/src/environments/environment.staging.ts index a46f5d2f20..f2a2e31eb0 100644 --- a/packages/frontend/projects/upgrade/src/environments/environment.staging.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.staging.ts @@ -13,5 +13,4 @@ export const environment: Environment = { withinSubjectExperimentSupportToggle: true, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: true, }; diff --git a/packages/frontend/projects/upgrade/src/environments/environment.ts b/packages/frontend/projects/upgrade/src/environments/environment.ts index dccf995fff..06ca6e8225 100755 --- a/packages/frontend/projects/upgrade/src/environments/environment.ts +++ b/packages/frontend/projects/upgrade/src/environments/environment.ts @@ -18,5 +18,4 @@ export const environment = { withinSubjectExperimentSupportToggle: false, errorLogsToggle: false, metricAnalyticsExperimentDisplayToggle: true, - moocletToggle: true, }; diff --git a/packages/types/CLAUDE.md b/packages/types/CLAUDE.md index 7cc12cf753..aafff82dba 100644 --- a/packages/types/CLAUDE.md +++ b/packages/types/CLAUDE.md @@ -19,7 +19,6 @@ src/ Experiment/ enums.ts # all enums + display map constants interfaces.ts # all interfaces - Mooclet/ # Mooclet-specific types and policy schemas index.ts # single re-export point for everything ``` diff --git a/packages/types/package.json b/packages/types/package.json index 1318b62954..c466f94714 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -23,9 +23,6 @@ "typescript": "~5.3.3" }, "dependencies": { - "class-transformer": "^0.5.1", - "class-validator": "^0.14.1", - "reflect-metadata": "^0.2.2", "tslib": "2.8.1" } } \ No newline at end of file diff --git a/packages/types/src/Experiment/enums.ts b/packages/types/src/Experiment/enums.ts index d61a74c59d..7b93d6a03b 100644 --- a/packages/types/src/Experiment/enums.ts +++ b/packages/types/src/Experiment/enums.ts @@ -37,13 +37,13 @@ export const CONDITION_ORDER_DISPLAY_MAP = { export enum ASSIGNMENT_ALGORITHM { RANDOM = 'random', STRATIFIED_RANDOM_SAMPLING = 'stratified random sampling', - MOOCLET_TS_CONFIGURABLE = 'ts_configurable', + THOMPSON_SAMPLING = 'thompson_sampling', } export const ASSIGNMENT_ALGORITHM_DISPLAY_MAP = { [ASSIGNMENT_ALGORITHM.RANDOM]: 'Random', [ASSIGNMENT_ALGORITHM.STRATIFIED_RANDOM_SAMPLING]: 'Stratified Random Sampling', - [ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE]: 'Thompson Sampling (Configurable)', + [ASSIGNMENT_ALGORITHM.THOMPSON_SAMPLING]: 'Thompson Sampling', }; export enum POST_EXPERIMENT_RULE { @@ -116,8 +116,6 @@ export enum SERVER_ERROR { MISSING_HEADER_USER_ID = 'Missing `User-Id` header', SEGMENT_DUPLICATE_NAME = 'Segment with same name already exists for this app-context.', INVALID_APP_CONTEXT = 'Invalid app context', - MOOCLET_REWARD_ERROR = 'Error processing Mooclet reward', - MOOCLET_ERROR = 'Mooclet synchronization error', } export enum MARKED_DECISION_POINT_STATUS { @@ -369,6 +367,7 @@ export enum CACHE_PREFIX { FEATURE_FLAG_PRECOMPUTED_SEGMENT_KEY_PREFIX = 'featureFlagPrecomputedSegments-', EXPERIMENT_PRECOMPUTED_SEGMENT_KEY_PREFIX = 'experimentPrecomputedSegments-', SETTING_KEY_PREFIX = 'setting-', + THOMPSON_SAMPLING_CONFIG_KEY_PREFIX = 'thompsonSamplingConfig-', } export enum STATUS_INDICATOR_CHIP_TYPE { diff --git a/packages/types/src/Experiment/interfaces.ts b/packages/types/src/Experiment/interfaces.ts index 8b12813f9b..0d7ae1be1f 100644 --- a/packages/types/src/Experiment/interfaces.ts +++ b/packages/types/src/Experiment/interfaces.ts @@ -335,3 +335,44 @@ export interface DuplicateSegmentNameError { context: string; httpCode: 400; } + +export interface Prior { + success: number; + failure: number; +} + +export enum BinaryRewardAllowedValue { + SUCCESS = 'SUCCESS', + FAILURE = 'FAILURE', +} + +export const BinaryRewardValueMap = { + [BinaryRewardAllowedValue.SUCCESS]: 1, + [BinaryRewardAllowedValue.FAILURE]: 0, +}; + +export interface ExperimentRewardsByCondition { + conditionCode: string; + successes: number; + failures: number; + successRate: string; + order: number; + priorSuccess?: number; + priorFailure?: number; + /** Estimated probability of winning a Thompson Sampling draw, as an integer percentage (0–100). */ + estimatedWeight?: number; +} + +export interface ExperimentRewardsSummary { + conditions: ExperimentRewardsByCondition[]; + /** + * Rewards buffered since the last batch flush, summed across all conditions. Cycles from 0 up + * to (but never reaching) batchSize, resetting to 0 once the shared batch closes. Always 0 when + * batchSize is 1 or less, since a reward is applied immediately rather than buffered. + */ + pendingRewardsCount: number; + /** Reward evidence collected so far (flushed + pending), the same measure warmupThreshold gates on. */ + totalRewardCount: number; + warmupThreshold: number; + batchSize: number; +} diff --git a/packages/types/src/Mooclet/MoocletPolicyParametersDTO.ts b/packages/types/src/Mooclet/MoocletPolicyParametersDTO.ts deleted file mode 100644 index bddc7c8d7f..0000000000 --- a/packages/types/src/Mooclet/MoocletPolicyParametersDTO.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { IsDefined, IsIn } from 'class-validator'; -import { ASSIGNMENT_ALGORITHM } from '../Experiment/enums'; - -// this will be a Union type of all possible policy parameters once more are introduced -export abstract class MoocletPolicyParametersDTO { - @IsDefined() - @IsIn(Object.values(ASSIGNMENT_ALGORITHM)) - assignmentAlgorithm: ASSIGNMENT_ALGORITHM; -} diff --git a/packages/types/src/Mooclet/MoocletTSConfigurablePolicyParametersDTO.ts b/packages/types/src/Mooclet/MoocletTSConfigurablePolicyParametersDTO.ts deleted file mode 100644 index 2673c589cb..0000000000 --- a/packages/types/src/Mooclet/MoocletTSConfigurablePolicyParametersDTO.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { IsNumber, IsString, ValidateNested, IsOptional, IsObject, IsDefined } from 'class-validator'; -import { Type } from 'class-transformer'; -import { MoocletPolicyParametersDTO } from './MoocletPolicyParametersDTO'; - -export class Prior { - @IsDefined() - @IsNumber() - @Type(() => Number) - failure = 1; - - @IsDefined() - @IsNumber() - @Type(() => Number) - success = 1; -} - -export class CurrentPosteriors { - @IsNumber() - @Type(() => Number) - failures = 0; - - @IsNumber() - @Type(() => Number) - successes = 0; -} - -export class MoocletTSConfigurablePolicyParametersDTO extends MoocletPolicyParametersDTO { - @IsOptional() - @IsObject() - prior?: Record; - - @IsOptional() - @IsObject() - @ValidateNested({ each: true }) - @Type(() => CurrentPosteriors) - current_posteriors?: Record; - - @IsNumber() - batch_size = 1; - - @IsNumber() - max_rating = 1; - - @IsNumber() - min_rating = 0; - - @IsNumber() - uniform_threshold = 0; - - @IsNumber() - tspostdiff_thresh = 0; - - @IsOptional() - @IsString() - outcome_variable_name?: string; -} diff --git a/packages/types/src/Mooclet/index.ts b/packages/types/src/Mooclet/index.ts deleted file mode 100644 index c535e3ad48..0000000000 --- a/packages/types/src/Mooclet/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import 'reflect-metadata'; -import { ASSIGNMENT_ALGORITHM } from '../Experiment/enums'; -import { - CurrentPosteriors, - MoocletTSConfigurablePolicyParametersDTO, - Prior, -} from './MoocletTSConfigurablePolicyParametersDTO'; -import { MoocletPolicyParametersDTO } from './MoocletPolicyParametersDTO'; - -const MOOCLET_POLICY_SCHEMA_MAP = { - [ASSIGNMENT_ALGORITHM.MOOCLET_TS_CONFIGURABLE]: MoocletTSConfigurablePolicyParametersDTO, -}; - -const SUPPORTED_MOOCLET_ALGORITHMS = Object.keys(MOOCLET_POLICY_SCHEMA_MAP); - -enum BinaryRewardAllowedValue { - SUCCESS = 'SUCCESS', - FAILURE = 'FAILURE', -} - -const BinaryRewardValueMap = { - [BinaryRewardAllowedValue.SUCCESS]: 1, - [BinaryRewardAllowedValue.FAILURE]: 0, -}; - -interface ExperimentRewardsByCondition { - conditionCode: string; - successes: number; - failures: number; - successRate: string; - order: number; - priorSuccess?: number; - priorFailure?: number; -} - -type ExperimentRewardsSummary = Array; - -export { - MOOCLET_POLICY_SCHEMA_MAP, - SUPPORTED_MOOCLET_ALGORITHMS, - Prior, - CurrentPosteriors, - MoocletPolicyParametersDTO, - MoocletTSConfigurablePolicyParametersDTO, - BinaryRewardAllowedValue, - BinaryRewardValueMap, - ExperimentRewardsSummary, - ExperimentRewardsByCondition, -}; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 4133ce13e3..9720688124 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -96,13 +96,8 @@ export { export { SYSTEM_USER_EMAIL, DEV_USER_EMAIL, FAKE_DEV_CREDENTIAL } from './User'; export { Prior, - CurrentPosteriors, - MoocletPolicyParametersDTO, - MoocletTSConfigurablePolicyParametersDTO, - MOOCLET_POLICY_SCHEMA_MAP, - SUPPORTED_MOOCLET_ALGORITHMS, BinaryRewardAllowedValue, BinaryRewardValueMap, ExperimentRewardsByCondition, ExperimentRewardsSummary, -} from './Mooclet'; +} from './Experiment/interfaces'; diff --git a/packages/types/tsconfig.json b/packages/types/tsconfig.json index f0d80d6d3d..1d5a0024f0 100644 --- a/packages/types/tsconfig.json +++ b/packages/types/tsconfig.json @@ -6,8 +6,6 @@ "sourceMap": true, "declaration": true, "moduleResolution": "node", - "emitDecoratorMetadata": true, - "experimentalDecorators": true, "importHelpers": true, "module": "commonjs", "target": "es2015", diff --git a/postman/ClientAPI.postman_collection.json b/postman/ClientAPI.postman_collection.json index a4e59e1621..cfb6fdd040 100644 --- a/postman/ClientAPI.postman_collection.json +++ b/postman/ClientAPI.postman_collection.json @@ -305,7 +305,7 @@ "host": ["{{baseUrl}}"], "path": ["v6", "reward"] }, - "description": "Send reward signal for adaptive experiments (Mooclets) using direct experiment ID lookup. Reward values: 'SUCCESS' or 'FAILURE'." + "description": "Send reward signal for adaptive (Thompson Sampling) experiments using direct experiment ID lookup. Reward values: 'SUCCESS' or 'FAILURE'." } }, {
@@ -16,10 +23,17 @@ + + + + {{ 'experiments.details.posteriors.successes.text' | translate }} + - {{ 'experiments.details.posteriors.successes.text' | translate }} + {{ 'experiments.details.posteriors.count.text' | translate }} {{ row.successes }} @@ -46,16 +60,41 @@ + {{ 'experiments.details.posteriors.failures.text' | translate }} + - {{ 'experiments.details.posteriors.failures.text' | translate }} + {{ 'experiments.details.posteriors.count.text' | translate }} {{ row.failures }} @@ -82,8 +121,30 @@ + {{ 'experiments.details.posteriors.estimated-weight.text' | translate }} + + {{ row.estimatedWeight != null ? '≈' + row.estimatedWeight + '%' : '—' }} +