From 34e824d56e9dd95b45ba9d29064d96b9ac11e7e1 Mon Sep 17 00:00:00 2001 From: 0xdevabir <261-15-001@diu.edu.bd> Date: Wed, 7 Oct 2026 09:05:15 +0600 Subject: [PATCH 01/12] Plan: label-realism experiment (under-reporting) --- docs/plans/04-label-realism.md | 68 ++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 docs/plans/04-label-realism.md diff --git a/docs/plans/04-label-realism.md b/docs/plans/04-label-realism.md new file mode 100644 index 0000000..ee6898e --- /dev/null +++ b/docs/plans/04-label-realism.md @@ -0,0 +1,68 @@ +# 04 — Label realism: what under-reporting costs, and what gets it back + +## Goal + +MODEL_CARD §12 admits that every model is trained on ground-truth labels, while a +real MFS only learns about the scams victims report (about half). Turn that +disclosed weakness into a measured result: train the same transaction model under +realistic label regimes, evaluate every one on the same ground-truth test period, +and show which scam types under-reporting hides and how much of the loss a +recovery method wins back. + +## Judge criteria it moves + +- **AI/ML depth (14.67/20).** Positive-unlabelled learning, network label + propagation and a simulated review loop, compared under one protocol with + several seeds and mean ± sd: a real experiment, not one more model. +- **Business impact (13.67/20).** The judges discounted the numbers as + synthetic-label numbers. This answers the obvious objection ("would it still work + with the labels a bank actually has?") with a measured cost in scams missed, by + typology, and names the cheapest fix (analyst verdicts, mule-network labels). + +## What the simulator already models + +`simulator/fraud.py` lets each victim transfer report with probability +`report_rate = 0.5`, uniformly at random, after a lognormal delay (median 1.5 +days, 5 days for the held-out investment scam). A report only flags the mule +wallet; which cases were reported is not written to the dataset. So the reporting +draw is re-simulated per case, from data that is in the dataset. + +## Steps + +1. `backend/src/fraudlens/models/label_realism.py`, `make label-realism`. + Model: the transaction LightGBM exactly as `train.fit_booster` builds it + (same params, train fold, early stopping on val_a), seeds varied. +2. Reporting process (per case, seeded): report probability from a logistic in + log loss, victim urban/rural, victim account tenure (the simulator has no + person age, so tenure stands in for it) and typology (unauthorised takeover + reported most, shame-heavy lottery/investment scams least); intercept solved + so the mean is 50%, matching `report_rate`. Report delay as the simulator's; + labels are those known at training time (end of val_b). A reported case labels + every transaction tagged with it (victim transfers, forwards, cash-outs). + A uniform 50% regime, the simulator's own process, is kept as a check. +3. Regimes, all evaluated on ground truth: + a. ground truth (baseline); + b. report-only labels, biased; b′. report-only, uniform 50%; + c1. b + PU learning (Elkan–Noto weighting with out-of-fold propensities); + c2. b + label propagation through the mule network (wallets that received + reported scam money and their next hop pass soft labels to their other + incoming and outgoing transfers in a time window); + d. b + the analyst-verdict loop: a model trained on early report-only labels + alerts on the later training days, simulated analysts label the alerts, + and the verdicts join through `train.add_feedback`, the code path + `mlops/feedback.py` feeds. +4. Metrics on the test period: PR-AUC (all fraud-chain rows), victim-transfer + and scam recall at the served model's operating false-positive rate (v4 warn + tier, read from its `test_scores.parquet`), recall by typology. Five seeds, + mean ± sd. +5. Write `backend/artifacts/reports/label_realism.json` (new file, nothing + existing is overwritten). Add a "Under-reporting" section to MODEL_CARD with + the measured numbers. +6. Additive API field on `/model/report` and a small card on the console model page. +7. Tests on the reporting draw, propagation and PU weights on a small frame. + +## Definition of done + +- `make label-realism` reproduces the JSON; the model card quotes it, number for number. +- `uv run pytest -q` and `make lint` pass; new tests cover the label regimes. +- No served model, default artifact or headline number changes. From bfb838eda83cf08aaaa04005b16a0fd6bca05903 Mon Sep 17 00:00:00 2001 From: 0xdevabir <261-15-001@diu.edu.bd> Date: Wed, 7 Oct 2026 09:18:13 +0600 Subject: [PATCH 02/12] Add business case: threshold sweep priced in taka, API, impact page section, sensitivity doc --- README.md | 1 + backend/src/fraudlens/api/routes/ops.py | 34 ++ backend/src/fraudlens/decision/business.py | 523 +++++++++++++++++++++ backend/tests/test_business.py | 170 +++++++ backend/tests/test_platform.py | 13 +- docs/BUSINESS_CASE.md | 185 ++++++++ docs/plans/03-business-case.md | 59 +++ frontend/app/(console)/impact/business.tsx | 270 +++++++++++ frontend/app/(console)/impact/page.tsx | 6 +- frontend/lib/types.ts | 78 +++ 10 files changed, 1337 insertions(+), 2 deletions(-) create mode 100644 backend/src/fraudlens/decision/business.py create mode 100644 backend/tests/test_business.py create mode 100644 docs/BUSINESS_CASE.md create mode 100644 docs/plans/03-business-case.md create mode 100644 frontend/app/(console)/impact/business.tsx diff --git a/README.md b/README.md index f62e121..623b72c 100644 --- a/README.md +++ b/README.md @@ -439,6 +439,7 @@ Keep two browser windows open: one as `analyst1`, one private window as `supervi | [DATA_ASSUMPTIONS.md](docs/DATA_ASSUMPTIONS.md) | What the synthetic world contains and what it leaves out | | [MODEL_CARD.md](docs/MODEL_CARD.md) | Models, results, ablations, fairness, limits | | [DECISION_POLICY.md](docs/DECISION_POLICY.md) | Tiers, rules, thresholds, explanations, the language model's role | +| [BUSINESS_CASE.md](docs/BUSINESS_CASE.md) | The threshold sweep as a monthly P&L in taka, assumptions and sensitivity | | [PLATFORM.md](docs/PLATFORM.md) | API, workflow, security, stream, measured latency | | [FRAUD_TAXONOMY.md](docs/FRAUD_TAXONOMY.md) | Eight kinds of fraud in the Bangladesh context, what detects each, the scam-message classifier and its limits | | [DEMO_SCRIPT.md](docs/DEMO_SCRIPT.md) | The walk-through | diff --git a/backend/src/fraudlens/api/routes/ops.py b/backend/src/fraudlens/api/routes/ops.py index 9bc3ab1..758f0ff 100644 --- a/backend/src/fraudlens/api/routes/ops.py +++ b/backend/src/fraudlens/api/routes/ops.py @@ -11,6 +11,7 @@ from fastapi import APIRouter, Query, Response from sqlalchemy import Date, cast, func, select, text +from ...decision.business import Assumptions, business_case from ...decision.evaluate import REPORT_FILE as POLICY_REPORT_FILE from ...decision.insights import INSIGHTS_FILE from ...models import registry @@ -266,6 +267,39 @@ def model_report( } +def _business(p: Plat, version: str | None, assumptions: Assumptions) -> dict: + if version is None and p.scorer.bundle is not None: + version = p.scorer.bundle.version + if version is None or version not in registry.versions(p.settings.models_dir): + raise WorkflowError(404, "model_not_found", f"no model version {version}") + insights = _report(p.settings.models_dir / version, INSIGHTS_FILE) + if not insights or not insights.get("impact"): + raise WorkflowError(404, "not_evaluated", f"model {version} has no threshold sweep") + return business_case(insights, assumptions) + + +@router.get("/model/business") +def business( + p: Plat, ctx: Staff, version: Annotated[str | None, Query(pattern=r"^v\d{1,6}$")] = None +) -> dict: + """The threshold sweep as a monthly profit and loss in taka, under default assumptions. + + Every assumption, its source and its range are in the answer; POST to change them. + """ + return _business(p, version, Assumptions()) + + +@router.post("/model/business") +def business_with( + assumptions: Assumptions, + p: Plat, + ctx: Staff, + version: Annotated[str | None, Query(pattern=r"^v\d{1,6}$")] = None, +) -> dict: + """The same, with some assumptions changed. Computes only: nothing is stored.""" + return _business(p, version, assumptions) + + @router.get("/policy") def policy(p: Plat, ctx: Staff) -> dict: """The whole policy as served: tiers, rules with their conditions, texts, fallback.""" diff --git a/backend/src/fraudlens/decision/business.py b/backend/src/fraudlens/decision/business.py new file mode 100644 index 0000000..bdd67e5 --- /dev/null +++ b/backend/src/fraudlens/decision/business.py @@ -0,0 +1,523 @@ +"""The threshold sweep turned into a monthly profit and loss in taka. + +`insights.py` says what a threshold does on the test period: alerts, false alerts, +victims' money stopped. This module puts a price on each of those at a provider's +own volume, so the question "is it worth running, and with how many people?" has a +number for every threshold in the sweep: + +- money kept from scammers: the scam money alerted in each tier, times the share + that tier actually stops (a warning is often clicked through; a hold rarely is); +- friction: honest payments abandoned after an interruption, and the calls to the + contact centre that interruptions cause; +- review: analyst time for every held payment, with a floor for round-the-clock cover; +- what is left: net benefit a month, prevented loss per taka of operating cost, + analysts needed, honest customers interrupted per 10,000 payments. + +Every assumption is a field of `Assumptions`, with its source or the reasoning +behind it in `ASSUMPTIONS`. The model numbers come from `insights.json` only, so +nothing is re-scored. Rates per payment are taken from the back-test and multiplied +by the provider's volume; the scam money and the true alerts are rescaled from the +simulator's scam rate (about 93 basis points of payment value, far denser than real +traffic) to the assumed real one. False alerts are not rescaled: honest customers +look the same whatever the scam rate. + + uv run python -m fraudlens.decision.business +""" + +from __future__ import annotations + +import argparse +import json +import math +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict, Field + +from ..config import settings +from ..models import registry +from .insights import INSIGHTS_FILE + +TIERS = ("warn", "step_up", "hold") + + +class Assumptions(BaseModel): + """What the business case assumes. Every field can be overridden; bounds keep it sane.""" + + model_config = ConfigDict(extra="forbid") + + monthly_payments: float = Field(20_000_000, ge=10_000, le=2_000_000_000) + avg_payment_taka: float = Field(2_800, ge=100, le=25_000) + scam_loss_bps: float = Field(2.0, ge=0.01, le=100) + stop_warn: float = Field(0.25, ge=0, le=1) + stop_step_up: float = Field(0.50, ge=0, le=1) + stop_hold: float = Field(0.90, ge=0, le=1) + abandon_rate: float = Field(0.05, ge=0, le=1) + abandon_cost_taka: float = Field(25, ge=0, le=5_000) + contacts_warn: float = Field(0.02, ge=0, le=5) + contacts_step_up: float = Field(0.10, ge=0, le=5) + contacts_hold: float = Field(0.25, ge=0, le=5) + contact_cost_taka: float = Field(50, ge=0, le=5_000) + review_minutes: float = Field(8, ge=0.5, le=240) + analyst_monthly_cost_taka: float = Field(60_000, ge=1_000, le=2_000_000) + analyst_hours_per_month: float = Field(132, ge=20, le=250) + min_analysts: int = Field(5, ge=0, le=500) + platform_monthly_cost_taka: float = Field(500_000, ge=0, le=1_000_000_000) + reimbursement_share: float = Field(0.25, ge=0, le=1) + reputation_per_taka: float = Field(0.25, ge=0, le=10) + + +@dataclass(frozen=True) +class Spec: + label: str + unit: str + low: float # the range the sensitivity table tries + high: float + source: str + + +ASSUMPTIONS: dict[str, Spec] = { + "monthly_payments": Spec( + "Send-money and cash-out payments a month", + "payments", + 10_000_000, + 40_000_000, + "Order of magnitude for a mid-sized provider (upay scale). Bangladesh Bank's monthly " + "MFS statistics put the whole industry in the hundreds of millions of transactions a " + "month; a bKash-scale provider is roughly 300 million scored payments. Replace with " + "the provider's own MIS figure.", + ), + "avg_payment_taka": Spec( + "Average payment", + "BDT", + 1_500, + 4_000, + "The simulator's scored payments average ৳2,830 (send-money ৳2,247, cash-out " + "৳3,990); rounded down. Bangladesh Bank's industry value-per-transaction is in the " + "same range.", + ), + "scam_loss_bps": Spec( + "Scam losses, share of payment value", + "basis points", + 0.5, + 8, + "Assumed. The simulator runs at 93 bp, deliberately dense so the model has cases to " + "learn from. UK push-payment scam losses are about 1-2 bp of Faster Payments value " + "(UK Finance, Pay.UK); MFS in Bangladesh is assumed more exposed. No public " + "Bangladeshi figure exists, so this is the assumption to replace first.", + ), + "stop_warn": Spec( + "Scam money stopped by a warning", + "share", + 0.10, + 0.50, + "Assumed. A coached victim often clicks through a warning; studies of payment " + "warnings report partial effect. The customer decides, so the share is low.", + ), + "stop_step_up": Spec( + "Scam money stopped by a step-up check", + "share", + 0.30, + 0.75, + "Assumed. Re-authentication defeats most account takeovers, and the 30-minute " + "cooling-off breaks the urgency a scam script relies on; a determined victim still pays.", + ), + "stop_hold": Spec( + "Scam money stopped by a hold", + "share", + 0.75, + 0.98, + "Assumed. The money stays in the wallet while an analyst calls the customer; some " + "victims insist. Also applied to mule cash-outs held after the victim's payment.", + ), + "abandon_rate": Spec( + "Honest payments abandoned after an interruption", + "share", + 0.01, + 0.15, + "Assumed. Most honest customers read the warning and continue; a step-up or a hold " + "loses some of them to another channel.", + ), + "abandon_cost_taka": Spec( + "Cost of one abandoned honest payment", + "BDT", + 10, + 60, + "Lost fee revenue and goodwill: cash-out pays about 1.85% (৳74 on ৳4,000), " + "app send-money ৳0-5; many abandoned payments are retried later.", + ), + "contacts_warn": Spec( + "Support contacts per warning", + "contacts", + 0.0, + 0.05, + "Assumed: a warning rarely leads to a call.", + ), + "contacts_step_up": Spec( + "Support contacts per step-up check", + "contacts", + 0.05, + 0.25, + "Assumed: a paused payment brings some customers to the helpline.", + ), + "contacts_hold": Spec( + "Support contacts per hold", + "contacts", + 0.10, + 0.50, + "Assumed: the analyst calls the customer anyway (counted as review time); this is " + "the extra inbound calls.", + ), + "contact_cost_taka": Spec( + "Cost of one support contact", + "BDT", + 25, + 100, + "About six minutes of a contact-centre agent at roughly ৳35,000 a month loaded, " + "plus telephony.", + ), + "review_minutes": Spec( + "Analyst minutes per held payment", + "minutes", + 4, + 20, + "The impact page's default: read the case note, call the customer, decide.", + ), + "analyst_monthly_cost_taka": Spec( + "Analyst cost a month, loaded", + "BDT", + 40_000, + 100_000, + "A fraud-operations analyst in Dhaka at about ৳45,000 a month, plus a third for " + "benefits, seat and supervision.", + ), + "analyst_hours_per_month": Spec( + "Hours of case work per analyst a month", + "hours", + 110, + 150, + "Six hours of case work in an eight-hour shift, 22 shifts a month.", + ), + "min_analysts": Spec( + "Minimum analysts for round-the-clock cover", + "people", + 3, + 10, + "Holds have a 30-minute review target at any hour: one seat staffed 24x7 takes " + "about five people.", + ), + "platform_monthly_cost_taka": Spec( + "Running the platform a month", + "BDT", + 250_000, + 1_500_000, + "Assumed: API and stream nodes, Postgres, Redis, and part of an engineer's time " + "for monitoring and retraining.", + ), + "reimbursement_share": Spec( + "Share of scam losses the provider reimburses", + "share", + 0.0, + 1.0, + "Assumed. Bangladesh has no mandatory reimbursement rule; the UK has required 100% " + "(up to £85,000) since October 2024, the upper end of the range.", + ), + "reputation_per_taka": Spec( + "Reputational cost per taka a customer loses", + "BDT per BDT", + 0.0, + 1.0, + "Assumed: victims and the people they tell use the wallet less, and the regulator " + "pays attention. The hardest number here to defend; shown separately.", + ), +} + +_RATE_KEYS = ("alert_rate", "precision", "taka_recall", "taka_recall_with_exit_holds") + + +def _cumulative(point: dict) -> dict[str, float]: + """Per-payment rates of everything at or above one threshold.""" + rate, precision = float(point["alert_rate"]), float(point["precision"]) + return { + "true": rate * precision, + "false": rate * (1 - precision), + "recall": float(point["taka_recall"]), + "recall_exit": float(point["taka_recall_with_exit_holds"]), + } + + +def _loss_per_payment(a: Assumptions) -> float: + return a.avg_payment_taka * a.scam_loss_bps / 10_000 + + +def _synthetic_loss_per_payment(insights: dict) -> float: + return insights["at_thresholds"]["warn"]["taka_at_risk"] / insights["rows"] + + +def _scale(insights: dict, a: Assumptions) -> float: + """Real scam money per payment over the simulator's: rescales the true alerts.""" + synthetic = _synthetic_loss_per_payment(insights) + return _loss_per_payment(a) / synthetic if synthetic else 0.0 + + +def nearest(points: list[dict], threshold: float) -> int: + """The sweep point closest to `threshold`, on a log scale (as the impact page does).""" + return min(range(len(points)), key=lambda i: abs(math.log(points[i]["threshold"] / threshold))) + + +def _point(insights: dict, point: dict, a: Assumptions, scale: float) -> dict: + """The monthly money of alerting at `point` as the warn threshold. + + Step-up and hold keep the policy's thresholds, or follow the slider once it passes + them. Each tier's band is the difference between two cumulative sweep rows. + """ + t = float(point["threshold"]) + here = _cumulative(point) + at = insights["at_thresholds"] + limits = insights["thresholds"] + s = here if t >= limits["step_up"] else _cumulative(at["step_up"]) + h = here if t >= limits["hold"] else _cumulative(at["hold"]) + cum = {"warn": here, "step_up": s, "hold": h} + following = {"warn": s, "step_up": h, "hold": None} + bands = {} + for tier in TIERS: + upper, lower = cum[tier], following[tier] + bands[tier] = {key: max(upper[key] - (lower[key] if lower else 0.0), 0.0) for key in upper} + + volume = a.monthly_payments + at_risk = volume * a.avg_payment_taka * a.scam_loss_bps / 10_000 + stop = {"warn": a.stop_warn, "step_up": a.stop_step_up, "hold": a.stop_hold} + contacts = {"warn": a.contacts_warn, "step_up": a.contacts_step_up, "hold": a.contacts_hold} + + prevented = {tier: at_risk * bands[tier]["recall"] * stop[tier] for tier in TIERS} + # Only a hold stops a mule's cash-out: a warning shown to a mule changes nothing. + exit_share = max(h["recall_exit"] - h["recall"], 0.0) + prevented["mule_cash_out_held"] = at_risk * exit_share * a.stop_hold + prevented_total = sum(prevented.values()) + + interrupted = { + tier: volume * (bands[tier]["true"] * scale + bands[tier]["false"]) for tier in TIERS + } + honest = {tier: volume * bands[tier]["false"] for tier in TIERS} + abandoned = sum(honest.values()) * a.abandon_rate + support_contacts = sum(interrupted[tier] * contacts[tier] for tier in TIERS) + friction = { + "abandoned_payments": abandoned * a.abandon_cost_taka, + "support_contacts": support_contacts * a.contact_cost_taka, + } + + held = interrupted["hold"] + review_hours = held * a.review_minutes / 60 + workload_fte = review_hours / a.analyst_hours_per_month + analysts = max(math.ceil(workload_fte - 1e-9), a.min_analysts) + analyst_cost = analysts * a.analyst_monthly_cost_taka + + operating = sum(friction.values()) + analyst_cost + a.platform_monthly_cost_taka + missed = at_risk - prevented_total + return { + "threshold": t, + "alert_rate": float(point["alert_rate"]), + "interrupted": {tier: round(n) for tier, n in interrupted.items()}, + "interrupted_total": round(sum(interrupted.values())), + "honest_interrupted": round(sum(honest.values())), + "honest_per_10k": round(sum(bands[t]["false"] for t in TIERS) * 10_000, 2), + "held_for_review": round(held), + "prevented": {k: round(v) for k, v in prevented.items()}, + "prevented_total": round(prevented_total), + "prevented_share": round(prevented_total / at_risk, 4) if at_risk else 0.0, + "missed": round(missed), + "friction": {k: round(v) for k, v in friction.items()}, + "friction_total": round(sum(friction.values())), + "abandoned_payments": round(abandoned), + "support_contacts": round(support_contacts), + "review_hours": round(review_hours), + "workload_fte": round(workload_fte, 1), + "analysts": int(analysts), + "analyst_cost": round(analyst_cost), + "platform_cost": round(a.platform_monthly_cost_taka), + "operating_cost": round(operating), + # Customers and provider together: every taka a customer keeps counts, plus the + # reputational cost avoided. + "net_benefit": round(prevented_total * (1 + a.reputation_per_taka) - operating), + # The provider's own cash: only what it would have reimbursed, plus reputation. + "provider_net_benefit": round( + prevented_total * (a.reimbursement_share + a.reputation_per_taka) - operating + ), + "prevented_per_taka_cost": round(prevented_total / operating, 2) if operating else None, + "missed_cost_provider": round(missed * (a.reimbursement_share + a.reputation_per_taka)), + } + + +def business_case(insights: dict, assumptions: Assumptions | None = None) -> dict: + """The monthly money at every threshold of the sweep, under `assumptions`.""" + a = assumptions or Assumptions() + sweep = insights.get("impact") or [] + if not sweep: + raise ValueError("the insights report has no threshold sweep") + scale = _scale(insights, a) + points = [_point(insights, p, a, scale) for p in sweep] + today = nearest(sweep, insights["thresholds"]["warn"]) + best = max(range(len(points)), key=lambda i: points[i]["net_benefit"]) + return { + "model_version": insights.get("model_version"), + "policy_version": insights.get("policy_version"), + "rows": insights["rows"], + "days": insights["days"], + "assumptions": a.model_dump(), + "defaults": Assumptions().model_dump(), + "sources": [ + { + "key": key, + "label": s.label, + "unit": s.unit, + "low": s.low, + "high": s.high, + "source": s.source, + } + for key, s in ASSUMPTIONS.items() + ], + "synthetic": { + "scam_loss_per_payment": round(_synthetic_loss_per_payment(insights), 2), + "scale": round(scale, 5), + }, + "scam_loss_at_risk": round(a.monthly_payments * _loss_per_payment(a)), + "today_index": today, + "best_index": best, + # The thresholds in force, measured exactly (the sweep only comes near them). + "policy": _point(insights, _policy_row(insights), a, scale), + "points": points, + "sensitivity": sensitivity(insights, a), + "break_even_bps": { + "net_benefit": break_even_bps(insights, a, "net_benefit"), + "provider_net_benefit": break_even_bps(insights, a, "provider_net_benefit"), + }, + } + + +def break_even_bps(insights: dict, a: Assumptions, key: str = "net_benefit") -> float | None: + """The scam rate (basis points of payment value) below which running the policy in + force costs more than it saves, everything else held. None if it never pays off + within the field's range.""" + row = _policy_row(insights) + low, high = 0.01, 100.0 # the field's bounds + + def net(bps: float) -> float: + changed = a.model_copy(update={"scam_loss_bps": bps}) + return _point(insights, row, changed, _scale(insights, changed))[key] + + if net(high) < 0: + return None + if net(low) >= 0: + return low + for _ in range(50): + mid = (low + high) / 2 + low, high = (mid, high) if net(mid) < 0 else (low, mid) + return round(high, 3) + + +def _policy_row(insights: dict) -> dict: + """The warn threshold in force, as a sweep row: exact, not the nearest sweep point.""" + return {**insights["at_thresholds"]["warn"], "threshold": insights["thresholds"]["warn"]} + + +def sensitivity(insights: dict, assumptions: Assumptions, row: dict | None = None) -> dict: + """Tornado table at one threshold (default: the policy's warn threshold). + + Each assumption is set to its low and then its high value with the others held; + rows are sorted by how far that swings the net benefit, largest first. The + provider's own net benefit is given too, since reimbursement only moves that one. + """ + row = row or _policy_row(insights) + base = _point(insights, row, assumptions, _scale(insights, assumptions)) + rows = [] + for key, spec in ASSUMPTIONS.items(): + out = {} + for side, value in (("low", spec.low), ("high", spec.high)): + changed = assumptions.model_copy(update={key: type(getattr(assumptions, key))(value)}) + point = _point(insights, row, changed, _scale(insights, changed)) + out[f"net_{side}"] = point["net_benefit"] + out[f"provider_{side}"] = point["provider_net_benefit"] + rows.append( + { + "key": key, + "label": spec.label, + "unit": spec.unit, + "low": spec.low, + "high": spec.high, + **out, + "swing": abs(out["net_high"] - out["net_low"]), + "provider_swing": abs(out["provider_high"] - out["provider_low"]), + } + ) + rows.sort(key=lambda r: (r["swing"], r["provider_swing"]), reverse=True) + return { + "threshold": float(row["threshold"]), + "net_benefit": base["net_benefit"], + "provider_net_benefit": base["provider_net_benefit"], + "rows": rows, + } + + +def _row(p: dict, mark: str = "") -> str: + return ( + f"{p['threshold']:>10.5f} {p['alert_rate']:>8.4%} {p['prevented_total']:>13,} " + f"{p['friction_total']:>11,} {p['analysts']:>8} {p['operating_cost']:>12,} " + f"{p['net_benefit']:>13,} {p['provider_net_benefit']:>13,} " + f"{p['prevented_per_taka_cost'] or 0:>7.2f} {p['honest_per_10k']:>10.2f}{mark}" + ) + + +def print_summary(case: dict) -> None: + print( + f"model {case['model_version']}, policy {case['policy_version']}: " + f"{case['assumptions']['monthly_payments']:,.0f} payments a month, " + f"scam money at risk ৳{case['scam_loss_at_risk']:,} a month " + f"(synthetic scale {case['synthetic']['scale']})" + ) + print( + f"{'threshold':>10} {'alerted':>8} {'prevented':>13} {'friction':>11} {'analysts':>8} " + f"{'operating':>12} {'net':>13} {'provider net':>13} {'per ৳1':>7} {'honest/10k':>10}" + ) + for i, p in enumerate(case["points"]): + mark = " <- nearest to today" if i == case["today_index"] else "" + print(_row(p, mark + (" <- best" if i == case["best_index"] else ""))) + policy = case["policy"] + print(_row(policy, " <- policy in force (exact)")) + print(f"\nat the policy in force: prevented {json.dumps(policy['prevented'])}") + print( + f" friction {json.dumps(policy['friction'])}, held {policy['held_for_review']:,}, " + f"review hours {policy['review_hours']:,}, workload {policy['workload_fte']} FTE, " + f"interrupted {json.dumps(policy['interrupted'])}" + ) + sens = case["sensitivity"] + print( + f"\nsensitivity at the policy's warn threshold: net ৳{sens['net_benefit']:,}, " + f"provider ৳{sens['provider_net_benefit']:,}" + ) + for row in sens["rows"]: + print( + f" {row['label']:<48} {row['low']:>12,} -> ৳{row['net_low']:>12,} " + f"(provider ৳{row['provider_low']:>12,}) {row['high']:>12,} -> " + f"৳{row['net_high']:>12,} (provider ৳{row['provider_high']:>12,})" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="The threshold sweep as a monthly P&L in taka") + parser.add_argument("--model", default=None, help="model version (default: current)") + parser.add_argument( + "--set", action="append", default=[], metavar="KEY=VALUE", help="override an assumption" + ) + parser.add_argument("--json", action="store_true", help="print the whole result as JSON") + args = parser.parse_args() + version = args.model or registry.current_version(settings.models_dir) + insights = json.loads((settings.models_dir / version / INSIGHTS_FILE).read_text()) + overrides = dict(item.split("=", 1) for item in args.set) + case = business_case(insights, Assumptions(**overrides)) + if args.json: + print(json.dumps(case, indent=2)) + else: + print_summary(case) + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_business.py b/backend/tests/test_business.py new file mode 100644 index 0000000..a69ebf7 --- /dev/null +++ b/backend/tests/test_business.py @@ -0,0 +1,170 @@ +"""The business case: the threshold sweep priced in taka.""" + +import json + +import pytest +from pydantic import ValidationError + +from fraudlens.config import settings +from fraudlens.decision import business +from fraudlens.decision.business import ASSUMPTIONS, Assumptions, business_case, sensitivity +from fraudlens.models import registry + + +def _row(threshold, alert_rate, precision, recall, recall_exit): + return { + "threshold": threshold, + "alert_rate": alert_rate, + "precision": precision, + "taka_recall": recall, + "taka_recall_with_exit_holds": recall_exit, + } + + +@pytest.fixture +def report(): + """1,000 payments, ৳10,000 of scam money; tiers at 0.1 / 0.5 / 0.9.""" + hold = _row(0.9, 0.004, 1.0, 0.5, 0.6) + step_up = _row(0.5, 0.006, 0.8, 0.7, 0.8) + warn = {**_row(0.1, 0.010, 0.6, 0.8, 0.9), "taka_at_risk": 10_000} + return { + "model_version": "v9", + "policy_version": "v1", + "rows": 1_000, + "days": 10, + "thresholds": {"warn": 0.1, "step_up": 0.5, "hold": 0.9}, + "at_thresholds": {"warn": warn, "step_up": step_up, "hold": hold}, + "impact": [_row(0.95, 0.002, 1.0, 0.3, 0.35), warn, _row(0.01, 0.05, 0.1, 0.9, 0.95)], + } + + +def _plain(**changes) -> Assumptions: + """Round numbers so the arithmetic can be checked by hand.""" + values = dict( + monthly_payments=1_000_000, + avg_payment_taka=1_000, + scam_loss_bps=100, # ৳10 a payment, the same as the report: scale 1 + stop_warn=0.2, + stop_step_up=0.5, + stop_hold=1.0, + abandon_rate=0.1, + abandon_cost_taka=10, + contacts_warn=0.0, + contacts_step_up=0.0, + contacts_hold=1.0, + contact_cost_taka=5, + review_minutes=6, + analyst_monthly_cost_taka=50_000, + analyst_hours_per_month=100, + min_analysts=0, + platform_monthly_cost_taka=100_000, + reimbursement_share=0.5, + reputation_per_taka=0.0, + ) + return Assumptions(**(values | changes)) + + +def test_the_policy_point_adds_up_by_hand(report): + case = business_case(report, _plain()) + p = case["policy"] + assert case["synthetic"]["scale"] == 1.0 and case["scam_loss_at_risk"] == 10_000_000 + # Bands of recall: warn 0.8-0.7, step-up 0.7-0.5, hold 0.5; exit holds 0.6-0.5. + assert p["prevented"] == { + "warn": 200_000, + "step_up": 1_000_000, + "hold": 5_000_000, + "mule_cash_out_held": 1_000_000, + } + assert p["prevented_total"] == 7_200_000 and p["missed"] == 2_800_000 + # Honest: warn band 0.004-0.0012, step-up 0.0012-0, hold 0 → 4,000 a month. + assert p["honest_interrupted"] == 4_000 and p["honest_per_10k"] == 40 + assert p["interrupted"] == {"warn": 4_000, "step_up": 2_000, "hold": 4_000} + assert p["friction"] == {"abandoned_payments": 4_000, "support_contacts": 20_000} + # 4,000 holds x 6 minutes = 400 hours = 4 analysts. + assert (p["review_hours"], p["analysts"], p["analyst_cost"]) == (400, 4, 200_000) + assert p["operating_cost"] == 324_000 + assert p["net_benefit"] == 7_200_000 - 324_000 + assert p["provider_net_benefit"] == 3_600_000 - 324_000 + assert p["prevented_per_taka_cost"] == round(7_200_000 / 324_000, 2) + + +def test_tiers_collapse_when_the_slider_passes_them(report): + case = business_case(report, _plain()) + strict = case["points"][0] # above the hold threshold: everything alerted is held + assert strict["prevented"]["warn"] == strict["prevented"]["step_up"] == 0 + assert strict["interrupted"]["hold"] == 2_000 == strict["interrupted_total"] + assert case["points"][1]["prevented_total"] == case["policy"]["prevented_total"] + + +def test_a_rarer_scam_rescales_true_alerts_but_not_false_ones(report): + dense, rare = (business_case(report, _plain(scam_loss_bps=b))["policy"] for b in (100, 10)) + assert rare["prevented_total"] == dense["prevented_total"] / 10 + assert rare["honest_interrupted"] == dense["honest_interrupted"] + assert rare["interrupted"]["hold"] == 400 # 4,000 true holds, a tenth as many + assert rare["analysts"] == 1 + + +def test_coverage_floor_and_friction_move_the_net_as_expected(report): + base = business_case(report, _plain())["policy"] + floored = business_case(report, _plain(min_analysts=10))["policy"] + assert floored["analysts"] == 10 and floored["net_benefit"] == base["net_benefit"] - 300_000 + sticky = business_case(report, _plain(abandon_rate=0.5))["policy"] + assert sticky["friction"]["abandoned_payments"] == 20_000 + + +def test_sensitivity_is_sorted_and_reimbursement_moves_only_the_provider(report): + table = sensitivity(report, _plain()) + swings = [row["swing"] for row in table["rows"]] + assert swings == sorted(swings, reverse=True) + assert {row["key"] for row in table["rows"]} == set(ASSUMPTIONS) + reimburse = next(r for r in table["rows"] if r["key"] == "reimbursement_share") + assert reimburse["swing"] == 0 and reimburse["provider_swing"] > 0 + + +def test_break_even_is_where_the_net_crosses_zero(report): + a = _plain() + bps = business.break_even_bps(report, a) + assert bps is not None and 0 < bps < 100 + at = business_case(report, a.model_copy(update={"scam_loss_bps": bps * 1.01}))["policy"] + below = business_case(report, a.model_copy(update={"scam_loss_bps": bps * 0.99}))["policy"] + assert at["net_benefit"] >= 0 > below["net_benefit"] + + +def test_assumptions_are_bounded_and_strict(): + with pytest.raises(ValidationError): + Assumptions(abandon_rate=1.5) + with pytest.raises(ValidationError): + Assumptions(monthly_payments=-1) + with pytest.raises(ValidationError): + Assumptions(wage=1) # unknown fields are refused, not ignored + for key, spec in ASSUMPTIONS.items(): + assert spec.low <= getattr(Assumptions(), key) <= spec.high, key + assert spec.source, key + assert set(ASSUMPTIONS) == set(Assumptions.model_fields) + + +def test_a_report_without_a_sweep_is_refused(report): + with pytest.raises(ValueError): + business_case({**report, "impact": []}) + + +def test_on_the_served_model_the_policy_pays_for_itself(): + version = registry.current_version(settings.models_dir) + path = settings.models_dir / str(version) / "insights.json" + if not version or not path.exists(): + pytest.skip("no evaluated model in artifacts/") + report = json.loads(path.read_text()) + case = business_case(report) + p = case["policy"] + assert len(case["points"]) == len(report["impact"]) + assert 0 < p["prevented_total"] <= case["scam_loss_at_risk"] + assert p["net_benefit"] > 0 and p["prevented_per_taka_cost"] > 1 + assert p["analysts"] >= Assumptions().min_analysts + # Honest payments interrupted per 10,000 is the back-test's false-alert rate. + warn = report["at_thresholds"]["warn"] + false_rate = warn["alert_rate"] * (1 - warn["precision"]) + assert p["honest_per_10k"] == pytest.approx(false_rate * 1e4, abs=0.01) + prevented = [point["prevented_total"] for point in case["points"]] + assert prevented == sorted(prevented) # looser thresholds never stop less + # The scam rate is the assumption that matters most, and the least known. + assert case["sensitivity"]["rows"][0]["key"] == "scam_loss_bps" diff --git a/backend/tests/test_platform.py b/backend/tests/test_platform.py index d6a96cb..5bd89f3 100644 --- a/backend/tests/test_platform.py +++ b/backend/tests/test_platform.py @@ -438,6 +438,17 @@ def test_dashboard_reads(client, tokens, replayed): assert report["model"]["test"]["risk_score"] and report["policy"]["tier_counts"] assert report["model_version"] == report["policy"]["model_version"] + case = client.get("/v1/model/business", headers=analyst).json() + assert len(case["points"]) == len(report["insights"]["impact"]) + assert case["assumptions"] == case["defaults"] and case["sensitivity"]["rows"] + bigger = client.post( + "/v1/model/business", headers=analyst, json={"monthly_payments": 40_000_000} + ).json() + assert bigger["scam_loss_at_risk"] == 2 * case["scam_loss_at_risk"] + bad = client.post("/v1/model/business", headers=analyst, json={"abandon_rate": 2}) + assert bad.status_code == 422 + assert client.post("/v1/model/business", headers=analyst, json={"wage": 1}).status_code == 422 + policy = client.get("/v1/policy", headers=analyst).json() assert policy["tiers"]["hold"]["human_review"] is True assert {"en", "bn"} == set(policy["messages"]["scam"]["hold"]) @@ -445,7 +456,7 @@ def test_dashboard_reads(client, tokens, replayed): warn, step_up, hold = (policy["resolved_thresholds"][t] for t in ("warn", "step_up", "hold")) assert 0 < warn <= step_up <= hold < 1 - for path in ("/v1/metrics/daily", "/v1/model/report", "/v1/policy"): + for path in ("/v1/metrics/daily", "/v1/model/report", "/v1/model/business", "/v1/policy"): assert client.get(path).status_code == 401 assert client.get(path, headers=tokens("upay-core")).status_code == 403 diff --git a/docs/BUSINESS_CASE.md b/docs/BUSINESS_CASE.md new file mode 100644 index 0000000..05e3414 --- /dev/null +++ b/docs/BUSINESS_CASE.md @@ -0,0 +1,185 @@ +# Business case — what the policy is worth in taka + +The model card and the decision policy report recall, precision and alerts a day. +This document turns those into a monthly profit and loss for a provider: money +kept from scammers, what the friction and the review team cost, and what is left. +It answers the three questions a CFO and a CRO ask first: is it worth running, +how many people does it need, and which of our own numbers would change the answer? + +**These are estimates on synthetic data.** The model rates come from the +back-test (model v4, policy v1 thresholds, 134,545 scored payments over 25 test +days the model never trained on). Everything that turns a rate into taka is an +assumption, listed in §2 with its source or reasoning. The simulator's fraud is +far denser than real traffic, so the scam money is rescaled to an assumed real +rate rather than taken from the simulation. A provider would replace the +assumptions with its own figures before relying on any number here. + +Code: `backend/src/fraudlens/decision/business.py`. API: +`GET /v1/model/business` (defaults) and `POST /v1/model/business` (any assumption +overridden). Console: the impact simulator, under "Business case". + +``` +cd backend && uv run python -m fraudlens.decision.business # the table below +uv run python -m fraudlens.decision.business --set monthly_payments=300000000 +``` + +## 1. How a threshold becomes taka + +The impact simulator's slider moves the warn threshold. Step-up and hold keep the +thresholds the policy gives them, or follow the slider once it passes them. For +each of the 40 thresholds in the sweep, and exactly at the policy in force: + +| Line | How it is computed | +| --- | --- | +| Scam money at risk | volume × average payment × scam rate | +| Kept from scammers | scam money alerted in each tier's band × the share that tier actually stops (warn, step-up, hold), plus mule cash-outs held after the victim's payment × the hold share. A warning shown to a mule stops nothing, so only holds count there | +| Friction | honest payments interrupted × abandonment rate × cost of an abandoned payment, plus every interruption × support contacts per tier × cost of a contact | +| Review | every held payment × analyst minutes; analysts = the larger of that workload and the round-the-clock floor, rounded up to whole people | +| Operating cost | friction + analysts + running the platform | +| Net benefit | kept × (1 + reputational cost per taka) − operating cost. Customers and provider together: every taka a customer keeps counts | +| Net benefit to the provider | kept × (reimbursement share + reputational cost per taka) − operating cost. Only the provider's own money | + +Rates per payment come from the back-test and are multiplied by the provider's +volume. The simulator's scams take ৳26.38 per scored payment (about 93 basis +points of payment value). At the assumed real rate of 2 basis points that is +scaled by 0.0212: scam money and true alerts shrink, false alerts do not, because +honest customers look the same whatever the scam rate. + +## 2. Assumptions + +| Assumption | Value | Range tried | Source or reasoning | +| --- | --- | --- | --- | +| Send-money and cash-out payments a month | 20 million | 10–40 million | Order of magnitude for a mid-sized provider (upay scale). Bangladesh Bank's monthly MFS statistics put the industry in the hundreds of millions of transactions a month; bKash scale is roughly 300 million scored payments. Replace with the provider's MIS figure | +| Average payment | ৳2,800 | ৳1,500–4,000 | The simulator's scored payments average ৳2,830 (send-money ৳2,247, cash-out ৳3,990) | +| Scam losses, share of payment value | 2 bp | 0.5–8 bp | Assumed. UK push-payment scam losses are about 1–2 bp of Faster Payments value; MFS in Bangladesh is assumed more exposed. No public Bangladeshi figure exists | +| Scam money stopped by a warning | 25% | 10–50% | Assumed: a coached victim often clicks through | +| … by a step-up check | 50% | 30–75% | Assumed: re-authentication stops most takeovers; the 30-minute cooling-off breaks a scam's urgency | +| … by a hold | 90% | 75–98% | Assumed: the money stays while an analyst calls; some victims insist | +| Honest payments abandoned after an interruption | 5% | 1–15% | Assumed | +| Cost of one abandoned payment | ৳25 | ৳10–60 | Lost fee and goodwill: cash-out pays about 1.85%, app send-money ৳0–5; many are retried | +| Support contacts per warning / step-up / hold | 0.02 / 0.10 / 0.25 | 0–0.05 / 0.05–0.25 / 0.10–0.50 | Assumed | +| Cost of one support contact | ৳50 | ৳25–100 | About six minutes of a contact-centre agent at ৳35,000 a month loaded, plus telephony | +| Analyst minutes per held payment | 8 | 4–20 | The impact page's default | +| Analyst cost a month, loaded | ৳60,000 | ৳40,000–100,000 | About ৳45,000 salary in Dhaka plus a third for benefits, seat, supervision | +| Hours of case work per analyst a month | 132 | 110–150 | Six hours of case work a shift, 22 shifts | +| Minimum analysts | 5 | 3–10 | Holds have a 30-minute target at any hour: one seat staffed 24x7 | +| Running the platform a month | ৳5 lakh | ৳2.5–15 lakh | Assumed: API and stream nodes, Postgres, Redis, part of an engineer | +| Share of scam losses the provider reimburses | 25% | 0–100% | Assumed. No mandatory rule in Bangladesh; the UK has required 100% since October 2024 | +| Reputational cost per taka a customer loses | ৳0.25 | ৳0–1 | Assumed, and the hardest to defend; shown separately in §4 | + +## 3. Results at upay scale (20 million payments a month) + +At the policy in force (warn at 1.49% of payments, measured exactly, not from the +nearest sweep point): + +| | A month | +| --- | --- | +| Scam money at risk | ৳1.12 crore | +| Kept from scammers | **৳88.6 lakh (79.1%)** — hold ৳73.4 lakh, mule cash-outs held ৳10.3 lakh, step-up ৳2.9 lakh, warn ৳2.0 lakh | +| Still lost | ৳23.4 lakh | +| Friction | ৳4.6 lakh — 5,164 abandoned honest payments (৳1.3 lakh), 6,664 support contacts (৳3.3 lakh) | +| Holds to review | 11,544 → 1,539 hours → 11.7 FTE of work → **12 analysts** (৳7.2 lakh) | +| Platform | ৳5.0 lakh | +| Operating cost | ৳16.8 lakh | +| **Net benefit** | **৳94.0 lakh a month (about ৳11.3 crore a year)** | +| Net benefit to the provider alone | ৳27.5 lakh a month (about ৳3.3 crore a year) | +| Kept per ৳1 of operating cost | **৳5.27** | +| Honest customers interrupted | **51.6 per 10,000 payments** (103,277 a month) | +| Break-even scam rate | 0.27 bp (0.69 bp for the provider alone) | + +Across the sweep (selected points): + +| Payments interrupted | Kept from scammers | Operating cost | Net benefit | Provider net | Analysts | Honest per 10,000 | Kept per ৳1 | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 0.05% | ৳9.3 lakh | ৳8.0 lakh | ৳3.5 lakh | −৳3.4 lakh | 5 | 0.0 | 1.15 | +| 0.53% | ৳58.1 lakh | ৳8.3 lakh | ৳64.3 lakh | ৳20.7 lakh | 5 | 0.1 | 7.00 | +| 0.96% | ৳86.1 lakh | ৳14.9 lakh | ৳92.8 lakh | ৳28.2 lakh | 12 | 13.2 | 5.79 | +| **1.21% (best)** | ৳88.0 lakh | ৳15.8 lakh | **৳94.2 lakh** | ৳28.2 lakh | 12 | 30.0 | 5.56 | +| **1.49% (policy)** | ৳88.6 lakh | ৳16.8 lakh | **৳94.0 lakh** | ৳27.5 lakh | 12 | 51.6 | 5.27 | +| 2.46% | ৳90.3 lakh | ৳20.8 lakh | ৳92.0 lakh | ৳24.3 lakh | 12 | 140.5 | 4.33 | +| 5.00% | ৳91.4 lakh | ৳31.9 lakh | ৳82.3 lakh | ৳13.7 lakh | 12 | 387.6 | 2.86 | + +What this says: + +- **The thresholds in force sit at the top of the curve.** The best point in the + sweep nets ৳22,600 a month more (0.2%) by interrupting 1.21% of payments instead + of 1.49%. Loosening further buys little: going from 1.49% to 5% keeps another + ৳2.8 lakh and costs ৳15 lakh more in friction. +- **Holds do the work.** 94% of the money kept comes from the hold tier and the + mule cash-outs it catches; warnings add 2%. That is why the assumed hold + effectiveness matters more than the warning's. +- **At real scam rates most interruptions are honest customers.** In the + simulator 65% of warn-level alerts are fraud; rescaled to 2 bp, 96% of the + 107,406 interruptions a month are honest payments, and about 8,500 of the + 11,544 holds are. The friction cost and the 12 analysts are sized for that. + The cost per honest customer is low (most see one warning), but a CRO should + see the count: about 1 in 200 payments. + +### At bKash scale (300 million payments a month) + +Same assumptions, volume ×15: ৳16.8 crore at risk, **৳13.3 crore kept**, operating +cost ৳1.79 crore (175 analysts, ৳1.05 crore; friction ৳69 lakh), **net benefit +৳14.8 crore a month**, ৳4.85 crore to the provider alone, ৳7.41 kept per ৳1. +The 175 analysts are a real constraint: 173,000 holds a month need a review +operation of that size, or a stricter hold threshold. + +## 4. Sensitivity + +Net benefit of the policy in force at upay scale (৳94.0 lakh a month; ৳27.5 lakh +to the provider), with one assumption at a time moved to the ends of its range. +Largest swing first. + +| Assumption | Low → net benefit (provider) | High → net benefit (provider) | Swing | +| --- | --- | --- | --- | +| Scam losses, share of payment value | 0.5 bp → ৳12.4 lakh (−৳4.2 lakh) | 8 bp → ৳4.20 crore (৳1.54 crore) | ৳4.08 crore | +| Payments a month | 10 M → ৳44.5 lakh (৳11.2 lakh) | 40 M → ৳1.93 crore (৳60.0 lakh) | ৳1.48 crore | +| Average payment | ৳1,500 → ৳43.3 lakh (৳7.7 lakh) | ৳4,000 → ৳1.41 crore (৳45.7 lakh) | ৳97.3 lakh | +| Reputational cost per taka lost | 0 → ৳71.8 lakh (৳5.3 lakh) | 1 → ৳1.60 crore (৳94.0 lakh) | ৳88.6 lakh | +| Scam money stopped by a hold | 75% → ৳76.5 lakh (৳20.5 lakh) | 98% → ৳1.03 crore (৳31.2 lakh) | ৳26.7 lakh | +| Analyst minutes per hold | 4 → ৳97.6 lakh (৳31.1 lakh) | 20 → ৳83.2 lakh (৳16.7 lakh) | ৳14.4 lakh | +| Running the platform | ৳2.5 lakh → ৳96.5 lakh | ৳15 lakh → ৳84.0 lakh | ৳12.5 lakh | +| Analyst cost a month | ৳40,000 → ৳96.4 lakh | ৳100,000 → ৳89.2 lakh | ৳7.2 lakh | +| Cost of one support contact | ৳25 → ৳95.6 lakh | ৳100 → ৳90.6 lakh | ৳5.0 lakh | +| Scam money stopped by a warning | 10% → ৳92.5 lakh | 50% → ৳96.4 lakh | ৳3.9 lakh | +| Honest payments abandoned | 1% → ৳95.0 lakh | 15% → ৳91.4 lakh | ৳3.6 lakh | +| Scam money stopped by a step-up | 30% → ৳92.5 lakh | 75% → ৳95.8 lakh | ৳3.3 lakh | +| Cost of an abandoned payment | ৳10 → ৳94.7 lakh | ৳60 → ৳92.1 lakh | ৳2.6 lakh | +| Support contacts per step-up | 0.05 → ৳94.5 lakh | 0.25 → ৳92.2 lakh | ৳2.3 lakh | +| Support contacts per hold | 0.10 → ৳94.8 lakh | 0.50 → ৳92.5 lakh | ৳2.3 lakh | +| Support contacts per warning | 0 → ৳94.7 lakh | 0.05 → ৳92.9 lakh | ৳1.8 lakh | +| Hours of case work per analyst | 110 → ৳92.8 lakh | 150 → ৳94.6 lakh | ৳1.8 lakh | +| Share reimbursed by the provider | 0% → ৳94.0 lakh (৳5.3 lakh) | 100% → ৳94.0 lakh (৳94.0 lakh) | 0 (provider ৳88.6 lakh) | +| Minimum analysts | 3 → ৳94.0 lakh | 10 → ৳94.0 lakh | 0 (workload needs 12 anyway) | + +Reading it: + +- **The answer depends on the scam rate and the volume, not on the cost lines.** + Every friction and staffing assumption together moves the net benefit by less + than the plausible range of the scam rate alone. The case stays positive for + customers and provider together down to 0.27 bp of scam losses, and for the + provider's own money down to 0.69 bp. +- **For the provider's own P&L, reimbursement and reputation decide it.** With no + reimbursement and no reputational cost the provider still nets ৳5.3 lakh at the + default scam rate; at 0.5 bp it would lose ৳4.2 lakh a month while its + customers kept ৳22.2 lakh (`--set scam_loss_bps=0.5`). Whether that is worth + paying for is a policy choice, and the table makes it visible. +- **The first number to replace is the scam rate.** It is the largest swing and + the least known; a provider's complaint and reimbursement records give it + directly. + +## 5. What this does not show + +- **The model rates are synthetic.** Recall and false-alert rate come from a + simulator ([DATA_ASSUMPTIONS.md](DATA_ASSUMPTIONS.md)). Real scams the model has + not seen will be caught less often; the held-out scam type is caught at 70.4% + against 86.9% overall (MODEL_CARD). +- **Rescaling assumes the scam mix and the alert ranking stay as they are** when + scams are rarer. Real precision would come from the provider's own back-test. +- **Tier effectiveness is assumed.** Nothing here measures how many warned victims + stop; a live A/B of warning texts would. +- **Warnings and step-ups cost no analyst time**, as the policy specifies; only + holds go to a person. A provider that reviewed step-ups would need more people. +- **The platform cost is flat.** At bKash scale it would be higher, though still + small next to the review team. +- **No churn model.** Abandonment is priced per payment; customers who leave + after a false hold are not counted beyond the reputational line. diff --git a/docs/plans/03-business-case.md b/docs/plans/03-business-case.md new file mode 100644 index 0000000..a699f58 --- /dev/null +++ b/docs/plans/03-business-case.md @@ -0,0 +1,59 @@ +# Plan 03 — Business case in taka + +## Goal + +Turn the back-test numbers (86.9% of scams caught at warn, 0.53% of payments +falsely interrupted) into a monthly profit-and-loss a provider's CFO and CRO can +check line by line: money kept from scammers, what the friction and the review +team cost, and what is left. Every assumption is explicit, editable and has a +stated source or rationale; every model number is read from the existing +`insights.json`, so nothing is re-scored and no artifact changes. + +## Judge criterion it moves + +**Business / customer impact (13.67/20).** Today the impact page speaks in +recall and alerts a day. A judge has to do the conversion to money and headcount +themselves, and the synthetic taka figures do not scale to a real provider. This +work answers "is it worth running, at our volume, and how many people does it +need?" in BDT, and shows which assumptions the answer depends on. + +## Steps + +1. `backend/src/fraudlens/decision/business.py`: a pure model over the insights + report. + - `Assumptions` (pydantic, bounded fields) with a source or rationale per + field: volume, average payment, real scam rate, share stopped by warn / + step-up / hold, abandonment, support contacts per interruption and their + cost, analyst minutes per hold, analyst cost, coverage floor, platform + cost, reimbursement share, reputational exposure. + - The slider moves the warn threshold; step-up and hold stay where the + policy puts them (or follow the slider when it passes them). Tier bands are + differences of the cumulative sweep and `at_thresholds`. + - Scaling: rates per payment from the back-test, multiplied by monthly + volume; true alerts and scam money rescaled from the synthetic scam rate to + the assumed real one. + - Output per sweep point: prevented loss, friction cost, analyst cost, net + benefit (customers + provider, and provider only), prevented loss per ৳1 + of operating cost, analyst FTE, honest customers interrupted per 10,000. + - `sensitivity()`: tornado table, each assumption at a low and a high value, + sorted by swing in net benefit. + - CLI `python -m fraudlens.decision.business` prints the table. +2. API: `GET /v1/model/business` (defaults) and `POST /v1/model/business` + (assumption overrides) in `routes/ops.py`, next to `/model/report`. +3. Console: extend `app/(console)/impact/page.tsx` with a business-case section: + money stats that follow the slider, editable assumptions (volume presets, + analyst cost, abandonment, scam rate), and the tornado table. Types added to + `lib/types.ts`. +4. Tests: `backend/tests/test_business.py` (arithmetic on a hand-made report, + monotonicity, sensitivity ordering, real artifact, API route). +5. `docs/BUSINESS_CASE.md` with measured numbers and the tornado table, honest + about the synthetic data; one link from README. + +## Definition of done + +- `uv run pytest -q` and `make lint` pass; new tests cover the model and route. +- Dragging the impact slider updates money saved, friction cost, net benefit and + analyst headcount; editing volume, analyst cost or abandonment recomputes. +- `docs/BUSINESS_CASE.md` numbers match `python -m fraudlens.decision.business` + on the current artifacts. +- No default artifact or headline number changed; shared-file edits are additive. diff --git a/frontend/app/(console)/impact/business.tsx b/frontend/app/(console)/impact/business.tsx new file mode 100644 index 0000000..6d9df96 --- /dev/null +++ b/frontend/app/(console)/impact/business.tsx @@ -0,0 +1,270 @@ +"use client"; + +import { useEffect, useState } from "react"; + +import { Legend, LineChart } from "@/components/charts"; +import { Button, Card, Chip, ErrorNote, Loading, Stat, Table, Td, cx, inputClass } from "@/components/ui"; +import { ApiError, api } from "@/lib/api"; +import { num, pct, taka } from "@/lib/format"; +import type { BusinessAssumptions, BusinessCase, BusinessPoint } from "@/lib/types"; + +/** Send-money and cash-out payments a month: orders of magnitude, not provider figures. */ +const VOLUMES = [ + { label: "upay scale", value: 20_000_000 }, + { label: "bKash scale", value: 300_000_000 }, +] as const; + +/** The assumptions a CFO is most likely to change, as the input shows them. */ +type Edited = Pick; + +function signed(value: number): string { + if (!value) return "same as today"; + return `${value > 0 ? "+" : "−"}${taka(Math.abs(value), true)} vs today`; +} + +function NumberField({ + label, value, onChange, min, max, step, suffix, +}: { + label: string; value: number; onChange: (value: number) => void; min: number; max: number; step: number; suffix?: string; +}) { + return ( + + ); +} + +/** Low and high net benefit of one assumption, drawn around today's net benefit. */ +function Tornado({ sensitivity, scale }: { sensitivity: BusinessCase["sensitivity"]; scale: [number, number] }) { + const [lo, hi] = scale; + const at = (value: number) => ((value - lo) / (hi - lo || 1)) * 100; + const base = at(sensitivity.net_benefit); + return ( + + {sensitivity.rows.map((row) => { + const left = Math.min(row.net_low, row.net_high); + const right = Math.max(row.net_low, row.net_high); + const fmt = (v: number) => (row.unit === "share" ? pct(v, 0) : num(v, 2)); + return ( + + + + + + + ); + })} +
{row.label}{fmt(row.low)} – {fmt(row.high)}{taka(row.net_low, true)} → {taka(row.net_high, true)} +
+
= row.net_low ? "bg-good/70" : "bg-bad/70")} + style={{ left: `${at(left)}%`, width: `${Math.max(at(right) - at(left), 0.5)}%` }} + /> +
+ {lo < 0 &&
} +
+
+ ); +} + +export function BusinessSection({ + index, setIndex, minutes, version, +}: { + index: number; setIndex: (index: number) => void; minutes: number; version: string; +}) { + const [edited, setEdited] = useState(null); + const [state, setState] = useState<{ data?: BusinessCase; error: ApiError | null }>({ error: null }); + + const body = edited ? { ...edited, review_minutes: minutes } : { review_minutes: minutes }; + const key = JSON.stringify(body); + useEffect(() => { + const controller = new AbortController(); + const timer = window.setTimeout(() => { + api(`/v1/model/business?version=${version}`, JSON.parse(key), controller.signal).then( + (data) => setState({ data, error: null }), + (error) => { + if (error?.name !== "AbortError") setState((old) => ({ data: old.data, error })); + }, + ); + }, 250); + return () => { + window.clearTimeout(timer); + controller.abort(); + }; + }, [key, version]); + + const data = state.data; + if (!data) return state.error ? : ; + + const a: Edited = edited ?? data.assumptions; + const change = (patch: Partial) => setEdited({ ...a, ...patch }); + const today = data.policy; + const point: BusinessPoint = index === data.today_index ? today : data.points[index]; + const best = data.points[data.best_index]; + const xs = data.points.map((p) => p.alert_rate * 100); + const labels = data.points.map((p) => pct(p.alert_rate, 2)); + const money = [ + { name: "Scam money kept from fraudsters", color: "var(--color-good)", values: data.points.map((p) => p.prevented_total) }, + { name: "Cost of running it (friction, analysts, platform)", color: "var(--color-bad)", values: data.points.map((p) => p.operating_cost) }, + ]; + const nets = data.sensitivity.rows.flatMap((r) => [r.net_low, r.net_high]); + const span: [number, number] = [Math.min(0, ...nets), Math.max(...nets)]; + const breakEven = data.break_even_bps; + + return ( + <> +
+
+

Business case, a month at your volume

+

+ The same threshold, priced in taka for {num(a.monthly_payments)} send-money and cash-out payments a month and a scam rate of{" "} + {num(a.scam_loss_bps, 2)} basis points of payment value ({taka(data.scam_loss_at_risk, true)} at risk a month). +

+
+ +
+ +
+ = 0 ? "good" : "bad"} + value={taka(point.net_benefit, true)} + sub={`${signed(point.net_benefit - today.net_benefit)} · to the provider alone ${taka(point.provider_net_benefit, true)}`} + /> + + today.friction_total ? "warn" : "plain"} + value={taka(point.friction_total, true)} + sub={`${num(point.abandoned_payments)} honest payments abandoned, ${num(point.support_contacts)} support contacts · ${num(point.honest_per_10k, 1)} honest customers interrupted per 10,000 payments`} + /> + +
+ +
+ + taka(v, true)} xTitle="Share of payments interrupted" mark={{ index, label: "selected" }} /> +
+
+ +
+
+
+ {VOLUMES.map((v) => ( + change({ monthly_payments: v.value })}> + {v.label} + + ))} +
+
+ change({ monthly_payments: Math.round(v * 1e6) })} + min={0.01} + max={2000} + step={1} + /> +
+
+ change({ analyst_monthly_cost_taka: v })} + min={1000} + max={2_000_000} + step={5000} + suffix="BDT" + /> + change({ abandon_rate: v / 100 })} + min={0} + max={100} + step={1} + suffix="%" + /> + change({ scam_loss_bps: v })} + min={0.01} + max={100} + step={0.5} + suffix="basis points" + /> +

+ Analyst minutes per hold come from the assumption above. Running the policy in force pays for itself above{" "} + {breakEven.net_benefit === null ? "–" : `${num(breakEven.net_benefit, 2)} bp`} of scam losses + {breakEven.provider_net_benefit === null ? "" : ` (${num(breakEven.provider_net_benefit, 2)} bp counting the provider’s own money only)`}. +

+ +
+
+
+ +
+ + + +
+ +
+ Where each assumption comes from + + {data.sources.map((s) => ( + + + + + + ))} +
{s.label} + {s.unit === "share" ? pct(data.assumptions[s.key], 0) : num(data.assumptions[s.key], 2)} + {s.unit === "share" ? "" : ` ${s.unit}`} + {s.source}
+
+ +

+ Rates come from the back-test of model {data.model_version}. The simulator’s scams take{" "} + {taka(data.synthetic.scam_loss_per_payment)} per payment, far more than real traffic, so scam money and true alerts are scaled by{" "} + {num(data.synthetic.scale, 4)} to the assumed rate. These are estimates on synthetic data to show the shape of the + trade-off, not a forecast: a provider would replace the assumptions with its own figures first. See docs/BUSINESS_CASE.md. +

+ + ); +} diff --git a/frontend/app/(console)/impact/page.tsx b/frontend/app/(console)/impact/page.tsx index c69728a..a140e1d 100644 --- a/frontend/app/(console)/impact/page.tsx +++ b/frontend/app/(console)/impact/page.tsx @@ -8,6 +8,8 @@ import { useApi } from "@/lib/api"; import { num, pct, taka } from "@/lib/format"; import type { ImpactPoint, Insights, Report } from "@/lib/types"; +import { BusinessSection } from "./business"; + const TIER_NAME = { warn: "Warn", step_up: "Step-up check", hold: "Hold for review" } as const; /** Hours of case work one reviewer gets through in a shift. */ const REVIEW_HOURS = 6; @@ -154,6 +156,8 @@ function Simulator({ insights, version }: { insights: Insights; version: string + +

Measured by replaying model {version} over {num(insights.rows)} payments from {days} days it never trained on. The fraud is simulated, so the taka figures show the shape of the trade-off, not a forecast for a real customer base. Moving the slider changes nothing in production: thresholds @@ -169,7 +173,7 @@ export default function ImpactPage() { <> {(data) => diff --git a/frontend/lib/types.ts b/frontend/lib/types.ts index 3e5818d..353e800 100644 --- a/frontend/lib/types.ts +++ b/frontend/lib/types.ts @@ -432,6 +432,84 @@ export interface ImpactPoint { legit_customers_alerted: number; } +/** What the business case assumes (backend: decision/business.py `Assumptions`). */ +export interface BusinessAssumptions { + monthly_payments: number; + avg_payment_taka: number; + scam_loss_bps: number; + stop_warn: number; + stop_step_up: number; + stop_hold: number; + abandon_rate: number; + abandon_cost_taka: number; + contacts_warn: number; + contacts_step_up: number; + contacts_hold: number; + contact_cost_taka: number; + review_minutes: number; + analyst_monthly_cost_taka: number; + analyst_hours_per_month: number; + min_analysts: number; + platform_monthly_cost_taka: number; + reimbursement_share: number; + reputation_per_taka: number; +} + +/** One threshold of the sweep, priced for a month at the assumed volume. */ +export interface BusinessPoint { + threshold: number; + alert_rate: number; + interrupted: Record<"warn" | "step_up" | "hold", number>; + interrupted_total: number; + honest_interrupted: number; + honest_per_10k: number; + held_for_review: number; + prevented: Record<"warn" | "step_up" | "hold" | "mule_cash_out_held", number>; + prevented_total: number; + prevented_share: number; + missed: number; + friction: { abandoned_payments: number; support_contacts: number }; + friction_total: number; + abandoned_payments: number; + support_contacts: number; + review_hours: number; + workload_fte: number; + analysts: number; + analyst_cost: number; + platform_cost: number; + operating_cost: number; + net_benefit: number; + provider_net_benefit: number; + prevented_per_taka_cost: number | null; + missed_cost_provider: number; +} + +export interface BusinessCase { + model_version: string; + policy_version: string; + rows: number; + days: number; + assumptions: BusinessAssumptions; + defaults: BusinessAssumptions; + sources: { key: keyof BusinessAssumptions; label: string; unit: string; low: number; high: number; source: string }[]; + synthetic: { scam_loss_per_payment: number; scale: number }; + scam_loss_at_risk: number; + today_index: number; + best_index: number; + policy: BusinessPoint; + points: BusinessPoint[]; + sensitivity: { + threshold: number; + net_benefit: number; + provider_net_benefit: number; + rows: { + key: keyof BusinessAssumptions; label: string; unit: string; low: number; high: number; + net_low: number; net_high: number; provider_low: number; provider_high: number; swing: number; provider_swing: number; + }[]; + }; + break_even_bps: { net_benefit: number | null; provider_net_benefit: number | null }; +} + export interface FairRow { group: string; transactions: number; From c53ba5b8f0f8d9b41db17380cd42534b9e9ff2ba Mon Sep 17 00:00:00 2001 From: 0xdevabir <261-15-001@diu.edu.bd> Date: Wed, 7 Oct 2026 09:23:15 +0600 Subject: [PATCH 03/12] Ground simulator and pitch in sourced Bangladesh MFS evidence; add survey kit --- Makefile | 5 +- backend/src/fraudlens/config.py | 3 + backend/src/fraudlens/simulator/config.py | 45 +++++- backend/src/fraudlens/simulator/engine.py | 4 +- backend/src/fraudlens/simulator/fraud.py | 6 + backend/src/fraudlens/simulator/generate.py | 14 +- backend/src/fraudlens/validation/__init__.py | 1 + .../fraudlens/validation/survey_analysis.py | 140 +++++++++++++++++ backend/tests/fixtures/survey_fake.csv | 6 + backend/tests/test_simulator.py | 37 +++++ backend/tests/test_survey_analysis.py | 64 ++++++++ docs/BANGLADESH_CONTEXT.md | 144 ++++++++++++++++++ docs/DATA_ASSUMPTIONS.md | 51 ++++++- docs/plans/02-bangladesh-calibration.md | 42 +++++ docs/validation/analyst_interview_guide.md | 95 ++++++++++++ docs/validation/customer_survey.md | 134 ++++++++++++++++ 16 files changed, 783 insertions(+), 8 deletions(-) create mode 100644 backend/src/fraudlens/validation/__init__.py create mode 100644 backend/src/fraudlens/validation/survey_analysis.py create mode 100644 backend/tests/fixtures/survey_fake.csv create mode 100644 backend/tests/test_survey_analysis.py create mode 100644 docs/BANGLADESH_CONTEXT.md create mode 100644 docs/plans/02-bangladesh-calibration.md create mode 100644 docs/validation/analyst_interview_guide.md create mode 100644 docs/validation/customer_survey.md diff --git a/Makefile b/Makefile index b55078b..5b08e5f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help setup up redis down data features train policy insights intel pipeline test lint fmt \ +.PHONY: help setup up redis down data data-calibrated features train policy insights intel pipeline test lint fmt \ migrate seed load api replay replay-live verify platform \ review retrain shadow models promote console console-build smoke \ demo demo-reset @@ -40,6 +40,9 @@ down: ## Stop everything (the demo too) and keep the data data: ## Generate the synthetic dataset cd backend && uv run python -m fraudlens.simulator.generate +data-calibrated: ## Generate the Bangladesh-calibrated dataset into data/full_calibrated + cd backend && uv run python -m fraudlens.simulator.generate --profile calibrated + features: ## Replay the dataset through the feature engine cd backend && uv run python -m fraudlens.features.build diff --git a/backend/src/fraudlens/config.py b/backend/src/fraudlens/config.py index 3a3b72d..8c9760a 100644 --- a/backend/src/fraudlens/config.py +++ b/backend/src/fraudlens/config.py @@ -16,6 +16,9 @@ class Settings(BaseSettings): data_dir: Path = BACKEND_DIR / "data" artifacts_dir: Path = BACKEND_DIR / "artifacts" dataset: str = "full" # sub-directory of data_dir the platform serves + # Simulator profile for `fraudlens.simulator.generate`: "default" or "calibrated" + # (Bangladesh-sourced parameters, docs/DATA_ASSUMPTIONS.md §11). + sim_profile: str = "default" # Model and policy the API serves. None means the promoted (CURRENT) model version. models_root: Path | None = None diff --git a/backend/src/fraudlens/simulator/config.py b/backend/src/fraudlens/simulator/config.py index 32c5189..6149c71 100644 --- a/backend/src/fraudlens/simulator/config.py +++ b/backend/src/fraudlens/simulator/config.py @@ -1,10 +1,17 @@ """Simulation parameters. Every number here is a documented synthetic assumption -(see docs/DATA_ASSUMPTIONS.md), not a measurement of any real MFS.""" +(see docs/DATA_ASSUMPTIONS.md), not a measurement of any real MFS. -from dataclasses import dataclass +Two profiles exist. "default" is the world every published number comes from. +"calibrated" replaces the parameters that a public Bangladesh source pins down +(DATA_ASSUMPTIONS.md §11, sources in docs/BANGLADESH_CONTEXT.md) and leaves +everything else as it is. +""" + +from dataclasses import dataclass, replace from datetime import datetime DAY = 86_400 +PROFILES = ("default", "calibrated") @dataclass(frozen=True) @@ -35,6 +42,16 @@ class SimConfig: max_mule_wallets: int = 3_000 farming_agent_share: float = 0.015 + profile: str = "default" + # Median (BDT, before the segment multiplier) of spontaneous send-money and + # cash-out amounts. The calibrated profile moves them toward Bangladesh Bank's + # published average ticket sizes. + send_median: float = 1_500.0 + cash_out_median: float = 3_000.0 + # Share of account-takeover cases kept; the rest are run as impersonation + # instead. 1.0 leaves the typology mix (and the random stream) untouched. + ato_keep_share: float = 1.0 + def split_of(self, day: int) -> str: if day < self.train_end_day: return "train" @@ -58,3 +75,27 @@ def small(cls, seed: int = 7) -> "SimConfig": n_heldout_cells=1, max_mule_wallets=600, ) + + def calibrated(self) -> "SimConfig": + """The same world with every parameter that has a public Bangladesh source + set from it. Values and sources: DATA_ASSUMPTIONS.md §11.""" + return replace( + self, + profile="calibrated", + # Smallest Bangladesh Bank daily limit from 27 March 2025 (cash-out, Tk 30,000). + txn_cap=30_000.0, + # Average P2P ticket Tk 3,552 and cash-out Tk 2,132 (Bangladesh Bank, Oct 2025). + # The send median puts the simulated send mean at about Tk 3,520. Most cash-outs + # are balance-driven withdrawals, so the cash-out mean barely moves (§11). + send_median=2_850.0, + cash_out_median=1_400.0, + # 41.2% of personal account holders filed a complaint (TIB 2025). + report_rate=0.412, + # Hacking is 12.3% of fraud reports (TIB 2025), half the default ATO share. + ato_keep_share=0.5, + ) + + def with_profile(self, profile: str) -> "SimConfig": + if profile not in PROFILES: + raise ValueError(f"unknown simulator profile {profile!r}; choose from {PROFILES}") + return self.calibrated() if profile == "calibrated" else self diff --git a/backend/src/fraudlens/simulator/engine.py b/backend/src/fraudlens/simulator/engine.py index 1eea0f8..b417711 100644 --- a/backend/src/fraudlens/simulator/engine.py +++ b/backend/src/fraudlens/simulator/engine.py @@ -187,7 +187,7 @@ def _schedule_day(self, day: int) -> None: continue times = self._times(idx, t0).tolist() if typ == SEND: - amounts = self._amounts(idx, 1500, 0.9, 50) + amounts = self._amounts(idx, cfg.send_median, 0.9, 50) elif typ == PAYMENT: amounts = self._amounts(idx, 450, 0.8, 20) elif typ == RECHARGE: @@ -195,7 +195,7 @@ def _schedule_day(self, day: int) -> None: elif typ == BILL_PAY: amounts = self._amounts(idx, 900, 0.6, 100) elif typ == CASH_OUT: - amounts = self._amounts(idx, 3000, 0.7, 500) + amounts = self._amounts(idx, cfg.cash_out_median, 0.7, 500) else: amounts = self._amounts(idx, 2500, 0.7, 500) topup = typ in (SEND, PAYMENT, RECHARGE, BILL_PAY) diff --git a/backend/src/fraudlens/simulator/fraud.py b/backend/src/fraudlens/simulator/fraud.py index 75f55de..a946f37 100644 --- a/backend/src/fraudlens/simulator/fraud.py +++ b/backend/src/fraudlens/simulator/fraud.py @@ -228,6 +228,12 @@ def _new_case(self, cell: Cell, t0: int) -> None: sim = self.sim rng = sim.rng typology = rng.choice(cell.typologies) + if ( + typology == "account_takeover" + and sim.cfg.ato_keep_share < 1.0 + and rng.random() >= sim.cfg.ato_keep_share + ): + typology = "impersonation" # calibrated profile: fewer takeovers, more deception if typology == "account_takeover": hour = rng.uniform(0, 6) if rng.random() < 0.5 else rng.uniform(0, 24) else: diff --git a/backend/src/fraudlens/simulator/generate.py b/backend/src/fraudlens/simulator/generate.py index 8126892..28228b4 100644 --- a/backend/src/fraudlens/simulator/generate.py +++ b/backend/src/fraudlens/simulator/generate.py @@ -17,7 +17,7 @@ import pandas as pd from ..config import settings -from .config import DAY, SimConfig +from .config import DAY, PROFILES, SimConfig from .engine import ADD_MONEY, CASH_IN, K_WALLET, KIND_NAMES, TYPE_NAMES, Simulation from .world import DISTRICTS, MERCHANT_CATEGORIES, SEGMENTS @@ -210,9 +210,19 @@ def main() -> None: parser.add_argument("--small", action="store_true", help="test-sized world") parser.add_argument("--seed", type=int, default=7) parser.add_argument("--out", type=Path, default=None) + parser.add_argument( + "--profile", + choices=PROFILES, + default=settings.sim_profile, + help="calibrated: Bangladesh-sourced parameters (env FRAUDLENS_SIM_PROFILE)", + ) args = parser.parse_args() cfg = SimConfig.small(args.seed) if args.small else SimConfig(seed=args.seed) - out = args.out or settings.data_dir / ("small" if args.small else "full") + cfg = cfg.with_profile(args.profile) + name = "small" if args.small else "full" + if cfg.profile != "default": + name = f"{name}_{cfg.profile}" # never overwrite the published dataset + out = args.out or settings.data_dir / name meta = generate(cfg, out) print(json.dumps(meta["profile"], indent=2, default=str)) print(f"wrote {out} in {meta['seconds']}s") diff --git a/backend/src/fraudlens/validation/__init__.py b/backend/src/fraudlens/validation/__init__.py new file mode 100644 index 0000000..6fd44b9 --- /dev/null +++ b/backend/src/fraudlens/validation/__init__.py @@ -0,0 +1 @@ +"""Tools for validating FraudLens with real people: survey and interview analysis.""" diff --git a/backend/src/fraudlens/validation/survey_analysis.py b/backend/src/fraudlens/validation/survey_analysis.py new file mode 100644 index 0000000..7e3b683 --- /dev/null +++ b/backend/src/fraudlens/validation/survey_analysis.py @@ -0,0 +1,140 @@ +"""Summarise the customer survey (docs/validation/customer_survey.md). + +uv run python -m fraudlens.validation.survey_analysis responses.csv +uv run python -m fraudlens.validation.survey_analysis responses.csv --by area + +One row per respondent, one column per question ID in the survey. Rows without +consent are dropped before anything is counted. Free text (q14) is never read. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import pandas as pd + +REQUIRED = ("respondent_id", "consent") +CATEGORICAL = { + "area": ("urban", "rural"), + "q01_contacted": ("yes", "no"), + "q02_lost_money": ("yes", "no"), + "q03_scam_type": ( + "impersonation", + "lottery_fee", + "wrong_send", + "investment", + "account_takeover", + "other", + ), + "q05_reported_to": ("nobody", "provider", "police", "cipc", "several"), + "q06_recovered": ("none", "partial", "full"), + "q12_trust": ("more", "same", "less"), + "q13_language": ("bangla", "english", "both"), +} +LIKERT = { + "q07_warn_would_stop": "warn", + "q08_step_up_ok": "step_up", + "q09_hold_ok": "hold", +} +NUMERIC = ("q04_loss_bdt", "q10_max_hold_minutes", "q11_false_alarms_per_year") + + +class SurveyError(ValueError): + pass + + +def load(path: Path) -> pd.DataFrame: + df = pd.read_csv(path, dtype=str, keep_default_na=False) + missing = [c for c in REQUIRED if c not in df.columns] + if missing: + raise SurveyError(f"missing columns: {', '.join(missing)}") + if df["respondent_id"].duplicated().any(): + raise SurveyError("respondent_id is not unique") + for col in CATEGORICAL: + if col in df: + df[col] = df[col].str.strip().str.lower() + return df + + +def _share(s: pd.Series, levels: tuple[str, ...]) -> dict: + s = s[s != ""] + unknown = sorted(set(s) - set(levels)) + if unknown: + raise SurveyError(f"unexpected answers {unknown} in {s.name}") + if s.empty: + return {"n": 0} + counts = s.value_counts() + return {"n": int(len(s)), **{k: round(int(counts.get(k, 0)) / len(s), 3) for k in levels}} + + +def _numbers(s: pd.Series) -> dict: + x = pd.to_numeric(s.replace("", None), errors="coerce").dropna() + if (x < 0).any(): + raise SurveyError(f"negative values in {s.name}") + if x.empty: + return {"n": 0} + return { + "n": int(len(x)), + "median": float(x.median()), + "mean": round(float(x.mean()), 1), + "p90": float(x.quantile(0.9)), + } + + +def _likert(s: pd.Series) -> dict: + x = pd.to_numeric(s.replace("", None), errors="coerce").dropna() + if not x.between(1, 5).all(): + raise SurveyError(f"{s.name} must be 1-5") + if x.empty: + return {"n": 0} + return { + "n": int(len(x)), + "mean": round(float(x.mean()), 2), + "agree": round(float((x >= 4).mean()), 3), # 4 or 5 + "disagree": round(float((x <= 2).mean()), 3), + } + + +def summarise(df: pd.DataFrame) -> dict: + consented = df[df["consent"].str.strip().str.lower() == "yes"] + out: dict = {"responses": len(df), "consented": len(consented)} + for col, levels in CATEGORICAL.items(): + if col in consented: + out[col] = _share(consented[col], levels) + # Scam type, loss, reporting and recovery describe victims only. + lost = consented.get("q02_lost_money") + victims = consented[lost == "yes"] if lost is not None else consented.iloc[0:0] + for col in ("q03_scam_type", "q05_reported_to", "q06_recovered"): + if col in victims: + out[col] = _share(victims[col], CATEGORICAL[col]) + for col in NUMERIC: + if col in consented: + src = victims if col == "q04_loss_bdt" else consented + out[col] = _numbers(src[col]) + out["screens"] = { + name: _likert(consented[col]) for col, name in LIKERT.items() if col in consented + } + return out + + +def summarise_by(df: pd.DataFrame, column: str) -> dict: + if column not in df: + raise SurveyError(f"no column {column}") + groups = {str(k): summarise(g) for k, g in df.groupby(column, sort=True)} + return {"all": summarise(df), "by": column, "groups": groups} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Summarise FraudLens customer survey responses") + parser.add_argument("csv", type=Path) + parser.add_argument("--by", default=None, help="also split by this column, e.g. area") + args = parser.parse_args() + df = load(args.csv) + result = summarise_by(df, args.by) if args.by else summarise(df) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/backend/tests/fixtures/survey_fake.csv b/backend/tests/fixtures/survey_fake.csv new file mode 100644 index 0000000..d0575f4 --- /dev/null +++ b/backend/tests/fixtures/survey_fake.csv @@ -0,0 +1,6 @@ +respondent_id,consent,area,q01_contacted,q02_lost_money,q03_scam_type,q04_loss_bdt,q05_reported_to,q06_recovered,q07_warn_would_stop,q08_step_up_ok,q09_hold_ok,q10_max_hold_minutes,q11_false_alarms_per_year,q12_trust,q13_language,q14_comments +FAKE-001,yes,urban,yes,yes,impersonation,5000,provider,none,5,4,3,30,2,more,bangla,FAKE ROW - not a real person +FAKE-002,yes,urban,yes,no,,,,,4,5,2,10,1,same,both,FAKE ROW - not a real person +FAKE-003,yes,rural,yes,yes,lottery_fee,1000,nobody,none,2,3,1,0,0,less,bangla,FAKE ROW - not a real person +FAKE-004,yes,rural,no,no,,,,,5,5,4,60,3,more,bangla,FAKE ROW - not a real person +FAKE-005,no,urban,yes,yes,wrong_send,99999,police,full,1,1,1,0,0,less,english,FAKE ROW - declined consent so must be ignored diff --git a/backend/tests/test_simulator.py b/backend/tests/test_simulator.py index bba6348..9c16a4e 100644 --- a/backend/tests/test_simulator.py +++ b/backend/tests/test_simulator.py @@ -1,4 +1,7 @@ +from dataclasses import fields + import pandas as pd +import pytest from fraudlens.simulator.config import SimConfig from fraudlens.simulator.engine import Simulation @@ -90,3 +93,37 @@ def test_same_seed_gives_identical_data(small_tables): sim.run() again = build_tables(sim)["transactions"] pd.testing.assert_frame_equal(again, small_tables["transactions"]) + + +def test_default_profile_is_the_published_world(): + assert SimConfig().with_profile("default") == SimConfig() + assert SimConfig.small().with_profile("default") == SimConfig.small() + + +def test_calibrated_profile_changes_only_sourced_parameters(): + base, cal = SimConfig.small(), SimConfig.small().with_profile("calibrated") + changed = {f.name for f in fields(SimConfig) if getattr(base, f.name) != getattr(cal, f.name)} + assert changed == { + "profile", + "txn_cap", + "send_median", + "cash_out_median", + "report_rate", + "ato_keep_share", + } + + +def test_unknown_profile_is_rejected(): + with pytest.raises(ValueError): + SimConfig().with_profile("real") + + +def test_calibrated_world_is_capped_and_has_fewer_takeovers(small_tables): + cfg = SimConfig.small().with_profile("calibrated") + sim = Simulation(cfg) + sim.run() + cal = build_tables(sim) + t = cal["transactions"] + assert t.loc[t["sender_type"] == "wallet", "amount"].max() <= cfg.txn_cap + ato = (cal["cases"]["typology"] == "account_takeover").mean() + assert ato < (small_tables["cases"]["typology"] == "account_takeover").mean() diff --git a/backend/tests/test_survey_analysis.py b/backend/tests/test_survey_analysis.py new file mode 100644 index 0000000..ca80ed1 --- /dev/null +++ b/backend/tests/test_survey_analysis.py @@ -0,0 +1,64 @@ +"""The survey fixture is invented: every respondent_id starts with FAKE-.""" + +from pathlib import Path + +import pandas as pd +import pytest + +from fraudlens.validation.survey_analysis import SurveyError, load, summarise, summarise_by + +FIXTURE = Path(__file__).parent / "fixtures" / "survey_fake.csv" + + +@pytest.fixture +def survey(): + return load(FIXTURE) + + +def test_fixture_is_obviously_fake(survey): + assert survey["respondent_id"].str.startswith("FAKE-").all() + + +def test_rows_without_consent_are_ignored(survey): + s = summarise(survey) + assert s["responses"] == 5 and s["consented"] == 4 + assert s["q04_loss_bdt"]["n"] == 2 # FAKE-005's Tk 99,999 is not counted + assert s["q04_loss_bdt"]["median"] == 3000.0 + + +def test_shares_and_victim_only_questions(survey): + s = summarise(survey) + assert s["q01_contacted"] == {"n": 4, "yes": 0.75, "no": 0.25} + assert s["q02_lost_money"]["yes"] == 0.5 + assert s["q03_scam_type"]["n"] == 2 + assert s["q03_scam_type"]["impersonation"] == 0.5 + assert s["q05_reported_to"]["nobody"] == 0.5 + + +def test_screen_reactions(survey): + screens = summarise(survey)["screens"] + assert screens["warn"] == {"n": 4, "mean": 4.0, "agree": 0.75, "disagree": 0.25} + assert screens["hold"]["agree"] == 0.25 + assert screens["step_up"]["mean"] == 4.25 + + +def test_split_by_area(survey): + out = summarise_by(survey, "area") + assert set(out["groups"]) == {"rural", "urban"} + assert out["groups"]["urban"]["consented"] == 2 + + +def test_bad_answers_are_rejected(tmp_path): + bad = pd.read_csv(FIXTURE, dtype=str, keep_default_na=False) + bad.loc[0, "q07_warn_would_stop"] = "7" + path = tmp_path / "bad.csv" + bad.to_csv(path, index=False) + with pytest.raises(SurveyError): + summarise(load(path)) + + +def test_missing_required_column(tmp_path): + path = tmp_path / "no_consent.csv" + path.write_text("respondent_id,area\nFAKE-001,urban\n") + with pytest.raises(SurveyError): + load(path) diff --git a/docs/BANGLADESH_CONTEXT.md b/docs/BANGLADESH_CONTEXT.md new file mode 100644 index 0000000..c816a86 --- /dev/null +++ b/docs/BANGLADESH_CONTEXT.md @@ -0,0 +1,144 @@ +# Bangladesh context: the problem in numbers + +FraudLens's model results come from synthetic data (DATA_ASSUMPTIONS.md). This +page covers the real problem they stand in for. Every figure is from a public +source, linked below and accessed on **7 October 2026**. "Derived" means we did +the arithmetic ourselves from the sourced figures. Where no public source could +be found, we say so. + +## 1. The problem in numbers + +### Scale of MFS + +| Figure | Value | Source | +| --- | --- | --- | +| Registered MFS accounts, Feb 2025 | 239.24 million (2,392.40 lakh) | [S1] | +| Active accounts, Feb 2025 | 87.15 million (derived: 36.4% of registered) | [S1] | +| Agents, Feb 2025 | 1,856,190 | [S1] | +| Providers | 13 | [S1] | +| Transactions, Feb 2025 | 671.3 million, worth Tk 164,726 crore (≈ Tk 1.65 trillion) | [S1] | +| Average transaction, Feb 2025 | ≈ Tk 2,454 (derived) | [S1] | +| Transactions, Oct 2025 | 678.63 million, worth Tk 1.58 trillion | [S2] | +| Average P2P send / cash-out / cash-in, Oct 2025 | Tk 3,552 / Tk 2,132 / Tk 2,898 (derived) | [S2] | +| Account holders by sex, Oct 2025 | 56% male, 44% female | [S2] | + +### Fraud and what happens to victims + +| Figure | Value | Source | +| --- | --- | --- | +| Payment fraud reported to Bangladesh Bank, 2025 | Tk 926.01 million across 81,423 cases; 10.7% recovered | [S3] | +| MFS share of it | Tk 813.26 million (88% of fraud value); only 8.7% recovered, Tk 742.56 million lost | [S3] | +| Personal MFS holders who were fraud victims | 6.3% (agents 17.0%, merchants 1.6%) | [S4] | +| Personal holders who lost money | 3.6% | [S4] | +| Personal losses reported | Tk 300 to Tk 83,000 | [S4] | +| How the fraud happened (multiple answers) | deceptive or false information 52.6%; phone call or SMS 42.1%; hacking 12.3% | [S4] | +| Victims who went to the police (case or GD) | 7.6% | [S4] | +| Holders with a problem who did not complain | 58.8%; the main reason was "no benefit" (65.8%) | [S4] | +| Holders aware of Bangladesh Bank's complaint centre (CIPC) | 6.2% | [S4] | +| Suspicious transaction/activity reports from MFS to BFIU | 451 (2022), 1,134 (2023), 1,107 (2024) | [S4] | +| Users who had experienced MFS fraud, 2021 survey of 9,279 | 9.3% ("1 in 10"); average loss above Tk 9,000; agents ≈ 13% | [S5] | +| Main fraud types in that survey | compromised PINs and impersonation | [S5] | + +The two surveys measure different things (any fraud in 2021 vs fraud victims +in the TIB sample, Nov 2023–May 2025), so they agree in size but should not be +read as a trend. + +### No public source found + +Per-provider scam loss rates; the share of scam money that is cashed out +within minutes; how long mule wallets stay active; the split of scams by +district; false-positive tolerance of Bangladeshi customers. These are what +the validation kit (docs/validation/) is designed to collect. + +## 2. Stakeholder map + +| Stakeholder | What they need | What FraudLens gives them | +| --- | --- | --- | +| **MFS customers** (239M accounts; 44% women) | Not to lose money; not to be blocked on a normal payment | Warn / step-up / hold before money leaves, in plain language; never a refused payment (DECISION_POLICY.md §2) | +| **MFS providers** (13; bKash, Nagad, Rocket, upay and others) | Lower fraud loss and complaint volume; regulatory compliance; low friction | Real-time scoring with reasons, analyst queue, mule network view, decision audit trail | +| **Agents** (1.86M; 17% have been fraud victims [S4]) | Protection from being used for cash-out; commission income | Agent risk score separating colluding/farming agents from transit hubs | +| **Bangladesh Bank** (PSD, FICSD, CIPC) | Providers that monitor patterns and resolve complaints in time | Monitoring that §10.1(ii) of the 2022 regulations asks for; case records kept and exportable | +| **BFIU** | Better suspicious-transaction reports | Case pages and network evidence that can back an STR | +| **Police / CID** | Evidence that follows the money | Mule chains and shared handsets across cases | + +## 3. Regulatory fit + +- **Bangladesh MFS Regulations 2022** (issued 15 Feb 2022) [S6]: + - §10.1(ii) — providers must monitor transaction patterns to identify + unauthorised or suspicious activity. FraudLens is that monitoring, in real + time. + - §15 — providers must run awareness programs "to combat against fraud and + forgery". The warn screen is awareness at the moment it matters. + - §17.3 — a 24-hour call centre and each dispute resolved within 10 working + days. The hold tier puts a human review on a 30-minute target. + - §17.4 — customers may complain to CIPC; §18.1 — records kept at least 6 + years. Every decision is logged with its model, policy version and reasons. +- **Bangladesh Bank, November 2025** — ordered the 13 MFS operators to deploy + "artificial intelligence–based systems to detect and flag illegal + transactions in real time" (issued about online gambling) [S7]. +- **TIB recommendations, May 2025** [S4] — an AI policy for suspicious + transaction reporting (#3), a central blacklist database (#9), a toll-free + CIPC line and a standard complaint process (#12–13). TIB also notes the + absence of proper use of AI in the sector. +- **The UK comparison** — since 7 October 2024 UK payment firms must reimburse + authorised-push-payment scam victims up to £85,000, split 50:50 between the + sending and receiving firm [S8]. In the first year 88% of money lost was + reimbursed (66% the year before) and 84% of claims were settled within 5 + business days [S9]. Bangladesh has no equivalent: 8.7% of MFS fraud value was + recovered in 2025 [S3]. If liability ever shifts to providers, catching the + payment before it leaves is far cheaper than paying it back. + +## 4. Pitch openers + +Each line is backed by the sources above. + +1. "In 2025, Bangladesh Bank recorded Tk 81 crore stolen through mobile + financial services — and only 8.7% of it came back." [S3] +2. "One in sixteen MFS users has been a fraud victim, and only 7.6% of them + ever went to the police." [S4] +3. "239 million MFS accounts move about Tk 1.6 trillion a month. Fraud + prevention today starts after the money is gone." [S1][S2][S3] +4. "Most MFS fraud is not hacking — 52.6% is deception. The customer types + their own PIN. Only a check before the payment can stop it." [S4] +5. "The UK now returns 88% of scam losses to victims. Bangladesh returns 8.7%. + Bangladesh Bank has already asked operators for real-time AI detection; this + is what that looks like." [S3][S7][S9] + +## Sources + +All accessed 7 October 2026. + +- **[S1]** Bangladesh Bank, *Mobile Financial Services (MFS) comparative + summary statement* (January and February 2025). + https://www.bb.org.bd/en/index.php/financialactivity/mfsdata +- **[S2]** The Financial Express, "MFS transactions maintain rising trend in + Oct '25", 3 January 2026, citing Bangladesh Bank. + https://thefinancialexpress.com.bd/trade/mfs-transactions-maintain-rising-trend-in-oct-25 +- **[S3]** The Financial Express, "Fraudsters gobble up Tk 926m in 2025", + 16 June 2026, citing Bangladesh Bank. + https://thefinancialexpress.com.bd/trade/fraudsters-gobble-up-tk-926m-in-2025 +- **[S4]** Transparency International Bangladesh, *Governance Challenges in + the Mobile Financial Services Sector and Way Forward* (executive summary), + 27 May 2025. Survey of 1,784 personal account holders, 664 agents and 429 + merchants in 32 districts, November 2023–May 2025. + https://www.ti-bangladesh.org/images/2025/report/mfs/Executive-Summary-Mobile-Financial-Services-Sector-En.pdf +- **[S5]** Policy Research Institute survey (Aug–Sep 2021, 9,279 respondents), + reported in Dhaka Tribune, 30 March 2022. + https://www.dhakatribune.com/business/266888/1-out-of-every-10-are-victims-of-mfs-fraud + and The Business Standard, https://www.tbsnews.net/node/394218 +- **[S6]** Bangladesh Bank, *Bangladesh Mobile Financial Services (MFS) + Regulations, 2022*. + https://www.bb.org.bd/aboutus/regulationguideline/psd/mfs_regulations_2022.pdf +- **[S7]** Yogonet, "Bangladesh Bank orders mobile financial service providers + to curb online gambling transactions", 6 November 2025. + https://www.yogonet.com/international/news/2025/11/06/116204-bangladesh-bank-orders-mobile-financial-service-providers-to-curb-online-gambling-transactions +- **[S8]** Payment Systems Regulator, *APP scams*, + https://www.psr.org.uk/our-work/app-scams/ ; DLA Piper, "PSR confirms APP + fraud reimbursement level to be reduced", 27 September 2024. + https://dlapiper.com/en-th/insights/publications/2024/09/psr-confirms-app-fraud-reimbursement-level-to-be-reduced +- **[S9]** Payment Systems Regulator, "One year on: impact of APP + reimbursement on victims", 8 October 2025. + https://www.psr.org.uk/news-and-updates/latest-news/news/one-year-on-impact-of-app-reimbursement-on-victims/ +- **[S10]** Dhaka Tribune, "bKash raises daily, monthly account transaction + limits", 29 March 2025 (Bangladesh Bank PSD notice of 27 March 2025). + https://www.dhakatribune.com/business/377554/bkash-raises-daily-monthly-account-transaction diff --git a/docs/DATA_ASSUMPTIONS.md b/docs/DATA_ASSUMPTIONS.md index eae9a6d..204088f 100644 --- a/docs/DATA_ASSUMPTIONS.md +++ b/docs/DATA_ASSUMPTIONS.md @@ -208,7 +208,8 @@ behaviour that was missing. But the test period is no longer strictly ## 10. Known limitations -- Rates, shares and amounts are informed guesses, not calibrated to real traffic. +- Rates, shares and amounts are informed guesses, not calibrated to real traffic + (the optional calibrated profile, §11, sets the few that have a public source). Absolute metrics will not transfer; the relative findings and the system design are what we claim. - Fraud prevalence is likely higher than in real traffic, to give enough positives @@ -224,3 +225,51 @@ behaviour that was missing. But the test period is no longer strictly review simulator (`make review`) closes cases with the simulator's ground truth, so the labels the feedback loop learns from are more accurate and more complete than real verdicts would be (MODEL_CARD §13). + +## 11. Calibration against Bangladesh sources + +Most of the parameters above have no public source. A few do, and the +**calibrated** profile (`SimConfig.calibrated()`) sets those from public +Bangladesh figures and leaves everything else alone. The **default** profile is +unchanged: every published model, policy and metric still comes from it, and a +test checks that `with_profile("default")` returns the same config. + +``` +cd backend +uv run python -m fraudlens.simulator.generate --profile calibrated # writes data/full_calibrated +FRAUDLENS_SIM_PROFILE=calibrated uv run python -m fraudlens.simulator.generate +``` + +Sources are listed in full, with access dates, in BANGLADESH_CONTEXT.md. + +| Parameter | Default | Calibrated | Sourced value | Source | +| --- | --- | --- | --- | --- | +| `txn_cap` (largest single wallet transaction) | ৳25,000 | ৳30,000 | Bangladesh Bank publishes daily limits, not a per-transaction one: cash-out ৳30,000/day, send money ৳50,000/day (from 27 Mar 2025). We use the smallest. | [S10] | +| `send_median` (send-money amount) | ৳1,500 → mean ৳2,425 | ৳2,850 → mean ৳3,520 | Average P2P transfer ৳3,552 (Oct 2025, derived from 134.26M txns worth ৳476.93bn) | [S2] | +| `cash_out_median` (spontaneous cash-out) | ৳3,000 → mean ৳4,336 | ৳1,400 → mean ৳4,337 | Average cash-out ৳2,132 (Oct 2025, derived). **Not matched**: most simulated cash-outs are balance-driven withdrawals after salary and remittance, which this parameter does not set. | [S2] | +| `report_rate` (share of victim transfers reported) | 0.50 (assumed) | 0.412 | 41.2% of personal holders with a problem complained (TIB). The denominator is all problems, not only fraud, so this is an approximation. | [S4] | +| Account-takeover share of base-typology cases (`ato_keep_share`) | 24.8% (keep 1.0) | 12.5% (keep 0.5; the rest become impersonation) | Hacking 12.3% of fraud reports; deceptive information 52.6% | [S4] | +| Cash-in amount | mean ৳2,978 | unchanged | Average cash-in ৳2,898 (Oct 2025, derived) — already close | [S2] | +| Overall mean transaction | ৳2,456 | ৳2,957 | ৳2,454 (Feb 2025, derived from ৳164,726 crore / 671.3M). The default matched without being tuned to it; the calibrated profile moves away because of the higher send amounts. | [S1] | +| Mean loss per case | ৳8,982 | ৳11,157 | "Over ৳9,000" average loss (2021 survey); TIB range ৳300–83,000 | [S5], [S4] | + +**Effect on the data (full world, seed 7).** 1,207 cases instead of 1,056; the +typology mix moves from 21% to 10.5% account takeover and from 19% to 31% +impersonation. The random stream of the default profile is untouched (the +extra draw for takeovers only happens when `ato_keep_share < 1`). + +**Not calibrated, and why.** + +- *Transaction mix by type.* Bangladesh Bank's Oct 2025 counts give P2P 19.8%, + cash-in 21.0%, cash-out 25.7%; the simulator has 29%, 20% and 15%. The mix + comes out of many behaviours (salary, remittance, agent farming) rather than + one parameter, and re-weighting it is a larger change than this profile makes. +- *Fraud prevalence.* No public source gives a per-transaction scam rate; the + simulator's rate stays deliberately high (§10). +- *No public source found* for district weights, segment shares, mule dwell + times and lifetimes, report delay, colluding-agent share or cell structure. + The validation kit (docs/validation/) is designed to collect several of these + from customers and analysts. + +The calibrated profile has not been used to retrain or re-evaluate the model; +doing so, and reporting the change, is the next step. diff --git a/docs/plans/02-bangladesh-calibration.md b/docs/plans/02-bangladesh-calibration.md new file mode 100644 index 0000000..8605f31 --- /dev/null +++ b/docs/plans/02-bangladesh-calibration.md @@ -0,0 +1,42 @@ +# Plan 02: Bangladesh evidence, simulator calibration, real-user validation kit + +## Goal + +Replace "informed guesses" with sourced Bangladesh figures wherever a public +source exists, say plainly where none does, and give the team the instruments +to collect real evidence from customers and analysts. + +## Judge criteria it moves + +- **Problem relevance (16/20).** The problem is stated in sourced numbers + (MFS scale, scam reports, regulation) instead of assertion. +- **Business impact (13.67/20).** Judges said results are "bounded by synthetic + data, real-world impact not yet validated". A calibration profile tied to + Bangladesh Bank statistics, plus a ready-to-run survey and interview guide, + turns that from an open gap into a concrete validation path. + +## Steps + +1. Research (web): Bangladesh Bank monthly MFS statistics, published MFS scam + figures, Bangladesh Bank MFS regulation and customer-protection guidance, the + UK PSR APP-scam reimbursement rules. Every figure gets a URL and access date. + Nothing unsourced is presented as sourced. +2. `docs/BANGLADESH_CONTEXT.md`: problem in numbers, stakeholder map, + regulatory fit, 3–5 cited pitch openers. +3. Simulator: compare `simulator/` parameters with sourced figures. Add an + opt-in `calibrated` profile (`SimConfig.profile`, `FRAUDLENS_SIM_PROFILE` + env var, `--profile` flag). The default profile and its output are unchanged + (same seed, same bytes). Add the comparison table to `DATA_ASSUMPTIONS.md`. +4. Validation kit: `docs/validation/customer_survey.md` (Bangla + English + consent, 10–15 questions), `docs/validation/analyst_interview_guide.md`, + `backend/src/fraudlens/validation/survey_analysis.py` (CSV in, summary out) + with a test on an obviously fake fixture. + +## Definition of done + +- Every number in `BANGLADESH_CONTEXT.md` and the calibration table has a source + URL and access date, or is marked "no public source found". +- `SimConfig()` default is unchanged; a test checks the calibrated profile + differs only in the sourced parameters and that default is untouched. +- `survey_analysis.py` runs on the fixture and is tested. +- `uv run pytest -q` and `make lint` pass; committed on the worktree branch. diff --git a/docs/validation/analyst_interview_guide.md b/docs/validation/analyst_interview_guide.md new file mode 100644 index 0000000..91427e8 --- /dev/null +++ b/docs/validation/analyst_interview_guide.md @@ -0,0 +1,95 @@ +# Analyst interview guide + +**Purpose.** Check FraudLens's assumptions against the people who would run it: +fraud, risk and AML analysts at an MFS provider or bank, and customer-care +staff who take scam calls. The simulator's typologies, report rates and +response times are assumptions (docs/DATA_ASSUMPTIONS.md); this interview is how +we find out which are wrong. + +**Format.** 45 minutes, one interviewee, two interviewers (one asks, one +notes). Remote or in person. No recording unless the interviewee agrees in +writing. + +**Ground rules (say these at the start).** + +- We want your professional view, not your employer's data. Please do not share + customer details, transaction records, internal thresholds or anything under + NDA. Ranges and "roughly" are fine. +- We will not name you or your organisation. Notes are kept by the team only + and summarised across interviews. +- You can skip any question or stop at any time. + +Note the interviewee's role, years in the job and type of organisation (MFS, +bank, PSP, regulator, other) only — no names. + +--- + +## 1. Today's process (10 min) + +1. Walk us through what happens from the moment a customer reports a scam to + the case being closed. Who touches it, and how long does each step take? +2. How do you find out about a scam other than a customer report? (Rules, + agent tips, other providers, police, BFIU.) +3. What rules or models run on transactions today, and do they act before the + money moves or after? +4. Roughly what share of scam reports end with any money recovered? What stops + recovery most often? + +*Compare with:* BB reported 8.7% recovery of MFS fraud value in 2025; TIB found +58.8% of personal holders who had a problem did not complain +(docs/BANGLADESH_CONTEXT.md). + +## 2. Typologies and mules (10 min) + +5. Which scam types do you see most? Put these in order: impersonation (company + or official), prize/lottery/job fee, "sent by mistake", investment scheme, + account takeover (PIN/OTP stolen). What is missing? +6. How quickly does a receiving wallet empty out after a scam payment? Cash-out + at an agent, onward transfers, or both? +7. How do mule wallets usually look: new accounts, bought accounts, agents' + own wallets? How long do they stay active before being blocked? +8. How often does the same mule or the same handset show up across providers? + Would a shared blacklist help, and what stops it today? + +*Compare with:* simulator typology mix, mule dwell time and cash-out delay +(DATA_ASSUMPTIONS.md §5) — all currently unsourced. + +## 3. The FraudLens console (15 min) + +Show the queue, a case page with its reasons, and the network view (make demo, +docs/DEMO_SCRIPT.md). Then ask: + +9. Would the reasons shown on a case let you decide faster? What is missing + that you would look up elsewhere? +10. How many alerts can one analyst clear in a shift? At what false-alarm rate + does a queue stop being worked? +11. Which of warn / step-up / hold could your organisation actually deploy + today? What would legal, compliance or customer care object to? +12. A hold delays a payment by up to 30 minutes. Who would have to approve + that, and what would make them say yes? + +*Compare with:* DECISION_POLICY.md bands and the analyst capacity assumed in +the review simulator. + +## 4. Data and deployment (5 min) + +13. Which of these signals could you get in real time: device ID, SIM-change + date, wallet age, receiver's inflow in the last hour, agent location? +14. Where would a system like this have to run (on-premise, in-country cloud), + and who signs off on a new model? + +## Close (5 min) + +15. If you could change one thing about how scams are handled, what would it + be? +16. Is there someone else we should talk to? + +Thank them. Within 24 hours, write up notes under the headings above, with +quotes marked as quotes and no identifying details. + +## After 5+ interviews + +Tally the answers to Q5, Q6, Q7 and Q10 in a table and compare with the +simulator's assumptions. Where most analysts disagree with an assumption, +change it in the calibrated profile (`SimConfig.calibrated`) and record the +source as "analyst interviews, n = X, " in DATA_ASSUMPTIONS.md §11. diff --git a/docs/validation/customer_survey.md b/docs/validation/customer_survey.md new file mode 100644 index 0000000..233bb65 --- /dev/null +++ b/docs/validation/customer_survey.md @@ -0,0 +1,134 @@ +# Customer survey: scam experience and reactions to FraudLens screens + +**Purpose.** Every FraudLens number so far comes from synthetic data. This survey +asks real MFS users two things: what scams they have actually met, and whether +they would accept the three interventions the decision policy uses (warn, +step-up, hold; docs/DECISION_POLICY.md). It takes about 8 minutes. + +**Who.** Adult (18+) personal MFS account holders. Aim for at least 100 +respondents, with both urban and rural areas and at least 30% women; TIB's 2025 +survey (1,784 respondents in 32 districts) is the benchmark to compare against. + +**How.** Face to face or by phone, by an enumerator who reads the questions in +Bangla. Record answers on the CSV template at the end; the column names are +the IDs `fraudlens.validation.survey_analysis` reads. + +**What we never ask or record.** Name, phone number, wallet number, PIN, OTP, +NID, or the exact amount in the respondent's account. If a respondent starts to +read out a PIN or OTP, stop them. + +--- + +## Consent (read aloud before any question) + +**বাংলা** + +> আসসালামু আলাইকুম। আমরা FraudLens নামে একটি ছাত্র প্রকল্পের পক্ষ থেকে এসেছি। +> আমরা মোবাইল ব্যাংকিং (যেমন বিকাশ, নগদ, রকেট) প্রতারণা কমানোর একটি পদ্ধতি নিয়ে +> কাজ করছি, এবং জানতে চাই প্রতারণার বিষয়ে আপনার অভিজ্ঞতা কী এবং কিছু সতর্কবার্তা +> আপনার কাছে কেমন লাগে। এতে প্রায় ৮ মিনিট লাগবে। +> +> আমরা আপনার নাম, ফোন নম্বর, অ্যাকাউন্ট নম্বর, পিন বা ওটিপি জানতে চাইব না — দয়া +> করে এগুলো বলবেন না। আপনার উত্তর বেনামে রাখা হবে এবং শুধু গবেষণার জন্য সামগ্রিক +> হিসাবে ব্যবহার করা হবে। আমরা কোনো এমএফএস কোম্পানি বা বাংলাদেশ ব্যাংকের প্রতিনিধি +> নই, এবং এই জরিপ কোনো হারানো টাকা ফেরত আনতে পারবে না। +> +> অংশ নেওয়া সম্পূর্ণ আপনার ইচ্ছা। যে কোনো প্রশ্ন বাদ দিতে পারেন, যে কোনো সময় থামতে +> পারেন। আপনি কি অংশ নিতে রাজি আছেন? + +**English** + +> Hello. We are from FraudLens, a student project working on a way to reduce +> mobile money (bKash, Nagad, Rocket and others) fraud. We would like to hear +> about your experience of scams and how some warning screens feel to you. It +> takes about 8 minutes. +> +> We will not ask for your name, phone number, account number, PIN or OTP — +> please do not tell us these. Your answers are anonymous and are used only in +> aggregate for research. We do not represent any MFS provider or Bangladesh +> Bank, and this survey cannot recover lost money. +> +> Taking part is entirely voluntary. You can skip any question or stop at any +> time. Do you agree to take part? + +Record `consent` = `yes` or `no`. If `no`, thank them and stop; the analysis +drops the row anyway. + +If the respondent has lost money: tell them they can complain to their +provider's helpline (providers must resolve disputes within 10 working days, +MFS Regulations 2022 §17.3) and to Bangladesh Bank's Customer Interests +Protection Centre (CIPC). + +--- + +## Questions + +Profile (enumerator fills in): `respondent_id` (e.g. `DHK-014`; never a phone +number), `area` = `urban` / `rural`. + +### Part A — scam experience (last 12 months) + +| ID | Question (English; read in Bangla) | Answers | +| --- | --- | --- | +| `q01_contacted` | Has anyone called, texted or messaged you trying to get money or a PIN/OTP from your mobile wallet? | `yes` / `no` | +| `q02_lost_money` | Did you lose money to a scam of this kind? | `yes` / `no` | +| `q03_scam_type` | *If yes:* Which is closest to what happened? (read options) | `impersonation` (posed as company, official, relative) / `lottery_fee` (prize, job or loan with a fee) / `wrong_send` ("sent by mistake, please return") / `investment` (high-return scheme) / `account_takeover` (someone got into your account) / `other` | +| `q04_loss_bdt` | *If yes:* Roughly how much did you lose, in taka? | number | +| `q05_reported_to` | *If yes:* Who did you tell? | `nobody` / `provider` / `police` / `cipc` / `several` | +| `q06_recovered` | *If yes:* Did you get any of it back? | `none` / `partial` / `full` | + +### Part B — reactions to the three screens + +Show each mock-up (screenshots from the FraudLens console, docs/screenshots) or +read the script below. Answers 1–5: 1 = strongly disagree, 5 = strongly agree. + +- **Warn.** *"Before you send Tk 8,000: this number was opened 2 days ago and + has received money from many people today. People are often told to send + money to 'return a mistake' or 'unlock a prize'. Do you know this person?"* + [Send anyway] [Cancel] +- **Step-up.** *"This payment looks like a common scam, so it is paused for 30 + minutes. Enter your PIN again to confirm it is really you; after the pause it + goes through unless you cancel."* +- **Hold.** *"We are holding this Tk 15,000 transfer for up to 30 minutes + so a person can check it. You can cancel now, or call your provider's + helpline. The money is still in your wallet."* + +| ID | Statement | Answers | +| --- | --- | --- | +| `q07_warn_would_stop` | The warning would make me stop and check before sending. | 1–5 | +| `q08_step_up_ok` | Confirming with my PIN again and waiting 30 minutes would be acceptable. | 1–5 | +| `q09_hold_ok` | Having a payment held for a while would be acceptable if it protects me. | 1–5 | +| `q10_max_hold_minutes` | What is the longest you would accept a payment being held? (minutes; 0 = never) | number | +| `q11_false_alarms_per_year` | How many times a year could a *normal* payment of yours be warned or held before you would stop trusting it? | number | +| `q12_trust` | If your wallet showed screens like these, would you trust it more, the same, or less? | `more` / `same` / `less` | +| `q13_language` | Which language should these screens be in? | `bangla` / `english` / `both` | +| `q14_comments` | Anything else you want to tell us? (free text; not analysed by the script) | text | + +--- + +## CSV template + +``` +respondent_id,consent,area,q01_contacted,q02_lost_money,q03_scam_type,q04_loss_bdt,q05_reported_to,q06_recovered,q07_warn_would_stop,q08_step_up_ok,q09_hold_ok,q10_max_hold_minutes,q11_false_alarms_per_year,q12_trust,q13_language,q14_comments +``` + +Leave a cell empty when a question was skipped or does not apply. Then: + +``` +cd backend +uv run python -m fraudlens.validation.survey_analysis responses.csv --by area +``` + +The output is JSON: shares for each categorical answer, loss median/mean/p90 +for victims only, and for each screen the mean score and the share who agree +(4–5) or disagree (1–2). + +## What the answers would change + +| Finding | Change in FraudLens | +| --- | --- | +| `q07` agree < 50% | Rewrite the warn text; test the scam-script wording against the message-check copy. | +| `q09` agree < 50% or `q10` median < 30 | Shorten the hold window in DECISION_POLICY.md or reserve hold for the top risk band only. | +| `q11` median low (≤ 2) | Tighten the warn threshold; the policy's false-positive budget is set too loose. | +| `q13` mostly `bangla` | Bangla-first copy for every customer-facing screen. | +| `q03` mix differs from the simulator's typology mix | Re-weight typologies in the calibrated simulator profile (DATA_ASSUMPTIONS.md §11). | From b7ab39162ec2bf229e97936241cc147b5d4b7e59 Mon Sep 17 00:00:00 2001 From: 0xdevabir <261-15-001@diu.edu.bd> Date: Wed, 7 Oct 2026 09:26:33 +0600 Subject: [PATCH 04/12] Add adaptive-adversary experiment: tactics that learn, frozen vs retrained vs drift-gated models Opt-in adversary mode in the simulator (default dataset unchanged), make adversary, tests, and MODEL_CARD section 14 with measured results. --- Makefile | 5 +- backend/src/fraudlens/models/adversary.py | 469 +++++++++++++++++++ backend/src/fraudlens/simulator/adversary.py | 390 +++++++++++++++ backend/src/fraudlens/simulator/engine.py | 7 +- backend/src/fraudlens/simulator/generate.py | 19 +- backend/tests/test_adversary.py | 123 +++++ docs/DATA_ASSUMPTIONS.md | 7 +- docs/MODEL_CARD.md | 106 ++++- docs/plans/05-adaptive-adversary.md | 83 ++++ 9 files changed, 1197 insertions(+), 12 deletions(-) create mode 100644 backend/src/fraudlens/models/adversary.py create mode 100644 backend/src/fraudlens/simulator/adversary.py create mode 100644 backend/tests/test_adversary.py create mode 100644 docs/plans/05-adaptive-adversary.md diff --git a/Makefile b/Makefile index b55078b..da3e43c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help setup up redis down data features train policy insights intel pipeline test lint fmt \ +.PHONY: help setup up redis down data features train policy insights intel adversary pipeline test lint fmt \ migrate seed load api replay replay-live verify platform \ review retrain shadow models promote console console-build smoke \ demo demo-reset @@ -55,6 +55,9 @@ insights: ## Threshold sweep, drift and fairness tables for the dashboards intel: ## Train and evaluate the scam-message classifier (docs/FRAUD_TAXONOMY.md) cd backend && uv run python -m fraudlens.intel.train +adversary: ## Adaptive scammers vs frozen, retrained and drift-gated models (after `make pipeline`) + cd backend && uv run python -m fraudlens.models.adversary + pipeline: data features train policy insights intel ## Data, features, models and policy end to end migrate: ## Bring the database schema up to date diff --git a/backend/src/fraudlens/models/adversary.py b/backend/src/fraudlens/models/adversary.py new file mode 100644 index 0000000..11a2cb6 --- /dev/null +++ b/backend/src/fraudlens/models/adversary.py @@ -0,0 +1,469 @@ +"""Adaptive-adversary experiment: does detection survive scammers who learn? + + uv run python -m fraudlens.models.adversary # backend/data/full and the served model + +The default world runs unchanged to its last day (checked against the stored +dataset, so the stored feature-engine snapshot can carry on from it). Then the +served model is "deployed", new scam cells set up for a week, and the adversary +runs in rounds (`simulator.adversary`): after each round it moves its tactic mix +toward whatever got past the model deployed in that arm. Four arms, each its own +simulation because the adversary reacts to its own arm's model: + +- control: frozen model, adversary does not adapt (the round-0 mix throughout) +- frozen: the served model, never changed +- retrain: retrained after every round, by the `models.train` recipe with the + labels `mlops.retrain` would have (see `labels`) +- drift_gated: retrained only after a round in which `mlops.drift.measure` raises + an alert on the served score + +Writes `artifacts/reports/adversary.json`. Nothing in the model registry or the +dataset is written. +""" + +from __future__ import annotations + +import argparse +import json +import time +from dataclasses import asdict, replace +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.ensemble import IsolationForest +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import StratifiedKFold, cross_val_predict + +from ..config import settings +from ..decision.insights import drift_reference, reference_bins +from ..features import FEATURES, RECIPIENT_FEATURES, FeatureEngine +from ..features.build import LABEL_COLUMNS, Replayer, epoch_seconds +from ..mlops import drift +from ..simulator.adversary import TACTICS, Adversary, AdversaryConfig, deploy +from ..simulator.config import DAY, SimConfig +from ..simulator.engine import K_WALLET, Simulation +from ..simulator.generate import DISTRICT_NAMES, _when, party_id, transaction_table +from . import metrics, registry +from .bundle import ANOMALY_FEATURES, ModelBundle, logit +from .data import AMBIGUOUS_ROLES, LOSS_ROLES, load_frame +from .train import FEEDBACK_COLUMNS, FUSION_MIN_GAIN, SEED, choose_thresholds, fit_booster + +ARMS = ("control", "frozen", "retrain", "drift_gated") +REPORT_FILE = "adversary.json" +TIER = "warn" # the operating point the card's headline case recall is quoted at +SHAP_TOP = 8 + + +# ----------------------------------------------------------------- the world + + +def sim_config(data_dir: Path) -> SimConfig: + """The configuration the stored dataset was generated with.""" + cfg = json.loads((data_dir / "meta.json").read_text())["config"] + return SimConfig(**{**cfg, "start": datetime.fromisoformat(cfg["start"])}) + + +class World: + """One arm's running simulation and the feature engine that follows it.""" + + def __init__(self, data_dir: Path, adversary: Adversary) -> None: + self.cfg = cfg = sim_config(data_dir) + self.sim = sim = Simulation(cfg) + sim.run() + stored = pd.read_parquet(data_dir / "transactions.parquet", columns=["amount"])["amount"] + if len(stored) != len(sim.cols["amount"]) or not np.isclose( + stored.sum(), float(np.sum(sim.cols["amount"])) + ): + raise RuntimeError( + f"the simulator no longer reproduces {data_dir}: regenerate the dataset " + "(make data features) before running the adversary experiment" + ) + self.engine = FeatureEngine.load(data_dir / "engine_after_test.pkl") + self.known_wallets = sim.w.n_wallets + self.done = len(sim.cols["ts"]) + self.day = cfg.days + self.planner = deploy(sim, adversary, cfg.days) + + def step(self, days: int) -> pd.DataFrame: + """Simulate `days` more days and replay them: one row per scored transaction.""" + sim, cfg, engine = self.sim, self.cfg, self.engine + sim.run_days(self.day, self.day + days) + self.day += days + end_ts = self.day * DAY + txns = transaction_table(sim, self.done) + self.done += len(txns) + + w = sim.w + new = range(self.known_wallets, w.n_wallets) + created = epoch_seconds(_when(cfg, [w.created_ts[i] for i in new])).tolist() + for i, ts in zip(new, created, strict=True): + engine.register_wallet(party_id(K_WALLET, i), ts, DISTRICT_NAMES[w.district[i]]) + self.known_wallets = w.n_wallets + + flagged = sorted( + (m.flagged_ts, party_id(K_WALLET, m.wallet)) + for m in self.planner.mules.values() + if m.flagged_ts < end_ts and party_id(K_WALLET, m.wallet) not in engine.flagged + ) + flags = pd.DataFrame( + { + "wallet_id": [f[1] for f in flagged], + "flagged_at": _when(cfg, [f[0] for f in flagged]), + } + ) + pos, x = Replayer(engine, flags).run(txns) + frame = txns.iloc[pos][list(LABEL_COLUMNS)].reset_index(drop=True) + frame = pd.concat([frame, pd.DataFrame(x, columns=list(FEATURES))], axis=1) + frame["y"] = frame["is_fraud"].astype("int8") + frame["y_loss"] = frame["fraud_role"].isin(LOSS_ROLES).astype("int8") + frame = frame[~frame["fraud_role"].isin(AMBIGUOUS_ROLES)].reset_index(drop=True) + mules = self.planner.mules + first = {party_id(K_WALLET, k): m.first_fraud_ts for k, m in mules.items()} + first_ts = _when(cfg, frame["receiver_id"].map(first).fillna(np.inf).to_numpy()) + frame["y_mule"] = ((frame["type"] == "SEND_MONEY") & (frame["ts"] >= first_ts)).astype( + "int8" + ) + frame["tactic"] = frame["case_id"].map(self.planner.tactic).fillna("") + return frame + + def reported_cases(self) -> set[int]: + """Scams a victim has reported by now: what a complaint desk would know.""" + now = self.day * DAY + return {case for case, ts in self.planner.reported.items() if ts < now} + + +# ---------------------------------------------------------------- the models + + +class Deployed: + """A model as served: bundle, tier thresholds and its drift reference.""" + + def __init__(self, bundle: ModelBundle, thresholds: dict[str, float], reference: dict): + self.bundle = bundle + self.thresholds = thresholds + self.reference = reference + + def risk(self, frame: pd.DataFrame) -> np.ndarray: + return self.bundle.score(frame[list(FEATURES)].to_numpy(dtype=np.float64))["risk"] + + +def fit_bundle(fold: dict[str, pd.DataFrame], version: str) -> tuple[ModelBundle, np.ndarray]: + """The `models.train.train` recipe without its reports: boosted transaction and + mule models, isolation forest, fusion and calibration on val_b, thresholds on val_b. + Returns the bundle (thresholds in its manifest) and its calibrated val_b risk.""" + cols, recipient = list(FEATURES), list(RECIPIENT_FEATURES) + send = {name: part[part["type"] == "SEND_MONEY"] for name, part in fold.items()} + txn = fit_booster(fold["train"], fold["val_a"], cols, "y") + mule = fit_booster(send["train"], send["val_a"], recipient, "y_mule") + behaviour = fold["train"][list(ANOMALY_FEATURES)] + sample = behaviour.fillna(behaviour.median()).sample( + min(len(behaviour), 200_000), random_state=SEED + ) + anomaly = IsolationForest(n_estimators=100, random_state=SEED, n_jobs=-1).fit(sample.to_numpy()) + bundle = ModelBundle( + version=version, txn=txn, mule=mule, anomaly=anomaly, + anomaly_fill=behaviour.median().to_numpy(), fusion_coef=np.zeros(4), + fusion_intercept=0.0, calibration=(1.0, 0.0), risk_source="txn", manifest={}, + ) # fmt: skip + vb = fold["val_b"] + parts = bundle.components(vb[cols].to_numpy()) + z, y = bundle.fusion_inputs(parts), vb["y"].to_numpy() + fusion = LogisticRegression(C=1.0, max_iter=2000) + cv = StratifiedKFold(5, shuffle=True, random_state=SEED) + fused_cv = cross_val_predict(fusion, z, y, cv=cv, method="predict_proba")[:, 1] + fusion.fit(z, y) + bundle.fusion_coef, bundle.fusion_intercept = fusion.coef_[0], float(fusion.intercept_[0]) + val_txn = metrics.ranking(y, parts["txn"])["pr_auc"] + val_fused = metrics.ranking(y, fused_cv)["pr_auc"] + bundle.risk_source = "fused" if val_fused >= val_txn + FUSION_MIN_GAIN else "txn" + raw = bundle.raw_risk(parts) + platt = LogisticRegression(C=1e6, max_iter=2000).fit(logit(raw).reshape(-1, 1), y) + bundle.calibration = (float(platt.coef_[0][0]), float(platt.intercept_[0])) + risk = bundle.calibrate(raw) + bundle.manifest = {"version": version, "thresholds": choose_thresholds(y, risk)} + return bundle, risk + + +class Lab: + """What every arm shares: the training folds, drift references, the served model.""" + + def __init__(self, data_dir: Path, models_root: Path | None, version: str | None) -> None: + self.data_dir = data_dir + frame = load_frame(data_dir) + clean = frame[~frame["ambiguous"]] + self.fold = {n: clean[clean["fold"] == n] for n in ("train", "val_a", "val_b", "test")} + served = registry.load(version, models_root) + self.version = served.version + thresholds = dict(served.manifest["thresholds"]) + ref_rows = pd.concat([self.fold["train"], self.fold["val_b"]], ignore_index=True) + risk = served.score(ref_rows[list(FEATURES)].to_numpy(dtype=np.float64))["risk"] + self.population_reference = drift_reference(ref_rows, risk, thresholds) + self.served = Deployed(served, thresholds, self.population_reference) + + test = self.fold["test"] + test_risk = self.served.risk(test) + self.served_check = { + "test_pr_auc": metrics.ranking(test["y"].to_numpy(), test_risk)["pr_auc"], + "manifest_pr_auc": served.manifest.get("headline", {}).get("pr_auc"), + } + + # The same monitor pointed at confirmed fraud only: training-period fraud as + # the feature reference, the model's scores on val_b fraud as the score reference. + fraud_train = self.fold["train"][self.fold["train"]["y"] == 1] + self.fraud_features = { + name: reference_bins(fraud_train[name].to_numpy(dtype=np.float64)) for name in FEATURES + } + self.val_b_fraud = self.fold["val_b"][self.fold["val_b"]["y"] == 1] + + def fraud_reference(self, model: Deployed) -> dict: + val_b_fraud_risk = model.risk(self.val_b_fraud) + thresholds = model.thresholds + return { + "features": self.fraud_features, + "score": reference_bins(val_b_fraud_risk), + "validation_alert_rate": round(float((val_b_fraud_risk >= thresholds[TIER]).mean()), 5), + "validation_hold_rate": round( + float((val_b_fraud_risk >= thresholds["hold"]).mean()), 5 + ), + } + + def retrain(self, feedback: pd.DataFrame, version: str) -> Deployed: + """Verdicts and complaints join the training fold, as `models.train.add_feedback` does.""" + fold = dict(self.fold) + rows = feedback[list(FEEDBACK_COLUMNS)].astype({"y": "int8", "y_mule": "int8"}) + fold["train"] = pd.concat([fold["train"], rows], ignore_index=True) + bundle, val_b_risk = fit_bundle(fold, version) + thresholds = bundle.manifest["thresholds"] + reference = { + **self.population_reference, + "score": reference_bins(val_b_risk), + "validation_alert_rate": round(float((val_b_risk >= thresholds[TIER]).mean()), 5), + "validation_hold_rate": round(float((val_b_risk >= thresholds["hold"]).mean()), 5), + } + return Deployed(bundle, thresholds, reference) + + +# -------------------------------------------------------------- measurement + + +def labels(frames: list[pd.DataFrame], alerted: list[np.ndarray], reported: set[int]): + """What a retrain could learn from by now, in `FEEDBACK_COLUMNS`. + + Verdicts: every alerted transaction, closed with its true label (as `make review` + closes cases with the simulation's ground truth). Complaints: every fraud-chain + transaction of a scam whose victim has reported it. Fraud nobody alerted and + nobody reported stays unlabelled, as it would in production. + """ + parts = [] + for frame, hit in zip(frames, alerted, strict=True): + complaint = frame["case_id"].isin(reported).to_numpy() & (frame["y"] == 1).to_numpy() + parts.append(frame[hit | complaint]) + rows = pd.concat(parts, ignore_index=True).drop_duplicates("txn_id") + rows = rows.assign(y_mule=(rows["y"] * (rows["type"] == "SEND_MONEY")).astype("int8")) + return rows + + +def outcomes(frame: pd.DataFrame, risk: np.ndarray, threshold: float) -> dict: + """Recall and precision at the tier threshold, overall and by tactic.""" + hit = risk >= threshold + overall = metrics.alert_outcomes(frame, hit) + by_tactic = metrics.alert_outcomes(frame.assign(typology=frame["tactic"]), hit)["by_typology"] + keep = ("loss_txns", "cases", "loss_txn_recall", "case_recall", "taka_recall", + "taka_recall_with_exit_holds") # fmt: skip + return { + "pr_auc": metrics.ranking(frame["y"].to_numpy(), risk)["pr_auc"], + **{k: overall[k] for k in ("alerts_per_day", "alert_rate", "precision", *keep)}, + "by_tactic": {t: {k: v[k] for k in keep} for t, v in by_tactic.items() if t}, + } + + +def tactic_results(frame: pd.DataFrame, risk: np.ndarray, threshold: float): + """Per tactic: (scams run, scams with no victim transfer alerted) — the adversary's view.""" + victim = frame[frame["y_loss"] == 1].assign( + hit=risk[(frame["y_loss"] == 1).to_numpy()] >= threshold + ) + caught = victim.groupby("case_id").agg(tactic=("tactic", "first"), hit=("hit", "any")) + out = {} + for tactic, rows in caught.groupby("tactic"): + out[str(tactic)] = (int(len(rows)), int((~rows["hit"]).sum())) + return out + + +def monitor(reference: dict, frame: pd.DataFrame, risk: np.ndarray, thresholds: dict) -> dict: + """`mlops.drift.measure`, summarised: does it raise an alert?""" + if len(frame) < drift.MIN_ROWS: + return {"status": "not_enough_data", "rows": int(len(frame)), "needed": drift.MIN_ROWS} + m = drift.measure(reference, frame[list(FEATURES)].to_numpy(dtype=np.float64), risk, thresholds) + return { + "rows": m["rows"], + "score_psi": m["score"]["psi"], + "score_status": m["score"]["status"], + "alert": m["score"]["status"] != "stable", + "alert_rate": m["score"]["alert_rate"], + "validation_alert_rate": m["score"]["validation_alert_rate"], + "feature_status": m["feature_status"], + "most_shifted": [{"feature": f["feature"], "psi": f["psi"]} for f in m["features"][:5]], + } + + +def shap_shares(model: Deployed, frame: pd.DataFrame) -> list[dict]: + """Mean |SHAP| of the transaction model on victim transfers, as shares of the total.""" + victims = frame[frame["y_loss"] == 1] + if victims.empty: + return [] + txn, _ = model.bundle.contributions(victims[list(FEATURES)].to_numpy(dtype=np.float64)) + mean = np.abs(txn).mean(axis=0) + order = np.argsort(-mean)[:SHAP_TOP] + total = mean.sum() or 1.0 + return [{"feature": FEATURES[i], "share": round(float(mean[i] / total), 4)} for i in order] + + +# ------------------------------------------------------------------- arms + + +def run_arm(arm: str, lab: Lab, acfg: AdversaryConfig, log=print) -> dict: + adversary = Adversary(replace(acfg, adapt=arm != "control")) + world = World(lab.data_dir, adversary) + world.step(acfg.warmup_days) # the new cells open and warm up their mules + model, model_name = lab.served, lab.version + frames, alerted, rounds = [], [], [] + for r in range(acfg.rounds): + mix = {t: round(v, 4) for t, v in adversary.mix.items()} + frame = world.step(acfg.round_days) + risk = model.risk(frame) + threshold = model.thresholds[TIER] + frames.append(frame.assign(risk=risk)) + alerted.append(risk >= threshold) + + result = outcomes(frame, risk, threshold) + population = monitor(model.reference, frame, risk, model.thresholds) + known = labels(frames, alerted, world.reported_cases()) + known_now = known[known["txn_id"].isin(frame["txn_id"]) & (known["y"] == 1)] + fraud_seg = monitor( + lab.fraud_reference(model), known_now, + model.risk(known_now) if len(known_now) else np.array([]), model.thresholds, + ) # fmt: skip + tactics = tactic_results(frame, risk, threshold) + new_mix = adversary.update(tactics) + + retrain = arm == "retrain" or (arm == "drift_gated" and population.get("alert", False)) + rounds.append( + { + "round": r, + "days": [world.day - acfg.round_days, world.day], + "model": model_name, + "mix": mix, + "scams_by_tactic": { + t: {"run": n, "got_through": g} for t, (n, g) in tactics.items() + }, + **result, + "drift_population": population, + "drift_confirmed_fraud": fraud_seg, + "labels_known": {"rows": int(len(known)), "fraud": int(known["y"].sum())}, + "retrain_after": bool(retrain and r < acfg.rounds - 1), + } + ) + log( + f" {arm} round {r}: case recall {result['case_recall']}, victim-txn recall " + f"{result['loss_txn_recall']}, score PSI {population.get('score_psi')}, " + f"model {model_name}" + ) + if retrain and r < acfg.rounds - 1: + model_name = f"{arm}-r{r + 1}" + model = lab.retrain(known, model_name) + rounds[-1]["mix_after"] = {t: round(v, 4) for t, v in new_mix.items()} + + pooled = pd.concat(frames, ignore_index=True) + risk_all = pooled["risk"].to_numpy() + by_tactic = metrics.alert_outcomes( + pooled.assign(typology=pooled["tactic"]), np.concatenate(alerted) + )["by_typology"] + return { + "rounds": rounds, + "pooled_pr_auc": metrics.ranking(pooled["y"].to_numpy(), risk_all)["pr_auc"], + "pooled_by_tactic": { + t: { + k: v[k] + for k in ( + "loss_txns", + "cases", + "loss_txn_recall", + "case_recall", + "taka_recall_with_exit_holds", + ) + } # fmt: skip + for t, v in by_tactic.items() + if t + }, + "shap_first_round": shap_shares(lab.served, frames[0]), + "shap_last_round": shap_shares(model, frames[-1]), + "last_model": model_name, + } + + +def run( + data_dir: Path, + models_root: Path | None = None, + version: str | None = None, + acfg: AdversaryConfig | None = None, + arms: tuple[str, ...] = ARMS, + log=print, +) -> dict: + t0 = time.perf_counter() + acfg = acfg or AdversaryConfig() + lab = Lab(data_dir, models_root, version) + log(f"served model {lab.version}: test PR-AUC {lab.served_check['test_pr_auc']}") + results = {} + for arm in arms: + t = time.perf_counter() + results[arm] = run_arm(arm, lab, acfg, log) + log(f"{arm} done in {time.perf_counter() - t:.0f}s") + summary = { + arm: { + "case_recall": [r["case_recall"] for r in res["rounds"]], + "loss_txn_recall": [r["loss_txn_recall"] for r in res["rounds"]], + "drift_alert": [r["drift_population"].get("alert") for r in res["rounds"]], + } + for arm, res in results.items() + } + return { + "generated_at": datetime.now(UTC).isoformat(timespec="seconds"), + "synthetic": True, + "data_dir": str(data_dir), + "served_model": lab.version, + "served_model_check": lab.served_check, + "tier": TIER, + "tactics": list(TACTICS), + "adversary": asdict(acfg), + "sampling": "none: every transaction of every round is replayed and scored", + "arms": results, + "summary": summary, + "seconds": round(time.perf_counter() - t0, 1), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Adaptive-adversary experiment") + parser.add_argument("--data", type=Path, default=None) + parser.add_argument("--version", default=None, help="model to deploy (default: the served one)") + parser.add_argument("--rounds", type=int, default=AdversaryConfig.rounds) + parser.add_argument("--round-days", type=int, default=AdversaryConfig.round_days) + parser.add_argument("--cells", type=int, default=AdversaryConfig.n_cells) + parser.add_argument("--arms", default=",".join(ARMS)) + parser.add_argument("--out", type=Path, default=None) + args = parser.parse_args() + arms = tuple(a for a in args.arms.split(",") if a) + if unknown := set(arms) - set(ARMS): + parser.error(f"unknown arms: {sorted(unknown)}") + acfg = AdversaryConfig(rounds=args.rounds, round_days=args.round_days, n_cells=args.cells) + report = run(args.data or settings.dataset_dir, settings.models_dir, args.version, acfg, arms) + out = args.out or settings.artifacts_dir / "reports" / REPORT_FILE + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2, default=str)) + print(json.dumps(report["summary"], indent=2)) + print(f"wrote {out} in {report['seconds']}s") + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/simulator/adversary.py b/backend/src/fraudlens/simulator/adversary.py new file mode 100644 index 0000000..9de120a --- /dev/null +++ b/backend/src/fraudlens/simulator/adversary.py @@ -0,0 +1,390 @@ +"""Adaptive adversaries: scam cells that change their scripts after FraudLens is deployed. + +Opt-in only. The default dataset never touches this module: the experiment +(`fraudlens.models.adversary`) runs the default world unchanged, then hands its +fraud planner to an `AdaptivePlanner` on the deployment day. From then on new +cells run in rounds; every scam draws one evasion tactic from the cells' shared +mix, and after each round the `Adversary` moves the mix toward the tactics whose +scams got past the deployed model. + +The tactics are the ones fraud teams report seeing once controls go live: + +- none: the original script +- split_amounts: each victim transfer split into a few payments below a cap +- seasoned_mule: an old customer wallet with years of normal history, bought or + rented, instead of a wallet opened for the purpose +- delayed_cashout: the mule waits hours to days, well past the 30-minute hold + review window, before forwarding or cashing out +- fan_out: a larger pool of mules, each victim paying the least-used one, so no + wallet collects many strangers' money +- mimic_hours: the scam call placed at the victim's own usual transacting hour +""" + +from __future__ import annotations + +import math +import random +from dataclasses import dataclass + +from .config import DAY +from .engine import CASH_OUT, SEND, VICTIM_ROLES, Intent +from .fraud import BASE_TYPOLOGIES, RECRUITABLE, WARMUP_DAYS, Case, Cell, FraudPlanner, Mule +from .world import DISTRICTS + +TACTICS = ("none", "split_amounts", "seasoned_mule", "delayed_cashout", "fan_out", "mimic_hours") +SPLIT_CAPS = (1000.0, 1500.0, 2000.0, 2500.0) # what a split payment is kept below +SEASONED_MIN_AGE_DAYS = 180 +DELAY_HOURS = (6.0, 48.0) +FAN_OUT_POOL = 8 # mules kept ready for fan-out scams, on top of the cell's usual three +SEASONED_POOL = 2 + + +@dataclass(frozen=True) +class AdversaryConfig: + """How the adversary behaves. The numbers are assumptions, stated in the model card.""" + + rounds: int = 6 + round_days: int = 7 + warmup_days: int = WARMUP_DAYS # new cells open and warm up mules before round 0 + n_cells: int = 8 + initial_none: float = 0.5 # round 0: half the scams run the old script + learning_rate: float = 3.0 # multiplicative-weights step on the success rate + floor: float = 0.03 # every tactic keeps being tried a little + min_cases: int = 5 # fewer scams than this with a tactic say nothing about it + adapt: bool = True # False keeps the round-0 mix: the non-adaptive control + seed: int = 11 + + +class Adversary: + """The cells' shared tactic mix and how it learns from the deployed model.""" + + def __init__(self, cfg: AdversaryConfig) -> None: + self.cfg = cfg + self.rng = random.Random(cfg.seed) + rest = (1.0 - cfg.initial_none) / (len(TACTICS) - 1) + self.mix = {t: (cfg.initial_none if t == "none" else rest) for t in TACTICS} + + def draw(self) -> str: + return self.rng.choices(TACTICS, weights=[self.mix[t] for t in TACTICS])[0] + + def update(self, outcomes: dict[str, tuple[int, int]]) -> dict[str, float]: + """`outcomes` maps a tactic to (scams run, scams that got through). Returns the new mix. + + Multiplicative weights on the success rate: a tactic that got through more + often gains share. A tactic tried fewer than `min_cases` times keeps its weight. + """ + cfg = self.cfg + if not cfg.adapt: + return dict(self.mix) + weights = {} + for t in TACTICS: + run, through = outcomes.get(t, (0, 0)) + gain = math.exp(cfg.learning_rate * through / run) if run >= cfg.min_cases else 1.0 + # Unproven tactics are scaled by the mean gain, so they neither win nor lose. + weights[t] = (self.mix[t], gain if run >= cfg.min_cases else None) + proven = [g for _, g in weights.values() if g is not None] + mean_gain = sum(proven) / len(proven) if proven else 1.0 + raw = {t: share * (mean_gain if g is None else g) for t, (share, g) in weights.items()} + total = sum(raw.values()) + free = 1.0 - cfg.floor * len(TACTICS) + self.mix = {t: cfg.floor + free * raw[t] / total for t in TACTICS} + return dict(self.mix) + + +class AdaptivePlanner(FraudPlanner): + """The default planner's state plus adaptive cells that start on `deploy_day`. + + Built from the running planner without drawing any random numbers, so the + world up to the deployment day is exactly the default one. + """ + + def __init__(self, base: FraudPlanner, adversary: Adversary, deploy_day: int) -> None: + self.sim = base.sim + self.mules = base.mules + self.victims = base.victims + self.cases = base.cases + self.cells = base.cells # shared: callbacks queued by `base` still find every cell + self.adversary = adversary + self.deploy_day = deploy_day + self.tactic: dict[int, str] = {} # case id -> tactic + self.reported: dict[int, float] = {} # case id -> when the victim first reported + self.first_adaptive = len(self.cells) + self.seasoned: dict[int, list[Mule]] = {} + self.spare: dict[int, list[Mule]] = {} + self._add_cells() + + @property + def round_start_day(self) -> int: + return self.deploy_day + self.adversary.cfg.warmup_days + + def is_adaptive(self, cell: Cell) -> bool: + return cell.id >= self.first_adaptive + + def _add_cells(self) -> None: + sim = self.sim + w, rng, acfg = sim.w, sim.rng, self.adversary.cfg + start = self.round_start_day + end = start + acfg.rounds * acfg.round_days - 1 + for i in range(acfg.n_cells): + a = BASE_TYPOLOGIES[i % 4] + b = BASE_TYPOLOGIES[(i // 4 + i + 1) % 4] + cell = Cell( + id=len(self.cells), + district=rng.randrange(len(DISTRICTS)), + start_day=start, + end_day=end, + typologies=(a, b) if a != b else (a,), + agents=[], + devices=[w.new_device() for _ in range(rng.randint(2, 4))], + recruited_share=rng.uniform(0.0, 0.35), + p_forward=rng.uniform(0.2, 0.5), + device_reuse=rng.uniform(0.3, 0.8), + heldout=False, + intensity=rng.uniform(0.7, 1.3), + ) + n_agents = rng.choices((0, 1, 2), weights=(0.35, 0.45, 0.20))[0] + free = [ + a + for a in w.agents_by_district[cell.district] + if not w.agent_colluding[a] and not w.agent_farming[a] + ] + for a in rng.sample(free, min(n_agents, len(free))): + w.agent_colluding[a] = True + cell.agents.append(a) + self.cells.append(cell) + self.seasoned[cell.id] = [] + self.spare[cell.id] = [] + + # ----------------------------------------------------------------- mules + + def _maintain_mules(self, cell: Cell, t0: int) -> None: + super()._maintain_mules(cell, t0) + if not self.is_adaptive(cell): + return + for pool, target, make in ( + (self.seasoned[cell.id], SEASONED_POOL, self._seasoned_mule), + (self.spare[cell.id], FAN_OUT_POOL, lambda c, t: self._create_mule(c, 1, t)), + ): + pool[:] = [m for m in pool if m.retire_ts > t0 and m.flagged_ts > t0] + ahead = sum(1 for m in pool if m.retire_ts > t0 + 6 * DAY) + for _ in range(target - ahead): + mule = make(cell, t0) + if mule is not None: + pool.append(mule) + + def _seasoned_mule(self, cell: Cell, t0: int) -> Mule | None: + """An old customer wallet with a normal history, taken over for the cell.""" + w, rng = self.sim.w, self.sim.rng + for _ in range(30): + c = rng.randrange(w.n_customers) + if ( + w.created_ts[c] < t0 - SEASONED_MIN_AGE_DAYS * DAY + and w.segment[c] in RECRUITABLE + and c not in self.mules + and c not in self.victims + and w.blocked_ts[c] == math.inf + ): + usable = t0 + rng.uniform(0, DAY) + mule = Mule(c, cell.id, 1, "seasoned", usable, usable + rng.uniform(10, 20) * DAY) + self.mules[c] = mule + return mule + return None + + # ----------------------------------------------------------------- scams + + def _new_case(self, cell: Cell, t0: int) -> None: + if not self.is_adaptive(cell): + super()._new_case(cell, t0) + return + sim = self.sim + rng, w, cap = sim.rng, sim.w, sim.cfg.txn_cap + tactic = self.adversary.draw() + typology = rng.choice(cell.typologies) + min_balance = {"impersonation": 1500, "wrong_send": 800, "account_takeover": 2000}.get( + typology, 0 + ) + need_app = typology == "account_takeover" + if tactic == "mimic_hours": + # Choose the victim first, then call at the hour they usually transact. + victim = self._pick_victim(t0, min_balance, need_app) + if victim is None: + return + hour = min(23.9, max(0.0, rng.gauss(float(w.peak_hour[victim]), 1.0))) + ts0 = t0 + int(hour * 3600) + else: + if typology == "account_takeover": + hour = rng.uniform(0, 6) if rng.random() < 0.5 else rng.uniform(0, 24) + else: + hour = min(22.0, max(9.0, rng.gauss(15, 3.5))) + ts0 = t0 + int(hour * 3600) + victim = None + + ready = self._ready(cell.l1, ts0) + if tactic == "seasoned_mule": + ready = self._ready(self.seasoned[cell.id], ts0) or ready + elif tactic == "fan_out": + ready = ready + self._ready(self.spare[cell.id], ts0) + if not ready: + if victim is not None: + self.victims.discard(victim) + return + if victim is None: + victim = self._pick_victim(ts0, min_balance, need_app) + if victim is None: + return + mule = rng.choice(ready) + + case = Case(len(self.cases), typology, cell.id, victim, mule.wallet, ts0) + self.cases.append(case) + self.tactic[case.id] = tactic + tag = {"case": case.id, "typology": typology, "cell": cell.id} + + sent_to: dict[int, int] = {} + + def receiver() -> int: + if tactic != "fan_out": + return mule.wallet + # The least-used ready mule takes the next payment. + least = min(ready, key=lambda m: (m.n_victims + sent_to.get(m.wallet, 0), rng.random())) + sent_to[least.wallet] = sent_to.get(least.wallet, 0) + 1 + return least.wallet + + def amounts( + amount: float | None, frac: float | None + ) -> list[tuple[float | None, float | None]]: + """One transfer, or with split_amounts several below a cap.""" + if tactic != "split_amounts": + return [(amount, frac)] + limit = rng.choice(SPLIT_CAPS) + total = amount if amount is not None else w.balance[victim] * (frac or 0.0) + k = max(2, min(6, math.ceil(total / limit))) + part = max(math.floor(total / k / 10) * 10, 50.0) + return [(float(part), None)] * k + + def victim_send(t: float, amount: float | None = None, frac: float | None = None, + **kw) -> float: # fmt: skip + for a, f in amounts(amount, frac): + sim.push( + t, + Intent(SEND, victim, receiver(), a or 0.0, frac=f, role="victim_transfer", + fraud=True, **tag, **kw), + ) # fmt: skip + t += rng.uniform(300, 1800) + return t + + def bait(t: float, amount: float) -> None: + sim.push(t, Intent(SEND, mule.wallet, victim, amount, role="bait", **tag)) + + if typology == "impersonation": + t = ts0 + for _ in range(rng.choices((1, 2, 3), weights=(0.55, 0.30, 0.15))[0]): + t = victim_send(t, frac=rng.uniform(0.5, 0.97)) + t += rng.uniform(60, 480) + + elif typology == "wrong_send": + if rng.random() < 0.15: + bait(ts0, float(rng.choice((20, 50, 100)))) + amount = rng.choices( + (1000, 1500, 2000, 3000, 5000, 8000, 10000, 15000), + weights=(18, 12, 20, 16, 16, 8, 7, 3), + )[0] + victim_send(ts0 + rng.uniform(300, 3600), amount=float(amount)) + + elif typology == "lottery_fee": + t, amount = ts0, float(rng.choice((500, 1000, 1500, 2000))) + for _ in range(rng.randint(2, 5)): + t = victim_send(t, amount=min(amount, cap), topup=True) + if rng.random() < 0.25: + break + amount = round(amount * rng.uniform(1.4, 2.4), -2) + t += rng.uniform(1200, 20 * 3600) + + else: # account_takeover + device = rng.choice(cell.devices) if rng.random() < 0.5 else w.new_device() + direct = bool(cell.agents) and rng.random() < 0.3 + t = ts0 + for _ in range(rng.randint(1, 4)): + for a, f in amounts(None, rng.uniform(0.5, 1.0)): + kw = dict(frac=f, device=device, district=cell.district, + role="ato_transfer", fraud=True, **tag) # fmt: skip + if direct: + sim.push( + t, Intent(CASH_OUT, victim, rng.choice(cell.agents), a or 0.0, **kw) + ) + else: + sim.push(t, Intent(SEND, victim, receiver(), a or 0.0, **kw)) + t += ( + rng.uniform(40, 300) + if tactic != "split_amounts" + else rng.uniform(300, 1800) + ) + + # -------------------------------------------------------------- reaction + + def on_case_transfer(self, ts: int, it: Intent, receiver: int | None, amount: float) -> None: + cell = self.cells[it.cell] + if not self.is_adaptive(cell): + super().on_case_transfer(ts, it, receiver, amount) + return + sim = self.sim + rng = sim.rng + tactic = self.tactic.get(it.case, "none") + if it.role in VICTIM_ROLES: + case = self.cases[it.case] + case.loss += amount + case.n_txn += 1 + if receiver is None or it.role == "bait": + return + mule = self.mules.get(receiver) + if mule is None: + return + mule.first_fraud_ts = min(mule.first_fraud_ts, ts) + mule.last_fraud_ts = max(mule.last_fraud_ts, ts) + tag = {"case": it.case, "typology": it.typology, "cell": it.cell, "fraud": True} + + if it.role in VICTIM_ROLES: + mule.n_victims += 1 + if rng.random() < sim.cfg.report_rate: + delay = rng.lognormvariate(math.log(1.5 * DAY), 0.8) + rng.uniform(3600, 12 * 3600) + first = self.reported.get(it.case, math.inf) + self.reported[it.case] = min(first, ts + delay) + sim.push(ts + delay, lambda t, mu=mule: self._flag(mu, t, "victim_report")) + + out = math.floor(amount * rng.uniform(0.92, 1.0) / 10) * 10 + if out < 50: + return + if tactic == "delayed_cashout": + dwell = rng.uniform(*DELAY_HOURS) * 3600 + elif rng.random() < 0.1: + dwell = rng.uniform(3600, 8 * 3600) + else: + dwell = rng.uniform(60, 1500) + + l2 = self._ready(cell.l2, ts) if mule.level == 1 else [] + if l2 and rng.random() < cell.p_forward: + nxt = rng.choice(l2).wallet + sim.push( + ts + dwell, Intent(SEND, receiver, nxt, float(out), role="mule_forward", **tag) + ) + return + + w = sim.w + if cell.agents and rng.random() < 0.8: + agent = rng.choice(cell.agents) + else: + agent = rng.choice(w.agents_by_district[w.district[receiver]]) + chunks = [out] + if out > 12_000 and rng.random() < 0.5: + first_part = math.floor(out * rng.uniform(0.35, 0.65) / 100) * 100 + chunks = [first_part, out - first_part] + t = ts + dwell + for chunk in chunks: + sim.push(t, Intent(CASH_OUT, receiver, agent, float(chunk), role="mule_cashout", **tag)) + t += rng.uniform(60, 400) + + +def deploy(sim, adversary: Adversary, deploy_day: int) -> AdaptivePlanner: + """Swap the running simulation's fraud planner for an adaptive one.""" + planner = AdaptivePlanner(sim.fraud, adversary, deploy_day) + sim.fraud = planner + return planner diff --git a/backend/src/fraudlens/simulator/engine.py b/backend/src/fraudlens/simulator/engine.py index 1eea0f8..c42b7bc 100644 --- a/backend/src/fraudlens/simulator/engine.py +++ b/backend/src/fraudlens/simulator/engine.py @@ -125,7 +125,12 @@ def push(self, ts: float, item: object) -> None: heapq.heappush(self._heap, (int(ts), self._seq, item)) def run(self) -> None: - for day in range(self.cfg.days): + self.run_days(0, self.cfg.days) + + def run_days(self, start: int, stop: int) -> None: + """Simulate days [start, stop). Called in order, it continues where it stopped, + which is how the adversary experiment steps the world a round at a time.""" + for day in range(start, stop): self._schedule_day(day) self.fraud.schedule_day(day) end = (day + 1) * DAY diff --git a/backend/src/fraudlens/simulator/generate.py b/backend/src/fraudlens/simulator/generate.py index 8126892..6aac6ae 100644 --- a/backend/src/fraudlens/simulator/generate.py +++ b/backend/src/fraudlens/simulator/generate.py @@ -39,21 +39,23 @@ def _when(cfg: SimConfig, seconds) -> pd.Series: return pd.Timestamp(cfg.start) + pd.to_timedelta(s.round(), unit="s") -def build_tables(sim: Simulation) -> dict[str, pd.DataFrame]: - cfg, w, c = sim.cfg, sim.w, sim.cols +def transaction_table(sim: Simulation, start: int = 0) -> pd.DataFrame: + """The transactions emitted so far, from the `start`-th one on (txn_id = position).""" + cfg = sim.cfg + c = {k: v[start:] for k, v in sim.cols.items()} ts = np.asarray(c["ts"], dtype=np.int64) day = ts // DAY - typ = np.asarray(c["type"]) + typ = np.asarray(c["type"], dtype=np.int64) app = np.asarray(c["app"]) - device = np.asarray(c["device"]) + device = np.asarray(c["device"], dtype=np.int64) channel = np.where(app == 1, "app", "ussd").astype(object) channel[typ == CASH_IN] = "agent" channel[typ == ADD_MONEY] = "bank" - txns = pd.DataFrame( + return pd.DataFrame( { - "txn_id": np.arange(len(ts), dtype=np.int64), + "txn_id": np.arange(start, start + len(ts), dtype=np.int64), "ts": _when(cfg, ts), "day": day, "split": np.select( @@ -77,6 +79,11 @@ def build_tables(sim: Simulation) -> dict[str, pd.DataFrame]: } ) + +def build_tables(sim: Simulation) -> dict[str, pd.DataFrame]: + cfg, w = sim.cfg, sim.w + txns = transaction_table(sim) + nw = w.n_wallets mules = sim.fraud.mules used = {wid: m for wid, m in mules.items() if m.n_victims > 0 or m.first_fraud_ts < math.inf} diff --git a/backend/tests/test_adversary.py b/backend/tests/test_adversary.py new file mode 100644 index 0000000..5a159ff --- /dev/null +++ b/backend/tests/test_adversary.py @@ -0,0 +1,123 @@ +import math + +import pytest + +from fraudlens.models import adversary as experiment +from fraudlens.simulator.adversary import ( + DELAY_HOURS, + SEASONED_MIN_AGE_DAYS, + TACTICS, + Adversary, + AdversaryConfig, + deploy, +) +from fraudlens.simulator.config import SimConfig +from fraudlens.simulator.engine import Simulation +from fraudlens.simulator.generate import build_tables, transaction_table + +DAY = 86_400 + + +def test_stepping_the_world_by_days_matches_one_run(small_tables, small_cfg): + sim = Simulation(small_cfg) + sim.run_days(0, 20) + sim.run_days(20, small_cfg.days) + txns = build_tables(sim)["transactions"] + expected = small_tables["transactions"] + assert len(txns) == len(expected) + assert txns["amount"].sum() == pytest.approx(expected["amount"].sum()) + + +def test_mix_moves_toward_what_got_through(): + adv = Adversary(AdversaryConfig()) + start = dict(adv.mix) + outcomes = {t: (20, 0) for t in TACTICS} + outcomes["fan_out"] = (20, 15) + mix = adv.update(outcomes) + assert mix["fan_out"] > start["fan_out"] + assert mix["none"] < start["none"] + assert sum(mix.values()) == pytest.approx(1.0) + assert min(mix.values()) >= AdversaryConfig().floor + + +def test_too_few_cases_say_nothing_and_control_never_adapts(): + adv = Adversary(AdversaryConfig()) + before = dict(adv.mix) + after = adv.update({t: (2, 2) for t in TACTICS}) # below min_cases everywhere + # Only the floor moves it: no tactic gains on another. + assert max(after, key=after.get) == max(before, key=before.get) + assert len({round(after[t], 9) for t in TACTICS if t != "none"}) == 1 + control = Adversary(AdversaryConfig(adapt=False)) + assert control.update({"fan_out": (50, 50)}) == control.mix + + +@pytest.fixture(scope="module") +def adaptive_world(): + """The small world's last 20 days with adaptive cells deployed.""" + cfg = SimConfig.small() + sim = Simulation(cfg) + deploy_day = cfg.days - 20 + sim.run_days(0, deploy_day) + start = len(sim.cols["amount"]) + planner = deploy(sim, Adversary(AdversaryConfig(n_cells=4, warmup_days=4)), deploy_day) + sim.run_days(deploy_day, cfg.days) + return sim, planner, transaction_table(sim, start) + + +def _rows(txns, planner, tactic, role): + cases = [c for c, t in planner.tactic.items() if t == tactic] + return txns[txns["case_id"].isin(cases) & (txns["fraud_role"] == role)] + + +def test_adaptive_cells_try_every_tactic(adaptive_world): + _, planner, txns = adaptive_world + assert set(planner.tactic.values()) == set(TACTICS) + assert txns["case_id"].isin(list(planner.tactic)).any() + + +def test_split_payments_are_smaller_and_more_numerous(adaptive_world): + _, planner, txns = adaptive_world + split = _rows(txns, planner, "split_amounts", "victim_transfer") + plain = _rows(txns, planner, "none", "victim_transfer") + assert len(split) and len(plain) + assert split["amount"].median() < plain["amount"].median() + assert split.groupby("case_id").size().mean() > plain.groupby("case_id").size().mean() + + +def test_delayed_cash_out_waits_hours(adaptive_world): + _, planner, txns = adaptive_world + first_in = _rows(txns, planner, "delayed_cashout", "victim_transfer") + first_out = _rows(txns, planner, "delayed_cashout", "mule_cashout") + first_in = first_in.groupby("case_id")["ts"].min() + first_out = first_out.groupby("case_id")["ts"].min() + both = first_in.index.intersection(first_out.index) + assert len(both) + waited = (first_out[both] - first_in[both]).dt.total_seconds() + assert (waited >= DELAY_HOURS[0] * 3600).all() + + +def test_seasoned_mules_are_old_wallets(adaptive_world): + sim, planner, _ = adaptive_world + seasoned = [m for pool in planner.seasoned.values() for m in pool] + assert seasoned + for m in seasoned: + assert m.kind == "seasoned" + assert m.usable_ts - float(sim.w.created_ts[m.wallet]) >= SEASONED_MIN_AGE_DAYS * DAY + + +def test_experiment_runs_end_to_end(trained): + data_dir, models_root, _ = trained + acfg = AdversaryConfig(rounds=2, round_days=3, n_cells=3, warmup_days=5) + report = experiment.run(data_dir, models_root, None, acfg, log=lambda *_: None) + assert set(report["arms"]) == set(experiment.ARMS) + for arm in experiment.ARMS: + rounds = report["arms"][arm]["rounds"] + assert len(rounds) == acfg.rounds + for r in rounds: + for key in ("case_recall", "loss_txn_recall"): + assert r[key] is None or 0 <= r[key] <= 1 + control = report["arms"]["control"]["rounds"] + assert control[0]["mix"] == control[-1]["mix_after"] + assert report["arms"]["frozen"]["last_model"] == report["served_model"] + assert report["arms"]["retrain"]["last_model"] == "retrain-r1" + assert not math.isnan(report["seconds"]) diff --git a/docs/DATA_ASSUMPTIONS.md b/docs/DATA_ASSUMPTIONS.md index eae9a6d..466514e 100644 --- a/docs/DATA_ASSUMPTIONS.md +++ b/docs/DATA_ASSUMPTIONS.md @@ -213,8 +213,11 @@ behaviour that was missing. But the test period is no longer strictly are what we claim. - Fraud prevalence is likely higher than in real traffic, to give enough positives to evaluate each typology. -- Fraud scripts are stylised. Real scammers adapt to controls; the simulator's do - not react to the model's decisions. +- Fraud scripts are stylised. Real scammers adapt to controls; the default + simulator's do not react to the model's decisions. The opt-in adversary mode + (`simulator/adversary.py`, MODEL_CARD §14) adds cells that shift between five + evasion tactics according to what got through. Its tactics, their parameters + and its learning rule are assumptions too. - No merchant fraud, no loan or credit fraud, no SIM-swap telemetry, no customer demographics beyond segment and district. - Reports are the only source of confirmed labels and arrive for about half of diff --git a/docs/MODEL_CARD.md b/docs/MODEL_CARD.md index ede2b93..970fa18 100644 --- a/docs/MODEL_CARD.md +++ b/docs/MODEL_CARD.md @@ -268,7 +268,8 @@ has not been done yet. thresholds must be re-fitted on real data. - **Labels.** Training uses ground-truth labels. In production only reported cases are labelled (about half), which biases what the model learns. -- **No adaptation.** Simulated scammers do not react to the controls. +- **No adaptation.** The scammers behind every number above do not react to the + controls. §14 adds scammers that do, and measures the decay. - **Young wallets and fast cash-out are risk signals.** New customers and remittance receivers legitimately show both. The data contains such customers as hard negatives. This is the main fairness risk, and the fairness report @@ -391,11 +392,112 @@ more wallets sit within three hops of one as time passes. A deployment that keeps this feature should expire or age old flags. The live endpoint measures the same on the latest decisions actually served. -## 14. Reproduce +## 14. Adaptive adversaries + +The scammers in §2 to §13 follow fixed scripts. Real scammers learn which +scripts get through. This section deploys scammers that do, against the served +model (v4), and asks three questions: how fast recall decays, whether weekly +retraining keeps up, and whether the drift monitor notices. + +**Set-up** (`make adversary`, about five minutes; writes +`artifacts/reports/adversary.json`). The default world is re-simulated to day 120 +(it reproduces the stored dataset exactly, which the script checks). Then 8 new +fraud cells open and run 6 one-week rounds after an 8-day warm-up. Every scam +draws one of six tactics: + +| Tactic | What changes | +| --- | --- | +| none | the usual script for the typology | +| split_amounts | each payment split into 2 to 6 parts below ৳1,000 to ৳2,500, 5 to 30 minutes apart | +| seasoned_mule | the money goes to an existing customer's wallet over 180 days old, not a young one | +| delayed_cashout | the mule waits 6 to 48 hours before forwarding or cashing out (past a 30-minute hold) | +| fan_out | each payment goes to the least-used of up to 11 mules | +| mimic_hours | the scam runs at the victim's own usual hour | + +Round 0 runs half the scams on the usual script and spreads the rest evenly. +After each round the cells see which scams had no victim transfer alerted at the +warn tier. They then shift their mix by multiplicative weights on that success +rate (step 3.0, floor 3% per tactic, tactics run fewer than 5 times keep their +weight). Each round is scored in full with no subsampling: 34,000 to 46,000 +transactions and 84 to 117 scams. There are four arms, all on the same world and +seed: + +- **control**: the mix never changes. This is natural decay. +- **frozen**: the scammers adapt and v4 stays deployed. +- **retrain**: after every round the model is refitted with the `make train` + recipe. It uses the training fold plus what an operator would know by then: + the verdict on every alerted transaction, and the complaints of reported scams. + Thresholds are re-chosen on val_b. +- **drift_gated**: retrain only when the live drift monitor + (`mlops/drift.py`: score PSI against training, or the alert rate) leaves "stable". + +**Scams caught (case recall) at the warn tier, by round:** + +| Arm | 0 | 1 | 2 | 3 | 4 | 5 | Pooled PR-AUC | +| --- | --- | --- | --- | --- | --- | --- | --- | +| control | 96.7% | 94.6% | 90.9% | 93.1% | 89.5% | 94.8% | 0.977 | +| frozen | 96.7% | 94.0% | 90.0% | 87.2% | 87.0% | **79.6%** | 0.961 | +| retrain | 96.7% | 97.0% | 90.5% | 92.6% | 97.8% | 98.0% | 0.960 | +| drift_gated | identical to frozen: the monitor never fired | | | | | | 0.961 | + +What the numbers say: + +- **The scammers found the weak spot, and it is seasoned mules.** Against the + frozen model, seasoned_mule's share grew from 10% to 48% of scams. Pooled over + the six rounds, the model caught 74.4% of seasoned-mule scams (191 scams), + against 92% to 100% for every other tactic. The other tactics barely hurt: + splitting, delaying, fan-out and hour-mimicking were each caught at 92% or + more. The model's two largest signals are the receiver's fast-exit share and + its age. Delaying cash-out does not hide the first signal on a mule that has + other victims, and only an old wallet hides the second. +- **Frozen decays, control does not.** By round 5 the frozen model caught 79.6% + of scams against 94.8% for the same model facing non-adaptive scammers. That is + 15 points lost to adaptation alone. Money is more robust: with holds on the + mule's exit transactions, 98.3% of the taka is still covered in round 5. The + scams that get through are caught at cash-out. +- **Weekly retraining keeps recall, at a price in alerts.** The retrained arm + catches 98.0% of scams in round 5 and 90.1% of seasoned-mule scams pooled. But + the first retrain raised alerts from 76 to 234 a day and cut precision from + 72% to 22%. By round 5 it was still at 118 alerts a day and 46% precision, + against 66 and 79% for the frozen model. The feedback rows are mostly alerted + rows and complaints, so they are far richer in fraud than real traffic, and + thresholds re-chosen on val_b do not correct for it. Retraining on verdicts + needs its alert budget re-checked before promotion. That is exactly what + shadow mode (§13) is for. +- **The drift monitor did not notice.** In the frozen arm the population score + PSI stayed between 0.0007 and 0.0044 and the alert rate between 1.29% and + 1.39% (1.24% on validation). So `drift_gated` never retrained and matched the + frozen arm round for round. Fraud is about 1% of traffic, and a tactic shift inside that 1% + barely moves the whole score distribution. Pointed at confirmed fraud only + (val_b fraud as the score reference), the same measure rose from 0.40 in round + 0 to 0.85 in round 5. But the control arm already reads 0.24 to 0.44, so the + standard 0.25 threshold does not separate the two. A fraud-segment monitor + needs its own baseline. The signals that did move are per-tactic catch rates + on confirmed cases, which only labels give. Drift on unlabelled traffic is no + substitute for them. +- **What carries detection after adaptation** (mean |SHAP| share on victim + transfers, round 0, then round 5). The frozen model keeps leaning on the same + features: receiver fast-exit share 14.8% then 12.9%, receiver age 10.8% then + 8.5%. The retrained model moves away from receiver age (to 7.8%) and fast exit + (to 9.5%). It moves toward the sender's handset age (7.7% to 13.2%, now the + largest), the receiver's 7-day fan-in (3.3% to 6.6%), its number of recipients + (3.4% to 6.4%) and prior payments between the pair (3.8%, new in the top 8). + These are the signals an old wallet does not fake: a victim on a new handset, + and many unrelated senders converging on one receiver. + +**Caveats.** One seed and one world. Each round has 84 to 117 scams and each +tactic 4 to 53, so a single round's number moves by a few points on a few scams. +Read the trend, not a single cell. The adversary sees ground truth on what was +alerted, which is more feedback than real scammers get. The tactic set and its +parameters are assumptions (DATA_ASSUMPTIONS.md). The retrained models are +experiment-only and are not registered. + +## 15. Reproduce ``` make data features train # about one minute; writes backend/artifacts/models// make policy insights # policy_report.json and insights.json (fairness, drift, threshold sweep) +make adversary # §14: adaptive scammers vs frozen, retrained and drift-gated models make test # 203 tests, including leakage and round-trip checks ``` diff --git a/docs/plans/05-adaptive-adversary.md b/docs/plans/05-adaptive-adversary.md new file mode 100644 index 0000000..3253723 --- /dev/null +++ b/docs/plans/05-adaptive-adversary.md @@ -0,0 +1,83 @@ +# Plan 05 — Adaptive adversaries + +## Goal + +`docs/DATA_ASSUMPTIONS.md` admits the weakest point of the evaluation: "Real +scammers adapt to controls; the simulator's do not react to the model's +decisions." This plan closes that gap with a measured experiment: after FraudLens +is deployed, scam cells change their scripts in rounds, each round moving toward +the evasion tactics that got past the deployed model in the previous one. We then +compare what keeps detection up: nothing (a frozen model), retraining every round +through the existing retrain recipe, or retraining only when the drift monitor +asks for it. + +## Judge criteria it moves + +- **AI/ML depth (14.67/20).** A closed-loop adversarial evaluation, retraining on + biased labels (verdicts on alerts plus victim complaints, not ground truth for + everything), drift detection measured against a non-adaptive control, and SHAP + on what carries detection after adaptation. This is the question a fraud team + actually faces after launch, answered with numbers instead of a sentence. +- **Innovation (7.67/10).** Few fraud demos model the attacker at all. A + simulator whose scam cells learn from the deployed model's decisions, and an + honest result on whether the standard drift monitor notices, is new for this + project and uncommon for hackathon fraud work. + +## Design + +- **Opt-in adversary mode** (`fraudlens.simulator.adversary`). Nothing in the + default dataset changes: the experiment runs the default world for its 120 days + (and checks it reproduces the stored dataset exactly, so the stored feature + engine snapshot can continue from it), then switches the fraud planner to an + adaptive one. New scam cells set up for a warm-up week, then run in rounds of a + week. +- **Tactics**, one drawn per scam from the cell's current mix: + `none` (the original script), `split_amounts` (victim transfers split below a + cap), `seasoned_mule` (an old recruited wallet with a normal history instead of + a young one), `delayed_cashout` (mule waits hours to days, past the 30-minute + hold-review window, before moving money), `fan_out` (each victim transfer to a + different mule from a larger pool), `mimic_hours` (scam timed at the victim's + own usual hour). +- **Adaptation**: after each round the adversary sees which of its scams got + through (no victim transfer alerted at the warn tier of that arm's deployed + model) and reweights its mix multiplicatively toward tactics with the higher + success rate, with an exploration floor. +- **Arms** (each its own simulation, because the adversary reacts to whichever + model is deployed in that arm): + - control: frozen model, adversary does not adapt (mix fixed at round 0) + - (a) frozen served model + - (b) retrained every round with the `models.train` recipe and the + `mlops.retrain` label semantics: verdicts on alerted transactions plus + complaint labels on reported scams, added to the training fold + - (c) retrained only in rounds where `mlops.drift.measure` raises an alert on + the round's traffic (the production trigger) +- **Drift**: `mlops.drift.measure` on every round's traffic against the model's + stored reference, for the adaptive and the control runs, plus the same monitor + pointed at confirmed fraud only (reference: training-period fraud). +- **SHAP**: mean |TreeSHAP| on victim transfers for the frozen model before and + after adaptation and for the last retrained model. +- **Runtime**: the four arms share nothing but the stored day-120 engine snapshot; + subsampling, if needed, is stated in the artifact and the model card. + +## Steps + +1. `Simulation.run_days` (additive; `run` calls it), `transaction_table` factored + out of `generate.build_tables` (behaviour-preserving). +2. `simulator/adversary.py`: tactics, mix update, `AdaptivePlanner`. +3. `models/adversary.py`: the experiment, artifact + `artifacts/reports/adversary.json`; `make adversary`. +4. Tests: default data unchanged by the adversary code, tactics change what they + claim to change, mix moves toward successful tactics, experiment smoke on the + small world. +5. `docs/MODEL_CARD.md` section "Adaptive adversaries" with the measured numbers; + `DATA_ASSUMPTIONS.md` updated to point at it. +6. If clean: an additive API field and a small chart on the model page. + +## Definition of done + +- `make adversary` runs end to end and writes `backend/artifacts/reports/adversary.json` + with recall per round per arm, per-tactic recall, mix per round, drift per round + and SHAP top features; no existing artifact is overwritten. +- The model card section quotes only numbers from that artifact. +- `uv run pytest -q` and `make lint` pass; new tests cover the adversary mode. +- Committed on the worktree branch. From 0d4fee3599fd7e68fe5f9ff81fcc87a3bc34567b Mon Sep 17 00:00:00 2001 From: 0xdevabir <261-15-001@diu.edu.bd> Date: Wed, 7 Oct 2026 09:28:00 +0600 Subject: [PATCH 05/12] Add signed ISO 20022 ingest webhook, key rotation, file-based secrets, TLS proxy and STRIDE threat model --- .gitignore | 1 + Makefile | 18 ++ README.md | 4 +- backend/src/fraudlens/api/app.py | 10 +- backend/src/fraudlens/api/deps.py | 15 +- backend/src/fraudlens/api/routes/ingest.py | 181 +++++++++++ backend/src/fraudlens/config.py | 26 +- backend/src/fraudlens/platform/callbacks.py | 178 ++++++++++ backend/src/fraudlens/platform/ingest.py | 342 ++++++++++++++++++++ backend/src/fraudlens/platform/keys.py | 269 +++++++++++++++ backend/src/fraudlens/platform/security.py | 97 +++++- backend/src/fraudlens/secret_sources.py | 94 ++++++ backend/tests/test_ingest.py | 341 +++++++++++++++++++ backend/tests/test_security.py | 310 ++++++++++++++++++ deploy/Caddyfile | 33 ++ docker-compose.yml | 27 ++ docs/INGEST.md | 203 ++++++++++++ docs/PLATFORM.md | 45 ++- docs/SECURITY.md | 162 ++++++++++ docs/plans/07-ingest-security.md | 67 ++++ sdk/python/example.py | 88 +++++ sdk/python/fraudlens_partner.py | 186 +++++++++++ 22 files changed, 2679 insertions(+), 18 deletions(-) create mode 100644 backend/src/fraudlens/api/routes/ingest.py create mode 100644 backend/src/fraudlens/platform/callbacks.py create mode 100644 backend/src/fraudlens/platform/ingest.py create mode 100644 backend/src/fraudlens/platform/keys.py create mode 100644 backend/src/fraudlens/secret_sources.py create mode 100644 backend/tests/test_ingest.py create mode 100644 backend/tests/test_security.py create mode 100644 deploy/Caddyfile create mode 100644 docs/INGEST.md create mode 100644 docs/SECURITY.md create mode 100644 docs/plans/07-ingest-security.md create mode 100644 sdk/python/example.py create mode 100644 sdk/python/fraudlens_partner.py diff --git a/.gitignore b/.gitignore index 026703d..d82d9b0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ out/ .env .env.* !.env.example +backend/secrets/ # OS / tooling .DS_Store diff --git a/Makefile b/Makefile index b55078b..4acb90a 100644 --- a/Makefile +++ b/Makefile @@ -111,5 +111,23 @@ lint: ## Lint and format check cd backend && uv run ruff check src tests && uv run ruff format --check src tests cd frontend && pnpm exec tsc --noEmit && pnpm exec eslint . +.PHONY: tls rotate-jwt partner-key keys +JWT_KEYRING ?= secrets/jwt.json +INGEST_KEYRING ?= secrets/ingest.json + +tls: backend/.env ## The demo behind HTTPS with HSTS on https://localhost:8443 (deploy/Caddyfile) + docker compose --profile demo --profile tls up --build + +rotate-jwt: ## New token-signing key; sessions signed with the old one last until they expire + cd backend && mkdir -p -m 700 secrets && uv run python -m fraudlens.platform.keys rotate-jwt $(JWT_KEYRING) + +partner-key: ## New ingest HMAC key: make partner-key PARTNER=upay [CALLBACK=https://...] + cd backend && mkdir -p -m 700 secrets && uv run python -m fraudlens.platform.keys rotate-partner \ + $(INGEST_KEYRING) --partner $(PARTNER) $(if $(CALLBACK),--callback-url $(CALLBACK)) + +keys: ## List the key ids in both keyrings (never the secrets) + cd backend && for f in $(JWT_KEYRING) $(INGEST_KEYRING); do \ + [ -f $$f ] && uv run python -m fraudlens.platform.keys list $$f; done; true + fmt: ## Auto-format cd backend && uv run ruff check --fix src tests && uv run ruff format src tests diff --git a/README.md b/README.md index f62e121..467f6a4 100644 --- a/README.md +++ b/README.md @@ -442,11 +442,13 @@ Keep two browser windows open: one as `analyst1`, one private window as `supervi | [PLATFORM.md](docs/PLATFORM.md) | API, workflow, security, stream, measured latency | | [FRAUD_TAXONOMY.md](docs/FRAUD_TAXONOMY.md) | Eight kinds of fraud in the Bangladesh context, what detects each, the scam-message classifier and its limits | | [DEMO_SCRIPT.md](docs/DEMO_SCRIPT.md) | The walk-through | +| [INGEST.md](docs/INGEST.md) | The signed webhook for core banking: ISO 20022 pacs.008 mapping, HMAC signing, replay protection, decision callbacks, a partner SDK | +| [SECURITY.md](docs/SECURITY.md) | STRIDE threat model with the status of each mitigation, personal data and retention | ## ⚠️ What this is not - **Everything runs on synthetic data,** so absolute numbers will not transfer to real traffic. Fraud is more common and more scripted here than in real life, and thresholds must be re-fitted on real data. -- **It is a prototype, not a deployment.** One scorer process holds the feature state, and there is no TLS, secrets manager or horizontal scaling. +- **It is a prototype, not a deployment.** One scorer process holds the feature state, and there is no horizontal scaling. TLS ends at a Caddy proxy, secrets come from files with no vault client, and keys rotate by command, not on a schedule ([SECURITY.md](docs/SECURITY.md)). - **The demo scaffolding is not the product.** The demo accounts, the phone demo endpoints and the review simulator exist only outside production mode. - **No model output moves or blocks money on its own.** A hold waits for an analyst and a freeze needs two people. - **The Bangla texts were written by the developers** and have not been reviewed by a professional translator. diff --git a/backend/src/fraudlens/api/app.py b/backend/src/fraudlens/api/app.py index d3d3ed6..5c17643 100644 --- a/backend/src/fraudlens/api/app.py +++ b/backend/src/fraudlens/api/app.py @@ -19,6 +19,7 @@ from ..config import Settings from ..intel.text import load_text_model from ..platform.audit import WorkflowError +from ..platform.callbacks import Dispatcher from ..platform.db import make_engine, make_sessions from ..platform.graph import Graph from ..platform.scoring import Scorer @@ -26,7 +27,7 @@ from ..platform.stream import Worker from .deps import Platform from .middleware import RequestContext, error_body -from .routes import alerts, auth, cases, customer, demo, intel, mlops, network, ops, scoring +from .routes import alerts, auth, cases, customer, demo, ingest, intel, mlops, network, ops, scoring log = logging.getLogger(__name__) @@ -79,6 +80,7 @@ def build_platform(settings: Settings) -> Platform: message_limit=RateLimiter(redis, "message-check", 20, 60), # Tighter: each call is a question about the ledger. proof_limit=RateLimiter(redis, "payment-verify", 10, 60), + callbacks=Dispatcher(settings, sessions, redis) if settings.ingest_keyring else None, ) @@ -96,10 +98,14 @@ async def lifespan(app: FastAPI): app.state.aredis = aioredis.from_url(settings.redis_url, decode_responses=True) if settings.run_worker: platform.worker.start() + if platform.callbacks is not None: + platform.callbacks.start() log.info("fraudlens api ready: scoring in %s mode", platform.scorer.mode) try: yield finally: + if platform.callbacks is not None: + platform.callbacks.stop() platform.worker.stop() if platform.worker.is_alive(): platform.worker.join(timeout=5.0) @@ -165,7 +171,7 @@ async def unavailable(request: Request, exc: Exception) -> JSONResponse: for kind in (OperationalError, InterfaceError, RedisError): app.add_exception_handler(kind, unavailable) - for module in (auth, scoring, alerts, cases, network, customer, intel, ops, mlops): + for module in (auth, scoring, ingest, alerts, cases, network, customer, intel, ops, mlops): app.include_router(module.router, prefix="/v1") if not settings.production: # stand-ins for the customer app; see routes/demo.py app.include_router(demo.router, prefix="/v1") diff --git a/backend/src/fraudlens/api/deps.py b/backend/src/fraudlens/api/deps.py index 8c8b79d..d7176f1 100644 --- a/backend/src/fraudlens/api/deps.py +++ b/backend/src/fraudlens/api/deps.py @@ -15,10 +15,17 @@ from ..config import Settings from ..intel.text import TextModel from ..platform.audit import Ctx, WorkflowError +from ..platform.callbacks import Dispatcher from ..platform.graph import Graph from ..platform.models import User from ..platform.scoring import Scorer -from ..platform.security import RateLimiter, TokenError, is_revoked, read_token +from ..platform.security import ( + RateLimiter, + TokenError, + client_address, + is_revoked, + read_token, +) from ..platform.stream import Worker REVIEWERS = ("analyst", "supervisor") @@ -47,6 +54,7 @@ class Platform: intel: TextModel | None # the scam-message classifier, when it has been trained message_limit: RateLimiter proof_limit: RateLimiter + callbacks: Dispatcher | None = None # decision callbacks to ingest partners, when configured def platform(request: Request) -> Platform: @@ -67,7 +75,10 @@ def get_session(p: Plat) -> Iterator[Session]: def client_ip(request: Request) -> str | None: - return request.client.host if request.client else None + """The caller's address, through the reverse proxies in FRAUDLENS_TRUSTED_PROXIES.""" + peer = request.client.host if request.client else None + trusted = request.app.state.platform.settings.trusted_proxies + return client_address(peer, request.headers.getlist("x-forwarded-for"), trusted) def current_user( diff --git a/backend/src/fraudlens/api/routes/ingest.py b/backend/src/fraudlens/api/routes/ingest.py new file mode 100644 index 0000000..f5ac6bb --- /dev/null +++ b/backend/src/fraudlens/api/routes/ingest.py @@ -0,0 +1,181 @@ +"""The signed webhook a core-banking system pushes transactions to (docs/INGEST.md). + +No bearer token: the partner signs each request with its HMAC key. The raw body is +read before anything is parsed, because the signature is over the bytes sent. +""" + +from __future__ import annotations + +import time +from typing import Literal + +from fastapi import APIRouter, Query, Request +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import ValidationError +from starlette.concurrency import run_in_threadpool + +from ...platform import callbacks +from ...platform.audit import WorkflowError +from ...platform.ingest import ( + IDEMPOTENCY_HEADER, + KEY_HEADER, + NONCE_HEADER, + SIGNATURE_HEADER, + TIMESTAMP_HEADER, + Idempotency, + IngestInvalid, + Pacs008, + claim_nonce, + first_sightings, + forget_sightings, + parse, + verify, +) +from ...platform.keys import Key, load_keyring +from ...platform.stream import MAX_BACKLOG, enqueue +from ..deps import Platform +from ..views import result_view + +router = APIRouter(prefix="/ingest", tags=["ingest"]) +_ERROR = "__error__" # a stored refusal, replayed as the same error + +_SIGNED = { + "parameters": [ + { + "name": h, + "in": "header", + "required": h != IDEMPOTENCY_HEADER, + "schema": {"type": "string"}, + } + for h in (KEY_HEADER, TIMESTAMP_HEADER, NONCE_HEADER, SIGNATURE_HEADER, IDEMPOTENCY_HEADER) + ], + "requestBody": { + "required": True, + "content": {"application/json": {"schema": Pacs008.model_json_schema()}}, + }, +} + + +def _authenticate(p: Platform, request: Request, body: bytes) -> Key: + if p.settings.ingest_keyring is None: + raise WorkflowError(404, "not_found", "ingest is not configured on this service") + path = request.url.path + (f"?{request.url.query}" if request.url.query else "") + window = p.settings.ingest_window_seconds + key = verify( + load_keyring(p.settings.ingest_keyring), request.headers, request.method, path, body, window + ) + claim_nonce(p.redis, key, request.headers[NONCE_HEADER], window) + return key + + +def _parse(body: bytes, single: bool): + try: + return parse(body, single) + except ValidationError as exc: + raise RequestValidationError(exc.errors()) from None + except IngestInvalid as exc: + raise RequestValidationError(exc.errors) from None + + +def _queue(p: Platform, key: Key, message: Pacs008, txns: list) -> dict: + backlog = p.worker.backlog() + if backlog > MAX_BACKLOG: + raise WorkflowError( + 503, "backlog_full", "the event queue is full; retry later", + retry_after_seconds=30, backlog=backlog, + ) # fmt: skip + ids = [t.txn_id for t in txns] + new = first_sightings(p.redis, ids) + fresh = [t for t, n in zip(txns, new, strict=True) if n] + try: + enqueue(p.redis, p.settings.events_stream, fresh) + except Exception: + forget_sightings(p.redis, [t.txn_id for t in fresh]) + raise + ring = load_keyring(p.settings.ingest_keyring) + if ring.callback_url(key.partner): + now = time.time() + for transfer, is_new in zip(message.CdtTrfTxInf, new, strict=True): + if is_new: + callbacks.schedule( + p.redis, key.partner, int(transfer.PmtId.TxId), transfer.PmtId.EndToEndId, now + ) + return { + "msg_id": message.GrpHdr.MsgId, + "accepted": len(fresh), + "duplicates": [ + t.PmtId.EndToEndId for t, n in zip(message.CdtTrfTxInf, new, strict=True) if not n + ], + "callback": bool(ring.callback_url(key.partner)), + } + + +def _decide(p: Platform, message: Pacs008, txns: list) -> dict: + [new] = first_sightings(p.redis, [txns[0].txn_id]) # a later queued copy is a duplicate + try: + [result] = p.scorer.process([txns[0].event()]) + except Exception: + if new: + forget_sightings(p.redis, [txns[0].txn_id]) + raise + if result.status == "stale": + raise WorkflowError( + 409, "stale_event", "this transaction is too far behind the stream to be scored" + ) + return {"end_to_end_id": message.CdtTrfTxInf[0].PmtId.EndToEndId, **result_view(result)} + + +async def _handle(request: Request, single: bool, wait: bool) -> JSONResponse: + p: Platform = request.app.state.platform + body = await request.body() + + def work() -> tuple[int, dict, bool]: + key = _authenticate(p, request, body) + idem = Idempotency(p.redis, key.partner, request.headers.get(IDEMPOTENCY_HEADER), body) + replay = idem.begin() + if replay is not None: + return *replay, True + try: + message, txns = _parse(body, single) + if wait: + status, answer = 200, jsonable_encoder(_decide(p, message, txns)) + else: + status, answer = 202, _queue(p, key, message, txns) + except WorkflowError as exc: + if exc.status >= 500: + idem.abandon() + else: # a refusal is an answer too: the same request gets it again + idem.finish(exc.status, {_ERROR: [exc.status, exc.code, exc.message]}) + raise + except Exception: # invalid, or a backing service failed: a retry may run it again + idem.abandon() + raise + idem.finish(status, answer) + return status, answer, False + + status, answer, replayed = await run_in_threadpool(work) + if _ERROR in answer: + code_status, code, message = answer[_ERROR] + raise WorkflowError(code_status, code, message) + headers = {"Idempotent-Replayed": "true"} if replayed else {} + return JSONResponse(answer, status, headers=headers) + + +@router.post("/transactions", status_code=202, openapi_extra=_SIGNED) +async def ingest_one( + request: Request, + wait: Literal["decision"] | None = Query( + default=None, description="`decision`: score now and answer with it (200)" + ), +) -> JSONResponse: + """One pacs.008 credit transfer, HMAC-signed by a partner. Queued for the stream + (202) or, with `?wait=decision`, decided now (200, the same answer as /v1/score).""" + return await _handle(request, single=True, wait=wait == "decision") + + +@router.post("/transactions/batch", status_code=202, openapi_extra=_SIGNED) +async def ingest_batch(request: Request) -> JSONResponse: + """Up to 500 credit transfers in one signed pacs.008 message, queued in order.""" + return await _handle(request, single=False, wait=False) diff --git a/backend/src/fraudlens/config.py b/backend/src/fraudlens/config.py index 3a3b72d..2e740f8 100644 --- a/backend/src/fraudlens/config.py +++ b/backend/src/fraudlens/config.py @@ -3,7 +3,9 @@ from pathlib import Path from pydantic import SecretStr -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict + +from .secret_sources import SecretFilesSource BACKEND_DIR = Path(__file__).resolve().parents[2] @@ -52,6 +54,28 @@ class Settings(BaseSettings): # Where the console is served from in development (`make console`). cors_origins: list[str] = ["http://localhost:3100", "http://127.0.0.1:3100"] + # Signing keys with ids, re-read when the file changes (`fraudlens.platform.keys`). + # When set they replace jwt_secret: the current key signs, unretired ones verify. + jwt_keyring: Path | None = None + # Partner HMAC keys for /v1/ingest (docs/INGEST.md); without them it answers 404. + ingest_keyring: Path | None = None + ingest_window_seconds: int = 300 # how far a request's timestamp may be from ours + # Reverse proxies (CIDRs) whose X-Forwarded-For is believed; nobody else's is. + trusted_proxies: list[str] = [] + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + # FRAUDLENS__FILE and FRAUDLENS_SECRETS_PROVIDER: see secret_sources.py. + files = SecretFilesSource(settings_cls) + return init_settings, env_settings, files, dotenv_settings, file_secret_settings + @property def dataset_dir(self) -> Path: return self.data_dir / self.dataset diff --git a/backend/src/fraudlens/platform/callbacks.py b/backend/src/fraudlens/platform/callbacks.py new file mode 100644 index 0000000..b606997 --- /dev/null +++ b/backend/src/fraudlens/platform/callbacks.py @@ -0,0 +1,178 @@ +"""Decision callbacks: tell the partner what was decided on a transaction it queued. + +Optional, per partner (`callback_url` in the ingest keyring). A queued transaction +puts a job on a Redis sorted set, scored by when it is next due. Any API process +may deliver it: removing the job from the set is the claim, so each attempt +happens once. Until the transaction has been decided the job waits; once it has, +the outcome is POSTed, signed like an ingest request with the partner's newest +key. A failed delivery is retried with exponential backoff and jitter, and after +the last attempt it goes to a dead list for an operator to look at. + +Delivery is at least once (a crash after the POST and before the claim is +dropped repeats it), so the partner should treat `txn_id` as the idempotency key. +""" + +from __future__ import annotations + +import json +import logging +import random +import secrets +import threading +import time +from collections.abc import Callable +from urllib.parse import urlsplit + +import httpx +from redis import Redis +from sqlalchemy.orm import Session, sessionmaker + +from ..config import Settings +from .ingest import signed_headers +from .keys import load_keyring +from .models import Decision, Transaction + +log = logging.getLogger(__name__) + +SCHEDULE = "fraudlens:callbacks" +DEAD = "fraudlens:callbacks:dead" +DEAD_MAXLEN = 10_000 +MAX_ATTEMPTS = 8 +BASE_DELAY = 2.0 # seconds; doubled each attempt +MAX_DELAY = 600.0 +POLL_DELAY = 2.0 # while the transaction is still in the stream +DECISION_WAIT = 600.0 # after this, report it as not processed (dead-lettered in the stream) +TIMEOUT = 5.0 + +Post = Callable[[str, dict[str, str], bytes], int] + + +def schedule(redis: Redis, partner: str, txn_id: int, end_to_end_id: str, now: float) -> None: + job = {"partner": partner, "txn_id": txn_id, "e2e": end_to_end_id, "queued": now, "try": 0} + redis.zadd(SCHEDULE, {json.dumps(job, sort_keys=True): now + POLL_DELAY}) + + +def backoff(attempt: int) -> float: + """Seconds before attempt `attempt + 1`: 2, 4, 8 ... capped, with ±20% jitter.""" + return min(MAX_DELAY, BASE_DELAY * 2**attempt) * random.uniform(0.8, 1.2) + + +def _http_post(url: str, headers: dict[str, str], body: bytes) -> int: + with httpx.Client(timeout=TIMEOUT, follow_redirects=False) as client: + return client.post(url, content=body, headers=headers).status_code + + +class Dispatcher(threading.Thread): + def __init__( + self, + settings: Settings, + sessions: sessionmaker[Session], + redis: Redis, + post: Post | None = None, + interval: float = 1.0, + ) -> None: + super().__init__(name="fraudlens-callbacks", daemon=True) + self.settings, self.sessions, self.redis = settings, sessions, redis + self.post = post or _http_post + self.interval = interval + self.delivered = self.failed = self.dead = 0 + self._stopping = threading.Event() + + def outcome(self, txn_id: int) -> dict | None: + """What the partner is told, or None while the transaction is not decided.""" + with self.sessions() as s: + txn = s.get(Transaction, txn_id) + if txn is None: + return None + decision = s.get(Decision, txn_id) + summary = None + if decision is not None: # what to do, not why: as /v1/score tells the service + summary = { + "tier": decision.tier, + "action": decision.action, + "requires_review": decision.requires_review, + "risk_score": decision.risk_score, + "risk_band": decision.risk_band, + "mode": decision.mode, + "model_version": decision.model_version, + "policy_version": decision.policy_version, + "decided_at": decision.decided_at.isoformat(), + } + return { + "status": txn.status, + "status_reason": txn.status_reason, + "scored": decision is not None, + "decision": summary, + } + + def _later(self, job: dict, at: float) -> None: + self.redis.zadd(SCHEDULE, {json.dumps(job, sort_keys=True): at}) + + def _deliver(self, job: dict, now: float) -> None: + outcome = self.outcome(job["txn_id"]) + if outcome is None: + if now - job["queued"] < DECISION_WAIT: + self._later(job, now + POLL_DELAY) + return + outcome = {"status": "not_processed", "scored": False, "decision": None} + ring = load_keyring(self.settings.ingest_keyring) + url, key = ring.callback_url(job["partner"]), ring.signing(job["partner"]) + if not url or key is None: # the callback was switched off, or every key retired + log.info("callback for %s dropped: no url or key", job["partner"]) + return + payload = {"txn_id": job["txn_id"], "end_to_end_id": job["e2e"], **outcome} + body = json.dumps(payload, separators=(",", ":")).encode() + parts = urlsplit(url) + path = parts.path or "/" + path += f"?{parts.query}" if parts.query else "" + headers = signed_headers(key, "POST", path, body, secrets.token_hex(16)) + try: + status = self.post(url, headers, body) + except httpx.HTTPError as exc: + status, error = 0, type(exc).__name__ + else: + error = f"HTTP {status}" + if 200 <= status < 300: + self.delivered += 1 + return + self.failed += 1 + job = job | {"try": job["try"] + 1, "error": error} + if job["try"] >= MAX_ATTEMPTS: + self.dead += 1 + log.warning( + "callback for txn %s to %s gave up: %s", job["txn_id"], job["partner"], error + ) + pipe = self.redis.pipeline(transaction=False) + pipe.lpush(DEAD, json.dumps(job | {"payload": payload}, sort_keys=True)) + pipe.ltrim(DEAD, 0, DEAD_MAXLEN - 1) + pipe.execute() + return + self._later(job, now + backoff(job["try"])) + + def run_once(self, now: float | None = None) -> int: + """Handle the jobs that are due. Returns how many were taken.""" + now = time.time() if now is None else now + due = self.redis.zrangebyscore(SCHEDULE, "-inf", now, start=0, num=50) + taken = 0 + for member in due: + if not self.redis.zrem(SCHEDULE, member): + continue # another process claimed it + taken += 1 + job = json.loads(member) + try: + self._deliver(job, now) + except Exception: + log.exception("callback for txn %s failed; retrying", job.get("txn_id")) + self._later(job, now + backoff(job.get("try", 0))) + return taken + + def run(self) -> None: + while not self._stopping.is_set(): + try: + self.run_once() + except Exception: + log.exception("callback dispatcher error") + self._stopping.wait(self.interval) + + def stop(self) -> None: + self._stopping.set() diff --git a/backend/src/fraudlens/platform/ingest.py b/backend/src/fraudlens/platform/ingest.py new file mode 100644 index 0000000..b2362f7 --- /dev/null +++ b/backend/src/fraudlens/platform/ingest.py @@ -0,0 +1,342 @@ +"""The signed ingest webhook: a core-banking system pushes transactions to FraudLens. + +The protocol is in docs/INGEST.md. In short, every request carries + + X-FraudLens-Key-Id: upay-20261007-77d1 which partner key signed it + X-FraudLens-Timestamp: 1791364502 Unix seconds, within ±5 minutes of ours + X-FraudLens-Nonce: 6f1c0e5d9a2b4c7e8f90 16-64 characters, used once + X-FraudLens-Signature: v1= over timestamp, nonce, method, path, body + +and the body is a pacs.008-shaped JSON message (ISO 20022 FI-to-FI customer +credit transfer). Each `CdtTrfTxInf` maps onto the same `TxnIn` that /v1/score +takes, so a transaction is validated by exactly one set of rules whichever way +it arrives. The same signature scheme, with the partner's newest key, signs the +decision callbacks going the other way. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import re +import time +from collections.abc import Mapping +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, IPvAnyAddress, ValidationError +from redis import Redis + +from .audit import WorkflowError +from .events import TxnIn +from .keys import Key, Keyring + +KEY_HEADER = "X-FraudLens-Key-Id" +TIMESTAMP_HEADER = "X-FraudLens-Timestamp" +NONCE_HEADER = "X-FraudLens-Nonce" +SIGNATURE_HEADER = "X-FraudLens-Signature" +IDEMPOTENCY_HEADER = "Idempotency-Key" +SCHEME = "v1" + +_NONCE = re.compile(r"^[A-Za-z0-9_-]{16,64}$") +_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$") +_SIGNATURE = re.compile(r"^[0-9a-f]{64}$") +IDEMPOTENCY_TTL = 24 * 3600 +SEEN_TXN_TTL = 7 * 24 * 3600 +MAX_BATCH = 500 +_PREFIX = "fraudlens:ingest" + +# ---------------------------------------------------------------- signatures + + +def canonical(timestamp: str, nonce: str, method: str, path: str, body: bytes) -> bytes: + """What is signed. The method and path (with its query) are in it, so a request + signed for one endpoint, or for a queued answer, cannot be replayed at another.""" + return f"{timestamp}\n{nonce}\n{method.upper()}\n{path}\n".encode() + body + + +def sign(secret: str, timestamp: str, nonce: str, method: str, path: str, body: bytes) -> str: + mac = hmac.new(secret.encode(), canonical(timestamp, nonce, method, path, body), "sha256") + return f"{SCHEME}={mac.hexdigest()}" + + +def signed_headers(key: Key, method: str, path: str, body: bytes, nonce: str) -> dict[str, str]: + """Headers for a request this side sends (the decision callback).""" + timestamp = str(int(time.time())) + return { + KEY_HEADER: key.kid, + TIMESTAMP_HEADER: timestamp, + NONCE_HEADER: nonce, + SIGNATURE_HEADER: sign(key.secret, timestamp, nonce, method, path, body), + "Content-Type": "application/json", + } + + +def _denied(code: str, message: str) -> WorkflowError: + return WorkflowError(401, code, message) + + +def verify( + ring: Keyring, + headers: Mapping[str, str], + method: str, + path: str, + body: bytes, + window: int, + now: float | None = None, +) -> Key: + """The partner key that signed this request, or a 401 saying what is wrong. + The answers name the check that failed and never what was expected.""" + kid, timestamp = headers.get(KEY_HEADER), headers.get(TIMESTAMP_HEADER) + nonce, signature = headers.get(NONCE_HEADER), headers.get(SIGNATURE_HEADER) + if not (kid and timestamp and nonce and signature): + raise _denied( + "unsigned", f"{KEY_HEADER}, {TIMESTAMP_HEADER}, {NONCE_HEADER} and " + f"{SIGNATURE_HEADER} are required", + ) # fmt: skip + key = ring.verifying(kid) + if key is None or key.partner is None: + raise _denied("unknown_key", "the key id is unknown or retired") + if not timestamp.isdigit() or len(timestamp) > 12: + raise _denied("stale_timestamp", "the timestamp must be Unix seconds") + now = time.time() if now is None else now + if abs(now - int(timestamp)) > window: + raise _denied("stale_timestamp", f"the timestamp must be within {window} seconds of now") + if not _NONCE.fullmatch(nonce): + raise _denied("bad_nonce", "the nonce must be 16 to 64 letters, digits, - or _") + expected = sign(key.secret, timestamp, nonce, method, path, body) + offered = [s.strip() for s in signature.split(",")] # several during a scheme change + if not any( + s.startswith(f"{SCHEME}=") + and _SIGNATURE.fullmatch(s[len(SCHEME) + 1 :]) + and hmac.compare_digest(s, expected) + for s in offered + ): + raise _denied("bad_signature", "the signature does not match the request") + return key + + +def claim_nonce(redis: Redis, key: Key, nonce: str, window: int) -> None: + """A signed request is accepted once. The nonce is remembered for twice the + timestamp window, after which the timestamp check refuses the request anyway.""" + if not redis.set(f"{_PREFIX}:nonce:{key.kid}:{nonce}", "1", nx=True, ex=2 * window): + raise WorkflowError(409, "replayed", "this signed request has already been received") + + +# ---------------------------------------------------------------- idempotency + + +class Idempotency: + """`Idempotency-Key`: a retry with the same key and body gets the first answer back. + + The key is per partner. The same key with a different body is refused, so a key + cannot be reused by mistake for another payment.""" + + def __init__(self, redis: Redis, partner: str, value: str | None, body: bytes) -> None: + if value is not None and not _IDEMPOTENCY_KEY.fullmatch(value): + raise WorkflowError( + 400, "bad_idempotency_key", "Idempotency-Key must be 1-128 of A-Z a-z 0-9 _ . : -" + ) + self.redis = redis + self.key = f"{_PREFIX}:idem:{partner}:{value}" if value else None + self.digest = hashlib.sha256(body).hexdigest() + + def begin(self) -> tuple[int, dict] | None: + """The stored answer to replay, or None when this request is the first.""" + if self.key is None: + return None + marker = json.dumps({"digest": self.digest}) + if self.redis.set(self.key, marker, nx=True, ex=IDEMPOTENCY_TTL): + return None + stored = json.loads(self.redis.get(self.key) or marker) + if stored["digest"] != self.digest: + raise WorkflowError( + 422, "idempotency_key_reused", "this Idempotency-Key was used for another body" + ) + if "status" not in stored: + raise WorkflowError( + 409, "in_progress", "a request with this Idempotency-Key is being processed", + retry_after_seconds=1, + ) # fmt: skip + return stored["status"], stored["body"] + + def finish(self, status: int, body: dict) -> None: + if self.key is not None: + record = {"digest": self.digest, "status": status, "body": body} + self.redis.set(self.key, json.dumps(record, default=str), ex=IDEMPOTENCY_TTL) + + def abandon(self) -> None: + """The request failed before an answer worth keeping: a retry may run it again.""" + if self.key is not None: + self.redis.delete(self.key) + + +def first_sightings(redis: Redis, txn_ids: list[int]) -> list[bool]: + """For each transaction id, whether this is the first time ingest has seen it.""" + pipe = redis.pipeline(transaction=False) + for txn_id in txn_ids: + pipe.set(f"{_PREFIX}:txn:{txn_id}", "1", nx=True, ex=SEEN_TXN_TTL) + return [bool(new) for new in pipe.execute()] + + +def forget_sightings(redis: Redis, txn_ids: list[int]) -> None: + if txn_ids: + redis.delete(*(f"{_PREFIX}:txn:{txn_id}" for txn_id in txn_ids)) + + +# ---------------------------------------------------------------- pacs.008 + + +Max35Text = Field(pattern=r"^[A-Za-z0-9/:._-]{1,35}$") + + +class _Iso(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class GroupHeader(_Iso): + MsgId: str = Max35Text + CreDtTm: datetime + NbOfTxs: int | None = Field(default=None, ge=1, le=MAX_BATCH) + + +class PaymentId(_Iso): + EndToEndId: str = Max35Text + TxId: str = Field(pattern=r"^[0-9]{1,18}$") # the platform's numeric transaction id + + +class Proprietary(_Iso): + Prtry: str = Field(max_length=35) + + +class CategoryPurpose(_Iso): + CtgyPurp: Proprietary # SEND_MONEY, CASH_OUT, ... + + +class Amount(_Iso): + Ccy: Literal["BDT"] + Amt: float = Field(gt=0, le=10_000_000, allow_inf_nan=False) + + +class SchemeName(_Iso): + Prtry: Literal["wallet", "agent", "bank", "merchant", "telco", "biller"] + + +class OtherId(_Iso): + Id: str = Field(pattern=r"^[A-Za-z0-9_-]{1,32}$") + SchmeNm: SchemeName + + +class AccountId(_Iso): + Othr: OtherId + + +class Account(_Iso): + Id: AccountId + + +class Envelope(_Iso): + """Context a credit transfer does not carry, but the channel knows.""" + + DbtrBalBefore: float | None = Field(default=None, ge=0, le=1e10, allow_inf_nan=False) + DvcId: str = Field(default="", max_length=64, pattern=r"^[A-Za-z0-9_-]*$") + Chanl: Literal["app", "ussd", "agent", "bank"] + Dstrct: str = Field(min_length=1, max_length=40, pattern=r"^[A-Za-z' .-]+$") + IPAddr: IPvAnyAddress | None = None + + +class SupplementaryData(_Iso): + Envlp: Envelope + + +class CreditTransfer(_Iso): + PmtId: PaymentId + PmtTpInf: CategoryPurpose + IntrBkSttlmAmt: Amount + AccptncDtTm: datetime # when the customer confirmed: the transaction's time + DbtrAcct: Account + CdtrAcct: Account + SplmtryData: SupplementaryData + + def txn(self) -> TxnIn: + """The platform's own event. Raises ValidationError with the platform's rules.""" + debtor, creditor = self.DbtrAcct.Id.Othr, self.CdtrAcct.Id.Othr + extra = self.SplmtryData.Envlp + return TxnIn.model_validate( + { + "txn_id": int(self.PmtId.TxId), + "ts": self.AccptncDtTm, + "type": self.PmtTpInf.CtgyPurp.Prtry, + "sender_id": debtor.Id, + "sender_type": debtor.SchmeNm.Prtry, + "receiver_id": creditor.Id, + "receiver_type": creditor.SchmeNm.Prtry, + "amount": self.IntrBkSttlmAmt.Amt, + "sender_balance_before": extra.DbtrBalBefore, + "device_id": extra.DvcId, + "channel": extra.Chanl, + "district": extra.Dstrct, + "ip": extra.IPAddr, + "source": "live", + } + ) + + +# TxnIn field -> where it sits in a CdtTrfTxInf. docs/INGEST.md has the same table. +ISO_PATH = { + "txn_id": ("PmtId", "TxId"), + "ts": ("AccptncDtTm",), + "type": ("PmtTpInf", "CtgyPurp", "Prtry"), + "sender_id": ("DbtrAcct", "Id", "Othr", "Id"), + "sender_type": ("DbtrAcct", "Id", "Othr", "SchmeNm", "Prtry"), + "receiver_id": ("CdtrAcct", "Id", "Othr", "Id"), + "receiver_type": ("CdtrAcct", "Id", "Othr", "SchmeNm", "Prtry"), + "amount": ("IntrBkSttlmAmt", "Amt"), + "sender_balance_before": ("SplmtryData", "Envlp", "DbtrBalBefore"), + "device_id": ("SplmtryData", "Envlp", "DvcId"), + "channel": ("SplmtryData", "Envlp", "Chanl"), + "district": ("SplmtryData", "Envlp", "Dstrct"), + "ip": ("SplmtryData", "Envlp", "IPAddr"), +} + + +class Pacs008(_Iso): + """FIToFICstmrCdtTrf, in JSON, with the elements FraudLens uses.""" + + GrpHdr: GroupHeader + CdtTrfTxInf: list[CreditTransfer] = Field(min_length=1, max_length=MAX_BATCH) + + +def parse(body: bytes, single: bool) -> tuple[Pacs008, list[TxnIn]]: + """The message and its transactions, or the `ValidationError`s with their place + in the message. Every transaction is checked before any is accepted.""" + message = Pacs008.model_validate_json(body) + count = len(message.CdtTrfTxInf) + problems: list[dict] = [] + if single and count != 1: + problems.append( + {"loc": ("CdtTrfTxInf",), "msg": "exactly one transaction; use /batch for more"} + ) + if message.GrpHdr.NbOfTxs is not None and message.GrpHdr.NbOfTxs != count: + problems.append({"loc": ("GrpHdr", "NbOfTxs"), "msg": "does not match CdtTrfTxInf"}) + txns = [] + for i, transfer in enumerate(message.CdtTrfTxInf): + if int(transfer.PmtId.TxId) >= 2**62: + problems.append({"loc": ("CdtTrfTxInf", i, "PmtId", "TxId"), "msg": "too large"}) + continue + try: + txns.append(transfer.txn()) + except ValidationError as exc: + for e in exc.errors(): # reported where the partner put it, not by our name + where = ISO_PATH.get(str(e["loc"][0]), ()) if e["loc"] else () + problems.append({"loc": ("CdtTrfTxInf", i, *where), "msg": e["msg"]}) + if problems: + raise IngestInvalid(problems) + return message, txns + + +class IngestInvalid(Exception): + def __init__(self, errors: list[dict]) -> None: + super().__init__(f"{len(errors)} problems") + self.errors = errors diff --git a/backend/src/fraudlens/platform/keys.py b/backend/src/fraudlens/platform/keys.py new file mode 100644 index 0000000..7ee492a --- /dev/null +++ b/backend/src/fraudlens/platform/keys.py @@ -0,0 +1,269 @@ +"""Keyrings: signing keys with ids, so a key can be replaced without a flag day. + +A keyring is a JSON file, kept wherever the deployment keeps secrets (a mounted +Docker or Kubernetes secret, a file only the service user can read): + + { + "keys": { + "jwt-20261007-3fa2": {"secret": "...", "created": "2026-10-07T09:00:00Z"}, + "jwt-20260901-91c0": {"secret": "...", "created": "...", "retire_at": "..."}, + "upay-20261007-77d1": {"secret": "...", "created": "...", "partner": "upay"} + }, + "partners": {"upay": {"callback_url": "https://core.upay.example/fraudlens"}} + } + +The newest unretired key signs; every unretired key verifies. A key past its +`retire_at` verifies nothing. The service re-reads the file when it changes, so a +rotation needs no restart. The JWT keyring holds keys without a partner; the +ingest keyring holds each partner's keys and, optionally, where to call it back. + + python -m fraudlens.platform.keys rotate-jwt secrets/jwt_keys.json + python -m fraudlens.platform.keys rotate-partner secrets/ingest_keys.json --partner upay + python -m fraudlens.platform.keys retire secrets/ingest_keys.json upay-20260901-91c0 + python -m fraudlens.platform.keys list secrets/ingest_keys.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import secrets +import tempfile +import threading +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path + +KID = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") +PARTNER = re.compile(r"^[a-z0-9][a-z0-9_-]{0,31}$") +JWT_PREFIX = "jwt" + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _iso(moment: datetime) -> str: + return moment.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _parse(value: str | None) -> datetime | None: + if value is None: + return None + moment = datetime.fromisoformat(value.replace("Z", "+00:00")) + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +@dataclass(frozen=True) +class Key: + kid: str + secret: str = field(repr=False) + created: datetime + partner: str | None = None + retire_at: datetime | None = None + + def active(self, now: datetime | None = None) -> bool: + return self.retire_at is None or (now or _now()) < self.retire_at + + +@dataclass +class Keyring: + keys: dict[str, Key] = field(default_factory=dict) + partners: dict[str, dict] = field(default_factory=dict) + + # ------------------------------------------------------------- reading + + def verifying(self, kid: str, now: datetime | None = None) -> Key | None: + key = self.keys.get(kid) + return key if key is not None and key.active(now) else None + + def signing(self, partner: str | None = None, now: datetime | None = None) -> Key | None: + """The newest active key of this partner (None: the keys without a partner).""" + own = [k for k in self.keys.values() if k.partner == partner and k.active(now)] + return max(own, key=lambda k: (k.created, k.kid)) if own else None + + def callback_url(self, partner: str) -> str | None: + return (self.partners.get(partner) or {}).get("callback_url") + + # ------------------------------------------------------------- changing + + def add(self, prefix: str, partner: str | None = None, now: datetime | None = None) -> Key: + now = now or _now() + while True: + kid = f"{prefix}-{now:%Y%m%d}-{secrets.token_hex(2)}" + if kid not in self.keys: + break + key = Key(kid, secrets.token_hex(32), now, partner) + self.keys[kid] = key + return key + + def retire(self, kid: str, at: datetime) -> None: + key = self.keys[kid] + if key.retire_at is None or at < key.retire_at: + self.keys[kid] = Key(key.kid, key.secret, key.created, key.partner, at) + + def prune(self, now: datetime | None = None) -> list[str]: + """Drop keys already retired: nothing can be verified with them any more.""" + gone = [kid for kid, key in self.keys.items() if not key.active(now)] + for kid in gone: + del self.keys[kid] + return gone + + # ------------------------------------------------------------- storage + + @classmethod + def from_json(cls, text: str) -> Keyring: + raw = json.loads(text) + ring = cls(partners=dict(raw.get("partners") or {})) + for kid, entry in (raw.get("keys") or {}).items(): + if not KID.fullmatch(kid): + raise ValueError(f"key id {kid!r} is not valid") + partner = entry.get("partner") + if partner is not None and not PARTNER.fullmatch(partner): + raise ValueError(f"partner name {partner!r} is not valid") + secret = entry["secret"] + if not isinstance(secret, str) or not secret: + raise ValueError(f"key {kid} has no secret") + created = _parse(entry.get("created")) or datetime(1970, 1, 1, tzinfo=UTC) + ring.keys[kid] = Key(kid, secret, created, partner, _parse(entry.get("retire_at"))) + return ring + + def to_json(self) -> str: + keys = {} + for kid, k in sorted(self.keys.items()): + entry = {"secret": k.secret, "created": _iso(k.created)} + if k.partner: + entry["partner"] = k.partner + if k.retire_at: + entry["retire_at"] = _iso(k.retire_at) + keys[kid] = entry + return json.dumps({"keys": keys, "partners": self.partners}, indent=2) + "\n" + + def save(self, path: Path) -> None: + """Write atomically, readable by the owner only.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.") + try: + with os.fdopen(fd, "w") as out: + out.write(self.to_json()) + os.chmod(tmp, 0o600) + os.replace(tmp, path) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + + +_cache: dict[Path, tuple[tuple[int, int], Keyring]] = {} +_cache_lock = threading.Lock() + + +def load_keyring(path: Path) -> Keyring: + """The keyring at `path`, re-read only when the file has changed.""" + path = Path(path) + stat = path.stat() + stamp = (stat.st_mtime_ns, stat.st_size) + with _cache_lock: + cached = _cache.get(path) + if cached and cached[0] == stamp: + return cached[1] + ring = Keyring.from_json(path.read_text()) + with _cache_lock: + _cache[path] = (stamp, ring) + return ring + + +def _open(path: Path) -> Keyring: + return Keyring.from_json(path.read_text()) if path.exists() else Keyring() + + +def rotate_jwt(path: Path, ttl_minutes: int, now: datetime | None = None) -> Key: + """A new signing key. The old ones keep verifying until the tokens they signed expire.""" + now = now or _now() + ring = _open(path) + ring.prune(now) + for key in list(ring.keys.values()): + if key.partner is None: + ring.retire(key.kid, now + timedelta(minutes=ttl_minutes, seconds=60)) + new = ring.add(JWT_PREFIX, now=now) + ring.save(path) + return new + + +def rotate_partner( + path: Path, + partner: str, + grace: timedelta, + callback_url: str | None = None, + now: datetime | None = None, +) -> Key: + """A new key for a partner. Its old keys stay valid for `grace`, time to switch over.""" + if not PARTNER.fullmatch(partner): + raise ValueError(f"partner name {partner!r} is not valid") + now = now or _now() + ring = _open(path) + ring.prune(now) + for key in list(ring.keys.values()): + if key.partner == partner: + ring.retire(key.kid, now + grace) + new = ring.add(partner, partner, now) + entry = ring.partners.setdefault(partner, {}) + if callback_url is not None: + entry["callback_url"] = callback_url or None + ring.save(path) + return new + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + jwt_cmd = sub.add_parser( + "rotate-jwt", help="new token-signing key; old ones verify until expiry" + ) + jwt_cmd.add_argument("path", type=Path) + jwt_cmd.add_argument("--ttl-minutes", type=int, default=None, help="default: from settings") + partner_cmd = sub.add_parser("rotate-partner", help="new HMAC key for an ingest partner") + partner_cmd.add_argument("path", type=Path) + partner_cmd.add_argument("--partner", required=True) + partner_cmd.add_argument("--grace-hours", type=float, default=24.0) + partner_cmd.add_argument("--callback-url", default=None, help="'' removes it") + retire_cmd = sub.add_parser("retire", help="stop accepting one key now (a leaked key)") + retire_cmd.add_argument("path", type=Path) + retire_cmd.add_argument("kid") + list_cmd = sub.add_parser("list", help="key ids and their dates, never the secrets") + list_cmd.add_argument("path", type=Path) + args = parser.parse_args(argv) + + if args.command == "rotate-jwt": + if args.ttl_minutes is None: + from ..config import Settings + + args.ttl_minutes = Settings().jwt_ttl_minutes + key = rotate_jwt(args.path, args.ttl_minutes) + print(f"signing with {key.kid}; earlier keys verify for {args.ttl_minutes} more minutes") + elif args.command == "rotate-partner": + grace = timedelta(hours=args.grace_hours) + key = rotate_partner(args.path, args.partner, grace, args.callback_url) + # The one time the secret is shown: hand it to the partner over a secure channel. + print(f"key id: {key.kid}\nsecret: {key.secret}") + print(f"the partner's earlier keys stay valid for {args.grace_hours:g} hours") + elif args.command == "retire": + ring = _open(args.path) + if args.kid not in ring.keys: + parser.error(f"no key {args.kid} in {args.path}") + ring.retire(args.kid, _now()) + ring.save(args.path) + print(f"{args.kid} is no longer accepted") + else: + ring = _open(args.path) + now = _now() + for kid, k in sorted(ring.keys.items(), key=lambda item: item[1].created): + state = "active" if k.active(now) else "retired" + until = f" until {_iso(k.retire_at)}" if k.retire_at and k.active(now) else "" + owner = k.partner or "tokens" + print(f"{kid:28} {owner:12} created {_iso(k.created)} {state}{until}") + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/platform/security.py b/backend/src/fraudlens/platform/security.py index 346fa57..b61fb9d 100644 --- a/backend/src/fraudlens/platform/security.py +++ b/backend/src/fraudlens/platform/security.py @@ -1,7 +1,9 @@ -"""Passwords, access tokens, rate limits and the checks that guard production.""" +"""Passwords, access tokens and their keys, client addresses, rate limits, production checks.""" from __future__ import annotations +import functools +import ipaddress import time import uuid from datetime import UTC, datetime, timedelta @@ -13,8 +15,11 @@ from redis import Redis from ..config import DEV_JWT_SECRET, Settings +from .keys import Key, Keyring, load_keyring ALGORITHM = "HS256" +DEFAULT_KID = "default" # the key id of FRAUDLENS_JWT_SECRET when there is no keyring +MIN_KEY_LENGTH = 32 ISSUER = "fraudlens" MIN_PASSWORD_LENGTH = 12 @@ -41,7 +46,18 @@ def verify_password(password_hash: str | None, password: str) -> bool: return False +def _jwt_keyring(settings: Settings) -> Keyring: + """The keyring file when there is one; otherwise the single secret, as key `default`.""" + if settings.jwt_keyring is not None: + return load_keyring(settings.jwt_keyring) + secret = settings.jwt_secret.get_secret_value() + return Keyring({DEFAULT_KID: Key(DEFAULT_KID, secret, datetime(1970, 1, 1, tzinfo=UTC))}) + + def issue_token(user_id: int, role: str, settings: Settings, now: datetime | None = None) -> str: + key = _jwt_keyring(settings).signing() + if key is None: + raise RuntimeError("the JWT keyring has no active key: run `keys rotate-jwt`") now = now or datetime.now(UTC) claims = { "sub": str(user_id), @@ -51,14 +67,19 @@ def issue_token(user_id: int, role: str, settings: Settings, now: datetime | Non "exp": now + timedelta(minutes=settings.jwt_ttl_minutes), "jti": uuid.uuid4().hex, # names this one token, so signing out can revoke it } - return jwt.encode(claims, settings.jwt_secret.get_secret_value(), algorithm=ALGORITHM) + # `kid` names the key, so a token outlives a rotation until it expires. + return jwt.encode(claims, key.secret, algorithm=ALGORITHM, headers={"kid": key.kid}) def read_token(token: str, settings: Settings) -> dict: try: + kid = jwt.get_unverified_header(token).get("kid", DEFAULT_KID) + key = _jwt_keyring(settings).verifying(kid) if isinstance(kid, str) else None + if key is None: + raise TokenError("signed with an unknown or retired key") return jwt.decode( token, - settings.jwt_secret.get_secret_value(), + key.secret, algorithms=[ALGORITHM], # never trust the algorithm named in the token issuer=ISSUER, options={"require": ["exp", "iat", "sub", "iss", "jti"]}, @@ -106,13 +127,77 @@ def reset(self, key: str) -> None: self.redis.delete(self._key(key)) +Network = ipaddress.IPv4Network | ipaddress.IPv6Network + + +@functools.cache +def _proxies(entries: tuple[str, ...]) -> tuple[Network, ...]: + nets = tuple(ipaddress.ip_network(e.strip(), strict=False) for e in entries) + if any(net.prefixlen == 0 for net in nets): + raise ValueError("FRAUDLENS_TRUSTED_PROXIES must not trust every address") + return nets + + +def proxies(entries: list[str]) -> tuple[Network, ...]: + return _proxies(tuple(entries)) + + +def _parse_ip(text: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None: + try: + ip = ipaddress.ip_address(text.strip()) + except ValueError: + return None + if ip.version == 6 and ip.ipv4_mapped is not None: + ip = ip.ipv4_mapped + return ip + + +def client_address(peer: str | None, forwarded_for: list[str], trusted: list[str]) -> str | None: + """The address a request came from. `X-Forwarded-For` is believed only as far + back as it was written by trusted proxies: walking from the nearest hop, the + first address that is not one of ours is the client. Anything to the left of it + was written by the client and could say anything.""" + nets = proxies(trusted) + if peer is None or not nets: + return peer + hop = _parse_ip(peer) + if hop is None or not any(hop in net for net in nets): + return peer # not from our proxy: its header means nothing + chain = [part for header in forwarded_for for part in header.split(",") if part.strip()] + for part in reversed(chain): + ip = _parse_ip(part) + if ip is None: + break # a malformed hop: stop at the last address we could vouch for + hop = ip + if not any(ip in net for net in nets): + break + return str(hop) + + def check_production(settings: Settings) -> None: """Refuse to run a production service on development defaults.""" + proxies(settings.trusted_proxies) # malformed or catch-all entries fail everywhere if not settings.production: return - secret = settings.jwt_secret.get_secret_value() - if secret == DEV_JWT_SECRET or len(secret) < 32: - raise RuntimeError("FRAUDLENS_JWT_SECRET must be a private key of 32+ characters") + if settings.jwt_keyring is None: + secret = settings.jwt_secret.get_secret_value() + if secret == DEV_JWT_SECRET or len(secret) < MIN_KEY_LENGTH: + raise RuntimeError("FRAUDLENS_JWT_SECRET must be a private key of 32+ characters") + else: + ring = load_keyring(settings.jwt_keyring) + if ring.signing() is None: + raise RuntimeError("FRAUDLENS_JWT_KEYRING has no active key") + if any(k.secret == DEV_JWT_SECRET or len(k.secret) < MIN_KEY_LENGTH + for k in ring.keys.values()): # fmt: skip + raise RuntimeError("every key in FRAUDLENS_JWT_KEYRING must be 32+ characters") + if settings.ingest_keyring is not None: + ring = load_keyring(settings.ingest_keyring) + if any(len(k.secret) < MIN_KEY_LENGTH for k in ring.keys.values()): + raise RuntimeError("every key in FRAUDLENS_INGEST_KEYRING must be 32+ characters") + for partner in ring.partners: + url = ring.callback_url(partner) + if url and urlsplit(url).scheme != "https": + raise RuntimeError(f"the callback for {partner} must use https") if "fraudlens_dev" in settings.database_url: raise RuntimeError("FRAUDLENS_DATABASE_URL still uses the development password") if not urlsplit(settings.redis_url).password: diff --git a/backend/src/fraudlens/secret_sources.py b/backend/src/fraudlens/secret_sources.py new file mode 100644 index 0000000..62fb201 --- /dev/null +++ b/backend/src/fraudlens/secret_sources.py @@ -0,0 +1,94 @@ +"""Settings from files and from a secrets store, so a secret never has to sit in the environment. + +Two sources, after the environment and before `.env`: + +- **Files.** `FRAUDLENS__FILE=/run/secrets/jwt_secret` reads `` from that + file (one trailing newline removed). This is the Docker and Kubernetes secrets + convention: the orchestrator mounts the file, the environment only names it. + Setting both `FRAUDLENS_` and `FRAUDLENS__FILE` is refused. +- **A provider.** `FRAUDLENS_SECRETS_PROVIDER=package.module:function` is called once + with the names of the settings still unset and returns a mapping of the ones it + knows, e.g. a function that reads Vault or AWS Secrets Manager. Nothing here + depends on any of those clients; the function brings its own. +""" + +from __future__ import annotations + +import importlib +import os +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any + +from pydantic.fields import FieldInfo +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource + +PREFIX = "FRAUDLENS_" +PROVIDER_VAR = PREFIX + "SECRETS_PROVIDER" +MAX_SECRET_BYTES = 64_000 + +Provider = Callable[[list[str]], Mapping[str, str]] + + +def read_secret_file(path: str | Path) -> str: + data = Path(path).read_bytes() + if len(data) > MAX_SECRET_BYTES: + raise ValueError(f"{path} is larger than a secret should be") + text = data.decode() + return text[:-1] if text.endswith("\n") else text + + +def load_provider(spec: str) -> Provider: + module, _, name = spec.partition(":") + if not module or not name: + raise ValueError(f"{PROVIDER_VAR} must look like 'package.module:function', not {spec!r}") + provider = getattr(importlib.import_module(module), name) + if not callable(provider): + raise TypeError(f"{spec} is not callable") + return provider + + +class SecretFilesSource(PydanticBaseSettingsSource): + """`FRAUDLENS__FILE` and the optional provider, for every setting.""" + + def __init__(self, settings_cls: type[BaseSettings], environ: Mapping[str, str] | None = None): + super().__init__(settings_cls) + self.environ = os.environ if environ is None else environ + + def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: + return None, field_name, False # everything is done in __call__ + + def _value(self, name: str, field: FieldInfo, raw: str) -> Any: + return self.prepare_field_value(name, field, raw, self.field_is_complex(field)) + + def __call__(self) -> dict[str, Any]: + values: dict[str, Any] = {} + fields = self.settings_cls.model_fields + for name, field in fields.items(): + path = self.environ.get(f"{PREFIX}{name.upper()}_FILE") + if not path: + continue + if f"{PREFIX}{name.upper()}" in self.environ: + raise ValueError(f"set {PREFIX}{name.upper()} or its _FILE variant, not both") + values[name] = self._value(name, field, read_secret_file(path)) + spec = self.environ.get(PROVIDER_VAR) + if spec: + unset = [ + n for n in fields if n not in values and f"{PREFIX}{n.upper()}" not in self.environ + ] + for name, raw in load_provider(spec)(unset).items(): + if name in unset and raw is not None: + values[name] = self._value(name, fields[name], str(raw)) + return values + + +def directory_provider(names: list[str]) -> dict[str, str]: + """An example provider: one file per setting in `FRAUDLENS_SECRETS_DIR` (default + /run/secrets), named after the setting. Swap in a function that asks your vault.""" + root = Path(os.environ.get(PREFIX + "SECRETS_DIR", "/run/secrets")) + found = {} + for name in names: + path = root / name + if path.is_file(): + found[name] = read_secret_file(path) + return found diff --git a/backend/tests/test_ingest.py b/backend/tests/test_ingest.py new file mode 100644 index 0000000..fddd33e --- /dev/null +++ b/backend/tests/test_ingest.py @@ -0,0 +1,341 @@ +"""The signed ingest webhook, decision callbacks and token-key rotation, on Postgres and Redis. + +Needs the containers from docker-compose (`make up`). Uses its own database +(`fraudlens_test_ingest`) and Redis database 14. +""" + +from __future__ import annotations + +import itertools +import json +import shutil +from datetime import timedelta + +import pytest +from fastapi.testclient import TestClient +from redis import Redis +from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url +from test_security import load_sdk + +from fraudlens.api import create_app +from fraudlens.config import Settings +from fraudlens.decision import evaluate, insights +from fraudlens.platform import callbacks +from fraudlens.platform.db import make_engine, make_sessions, migrate +from fraudlens.platform.events import moment +from fraudlens.platform.keys import load_keyring, main, rotate_jwt, rotate_partner +from fraudlens.platform.load import load +from fraudlens.platform.models import Transaction +from fraudlens.platform.seed import seed_users + +pytestmark = pytest.mark.integration + +PASSWORD = "correct-horse-battery-staple" +TEST_DB = "fraudlens_test_ingest" +PATH = "/v1/ingest/transactions" +CALLBACK = "http://partner.test/fraudlens/decisions?v=1" + + +@pytest.fixture(scope="module") +def keyrings(tmp_path_factory): + root = tmp_path_factory.mktemp("keys") + rotate_jwt(root / "jwt.json", 480) + rotate_partner(root / "ingest.json", "upay", timedelta(hours=24), CALLBACK) + rotate_partner(root / "ingest.json", "quietbank", timedelta(hours=24)) # no callback + return root + + +@pytest.fixture(scope="module") +def settings(trained, tmp_path_factory, keyrings): + data_dir, shared_root, _ = trained + base = Settings() + admin_url = make_url(base.database_url) + redis_url = base.redis_url.rsplit("/", 1)[0] + "/14" + try: + admin = create_engine(admin_url, isolation_level="AUTOCOMMIT") + with admin.connect() as connection: + connection.execute(text(f"DROP DATABASE IF EXISTS {TEST_DB} WITH (FORCE)")) + connection.execute(text(f"CREATE DATABASE {TEST_DB}")) + admin.dispose() + Redis.from_url(redis_url).flushdb() + except Exception as exc: # the containers are not running + pytest.skip(f"Postgres/Redis not reachable: {type(exc).__name__}") + models_root = tmp_path_factory.mktemp("ingest") / "models" + shutil.copytree(shared_root, models_root) + evaluate.run(data_dir, models_root=models_root) + insights.run(data_dir, models_root=models_root) + return Settings( + data_dir=data_dir.parent, + dataset=data_dir.name, + models_root=models_root, + database_url=admin_url.set(database=TEST_DB).render_as_string(hide_password=False), + redis_url=redis_url, + events_stream="fraudlens-ingest-test:events", + alerts_channel="fraudlens-ingest-test:alerts", + run_worker=False, + seed_password=PASSWORD, + llm_notes=False, + jwt_keyring=keyrings / "jwt.json", + ingest_keyring=keyrings / "ingest.json", + ) + + +@pytest.fixture(scope="module") +def client(settings): + migrate(settings.database_url) + engine = make_engine(settings.database_url) + redis = Redis.from_url(settings.redis_url, decode_responses=True) + with make_sessions(engine)() as session: + seed_users(session, settings) + load(engine, redis, settings, reset=True) + engine.dispose() + with TestClient(create_app(settings)) as test_client: + yield test_client + + +@pytest.fixture(scope="module") +def p(client): + return client.app.state.platform + + +@pytest.fixture(scope="module") +def sdk(): + return load_sdk() + + +@pytest.fixture(scope="module") +def wallets(p): + engine = p.scorer.engine + pool = iter( + w for w, state in sorted(engine.wallets.items()) if w not in engine.flagged and state.home + ) + return lambda: next(pool) + + +_ids = itertools.count(9_100_000_000) + + +def transfer(sdk, p, sender: str, receiver: str, amount: float = 900.0, **over) -> dict: + txn_id = next(_ids) + state = p.scorer.engine.wallets[sender] + args = { + "txn_id": txn_id, + "end_to_end_id": f"E2E-{txn_id}", + "accepted_at": moment(p.scorer.clock.now()), + "purpose": "SEND_MONEY", + "amount": amount, + "debtor": (sender, "wallet"), + "creditor": (receiver, "wallet"), + "channel": "app", + "district": state.home, + "debtor_balance_before": 20_000.0, + "device_id": next(iter(state.devices), ""), + } + return sdk.credit_transfer(**(args | over)) + + +def key_of(settings, partner: str = "upay"): + return load_keyring(settings.ingest_keyring).signing(partner) + + +def post(client, sdk, settings, message, path=PATH, partner="upay", idem=None, **header_over): + body = json.dumps(message).encode() + key = key_of(settings, partner) + headers = sdk.signed_headers(key.kid, key.secret, "POST", path, body) + if idem: + headers["Idempotency-Key"] = idem + headers.update(header_over) + return client.post(path, content=body, headers=headers), headers, body + + +def code(response) -> str: + return response.json()["error"]["code"] + + +# ---------------------------------------------------------------- authentication + + +def test_unsigned_and_badly_signed_requests_are_refused(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-A", [transfer(sdk, p, wallets(), wallets())]) + body = json.dumps(message).encode() + assert code(client.post(PATH, content=body)) == "unsigned" + response, headers, _ = post(client, sdk, settings, message) + assert response.status_code == 202, response.text + # The same signature on a body with one figure changed. + tampered = body.replace(b'"Amt": 900.0', b'"Amt": 9000.0') + assert tampered != body + forged = client.post(PATH, content=tampered, headers=headers | {"X-FraudLens-Nonce": "n" * 20}) + assert (forged.status_code, code(forged)) == (401, "bad_signature") + # Signed for the queue, sent asking for a decision. + wrong_path = client.post(PATH + "?wait=decision", content=body, headers=headers) + assert code(wrong_path) == "bad_signature" + + +def test_a_stale_timestamp_is_refused(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-B", [transfer(sdk, p, wallets(), wallets())]) + body = json.dumps(message).encode() + key = key_of(settings) + import time + + for skew in (-301, 301): + headers = sdk.signed_headers(key.kid, key.secret, "POST", PATH, body, time.time() + skew) + response = client.post(PATH, content=body, headers=headers) + assert (response.status_code, code(response)) == (401, "stale_timestamp") + + +def test_a_replayed_request_is_refused(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-C", [transfer(sdk, p, wallets(), wallets())]) + first, headers, body = post(client, sdk, settings, message) + assert first.status_code == 202 + again = client.post(PATH, content=body, headers=headers) + assert (again.status_code, code(again)) == (409, "replayed") + + +def test_a_retired_or_unknown_key_is_refused(client, sdk, settings, p, wallets): + old = key_of(settings, "quietbank") + rotate_partner(settings.ingest_keyring, "quietbank", timedelta(hours=1)) + message = sdk.pacs008("MSG-D", [transfer(sdk, p, wallets(), wallets())]) + body = json.dumps(message).encode() + # During the grace period the old key still works. + headers = sdk.signed_headers(old.kid, old.secret, "POST", PATH, body) + assert client.post(PATH, content=body, headers=headers).status_code == 202 + main(["retire", str(settings.ingest_keyring), old.kid]) + headers = sdk.signed_headers(old.kid, old.secret, "POST", PATH, body) + response = client.post(PATH, content=body, headers=headers) + assert (response.status_code, code(response)) == (401, "unknown_key") + + +# ---------------------------------------------------------------- what comes in + + +def test_a_decision_can_be_had_synchronously(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-E", [transfer(sdk, p, wallets(), wallets())]) + response, _, _ = post(client, sdk, settings, message, path=PATH + "?wait=decision") + assert response.status_code == 200, response.text + answer = response.json() + txn = message["CdtTrfTxInf"][0] + assert answer["end_to_end_id"] == txn["PmtId"]["EndToEndId"] + assert answer["txn_id"] == int(txn["PmtId"]["TxId"]) and answer["scored"] + assert answer["decision"]["tier"] in {"allow", "warn", "step_up", "hold"} + # Queued again under another message: recognised, not scored twice. + again = sdk.pacs008("MSG-E2", message["CdtTrfTxInf"]) + response, _, _ = post(client, sdk, settings, again) + assert response.json()["accepted"] == 0 + assert response.json()["duplicates"] == [txn["PmtId"]["EndToEndId"]] + + +def test_an_idempotency_key_returns_the_first_answer(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-F", [transfer(sdk, p, wallets(), wallets())]) + path = PATH + "?wait=decision" + first, _, _ = post(client, sdk, settings, message, path=path, idem="retry-F") + retry, _, _ = post(client, sdk, settings, message, path=path, idem="retry-F") # new nonce + assert first.status_code == retry.status_code == 200 + assert retry.json() == first.json() and not retry.json()["duplicate"] + assert retry.headers["Idempotent-Replayed"] == "true" + other = sdk.pacs008("MSG-F2", [transfer(sdk, p, wallets(), wallets())]) + reused, _, _ = post(client, sdk, settings, other, path=path, idem="retry-F") + assert (reused.status_code, code(reused)) == (422, "idempotency_key_reused") + + +def test_a_batch_is_queued_in_order_and_scored_by_the_stream(client, sdk, settings, p, wallets): + transfers = [transfer(sdk, p, wallets(), wallets()) for _ in range(3)] + transfers[1]["PmtTpInf"]["CtgyPurp"]["Prtry"] = "CASH_IN" # an agent pays a wallet + transfers[1]["DbtrAcct"]["Id"]["Othr"] = {"Id": "AG_NOBODY", "SchmeNm": {"Prtry": "agent"}} + transfers[1]["SplmtryData"]["Envlp"]["Chanl"] = "agent" + message = sdk.pacs008("MSG-G", transfers) + single, _, _ = post(client, sdk, settings, message) + assert code(single) == "invalid_request" # one endpoint, one transaction + response, _, _ = post(client, sdk, settings, message, path=PATH + "/batch") + assert response.status_code == 202, response.text + assert response.json()["accepted"] == 3 and response.json()["callback"] + p.worker.drain() + with p.sessions() as s: + stored = [s.get(Transaction, int(t["PmtId"]["TxId"])) for t in transfers] + assert all(t is not None and t.source == "live" for t in stored) + + +def test_schema_problems_name_the_iso_element(client, sdk, settings, p, wallets): + bad = transfer(sdk, p, wallets(), wallets()) + bad["IntrBkSttlmAmt"] = {"Ccy": "USD", "Amt": -5} + wrong_kind = transfer(sdk, p, wallets(), wallets()) + wrong_kind["CdtrAcct"]["Id"]["Othr"]["SchmeNm"]["Prtry"] = "agent" # SEND_MONEY to an agent + for message, where in [ + (sdk.pacs008("MSG-H", [bad]), "CdtTrfTxInf.0.IntrBkSttlmAmt"), + (sdk.pacs008("MSG-I", [wrong_kind]), "CdtTrfTxInf.0"), + ]: + response, _, _ = post(client, sdk, settings, message) + assert response.status_code == 422 + fields = [f["field"] for f in response.json()["error"]["fields"]] + assert all(f.startswith(where) for f in fields), fields + assert "-5" not in response.text # the value sent is not repeated + + +# ---------------------------------------------------------------- callbacks + + +def test_the_decision_is_called_back_signed_with_retries(client, sdk, settings, p, wallets): + received, statuses = [], iter([503, 204]) + + def fake_post(url: str, headers: dict, body: bytes) -> int: + received.append((url, headers, body)) + return next(statuses) + + dispatcher = callbacks.Dispatcher(settings, p.sessions, p.redis, post=fake_post) + p.redis.delete(callbacks.SCHEDULE) + message = sdk.pacs008("MSG-J", [transfer(sdk, p, wallets(), wallets())]) + response, _, _ = post(client, sdk, settings, message) + assert response.status_code == 202 + import time + + now = time.time() + # Not scored yet: the job waits and nothing is sent. + assert dispatcher.run_once(now + 3) == 1 and received == [] + p.worker.drain() + assert dispatcher.run_once(now + 6) == 1 and len(received) == 1 # 503: retried later + [(member, due)] = p.redis.zrange(callbacks.SCHEDULE, 0, -1, withscores=True) + assert json.loads(member)["try"] == 1 and now + 6 + 1.5 <= due <= now + 6 + 5 + assert dispatcher.run_once(now + 6) == 0 # not due yet + assert dispatcher.run_once(due) == 1 and len(received) == 2 + assert p.redis.zcard(callbacks.SCHEDULE) == 0 and dispatcher.delivered == 1 + + url, headers, body = received[-1] + assert url == CALLBACK + key = key_of(settings) + assert sdk.verify({key.kid: key.secret}, headers, "POST", "/fraudlens/decisions?v=1", body) + payload = json.loads(body) + txn = message["CdtTrfTxInf"][0] + assert payload["end_to_end_id"] == txn["PmtId"]["EndToEndId"] + assert payload["decision"]["tier"] in {"allow", "warn", "step_up", "hold"} + assert "reasons" not in payload["decision"] # what to do, not why + + +def test_a_callback_that_keeps_failing_ends_in_the_dead_list(client, sdk, settings, p): + dispatcher = callbacks.Dispatcher(settings, p.sessions, p.redis, post=lambda *a: 500) + p.redis.delete(callbacks.SCHEDULE, callbacks.DEAD) + callbacks.schedule(p.redis, "upay", 2**61, "E2E-NEVER", now=0.0) # never arrives + now = callbacks.DECISION_WAIT + 10 # long enough: reported as not processed + for _ in range(callbacks.MAX_ATTEMPTS): + now += callbacks.MAX_DELAY * 1.3 + assert dispatcher.run_once(now) == 1 + assert p.redis.zcard(callbacks.SCHEDULE) == 0 + [dead] = p.redis.lrange(callbacks.DEAD, 0, -1) + assert json.loads(dead)["payload"]["status"] == "not_processed" + + +# ---------------------------------------------------------------- token keys + + +def test_sessions_survive_a_signing_key_rotation(client, settings): + login = client.post("/v1/auth/login", json={"username": "analyst1", "password": PASSWORD}) + assert login.status_code == 200, login.text + old = {"Authorization": f"Bearer {login.json()['access_token']}"} + import jwt + + old_kid = jwt.get_unverified_header(login.json()["access_token"])["kid"] + rotate_jwt(settings.jwt_keyring, settings.jwt_ttl_minutes) # no restart + again = client.post("/v1/auth/login", json={"username": "analyst1", "password": PASSWORD}) + assert jwt.get_unverified_header(again.json()["access_token"])["kid"] != old_kid + assert client.get("/v1/auth/me", headers=old).status_code == 200 + main(["retire", str(settings.jwt_keyring), old_kid]) + assert client.get("/v1/auth/me", headers=old).status_code == 401 diff --git a/backend/tests/test_security.py b/backend/tests/test_security.py new file mode 100644 index 0000000..12bf0c7 --- /dev/null +++ b/backend/tests/test_security.py @@ -0,0 +1,310 @@ +"""Keyrings and rotation, secrets from files, client addresses behind a proxy, ingest signatures. + +No database or Redis: the endpoint behaviour is in test_ingest.py. +""" + +from __future__ import annotations + +import importlib.util +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from fraudlens.config import BACKEND_DIR, DEV_JWT_SECRET, Settings +from fraudlens.platform import ingest +from fraudlens.platform.audit import WorkflowError +from fraudlens.platform.keys import Keyring, load_keyring, main, rotate_jwt, rotate_partner +from fraudlens.platform.security import ( + TokenError, + check_production, + client_address, + issue_token, + read_token, +) +from fraudlens.secret_sources import SecretFilesSource + +SDK_PATH = BACKEND_DIR.parent / "sdk" / "python" / "fraudlens_partner.py" + + +def load_sdk(): + spec = importlib.util.spec_from_file_location("fraudlens_partner", SDK_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# ---------------------------------------------------------------- keyrings + + +def test_a_rotated_jwt_key_keeps_verifying_until_its_tokens_expire(tmp_path): + path = tmp_path / "jwt.json" + t0 = datetime(2026, 10, 7, 9, tzinfo=UTC) + first = rotate_jwt(path, ttl_minutes=480, now=t0) + second = rotate_jwt(path, ttl_minutes=480, now=t0 + timedelta(hours=1)) + ring = Keyring.from_json(path.read_text()) + later = t0 + timedelta(hours=2) + assert ring.signing(now=later).kid == second.kid + assert ring.verifying(first.kid, now=later) is not None + # 480 minutes after the rotation (and a minute's grace), the old key is done. + gone = t0 + timedelta(hours=1, minutes=482) + assert ring.verifying(first.kid, now=gone) is None + assert ring.verifying(second.kid, now=gone) is not None + # The next rotation prunes it; the file is for the owner only. + rotate_jwt(path, ttl_minutes=480, now=gone) + assert first.kid not in Keyring.from_json(path.read_text()).keys + assert path.stat().st_mode & 0o077 == 0 + + +def test_tokens_carry_the_key_id_and_survive_a_rotation(tmp_path): + path = tmp_path / "jwt.json" + settings = Settings(jwt_keyring=path) + rotate_jwt(path, settings.jwt_ttl_minutes) + old = issue_token(7, "analyst", settings) + rotate_jwt(path, settings.jwt_ttl_minutes) + new = issue_token(7, "analyst", settings) + import jwt + + assert jwt.get_unverified_header(old)["kid"] != jwt.get_unverified_header(new)["kid"] + assert read_token(old, settings)["sub"] == read_token(new, settings)["sub"] == "7" + # Retiring the old key (a leak) refuses its tokens at once, without a restart. + main(["retire", str(path), jwt.get_unverified_header(old)["kid"]]) + with pytest.raises(TokenError): + read_token(old, settings) + assert read_token(new, settings)["sub"] == "7" + + +def test_without_a_keyring_the_single_secret_still_works(): + settings = Settings(jwt_secret="x" * 40) + token = issue_token(1, "admin", settings) + assert read_token(token, settings)["role"] == "admin" + with pytest.raises(TokenError): + read_token(token, Settings(jwt_secret="y" * 40)) + + +def test_partner_rotation_has_a_grace_period(tmp_path): + path = tmp_path / "ingest.json" + t0 = datetime(2026, 10, 7, tzinfo=UTC) + old = rotate_partner(path, "upay", timedelta(hours=24), "https://core.example/cb", now=t0) + new = rotate_partner(path, "upay", timedelta(hours=24), now=t0 + timedelta(days=30)) + ring = load_keyring(path) + assert ring.signing("upay", now=t0 + timedelta(days=30, hours=1)).kid == new.kid + assert ring.verifying(old.kid, now=t0 + timedelta(days=30, hours=23)) is not None + assert ring.verifying(old.kid, now=t0 + timedelta(days=31, hours=1)) is None + assert ring.callback_url("upay") == "https://core.example/cb" # kept across rotations + assert ring.signing(None) is None # partner keys never sign tokens + with pytest.raises(ValueError): + rotate_partner(path, "Not A Name", timedelta(hours=1)) + + +def test_the_production_guard_covers_the_keyrings(tmp_path): + prod = { + "environment": "production", + "database_url": "postgresql+psycopg://fraudlens:s3cret-long@db:5432/fraudlens", + "redis_url": "redis://:pw@redis:6379/0", + "cors_origins": ["https://console.example"], + } + jwt_path = tmp_path / "jwt.json" + rotate_jwt(jwt_path, 480) + check_production(Settings(jwt_keyring=jwt_path, **prod)) # the dev secret is not used + ring = Keyring.from_json(jwt_path.read_text()) + ring.add("jwt") + weak = next(iter(ring.keys.values())) + ring.keys[weak.kid] = type(weak)(weak.kid, DEV_JWT_SECRET[:20], weak.created) + ring.save(jwt_path) + with pytest.raises(RuntimeError, match="32"): + check_production(Settings(jwt_keyring=jwt_path, **prod)) + ingest_path = tmp_path / "ingest.json" + rotate_partner(ingest_path, "upay", timedelta(hours=1), "http://core.example/cb") + with pytest.raises(RuntimeError, match="https"): + check_production(Settings(jwt_secret="k" * 40, ingest_keyring=ingest_path, **prod)) + with pytest.raises(ValueError, match="every address"): + check_production(Settings(trusted_proxies=["0.0.0.0/0"])) + + +# ---------------------------------------------------------------- secrets from files + + +def test_settings_read_secrets_from_files(tmp_path, monkeypatch): + (tmp_path / "jwt").write_text("from-a-file-" + "z" * 30 + "\n") + (tmp_path / "origins").write_text('["https://console.example"]') + monkeypatch.setenv("FRAUDLENS_JWT_SECRET_FILE", str(tmp_path / "jwt")) + monkeypatch.setenv("FRAUDLENS_CORS_ORIGINS_FILE", str(tmp_path / "origins")) + settings = Settings() + assert settings.jwt_secret.get_secret_value() == "from-a-file-" + "z" * 30 # newline gone + assert settings.cors_origins == ["https://console.example"] + monkeypatch.setenv("FRAUDLENS_JWT_SECRET", "both") + with pytest.raises(ValueError, match="not both"): + Settings() + + +def provide(names: list[str]) -> dict[str, str]: + """A secrets provider for the test: knows one secret.""" + return {"seed_password": "from-the-vault-123"} if "seed_password" in names else {} + + +def test_a_secrets_provider_fills_what_is_unset(monkeypatch): + monkeypatch.setenv("FRAUDLENS_SECRETS_PROVIDER", f"{__name__}:provide") + assert Settings().seed_password.get_secret_value() == "from-the-vault-123" + monkeypatch.setenv("FRAUDLENS_SEED_PASSWORD", "the-environment-wins") + assert Settings().seed_password.get_secret_value() == "the-environment-wins" + source = SecretFilesSource(Settings, environ={"FRAUDLENS_SECRETS_PROVIDER": "nonsense"}) + with pytest.raises(ValueError, match="package.module:function"): + source() + + +# ---------------------------------------------------------------- client address + + +@pytest.mark.parametrize( + ("peer", "forwarded", "expected"), + [ + ("203.0.113.9", [], "203.0.113.9"), # no proxy in the way + ("203.0.113.9", ["1.2.3.4"], "203.0.113.9"), # not our proxy: the header is ignored + ("10.0.0.2", ["198.51.100.7"], "198.51.100.7"), # our proxy says who it was + # A client that writes its own header gets its own address, not the one it chose. + ("10.0.0.2", ["6.6.6.6, 198.51.100.7"], "198.51.100.7"), + ("10.0.0.2", ["6.6.6.6", "198.51.100.7, 10.0.0.3"], "198.51.100.7"), # two of ours + ("10.0.0.2", ["garbage, 198.51.100.7"], "198.51.100.7"), + ("10.0.0.2", ["198.51.100.7, garbage"], "10.0.0.2"), # broken hop: stop at ours + ("10.0.0.2", [], "10.0.0.2"), + ("::ffff:10.0.0.2", ["2001:db8::1"], "2001:db8::1"), + ], +) +def test_forwarded_for_is_believed_only_from_trusted_proxies(peer, forwarded, expected): + assert client_address(peer, forwarded, ["10.0.0.0/24"]) == expected + assert client_address(peer, forwarded, []) == peer # nothing trusted: the peer + + +# ---------------------------------------------------------------- signatures + + +def _ring(secret: str = "s" * 64) -> Keyring: + ring = Keyring() + key = ring.add("upay", "upay") + ring.keys[key.kid] = type(key)(key.kid, secret, key.created, "upay") + return ring + + +def test_sdk_and_server_agree_on_the_signature(): + sdk, ring = load_sdk(), _ring() + key = ring.signing("upay") + body = b'{"GrpHdr":{}}' + headers = sdk.signed_headers(key.kid, key.secret, "POST", "/v1/ingest/transactions", body) + assert ingest.verify(ring, headers, "POST", "/v1/ingest/transactions", body, 300) == key + # And the other way: a callback signed here is accepted by the partner's SDK. + back = ingest.signed_headers(key, "POST", "/cb?x=1", body, "n" * 32) + assert sdk.verify({key.kid: key.secret}, back, "POST", "/cb?x=1", body) + assert not sdk.verify({key.kid: key.secret}, back, "POST", "/cb?x=2", body) + assert not sdk.verify({key.kid: "other"}, back, "POST", "/cb?x=1", body) + seen = set() + assert sdk.verify({key.kid: key.secret}, back, "POST", "/cb?x=1", body, seen_nonce(seen)) + assert not sdk.verify({key.kid: key.secret}, back, "POST", "/cb?x=1", body, seen_nonce(seen)) + + +def seen_nonce(seen: set): + def check(nonce: str) -> bool: + if nonce in seen: + return True + seen.add(nonce) + return False + + return check + + +@pytest.mark.parametrize( + ("change", "code"), + [ + ({"X-FraudLens-Signature": None}, "unsigned"), + ({"X-FraudLens-Key-Id": "nobody-20261007-0000"}, "unknown_key"), + ({"X-FraudLens-Timestamp": "1000"}, "stale_timestamp"), + ({"X-FraudLens-Timestamp": "+1000"}, "stale_timestamp"), + ({"X-FraudLens-Nonce": "short"}, "bad_nonce"), + ({"X-FraudLens-Signature": "v1=" + "0" * 64}, "bad_signature"), + ({"X-FraudLens-Signature": "md5=abc"}, "bad_signature"), + ], +) +def test_bad_requests_are_refused_with_the_check_that_failed(change, code): + sdk, ring = load_sdk(), _ring() + key = ring.signing("upay") + body = b"{}" + headers = sdk.signed_headers(key.kid, key.secret, "POST", "/v1/ingest/transactions", body) + for name, value in change.items(): + if value is None: + headers.pop(name) + else: + headers[name] = value + with pytest.raises(WorkflowError) as caught: + ingest.verify(ring, headers, "POST", "/v1/ingest/transactions", body, 300) + assert (caught.value.status, caught.value.code) == (401, code) + + +def test_the_signature_covers_the_body_path_and_time(): + sdk, ring = load_sdk(), _ring() + key = ring.signing("upay") + headers = sdk.signed_headers(key.kid, key.secret, "POST", "/v1/ingest/transactions", b"{}") + for path, body in [ + ("/v1/ingest/transactions", b'{"a":1}'), # body changed + ("/v1/ingest/transactions?wait=decision", b"{}"), # asks for something else + ("/v1/ingest/transactions/batch", b"{}"), # another endpoint + ]: + with pytest.raises(WorkflowError, match="does not match"): + ingest.verify(ring, headers, "POST", path, body, 300) + then = int(headers["X-FraudLens-Timestamp"]) + with pytest.raises(WorkflowError, match="within 300"): + ingest.verify(ring, headers, "POST", "/v1/ingest/transactions", b"{}", 300, then + 301) + ingest.verify(ring, headers, "POST", "/v1/ingest/transactions", b"{}", 300, then - 299) + + +# ---------------------------------------------------------------- pacs.008 + + +def _transfer(**over) -> dict: + sdk = load_sdk() + args = { + "txn_id": 42, + "end_to_end_id": "E2E-42", + "accepted_at": datetime(2026, 10, 7, 9, tzinfo=UTC), + "purpose": "SEND_MONEY", + "amount": 900.0, + "debtor": ("W1", "wallet"), + "creditor": ("W2", "wallet"), + "channel": "app", + "district": "Dhaka", + "debtor_balance_before": 20_000.0, + "ip": "203.0.113.9", + } + return sdk.credit_transfer(**(args | over)) + + +def test_a_pacs008_message_maps_onto_the_platform_event(): + sdk = load_sdk() + body = json.dumps(sdk.pacs008("MSG-1", [_transfer()])).encode() + message, [txn] = ingest.parse(body, single=True) + assert message.GrpHdr.MsgId == "MSG-1" + assert (txn.txn_id, txn.type, txn.sender_id, txn.receiver_id) == (42, "SEND_MONEY", "W1", "W2") + assert (txn.amount, txn.sender_balance_before, txn.channel) == (900.0, 20_000.0, "app") + assert txn.event().txn.network == "203.0.113.0/24" # the address itself is not kept + + +def test_pacs008_problems_are_reported_where_the_partner_put_them(): + sdk = load_sdk() + wrong = _transfer(purpose="CASH_OUT", debtor_balance_before=None) # a wallet to a wallet + body = json.dumps(sdk.pacs008("MSG-2", [_transfer(), wrong])).encode() + with pytest.raises(ingest.IngestInvalid) as caught: + ingest.parse(body, single=False) + assert all(e["loc"][:2] == ("CdtTrfTxInf", 1) for e in caught.value.errors) + with pytest.raises(ingest.IngestInvalid) as caught: + ingest.parse(body, single=True) + assert caught.value.errors[0]["loc"] == ("CdtTrfTxInf",) + unknown = json.loads(body) + unknown["CdtTrfTxInf"][0]["Surprise"] = 1 + with pytest.raises(ValueError): + ingest.parse(json.dumps(unknown).encode(), single=False) + + +def test_the_example_and_the_sdk_have_no_third_party_imports(): + for path in SDK_PATH.parent.glob("*.py"): + text = Path(path).read_text() + assert "import httpx" not in text and "import requests" not in text diff --git a/deploy/Caddyfile b/deploy/Caddyfile new file mode 100644 index 0000000..d71d200 --- /dev/null +++ b/deploy/Caddyfile @@ -0,0 +1,33 @@ +# TLS in front of the API and the console: `make tls` (docker compose --profile tls). +# FRAUDLENS_SITE is the public host name. Left at localhost, Caddy issues a +# certificate from its own local CA (expect a browser warning, or trust it); with +# a real domain it gets one from Let's Encrypt. +# +# Caddy is the edge: it drops any X-Forwarded-For a client sends and writes the +# real peer address, so the API can believe this one proxy (FRAUDLENS_TRUSTED_PROXIES). + +{$FRAUDLENS_SITE:localhost} { + encode zstd gzip + + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains" + X-Content-Type-Options nosniff + Referrer-Policy no-referrer + X-Frame-Options DENY + -Server + } + + # Nothing the API takes is this big; refuse it here, not in Python. + request_body { + max_size 2MB + } + + @api path /v1/* /health /ready /docs /openapi.json + handle @api { + reverse_proxy api:8010 + } + + handle { + reverse_proxy console:3100 + } +} diff --git a/docker-compose.yml b/docker-compose.yml index c5b9adc..14e26dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,9 @@ services: environment: FRAUDLENS_DATABASE_URL: postgresql+psycopg://fraudlens:${POSTGRES_PASSWORD:-fraudlens_dev}@postgres:5432/fraudlens FRAUDLENS_REDIS_URL: redis://:${REDIS_PASSWORD:-}@redis:6379/0 + # Only the TLS proxy's X-Forwarded-For is believed (its fixed address below). + FRAUDLENS_TRUSTED_PROXIES: '["172.30.250.10/32"]' + networks: [default, edge] depends_on: postgres: condition: service_healthy @@ -74,8 +77,32 @@ services: condition: service_healthy ports: - "127.0.0.1:3100:3100" + networks: [default, edge] + + # `make tls`: HTTPS with HSTS in front of the API and the console (deploy/Caddyfile). + proxy: + profiles: [tls] + image: caddy:2-alpine + environment: + FRAUDLENS_SITE: ${FRAUDLENS_SITE:-localhost} + depends_on: [api, console] + ports: + - "127.0.0.1:${TLS_HTTPS_PORT:-8443}:443" + volumes: + - ./deploy/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy:/data + networks: + edge: + ipv4_address: 172.30.250.10 + +networks: + edge: + ipam: + config: + - subnet: 172.30.250.0/24 volumes: pgdata: data: artifacts: + caddy: diff --git a/docs/INGEST.md b/docs/INGEST.md new file mode 100644 index 0000000..4ec5a39 --- /dev/null +++ b/docs/INGEST.md @@ -0,0 +1,203 @@ +# Ingest — the signed webhook for a core-banking system + +`/v1/score` and `/v1/events` need a service account's bearer token. A core-banking +or switch system usually works the other way round: it pushes each payment to a +fraud engine as it is accepted, signs the request with a shared key, retries +until it gets an answer, and wants the verdict back on its own endpoint. That is +this webhook. It takes ISO 20022 pacs.008 credit transfers (in JSON), so a bank +can send what its payment rail already produces. + +Code: `backend/src/fraudlens/platform/ingest.py` (signatures, replay, +idempotency, the pacs.008 model), `api/routes/ingest.py`, `platform/callbacks.py`, +`platform/keys.py`. Partner SDK (Python standard library only): +`sdk/python/fraudlens_partner.py`, with `sdk/python/example.py`. + +## 1. Endpoints + +| Endpoint | Body | Answer | +| --- | --- | --- | +| `POST /v1/ingest/transactions` | pacs.008 with exactly one `CdtTrfTxInf` | `202` queued for the stream | +| `POST /v1/ingest/transactions?wait=decision` | the same | `200` the decision now, the same body as `/v1/score` plus `end_to_end_id` | +| `POST /v1/ingest/transactions/batch` | pacs.008 with 1 to 500 `CdtTrfTxInf` | `202` all queued, in order | + +The queued answer: + +```json +{"msg_id": "MSG-20260415-0001", "accepted": 2, "duplicates": ["E2E-881"], "callback": true} +``` + +`duplicates` lists the `EndToEndId`s of transactions whose `TxId` was already +received in the last 7 days. They are not queued again; the rest of the batch is. + +The endpoints exist only when `FRAUDLENS_INGEST_KEYRING` names a keyring file; +otherwise they answer `404`. + +## 2. Signing a request + +Every request carries four headers: + +| Header | Value | +| --- | --- | +| `X-FraudLens-Key-Id` | the partner's key id, e.g. `upay-20261007-9ef5` | +| `X-FraudLens-Timestamp` | Unix seconds when the request was signed | +| `X-FraudLens-Nonce` | 16 to 64 of `A-Z a-z 0-9 - _`, new for every request | +| `X-FraudLens-Signature` | `v1=` + hex HMAC-SHA256 of the string below | + +What is signed is the timestamp, the nonce, the method, the path with its query, +each followed by a newline, and then the raw body bytes: + +``` +1776234000\n +4f1c0d6a9b2e47c8a1f3\n +POST\n +/v1/ingest/transactions?wait=decision\n +{"GrpHdr":{...},"CdtTrfTxInf":[...]} +``` + +The method and path are signed so that a request made for the queue cannot be +replayed as a request for a decision, or at another endpoint. The body is signed +as sent: sign the exact bytes you put on the wire, not a re-serialisation. + +The server checks, in this order, and stops at the first failure: + +| Check | Refusal | +| --- | --- | +| all four headers present | `401 unsigned` | +| the key id exists, belongs to a partner and is not retired | `401 unknown_key` | +| the timestamp is within ±5 minutes of the server's clock (`FRAUDLENS_INGEST_WINDOW_SECONDS`) | `401 stale_timestamp` | +| the nonce has the right form | `401 bad_nonce` | +| the signature matches (constant-time comparison) | `401 bad_signature` | +| the nonce has not been seen with this key in the last 10 minutes | `409 replayed` | + +A refusal names the check that failed, never the value that was expected. The +nonce is remembered in Redis for twice the window; after that the timestamp check +refuses the same request anyway, so a captured request cannot be sent again. + +## 3. Retries and idempotency + +A retry is a new request: new timestamp, new nonce, new signature. To make it +safe, send `Idempotency-Key` (1 to 128 of `A-Z a-z 0-9 _ . : -`; the SDK uses +the `MsgId`). For 24 hours, per partner: + +- the same key with the same body gets the first answer again, with + `Idempotent-Replayed: true`; refusals such as `409 stale_event` are replayed too; +- the same key with a different body is refused, `422 idempotency_key_reused`; +- while the first request is still being processed, `409 in_progress` with + `retry_after_seconds`; +- a `5xx` or a malformed body is not remembered, so the retry runs. + +Without an idempotency key, the `TxId` check still stops a transaction from being +queued twice; a retried `?wait=decision` for a transaction already decided +answers from the scorer's own duplicate handling. + +## 4. The message: pacs.008 to FraudLens + +JSON with the ISO 20022 element names, the elements FraudLens uses and nothing +else (unknown elements are refused). One `CdtTrfTxInf` is one transaction. + +| FraudLens field | pacs.008 element | Notes | +| --- | --- | --- | +| — | `GrpHdr.MsgId` | 1–35 characters; returned as `msg_id`, the SDK's idempotency key | +| — | `GrpHdr.CreDtTm` | when the message was created | +| — | `GrpHdr.NbOfTxs` | optional; must equal the number of `CdtTrfTxInf` | +| — | `CdtTrfTxInf[].PmtId.EndToEndId` | the bank's reference; echoed in answers and callbacks | +| `txn_id` | `PmtId.TxId` | digits only, below 2^62 | +| `ts` | `AccptncDtTm` | when the payment was accepted, with a time zone | +| `type` | `PmtTpInf.CtgyPurp.Prtry` | `SEND_MONEY`, `CASH_OUT`, `PAYMENT`, … as in `/v1/score` | +| `amount` | `IntrBkSttlmAmt.Amt` | with `IntrBkSttlmAmt.Ccy` = `BDT`; other currencies are refused | +| `sender_id` | `DbtrAcct.Id.Othr.Id` | the wallet or agent number | +| `sender_type` | `DbtrAcct.Id.Othr.SchmeNm.Prtry` | `wallet`, `agent` or `bank` | +| `receiver_id` | `CdtrAcct.Id.Othr.Id` | | +| `receiver_type` | `CdtrAcct.Id.Othr.SchmeNm.Prtry` | `wallet`, `agent`, `merchant`, `telco` or `biller` | +| `sender_balance_before` | `SplmtryData.Envlp.DbtrBalBefore` | | +| `device_id` | `SplmtryData.Envlp.DvcId` | optional | +| `channel` | `SplmtryData.Envlp.Chanl` | `app`, `ussd`, `agent` or `bank` | +| `district` | `SplmtryData.Envlp.Dstrct` | | +| `ip` | `SplmtryData.Envlp.IPAddr` | optional; reduced to its network at once and never stored ([PLATFORM.md](PLATFORM.md) §3) | + +pacs.008 has no place for a wallet's balance, the customer's device or the +channel, so those travel in `SplmtryData`, the standard's extension envelope. +Everything ingested is `source: live`. + +Validation errors are `422 invalid_request`, and each one names the place in the +message, not the FraudLens field, for example +`["CdtTrfTxInf", 0, "IntrBkSttlmAmt", "Ccy"]`. Every transaction in a batch is +checked before any is queued: a batch is accepted whole or not at all. + +A `202` means the transactions are on the stream (`backlog_full`, `503`, when it +is too long). They are decided by the stream worker like `/v1/events`, with the +same ordering and dead-letter rules. + +## 5. Decision callback + +When the partner has a `callback_url` in the keyring, each transaction it queued +is followed by one `POST` to that URL once it has been decided: + +```json +{"txn_id": 8812345678, "end_to_end_id": "E2E-881", "status": "held", "status_reason": null, + "scored": true, + "decision": {"tier": "hold", "action": "hold_for_review", "requires_review": true, + "risk_score": 0.91, "risk_band": "very high", "mode": "...", + "model_version": "...", "policy_version": "...", "decided_at": "..."}} +``` + +It is signed exactly like an ingest request, with the partner's newest key, so +the partner verifies it with the same code (`fraudlens_partner.verify`). The +decision's reasons are not sent: like `/v1/score`, the partner learns what to do, +and the reasons stay with the analysts. + +- **When.** The job waits for the decision, checking every 2 seconds. A + transaction the stream could not process (dead-lettered, stale) is reported + after 10 minutes as `"status": "not_processed"`. +- **Retries.** Any answer other than `2xx`, a timeout (5 s) or a connection error + is retried after 2, 4, 8 … seconds (±20%, at most 10 minutes), 8 attempts in + all. Then the job goes to the Redis list `fraudlens:callbacks:dead` for an + operator. +- **At least once.** The jobs are in a Redis sorted set; whichever API process + removes a due job delivers it, so there is one attempt at a time. A crash + between the `POST` and the bookkeeping can repeat a delivery: treat `txn_id` as + the idempotency key. +- Redirects are not followed. In production the URL must be `https`. + +## 6. Keys + +The keyring is a JSON file, mode `0600`, outside the repository +(`backend/secrets/` is ignored by git): + +```json +{"keys": {"upay-20261007-9ef5": {"secret": "…", "created": "2026-10-07T03:21:34Z", "partner": "upay"}}, + "partners": {"upay": {"callback_url": "https://upay.example/fraudlens/decisions"}}} +``` + +```bash +make partner-key PARTNER=upay CALLBACK=https://upay.example/fraudlens/decisions +# key id: upay-20261007-9ef5 +# secret: 0706… (shown once; hand it over out of band) +make keys # ids, owners and dates, never the secrets +cd backend && uv run python -m fraudlens.platform.keys retire upay-20260901-1a2b +``` + +A new key does not cut the old ones off: they keep verifying for a grace period +(24 hours by default, `--grace-hours`), so the partner can switch over without a +failed request. The newest key signs callbacks. `retire` stops a leaked key at +once. The API re-reads the file when it changes; no restart. + +## 7. Try it + +```bash +make partner-key PARTNER=upay CALLBACK=http://127.0.0.1:9099/fraudlens +FRAUDLENS_INGEST_KEYRING=secrets/ingest.json make api # in another terminal +export FRAUDLENS_PARTNER_KEY_ID=upay-… FRAUDLENS_PARTNER_SECRET=… +python sdk/python/example.py --sender W000123 --receiver W000456 --amount 900 \ + --at 2026-04-15T10:00:00+06:00 # decided now +python sdk/python/example.py ... --queue --listen 9099 # queued; prints the callback +``` + +`--at` sets `AccptncDtTm`. The demo's clock follows the transactions it is sent +([PLATFORM.md](PLATFORM.md) §14), so use a time inside the replayed period. +Outside production a callback URL may be plain `http`. + +Tests: `backend/tests/test_security.py` (signatures, the SDK against the server, +the message mapping) and `backend/tests/test_ingest.py` (bad signature, stale +timestamp, replay, retired key, duplicates, idempotency, batches, callbacks with +retries and the dead list, against Postgres and Redis). diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index 78fc2bf..d8e295f 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -70,6 +70,7 @@ served at `/docs` outside production. | Auth | `POST /auth/login`, `GET /auth/me`, `POST /auth/logout` | anyone / signed in / signed in | | Demo sign-in (§7, off by default) | `GET /auth/demo`, `POST /auth/demo-login` | anyone, when enabled | | Scoring | `POST /score`, `POST /events` (up to 500 per call, queued), `POST /wallet-flags` | service | +| Ingest ([INGEST.md](INGEST.md)), off without a partner keyring | `POST /ingest/transactions[?wait=decision]`, `POST /ingest/transactions/batch` (pacs.008, HMAC-signed, no token) | partner key | | | `POST /score/what-if` | service, analyst, supervisor | | Alerts | `GET /alerts`, `GET /decisions/{txn_id}`, `GET /decisions/{txn_id}/narrative?lang=en\|bn`, `GET /stream/alerts` (server-sent events) | analyst, supervisor | | Cases | `GET /cases`, `POST /cases`, `GET /cases/{id}`, `POST /cases/{id}/assign\|notes\|escalate\|verdict`, `GET /users/reviewers` | analyst, supervisor | @@ -109,6 +110,11 @@ repeat the value that was sent. `POST /v1/events`. They keep the feature state current. 5. When your own investigation confirms a wallet as fraud: `POST /v1/wallet-flags`. +A core-banking system that would rather push than call with a token uses the +signed webhook instead of steps 2 and 4: ISO 20022 pacs.008 credit transfers, +HMAC-signed per request, with replay protection, idempotency keys and an optional +signed decision callback ([INGEST.md](INGEST.md)). + A transaction may carry `ip`, the address the customer's request came from (optional, IPv4 or IPv6). The platform reduces it to its network at the edge (the /24 of an IPv4 address, the /48 of an IPv6 one) and keeps only that, in @@ -214,7 +220,9 @@ and each request still needs its own approval by a second person. - **Production guard.** With `FRAUDLENS_ENVIRONMENT=production` the API refuses to start with the development signing secret, the development database password, a Redis URL without a password, `*` as a CORS origin, or demo sign-in enabled, - and the interactive docs are off. + and the interactive docs are off. It also refuses keyring keys shorter than 32 + characters, partner callbacks that are not `https`, and a trusted-proxy list + that trusts everyone (`0.0.0.0/0`). - **Demo sign-in.** Off by default. `FRAUDLENS_DEMO_LOGIN=true` makes `POST /auth/demo-login` sign in as one of the five seeded console accounts without a password, and the login page shows a button for each. Anyone who can @@ -222,6 +230,22 @@ and each request still needs its own approval by a second person. you can reach. Service accounts are never offered, each use is audited as `auth.login` with `demo: true`, and the endpoint answers 404 when the setting is off or the environment is production. +- **Key rotation.** Each token names its signing key (`kid`). With + `FRAUDLENS_JWT_KEYRING` set, `make rotate-jwt` adds a key that signs from then + on, and the earlier ones keep verifying until the tokens they signed have + expired, so nobody is signed out. Partner HMAC keys rotate the same way + (`make partner-key`, with a grace period), and `keys retire` cuts off a leaked + key at once. The keyring files are re-read when they change; no restart. +- **Secrets from files.** Every setting can come from a file: + `FRAUDLENS_JWT_SECRET_FILE=/run/secrets/jwt` (Docker or Kubernetes secrets), + never both the value and the file. `FRAUDLENS_SECRETS_PROVIDER=module:function` + plugs in a secrets manager; only a directory reader ships + (`fraudlens.secret_sources.directory_provider`). +- **TLS and the client address.** `make tls` puts Caddy in front of the API and + the console (HTTPS, HSTS, a 2 MB body cap; `deploy/Caddyfile`). The API believes + `X-Forwarded-For` only from the addresses in `FRAUDLENS_TRUSTED_PROXIES`, and + only as far back as those proxies wrote it; a client's own header is ignored. + The threat model is [SECURITY.md](SECURITY.md). - **Redis password.** Optional in development: `REDIS_PASSWORD` in a `.env` next to `docker-compose.yml` turns it on, and `FRAUDLENS_REDIS_URL` carries the same password. Without it Redis relies on its port being bound to 127.0.0.1. @@ -262,9 +286,10 @@ written before they existed is refused at load (`make features` rebuilds it). - **Recovery time grows** with the number of transactions since the snapshot. A production system would write snapshots periodically; this one has only the end-of-history snapshot. -- **Rate limits and the login lockout use the address the API sees.** Behind a - proxy, run uvicorn with `--proxy-headers` and a trusted proxy list, or every - client shares one address. +- **Rate limits and the login lockout use the client address** worked out from + `FRAUDLENS_TRUSTED_PROXIES`. Left empty behind a proxy, every client shares the + proxy's address; listing a proxy that passes on its clients' own headers lets + them choose their address. - **The live feed authenticates with the Bearer header**, which the browser's `EventSource` cannot send; the console reads it with `fetch`. - **Step-up is asserted by the caller** (`step_up_passed`). The platform trusts @@ -277,8 +302,16 @@ written before they existed is refused at load (`make features` rebuilds it). sent to the language model is masked in the backend, before it leaves. - **The free-text mask is a pattern match.** It catches long numbers and e-mail addresses, not a name or an address written in words. -- **No TLS, secrets manager or key rotation**: deployment concerns outside this - prototype. +- **Security is partly built** ([SECURITY.md](SECURITY.md) lists each threat as + done, partial or not done). TLS ends at the proxy: the hops behind it (API, + Postgres, Redis) are plain on a private network. Secrets come from files or a + provider hook, but no secrets-manager client ships. Keys rotate when someone + runs the command; nothing rotates them on a schedule, and moving from the single + `FRAUDLENS_JWT_SECRET` to a keyring signs everyone out once. +- **Decision callbacks** cover transactions queued through the ingest webhook, + not `/v1/events`. Delivery is at least once, and the outbound request does not + check where the partner's address resolves to: callback URLs are set by an + operator, not by the partner. ## 11. Measured diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..8ed4183 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,162 @@ +# Security — threat model, personal data and retention + +What FraudLens protects, from whom, and how far each protection goes. Each +threat is marked **done** (built and tested), **partial** (built, with a gap +named) or **not done**. The controls themselves are described in +[PLATFORM.md](PLATFORM.md) §7 and [INGEST.md](INGEST.md). + +## 1. Assets + +| Asset | Where | Why it matters | +| --- | --- | --- | +| Transactions and decisions | Postgres `transactions`, `decisions` | Customers' payments; a changed decision moves or holds money | +| Wallet state (frozen, flagged) | Postgres `wallets`, `freeze_requests`, `wallet_flags` | A freeze stops a customer's money; an unfreeze releases a mule's | +| Cases, notes and verdicts | Postgres `cases`, `case_events` | Evidence, and the labels the next model is trained on | +| Customer scam reports | Postgres `customer_reports` | Free text a customer wrote, possibly with names and numbers | +| Audit log | Postgres `audit_log` (append-only) | The record of who saw and did what | +| Token signing keys | `FRAUDLENS_JWT_SECRET` or the JWT keyring | Whoever has one can be any user | +| Partner HMAC keys | the ingest keyring | Whoever has one can inject transactions as that bank | +| Database and Redis credentials | environment or secret files | Direct access to everything above | +| Models and the decision policy | `backend/artifacts/`, `decision/policies/` | Changing them changes every decision | +| Feature state | the scorer's memory, snapshots in Postgres | Wrong state, wrong scores | + +## 2. Trust boundaries + +``` + customer ──► payment app/upay ──(bearer token: service)──────────┐ + │ + core banking ──(HMAC-signed pacs.008, §INGEST)───────────────────┤ + ▼ + analyst's browser ──(HTTPS, Caddy)──► console ──(bearer token)──► API ──► Postgres + │ └───► Redis (stream, revocations, nonces) + │ + partner callback URL ◄──(HMAC-signed POST) + language model (optional) ◄──(masked evidence) +``` + +1. **Internet to the edge.** Browsers and partners reach Caddy over TLS + (`make tls`). Caddy replaces any `X-Forwarded-For` a client sends. +2. **Edge to the API.** A private Docker network; the API believes forwarded + addresses only from `FRAUDLENS_TRUSTED_PROXIES`. +3. **Caller to the API.** Every request is authenticated: a bearer token with a + role, or a partner's HMAC signature on the ingest endpoints. Nothing else. +4. **API to its stores.** Postgres and Redis on the private network, with + passwords; their ports are bound to 127.0.0.1 on the host. +5. **API outwards.** Decision callbacks to partner URLs set by an operator, and + (off by default) case-note wording from a language model. + +## 3. Threats (STRIDE) + +### Spoofing + +| Threat | Mitigation | Status | +| --- | --- | --- | +| Guessing an analyst's password | Argon2id; five failures in five minutes lock the username from that address; a per-address limit | done | +| A forged or altered token | HS256 with a key of at least 32 characters (production guard); `kid` must name a known, unretired key | done | +| A stolen token | 8-hour expiry, sign-out revokes it in Redis (fails closed), the user is re-read on every request | partial: no binding to a device, no refresh-token rotation | +| A forged ingest request | HMAC-SHA256 over timestamp, nonce, method, path and body; constant-time comparison | done (`test_ingest.py`) | +| Someone posing as FraudLens to a partner's callback endpoint | Callbacks are signed with the partner's own key, verified by the SDK | done | +| Spoofing the client address to dodge the lockout | `X-Forwarded-For` is read only from configured proxies, right to left; `0.0.0.0/0` is refused | done (`test_security.py`) | +| The demo sign-in left on | Off by default, refused in production, every use audited | done | + +### Tampering + +| Threat | Mitigation | Status | +| --- | --- | --- | +| A request changed in transit | TLS at the edge; ingest bodies are signed whatever the transport | partial: TLS ends at the proxy, the private hops are plain | +| Rewriting the audit trail | A trigger refuses `UPDATE`, `DELETE`, `TRUNCATE` on `audit_log` | partial: a database superuser can drop the trigger; no external copy | +| Changing a decision or a freeze | Roles; freezes need a second person (§5 of PLATFORM.md); every step audited | done | +| Injection through input | Every body validated, unknown fields refused, identifiers match a pattern, SQL parameterised, 1 MB body cap (2 MB at the proxy) | done | +| Poisoning the models through analyst verdicts | Retraining registers a challenger only; promotion is a separate, manual step with a shadow comparison | partial: no automatic check for a run of suspicious verdicts | +| Swapping the model or policy files | Loaded once at start-up from the artifacts directory and the image | not done: no signature on artifacts | + +### Repudiation + +| Threat | Mitigation | Status | +| --- | --- | --- | +| "I never froze that wallet" | Every case action, freeze step, flag, login and profile view is in the append-only audit log with the user and request id | done | +| "I never looked at that number" | A full identifier is shown only after `POST /audit/reveals` | partial: the API returns full numbers to reviewers (PLATFORM.md §10) | +| "The bank never sent that payment" | Only a holder of the partner's key could have signed it | partial: the transaction is stored, but not the key id, `MsgId` or signature it came with | + +### Information disclosure + +| Threat | Mitigation | Status | +| --- | --- | --- | +| Reading data with too broad a role | `admin` sees metrics and audit, not customer data; `service` scores but cannot read alerts or cases | done | +| Customer IP addresses | Reduced to the /24 or /48 at the edge; the address is never stored, logged or returned | done | +| Numbers and e-mails in scam reports | Masked on the way out | partial: pattern-based, names in words get through | +| Scam messages a customer checks | Classified and discarded; not stored | done | +| Evidence sent to a language model | Off by default; identifiers masked before it leaves | done | +| Error messages revealing internals | One error shape; validation errors never echo the value; signature refusals never show the expected value | done | +| Secrets in the repository or the image | `.env` and `backend/secrets/` ignored by git; `*_FILE` variables read Docker/Kubernetes secrets; keyrings written with mode 0600 | done | +| Secrets in the environment of the process | `*_FILE` and a provider hook avoid it | partial: only a directory provider ships, not a vault client | +| Reasons for a decision leaking to a fraudster | `/v1/score` and callbacks return the action, not the reasons | done | +| A captured backup | — | not done: no encryption at rest beyond the disk's own | + +### Denial of service + +| Threat | Mitigation | Status | +| --- | --- | --- | +| Flooding the login or customer endpoints | Per-address and per-wallet rate limits in Redis | done | +| Flooding the stream | `503 backlog_full` above the backlog limit; batches capped at 500 | done | +| Replaying a signed ingest request | Nonce kept in Redis for twice the ±5 minute window; `TxId` deduplicated for 7 days | done | +| Large bodies | 2 MB at the proxy, 1 MB in the API | done | +| A partner's slow or dead callback endpoint | 5 s timeout, backoff, 8 attempts, then a dead list; delivery is off the request path | done | +| Redis down | Token checks fail closed (`503`); scoring needs it for the stream | partial: no Redis replica | +| One scorer process | — | see PLATFORM.md §10 | + +### Elevation of privilege + +| Threat | Mitigation | Status | +| --- | --- | --- | +| A service account reading cases | A role check on every route | done | +| An analyst approving their own freeze | Two-person rule in the database workflow | done | +| A partner key used for other partners or the console | Partner keys are accepted only on `/v1/ingest`, and JWT keys only for tokens | done | +| A leaked key staying valid | `keys retire` cuts one off at once; rotation keeps old keys only for a grace period | partial: rotation is manual, not scheduled | +| Server-side request forgery via callbacks | Callback URLs are set by operators in the keyring, `https` only in production, redirects not followed | partial: no check of where the name resolves | + +## 4. Personal data + +| Data | Kept | Shown to | +| --- | --- | --- | +| Wallet, agent and device numbers | In full, in transactions and cases | Analysts and supervisors (masked on screen until a recorded reveal) | +| Amounts, times, districts, channels | In full | Analysts and supervisors | +| Customer IP address | Only its network (/24, /48) | Nobody: used by the policy only | +| Scam-report text | As written | Reviewers, with numbers and e-mails masked | +| Scam messages checked by a customer | Not kept | — | +| Names | Not collected | — | +| Analysts' accounts | Username, role, Argon2id hash | Admins | + +Nothing personal goes to the language model unmasked, and nothing goes to it at +all unless it is switched on. + +## 5. Retention + +The policy below is what FraudLens is designed for. **Enforcing it is not built**: +there is no scheduled job that deletes or anonymises rows. The Redis entries are +the exception: they expire by themselves. + +| Data | Kept for | Then | +| --- | --- | --- | +| Transactions and decisions | 5 years (the Money Laundering Prevention Act 2012 asks for at least 5 years of transaction records) | Wallet and device numbers replaced by a keyed hash; amounts and times kept for models | +| Cases, case events, verdicts, freeze requests | 5 years after the case is closed | As above | +| Scam reports | 2 years | Text deleted, category kept | +| Audit log | 5 years | Exported and archived, then deleted | +| Redis: ingest nonces | 10 minutes | Expire | +| Redis: idempotency answers | 24 hours | Expire | +| Redis: seen `TxId`s | 7 days | Expire | +| Redis: revoked tokens | Until the token would have expired | Expire | +| Redis: callback dead list | Last 10,000 | Trimmed | +| Model training data | The synthetic dataset; no real customer data is used | — | + +A customer who asks for their data gets what is held under their wallet number; +deleting it before the period ends is refused where the law requires keeping it. + +## 6. Not done, in order of what to build next + +1. TLS on the private hops (API to Postgres and Redis), or a service mesh. +2. A secrets-manager client behind the provider hook, and scheduled rotation. +3. A retention job implementing §5. +4. Signed model and policy artifacts, checked at start-up. +5. The audit log shipped to write-once storage outside the database. +6. Encrypted backups. diff --git a/docs/plans/07-ingest-security.md b/docs/plans/07-ingest-security.md new file mode 100644 index 0000000..15f395e --- /dev/null +++ b/docs/plans/07-ingest-security.md @@ -0,0 +1,67 @@ +# Plan 07 — Core-banking ingest and security hardening + +## Goal + +A core-banking system or payment switch can push transactions to FraudLens over +a signed webhook, the way a bank partner would, and get a decision back (in the +response, or later by a signed callback). The service's secrets can come from +files or a secrets store, its signing keys can be rotated without logging +everyone out, it can sit behind TLS, and rate limits see the real client behind +a proxy. A STRIDE threat model says what is protected and what is not. + +## Which judge criterion it moves, and why + +- **Scalability & integration (6.67/10)**: "without live core banking webhooks". + The ingest endpoint is that webhook: HMAC-signed, replay-protected, + idempotent, ISO 20022 pacs.008-shaped, with a partner SDK and a decision + callback. It feeds the existing Redis stream, so it adds an integration path, + not a second scoring path. +- **Responsible AI & security (4/5)** and "security hardening still pending": + key rotation with `kid`, secrets from files, TLS with HSTS, a trusted-proxy + setting, and docs/SECURITY.md with every threat's mitigation and an honest + status. + +## Steps + +1. `platform/keys.py`: keyrings (JSON files) for JWT signing keys and partner + HMAC keys, re-read when the file changes; a rotation command + (`python -m fraudlens.platform.keys rotate-jwt|rotate-partner|list|retire`). +2. `platform/security.py`: tokens carry `kid`; the current key signs, any + unretired key verifies. Without a keyring the single `FRAUDLENS_JWT_SECRET` + still works as before. Production guard extended. +3. `secrets.py` + a settings source in `config.py`: `FRAUDLENS__FILE` + reads a setting from a file (Docker/Kubernetes secrets); an optional + `FRAUDLENS_SECRETS_PROVIDER=module:function` fills what is still unset. +4. `platform/ingest.py` + `api/routes/ingest.py`: + `POST /v1/ingest/transactions` (one pacs.008-style message, queued, or + decided synchronously with `?wait=decision`) and + `POST /v1/ingest/transactions/batch` (up to 500, queued). Checks, in order: + key id, timestamp within ±5 minutes, HMAC-SHA256 over + timestamp, nonce, method, path and body; nonce single use in Redis; + `Idempotency-Key` replays the stored answer; schema; then the stream. +5. `platform/callbacks.py`: optional signed decision callback per partner, from + a Redis schedule with exponential backoff and a dead list; any API process + can deliver, a claim makes each delivery happen once. +6. `api/deps.py`: `client_ip` honours `X-Forwarded-For` only from + `FRAUDLENS_TRUSTED_PROXIES`. +7. `deploy/Caddyfile` + a `tls` profile in docker-compose: HTTPS, HSTS, the API + and console behind one origin, the proxy on a fixed address the API trusts. +8. `sdk/python/fraudlens_partner.py` (standard library only) and an example. +9. Tests: bad signature, unknown key, stale timestamp, replayed nonce, + idempotent retry, conflicting idempotency key, duplicate transaction, schema, + sync decision, batch, callback with retry; keyrings, JWT rotation, secret + files, trusted proxies. +10. Docs: docs/INGEST.md (protocol, pacs.008 mapping), docs/SECURITY.md (STRIDE, + PII and retention), PLATFORM.md "not built" list updated honestly, README. + +## Definition of done + +- The ingest tests and the existing suite pass against real Postgres and Redis; + `make lint` is clean. +- A request signed by the SDK is accepted; tampered, stale, replayed and + unknown-key requests are refused with distinct error codes. +- Rotating the JWT key leaves existing sessions valid until their tokens expire; + rotating a partner key leaves the old one valid for a stated grace period. +- `docker compose --profile tls config` is valid and the Caddyfile validates. +- SECURITY.md lists every threat with done / partial / not done, and + PLATFORM.md no longer claims anything that is not built. diff --git a/sdk/python/example.py b/sdk/python/example.py new file mode 100644 index 0000000..ff12e86 --- /dev/null +++ b/sdk/python/example.py @@ -0,0 +1,88 @@ +"""Send one signed transaction to FraudLens and print the decision; optionally receive callbacks. + + export FRAUDLENS_PARTNER_KEY_ID=upay-20261007-77d1 FRAUDLENS_PARTNER_SECRET=... + python sdk/python/example.py --api http://127.0.0.1:8010 \\ + --sender W000123 --receiver W000456 --amount 900 --balance 20000 --district Dhaka + + # queued instead of decided, with a receiver for the signed decision callback + # (the partner's callback_url in the keyring: http://127.0.0.1:9099/fraudlens) + python sdk/python/example.py ... --queue --listen 9099 +""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import threading +from datetime import UTC, datetime +from http.server import BaseHTTPRequestHandler, HTTPServer + +from fraudlens_partner import Client, credit_transfer, pacs008, verify + + +def receiver(port: int, key_id: str, secret: str) -> HTTPServer: + seen: set[str] = set() + + def seen_nonce(nonce: str) -> bool: + if nonce in seen: + return True + seen.add(nonce) + return False + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + ok = verify({key_id: secret}, dict(self.headers), "POST", self.path, body, seen_nonce) + self.send_response(204 if ok else 401) + self.end_headers() + print("callback", "verified" if ok else "REJECTED", json.loads(body) if ok else "") + + def log_message(self, *args) -> None: + pass + + server = HTTPServer(("127.0.0.1", port), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--api", default="http://127.0.0.1:8010") + parser.add_argument("--sender", required=True) + parser.add_argument("--receiver", required=True) + parser.add_argument("--amount", type=float, default=900.0) + parser.add_argument("--balance", type=float, default=20_000.0) + parser.add_argument("--district", default="Dhaka") + parser.add_argument("--queue", action="store_true", help="queue it; do not wait") + # The demo world runs on recorded time: pass its clock (`now` in GET /v1/demo/scenarios). + parser.add_argument("--at", type=datetime.fromisoformat, help="default: the wall clock") + parser.add_argument("--listen", type=int, help="receive callbacks on this port") + args = parser.parse_args() + key_id, secret = os.environ["FRAUDLENS_PARTNER_KEY_ID"], os.environ["FRAUDLENS_PARTNER_SECRET"] + + server = receiver(args.listen, key_id, secret) if args.listen else None + txn_id = random.randrange(8_000_000_000, 9_000_000_000) + transfer = credit_transfer( + txn_id=txn_id, + end_to_end_id=f"E2E-{txn_id}", + accepted_at=args.at or datetime.now(UTC), + purpose="SEND_MONEY", + amount=args.amount, + debtor=(args.sender, "wallet"), + creditor=(args.receiver, "wallet"), + channel="app", + district=args.district, + debtor_balance_before=args.balance, + ) + client = Client(args.api, key_id, secret) + status, answer = client.send(pacs008(f"MSG-{txn_id}", [transfer]), not args.queue) + print(status, json.dumps(answer, indent=2)) + if server: + input("waiting for the callback; press Enter to stop\n") + server.shutdown() + + +if __name__ == "__main__": + main() diff --git a/sdk/python/fraudlens_partner.py b/sdk/python/fraudlens_partner.py new file mode 100644 index 0000000..c44f00e --- /dev/null +++ b/sdk/python/fraudlens_partner.py @@ -0,0 +1,186 @@ +"""FraudLens partner client: sign ingest requests, verify decision callbacks. + +One file, standard library only, so a core-banking team can vendor it. The +protocol is in docs/INGEST.md; this file is its reference implementation. + + from fraudlens_partner import Client, credit_transfer, pacs008 + + client = Client("https://fraudlens.example", key_id="upay-20261007-77d1", secret="...") + message = pacs008("UPAY-0001", [credit_transfer(...)]) + status, answer = client.send(message, wait_for_decision=True) + +A callback receiver checks what FraudLens sends with `verify(...)` before +trusting it. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import secrets +import time +import urllib.error +import urllib.request +from collections.abc import Mapping +from datetime import UTC, datetime +from urllib.parse import urlsplit + +KEY_HEADER = "X-FraudLens-Key-Id" +TIMESTAMP_HEADER = "X-FraudLens-Timestamp" +NONCE_HEADER = "X-FraudLens-Nonce" +SIGNATURE_HEADER = "X-FraudLens-Signature" +WINDOW_SECONDS = 300 + + +def canonical(timestamp: str, nonce: str, method: str, path: str, body: bytes) -> bytes: + return f"{timestamp}\n{nonce}\n{method.upper()}\n{path}\n".encode() + body + + +def sign(secret: str, timestamp: str, nonce: str, method: str, path: str, body: bytes) -> str: + digest = hmac.new(secret.encode(), canonical(timestamp, nonce, method, path, body), "sha256") + return "v1=" + digest.hexdigest() + + +def signed_headers( + key_id: str, secret: str, method: str, path: str, body: bytes, now: float | None = None +) -> dict[str, str]: + """Fresh headers for one attempt. A retry needs new ones: the nonce is single use.""" + timestamp = str(int(time.time() if now is None else now)) + nonce = secrets.token_hex(16) + return { + KEY_HEADER: key_id, + TIMESTAMP_HEADER: timestamp, + NONCE_HEADER: nonce, + SIGNATURE_HEADER: sign(secret, timestamp, nonce, method, path, body), + "Content-Type": "application/json", + } + + +def verify( + secrets_by_kid: Mapping[str, str], + headers: Mapping[str, str], + method: str, + path: str, + body: bytes, + seen_nonce=None, + now: float | None = None, +) -> bool: + """Is this callback from FraudLens? `secrets_by_kid` holds your current and, during + a rotation, previous key. `seen_nonce(nonce) -> bool` should record the nonce and + say whether it was already used (pass one backed by your cache to stop replays).""" + lower = {k.lower(): v for k, v in headers.items()} + kid = lower.get(KEY_HEADER.lower(), "") + timestamp = lower.get(TIMESTAMP_HEADER.lower(), "") + nonce = lower.get(NONCE_HEADER.lower(), "") + offered = lower.get(SIGNATURE_HEADER.lower(), "") + secret = secrets_by_kid.get(kid) + if not (secret and timestamp.isdigit() and nonce and offered): + return False + if abs((time.time() if now is None else now) - int(timestamp)) > WINDOW_SECONDS: + return False + expected = sign(secret, timestamp, nonce, method, path, body) + if not hmac.compare_digest(offered, expected): + return False + return not (seen_nonce and seen_nonce(nonce)) + + +# ------------------------------------------------------------------ pacs.008 + + +def credit_transfer( + *, + txn_id: int, + end_to_end_id: str, + accepted_at: datetime, + purpose: str, + amount: float, + debtor: tuple[str, str], + creditor: tuple[str, str], + channel: str, + district: str, + debtor_balance_before: float | None = None, + device_id: str = "", + ip: str | None = None, +) -> dict: + """One CdtTrfTxInf. `debtor` and `creditor` are (account id, kind), the kind being + wallet, agent, bank, merchant, telco or biller.""" + envelope = {"Chanl": channel, "Dstrct": district, "DvcId": device_id} + if debtor_balance_before is not None: + envelope["DbtrBalBefore"] = debtor_balance_before + if ip: + envelope["IPAddr"] = ip + + def account(party: tuple[str, str]) -> dict: + return {"Id": {"Othr": {"Id": party[0], "SchmeNm": {"Prtry": party[1]}}}} + + return { + "PmtId": {"EndToEndId": end_to_end_id, "TxId": str(txn_id)}, + "PmtTpInf": {"CtgyPurp": {"Prtry": purpose}}, + "IntrBkSttlmAmt": {"Ccy": "BDT", "Amt": amount}, + "AccptncDtTm": accepted_at.isoformat(), + "DbtrAcct": account(debtor), + "CdtrAcct": account(creditor), + "SplmtryData": {"Envlp": envelope}, + } + + +def pacs008(msg_id: str, transfers: list[dict]) -> dict: + return { + "GrpHdr": { + "MsgId": msg_id, + "CreDtTm": datetime.now(UTC).isoformat(), + "NbOfTxs": len(transfers), + }, + "CdtTrfTxInf": transfers, + } + + +# ------------------------------------------------------------------ client + + +class Client: + def __init__(self, base_url: str, key_id: str, secret: str, timeout: float = 10.0) -> None: + self.base_url, self.key_id, self.secret = base_url.rstrip("/"), key_id, secret + self.timeout = timeout + + def request(self, path: str, body: bytes, idempotency_key: str | None) -> tuple[int, dict]: + signed_path = urlsplit(self.base_url).path + path + headers = signed_headers(self.key_id, self.secret, "POST", signed_path, body) + if idempotency_key: + headers["Idempotency-Key"] = idempotency_key + req = urllib.request.Request(self.base_url + path, body, headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=self.timeout) as response: + return response.status, json.loads(response.read() or b"{}") + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read() or b"{}") + + def send( + self, + message: dict, + wait_for_decision: bool = False, + idempotency_key: str | None = None, + attempts: int = 4, + ) -> tuple[int, dict]: + """POST a message, retrying network errors, 409 in_progress and 5xx with backoff. + Each attempt is signed afresh; the idempotency key (default: the MsgId) makes + the retries safe.""" + batch = len(message["CdtTrfTxInf"]) > 1 + path = "/v1/ingest/transactions" + ("/batch" if batch else "") + if wait_for_decision and not batch: + path += "?wait=decision" + body = json.dumps(message, separators=(",", ":")).encode() + key = idempotency_key or message["GrpHdr"]["MsgId"] + for attempt in range(attempts): + try: + status, answer = self.request(path, body, key) + except (urllib.error.URLError, TimeoutError): + if attempt == attempts - 1: + raise + else: + code = answer.get("error", {}).get("code") + if status < 500 and code != "in_progress" or attempt == attempts - 1: + return status, answer + time.sleep(min(30.0, 0.5 * 2**attempt)) + raise AssertionError("unreachable") From 4839dad3c4effca385f2174374d3d3d4d0332bab Mon Sep 17 00:00:00 2001 From: 0xdevabir <261-15-001@diu.edu.bd> Date: Wed, 7 Oct 2026 09:37:15 +0600 Subject: [PATCH 06/12] Label-realism experiment: cost of under-reporting and what recovers it Retrains the transaction model on reported-only labels (biased, delayed, censored) and with PU learning, mule-network label propagation and analyst verdicts; scores every regime on ground truth at the v4 warn-tier FPR over five seeds. Adds make label-realism, MODEL_CARD section 15, an additive label_realism field on /model/report and a console card. --- Makefile | 5 +- backend/src/fraudlens/api/routes/ops.py | 11 + backend/src/fraudlens/models/label_realism.py | 497 ++++++++++++++++++ backend/tests/test_label_realism.py | 103 ++++ docs/MODEL_CARD.md | 87 ++- frontend/app/(console)/model/page.tsx | 45 +- frontend/lib/types.ts | 17 + 7 files changed, 762 insertions(+), 3 deletions(-) create mode 100644 backend/src/fraudlens/models/label_realism.py create mode 100644 backend/tests/test_label_realism.py diff --git a/Makefile b/Makefile index b55078b..6adc5ee 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: help setup up redis down data features train policy insights intel pipeline test lint fmt \ migrate seed load api replay replay-live verify platform \ - review retrain shadow models promote console console-build smoke \ + review retrain shadow models promote console console-build smoke label-realism \ demo demo-reset API_PORT ?= 8010 @@ -55,6 +55,9 @@ insights: ## Threshold sweep, drift and fairness tables for the dashboards intel: ## Train and evaluate the scam-message classifier (docs/FRAUD_TAXONOMY.md) cd backend && uv run python -m fraudlens.intel.train +label-realism: ## Retrain on reported-only labels, and with recovery methods; compare on ground truth + cd backend && uv run python -m fraudlens.models.label_realism + pipeline: data features train policy insights intel ## Data, features, models and policy end to end migrate: ## Bring the database schema up to date diff --git a/backend/src/fraudlens/api/routes/ops.py b/backend/src/fraudlens/api/routes/ops.py index 9bc3ab1..8b5f05c 100644 --- a/backend/src/fraudlens/api/routes/ops.py +++ b/backend/src/fraudlens/api/routes/ops.py @@ -14,6 +14,7 @@ from ...decision.evaluate import REPORT_FILE as POLICY_REPORT_FILE from ...decision.insights import INSIGHTS_FILE from ...models import registry +from ...models.label_realism import REPORT_FILE as LABEL_REALISM_FILE from ...platform.audit import WorkflowError from ...platform.models import AuditLog, Case, Decision, FreezeRequest, Transaction from ..deps import Db, Oversight, Plat, Staff @@ -263,9 +264,19 @@ def model_report( "model": _report(directory, "report.json"), "policy": _report(directory, POLICY_REPORT_FILE), "insights": _report(directory, INSIGHTS_FILE), + "label_realism": _label_realism(p.settings.artifacts_dir), } +def _label_realism(artifacts: Path) -> dict | None: + """The under-reporting experiment (`make label-realism`), averaged over seeds.""" + report = _report(artifacts / "reports", LABEL_REALISM_FILE) + if report is None: + return None + keep = ("created_at", "seeds", "operating_point", "regimes", "summary") + return {key: report.get(key) for key in keep} + + @router.get("/policy") def policy(p: Plat, ctx: Staff) -> dict: """The whole policy as served: tiers, rules with their conditions, texts, fallback.""" diff --git a/backend/src/fraudlens/models/label_realism.py b/backend/src/fraudlens/models/label_realism.py new file mode 100644 index 0000000..a53ff6d --- /dev/null +++ b/backend/src/fraudlens/models/label_realism.py @@ -0,0 +1,497 @@ +"""Label realism: what training only on reported scams costs, and what wins it back. + +uv run python -m fraudlens.models.label_realism # backend/data/full, 5 seeds +uv run python -m fraudlens.models.label_realism --seeds 7 11 # fewer seeds + +The served models are trained on the simulator's ground truth. A real wallet only +learns about the scams its victims report. This experiment trains the transaction +model the way `train.fit_booster` does (same parameters, train fold, early stopping +on val_a) under several label regimes and scores every one on the same ground-truth +test period, at the false-positive rate the served model runs at. + +Regimes: + +- ground_truth: every fraud-chain transaction labelled (what the served model had). +- reported_uniform: a case is labelled only if its victim reported it, with the + simulator's own process (50%, independent of anything). +- reported: the same 50% on average, but larger losses, urban victims, long-held + accounts and unauthorised takeovers report more, shame-heavy scams less. +- reported_pu: reported, then positive-unlabelled learning (Elkan & Noto 2008): + each unlabelled transaction counts as fraud with the weight its propensity implies. +- reported_propagated: reported, then soft labels through the mule network: other + money into or out of a wallet a reported scam paid, near the reported transfers. +- reported_feedback: reported, then the analyst-verdict loop: a model trained on + earlier reports alerts on the last training weeks, analysts label the alerts, and + the verdicts join the training data through `train.add_feedback`, the path + `mlops/feedback.py` feeds in production. +- reported_propagated_feedback: both. + +A reported case labels every transaction tagged with it (the victim's transfers and +the mule's forwards and cash-outs), as an investigation following the money would. +Reports arrive after a delay; only those in by the start of the test period, when +the model is trained, count. Unreported fraud is labelled clean, as it would be. +""" + +from __future__ import annotations + +import argparse +import json +import time +from datetime import UTC, datetime +from pathlib import Path + +import lightgbm as lgb +import numpy as np +import pandas as pd +from sklearn.model_selection import StratifiedKFold + +from ..config import settings +from ..features import FEATURES +from . import metrics, registry +from .data import load_frame +from .train import FEEDBACK_COLUMNS, LGB_PARAMS, add_feedback + +REPORT_FILE = "label_realism.json" +SEEDS = (7, 11, 13, 17, 19) +DAY = pd.Timedelta(days=1) + +# Report propensity, as log-odds terms. Synthetic assumptions, not measurements: the +# intercept is solved so that the average matches the simulator's `report_rate`. +# The simulator has no person age, so account tenure stands in for an older customer. +REPORT_LOGIT = {"log_loss": 0.6, "urban": 0.5, "tenure_1y": 0.4} +TYPOLOGY_LOGIT = { + "account_takeover": 1.0, # unauthorised: the victim disputes it with the provider + "impersonation": 0.0, + "wrong_send": 0.0, + "lottery_fee": -0.5, # victims who paid a "fee" are embarrassed to report + "investment_scam": -0.8, # and those who believed in the returns report late, if at all +} +# Report delay, as in the simulator: lognormal around a median, plus a few hours. +DELAY_MEDIAN_DAYS = {"investment_scam": 5.0} +DELAY_MEDIAN_DEFAULT_DAYS = 1.5 + +PROPAGATION_WEIGHT = 0.7 # fraud weight of a transfer that touched a reported mule +PROPAGATION_WINDOW = 3 * DAY # around the wallet's first and last reported transfer +PU_FOLDS = 3 +REVIEW_BUDGET = 0.01 # analysts review the top 1% of the review window, the headline budget +SERVED_TIER = "warn" # the operating point compared at: its false-positive rate on test +REGIMES = ( + "ground_truth", + "reported_uniform", + "reported", + "reported_pu", + "reported_pu_known_rate", + "reported_propagated", + "reported_feedback", + "reported_propagated_feedback", +) + + +# ---------------------------------------------------------------- reporting + + +def case_table(data_dir: Path) -> pd.DataFrame: + """Cases with what their reporting depends on: loss, victim's area and tenure.""" + cases = pd.read_parquet(data_dir / "cases.parquet") + wallets = pd.read_parquet(data_dir / "wallets.parquet").set_index("wallet_id") + victim = wallets.reindex(cases["victim_id"]) + cases["urban"] = (victim["area_type"].to_numpy() == "urban").astype(float) + tenure = (cases["started_at"] - victim["created_at"].to_numpy()) / DAY + cases["tenure_days"] = tenure.to_numpy() + return cases + + +def report_probability(cases: pd.DataFrame, rate: float, biased: bool = True) -> np.ndarray: + """Probability that a case is reported, averaging `rate` over the cases.""" + if not biased: + return np.full(len(cases), rate) + z = ( + REPORT_LOGIT["log_loss"] * np.log(cases["loss"].clip(lower=1.0) / cases["loss"].median()) + + REPORT_LOGIT["urban"] * cases["urban"].to_numpy() + + REPORT_LOGIT["tenure_1y"] * (cases["tenure_days"].to_numpy() >= 365) + + cases["typology"].map(TYPOLOGY_LOGIT).fillna(0.0).to_numpy() + ) + lo, hi = -20.0, 20.0 + for _ in range(100): # mean of a logistic is monotone in the intercept + mid = (lo + hi) / 2 + if (1 / (1 + np.exp(-(z + mid)))).mean() < rate: + lo = mid + else: + hi = mid + return 1 / (1 + np.exp(-(z + (lo + hi) / 2))) + + +def draw_reports(cases: pd.DataFrame, p: np.ndarray, rng: np.random.Generator) -> pd.Series: + """When each case's report arrives (NaT if never), indexed by case id.""" + reported = rng.random(len(cases)) < p + median = cases["typology"].map(DELAY_MEDIAN_DAYS).fillna(DELAY_MEDIAN_DEFAULT_DAYS) + days = rng.lognormal(np.log(median.to_numpy()), 0.8) + rng.uniform(1 / 24, 0.5, len(cases)) + at = cases["started_at"] + pd.to_timedelta(days, unit="D") + return pd.Series(at.where(reported).to_numpy(), index=cases["case_id"].to_numpy()) + + +def observed_labels(frame: pd.DataFrame, report_at: pd.Series, cutoff: pd.Timestamp) -> np.ndarray: + """A fraud-chain row is labelled fraud only if its case was reported before `cutoff`.""" + known = report_at[report_at <= cutoff].index + return (frame["y"].to_numpy() == 1) & frame["case_id"].isin(known).to_numpy() + + +# ---------------------------------------------------------------- recovery + + +def propagate(frame: pd.DataFrame, y_obs: np.ndarray, window: pd.Timedelta = PROPAGATION_WINDOW): + """Rows labelled clean that moved money into or out of a reported mule wallet. + + A wallet that received a reported scam transfer is a known mule from then on. + Other transfers it received, and the money it sent on, within `window` of its + reported transfers inherit a soft fraud label. Recruited mules are ordinary + customers too, so some of what this labels is legitimate. + """ + paid = frame[y_obs & (frame["type"] == "SEND_MONEY").to_numpy()] + span = paid.groupby("receiver_id")["ts"].agg(["min", "max"]) + lo, hi = span["min"] - window, span["max"] + window + hit = np.zeros(len(frame), dtype=bool) + for side in ("receiver_id", "sender_id"): + start = frame[side].map(lo).to_numpy() + end = frame[side].map(hi).to_numpy() + ts = frame["ts"].to_numpy() + hit |= (ts >= start) & (ts <= end) + return hit & ~y_obs + + +def labelling_propensity( + x: np.ndarray, s: np.ndarray, x_val: np.ndarray, s_val: np.ndarray, seed: int +) -> np.ndarray: + """g(x) = P(labelled | x), fitted out-of-fold so no row is scored by a model that saw it.""" + g = np.zeros(len(s)) + folds = StratifiedKFold(PU_FOLDS, shuffle=True, random_state=seed) + for fit_idx, held_idx in folds.split(x, s): + booster = fit_weighted(x[fit_idx], s[fit_idx], None, x_val, s_val, None, seed) + g[held_idx] = booster.predict(x[held_idx]) + return g + + +def elkan_noto(g: np.ndarray, s: np.ndarray, c: float | None = None) -> tuple[np.ndarray, float]: + """Elkan & Noto (2008): how likely each unlabelled row is fraud, and the label rate c. + + c = P(labelled | fraud) is the mean of g over labelled rows unless it is given. An + unlabelled row is fraud with weight (1 - c)/c * g/(1 - g), clipped to [0, 1]; + labelled rows get NaN (they keep their hard label). This assumes labels are + missing at random among frauds, which biased reporting breaks. + """ + if c is None: + c = float(g[s == 1].mean()) + g = np.clip(g, 1e-6, 1 - 1e-6) + w = np.clip((1 - c) / c * g / (1 - g), 0.0, 1.0) + return np.where(s == 1, np.nan, w), c + + +def with_soft_labels(x: np.ndarray, y: np.ndarray, soft: np.ndarray | None): + """Rows whose `soft` is set appear twice: as fraud with that weight, as clean with the rest.""" + if soft is None: + return x, y, None + m = ~np.isnan(soft) & (soft > 1e-4) + keep = ~m + w = soft[m] + return ( + np.vstack([x[keep], x[m], x[m]]), + np.concatenate([y[keep], np.ones(m.sum()), np.zeros(m.sum())]), + np.concatenate([np.ones(keep.sum()), w, 1 - w]), + ) + + +def arrays(part: pd.DataFrame): + """Features, labels and weights of a labelled frame, `soft` expanded by `with_soft_labels`.""" + soft = part["soft"].to_numpy(dtype=float) if "soft" in part else None + if soft is not None and np.isnan(soft).all(): + soft = None + return with_soft_labels(part[list(FEATURES)].to_numpy(), part["y"].to_numpy(float), soft) + + +def fit_weighted(x, y, w, x_val, y_val, w_val, seed: int) -> lgb.Booster: + """`train.fit_booster`, with sample weights and a seed.""" + params = {**LGB_PARAMS, "seed": seed} + dtrain = lgb.Dataset(x, y, weight=w, feature_name=list(FEATURES)) + dval = lgb.Dataset(x_val, y_val, weight=w_val, reference=dtrain) + booster = lgb.train( + params, + dtrain, + num_boost_round=2000, + valid_sets=[dval], + callbacks=[lgb.early_stopping(100, verbose=False)], + ) + return lgb.Booster(model_str=booster.model_to_string(num_iteration=booster.best_iteration)) + + +# ---------------------------------------------------------------- feedback loop + + +def review_window(cfg: dict) -> tuple[int, int, int]: + """Days (fit until, review from, review until) for the model whose alerts get reviewed.""" + train_end = cfg["train_end_day"] + es = (train_end + cfg["val_end_day"]) // 2 - train_end # as long as val_a + review = max(1, round(0.2 * train_end)) + return train_end - review - es, train_end - review, train_end + + +def verdicts( + clean: pd.DataFrame, + report_at: pd.Series, + cfg: dict, + start: pd.Timestamp, + cutoff: pd.Timestamp, + seed: int, +) -> tuple[pd.DataFrame, dict]: + """Analysts' verdicts on the alerts of a model trained on earlier reports. + + The model is fitted on reports known when the review window opens, alerts on the + top REVIEW_BUDGET of the window, and every alert is closed with the true answer + (the same simulated, always-right reviewer as `make review`). + """ + fit_until, review_from, review_until = review_window(cfg) + opens = start + review_from * DAY + y_then = observed_labels(clean, report_at, opens) + day = clean["day"].to_numpy() + fit, stop = day < fit_until, (day >= fit_until) & (day < review_from) + x = clean[list(FEATURES)].to_numpy() + booster = fit_weighted(x[fit], y_then[fit], None, x[stop], y_then[stop], None, seed) + window = clean[(day >= review_from) & (day < review_until)] + score = booster.predict(window[list(FEATURES)].to_numpy()) + alerted = window[score >= np.quantile(score, 1 - REVIEW_BUDGET)].copy() + alerted["y_mule"] = (alerted["y"] * (alerted["type"] == "SEND_MONEY")).astype("int8") + rows = alerted[list(FEEDBACK_COLUMNS)] + truth = alerted["y"].to_numpy() == 1 + unreported = int((truth & ~observed_labels(alerted, report_at, cutoff)).sum()) + return rows, { + "window_days": [review_from, review_until], + "reviewed": int(len(rows)), + "fraud": int(truth.sum()), + "fraud_never_reported_by_training": unreported, + } + + +# ---------------------------------------------------------------- evaluation + + +def operating_fpr(test: pd.DataFrame, models_root: Path) -> dict: + """The served version's false-positive rate on test at its SERVED_TIER threshold.""" + version = registry.current_version(models_root) + if version is None: + raise SystemExit("no served model: run `make train` first") + directory = models_root / version + threshold = json.loads((directory / "manifest.json").read_text())["thresholds"][SERVED_TIER] + scores = pd.read_parquet(directory / "test_scores.parquet", columns=["txn_id", "risk"]) + risk = test[["txn_id", "y"]].merge(scores, on="txn_id", how="inner") + clean = risk[risk["y"] == 0] + return { + "version": version, + "tier": SERVED_TIER, + "threshold": threshold, + "fpr": float((clean["risk"] >= threshold).mean()), + } + + +def evaluate(test: pd.DataFrame, score: np.ndarray, fpr: float) -> dict: + """Ground-truth outcomes when as many clean rows are alerted as the served model alerts.""" + y = test["y"].to_numpy() + threshold = float(np.quantile(score[y == 0], 1 - fpr)) + flagged = score > threshold + out = metrics.alert_outcomes(test, flagged) + return { + "pr_auc": metrics.ranking(y, score)["pr_auc"], + "fpr": round(float(flagged[y == 0].mean()), 5), + "precision": out["precision"], + "loss_txn_recall": out["loss_txn_recall"], + "case_recall": out["case_recall"], + "taka_recall": out["taka_recall"], + "by_typology": { + t: {"loss_txn_recall": v["loss_txn_recall"], "case_recall": v["case_recall"]} + for t, v in out["by_typology"].items() + }, + } + + +def label_summary(part: pd.DataFrame, y_obs: np.ndarray) -> dict: + """How much of the true fraud the training labels contain, by typology.""" + y = part["y"].to_numpy() == 1 + fraud = part[y].assign(labelled=y_obs[y]) + cases = fraud.groupby("case_id").agg( + typology=("typology", "first"), labelled=("labelled", "max") + ) + return { + "fraud_rows": int(y.sum()), + "fraud_rows_labelled": int(y_obs[y].sum()), + "clean_rows_labelled_fraud": int((y_obs & ~y).sum()), + "case_label_rate": metrics._ratio(cases["labelled"].sum(), len(cases)), + "case_label_rate_by_typology": { + str(t): metrics._ratio(g["labelled"].sum(), len(g)) + for t, g in cases.groupby("typology") + }, + } + + +# ---------------------------------------------------------------- experiment + + +def run_seed(clean: pd.DataFrame, cases: pd.DataFrame, cfg: dict, fpr: float, seed: int) -> dict: + start = pd.Timestamp(cfg["start"]) + cutoff = start + cfg["val_end_day"] * DAY # the model is trained when the test period opens + rng = np.random.default_rng(seed) + rate = cfg["report_rate"] + biased_at = draw_reports(cases, report_probability(cases, rate), rng) + uniform_at = draw_reports(cases, report_probability(cases, rate, biased=False), rng) + + fold = clean["fold"].to_numpy() + tr, va, te = fold == "train", fold == "val_a", fold == "test" + x = clean[list(FEATURES)].to_numpy() + test = clean[te] + y_true = clean["y"].to_numpy() + y_rep = observed_labels(clean, biased_at, cutoff) + y_uni = observed_labels(clean, uniform_at, cutoff) + spread = propagate(clean, y_rep) + soft_prop = np.where(spread, PROPAGATION_WEIGHT, np.nan) + + out: dict = {"labels": {}, "regimes": {}, "diagnostics": {}} + out["labels"] = { + name: label_summary(clean[tr | va], y[tr | va]) + for name, y in (("ground_truth", y_true == 1), ("reported_uniform", y_uni), + ("reported", y_rep)) + } # fmt: skip + in_fit = tr | va + out["diagnostics"]["propagation"] = { + "rows_given_soft_label": int(spread[in_fit].sum()), + "of_which_true_fraud": metrics._ratio( + (spread & (y_true == 1))[in_fit].sum(), spread[in_fit].sum() + ), # fmt: skip + } + + def labelled(y: np.ndarray, soft: np.ndarray | None = None) -> pd.DataFrame: + return clean.assign(y=y.astype("int8"), soft=np.nan if soft is None else soft) + + def fit(part: pd.DataFrame) -> np.ndarray: + train, val = part[part["fold"] == "train"], part[part["fold"] == "val_a"] + booster = fit_weighted(*arrays(train), *arrays(val), seed) + return booster.predict(x[te]) + + def with_verdicts(part: pd.DataFrame) -> pd.DataFrame: + # Verdicts replace the labels of the transactions they cover, exactly as + # `train.add_feedback` does when `make retrain` runs. + sub = {"train": part[part["fold"] == "train"], "test": test} + add_feedback(sub, rows) + sub["train"] = sub["train"].assign(fold="train") + return pd.concat([sub["train"], part[part["fold"] == "val_a"]], ignore_index=True) + + out["regimes"]["ground_truth"] = evaluate(test, fit(labelled(y_true)), fpr) + out["regimes"]["reported_uniform"] = evaluate(test, fit(labelled(y_uni)), fpr) + out["regimes"]["reported"] = evaluate(test, fit(labelled(y_rep)), fpr) + + s_tr = y_rep[tr].astype(float) + g = labelling_propensity(x[tr], s_tr, x[va], y_rep[va].astype(float), seed) + soft = np.full(len(clean), np.nan) + soft[tr], c = elkan_noto(g, s_tr) + out["regimes"]["reported_pu"] = evaluate(test, fit(labelled(y_rep, soft)), fpr) + # The same with c taken as the report rate a survey would give, instead of estimated. + soft[tr], _ = elkan_noto(g, s_tr, rate) + out["regimes"]["reported_pu_known_rate"] = evaluate(test, fit(labelled(y_rep, soft)), fpr) + true_c = metrics._ratio(y_rep[tr].sum(), (y_true[tr] == 1).sum()) + out["diagnostics"]["pu"] = {"estimated_label_rate": round(c, 4), "actual_label_rate": true_c} + + propagated = labelled(y_rep, soft_prop) + out["regimes"]["reported_propagated"] = evaluate(test, fit(propagated), fpr) + + rows, out["diagnostics"]["feedback"] = verdicts(clean, biased_at, cfg, start, cutoff, seed) + out["regimes"]["reported_feedback"] = evaluate(test, fit(with_verdicts(labelled(y_rep))), fpr) + out["regimes"]["reported_propagated_feedback"] = evaluate( + test, fit(with_verdicts(propagated)), fpr + ) + return out + + +def summarise(runs: list[dict]) -> dict: + """Mean and standard deviation across seeds of every number in the runs.""" + + def walk(items: list): + first = items[0] + if isinstance(first, list): + return first # a setting, the same for every seed + if isinstance(first, dict): + return {k: walk([i[k] for i in items if k in i]) for k in first} + values = np.array([np.nan if v is None else v for v in items], dtype=float) + sd = float(np.nanstd(values, ddof=1)) if np.isfinite(values).sum() > 1 else 0.0 + return {"mean": round(float(np.nanmean(values)), 4), "sd": round(sd, 4)} + + return walk(runs) + + +def run(data_dir: Path, models_root: Path, seeds: tuple[int, ...] = SEEDS) -> dict: + t0 = time.perf_counter() + frame = load_frame(data_dir) + cfg = json.loads((data_dir / "meta.json").read_text())["config"] + clean = frame[~frame["ambiguous"]].reset_index(drop=True) + cases = case_table(data_dir) + served = operating_fpr(clean[clean["fold"] == "test"], models_root) + runs = {} + for seed in seeds: + runs[seed] = run_seed(clean, cases, cfg, served["fpr"], seed) + print(f"seed {seed}: " + ", ".join( + f"{k} {v['case_recall']:.3f}" for k, v in runs[seed]["regimes"].items() + )) # fmt: skip + return { + "created_at": datetime.now(UTC).isoformat(timespec="seconds"), + "data_dir": str(data_dir), + "data_seed": cfg["seed"], + "seeds": list(seeds), + "operating_point": served, + "assumptions": { + "report_rate": cfg["report_rate"], + "report_logit": REPORT_LOGIT, + "typology_logit": TYPOLOGY_LOGIT, + "delay_median_days": {**DELAY_MEDIAN_DAYS, "other": DELAY_MEDIAN_DEFAULT_DAYS}, + "labels_known_at_day": cfg["val_end_day"], + "propagation_weight": PROPAGATION_WEIGHT, + "propagation_window_days": PROPAGATION_WINDOW / DAY, + "pu_folds": PU_FOLDS, + "review_budget": REVIEW_BUDGET, + }, + "regimes": list(REGIMES), + "summary": summarise(list(runs.values())), + "runs": {str(k): v for k, v in runs.items()}, + "seconds": round(time.perf_counter() - t0, 1), + } + + +def print_summary(report: dict) -> None: + s = report["summary"]["regimes"] + op = report["operating_point"] + print(f"\nat the {op['version']} {op['tier']} tier's false-positive rate, {op['fpr']:.3%}" + f" ({len(report['seeds'])} seeds, mean ± sd):") # fmt: skip + keys = ("pr_auc", "loss_txn_recall", "case_recall", "taka_recall") + rows = {r: {k: f"{s[r][k]['mean']:.3f} ± {s[r][k]['sd']:.3f}" for k in keys} for r in s} + print(pd.DataFrame(rows).T.to_string()) + print("\nscams caught by typology:") + typ = { + r: {t: f"{v['case_recall']['mean']:.3f}" for t, v in s[r]["by_typology"].items()} for r in s + } + print(pd.DataFrame(typ).T.to_string()) + print("\ndiagnostics:", json.dumps(report["summary"]["diagnostics"])) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Train under realistic labels and compare") + parser.add_argument("--small", action="store_true") + parser.add_argument("--data", type=Path, default=None) + parser.add_argument("--seeds", type=int, nargs="+", default=list(SEEDS)) + parser.add_argument( + "--out", type=Path, default=settings.artifacts_dir / "reports" / REPORT_FILE + ) + args = parser.parse_args() + data_dir = args.data or settings.data_dir / ("small" if args.small else "full") + report = run(data_dir, registry.models_dir(), tuple(args.seeds)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2, default=str)) + print_summary(report) + print(f"\nwrote {args.out} in {report['seconds']}s") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_label_realism.py b/backend/tests/test_label_realism.py new file mode 100644 index 0000000..9f14e5c --- /dev/null +++ b/backend/tests/test_label_realism.py @@ -0,0 +1,103 @@ +import numpy as np +import pandas as pd +import pytest + +from fraudlens.models import label_realism as L + + +def _cases(n: int = 400) -> pd.DataFrame: + rng = np.random.default_rng(0) + typologies = ["account_takeover", "lottery_fee", "wrong_send", "impersonation"] + return pd.DataFrame( + { + # Case ids are not positions: some cases never produced a transfer. + "case_id": np.arange(n) * 3 + 5, + "typology": [typologies[i % 4] for i in range(n)], + "loss": rng.lognormal(8, 1, n), + "urban": (rng.random(n) < 0.5).astype(float), + "tenure_days": rng.uniform(10, 900, n), + "started_at": pd.Timestamp("2026-01-01") + pd.to_timedelta(np.arange(n), unit="h"), + } + ) + + +def test_biased_reporting_averages_the_simulator_rate_and_favours_takeovers(): + cases = _cases() + p = np.asarray(L.report_probability(cases, 0.5)) + assert p.mean() == pytest.approx(0.5, abs=1e-6) + by_type = pd.Series(p).groupby(cases["typology"]).mean() + assert by_type["account_takeover"] > by_type["impersonation"] > by_type["lottery_fee"] + big = cases["loss"] > cases["loss"].median() + assert p[big].mean() > p[~big].mean() + assert np.all(np.asarray(L.report_probability(cases, 0.5, biased=False)) == 0.5) + + +def test_reports_stay_with_their_case_when_case_ids_are_not_positions(): + cases = _cases() + p = np.where(cases["typology"] == "account_takeover", 1.0, 0.0) + at = L.draw_reports(cases, p, np.random.default_rng(1)) + assert list(at.index) == list(cases["case_id"]) + reported = at.notna().to_numpy() + assert np.array_equal(reported, cases["typology"].to_numpy() == "account_takeover") + assert (at.dropna() > cases.set_index("case_id")["started_at"].reindex(at.dropna().index)).all() + + +def test_only_reports_in_by_the_cutoff_label_their_case(): + ts = pd.Timestamp("2026-02-01") + report_at = pd.Series([ts, ts + pd.Timedelta(days=5), pd.NaT], index=[10, 11, 12]) + frame = pd.DataFrame({"y": [1, 1, 1, 0], "case_id": [10, 11, 12, -1]}) + y = L.observed_labels(frame, report_at, ts + pd.Timedelta(days=1)) + assert y.tolist() == [True, False, False, False] + + +def test_propagation_reaches_other_money_through_a_reported_mule_within_the_window(): + t = pd.Timestamp("2026-02-01") + frame = pd.DataFrame( + { + "type": ["SEND_MONEY", "SEND_MONEY", "CASH_OUT", "SEND_MONEY", "SEND_MONEY"], + "sender_id": ["V1", "V2", "M1", "V3", "X"], + "receiver_id": ["M1", "M1", "A1", "M1", "Y"], + "ts": [t, t + pd.Timedelta(hours=5), t + pd.Timedelta(hours=6), + t + pd.Timedelta(days=10), t], + } + ) # fmt: skip + y_obs = np.array([True, False, False, False, False]) + hit = L.propagate(frame, y_obs) + # Another victim's transfer and the mule's cash-out; not the one 10 days later, + # not unrelated traffic, and never the already-labelled row. + assert hit.tolist() == [False, True, True, False, False] + + +def test_elkan_noto_weights(): + g = np.array([0.4, 0.4, 0.2, 0.01]) + s = np.array([1.0, 1.0, 0.0, 0.0]) + w, c = L.elkan_noto(g, s) + assert c == pytest.approx(0.4) + assert np.isnan(w[:2]).all() + assert w[2] == pytest.approx(1.5 * 0.2 / 0.8) + assert 0 <= w[3] < w[2] <= 1 + w_known, c_known = L.elkan_noto(g, s, 0.5) + assert c_known == 0.5 and w_known[2] == pytest.approx(0.25) + + +def test_soft_rows_are_split_into_weighted_fraud_and_clean_copies(): + x = np.arange(6, dtype=float).reshape(3, 2) + y = np.array([1.0, 0.0, 0.0]) + xs, ys, ws = L.with_soft_labels(x, y, np.array([np.nan, 0.7, np.nan])) + assert len(xs) == 4 and ys.tolist() == [1.0, 0.0, 1.0, 0.0] + assert ws.tolist() == pytest.approx([1.0, 1.0, 0.7, 0.3]) + assert L.with_soft_labels(x, y, None)[2] is None + + +def test_experiment_runs_every_regime_at_the_served_false_positive_rate(trained): + data_dir, models_root, _ = trained + report = L.run(data_dir, models_root, seeds=(7,)) + assert list(report["summary"]["regimes"]) == list(L.REGIMES) + target = report["operating_point"]["fpr"] + for result in report["runs"]["7"]["regimes"].values(): + assert result["fpr"] <= target + 1e-3 + assert 0 <= result["pr_auc"] <= 1 + labels = report["runs"]["7"]["labels"] + assert labels["ground_truth"]["case_label_rate"] == 1.0 + assert labels["reported"]["fraud_rows_labelled"] < labels["ground_truth"]["fraud_rows"] + assert labels["reported"]["clean_rows_labelled_fraud"] == 0 diff --git a/docs/MODEL_CARD.md b/docs/MODEL_CARD.md index ede2b93..50e1ce0 100644 --- a/docs/MODEL_CARD.md +++ b/docs/MODEL_CARD.md @@ -267,7 +267,8 @@ has not been done yet. real fraud. Precision in production would be lower at the same thresholds; thresholds must be re-fitted on real data. - **Labels.** Training uses ground-truth labels. In production only reported - cases are labelled (about half), which biases what the model learns. + cases are labelled (about half), which biases what the model learns. §15 + measures what that costs and what wins it back. - **No adaptation.** Simulated scammers do not react to the controls. - **Young wallets and fast cash-out are risk signals.** New customers and remittance receivers legitimately show both. The data contains such customers @@ -402,3 +403,87 @@ make test # 203 tests, including leakage and round-trip check `report.json` holds every number above; `manifest.json` holds thresholds, calibration, feature list and data seed; `test_scores.parquet` holds the score of every test transaction. + +## 15. Under-reporting: training on reported scams only + +Every number here is read from `backend/artifacts/reports/label_realism.json`, +which `make label-realism` writes (about 5 to 15 minutes). It does not change +the served model or any number above. + +The served model learnt from every scam the simulator ran. A wallet provider +only learns about the ones victims report. To measure what that costs, the +transaction model is retrained, with the same LightGBM settings, training fold +and early stopping as `make train`, on the labels a provider would have had, +and every version is scored on **every** scam in the test period. Each is +compared at the false-positive rate the served model (v4, warn tier) runs at +on test, 0.523% of honest payments, so the rows differ only in how they rank. +Five seeds (7, 11, 13, 17, 19) vary both the reporting draw and the model; +values are mean ± sd. + +**How reporting is simulated.** A case is reported with a probability that +averages the simulator's `report_rate` of 50% but is higher for larger losses, +urban victims, accounts held for a year or more (the data has no person age) +and account takeovers, and lower for lottery-fee and investment scams. Reports +arrive after the simulator's delay (median 1.5 days, 5 for investment scams); +only those in by the start of the test period count. A reported case labels all +its transactions, including the mule's forwards and cash-outs. Unreported fraud +is labelled clean. These weights are assumptions, listed in the JSON, not +measurements. In the training data this labels 82% of takeovers, 56% of +impersonations, 40% of wrong-send and 35% of lottery-fee cases. + +| Training labels | Scams caught | Loss transfers caught | Taka stopped | PR-AUC | +| --- | --- | --- | --- | --- | +| Every scam (as served) | 87.3% ± 0.5 | 81.7% ± 0.7 | 85.0% ± 0.6 | 0.831 ± 0.003 | +| Reported only, half at random | 72.9% ± 5.3 | 66.4% ± 6.0 | 72.3% ± 7.4 | 0.595 ± 0.038 | +| **Reported only, biased** | **78.8% ± 2.7** | 71.4% ± 2.2 | 77.9% ± 1.9 | 0.724 ± 0.019 | +| + positive-unlabelled learning | 79.4% ± 1.5 | 72.1% ± 1.9 | 79.5% ± 1.9 | 0.762 ± 0.020 | +| + same, report rate given as 50% | 79.0% ± 1.8 | 71.9% ± 1.8 | 78.8% ± 2.0 | 0.754 ± 0.020 | +| + labels through mule wallets | 82.8% ± 2.2 | 75.6% ± 1.7 | 80.9% ± 1.5 | 0.770 ± 0.043 | +| + analyst verdicts on alerts | 83.0% ± 2.0 | 76.1% ± 2.1 | 81.9% ± 1.6 | 0.769 ± 0.033 | +| + mule-wallet labels and verdicts | 82.4% ± 1.4 | 75.0% ± 1.4 | 79.7% ± 2.1 | 0.781 ± 0.023 | + +Scams caught, by typology (mean over seeds): + +| Training labels | account_takeover | impersonation | lottery_fee | wrong_send | **investment_scam** (unseen) | +| --- | --- | --- | --- | --- | --- | +| Every scam | 100% | 96.8% | 98.5% | 100% | 71.2% ± 1.3 | +| Reported only, biased | 100% | 96.8% | 95.8% | 100% | **52.4% ± 5.8** | +| + positive-unlabelled | 100% | 96.8% | 95.2% | 100% | 54.1% ± 3.5 | +| + labels through mule wallets | 100% | 96.8% | 96.7% | 100% | 61.5% ± 4.9 | +| + analyst verdicts | 100% | 96.8% | 96.7% | 100% | 61.8% ± 4.5 | + +What this shows: + +- **Under-reporting costs about 8.5 points of scams caught** (87.3% to 78.8%) + at the same false-alert rate, and PR-AUC falls from 0.831 to 0.724. +- **It hides the scam the model has not seen, not the ones it has.** Known + typologies stay at 96 to 100% caught: scripted fraud is easy enough that half + the examples suffice. The held-out investment scam falls from 71.2% to 52.4%. + With fewer examples the model learns the narrow signature of each reported + pattern rather than the general shape of a mule chain, which is what caught + the new scam. +- **Who reports matters less than how many.** Biased reporting did better than + reporting at random (78.8% against 72.9%), with far less spread between + seeds, because it keeps the large, cleanly labelled takeover and + impersonation cases; it also labels more fraud rows (1,293 against 1,066 on + average). The random-reporting result swings by ±5 points with the draw. +- **Positive-unlabelled learning does little.** Elkan–Noto weighting lifts + PR-AUC to 0.762 but scams caught by under one point, within noise, and + telling it the true report rate does not help. Its assumption, that labels go + missing at random among frauds, is what biased reporting breaks. +- **What helps is the network and the analysts.** Spreading soft labels (weight + 0.7) from reported mule wallets to their other incoming and outgoing money + within three days labels about 755 extra rows, 91% of them really fraud, and + wins back 4 points of scams caught and 9 of the unseen typology. Analyst + verdicts on the top 1% of a model's alerts over the last 15 training days + (about 978 reviews, 230 of them fraud no victim had reported) win back the + same. Together they are no better than either alone: they recover largely the + same rows. Roughly half the cost of under-reporting is recovered; the unseen + typology stays about 10 points below the ground-truth model. + +Cautions: the reviewers are simulated and always right (as in §13); the +operating threshold is set on test negatives so every row has the same false- +alert rate, which isolates ranking but is not how a threshold would be chosen in +production; the report propensities are assumptions; and five seeds give +standard deviations of 1.5 to 5 points, so differences under about 3 points +between recovery methods are not meaningful. diff --git a/frontend/app/(console)/model/page.tsx b/frontend/app/(console)/model/page.tsx index 5ac4c8c..1eabef2 100644 --- a/frontend/app/(console)/model/page.tsx +++ b/frontend/app/(console)/model/page.tsx @@ -6,7 +6,7 @@ import { HBars } from "@/components/charts"; import { Async, Badge, Card, Empty, Facts, PageHeader, Stat, StatusBadge, Table, Tabs, Td } from "@/components/ui"; import { useApi } from "@/lib/api"; import { day, num, pct, shortDay, TIER_LABEL, TIERS, when, words } from "@/lib/format"; -import type { Drift, Feedback, Insights, Registry, Report, Shadow, Summary } from "@/lib/types"; +import type { Drift, Feedback, Insights, LabelRealism, Registry, Report, Shadow, Spread, Summary } from "@/lib/types"; type Tab = "performance" | "drift" | "versions" | "feedback"; @@ -197,6 +197,7 @@ function Performance({ report, summary }: { report: Report; summary?: Summary }) /> + {report.label_realism && }

Model {report.model_version}, trained on {num(report.model.data.rows.train)} payments and measured on {num(report.model.data.rows.test)} later ones it never saw. The data is simulated: these numbers say the pipeline works end to end, not how it would do on real traffic. @@ -205,6 +206,48 @@ function Performance({ report, summary }: { report: Report; summary?: Summary }) ); } +const LABEL_REGIME: Record = { + ground_truth: "Every scam labelled (what the model was trained on)", + reported_uniform: "Only reported scams, half at random", + reported: "Only reported scams, as victims really report", + reported_pu: "+ positive-unlabelled learning", + reported_pu_known_rate: "+ positive-unlabelled, report rate known", + reported_propagated: "+ labels spread through mule wallets", + reported_feedback: "+ analyst verdicts on alerts", + reported_propagated_feedback: "+ mule-wallet labels and verdicts", +}; + +function spread(value: Spread | undefined): string { + return value ? `${pct(value.mean)} ± ${num(value.sd * 100, 1)}` : "–"; +} + +function UnderReporting({ data }: { data: LabelRealism }) { + const rows = data.regimes.filter((name) => data.summary.regimes[name]); + return ( +

+ + + {rows.map((name) => { + const row = data.summary.regimes[name]; + return ( + + + + + + + ); + })} +
{LABEL_REGIME[name] ?? words(name)}{spread(row.case_recall)}{spread(row.by_typology.investment_scam?.case_recall)}{row.pr_auc ? `${num(row.pr_auc.mean, 3)} ± ${num(row.pr_auc.sd, 3)}` : "–"}
+
+
+ ); +} + function DriftView({ drift, backtest }: { drift: Drift; backtest?: Insights["drift"] }) { const worst = [...drift.features].sort((a, b) => b.psi - a.psi).slice(0, 14); return ( diff --git a/frontend/lib/types.ts b/frontend/lib/types.ts index 3e5818d..d02328b 100644 --- a/frontend/lib/types.ts +++ b/frontend/lib/types.ts @@ -515,6 +515,23 @@ export interface Report { single_decisions: { decisions: number; alerts: number; decide_ms: { p50: number; p95: number; max: number } } & Record; } & Record; insights: Insights | null; + label_realism?: LabelRealism | null; +} + +/** Mean and standard deviation across seeds. */ +export interface Spread { mean: number; sd: number } + +export interface LabelRealism { + created_at: string; + seeds: number[]; + operating_point: { version: string; tier: string; threshold: number; fpr: number }; + regimes: string[]; + summary: { + regimes: Record; + }>; + } & Record; } export interface ModelInfo { From a0a53ec531f496ac43d676d84022e6a821863f8b Mon Sep 17 00:00:00 2001 From: 0xdevabir <261-15-001@diu.edu.bd> Date: Wed, 7 Oct 2026 09:40:45 +0600 Subject: [PATCH 07/12] Bangla/English customer intervention screens and customer appeals with analyst queue --- Makefile | 5 +- README.md | 5 +- .../alembic/versions/0002_shadow_scores.py | 5 +- .../alembic/versions/0004_customer_appeals.py | 78 +++ backend/src/fraudlens/api/app.py | 16 +- backend/src/fraudlens/api/routes/appeals.py | 134 +++++ backend/src/fraudlens/api/routes/demo.py | 27 + backend/src/fraudlens/api/schemas.py | 26 + backend/src/fraudlens/api/views.py | 25 +- backend/src/fraudlens/decision/customer.py | 53 ++ backend/src/fraudlens/mlops/feedback.py | 64 ++- backend/src/fraudlens/platform/appeals.py | 130 +++++ backend/src/fraudlens/platform/load.py | 2 +- backend/src/fraudlens/platform/models.py | 44 ++ backend/src/fraudlens/platform/scoring.py | 4 + backend/tests/test_platform.py | 150 +++++- docs/PLATFORM.md | 23 +- docs/plans/08-customer-experience.md | 128 +++++ frontend/app/(console)/appeals/page.tsx | 140 +++++ frontend/app/(console)/phone/copy.ts | 196 +++++++ frontend/app/(console)/phone/page.tsx | 478 +++++++++++++----- frontend/components/session.tsx | 1 + frontend/lib/types.ts | 27 + frontend/scripts/phone-e2e.cjs | 134 +++++ frontend/scripts/smoke.cjs | 2 +- 25 files changed, 1754 insertions(+), 143 deletions(-) create mode 100644 backend/alembic/versions/0004_customer_appeals.py create mode 100644 backend/src/fraudlens/api/routes/appeals.py create mode 100644 backend/src/fraudlens/decision/customer.py create mode 100644 backend/src/fraudlens/platform/appeals.py create mode 100644 docs/plans/08-customer-experience.md create mode 100644 frontend/app/(console)/appeals/page.tsx create mode 100644 frontend/app/(console)/phone/copy.ts create mode 100644 frontend/scripts/phone-e2e.cjs diff --git a/Makefile b/Makefile index b55078b..32edd51 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: help setup up redis down data features train policy insights intel pipeline test lint fmt \ migrate seed load api replay replay-live verify platform \ - review retrain shadow models promote console console-build smoke \ + review retrain shadow models promote console console-build smoke e2e-phone \ demo demo-reset API_PORT ?= 8010 @@ -104,6 +104,9 @@ console-build: ## Production build of the console smoke: ## Open every console page in a headless browser as each role (needs the API and the console) cd frontend && pnpm exec playwright install chromium-headless-shell && node scripts/smoke.cjs +e2e-phone: ## The customer phone in Bangla and English, and an appeal round trip (needs the API and the console) + cd frontend && pnpm exec playwright install chromium-headless-shell && node scripts/phone-e2e.cjs + test: ## Run backend tests (the platform tests need `make up`) cd backend && uv run pytest -q diff --git a/README.md b/README.md index f62e121..92bac6c 100644 --- a/README.md +++ b/README.md @@ -403,12 +403,13 @@ To run a challenger in shadow mode, set `FRAUDLENS_SHADOW_MODEL_VERSION` to its Check it ```bash -make test # 227 backend tests; the platform tests need `make up` +make test # 230 backend tests; the platform tests need `make up` make lint # ruff, tsc, eslint make smoke # opens every console page as each role in a headless browser +make e2e-phone # the customer phone in Bangla and English, plus an appeal round trip ``` -`make smoke` needs the demo (or `make api` and `make console`) running and `make setup` done. CI (`.github/workflows/ci.yml`) runs the tests and the lint against real Postgres and Redis, builds the console and builds both images. +`make smoke` and `make e2e-phone` need the demo (or `make api` and `make console`) running and `make setup` done. CI (`.github/workflows/ci.yml`) runs the tests and the lint against real Postgres and Redis, builds the console and builds both images. diff --git a/backend/alembic/versions/0002_shadow_scores.py b/backend/alembic/versions/0002_shadow_scores.py index 0c57bac..b43db63 100644 --- a/backend/alembic/versions/0002_shadow_scores.py +++ b/backend/alembic/versions/0002_shadow_scores.py @@ -28,7 +28,10 @@ def upgrade() -> None: sa.Column("tier", sa.String(length=8), nullable=False), sa.Column("latency_ms", sa.Float(), nullable=True), sa.Column( - "created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, ), sa.CheckConstraint( "tier IN ('allow', 'warn', 'step_up', 'hold')", name=op.f("ck_shadow_scores_tier") diff --git a/backend/alembic/versions/0004_customer_appeals.py b/backend/alembic/versions/0004_customer_appeals.py new file mode 100644 index 0000000..cbd6bf4 --- /dev/null +++ b/backend/alembic/versions/0004_customer_appeals.py @@ -0,0 +1,78 @@ +"""customer appeals + +Revision ID: 0004_appeals +Revises: 0003 +Create Date: 2026-10-07 12:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0004_appeals" +down_revision: str | Sequence[str] | None = "0003" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "appeals", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("txn_id", sa.BigInteger(), nullable=False), + sa.Column("wallet_id", sa.String(length=32), nullable=False), + sa.Column("case_id", sa.Integer(), nullable=True), + sa.Column("tier", sa.String(length=8), nullable=False), + sa.Column("relation", sa.String(length=16), nullable=False), + sa.Column("reason", sa.Text(), nullable=False), + sa.Column("status", sa.String(length=12), server_default="pending", nullable=False), + sa.Column("filed_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("sla_due_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("decided_by", sa.Integer(), nullable=True), + sa.Column("decision_note", sa.Text(), nullable=True), + sa.Column("decided_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "(status = 'pending') = (decided_by IS NULL)", + name=op.f("ck_appeals_decided_has_decider"), + ), + sa.CheckConstraint( + "relation IN ('family', 'friend', 'business', 'seller', 'landlord', 'employer', " + "'other', 'none')", + name=op.f("ck_appeals_relation"), + ), + sa.CheckConstraint( + "status IN ('pending', 'approved', 'rejected')", name=op.f("ck_appeals_status") + ), + sa.CheckConstraint("tier IN ('warn', 'step_up', 'hold')", name=op.f("ck_appeals_tier")), + sa.ForeignKeyConstraint(["case_id"], ["cases.id"], name=op.f("fk_appeals_case_id_cases")), + sa.ForeignKeyConstraint( + ["decided_by"], ["users.id"], name=op.f("fk_appeals_decided_by_users") + ), + sa.ForeignKeyConstraint( + ["txn_id"], + ["transactions.txn_id"], + name=op.f("fk_appeals_txn_id_transactions"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_appeals")), + sa.UniqueConstraint("txn_id", name=op.f("uq_appeals_txn_id")), + ) + op.create_index("ix_appeals_status_sla", "appeals", ["status", "sla_due_at"], unique=False) + op.create_index(op.f("ix_appeals_wallet_id"), "appeals", ["wallet_id"], unique=False) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index(op.f("ix_appeals_wallet_id"), table_name="appeals") + op.drop_index("ix_appeals_status_sla", table_name="appeals") + op.drop_table("appeals") diff --git a/backend/src/fraudlens/api/app.py b/backend/src/fraudlens/api/app.py index d3d3ed6..1791b0f 100644 --- a/backend/src/fraudlens/api/app.py +++ b/backend/src/fraudlens/api/app.py @@ -26,7 +26,19 @@ from ..platform.stream import Worker from .deps import Platform from .middleware import RequestContext, error_body -from .routes import alerts, auth, cases, customer, demo, intel, mlops, network, ops, scoring +from .routes import ( + alerts, + appeals, + auth, + cases, + customer, + demo, + intel, + mlops, + network, + ops, + scoring, +) log = logging.getLogger(__name__) @@ -165,7 +177,7 @@ async def unavailable(request: Request, exc: Exception) -> JSONResponse: for kind in (OperationalError, InterfaceError, RedisError): app.add_exception_handler(kind, unavailable) - for module in (auth, scoring, alerts, cases, network, customer, intel, ops, mlops): + for module in (auth, scoring, alerts, cases, appeals, network, customer, intel, ops, mlops): app.include_router(module.router, prefix="/v1") if not settings.production: # stand-ins for the customer app; see routes/demo.py app.include_router(demo.router, prefix="/v1") diff --git a/backend/src/fraudlens/api/routes/appeals.py b/backend/src/fraudlens/api/routes/appeals.py new file mode 100644 index 0000000..1c8abf6 --- /dev/null +++ b/backend/src/fraudlens/api/routes/appeals.py @@ -0,0 +1,134 @@ +"""Customer appeals: filed by the customer's app, answered by a reviewer within a deadline.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from fastapi import APIRouter, Query +from sqlalchemy import case as sql_case +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from ...platform import appeals as workflow +from ...platform.audit import WorkflowError +from ...platform.models import Appeal, Transaction, User +from ..deps import Db, Plat, Platform, Reviewer, RowId, Service, TxnId +from ..schemas import AppealDecision, AppealIn +from ..views import appeal_view +from .customer import _limit + +router = APIRouter(tags=["appeals"]) + +_TIER = sql_case({"hold": 3, "step_up": 2, "warn": 1}, value=Appeal.tier, else_=0) + + +def customer_appeal_view(p: Platform, s: Session, appeal: Appeal) -> dict: + """What the customer's app shows: where the appeal stands, never who or why internally.""" + txn = s.get(Transaction, appeal.txn_id) + return { + "id": appeal.id, + "txn_id": appeal.txn_id, + "tier": appeal.tier, + "status": appeal.status, + "filed_at": appeal.filed_at, + "sla_due_at": appeal.sla_due_at, + "decided_at": appeal.decided_at, + "txn_status": txn.status if txn else None, + "now": p.scorer.now(), + } + + +def find_appeal(s: Session, wallet_id: str, txn_id: int) -> Appeal: + appeal = s.scalar(select(Appeal).where(Appeal.txn_id == txn_id)) + # The same answer for "no appeal" and "not yours". + if appeal is None or appeal.wallet_id != wallet_id: + raise WorkflowError(404, "appeal_not_found", "no appeal for this payment") + return appeal + + +# ------------------------------------------------------------- the customer's app + + +@router.post("/customer/transactions/{txn_id}/appeal", status_code=201) +def file_appeal(txn_id: TxnId, body: AppealIn, p: Plat, s: Db, ctx: Service) -> dict: + """'This is a genuine payment.' A person answers it; it decides nothing by itself.""" + _limit(p.report_limit, f"appeal:{body.wallet_id}") + appeal = workflow.file_appeal(p.scorer, ctx, body.wallet_id, txn_id, body.relation, body.reason) + return customer_appeal_view(p, s, appeal) + + +@router.get("/customer/transactions/{txn_id}/appeal") +def appeal_status( + txn_id: TxnId, + p: Plat, + s: Db, + ctx: Service, + wallet_id: Annotated[str, Query(pattern=r"^[A-Za-z0-9_-]{1,32}$")], +) -> dict: + return customer_appeal_view(p, s, find_appeal(s, wallet_id, txn_id)) + + +# ------------------------------------------------------------------ the reviewers + + +@router.get("/appeals") +def list_appeals( + p: Plat, + s: Db, + ctx: Reviewer, + status: Literal["pending", "approved", "rejected"] | None = "pending", + limit: Annotated[int, Query(ge=1, le=200)] = 50, +) -> dict: + """The queue: money waiting first, then the nearest deadline.""" + where = [Appeal.status == status] if status else [] + rows = s.execute( + select(Appeal, Transaction) + .join(Transaction, Transaction.txn_id == Appeal.txn_id) + .where(*where) + .order_by( + (Appeal.status != "pending"), + (Transaction.status != "held"), + _TIER.desc(), + Appeal.sla_due_at.asc(), + Appeal.id.desc(), + ) + .limit(limit) + ).all() + ids = {a.decided_by for a, _ in rows if a.decided_by is not None} + names = ( + dict(s.execute(select(User.id, User.username).where(User.id.in_(ids))).all()) if ids else {} + ) + counts = dict(s.execute(select(Appeal.status, func.count()).group_by(Appeal.status)).all()) + now = p.scorer.now() + return { + "now": now, + "counts": {k: counts.get(k, 0) for k in ("pending", "approved", "rejected")}, + "appeals": [ + appeal_view(a, now, names) + | { + "amount": t.amount, + "receiver_id": t.receiver_id, + "txn_status": t.status, + "ts": t.ts, + } + for a, t in rows + ], + } + + +@router.post("/appeals/{appeal_id}/approve") +def approve(appeal_id: RowId, body: AppealDecision, p: Plat, ctx: Reviewer) -> dict: + """The payment is genuine: a held one is released now.""" + done = workflow.decide_appeal(p.scorer, ctx, appeal_id, True, body.note) + return {"appeal": appeal_view(done["appeal"], p.scorer.now()), **_outcome(done)} + + +@router.post("/appeals/{appeal_id}/reject") +def reject(appeal_id: RowId, body: AppealDecision, p: Plat, ctx: Reviewer) -> dict: + """Not convinced: a held payment stays held, and its case decides.""" + done = workflow.decide_appeal(p.scorer, ctx, appeal_id, False, body.note) + return {"appeal": appeal_view(done["appeal"], p.scorer.now()), **_outcome(done)} + + +def _outcome(done: dict) -> dict: + return {"txn_status": done["txn_status"], "released": done["released"]} diff --git a/backend/src/fraudlens/api/routes/demo.py b/backend/src/fraudlens/api/routes/demo.py index f47515e..6aa19fa 100644 --- a/backend/src/fraudlens/api/routes/demo.py +++ b/backend/src/fraudlens/api/routes/demo.py @@ -21,12 +21,14 @@ from ...decision.policy import TIERS from ...features.engine import DISTRICT_COORDS, MIN_NETWORK_HISTORY +from ...platform import appeals as appeal_flow from ...platform import cases as workflow from ...platform.audit import WorkflowError, audit from ...platform.events import Identifier, TxnIn from ...platform.models import Decision, Transaction from ..deps import Db, Plat, Platform, Staff from ..schemas import ( + DemoAppeal, DemoClock, DemoPayment, DemoResponse, @@ -36,6 +38,7 @@ ScamReport, ) from ..views import result_view +from .appeals import customer_appeal_view, find_appeal from .customer import _limit, check_recipient, check_text, verify_payment router = APIRouter(prefix="/demo", tags=["demo"]) @@ -212,6 +215,30 @@ def report(body: ScamReport, p: Plat, ctx: Staff) -> dict: return {"report_id": made.id, "case_id": made.case_id} +def _sender(s: Session, txn_id: int) -> str: + sender = s.scalar(select(Transaction.sender_id).where(Transaction.txn_id == txn_id)) + if sender is None: + raise WorkflowError(404, "txn_not_found", "no such transaction") + return sender + + +@router.post("/appeal", status_code=201) +def appeal(body: DemoAppeal, p: Plat, s: Db, ctx: Staff) -> dict: + """The demo customer's 'this is a genuine payment'. A reviewer answers it on /appeals.""" + sender = _sender(s, body.txn_id) + _limit(p.report_limit, f"appeal:{sender}") + made = appeal_flow.file_appeal(p.scorer, ctx, sender, body.txn_id, body.relation, body.reason) + return customer_appeal_view(p, s, made) + + +@router.get("/appeal") +def appeal_status( + txn_id: Annotated[int, Query(ge=0, lt=2**62)], p: Plat, s: Db, ctx: Staff +) -> dict: + """Where the demo customer's appeal stands, as their app would show it.""" + return customer_appeal_view(p, s, find_appeal(s, _sender(s, txn_id), txn_id)) + + @router.post("/recipient-check") def recipient_check(body: RecipientCheck, p: Plat, ctx: Staff) -> dict: """What the app would show before the customer types an amount.""" diff --git a/backend/src/fraudlens/api/schemas.py b/backend/src/fraudlens/api/schemas.py index fd792de..707b90d 100644 --- a/backend/src/fraudlens/api/schemas.py +++ b/backend/src/fraudlens/api/schemas.py @@ -121,6 +121,32 @@ class ScamReport(Body): ] +AppealReason = Annotated[ + str, StringConstraints(strip_whitespace=True, min_length=3, max_length=500) +] +Relation = Literal[ + "family", "friend", "business", "seller", "landlord", "employer", "other", "none" +] + + +class AppealIn(Body): + """'This is a genuine payment.' Typed by the customer: shown masked, never read to decide.""" + + wallet_id: Identifier + relation: Relation + reason: AppealReason + + +class DemoAppeal(Body): + txn_id: int = Field(ge=0, lt=2**62) + relation: Relation + reason: AppealReason + + +class AppealDecision(Body): + note: Reason + + class MessageCheck(Body): """A message the customer received and wants checked. It is read, never stored.""" diff --git a/backend/src/fraudlens/api/views.py b/backend/src/fraudlens/api/views.py index beb0594..46324f3 100644 --- a/backend/src/fraudlens/api/views.py +++ b/backend/src/fraudlens/api/views.py @@ -6,7 +6,8 @@ from ..intel.attribute import categorise from ..intel.taxonomy import load_taxonomy -from ..platform.models import Case, CaseEvent, Decision, FreezeRequest, Transaction, User +from ..platform.models import Appeal, Case, CaseEvent, Decision, FreezeRequest, Transaction, User +from ..platform.pii import redact from ..platform.scoring import Result @@ -74,6 +75,28 @@ def freeze_view(r: FreezeRequest, names: dict[int, str] | None = None) -> dict: } +def appeal_view(a: Appeal, now: datetime, names: dict[int, str] | None = None) -> dict: + names = names or {} + return { + "id": a.id, + "txn_id": a.txn_id, + "wallet_id": a.wallet_id, + "case_id": a.case_id, + "tier": a.tier, + "relation": a.relation, + # Typed by a customer: numbers and addresses in it are masked. + "reason": redact(a.reason), + "status": a.status, + "filed_at": a.filed_at, + "sla_due_at": a.sla_due_at, + "overdue": a.status == "pending" and a.sla_due_at < now, + "decided_by": a.decided_by, + "decider": names.get(a.decided_by), + "decision_note": a.decision_note, + "decided_at": a.decided_at, + } + + def alert_view(t: Transaction, d: Decision) -> dict: """One line of the alert queue.""" return { diff --git a/backend/src/fraudlens/decision/customer.py b/backend/src/fraudlens/decision/customer.py new file mode 100644 index 0000000..0e6d4f7 --- /dev/null +++ b/backend/src/fraudlens/decision/customer.py @@ -0,0 +1,53 @@ +"""Which scam pattern a customer-facing screen should name. + +The policy's customer message is a fixed text per scenario and tier. A good +warning also asks the one question that fits the scam in front of the customer +("did someone claiming to be from your bank ask you to send this?"). This picks +that question's key from what the decision already recorded: the rules that +fired, its scenario, and the typology of the past cases it most resembles. + +Nothing is scored here, and the key says no more than the message already does: +no score, no reason detail, nothing about the receiving wallet beyond what the +recipient check would say. +""" + +from __future__ import annotations + +from collections import defaultdict + +CUES = ( + "reported_recipient", # the receiving wallet is confirmed fraud (rule R01) + "impersonation", # someone posing as upay, bKash, a bank or an official + "prize", # a prize or lottery that needs a fee + "investment", # an investment that promises high returns + "wrong_send", # "I sent you money by mistake, send it back" + "not_you", # account takeover, or use from an unusual place or network + "generic", +) + +_TYPOLOGY = { + "impersonation": "impersonation", + "lottery_fee": "prize", + "investment_scam": "investment", + "wrong_send": "wrong_send", + "account_takeover": "not_you", +} + + +def cue(tier: str, scenario: str | None, detail: dict | None) -> str | None: + """The pattern key for an alert, or None for a payment that went straight through.""" + if tier == "allow": + return None + detail = detail or {} + fired = {r.get("id") for r in detail.get("rule_trace") or [] if r.get("status") == "fired"} + if "R01_RECIPIENT_CONFIRMED_FRAUD" in fired: + return "reported_recipient" + if scenario in ("takeover", "unusual_access"): + return "not_you" + weight: dict[str, float] = defaultdict(float) + for case in detail.get("similar_cases") or []: + if case.get("typology"): + weight[case["typology"]] += case.get("similarity") or 0.0 + if weight: + return _TYPOLOGY.get(max(weight, key=weight.__getitem__), "generic") + return "generic" diff --git a/backend/src/fraudlens/mlops/feedback.py b/backend/src/fraudlens/mlops/feedback.py index f50ddbf..ebe5327 100644 --- a/backend/src/fraudlens/mlops/feedback.py +++ b/backend/src/fraudlens/mlops/feedback.py @@ -8,49 +8,69 @@ Only what was alerted can be reviewed, so these labels are a biased sample: they say where the model was right or wrong among its alerts and nothing about the fraud it let through. They are added to the training data, never used in its place. + +A customer appeal a reviewer approved ("this is a genuine payment", checked by a +person) labels that one payment `legitimate`, unless its case reached a verdict: +the case looked at more and outranks it. A rejected appeal labels nothing. """ from __future__ import annotations import numpy as np import pandas as pd -from sqlalchemy import func, select +from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session from ..features import FEATURES -from ..platform.models import Case, Decision, Transaction +from ..platform.models import Appeal, Case, Decision, Transaction LABELS = {"confirmed_fraud": 1, "legitimate": 0} +# An approved appeal whose payment has no case verdict to outrank it. +_APPEAL_ONLY = and_( + Appeal.status == "approved", + or_(Decision.case_id.is_(None), Case.verdict.is_(None), Case.verdict.not_in(LABELS)), +) + def labels(s: Session) -> pd.DataFrame: - """One row per reviewed alert, in `models.train.FEEDBACK_COLUMNS` plus case and verdict.""" - rows = s.execute( - select( - Decision.txn_id, Transaction.ts, Transaction.type, - Case.id, Case.verdict, Decision.features, - ) + """One row per reviewed alert, in `models.train.FEEDBACK_COLUMNS` plus case, verdict + and where the label came from (`case` or `appeal`).""" + columns = (Decision.txn_id, Transaction.ts, Transaction.type, Case.id, Decision.features) + reviewed = s.execute( + select(*columns, Case.verdict) .join(Transaction, Transaction.txn_id == Decision.txn_id) .join(Case, Case.id == Decision.case_id) .where(Case.verdict.in_(LABELS)) - .order_by(Decision.txn_id) - ).all() # fmt: skip + ).all() + appealed = s.execute( + select(*columns) + .join(Transaction, Transaction.txn_id == Decision.txn_id) + .join(Appeal, Appeal.txn_id == Decision.txn_id) + .outerjoin(Case, Case.id == Decision.case_id) + .where(_APPEAL_ONLY) + ).all() + rows = sorted( + [(*row, "case") for row in reviewed] + [(*row, "legitimate", "appeal") for row in appealed], + key=lambda row: row[0], + ) # A decision made by code with another feature list cannot be trained on. - rows = [row for row in rows if len(row.features) == len(FEATURES)] + rows = [row for row in rows if len(row[4]) == len(FEATURES)] frame = pd.DataFrame( { - "txn_id": np.array([row.txn_id for row in rows], dtype=np.int64), + "txn_id": np.array([row[0] for row in rows], dtype=np.int64), # Naive UTC, as in the dataset the training frame is read from. - "ts": pd.to_datetime([row.ts for row in rows], utc=True).tz_localize(None), - "type": [row.type for row in rows], - "case_id": [row.id for row in rows], - "verdict": [row.verdict for row in rows], + "ts": pd.to_datetime([row[1] for row in rows], utc=True).tz_localize(None), + "type": [row[2] for row in rows], + "case_id": pd.array([row[3] for row in rows], dtype="Int64"), + "verdict": [row[5] for row in rows], + "source": [row[6] for row in rows], } ) frame["y"] = frame["verdict"].map(LABELS).astype("int8") # A confirmed payment into a wallet is what the mule model learns from. frame["y_mule"] = (frame["y"] * (frame["type"] == "SEND_MONEY")).astype("int8") - features = np.array([row.features for row in rows], dtype=np.float64).reshape(-1, len(FEATURES)) + features = np.array([row[4] for row in rows], dtype=np.float64).reshape(-1, len(FEATURES)) return pd.concat([frame, pd.DataFrame(features, columns=list(FEATURES))], axis=1) @@ -71,11 +91,19 @@ def summary(s: Session) -> dict: .group_by(Case.verdict) ).all() ) + appeals = s.scalar( + select(func.count()) + .select_from(Appeal) + .join(Decision, Decision.txn_id == Appeal.txn_id) + .outerjoin(Case, Case.id == Decision.case_id) + .where(_APPEAL_ONLY) + ) return { "cases": {verdict: cases.get(verdict, 0) for verdict in (*LABELS, "inconclusive")}, "labels": { "fraud": alerts.get("confirmed_fraud", 0), - "legitimate": alerts.get("legitimate", 0), + "legitimate": alerts.get("legitimate", 0) + appeals, }, + "approved_appeals": appeals, "last_verdict_at": s.scalar(select(func.max(Case.closed_at))), } diff --git a/backend/src/fraudlens/platform/appeals.py b/backend/src/fraudlens/platform/appeals.py new file mode 100644 index 0000000..ab6cd4c --- /dev/null +++ b/backend/src/fraudlens/platform/appeals.py @@ -0,0 +1,130 @@ +"""Customer appeals: "this is a genuine payment", answered by a named person. + +A customer whose payment was warned, put through a step-up or held can say it +is genuine, with how they know the recipient and why. An analyst or supervisor +answers within a deadline: + +- Approving a held payment releases it, through the same path a `legitimate` + verdict uses. Money is never released into a wallet that is confirmed fraud + or frozen, whatever the appeal says. +- Rejecting keeps the hold; the case still decides. +- An appeal against a warning or a step-up moves no money: the customer + already decides those. It records that the warning was wrong, and an approved + one becomes a `legitimate` label (mlops/feedback.py). It never shortens a + cooling-off period: "let it through now, it is genuine" is exactly what a + scammer coaching a victim would have them say. + +What the customer typed is shown masked and is never read to decide anything. +""" + +from __future__ import annotations + +from datetime import timedelta + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +from .audit import Ctx, WorkflowError, audit +from .cases import _event +from .models import Appeal, Case, Decision, Transaction +from .scoring import Scorer + +APPEALABLE = ("warn", "step_up", "hold") +# A warning or step-up appeal is feedback: nothing is waiting on it, so a day. +FEEDBACK_SLA_MINUTES = 24 * 60 + + +def sla_minutes(scorer: Scorer, tier: str) -> int: + if tier == "hold": + return scorer.policy.tiers["hold"].review_sla_minutes or FEEDBACK_SLA_MINUTES + return FEEDBACK_SLA_MINUTES + + +def file_appeal( + scorer: Scorer, ctx: Ctx, wallet_id: str, txn_id: int, relation: str, reason: str +) -> Appeal: + with scorer.transaction() as s: + txn = s.scalar( + select(Transaction).where(Transaction.txn_id == txn_id).with_for_update(key_share=True) + ) + # The same answer for "no such transaction" and "not yours". + if txn is None or txn.sender_id != wallet_id: + raise WorkflowError(404, "txn_not_found", "no such transaction for this wallet") + decision = s.get(Decision, txn_id) + if decision is None or decision.tier not in APPEALABLE: + raise WorkflowError( + 409, "not_appealable", "only a warned or held payment can be appealed" + ) + if decision.tier == "hold" and txn.status != "held": + raise WorkflowError(409, "already_decided", f"the payment is already {txn.status}") + now = scorer.now() + appeal = Appeal( + txn_id=txn_id, + wallet_id=wallet_id, + case_id=decision.case_id, + tier=decision.tier, + relation=relation, + reason=reason, + filed_at=now, + sla_due_at=now + timedelta(minutes=sla_minutes(scorer, decision.tier)), + ) + s.add(appeal) + try: + s.flush() + except IntegrityError: + raise WorkflowError( + 409, "appeal_exists", "this payment has already been appealed" + ) from None + if appeal.case_id is not None: + _event( + s, appeal.case_id, None, "appeal_filed", + appeal_id=appeal.id, txn_id=txn_id, relation=relation, tier=decision.tier, + ) # fmt: skip + audit( + s, ctx, "appeal.file", "appeal", appeal.id, + txn_id=txn_id, wallet_id=wallet_id, tier=decision.tier, relation=relation, + ) # fmt: skip + return appeal + + +def decide_appeal(scorer: Scorer, ctx: Ctx, appeal_id: int, approve: bool, note: str) -> dict: + """The person's answer. Approving releases a payment that is still held.""" + with scorer.transaction() as s: + appeal = s.scalar(select(Appeal).where(Appeal.id == appeal_id).with_for_update()) + if appeal is None: + raise WorkflowError(404, "appeal_not_found", f"no appeal {appeal_id}") + if appeal.status != "pending": + raise WorkflowError(409, "already_decided", f"the appeal was {appeal.status}") + if appeal.case_id is not None and ctx.role != "supervisor": + case = s.get(Case, appeal.case_id) + if case is not None and case.status == "escalated": + raise WorkflowError(403, "needs_supervisor", "an escalated case needs a supervisor") + txn = s.scalar( + select(Transaction) + .where(Transaction.txn_id == appeal.txn_id) + .with_for_update(key_share=True) + ) + released = False + if approve and txn.status == "held": + if txn.receiver_id in scorer.engine.flagged: + raise WorkflowError( + 409, "recipient_confirmed_fraud", + "the receiving wallet is confirmed fraud: an appeal cannot release money to it", + ) # fmt: skip + scorer.complete(txn) # a frozen party turns this into `rejected` + released = txn.status == "completed" + now = scorer.now() + appeal.status = "approved" if approve else "rejected" + appeal.decided_by, appeal.decision_note, appeal.decided_at = ctx.user_id, note, now + if appeal.case_id is not None: + _event( + s, appeal.case_id, ctx, f"appeal_{appeal.status}", note, + appeal_id=appeal.id, txn_id=appeal.txn_id, released=released, + ) # fmt: skip + audit( + s, ctx, f"appeal.{'approve' if approve else 'reject'}", "appeal", appeal.id, + txn_id=appeal.txn_id, released=released, txn_status=txn.status, + within_sla=now <= appeal.sla_due_at, + ) # fmt: skip + s.flush() + return {"appeal": appeal, "txn_status": txn.status, "released": released} diff --git a/backend/src/fraudlens/platform/load.py b/backend/src/fraudlens/platform/load.py index 1471a47..92b2dfe 100644 --- a/backend/src/fraudlens/platform/load.py +++ b/backend/src/fraudlens/platform/load.py @@ -33,7 +33,7 @@ SYSTEM = Ctx(None, "system", None) # Everything the platform writes at run time. Never users, never the audit log. RUNTIME_TABLES = ( - "shadow_scores", "decisions", "case_events", "freeze_requests", "customer_reports", + "appeals", "shadow_scores", "decisions", "case_events", "freeze_requests", "customer_reports", "wallet_flags", "cases", "transactions", "past_cases", "wallets", "agents", ) # fmt: skip TXN_COLUMNS = ( diff --git a/backend/src/fraudlens/platform/models.py b/backend/src/fraudlens/platform/models.py index c69528a..c0308de 100644 --- a/backend/src/fraudlens/platform/models.py +++ b/backend/src/fraudlens/platform/models.py @@ -39,6 +39,18 @@ VERDICTS = ("confirmed_fraud", "legitimate", "inconclusive") CASE_SOURCE = ("alert", "customer_report", "manual") FREEZE_STATUS = ("pending", "approved", "rejected") +APPEAL_STATUS = ("pending", "approved", "rejected") +# How the customer says they know the person they are paying. +APPEAL_RELATIONS = ( + "family", + "friend", + "business", + "seller", + "landlord", + "employer", + "other", + "none", +) def _one_of(column: str, values: tuple[str, ...]) -> str: @@ -305,6 +317,38 @@ class CustomerReport(Base): created_at: Mapped[datetime] = mapped_column(Timestamp, server_default=func.now()) +class Appeal(Base): + """A customer says a warned or held payment is genuine. A person answers it.""" + + __tablename__ = "appeals" + __table_args__ = ( + CheckConstraint(_one_of("status", APPEAL_STATUS), name="status"), + CheckConstraint(_one_of("relation", APPEAL_RELATIONS), name="relation"), + CheckConstraint("tier IN ('warn', 'step_up', 'hold')", name="tier"), + CheckConstraint("(status = 'pending') = (decided_by IS NULL)", name="decided_has_decider"), + Index("ix_appeals_status_sla", "status", "sla_due_at"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + # One appeal per payment. + txn_id: Mapped[int] = mapped_column( + BigInteger, ForeignKey("transactions.txn_id", ondelete="CASCADE"), unique=True + ) + wallet_id: Mapped[str] = mapped_column(String(32), index=True) # the sender, who appeals + case_id: Mapped[int | None] = mapped_column(ForeignKey("cases.id")) + tier: Mapped[str] = mapped_column(String(8)) # the decision's tier when it was appealed + relation: Mapped[str] = mapped_column(String(16)) + # Free text typed by a customer: shown masked, never used to decide anything. + reason: Mapped[str] = mapped_column(Text) + status: Mapped[str] = mapped_column(String(12), server_default="pending") + filed_at: Mapped[datetime] = mapped_column(Timestamp) # domain time, like opened_at + sla_due_at: Mapped[datetime] = mapped_column(Timestamp) + decided_by: Mapped[int | None] = mapped_column(ForeignKey("users.id")) + decision_note: Mapped[str | None] = mapped_column(Text) + decided_at: Mapped[datetime | None] = mapped_column(Timestamp) + created_at: Mapped[datetime] = mapped_column(Timestamp, server_default=func.now()) + + class PastCase(Base): """Confirmed fraud cases from before the platform went live (the training period).""" diff --git a/backend/src/fraudlens/platform/scoring.py b/backend/src/fraudlens/platform/scoring.py index 10d6b3f..a613e7b 100644 --- a/backend/src/fraudlens/platform/scoring.py +++ b/backend/src/fraudlens/platform/scoring.py @@ -38,6 +38,7 @@ from ..config import Settings from ..decision import Decision as EngineDecision from ..decision import DecisionEngine, SimilarCases, context_from_engine, load_policy +from ..decision.customer import cue as customer_cue from ..decision.policy import RANK from ..features import SCORED_TYPES, FeatureEngine, Txn from ..mlops.shadow import Shadow @@ -468,6 +469,9 @@ def summary(self, row) -> dict: "review_sla_minutes": spec.review_sla_minutes, "case_id": get("case_id"), "latency_ms": round(get("latency_ms"), 2), + # Which scam pattern the app's warning should ask about; see decision/customer.py. + "scenario": get("scenario"), + "cue": customer_cue(get("tier"), get("scenario"), get("detail")), } def _recorded(self, s: Session, result: Result) -> None: diff --git a/backend/tests/test_platform.py b/backend/tests/test_platform.py index d6a96cb..38b8780 100644 --- a/backend/tests/test_platform.py +++ b/backend/tests/test_platform.py @@ -12,6 +12,7 @@ import itertools import json import shutil +from datetime import datetime import numpy as np import pytest @@ -828,8 +829,12 @@ def test_a_freeze_needs_two_people(client, tokens, p, replayed, spare): # --------------------------------------------------------------- customers -def _pending(client, tokens, p, tier: str) -> dict: - """A live payment the policy answers with `tier`, found by asking what-if first.""" +_scored: dict[int, dict] = {} # what `_pending` was answered, by txn_id + + +def _pending(client, tokens, p, tier: str, by_model: bool = False) -> dict: + """A live payment the policy answers with `tier`, found by asking what-if first. + `by_model` skips payments into confirmed-fraud wallets, which a rule holds.""" with p.sessions() as s: candidates = s.execute( select(Transaction) @@ -843,6 +848,8 @@ def _pending(client, tokens, p, tier: str) -> dict: ] service = tokens("upay-core") for sender, receiver, amount, device in candidates: + if by_model and receiver in p.scorer.engine.flagged: + continue for scale in (1.0, 0.5, 2.0, 0.25): body = send(p, sender, receiver, round(amount * scale, 2), device_id=device) probe = client.post("/v1/score/what-if", headers=service, json=body).json() @@ -850,6 +857,7 @@ def _pending(client, tokens, p, tier: str) -> dict: continue result = client.post("/v1/score", headers=service, json=body).json() assert result["decision"]["tier"] == tier # what-if and scoring agree + _scored[body["txn_id"]] = result return body pytest.fail(f"no live payment produced a {tier} decision") @@ -897,6 +905,144 @@ def test_a_step_up_needs_verification_and_the_cooling_off(client, tokens, p, rep assert txn.applied_at > txn.ts and decision.customer_response == "proceed" +# ----------------------------------------------------------------- appeals + + +def _at(iso: str) -> datetime: + return datetime.fromisoformat(iso.replace("Z", "+00:00")) + + +def _appeal(client, service, body: dict, **over) -> object: + answer = {"wallet_id": body["sender_id"], "relation": "family", "reason": "My brother's rent"} + return client.post( + f"/v1/customer/transactions/{body['txn_id']}/appeal", headers=service, json=answer | over + ) + + +def test_a_customer_appeal_releases_a_held_payment_once_a_person_agrees( + client, tokens, p, replayed +): + service, analyst = tokens("upay-core"), tokens("analyst1") + body = _pending(client, tokens, p, "hold", by_model=True) + txn_id, sender = body["txn_id"], body["sender_id"] + assert _scored[txn_id]["decision"]["cue"] != "reported_recipient" + record = client.get(f"/v1/decisions/{txn_id}", headers=analyst).json() + initiated = p.scorer.engine.wallets[sender].n_init + + assert _appeal(client, service, body, wallet_id="W0").status_code == 404 # not theirs + assert _appeal(client, analyst, body).status_code == 403 # only the app files one + assert _appeal(client, service, body, relation="cousin").status_code == 422 + filed = _appeal(client, service, body, reason="Rent for 01711000000, my brother") + assert filed.status_code == 201, filed.text + appeal = filed.json() + assert (appeal["status"], appeal["tier"], appeal["txn_status"]) == ("pending", "hold", "held") + assert "reason" not in appeal and "decided_by" not in appeal # the app sees where, not who + waited = _at(appeal["sla_due_at"]) - _at(appeal["filed_at"]) + assert waited.total_seconds() == 30 * 60 # the hold's own review deadline + again = _appeal(client, service, body) + assert again.status_code == 409 and again.json()["error"]["code"] == "appeal_exists" + status = f"/v1/customer/transactions/{txn_id}/appeal" + assert client.get(status, headers=service, params={"wallet_id": "W0"}).status_code == 404 + assert ( + client.get(status, headers=service, params={"wallet_id": sender}).json()["id"] + == (appeal["id"]) + ) + + assert client.get("/v1/appeals", headers=service).status_code == 403 + queue = client.get("/v1/appeals", headers=analyst).json() + [row] = [a for a in queue["appeals"] if a["id"] == appeal["id"]] + assert row["txn_status"] == "held" and row["relation"] == "family" + assert "01711000000" not in row["reason"] # what a customer typed is masked + assert queue["counts"]["pending"] >= 1 + + path = f"/v1/appeals/{appeal['id']}/approve" + assert client.post(path, headers=analyst, json={"note": "ok"}).status_code == 422 + done = client.post(path, headers=analyst, json={"note": REASON}) + assert done.status_code == 200, done.text + assert done.json()["released"] is True and done.json()["txn_status"] == "completed" + assert done.json()["appeal"]["status"] == "approved" + assert done.json()["appeal"]["decided_by"] is not None + assert p.scorer.engine.wallets[sender].n_init == initiated + 1 + twice = client.post(path, headers=analyst, json={"note": REASON}) + assert twice.status_code == 409 and twice.json()["error"]["code"] == "already_decided" + + with p.sessions() as s: + actions = set( + s.scalars( + select(AuditLog.action).where( + AuditLog.object_type == "appeal", AuditLog.object_id == str(appeal["id"]) + ) + ) + ) + labels = feedback.labels(s).set_index("txn_id") + assert actions == {"appeal.file", "appeal.approve"} + # A person agreed the payment was genuine: it becomes a `legitimate` label. + assert (labels.loc[txn_id, "y"], labels.loc[txn_id, "source"]) == (0, "appeal") + if record["case_id"] is not None: + timeline = client.get(f"/v1/cases/{record['case_id']}", headers=analyst).json() + kinds = [e["kind"] for e in timeline["timeline"]] + assert "appeal_filed" in kinds and "appeal_approved" in kinds + + +def test_an_appeal_never_releases_money_to_confirmed_fraud(client, tokens, p, replayed, spare): + service, analyst, supervisor = tokens("upay-core"), tokens("analyst1"), tokens("supervisor1") + body, result = held_payment(client, tokens, p, spare) + assert result["decision"]["cue"] == "reported_recipient" + appeal = _appeal(client, service, body, relation="friend").json() + case_id = result["decision"]["case_id"] + client.post(f"/v1/cases/{case_id}/escalate", headers=analyst, json={"reason": REASON}) + path = f"/v1/appeals/{appeal['id']}" + refused = client.post(f"{path}/reject", headers=analyst, json={"note": REASON}) + assert refused.status_code == 403 and refused.json()["error"]["code"] == "needs_supervisor" + blocked = client.post(f"{path}/approve", headers=supervisor, json={"note": REASON}) + assert blocked.status_code == 409 + assert blocked.json()["error"]["code"] == "recipient_confirmed_fraud" + rejected = client.post(f"{path}/reject", headers=supervisor, json={"note": REASON}) + assert rejected.status_code == 200, rejected.text + assert (rejected.json()["released"], rejected.json()["txn_status"]) == (False, "held") + with p.sessions() as s: + assert s.get(Transaction, body["txn_id"]).status == "held" # the case still decides + assert body["txn_id"] not in set(feedback.labels(s)["txn_id"]) # no label from a rejection + + +def test_an_appeal_against_a_warning_is_feedback_and_moves_no_money(client, tokens, p, replayed): + staff = tokens("analyst2") + body = _pending(client, tokens, p, "warn") + txn_id = body["txn_id"] + # The warning names the scam it looks like, for the app to explain in words. + assert _scored[txn_id]["decision"]["cue"] in { + "impersonation", "prize", "investment", "wrong_send", "not_you", "generic", + "reported_recipient", + } # fmt: skip + # The demo phone files it as the app would. + filed = client.post( + "/v1/demo/appeal", + headers=staff, + json={"txn_id": txn_id, "relation": "seller", "reason": "I buy from this shop every week"}, + ) + assert filed.status_code == 201, filed.text + appeal = filed.json() + assert appeal["tier"] == "warn" and appeal["txn_status"] == "pending_customer" + waited = _at(appeal["sla_due_at"]) - _at(appeal["filed_at"]) + assert waited.total_seconds() == 24 * 3600 # nothing waits on it + shown = client.get("/v1/demo/appeal", headers=staff, params={"txn_id": txn_id}).json() + assert shown["id"] == appeal["id"] + done = client.post( + f"/v1/appeals/{appeal['id']}/approve", headers=tokens("analyst1"), json={"note": REASON} + ) + assert done.status_code == 200 + # The customer still decides a warning; an appeal never answers for them. + assert (done.json()["released"], done.json()["txn_status"]) == (False, "pending_customer") + + allowed = _pending(client, tokens, p, "allow") + nothing = client.post( + "/v1/demo/appeal", + headers=staff, + json={"txn_id": allowed["txn_id"], "relation": "none", "reason": "just checking"}, + ) + assert nothing.status_code == 409 and nothing.json()["error"]["code"] == "not_appealable" + + def test_a_customer_report_opens_a_case(client, tokens, p, replayed, spare): service = tokens("upay-core") reporter, reported = spare(), spare() diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index 78fc2bf..51604b4 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -79,12 +79,14 @@ served at `/docs` outside production. | Network | `GET /wallets/{id}`, `GET /wallets/{id}/network`, `GET /rings`, `GET /rings/{ring_id}`, `GET /agents/risk`, `GET /agents/{id}`, `GET /past-cases/{id}` | analyst, supervisor | | Customer | `POST /customer/recipient-check`, `POST /customer/transactions/{txn_id}/respond`, `POST /customer/reports` | service | | | `POST /customer/message-check`, `POST /customer/payment-verify` ([FRAUD_TAXONOMY.md](FRAUD_TAXONOMY.md)) | service | +| | `POST\|GET /customer/transactions/{txn_id}/appeal` (§6) | service | +| Appeals | `GET /appeals`, `POST /appeals/{id}/approve\|reject` (§6) | analyst, supervisor | | Fraud types | `GET /intel/taxonomy` | analyst, supervisor, admin | | Operations | `GET /metrics/summary`, `GET /metrics/daily`, `GET /model`, `GET /model/report`, `GET /policy` | analyst, supervisor, admin | | After deployment (§12) | `GET /models`, `GET /model/shadow`, `GET /metrics/drift`, `GET /feedback` | analyst, supervisor, admin | | Audit | `GET /audit` | supervisor, admin | | | `POST /audit/reveals` (§7) | analyst, supervisor | -| Demo (§14), absent in production | `GET /demo/scenarios`, `GET /demo/payment-draft`, `GET /demo/payment-claims`, `POST /demo/pay\|respond\|report\|recipient-check\|message-check\|payment-verify\|advance-clock` | analyst, supervisor, admin | +| Demo (§14), absent in production | `GET /demo/scenarios`, `GET /demo/payment-draft`, `GET /demo/payment-claims`, `GET /demo/appeal`, `POST /demo/pay\|respond\|report\|appeal\|recipient-check\|message-check\|payment-verify\|advance-clock` | analyst, supervisor, admin | | | `GET /health`, `GET /ready` | none | Errors always have the same shape, with the request id that is also in the @@ -172,6 +174,25 @@ and each request still needs its own approval by a second person. from the SMS or screenshot shown, and only for payments made to the asking wallet. 10 a minute per wallet. Both are described in [FRAUD_TAXONOMY.md](FRAUD_TAXONOMY.md). +- **Which scam it looks like.** `/v1/score` and the demo payment return + `decision.scenario` and `decision.cue` (`reported_recipient`, `impersonation`, + `prize`, `investment`, `wrong_send`, `not_you` or `generic`), so the app can + word its warning for that scam (`decision/customer.py`). The phone demo does, + in Bangla and English, with a live cooling-off or review countdown and a "what + happens next" timeline. +- **Appealing.** `POST /customer/transactions/{txn_id}/appeal` (the sender only, + on a warned, step-up or held payment; one per payment; 5 an hour) with a + relation to the recipient and a reason (digits redacted). A person answers + every appeal: `GET /appeals`, `POST /appeals/{id}/approve|reject` (analyst, + supervisor; same escalation rules as the case). Deadlines: the hold's review + SLA (30 minutes) for a held payment, 24 hours for a warning. Approving a held + payment releases it, except into a confirmed-fraud or frozen wallet (`409`). + Approving a warning moves no money and grants no cooling-off waiver, so an + appeal cannot be used to coach a victim past the check. An approved appeal is a + `legitimate` label in `/feedback` (`source: appeal`) unless a case verdict says + otherwise; a rejected one adds nothing. The customer's view + (`GET /customer/transactions/{txn_id}/appeal`) shows status and deadline only. + Everything is audited. Migration `0004_appeals`. ## 7. Security diff --git a/docs/plans/08-customer-experience.md b/docs/plans/08-customer-experience.md new file mode 100644 index 0000000..39f2d05 --- /dev/null +++ b/docs/plans/08-customer-experience.md @@ -0,0 +1,128 @@ +# 08 · Customer experience and customer appeals + +## Goal + +Make the sender's side of FraudLens the best part of the demo, and give the +customer a way to say "this is a genuine payment" that a person answers. + +1. The phone mock (`frontend/app/(console)/phone/page.tsx`) shows every + intervention tier (allow, warn, step-up, hold) in Bangla or English, with + natural Bangla copy, a warning that names the scam pattern the decision + points to, a visible cooling-off countdown, a "what happens next" timeline, + and proper accessibility at phone width. +2. A customer can appeal a hold or a warning from the phone. An analyst or + supervisor answers it from an appeals queue in the console: approving + releases a held payment, rejecting keeps it held for the case. Every step is + audited, every appeal has a deadline, and an approved appeal becomes a + training label through the existing feedback path. + +## Judge criteria it moves + +- **Business / customer impact (13.67/20, "low customer interruption").** The + cost of APP-fraud controls is borne by honest customers who get interrupted. + This work makes each interruption shorter and clearer (the customer is told + *which* scam it looks like, how long the wait is, and what happens next) and + gives a wrongly-stopped customer recourse with a stated deadline, instead of + waiting on a helpline. Approved appeals feed back as "legitimate" labels, so + the false positives that interrupt customers are what the next model learns from. +- **Prototype quality (11.67/15).** An end-to-end customer journey in both + languages, a new reviewed workflow that reuses the case/approval/audit + patterns, a migration, tests and a browser test. +- **Responsible AI.** Contestability: an automated hold can be challenged and is + decided by a named person within an SLA; the decision and the reasoning are in + the audit log. + +## Design + +### Customer copy keyed by scam pattern + +`POST /v1/demo/pay` and `/v1/score` already return a decision summary. It gains +two additive fields: `scenario` (scam / takeover / unusual_access / cash_out) +and `cue`, one customer-safe pattern name derived from what the decision +already recorded (the rule that fired, the scenario, and the typology of the +most similar past cases): `reported_recipient`, `impersonation`, `prize`, +`investment`, `wrong_send`, `not_you`, or `generic`. No score, no reason +detail and no identifier is added. The phone turns the cue into one plain +question per language ("Did someone claiming to be from upay, bKash or your +bank ask you to send this?"). The policy's own fixed `customer_message` is +still shown unchanged. + +### Appeals + +- Table `appeals` (migration `0004_customer_appeals`, additive): one per + transaction; wallet, case, tier at the time, relation to the recipient, the + customer's reason, status (`pending` / `approved` / `rejected`), `filed_at`, + `sla_due_at`, decider, note, decided_at. +- Who can appeal: the sender of a payment whose decision was warn, step-up or + hold. A hold can be appealed only while it is still held. +- SLA: a hold's appeal is due within the hold review time from the policy + (30 minutes); a warning or step-up appeal (feedback only, the money is the + customer's to send) within 24 hours. +- Decide: analyst or supervisor. An escalated case needs a supervisor (same + rule as verdicts). Approve releases a still-held payment through the same + `Scorer.complete` the verdict uses; it refuses to release money into a wallet + that is confirmed fraud or frozen. Reject keeps the hold; the case still + decides. Approving a warning appeal changes no money: it records that the + warning was wrong. Neither shortens a step-up's cooling-off (that would be + exactly what a scammer coaching a victim would ask for). +- Audit: `appeal.file`, `appeal.approve`, `appeal.reject`; case timeline events + `appeal_filed` / `appeal_approved` / `appeal_rejected`. +- Feedback: an approved appeal labels its decision `legitimate` (y = 0) in + `mlops.feedback.labels`, unless the case it belongs to has its own verdict, + which wins. A rejected appeal is not a label: "keep it held" is not "fraud". +- Endpoints: `POST /v1/customer/transactions/{txn_id}/appeal` and + `GET .../appeal` (service); `POST /v1/demo/appeal`, `GET /v1/demo/appeal` + (staff playing the phone); `GET /v1/appeals`, `POST /v1/appeals/{id}/approve|reject` + (analyst, supervisor). + +### Console + +New page `/appeals` (nav: Investigations), modelled on Freeze approvals: status +chips, a table with SLA countdown/overdue, the customer's relation and reason +(masked), the tier, the case link, and Approve / Reject through the existing +`ReasonDialog`. + +### Phone + +- Language toggle (Bangla default, English), stored per viewer; every phone + string from one copy table; `lang` attributes set. +- Warn: pattern question, policy message, "Cancel" as the primary action, + "Send anyway" (dismissible: the policy lets the customer decide), "This is a + genuine payment" (appeal), report. +- Step-up: live mm:ss cooling-off countdown (`role=timer`), progress bar, PIN, + "Verify and send" enabled once the wait is over; cancel; appeal. +- Hold: "Your money has not left your wallet", review deadline countdown, + timeline (paused → reviewer → decision), appeal and report. +- Appeal screen: relation (radio group), reason (textarea with counter), + submit; then appeal status with its deadline and a refresh. +- Accessibility: focus moves to each new screen's heading, `aria-live` status, + labelled controls, 44px touch targets, contrast checked, phone frame fits a + 360px viewport without horizontal scroll. + +## Steps + +1. Plan (this file). +2. Backend: cue in the decision summary; `Appeal` model + migration 0004; + `platform/appeals.py` workflow; routes (customer, demo, reviewer queue); + feedback labels; tests in `test_platform.py`. +3. Console: types, `/appeals` page, nav entry. +4. Phone rewrite: language toggle, per-tier screens, countdowns, timeline, + appeal flow, accessibility. +5. Playwright: `frontend/scripts/phone-e2e.cjs` (`make e2e-phone`): pay a warn + and a hold in both languages, file an appeal, approve it in `/appeals`, see + the payment released on the phone. Add `/appeals` to the smoke pages. +6. Docs: PLATFORM.md customer section and endpoint table. +7. `uv run pytest -q`, `make lint`, e2e against a private stack + (`docker compose -p fl-cx`, ports 55434/56381, API 8028, console 3208). + +## Definition of done + +- Every phone screen reads correctly in both languages; no mixed-language + strings except brand names. +- Warn and step-up stay dismissible exactly as the policy says; hold is not. +- Appeal round trip works: file from the phone → appears in `/appeals` with an + SLA → approve releases the held payment → phone shows it sent; reject keeps + it held. All audited. +- Approved appeals appear in `/v1/feedback` and `feedback.labels()`. +- Migration matches the models (`test_the_schema_matches_the_models`). +- `pytest`, `make lint` pass; the e2e passes locally if the stack runs. diff --git a/frontend/app/(console)/appeals/page.tsx b/frontend/app/(console)/appeals/page.tsx new file mode 100644 index 0000000..c6cc30a --- /dev/null +++ b/frontend/app/(console)/appeals/page.tsx @@ -0,0 +1,140 @@ +"use client"; + +import Link from "next/link"; +import { useState } from "react"; + +import { Id, useSession } from "@/components/session"; +import { Async, Badge, Button, Card, Chip, Empty, PageHeader, ReasonDialog, StatusBadge, Table, Td, TierBadge } from "@/components/ui"; +import { api, useApi } from "@/lib/api"; +import { taka, when, words } from "@/lib/format"; +import type { Appeal, AppealQueue, AppealStatus } from "@/lib/types"; + +const STATUSES: AppealStatus[] = ["pending", "approved", "rejected"]; +const OUTCOME_TONE = { pending: "amber", approved: "green", rejected: "slate" } as const; + +/** Time left until the deadline, or how late it is: "12 min left", "1 h 5 min late". */ +function due(appeal: Appeal, now: string): { text: string; late: boolean } { + const minutes = Math.round((Date.parse(appeal.sla_due_at) - Date.parse(now)) / 60_000); + const span = Math.abs(minutes); + const text = span >= 60 ? `${Math.floor(span / 60)} h ${span % 60} min` : `${span} min`; + return minutes >= 0 ? { text: `${text} left`, late: false } : { text: `${text} late`, late: true }; +} + +interface Pending { + appeal: Appeal; + approve: boolean; +} + +export default function AppealsPage() { + const { canReview } = useSession(); + const [status, setStatus] = useState("pending"); + const queue = useApi(`/v1/appeals?status=${status}&limit=200`); + const [dialog, setDialog] = useState(null); + const [outcome, setOutcome] = useState(null); + + async function decide({ appeal, approve }: Pending, note: string) { + const done = await api<{ released: boolean; txn_status: string }>( + `/v1/appeals/${appeal.id}/${approve ? "approve" : "reject"}`, + { note }, + ); + setOutcome( + `Appeal #${appeal.id} ${approve ? "approved" : "rejected"}. ` + + (done.released ? "The payment was released to the recipient." : `The payment is ${words(done.txn_status)}.`), + ); + queue.reload(); + } + + return ( + <> + + {outcome &&
{outcome}
} + + +
+ {STATUSES.map((value) => ( + setStatus(value)}> + {words(value)} + {queue.data ? ` (${queue.data.counts[value]})` : ""} + + ))} +
+ + {({ appeals, now }) => + appeals.length ? ( + + {appeals.map((appeal) => { + const deadline = due(appeal, now); + return ( + + + + + + + + + + ); + })} +
+ #{appeal.id} +
{when(appeal.filed_at)}
+
+ {taka(appeal.amount)} +
+ → {appeal.receiver_id ? : "–"} +
+ {appeal.txn_status &&
} +
+ {words(appeal.relation)} +
{appeal.reason}
+
+ {appeal.status === "pending" ? ( + {deadline.text} + ) : ( + {when(appeal.sla_due_at)} + )} + + {appeal.case_id ? #{appeal.case_id} : –} + + {appeal.status !== "pending" ? ( + + {words(appeal.status)} {appeal.decider}: {appeal.decision_note} + + ) : canReview ? ( + + + + + ) : null} +
+ ) : ( + No {status} appeals. + ) + } +
+
+ + {dialog && ( + setDialog(null)} + onSubmit={(note) => decide(dialog, note)} + /> + )} + + ); +} diff --git a/frontend/app/(console)/phone/copy.ts b/frontend/app/(console)/phone/copy.ts new file mode 100644 index 0000000..8b24688 --- /dev/null +++ b/frontend/app/(console)/phone/copy.ts @@ -0,0 +1,196 @@ +// What the demo phone says, in the customer's language. The Bangla is written for +// the screen, the way upay's app speaks to its customers, not translated word by word. + +import type { AppealRelation, Cue, Tier } from "@/lib/types"; + +export type Lang = "bn" | "en"; +export type Words = Record; + +const BN_DIGITS = "০১২৩৪৫৬৭৮৯"; + +/** Bangla numerals for a Bangla screen; amounts, times and counts all read naturally. */ +export function digits(text: string | number, lang: Lang): string { + const value = String(text); + return lang === "bn" ? value.replace(/[0-9]/g, (d) => BN_DIGITS[Number(d)]) : value; +} + +/** mm:ss, or h:mm:ss past an hour. */ +export function clockText(seconds: number, lang: Lang): string { + const s = Math.max(0, Math.ceil(seconds)); + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + const pad = (n: number) => String(n).padStart(2, "0"); + return digits(h ? `${h}:${pad(m)}:${pad(s % 60)}` : `${pad(m)}:${pad(s % 60)}`, lang); +} + +/** The spoken form for screen readers: "12 minutes 5 seconds". */ +export function clockSpoken(seconds: number, lang: Lang): string { + const s = Math.max(0, Math.ceil(seconds)); + const m = Math.floor(s / 60); + return lang === "bn" ? `${digits(m, lang)} মিনিট ${digits(s % 60, lang)} সেকেন্ড` : `${m} minutes ${s % 60} seconds`; +} + +/** The warning itself, by the kind of scam the decision looks like (the API's `cue`). */ +export const CUE: Record = { + reported_recipient: { + title: { bn: "এই নম্বরের বিরুদ্ধে প্রতারণার অভিযোগ আছে", en: "This number has been reported for fraud" }, + body: { + bn: "অন্য গ্রাহকেরা এই নম্বরে টাকা পাঠিয়ে প্রতারিত হয়েছেন। যাকে চেনেন না, বা কেউ তাড়া দিচ্ছে বলে পাঠাচ্ছেন — এমন হলে টাকা পাঠাবেন না।", + en: "Other customers lost money sending to this number. Don't send if you don't know them, or if someone is rushing you.", + }, + check: { bn: "প্রাপককে নিজে ফোন করে নিশ্চিত হোন।", en: "Call the person yourself to make sure it's them." }, + }, + impersonation: { + title: { bn: "কেউ কি upay, ব্যাংক বা সরকারি অফিসের লোক পরিচয় দিয়েছে?", en: "Did someone say they're from upay, a bank or the government?" }, + body: { + bn: "upay কখনো ফোন করে পিন, ওটিপি বা টাকা চায় না। কেউ চাইলে ফোন কেটে দিন।", + en: "upay never calls to ask for your PIN, OTP or money. If someone does, hang up.", + }, + check: { bn: "সন্দেহ হলে নিজে ১৬২৬৮ নম্বরে কল করুন।", en: "If in doubt, call 16268 yourself." }, + }, + prize: { + title: { bn: "পুরস্কার পেতে টাকা লাগে না", en: "Real prizes don't cost money" }, + body: { + bn: "লটারি, পুরস্কার বা উপহার পাওয়ার আগে ফি, ট্যাক্স বা চার্জ পাঠাতে বললে সেটা প্রতারণা।", + en: "If you're asked to pay a fee, tax or charge before you get a prize or gift, it's a scam.", + }, + check: { bn: "আপনি কি সত্যিই কোনো প্রতিযোগিতায় অংশ নিয়েছিলেন?", en: "Did you actually enter any contest?" }, + }, + investment: { + title: { bn: "নিশ্চিত লাভের প্রতিশ্রুতি? সাবধান", en: "Guaranteed profit? Be careful" }, + body: { + bn: "অল্প দিনে টাকা দ্বিগুণ বা নিশ্চিত মুনাফার কথা বলে টাকা নেওয়া একটি পরিচিত প্রতারণা।", + en: "Promises to double your money quickly, or of guaranteed profit, are a well-known scam.", + }, + check: { bn: "পাঠানোর আগে পরিবারের কারো সাথে কথা বলুন।", en: "Talk to someone in your family before you send." }, + }, + wrong_send: { + title: { bn: "কেউ কি 'ভুল করে পাঠিয়েছি' বলে টাকা ফেরত চাইছে?", en: "Is someone asking for money back they 'sent by mistake'?" }, + body: { + bn: "আগে নিজের ব্যালেন্স আর লেনদেনের তালিকা দেখুন। টাকা সত্যিই এসেছে কি না না দেখে কিছু ফেরত পাঠাবেন না।", + en: "Check your balance and your transaction list first. Don't send anything back until you've seen the money arrive.", + }, + check: { bn: "এসএমএস নয়, অ্যাপের লেনদেন তালিকা দেখুন — এসএমএস নকল হতে পারে।", en: "Check the app's history, not an SMS: an SMS can be faked." }, + }, + not_you: { + title: { bn: "লেনদেনটি কি আপনি নিজেই করছেন?", en: "Is this really you?" }, + body: { + bn: "এটি আপনার স্বাভাবিক লেনদেনের সাথে মিলছে না — নতুন ফোন, নতুন জায়গা, বা অস্বাভাবিক অঙ্ক।", + en: "This doesn't look like how you usually pay: a new phone, a new place or an unusual amount.", + }, + check: { bn: "আপনি না করে থাকলে বাতিল করে ১৬২৬৮ নম্বরে কল করুন।", en: "If it isn't you, cancel and call 16268." }, + }, + generic: { + title: { bn: "পাঠানোর আগে একটু ভাবুন", en: "Take a moment before you send" }, + body: { + bn: "এই লেনদেনটি পরিচিত প্রতারণার ধরনের সাথে মিলে যাচ্ছে।", + en: "This payment looks like a known scam pattern.", + }, + check: { bn: "প্রাপককে চেনেন তো? কেউ তাড়া দিচ্ছে না তো?", en: "Do you know the person? Is anyone rushing you?" }, + }, +}; + +/** The strip under the bar: what kind of interruption this is. */ +export const STRIP_TEXT: Record, Words> = { + warn: { bn: "একটু থামুন", en: "Pause a moment" }, + step_up: { bn: "আপনাকে আবার যাচাই করতে হবে", en: "Verify it's you" }, + hold: { bn: "লেনদেন সাময়িকভাবে স্থগিত", en: "Payment paused for review" }, +}; + +export const RELATIONS: [AppealRelation, Words][] = [ + ["family", { bn: "পরিবারের সদস্য", en: "Family" }], + ["friend", { bn: "বন্ধু", en: "Friend" }], + ["seller", { bn: "দোকান বা বিক্রেতা", en: "A shop or seller" }], + ["business", { bn: "ব্যবসার কাজে", en: "Business" }], + ["landlord", { bn: "বাড়িওয়ালা", en: "Landlord" }], + ["employer", { bn: "অফিস বা মালিক", en: "Employer" }], + ["other", { bn: "অন্য কেউ", en: "Someone else" }], + ["none", { bn: "চিনি না", en: "I don't know them" }], +]; + +/** Everything else on the phone, keyed by what it is for. */ +export const T = { + appName: { bn: "সেন্ড মানি", en: "Send money" }, + switchTo: { bn: "English", en: "বাংলা" }, + switchLabel: { bn: "Switch the app to English", en: "অ্যাপটি বাংলায় দেখুন" }, + back: { bn: "পেছনে যান", en: "Back" }, + to: { bn: "প্রাপক", en: "To" }, + amount: { bn: "পরিমাণ", en: "Amount" }, + balance: { bn: "বর্তমান ব্যালেন্স", en: "Balance" }, + from: { bn: "যে অ্যাকাউন্ট থেকে", en: "From" }, + chooseLeft: { bn: "শুরু করতে বাঁ দিক থেকে একটি লেনদেন বেছে নিন।", en: "Choose a payment on the left to begin." }, + helpline: { bn: "হেল্পলাইন ১৬২৬৮", en: "Helpline 16268" }, + beforeYouSend: { bn: "পাঠানোর আগে দেখুন", en: "Before you send" }, + send: { bn: "সেন্ড মানি করুন", en: "Send" }, + sending: { bn: "পাঠানো হচ্ছে…", en: "Sending…" }, + cancel: { bn: "বাতিল করুন", en: "Cancel the payment" }, + sendAnyway: { bn: "বুঝেছি, তবুও পাঠাব", en: "I understand, send anyway" }, + pin: { bn: "পিন দিন", en: "Enter your PIN" }, + verifySend: { bn: "যাচাই করে পাঠান", en: "Verify and send" }, + availableIn: { bn: "পাঠানো যাবে আর", en: "You can send in" }, + coolingOff: { + bn: "নিরাপত্তার জন্য এই অপেক্ষা। এই সময়ে ভেবে দেখুন কে আপনাকে টাকা পাঠাতে বলেছে। চাইলে এখনই বাতিল করতে পারেন।", + en: "This wait is for your safety. Use it to think about who asked you to pay. You can cancel at any time.", + }, + waitOver: { bn: "অপেক্ষার সময় শেষ। পিন দিয়ে পাঠাতে পারেন।", en: "The wait is over. Enter your PIN to send." }, + heldMoney: { bn: "আপনার টাকা আপনার অ্যাকাউন্টেই আছে।", en: "Your money is still in your account." }, + reviewIn: { bn: "একজন কর্মকর্তা দেখবেন, বাকি সময়", en: "A person will review it within" }, + reviewLate: { + bn: "পর্যালোচনায় নির্ধারিত সময়ের চেয়ে বেশি লাগছে। আমরা দুঃখিত — দরকার হলে ১৬২৬৮ নম্বরে কল করুন।", + en: "The review is taking longer than promised. We're sorry; call 16268 if you need to.", + }, + whatNext: { bn: "এরপর কী হবে", en: "What happens next" }, + report: { bn: "প্রতারণার অভিযোগ করুন", en: "Report a scam" }, + whatHappened: { bn: "কী হয়েছিল?", en: "What happened?" }, + reportDone: { bn: "অভিযোগ গ্রহণ করা হয়েছে।", en: "Report received." }, + appeal: { bn: "এটা আসল লেনদেন — আপিল করুন", en: "This payment is genuine: appeal" }, + appealTitle: { bn: "আপিল করুন", en: "Appeal" }, + appealIntro: { + bn: "আপনার মনে হলে লেনদেনটি সঠিক, আমাদের জানান। একজন কর্মকর্তা দেখে সিদ্ধান্ত নেবেন।", + en: "If you think this payment is fine, tell us. A person will look at it and decide.", + }, + relation: { bn: "প্রাপক আপনার কে হন?", en: "How do you know the person?" }, + reason: { bn: "কেন পাঠাচ্ছেন? (সংক্ষেপে)", en: "What is the payment for? (briefly)" }, + reasonHint: { bn: "পিন বা ওটিপি এখানে লিখবেন না।", en: "Never write your PIN or OTP here." }, + submitAppeal: { bn: "আপিল জমা দিন", en: "Send the appeal" }, + appealSent: { bn: "আপিল জমা হয়েছে", en: "Appeal received" }, + appealBy: { bn: "উত্তর পাবেন যত দেরিতে হলেও", en: "You'll get an answer by" }, + appealNoMoney: { + bn: "এই আপিলে অপেক্ষার সময় কমবে না; এটি আমাদের সতর্কবার্তা আরও ভালো করতে সাহায্য করবে।", + en: "An appeal doesn't shorten the wait; it helps us make our warnings better.", + }, + appealApproved: { bn: "আপিল গ্রহণ করা হয়েছে", en: "Your appeal was accepted" }, + appealRejected: { bn: "আপিল গ্রহণ করা হয়নি — কর্মকর্তা তদন্ত চালিয়ে যাচ্ছেন", en: "Your appeal wasn't accepted; the review continues" }, + checkStatus: { bn: "অবস্থা দেখুন", en: "Check the status" }, + sent: { bn: "টাকা পাঠানো হয়েছে", en: "Sent" }, + cancelled: { bn: "বাতিল হয়েছে", en: "Cancelled" }, + notSent: { bn: "লেনদেন হয়নি", en: "Not sent" }, + sentBody: { bn: "লেনদেন সফল হয়েছে।", en: "The payment went through." }, + cancelledBody: { bn: "কিছুই পাঠানো হয়নি। টাকা আপনার অ্যাকাউন্টেই আছে।", en: "Nothing was sent. Your money is still in your account." }, + refusedBody: { bn: "এই লেনদেনটি করা যায়নি। টাকা আপনার অ্যাকাউন্টেই আছে।", en: "This payment was refused. Your money is still in your account." }, + newPayment: { bn: "নতুন লেনদেন", en: "New payment" }, + minutes: { bn: "মিনিট", en: "minutes" }, +} satisfies Record; + +/** The steps after each kind of interruption: done, happening now, still to come. */ +export function steps(tier: Exclude, minutes: number | null, lang: Lang): { text: string; state: "done" | "now" | "next" }[] { + const n = digits(minutes ?? 30, lang); + const words = { + warn: [ + { bn: "আমরা আপনাকে সতর্ক করেছি", en: "We've warned you" }, + { bn: "আপনি ঠিক করুন: বাতিল, নাকি তবুও পাঠাবেন", en: "You decide: cancel, or send anyway" }, + { bn: "পাঠালে টাকা সাথে সাথে চলে যাবে, আর ফেরত আনা কঠিন", en: "If you send, the money goes at once and is hard to get back" }, + ], + step_up: [ + { bn: "আমরা লেনদেনটি থামিয়েছি", en: "We've paused the payment" }, + { bn: `${n} মিনিট অপেক্ষা — এর মধ্যে যেকোনো সময় বাতিল করা যায়`, en: `A ${n}-minute wait; you can cancel at any time` }, + { bn: "এরপর পিন দিয়ে পাঠাতে পারবেন", en: "Then you can send it with your PIN" }, + ], + hold: [ + { bn: "টাকা আপনার অ্যাকাউন্টেই আছে, কোথাও যায়নি", en: "Your money hasn't left your account" }, + { bn: `একজন কর্মকর্তা ${n} মিনিটের মধ্যে দেখবেন`, en: `A person reviews it within ${n} minutes` }, + { bn: "ঠিক থাকলে টাকা পৌঁছে যাবে; না হলে আপনার কাছেই থাকবে", en: "If it's fine, it's delivered; if not, it stays with you" }, + ], + }[tier]; + return words.map((step, index) => ({ text: step[lang], state: index === 0 ? "done" : index === 1 ? "now" : "next" })); +} diff --git a/frontend/app/(console)/phone/page.tsx b/frontend/app/(console)/phone/page.tsx index b35f1f6..d6b86b4 100644 --- a/frontend/app/(console)/phone/page.tsx +++ b/frontend/app/(console)/phone/page.tsx @@ -2,15 +2,16 @@ import Image from "next/image"; import Link from "next/link"; -import { type ReactNode, useEffect, useState } from "react"; +import { type ReactNode, useEffect, useId, useRef, useState } from "react"; import { ScoreBar } from "@/components/charts"; import { useSession } from "@/components/session"; import { Async, Badge, Button, Card, cx, ErrorNote, Facts, inputClass, PageHeader, StatusBadge, TierBadge } from "@/components/ui"; import { api, ApiError, qs, useApi } from "@/lib/api"; -import { maskId, num, taka, when, words } from "@/lib/format"; -import type { PayResult, Scenario, Text2, Tier } from "@/lib/types"; +import { maskId, taka, when, words } from "@/lib/format"; +import type { AppealRelation, CustomerAppeal, PayResult, Scenario, Text2, Tier } from "@/lib/types"; +import { clockSpoken, clockText, CUE, digits, type Lang, RELATIONS, steps, STRIP_TEXT, T, type Words } from "./copy"; // upay's app icon. The name and the mark belong to UCB Fintech Company Limited. import upayLogo from "./upay-logo.png"; @@ -42,9 +43,9 @@ const NETWORKS = [ const SCENARIO: Record = { allow: { title: "An ordinary payment", text: "Goes straight through; the customer never sees FraudLens." }, - warn: { title: "Looks like a scam", text: "The customer gets a Bangla warning and chooses." }, + warn: { title: "Looks like a scam", text: "The customer gets a warning naming the scam, and chooses." }, step_up: { title: "Riskier", text: "Verify again, then wait out a cooling-off period." }, - hold: { title: "Very likely fraud", text: "Paused until a reviewer decides." }, + hold: { title: "Very likely fraud", text: "Paused until a reviewer decides; the customer can appeal." }, confirmed_fraud: { title: "To a confirmed-fraud wallet", text: "Held by a hard rule, whatever the model says." }, }; const CATEGORIES: [string, string, string][] = [ @@ -60,27 +61,39 @@ const CATEGORIES: [string, string, string][] = [ ["other", "অন্য কিছু", "Something else"], ]; const WALLET_ID = /^[A-Za-z0-9_-]{1,32}$/; +const LANG_KEY = "fraudlens.phone.lang"; +const REASON_MAX = 500; // The phone is drawn the way upay's app looks: its yellow bar, blue buttons and white pages, // whatever the console around it looks like. What FraudLens adds is the strip under the bar. +// Every pairing here passes WCAG AA contrast for its text size. const STRIP: Record = { allow: "border-green-300 bg-green-50 text-green-900", - warn: "border-amber-300 bg-amber-50 text-amber-900", - step_up: "border-orange-300 bg-orange-50 text-orange-900", + warn: "border-amber-300 bg-amber-50 text-amber-950", + step_up: "border-orange-300 bg-orange-50 text-orange-950", hold: "border-red-300 bg-red-50 text-red-900", }; // The services on upay's home screen. Only send money does anything here. const SERVICES: [string, string, string][] = [ ["সেন্ড মানি", "Send money", "M3 11.5 21 3l-6.5 18-3-7.5z"], - ["মোবাইল রিচার্জ", "Mobile recharge", "M8 2.5h8A1.5 1.5 0 0 1 17.5 4v16a1.5 1.5 0 0 1-1.5 1.5H8A1.5 1.5 0 0 1 6.5 20V4A1.5 1.5 0 0 1 8 2.5zM10.5 18.5h3"], + ["মোবাইল রিচার্জ", "Recharge", "M8 2.5h8A1.5 1.5 0 0 1 17.5 4v16a1.5 1.5 0 0 1-1.5 1.5H8A1.5 1.5 0 0 1 6.5 20V4A1.5 1.5 0 0 1 8 2.5zM10.5 18.5h3"], ["ক্যাশ আউট", "Cash out", "M12 3v12m0 0-4-4m4 4 4-4M4 20h16"], - ["মেক পেমেন্ট", "Make payment", "M4 4h6v6H4zM14 4h6v6h-6zM4 14h6v6H4zM14 14h2v2h-2zM18 18h2v2h-2zM18 14h2M14 18v2"], + ["মেক পেমেন্ট", "Payment", "M4 4h6v6H4zM14 4h6v6h-6zM4 14h6v6H4zM14 14h2v2h-2zM18 18h2v2h-2zM18 14h2M14 18v2"], ["অ্যাড মানি", "Add money", "M12 3a9 9 0 1 0 0 18 9 9 0 0 0 0-18zM12 8v8M8 12h8"], ["পে বিল", "Pay bill", "M6 3h12v18l-3-2-3 2-3-2-3 2zM9 8h6M9 12h6"], - ["রিকোয়েস্ট মানি", "Request money", "M12 21V9m0 0-4 4m4-4 4 4M4 4h16"], - ["ফান্ড ট্রান্সফার", "Fund transfer", "M3 9.5 12 4l9 5.5M5 10v8M9.5 10v8M14.5 10v8M19 10v8M3 20h18"], + ["রিকোয়েস্ট মানি", "Request", "M12 21V9m0 0-4 4m4-4 4 4M4 4h16"], + ["ফান্ড ট্রান্সফার", "Transfer", "M3 9.5 12 4l9 5.5M5 10v8M9.5 10v8M14.5 10v8M19 10v8M3 20h18"], ]; +/** The language the demo customer last chose. Storage can be missing or blocked: Bangla then. */ +function savedLang(): Lang { + try { + return window.localStorage.getItem(LANG_KEY) === "en" ? "en" : "bn"; + } catch { + return "bn"; + } +} + function Icon({ d, className }: { d: string; className?: string }) { return ( {T.switchTo[lang]} + + ); +} + // The buttons stay at the bottom of the screen while a long message scrolls behind them. const FOOT = "sticky -bottom-4 -mx-4 -mb-4 mt-auto space-y-2 border-t border-neutral-200 bg-white p-4"; +/** One screen of the app. Its title takes focus when it appears, so a screen reader + * announces the new screen and keyboard users start from the top of it. */ function Screen({ - title, tone = "plain", heading, onBack, children, -}: { title: ReactNode; tone?: Tier | "plain"; heading?: ReactNode; onBack?: () => void; children: ReactNode }) { + title, tone = "plain", heading, onBack, lang, onLang, children, +}: { + title: ReactNode; tone?: Tier | "plain"; heading?: ReactNode; onBack?: () => void; + lang: Lang; onLang: () => void; children: ReactNode; +}) { + const titleRef = useRef(null); + useEffect(() => titleRef.current?.focus(), []); return (
-
+
{onBack && ( - )} - {title} +

{title}

+
{tone !== "plain" && heading && (
{heading}
@@ -128,43 +164,31 @@ function Screen({ ); } -function Home() { +function Home({ lang, onLang }: { lang: Lang; onLang: () => void }) { return (
upay
-
Demo customer
-
01XXXXXXXXX
+
{lang === "bn" ? "ডেমো গ্রাহক" : "Demo customer"}
+
01XXXXXXXXX
- ব্যালেন্স +
    {SERVICES.map(([bn, en, d], index) => ( -
  • 0 && "opacity-60")}> +
  • 0 && "opacity-60")}> - {bn} + {lang === "bn" ? bn : en}
  • ))}
-
সেন্ড মানি
- Choose a payment on the left to begin. +
{T.appName[lang]}
+ {T.chooseLeft[lang]}
-

- হেল্পলাইন ১৬২৬৮ · Helpline 16268 -

-
- ); -} - -function Message({ text }: { text: Text2 | null | undefined }) { - if (!text) return null; - return ( -
-

{text.bn}

-

{text.en}

+

{T.helpline[lang]}

); } @@ -173,13 +197,82 @@ function PhoneButton({ tone = "dark", ...rest }: React.ButtonHTMLAttributes; + return ( + + )} +
); - const label = "text-xs font-semibold text-neutral-500"; + const label = "text-xs font-semibold text-neutral-600"; const avatar = ( ); + const amount = payment ? digits(taka(payment.amount), lang) : ""; // One row once there is a decision to read, so the message and the buttons get the room. const party = payment && (result ? (
{avatar} - প্রাপক · To + {w(T.to)} {maskId(payment.receiver_id)} - {taka(payment.amount)} + {amount}
) : (
-
প্রাপক · To
+
{w(T.to)}
{avatar} {maskId(payment.receiver_id)}
-
অ্যামাউন্ট · Amount
-
{taka(payment.amount)}
+
{w(T.amount)}
+
{amount}
)); - const sendMoney = <>সেন্ড মানি · Send money; + const sendMoney = w(T.appName); // Back to the home screen. Not offered while a warning waits for the customer's answer. - const home = busy ? undefined : () => { setPayment(null); setPicked(null); setResult(null); setOutcome(null); setReported(null); setError(null); }; + const home = busy ? undefined : () => { setPayment(null); setPicked(null); setResult(null); setOutcome(null); setReported(null); setAppeal(null); setAppealing(false); setError(null); setWait(null); setReview(null); }; + const screenProps = { lang, onLang: switchLang }; let screen: ReactNode; + let screenKey: string; if (!payment) { - screen = ; + screenKey = "home"; + screen = ; } else if (reporting) { + screenKey = "report"; screen = ( - কী হয়েছিল? · What happened?} onBack={() => setReporting(false)}> + setReporting(false)} {...screenProps}> {CATEGORIES.map(([id, bn, en]) => ( ))} ); + } else if (appealing && decision) { + screenKey = "appeal"; + const left = REASON_MAX - reason.length; + screen = ( + setAppealing(false)} {...screenProps}> +
+

{w(T.appealIntro)}

+
+ {w(T.relation)} +
+ {RELATIONS.map(([value, text]) => ( + + ))} +
+
+