From 640b06c90dc93dbcc65c000e769e17664f443fb7 Mon Sep 17 00:00:00 2001 From: Luke Mainwaring Date: Thu, 16 Jul 2026 15:24:18 -0400 Subject: [PATCH 1/2] test: adopt three-tier test layout with real-Postgres integration tier - Restructure backend/tests into unit//, integration/, evals/ tiers; default pytest run is the DB-free unit tier, integration is opt-in via -m integration (CI runs both). - Add tests/integration/conftest.py: schema from the shipped Alembic migrations, per-test outer-transaction rollback, httpx ASGITransport client with the session dependency overridden. Guards refuse non-test database names and skip when Postgres is down. - Add tests/factories.py (make_* unpersisted / create_* persisted builders). - New integration tests: sessions/thread/retrieval routes, models CRUD, pgvector cosine-ordering smoke, migrations round-trip, DB health. New unit tests: title_generator, message_serialization, emotion. Coverage 43% -> 53%. - Switch async tests to anyio's pytest plugin (drop raw asyncio.run wrappers); drop unused main/additional markers. - Document the commit-owning session dependency contract in dependencies/db.py and remove redundant commits in session-context callers (title_generator, seed_sessions); services keep flush-only. - Alembic env: fileConfig(disable_existing_loggers=False) so in-process migrations (integration tier) don't silence app loggers. - CI: pgvector/pgvector:pg17 service container, POSTGRES_DB=cortexdj_test. - backend/scripts/create-test-db.sh + DEVELOPMENT.md three-tier docs. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 25 +++- .gitignore | 1 + DEVELOPMENT.md | 19 ++- backend/pyproject.toml | 10 +- backend/scripts/create-test-db.sh | 19 +++ backend/src/cortexdj/dependencies/db.py | 8 + backend/src/cortexdj/migrations/env.py | 6 +- .../src/cortexdj/scripts/build_track_index.py | 2 + backend/src/cortexdj/scripts/seed_sessions.py | 2 - .../src/cortexdj/services/title_generator.py | 2 - backend/tests/conftest.py | 12 ++ backend/tests/evals/test_brain_agent_evals.py | 6 +- backend/tests/evals/test_prepare_tools.py | 46 +++--- backend/tests/factories.py | 106 ++++++++++++++ backend/tests/integration/__init__.py | 0 backend/tests/integration/conftest.py | 117 +++++++++++++++ backend/tests/integration/models/__init__.py | 0 .../integration/models/test_models_crud.py | 138 ++++++++++++++++++ backend/tests/integration/routers/__init__.py | 0 .../integration/routers/test_retrieval_api.py | 18 +++ .../integration/routers/test_sessions_api.py | 87 +++++++++++ .../integration/routers/test_thread_api.py | 81 ++++++++++ backend/tests/integration/test_health.py | 11 ++ backend/tests/integration/test_migrations.py | 22 +++ backend/tests/unit/__init__.py | 0 backend/tests/unit/agents/__init__.py | 0 .../agents}/test_brain_agent_hooks.py | 23 ++- .../{ => unit/agents}/test_retrieval_tool.py | 28 ++-- backend/tests/unit/ml/__init__.py | 0 .../tests/{ => unit/ml}/test_contrastive.py | 0 .../ml}/test_contrastive_dataset_paths.py | 0 backend/tests/{ => unit/ml}/test_dataset.py | 0 .../tests/{ => unit/ml}/test_deap_dataset.py | 0 .../{ => unit/ml}/test_majority_baseline.py | 0 backend/tests/{ => unit/ml}/test_metrics.py | 0 .../tests/{ => unit/ml}/test_preprocessing.py | 0 .../tests/{ => unit/ml}/test_pretrained.py | 0 .../tests/{ => unit/ml}/test_resume_state.py | 0 .../tests/{ => unit/ml}/test_train_smoke.py | 0 backend/tests/unit/scripts/__init__.py | 0 .../scripts}/test_build_track_index.py | 25 ++-- backend/tests/unit/services/__init__.py | 0 .../{ => unit/services}/test_audio_catalog.py | 0 .../services}/test_retrieval_service.py | 23 +-- .../unit/services/test_title_generator.py | 97 ++++++++++++ .../{ => unit/services}/test_trajectory.py | 0 backend/tests/unit/utils/__init__.py | 0 backend/tests/unit/utils/test_emotion.py | 27 ++++ .../unit/utils/test_message_serialization.py | 54 +++++++ backend/uv.lock | 2 + 50 files changed, 930 insertions(+), 87 deletions(-) create mode 100755 backend/scripts/create-test-db.sh create mode 100644 backend/tests/factories.py create mode 100644 backend/tests/integration/__init__.py create mode 100644 backend/tests/integration/conftest.py create mode 100644 backend/tests/integration/models/__init__.py create mode 100644 backend/tests/integration/models/test_models_crud.py create mode 100644 backend/tests/integration/routers/__init__.py create mode 100644 backend/tests/integration/routers/test_retrieval_api.py create mode 100644 backend/tests/integration/routers/test_sessions_api.py create mode 100644 backend/tests/integration/routers/test_thread_api.py create mode 100644 backend/tests/integration/test_health.py create mode 100644 backend/tests/integration/test_migrations.py create mode 100644 backend/tests/unit/__init__.py create mode 100644 backend/tests/unit/agents/__init__.py rename backend/tests/{ => unit/agents}/test_brain_agent_hooks.py (79%) rename backend/tests/{ => unit/agents}/test_retrieval_tool.py (81%) create mode 100644 backend/tests/unit/ml/__init__.py rename backend/tests/{ => unit/ml}/test_contrastive.py (100%) rename backend/tests/{ => unit/ml}/test_contrastive_dataset_paths.py (100%) rename backend/tests/{ => unit/ml}/test_dataset.py (100%) rename backend/tests/{ => unit/ml}/test_deap_dataset.py (100%) rename backend/tests/{ => unit/ml}/test_majority_baseline.py (100%) rename backend/tests/{ => unit/ml}/test_metrics.py (100%) rename backend/tests/{ => unit/ml}/test_preprocessing.py (100%) rename backend/tests/{ => unit/ml}/test_pretrained.py (100%) rename backend/tests/{ => unit/ml}/test_resume_state.py (100%) rename backend/tests/{ => unit/ml}/test_train_smoke.py (100%) create mode 100644 backend/tests/unit/scripts/__init__.py rename backend/tests/{ => unit/scripts}/test_build_track_index.py (88%) create mode 100644 backend/tests/unit/services/__init__.py rename backend/tests/{ => unit/services}/test_audio_catalog.py (100%) rename backend/tests/{ => unit/services}/test_retrieval_service.py (89%) create mode 100644 backend/tests/unit/services/test_title_generator.py rename backend/tests/{ => unit/services}/test_trajectory.py (100%) create mode 100644 backend/tests/unit/utils/__init__.py create mode 100644 backend/tests/unit/utils/test_emotion.py create mode 100644 backend/tests/unit/utils/test_message_serialization.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92a6694..e6ccede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,11 +57,28 @@ jobs: test: name: Test runs-on: ubuntu-latest + services: + # Same image as local docker compose; backs the integration tier. + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: cortexdj_test + ports: + - "5432:5432" + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 env: - # Settings require these at import; the tests never connect to a DB. + # The integration tier connects to the service container above and + # refuses to run unless the DB name contains "test" (it runs alembic + # upgrade/downgrade). The unit tier stays DB-free. POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: cortexdj + POSTGRES_DB: cortexdj_test POSTGRES_HOST: localhost POSTGRES_PORT: "5432" # Dummy provider key: tests use TestModel/monkeypatch, never a real call. @@ -86,6 +103,10 @@ jobs: - name: Run tests + coverage working-directory: ./backend # Coverage floor at 40% (baseline ~43%); ratchet up as coverage grows. + # The explicit -m "not eval" OVERRIDES addopts' 'not integration' + # (the last -m wins), which is what selects the integration tier + # here — don't remove it as "redundant" or the tier silently + # deselects while CI stays green. run: uv run pytest -m "not eval" --cov=cortexdj --cov-report=term-missing --cov-report=xml --cov-fail-under=40 - name: Upload coverage report diff --git a/.gitignore b/.gitignore index 7d6a55c..103dbc2 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,7 @@ Thumbs.db # Testing .coverage +coverage.xml htmlcov/ .pytest_cache/ .hypothesis/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index a131dfb..8969c8d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -47,14 +47,27 @@ uv run --directory backend pre-commit run --all-files ## Tests +The suite has three tiers under `backend/tests/`: + +- **`unit//`** — fast, DB-free, provider-free. This is the default `pytest` run. +- **`integration/`** — real Postgres + HTTP (`httpx` against the app). Schema comes from the shipped Alembic migrations; each test rolls back an outer transaction, so tests never pollute each other. Opt-in via `-m integration`. +- **`evals/`** — real-LLM `brain_agent` evals. Opt-in via `-m eval`; use as a nightly safety net on `main` or manual spot-checks, not on every PR. See `.claude/rules/backend/pydantic-ai.md`. + ```bash -uv run --directory backend pytest # unit tests (eval suite excluded) +uv run --directory backend pytest # unit tier (integration + eval excluded) uv run --directory backend pytest -v # verbose output -uv run --directory backend pytest tests/test_preprocessing.py # single file +uv run --directory backend pytest tests/unit/ml/test_preprocessing.py # single file uv run --directory backend pytest -m eval # real-model brain_agent eval suite (opt-in) ``` -The `eval` marker gates tests that call the real OpenAI API via `brain_agent` — the default `pytest` invocation excludes them via `addopts = "-m 'not eval'"`. Use them as a nightly safety net on `main` or manual spot-checks, not on every PR. See `.claude/rules/backend/pydantic-ai.md` and `backend/tests/evals/` for the suite layout. +To run the integration tier locally, create the test database once, then point `POSTGRES_DB` at it (the env var overrides the `.env` value; host/port still come from `.env`): + +```bash +./backend/scripts/create-test-db.sh +POSTGRES_DB=cortexdj_test uv run --directory backend pytest -m integration +``` + +The tier refuses to run unless the database name contains `test`, because it runs `alembic upgrade`/`downgrade` against whatever database the env points at. Shared test-data builders live in `backend/tests/factories.py` (`make_*` = unpersisted, `create_*` = persisted via flush). ## Database migrations diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f7aa593..fca9ed5 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ [dependency-groups] dev = [ + "anyio>=4.14.2", "mypy>=2.3.0", "pre-commit>=4.6.0", "pydantic-evals>=2.11.0", @@ -43,12 +44,15 @@ dev = [ ] [tool.pytest.ini_options] +testpaths = ["tests"] markers = [ - "main: marks tests for main demo examples", - "additional: marks tests for additional demo examples", + "integration: real-DB/HTTP tests; needs Postgres (CI service or `docker compose up -d postgres`)", "eval: marks tests that call a real LLM (opt-in; run with `pytest -m eval`)", ] -addopts = "-m 'not eval'" +# Default run = the fast, DB-free unit tier; `integration` and `eval` are +# opt-in via -m. Test dirs are packages (an __init__.py each) so the same +# basename can recur across tiers. +addopts = "-m 'not eval and not integration'" [tool.mypy] strict = true diff --git a/backend/scripts/create-test-db.sh b/backend/scripts/create-test-db.sh new file mode 100755 index 0000000..eb16aad --- /dev/null +++ b/backend/scripts/create-test-db.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Create the integration-test database (idempotent). The integration tier +# refuses to run unless POSTGRES_DB contains "test", because it runs +# `alembic upgrade`/`downgrade` against whatever database the env points at. +set -euo pipefail + +docker compose up -d postgres + +echo "Waiting for Postgres to become ready..." +until docker compose exec postgres pg_isready -U postgres -q; do + sleep 1 +done + +docker compose exec postgres psql -U postgres -tc \ + "SELECT 1 FROM pg_database WHERE datname = 'cortexdj_test'" | grep -q 1 || + docker compose exec postgres createdb -U postgres cortexdj_test + +echo "cortexdj_test is ready. Run the tier with:" +echo " POSTGRES_DB=cortexdj_test uv run --directory backend pytest -m integration" diff --git a/backend/src/cortexdj/dependencies/db.py b/backend/src/cortexdj/dependencies/db.py index bee3dae..b753f49 100644 --- a/backend/src/cortexdj/dependencies/db.py +++ b/backend/src/cortexdj/dependencies/db.py @@ -35,6 +35,14 @@ def get_postgres_url(prefix: str) -> str: async def _get_async_sqlalchemy_session_dependency() -> AsyncGenerator[AsyncSession, None]: + """Yield a session that owns its transaction: commit on success, rollback on error. + + This is the single commit point for request-scoped sessions (and for + background work via ``get_async_sqlalchemy_session``). Services and model + classmethods must only ``flush()``, never ``commit()`` — the integration + test tier overrides this dependency and rolls back an outer transaction + per test, which only isolates tests if no other code commits. + """ async with AsyncSessionMaker() as session: try: yield session diff --git a/backend/src/cortexdj/migrations/env.py b/backend/src/cortexdj/migrations/env.py index 675c837..1cf0702 100644 --- a/backend/src/cortexdj/migrations/env.py +++ b/backend/src/cortexdj/migrations/env.py @@ -9,7 +9,11 @@ config.set_main_option("sqlalchemy.url", get_postgres_url("postgresql+psycopg")) if config.config_file_name is not None: - fileConfig(config.config_file_name) + # Keep existing loggers alive: migrations also run in-process (the + # integration test tier calls `command.upgrade`), and fileConfig's + # default disable_existing_loggers=True would silence every logger + # created before this point. + fileConfig(config.config_file_name, disable_existing_loggers=False) from cortexdj.models import Base # noqa: F401, E402 diff --git a/backend/src/cortexdj/scripts/build_track_index.py b/backend/src/cortexdj/scripts/build_track_index.py index 000e397..f9e6771 100644 --- a/backend/src/cortexdj/scripts/build_track_index.py +++ b/backend/src/cortexdj/scripts/build_track_index.py @@ -209,6 +209,8 @@ async def _main_async() -> int: MISS_LOG_PATH.unlink(missing_ok=True) + # Raw script-owned session (not the commit-owning dependency), so the + # explicit commits below are intentional. async with AsyncSessionMaker() as db: from cortexdj.services.spotify import get_user_spotify_client diff --git a/backend/src/cortexdj/scripts/seed_sessions.py b/backend/src/cortexdj/scripts/seed_sessions.py index 02f1cdb..a10124d 100644 --- a/backend/src/cortexdj/scripts/seed_sessions.py +++ b/backend/src/cortexdj/scripts/seed_sessions.py @@ -193,8 +193,6 @@ async def seed_participant( seeded += 1 - await db.commit() - return seeded diff --git a/backend/src/cortexdj/services/title_generator.py b/backend/src/cortexdj/services/title_generator.py index f98d98c..7dbfed2 100644 --- a/backend/src/cortexdj/services/title_generator.py +++ b/backend/src/cortexdj/services/title_generator.py @@ -51,7 +51,6 @@ async def generate_thread_title( title = title.strip().strip('"').strip("'") async with get_async_sqlalchemy_session() as db: await Thread.update_title(db, thread_id, agent_type, title) - await db.commit() logger.info(f"Generated title for thread {thread_id}: {title}") except Exception: @@ -60,7 +59,6 @@ async def generate_thread_title( try: async with get_async_sqlalchemy_session() as db: await Thread.update_title(db, thread_id, agent_type, fallback) - await db.commit() logger.info(f"Used fallback title for thread {thread_id}: {fallback}") except Exception: logger.exception(f"Failed to save fallback title for thread {thread_id}") diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 888101f..f737bf8 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,8 +1,20 @@ import os +import pytest + # ``OpenAIResponsesModel`` constructs its provider client eagerly at module # import time, which requires ``OPENAI_API_KEY``. Any test that imports # ``brain_agent`` — directly or transitively — would otherwise fail at # collection. Tests that actually invoke the model use ``agent.override`` # with ``TestModel``; this dummy value never reaches a real API call. os.environ.setdefault("OPENAI_API_KEY", "sk-test-deterministic-no-real-calls") + +# Deliberately no POSTGRES_* defaults here: settings load from the repo-root +# ``.env`` (port 5433), and real env vars take precedence over dotenv — a +# setdefault would silently override ``.env`` and break local integration runs. + + +@pytest.fixture +def anyio_backend() -> str: + """Run ``@pytest.mark.anyio`` async tests on the asyncio backend.""" + return "asyncio" diff --git a/backend/tests/evals/test_brain_agent_evals.py b/backend/tests/evals/test_brain_agent_evals.py index ef9a942..5ebc708 100644 --- a/backend/tests/evals/test_brain_agent_evals.py +++ b/backend/tests/evals/test_brain_agent_evals.py @@ -21,7 +21,6 @@ agent is allowed to chain tools as long as the critical one fires. """ -import asyncio from dataclasses import dataclass, field import pytest @@ -159,8 +158,9 @@ def evaluate(self, ctx: EvaluatorContext[BrainAgentInput, BrainAgentOutput, None @pytest.mark.eval -def test_brain_agent_tool_routing() -> None: - report = asyncio.run(_dataset.evaluate(_run_brain_agent)) +@pytest.mark.anyio +async def test_brain_agent_tool_routing() -> None: + report = await _dataset.evaluate(_run_brain_agent) execution_failures = [f.name for f in report.failures] assert not execution_failures, f"Eval execution errors: {execution_failures}" diff --git a/backend/tests/evals/test_prepare_tools.py b/backend/tests/evals/test_prepare_tools.py index 7b728f3..3721668 100644 --- a/backend/tests/evals/test_prepare_tools.py +++ b/backend/tests/evals/test_prepare_tools.py @@ -12,8 +12,7 @@ model-routing harness, but the wiring they depend on is validated here. """ -import asyncio - +import pytest import spotipy from pydantic_ai.models.test import TestModel @@ -28,7 +27,10 @@ def _offered_tool_names(model: TestModel) -> set[str]: return {t.name for t in params.function_tools} -def _run_agent_with_test_model( +pytestmark = pytest.mark.anyio + + +async def _run_agent_with_test_model( *, spotify_client: spotipy.Spotify | None, eeg_model: EEGModel | None, @@ -36,17 +38,15 @@ def _run_agent_with_test_model( test_model = TestModel(call_tools=[], custom_output_text="ok") deps = make_fake_deps(spotify_client=spotify_client, eeg_model=eeg_model) - async def _run() -> None: - with brain_agent.override(model=test_model, deps=deps): - await brain_agent.run("hello") + with brain_agent.override(model=test_model, deps=deps): + await brain_agent.run("hello") - asyncio.run(_run()) return test_model class TestPlaylistCapabilityPrepareTools: - def test_hides_user_spotify_tools_when_disconnected(self) -> None: - model = _run_agent_with_test_model(spotify_client=None, eeg_model=None) + async def test_hides_user_spotify_tools_when_disconnected(self) -> None: + model = await _run_agent_with_test_model(spotify_client=None, eeg_model=None) offered = _offered_tool_names(model) for hidden in ( @@ -57,24 +57,24 @@ def test_hides_user_spotify_tools_when_disconnected(self) -> None: ): assert hidden not in offered, f"{hidden} should be hidden when spotify_client is None" - def test_public_spotify_tools_always_available(self) -> None: - model = _run_agent_with_test_model(spotify_client=None, eeg_model=None) + async def test_public_spotify_tools_always_available(self) -> None: + model = await _run_agent_with_test_model(spotify_client=None, eeg_model=None) offered = _offered_tool_names(model) assert "search_tracks" in offered assert "get_track_info" in offered - def test_eeg_tools_always_available_regardless_of_spotify(self) -> None: - model = _run_agent_with_test_model(spotify_client=None, eeg_model=None) + async def test_eeg_tools_always_available_regardless_of_spotify(self) -> None: + model = await _run_agent_with_test_model(spotify_client=None, eeg_model=None) offered = _offered_tool_names(model) assert "find_relaxing_tracks" in offered assert "build_mood_playlist" in offered - def test_shows_user_spotify_tools_when_connected(self) -> None: + async def test_shows_user_spotify_tools_when_connected(self) -> None: from tests.evals.conftest import fake_spotify_client - model = _run_agent_with_test_model(spotify_client=fake_spotify_client(), eeg_model=None) + model = await _run_agent_with_test_model(spotify_client=fake_spotify_client(), eeg_model=None) offered = _offered_tool_names(model) assert "get_my_playlists" in offered @@ -82,30 +82,30 @@ def test_shows_user_spotify_tools_when_connected(self) -> None: class TestClassificationCapabilityPrepareTools: - def test_hides_model_tools_when_eeg_model_missing(self) -> None: - model = _run_agent_with_test_model(spotify_client=None, eeg_model=None) + async def test_hides_model_tools_when_eeg_model_missing(self) -> None: + model = await _run_agent_with_test_model(spotify_client=None, eeg_model=None) offered = _offered_tool_names(model) assert "get_model_info" not in offered - def test_set_brain_context_always_available(self) -> None: - model = _run_agent_with_test_model(spotify_client=None, eeg_model=None) + async def test_set_brain_context_always_available(self) -> None: + model = await _run_agent_with_test_model(spotify_client=None, eeg_model=None) offered = _offered_tool_names(model) assert "set_brain_context" in offered - def test_shows_model_tools_when_eeg_model_loaded(self) -> None: + async def test_shows_model_tools_when_eeg_model_loaded(self) -> None: from tests.evals.conftest import fake_eeg_model - model = _run_agent_with_test_model(spotify_client=None, eeg_model=fake_eeg_model()) + model = await _run_agent_with_test_model(spotify_client=None, eeg_model=fake_eeg_model()) offered = _offered_tool_names(model) assert "get_model_info" in offered class TestAlwaysAvailableTools: - def test_session_and_insight_tools_always_present(self) -> None: - model = _run_agent_with_test_model(spotify_client=None, eeg_model=None) + async def test_session_and_insight_tools_always_present(self) -> None: + model = await _run_agent_with_test_model(spotify_client=None, eeg_model=None) offered = _offered_tool_names(model) for always in ( diff --git a/backend/tests/factories.py b/backend/tests/factories.py new file mode 100644 index 0000000..c3c58dd --- /dev/null +++ b/backend/tests/factories.py @@ -0,0 +1,106 @@ +"""Test-data builders shared across test tiers. + +``make_*`` returns an unpersisted ORM instance (unit tier). ``create_*`` +persists via the given session — add, flush, refresh — for the integration +tier, where the surrounding fixture rolls an outer transaction back after +each test. Builders flush and never commit, same as services: the session's +owner holds the only commit. +""" + +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +from sqlalchemy.ext.asyncio import AsyncSession + +from cortexdj.models.eeg_segment import EegSegment +from cortexdj.models.session import Session +from cortexdj.models.thread import Thread +from cortexdj.models.track_audio_embedding import EMBEDDING_DIM, TrackAudioEmbedding +from cortexdj.schemas.agent_type import AgentType + + +def make_session(**overrides: Any) -> Session: + data: dict[str, Any] = { + "id": str(uuid4()), + "participant_id": "P01", + "dataset_source": "deap", + "recorded_at": datetime(2024, 1, 1, tzinfo=UTC), + "duration_seconds": 60.0, + } + data.update(overrides) + return Session(**data) + + +async def create_session(db: AsyncSession, **overrides: Any) -> Session: + session = make_session(**overrides) + db.add(session) + await db.flush() + await db.refresh(session) + return session + + +def make_eeg_segment(session_id: str, **overrides: Any) -> EegSegment: + data: dict[str, Any] = { + "id": str(uuid4()), + "session_id": session_id, + "segment_index": 0, + "start_time": 0.0, + "end_time": 4.0, + "arousal_score": 0.7, + "valence_score": 0.6, + "dominant_state": "excited", + "band_powers": {"alpha": 1.0, "beta": 0.5, "theta": 0.3, "delta": 0.2, "gamma": 0.1}, + } + data.update(overrides) + return EegSegment(**data) + + +async def create_eeg_segment(db: AsyncSession, session_id: str, **overrides: Any) -> EegSegment: + segment = make_eeg_segment(session_id, **overrides) + db.add(segment) + await db.flush() + await db.refresh(segment) + return segment + + +def make_track_audio_embedding(**overrides: Any) -> TrackAudioEmbedding: + # Default is a unit vector, not zeros — cosine distance against a zero + # vector is undefined, which would poison similarity-ordering tests. + embedding = [0.0] * EMBEDDING_DIM + embedding[0] = 1.0 + data: dict[str, Any] = { + "spotify_id": f"sp-{uuid4().hex[:16]}", + "title": "Test Track", + "artist": "Test Artist", + "source": "user_library", + "embedding": embedding, + } + data.update(overrides) + return TrackAudioEmbedding(**data) + + +async def create_track_audio_embedding(db: AsyncSession, **overrides: Any) -> TrackAudioEmbedding: + row = make_track_audio_embedding(**overrides) + db.add(row) + await db.flush() + await db.refresh(row) + return row + + +def make_thread(**overrides: Any) -> Thread: + data: dict[str, Any] = { + "thread_id": f"t-{uuid4().hex[:8]}", + "agent_type": AgentType.CHAT.value, + "title": None, + } + data.update(overrides) + return Thread(**data) + + +async def create_thread(db: AsyncSession, **overrides: Any) -> Thread: + thread = make_thread(**overrides) + db.add(thread) + await db.flush() + await db.refresh(thread) + return thread diff --git a/backend/tests/integration/__init__.py b/backend/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py new file mode 100644 index 0000000..b592563 --- /dev/null +++ b/backend/tests/integration/conftest.py @@ -0,0 +1,117 @@ +"""Composition root for DB/HTTP integration tests — the test-tier analog of app.py. + +Schema comes from the shipped Alembic migrations, not ``create_all``: the +pgvector extension and the HNSW index exist only in the migrations, so this +tier exercises the real schema. Each test runs inside an outer transaction +on a single connection that is rolled back at teardown, so tests never see +each other's writes. The ``client`` fixture overrides the app's session +dependency with that transactional session and never commits — isolation +holds only because the app's sole commit point is the overridden dependency +itself (services and model classmethods flush, never commit). + +Caveats: +- ``ASGITransport`` skips lifespan, so ``app.state.eeg_model`` is never set. + Don't integration-test ``/agent/chat``; its behavior is covered at the + unit tier with pydantic-ai's ``TestModel``. +- ``/api/health/db`` uses the raw psycopg dependency, which is not + overridden; it opens a real (read-only) connection to the test database. +- Guards read ``get_settings()`` rather than ``os.environ`` because settings + may come from the repo-root ``.env``; run locally with + ``POSTGRES_DB=cortexdj_test`` to override the ``.env`` database name. +""" + +import socket +from collections.abc import AsyncGenerator + +import pytest +from alembic import command +from alembic.config import Config +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) +from sqlalchemy.pool import NullPool + +from cortexdj.core.config import get_settings +from cortexdj.dependencies.db import ( + _get_async_sqlalchemy_session_dependency, + get_async_postgres_url, +) + +_ALEMBIC_INI = "src/cortexdj/alembic.ini" # relative to backend/, where pytest runs + + +def _db_reachable() -> bool: + settings = get_settings() + try: + with socket.create_connection((settings.POSTGRES_HOST, settings.POSTGRES_PORT), timeout=1.0): + return True + except OSError: + return False + + +def _is_test_db() -> bool: + return "test" in get_settings().POSTGRES_DB + + +@pytest.fixture(scope="session", autouse=True) +def _require_test_db() -> None: + if not _db_reachable(): + pytest.skip("integration tier needs Postgres (`docker compose up -d postgres`)") + if not _is_test_db(): + pytest.skip( + "refusing to run against a non-test database — this tier runs alembic " + "upgrade/downgrade; set POSTGRES_DB=cortexdj_test " + "(see backend/scripts/create-test-db.sh)" + ) + + +@pytest.fixture(scope="session") +def _migrated(_require_test_db: None) -> None: + command.upgrade(Config(_ALEMBIC_INI), "head") + + +@pytest.fixture +async def engine(_migrated: None) -> AsyncGenerator[AsyncEngine, None]: + # Function-scoped with NullPool on purpose: anyio gives each test its own + # event loop, and pooled connections bound to a previous test's loop fail. + # Don't "optimize" this to session scope. + engine = create_async_engine(get_async_postgres_url(), poolclass=NullPool) + try: + yield engine + finally: + await engine.dispose() + + +@pytest.fixture +async def db_session(engine: AsyncEngine) -> AsyncGenerator[AsyncSession, None]: + conn = await engine.connect() + outer = await conn.begin() # rolled back at teardown → no cross-test pollution + session = async_sessionmaker(bind=conn, expire_on_commit=False, class_=AsyncSession)() + try: + yield session + finally: + await session.close() + await outer.rollback() + await conn.close() + + +@pytest.fixture +async def client(db_session: AsyncSession) -> AsyncGenerator[AsyncClient, None]: + # Imported lazily so collecting the unit tier never pulls in the app. + from cortexdj.app import app + + async def _override() -> AsyncGenerator[AsyncSession, None]: + # Deliberately no commit: the outer transaction owns the data. + yield db_session + + app.dependency_overrides[_get_async_sqlalchemy_session_dependency] = _override + try: + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + finally: + app.dependency_overrides.pop(_get_async_sqlalchemy_session_dependency, None) diff --git a/backend/tests/integration/models/__init__.py b/backend/tests/integration/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/integration/models/test_models_crud.py b/backend/tests/integration/models/test_models_crud.py new file mode 100644 index 0000000..02e479a --- /dev/null +++ b/backend/tests/integration/models/test_models_crud.py @@ -0,0 +1,138 @@ +"""CRUD classmethod tests against the real (migrated) schema. + +Covers the model behaviors the HTTP tier doesn't reach directly: pagination +semantics, idempotent get_or_create, append-only message history, the +Spotify token singleton, and a pgvector cosine-ordering smoke test that +exercises the extension + HNSW index created by the migrations. +""" + +from datetime import UTC, datetime + +import numpy as np +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from cortexdj.models.message import Message +from cortexdj.models.session import Session +from cortexdj.models.spotify_token import SpotifyToken +from cortexdj.models.thread import Thread +from cortexdj.models.track_audio_embedding import EMBEDDING_DIM, TrackAudioEmbedding +from cortexdj.schemas.agent_type import AgentType +from cortexdj.schemas.thread import BrainContext +from tests.factories import create_session, create_track_audio_embedding + +pytestmark = [pytest.mark.integration, pytest.mark.anyio] + +_CHAT = AgentType.CHAT.value + + +class TestSession: + async def test_get_all_paginates_newest_first(self, db_session: AsyncSession) -> None: + for day in (1, 2, 3): + await create_session(db_session, recorded_at=datetime(2024, 2, day, tzinfo=UTC)) + + sessions, total = await Session.get_all(db_session, limit=2, offset=0) + assert total == 3 + assert len(sessions) == 2 + assert sessions[0].recorded_at.day == 3 + + async def test_get_by_participant(self, db_session: AsyncSession) -> None: + await create_session(db_session, participant_id="P01") + await create_session(db_session, participant_id="P02") + + found = await Session.get_by_participant(db_session, "P02") + assert [s.participant_id for s in found] == ["P02"] + + +class TestThread: + async def test_get_or_create_is_idempotent(self, db_session: AsyncSession) -> None: + first = await Thread.get_or_create(db_session, "t-idem", _CHAT) + second = await Thread.get_or_create(db_session, "t-idem", _CHAT) + assert first.thread_id == second.thread_id + assert len(await Thread.list_all(db_session, _CHAT)) == 1 + + async def test_update_title(self, db_session: AsyncSession) -> None: + await Thread.get_or_create(db_session, "t-title", _CHAT) + await Thread.update_title(db_session, "t-title", _CHAT, "Morning focus") + + thread = await Thread.get(db_session, "t-title", _CHAT) + assert thread is not None + assert thread.title == "Morning focus" + + async def test_update_brain_context_merges(self, db_session: AsyncSession) -> None: + # Creates the thread on first write, then merges only set fields. + merged = await Thread.update_brain_context( + db_session, "t-ctx", _CHAT, BrainContext(latest_session_id="sess-1", dominant_mood="relaxed") + ) + assert merged.latest_session_id == "sess-1" + + merged = await Thread.update_brain_context(db_session, "t-ctx", _CHAT, BrainContext(dominant_mood="excited")) + assert merged.dominant_mood == "excited" + assert merged.latest_session_id == "sess-1", "unset fields must survive the merge" + + async def test_delete_by_id(self, db_session: AsyncSession) -> None: + await Thread.get_or_create(db_session, "t-del", _CHAT) + await Thread.delete_by_id(db_session, "t-del", _CHAT) + assert await Thread.get(db_session, "t-del", _CHAT) is None + + +class TestMessage: + async def test_save_history_is_append_only(self, db_session: AsyncSession) -> None: + await Thread.get_or_create(db_session, "t-msg", _CHAT) + first_batch: list[dict[str, object]] = [{"kind": "request", "n": 1}, {"kind": "response", "n": 2}] + await Message.save_history(db_session, "t-msg", _CHAT, first_batch) + + # Re-saving the full history plus one new message must insert only + # the new one — existing rows keep their ids and timestamps. + await Message.save_history(db_session, "t-msg", _CHAT, [*first_batch, {"kind": "request", "n": 3}]) + + history = await Message.get_history(db_session, "t-msg", _CHAT) + assert [m["n"] for m in history] == [1, 2, 3] + + +class TestSpotifyToken: + async def test_upsert_then_update_keeps_singleton(self, db_session: AsyncSession) -> None: + expires = datetime(2030, 1, 1, tzinfo=UTC) + assert not await SpotifyToken.is_connected(db_session) + + await SpotifyToken.upsert(db_session, "access-1", "refresh-1", expires) + token = await SpotifyToken.upsert(db_session, "access-2", "refresh-2", expires) + + assert token.access_token == "access-2" + assert await SpotifyToken.is_connected(db_session) + + async def test_clear(self, db_session: AsyncSession) -> None: + await SpotifyToken.upsert(db_session, "a", "r", datetime(2030, 1, 1, tzinfo=UTC)) + await SpotifyToken.clear(db_session) + assert not await SpotifyToken.is_connected(db_session) + + +class TestTrackAudioEmbeddingPgvector: + async def test_cosine_search_orders_nearest_first(self, db_session: AsyncSession) -> None: + e1 = [0.0] * EMBEDDING_DIM + e1[0] = 1.0 + e2 = [0.0] * EMBEDDING_DIM + e2[1] = 1.0 + await create_track_audio_embedding(db_session, spotify_id="sp-near", embedding=e1) + await create_track_audio_embedding(db_session, spotify_id="sp-far", embedding=e2) + + hits = await TrackAudioEmbedding.get_top_k_similar(db_session, np.array(e1, dtype=np.float32), k=2) + + assert [row.spotify_id for row, _ in hits] == ["sp-near", "sp-far"] + near_distance, far_distance = hits[0][1], hits[1][1] + assert near_distance == pytest.approx(0.0, abs=1e-5) + assert far_distance == pytest.approx(1.0, abs=1e-5) + + async def test_upsert_replaces_on_spotify_id_conflict(self, db_session: AsyncSession) -> None: + vec = np.zeros(EMBEDDING_DIM, dtype=np.float32) + vec[0] = 1.0 + await TrackAudioEmbedding.upsert( + db_session, spotify_id="sp-up", title="Old", artist="A", source="user_library", embedding=vec + ) + await TrackAudioEmbedding.upsert( + db_session, spotify_id="sp-up", title="New", artist="A", source="user_library", embedding=vec + ) + + assert await TrackAudioEmbedding.count(db_session) == 1 + hits = await TrackAudioEmbedding.get_top_k_similar(db_session, vec, k=1) + assert hits[0][0].title == "New" diff --git a/backend/tests/integration/routers/__init__.py b/backend/tests/integration/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/integration/routers/test_retrieval_api.py b/backend/tests/integration/routers/test_retrieval_api.py new file mode 100644 index 0000000..33a7888 --- /dev/null +++ b/backend/tests/integration/routers/test_retrieval_api.py @@ -0,0 +1,18 @@ +"""Route-level test for the similar-tracks endpoint's 404 path. + +Deliberately no happy-path HTTP test: retrieval needs the contrastive +checkpoint and DEAP ``.dat`` files, neither of which exists in CI. The +503 (missing checkpoint) and 500 (missing DEAP file) mappings are covered +at the unit tier in ``tests/unit/services/test_retrieval_service.py`` and +``tests/unit/agents/test_retrieval_tool.py``. +""" + +import pytest +from httpx import AsyncClient + +pytestmark = [pytest.mark.integration, pytest.mark.anyio] + + +async def test_similar_tracks_for_unknown_session_returns_404(client: AsyncClient) -> None: + response = await client.get("/api/sessions/no-such-session/similar-tracks") + assert response.status_code == 404 diff --git a/backend/tests/integration/routers/test_sessions_api.py b/backend/tests/integration/routers/test_sessions_api.py new file mode 100644 index 0000000..e7c6d29 --- /dev/null +++ b/backend/tests/integration/routers/test_sessions_api.py @@ -0,0 +1,87 @@ +"""Route-level tests for the sessions endpoints, asserted through HTTP.""" + +from datetime import UTC, datetime + +import pytest +from httpx import AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession + +from tests.factories import create_eeg_segment, create_session + +pytestmark = [pytest.mark.integration, pytest.mark.anyio] + + +async def test_list_sessions_empty(client: AsyncClient) -> None: + response = await client.get("/api/sessions") + assert response.status_code == 200 + body = response.json() + assert body["sessions"] == [] + assert body["total"] == 0 + + +async def test_list_sessions_pagination(client: AsyncClient, db_session: AsyncSession) -> None: + for day in (1, 2, 3): + await create_session(db_session, recorded_at=datetime(2024, 1, day, tzinfo=UTC)) + + first_page = (await client.get("/api/sessions", params={"limit": 2, "offset": 0})).json() + assert first_page["total"] == 3 + assert len(first_page["sessions"]) == 2 + # Newest first. + assert first_page["sessions"][0]["recorded_at"].startswith("2024-01-03") + + second_page = (await client.get("/api/sessions", params={"limit": 2, "offset": 2})).json() + assert second_page["total"] == 3 + assert len(second_page["sessions"]) == 1 + + +async def test_get_session_by_id(client: AsyncClient, db_session: AsyncSession) -> None: + session = await create_session(db_session, participant_id="P07") + + response = await client.get(f"/api/sessions/{session.id}") + assert response.status_code == 200 + body = response.json() + assert body["id"] == session.id + assert body["participant_id"] == "P07" + + +async def test_get_unknown_session_returns_404(client: AsyncClient) -> None: + response = await client.get("/api/sessions/no-such-session") + assert response.status_code == 404 + + +async def test_get_session_segments_ordered_with_trajectory(client: AsyncClient, db_session: AsyncSession) -> None: + session = await create_session(db_session) + # Insert out of order to prove ordering comes from segment_index. + await create_eeg_segment(db_session, session.id, segment_index=1, start_time=4.0, end_time=8.0) + await create_eeg_segment(db_session, session.id, segment_index=0, start_time=0.0, end_time=4.0) + + response = await client.get(f"/api/sessions/{session.id}/segments") + assert response.status_code == 200 + body = response.json() + assert body["total"] == 2 + assert [s["segment_index"] for s in body["segments"]] == [0, 1] + assert body["trajectory_summary"] is not None + + +async def test_segments_for_unknown_session_is_empty_not_404(client: AsyncClient) -> None: + # Pins current behavior: the endpoint returns an empty list rather than 404. + response = await client.get("/api/sessions/no-such-session/segments") + assert response.status_code == 200 + assert response.json()["total"] == 0 + + +async def test_list_sessions_enriched(client: AsyncClient, db_session: AsyncSession) -> None: + session = await create_session(db_session) + for i in range(3): + await create_eeg_segment(db_session, session.id, segment_index=i, dominant_state="relaxed") + + response = await client.get("/api/sessions/enriched") + assert response.status_code == 200 + body = response.json() + assert body["total"] == 1 + summary = body["sessions"][0] + assert summary["display_index"] == 1 + assert summary["dominant_state"] == "relaxed" + assert summary["label"] == "Relaxed throughout" + assert summary["segment_count"] == 3 + assert summary["state_distribution"]["relaxed"] == 1.0 diff --git a/backend/tests/integration/routers/test_thread_api.py b/backend/tests/integration/routers/test_thread_api.py new file mode 100644 index 0000000..6332dcd --- /dev/null +++ b/backend/tests/integration/routers/test_thread_api.py @@ -0,0 +1,81 @@ +"""Route-level tests for the thread endpoints, asserted through HTTP.""" + +import pytest +from httpx import AsyncClient +from pydantic_ai.messages import ModelRequest, ModelResponse, TextPart, UserPromptPart +from sqlalchemy.ext.asyncio import AsyncSession + +from cortexdj.models.message import Message +from cortexdj.schemas.agent_type import AgentType +from cortexdj.utils.message_serialization import prepare_messages_for_storage +from tests.factories import create_thread + +pytestmark = [pytest.mark.integration, pytest.mark.anyio] + +_CHAT = AgentType.CHAT.value + + +def _history_payload() -> list[dict[str, object]]: + return prepare_messages_for_storage( + [ + ModelRequest(parts=[UserPromptPart(content="How was my last session?")]), + ModelResponse(parts=[TextPart(content="It was mostly relaxed.")]), + ] + ) + + +async def test_list_threads_empty(client: AsyncClient) -> None: + response = await client.get("/api/threads") + assert response.status_code == 200 + assert response.json()["threads"] == [] + + +async def test_thread_messages_round_trip(client: AsyncClient, db_session: AsyncSession) -> None: + thread = await create_thread(db_session) + await Message.save_history(db_session, thread.thread_id, _CHAT, _history_payload()) + + response = await client.get(f"/api/threads/{thread.thread_id}/messages") + assert response.status_code == 200 + body = response.json() + assert body["thread_id"] == thread.thread_id + roles = [m["role"] for m in body["messages"]] + assert roles == ["user", "assistant"] + + +async def test_messages_for_unknown_thread_returns_404(client: AsyncClient) -> None: + response = await client.get("/api/threads/no-such-thread/messages") + assert response.status_code == 404 + + +async def test_rename_thread(client: AsyncClient, db_session: AsyncSession) -> None: + thread = await create_thread(db_session) + + response = await client.patch(f"/api/threads/{thread.thread_id}", json={"title": "Focus session recap"}) + assert response.status_code == 200 + assert response.json()["title"] == "Focus session recap" + + listed = (await client.get("/api/threads")).json()["threads"] + assert listed[0]["title"] == "Focus session recap" + + +async def test_rename_unknown_thread_returns_404(client: AsyncClient) -> None: + # Pins the ThreadNotFound(HTTPException) convention end-to-end. + response = await client.patch("/api/threads/no-such-thread", json={"title": "x"}) + assert response.status_code == 404 + + +async def test_delete_thread_cascades_messages(client: AsyncClient, db_session: AsyncSession) -> None: + thread = await create_thread(db_session) + await Message.save_history(db_session, thread.thread_id, _CHAT, _history_payload()) + + response = await client.delete(f"/api/threads/{thread.thread_id}") + assert response.status_code == 200 + + assert (await client.get("/api/threads")).json()["threads"] == [] + # FK ondelete=CASCADE — exercised against the real schema. + assert await Message.get_history(db_session, thread.thread_id, _CHAT) == [] + + +async def test_delete_unknown_thread_returns_404(client: AsyncClient) -> None: + response = await client.delete("/api/threads/no-such-thread") + assert response.status_code == 404 diff --git a/backend/tests/integration/test_health.py b/backend/tests/integration/test_health.py new file mode 100644 index 0000000..da63504 --- /dev/null +++ b/backend/tests/integration/test_health.py @@ -0,0 +1,11 @@ +import pytest +from httpx import AsyncClient + +pytestmark = [pytest.mark.integration, pytest.mark.anyio] + + +async def test_db_health_check(client: AsyncClient) -> None: + # Exercises the raw psycopg dependency against the real test database. + response = await client.get("/api/health/db") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} diff --git a/backend/tests/integration/test_migrations.py b/backend/tests/integration/test_migrations.py new file mode 100644 index 0000000..1b40b9d --- /dev/null +++ b/backend/tests/integration/test_migrations.py @@ -0,0 +1,22 @@ +"""Prove the shipped migrations round-trip cleanly (downgrade base → head). + +Safe only because the integration tier refuses to run against a database +whose name lacks "test" (see conftest ``_require_test_db``). +""" + +import pytest +from alembic import command +from alembic.config import Config + +pytestmark = pytest.mark.integration + +_ALEMBIC_INI = "src/cortexdj/alembic.ini" + + +def test_migrations_round_trip(_migrated: None) -> None: + cfg = Config(_ALEMBIC_INI) + try: + command.downgrade(cfg, "base") + finally: + # Leave the schema at head for the rest of the tier regardless of outcome. + command.upgrade(cfg, "head") diff --git a/backend/tests/unit/__init__.py b/backend/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/unit/agents/__init__.py b/backend/tests/unit/agents/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/test_brain_agent_hooks.py b/backend/tests/unit/agents/test_brain_agent_hooks.py similarity index 79% rename from backend/tests/test_brain_agent_hooks.py rename to backend/tests/unit/agents/test_brain_agent_hooks.py index f032dd7..a97e981 100644 --- a/backend/tests/test_brain_agent_hooks.py +++ b/backend/tests/unit/agents/test_brain_agent_hooks.py @@ -6,8 +6,7 @@ conversationally) lives in ``tests/evals/``. """ -import asyncio - +import pytest from pydantic_ai import ToolDefinition from pydantic_ai.capabilities import Hooks from pydantic_ai.messages import ToolCallPart @@ -36,20 +35,18 @@ def test_payload_shape_is_stable_across_exception_types(self) -> None: class TestRecoverToolErrorHandler: - def test_returns_recovery_payload_for_unexpected_exception(self) -> None: + @pytest.mark.anyio + async def test_returns_recovery_payload_for_unexpected_exception(self) -> None: call = ToolCallPart(tool_name="search_tracks", tool_call_id="call-1") tool_def = ToolDefinition(name="search_tracks") - async def _invoke() -> dict[str, object]: - return await _recover_tool_error( - None, # type: ignore[arg-type] # handler doesn't touch ctx - call=call, - tool_def=tool_def, - args={}, - error=RuntimeError("spotify 500"), - ) - - result = asyncio.run(_invoke()) + result = await _recover_tool_error( + None, # type: ignore[arg-type] # handler doesn't touch ctx + call=call, + tool_def=tool_def, + args={}, + error=RuntimeError("spotify 500"), + ) assert result["error"] == "tool_failed" assert result["tool"] == "search_tracks" assert result["exception_type"] == "RuntimeError" diff --git a/backend/tests/test_retrieval_tool.py b/backend/tests/unit/agents/test_retrieval_tool.py similarity index 81% rename from backend/tests/test_retrieval_tool.py rename to backend/tests/unit/agents/test_retrieval_tool.py index 7e98789..8631a11 100644 --- a/backend/tests/test_retrieval_tool.py +++ b/backend/tests/unit/agents/test_retrieval_tool.py @@ -7,7 +7,6 @@ pydantic-ai convention (let hooks handle tool errors, don't swallow them). """ -import asyncio import json from unittest.mock import AsyncMock, MagicMock, patch @@ -23,8 +22,11 @@ def _make_ctx() -> MagicMock: return ctx +pytestmark = pytest.mark.anyio + + class TestRetrieveTracksFromBrainState: - def test_populated_index_returns_json_with_ranked_hits(self) -> None: + async def test_populated_index_returns_json_with_ranked_hits(self) -> None: hits = [ TrackHit( spotify_id="spid1", @@ -47,7 +49,7 @@ def test_populated_index_returns_json_with_ranked_hits(self) -> None: "cortexdj.agents.tools.retrieval_tools.retrieval_service.retrieve_similar_tracks", new=AsyncMock(return_value=hits), ): - payload_json = asyncio.run(retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=5)) + payload_json = await retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=5) payload = json.loads(payload_json) assert payload["session_id"] == "sess-1" @@ -59,12 +61,12 @@ def test_populated_index_returns_json_with_ranked_hits(self) -> None: # `note` is only set on the empty-index path; never leak it on success. assert "note" not in payload - def test_empty_index_returns_note_field(self) -> None: + async def test_empty_index_returns_note_field(self) -> None: with patch( "cortexdj.agents.tools.retrieval_tools.retrieval_service.retrieve_similar_tracks", new=AsyncMock(return_value=[]), ): - payload_json = asyncio.run(retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=10)) + payload_json = await retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=10) payload = json.loads(payload_json) assert payload["tracks"] == [] @@ -73,7 +75,7 @@ def test_empty_index_returns_note_field(self) -> None: # command instead of hallucinating a recovery. assert "seed-track-index" in payload["note"] - def test_lookup_error_propagates_to_hooks(self) -> None: + async def test_lookup_error_propagates_to_hooks(self) -> None: # Per .claude/rules/backend/pydantic-ai.md: tools let exceptions # propagate so on_tool_execute_error can produce a structured # recovery payload. The tool must NOT catch LookupError. @@ -82,9 +84,9 @@ def test_lookup_error_propagates_to_hooks(self) -> None: new=AsyncMock(side_effect=LookupError("session sess-bogus not found")), ): with pytest.raises(LookupError, match="sess-bogus"): - asyncio.run(retrieve_tracks_from_brain_state(_make_ctx(), "sess-bogus", k=5)) + await retrieve_tracks_from_brain_state(_make_ctx(), "sess-bogus", k=5) - def test_deap_file_missing_returns_structured_error(self) -> None: + async def test_deap_file_missing_returns_structured_error(self) -> None: # `DeapFileMissingError` is server misconfig — the hooks recovery # template strips the exception message, so we catch it specifically # and return an actionable JSON payload the agent can relay verbatim. @@ -92,7 +94,7 @@ def test_deap_file_missing_returns_structured_error(self) -> None: "cortexdj.agents.tools.retrieval_tools.retrieval_service.retrieve_similar_tracks", new=AsyncMock(side_effect=DeapFileMissingError("DEAP file for P99 not found at /x/s99.dat")), ): - payload_json = asyncio.run(retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=5)) + payload_json = await retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=5) payload = json.loads(payload_json) assert payload["error"] == "deap_data_missing" @@ -102,17 +104,17 @@ def test_deap_file_missing_returns_structured_error(self) -> None: # render an empty list as if retrieval succeeded. assert "tracks" not in payload - def test_passes_k_through_to_service(self) -> None: + async def test_passes_k_through_to_service(self) -> None: mock = AsyncMock(return_value=[]) with patch("cortexdj.agents.tools.retrieval_tools.retrieval_service.retrieve_similar_tracks", new=mock): - asyncio.run(retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=25)) + await retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=25) assert mock.call_args.kwargs["k"] == 25 - def test_negative_k_passes_through_unclamped_at_tool_layer(self) -> None: + async def test_negative_k_passes_through_unclamped_at_tool_layer(self) -> None: # Pins the "clamping lives in the service, not the tool" contract — # otherwise a well-meaning future refactor might add `max(1, k)` at # the tool layer and silently mask bugs in the service-side clamp. mock = AsyncMock(return_value=[]) with patch("cortexdj.agents.tools.retrieval_tools.retrieval_service.retrieve_similar_tracks", new=mock): - asyncio.run(retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=-1)) + await retrieve_tracks_from_brain_state(_make_ctx(), "sess-1", k=-1) assert mock.call_args.kwargs["k"] == -1 diff --git a/backend/tests/unit/ml/__init__.py b/backend/tests/unit/ml/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/test_contrastive.py b/backend/tests/unit/ml/test_contrastive.py similarity index 100% rename from backend/tests/test_contrastive.py rename to backend/tests/unit/ml/test_contrastive.py diff --git a/backend/tests/test_contrastive_dataset_paths.py b/backend/tests/unit/ml/test_contrastive_dataset_paths.py similarity index 100% rename from backend/tests/test_contrastive_dataset_paths.py rename to backend/tests/unit/ml/test_contrastive_dataset_paths.py diff --git a/backend/tests/test_dataset.py b/backend/tests/unit/ml/test_dataset.py similarity index 100% rename from backend/tests/test_dataset.py rename to backend/tests/unit/ml/test_dataset.py diff --git a/backend/tests/test_deap_dataset.py b/backend/tests/unit/ml/test_deap_dataset.py similarity index 100% rename from backend/tests/test_deap_dataset.py rename to backend/tests/unit/ml/test_deap_dataset.py diff --git a/backend/tests/test_majority_baseline.py b/backend/tests/unit/ml/test_majority_baseline.py similarity index 100% rename from backend/tests/test_majority_baseline.py rename to backend/tests/unit/ml/test_majority_baseline.py diff --git a/backend/tests/test_metrics.py b/backend/tests/unit/ml/test_metrics.py similarity index 100% rename from backend/tests/test_metrics.py rename to backend/tests/unit/ml/test_metrics.py diff --git a/backend/tests/test_preprocessing.py b/backend/tests/unit/ml/test_preprocessing.py similarity index 100% rename from backend/tests/test_preprocessing.py rename to backend/tests/unit/ml/test_preprocessing.py diff --git a/backend/tests/test_pretrained.py b/backend/tests/unit/ml/test_pretrained.py similarity index 100% rename from backend/tests/test_pretrained.py rename to backend/tests/unit/ml/test_pretrained.py diff --git a/backend/tests/test_resume_state.py b/backend/tests/unit/ml/test_resume_state.py similarity index 100% rename from backend/tests/test_resume_state.py rename to backend/tests/unit/ml/test_resume_state.py diff --git a/backend/tests/test_train_smoke.py b/backend/tests/unit/ml/test_train_smoke.py similarity index 100% rename from backend/tests/test_train_smoke.py rename to backend/tests/unit/ml/test_train_smoke.py diff --git a/backend/tests/unit/scripts/__init__.py b/backend/tests/unit/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/test_build_track_index.py b/backend/tests/unit/scripts/test_build_track_index.py similarity index 88% rename from backend/tests/test_build_track_index.py rename to backend/tests/unit/scripts/test_build_track_index.py index 5915c4c..fa666ea 100644 --- a/backend/tests/test_build_track_index.py +++ b/backend/tests/unit/scripts/test_build_track_index.py @@ -6,12 +6,15 @@ behavior so a future "optimization" can't silently regress it. """ -import asyncio from typing import Any from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from cortexdj.scripts.build_track_index import _dedupe_by_spotify_id, _gather_candidates +pytestmark = pytest.mark.anyio + def _candidate(spotify_id: str, source: str = "user_library") -> dict[str, Any]: return { @@ -45,7 +48,7 @@ def test_all_unique(self) -> None: class TestGatherCandidates: - def test_library_only_when_full(self) -> None: + async def test_library_only_when_full(self) -> None: # Library returns exactly `limit` candidates — topup should be skipped. client = MagicMock() with ( @@ -58,11 +61,11 @@ def test_library_only_when_full(self) -> None: new=AsyncMock(return_value=[]), ) as mock_seeds, ): - pool = asyncio.run(_gather_candidates(client, limit=10, skip_library=False)) + pool = await _gather_candidates(client, limit=10, skip_library=False) assert len(pool) == 10 mock_seeds.assert_not_called() - def test_library_shortfall_triggers_seed_topup(self) -> None: + async def test_library_shortfall_triggers_seed_topup(self) -> None: # Library returns 3, we need 10 — seeds must top up the remaining 7. # The corrected implementation overshoots with `shortfall * 2 = 14` # to absorb seed-side dedupe losses. @@ -77,7 +80,7 @@ def test_library_shortfall_triggers_seed_topup(self) -> None: new=AsyncMock(return_value=[_candidate(f"seed{i}", source="seed_search") for i in range(14)]), ) as mock_seeds, ): - pool = asyncio.run(_gather_candidates(client, limit=10, skip_library=False)) + pool = await _gather_candidates(client, limit=10, skip_library=False) assert len(pool) == 10 # Seeds should have been asked for shortfall * 2 = 14. mock_seeds.assert_called_once() @@ -85,7 +88,7 @@ def test_library_shortfall_triggers_seed_topup(self) -> None: # The 3 library candidates should be first (source preserved). assert [c["source"] for c in pool[:3]] == ["user_library"] * 3 - def test_library_and_seed_dedupe(self) -> None: + async def test_library_and_seed_dedupe(self) -> None: # Library returns 5 unique tracks; seeds return 5 that overlap with # library + 3 fresh ones. After dedupe the pool should contain # 5 library + 3 fresh seed = 8, sliced to limit=10 → all 8. @@ -102,7 +105,7 @@ def test_library_and_seed_dedupe(self) -> None: new=AsyncMock(return_value=[_candidate(i, source="seed_search") for i in seed_ids]), ), ): - pool = asyncio.run(_gather_candidates(client, limit=10, skip_library=False)) + pool = await _gather_candidates(client, limit=10, skip_library=False) spotify_ids = [c["spotify_id"] for c in pool] assert len(spotify_ids) == 8 # Library order preserved, fresh seeds appended. @@ -111,7 +114,7 @@ def test_library_and_seed_dedupe(self) -> None: # Dedupe kept the library copy, not the seed duplicate. assert all(c["source"] == "user_library" for c in pool[:5]) - def test_skip_library_uses_only_seeds(self) -> None: + async def test_skip_library_uses_only_seeds(self) -> None: client = MagicMock() with ( patch( @@ -122,12 +125,12 @@ def test_skip_library_uses_only_seeds(self) -> None: new=AsyncMock(return_value=[_candidate(f"seed{i}", source="seed_search") for i in range(20)]), ), ): - pool = asyncio.run(_gather_candidates(client, limit=10, skip_library=True)) + pool = await _gather_candidates(client, limit=10, skip_library=True) mock_lib.assert_not_called() assert len(pool) == 10 assert all(c["source"] == "seed_search" for c in pool) - def test_slice_to_limit(self) -> None: + async def test_slice_to_limit(self) -> None: # Library + seeds far exceed limit — final pool must be sliced down. client = MagicMock() with ( @@ -140,7 +143,7 @@ def test_slice_to_limit(self) -> None: new=AsyncMock(return_value=[_candidate(f"seed{i}", source="seed_search") for i in range(50)]), ) as mock_seeds, ): - pool = asyncio.run(_gather_candidates(client, limit=10, skip_library=False)) + pool = await _gather_candidates(client, limit=10, skip_library=False) # Library already fills the limit — seeds never fetched. assert len(pool) == 10 mock_seeds.assert_not_called() diff --git a/backend/tests/unit/services/__init__.py b/backend/tests/unit/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/test_audio_catalog.py b/backend/tests/unit/services/test_audio_catalog.py similarity index 100% rename from backend/tests/test_audio_catalog.py rename to backend/tests/unit/services/test_audio_catalog.py diff --git a/backend/tests/test_retrieval_service.py b/backend/tests/unit/services/test_retrieval_service.py similarity index 89% rename from backend/tests/test_retrieval_service.py rename to backend/tests/unit/services/test_retrieval_service.py index 9c3aee3..90edaff 100644 --- a/backend/tests/test_retrieval_service.py +++ b/backend/tests/unit/services/test_retrieval_service.py @@ -7,7 +7,6 @@ lookup failures, cosine→similarity conversion). """ -import asyncio from unittest.mock import AsyncMock, MagicMock, patch import numpy as np @@ -20,6 +19,8 @@ serialize_hits, ) +pytestmark = pytest.mark.anyio + class TestParticipantDatPath: def test_parses_standard_format(self) -> None: @@ -74,26 +75,26 @@ def test_handles_null_preview_url(self) -> None: class TestRetrieveSimilarTracks: - def test_missing_session_raises_lookup_error(self) -> None: + async def test_missing_session_raises_lookup_error(self) -> None: db = MagicMock() with patch("cortexdj.services.retrieval.Session.get", new=AsyncMock(return_value=None)): with pytest.raises(LookupError, match="sess-bogus"): - asyncio.run(retrieve_similar_tracks(db, "sess-bogus", k=10)) + await retrieve_similar_tracks(db, "sess-bogus", k=10) - def test_empty_index_returns_empty_list(self) -> None: + async def test_empty_index_returns_empty_list(self) -> None: db = MagicMock() with ( patch("cortexdj.services.retrieval.Session.get", new=AsyncMock(return_value=MagicMock())), patch("cortexdj.services.retrieval.TrackAudioEmbedding.count", new=AsyncMock(return_value=0)), patch("cortexdj.services.retrieval.encode_session_to_clap_space") as mock_encode, ): - hits = asyncio.run(retrieve_similar_tracks(db, "sess-1", k=10)) + hits = await retrieve_similar_tracks(db, "sess-1", k=10) assert hits == [] # Importantly, we did NOT call the encoder if the index is empty — # encoding is the expensive part and there's nothing to query against. mock_encode.assert_not_called() - def test_k_is_clamped_to_valid_range(self) -> None: + async def test_k_is_clamped_to_valid_range(self) -> None: db = MagicMock() mock_row = MagicMock( spotify_id="abc", @@ -113,11 +114,11 @@ def test_k_is_clamped_to_valid_range(self) -> None: new=AsyncMock(return_value=[(mock_row, 0.2)]), ) as mock_topk, ): - asyncio.run(retrieve_similar_tracks(db, "sess-1", k=500)) + await retrieve_similar_tracks(db, "sess-1", k=500) # k=500 should be clamped to 100. assert mock_topk.call_args.kwargs["k"] == 100 - def test_k_floor_is_one(self) -> None: + async def test_k_floor_is_one(self) -> None: db = MagicMock() with ( patch("cortexdj.services.retrieval.Session.get", new=AsyncMock(return_value=MagicMock())), @@ -131,10 +132,10 @@ def test_k_floor_is_one(self) -> None: new=AsyncMock(return_value=[]), ) as mock_topk, ): - asyncio.run(retrieve_similar_tracks(db, "sess-1", k=0)) + await retrieve_similar_tracks(db, "sess-1", k=0) assert mock_topk.call_args.kwargs["k"] == 1 - def test_cosine_distance_converts_to_similarity(self) -> None: + async def test_cosine_distance_converts_to_similarity(self) -> None: # pgvector returns cosine distance in [0, 2] (0 = identical, 1 = orthogonal, # 2 = antipodal). We convert to similarity = 1 - distance so callers get # the standard [-1, 1] cosine-similarity convention where higher is closer. @@ -156,6 +157,6 @@ def test_cosine_distance_converts_to_similarity(self) -> None: new=AsyncMock(return_value=rows), ), ): - hits = asyncio.run(retrieve_similar_tracks(db, "sess-1", k=3)) + hits = await retrieve_similar_tracks(db, "sess-1", k=3) similarities = [h.similarity for h in hits] assert similarities == [1.0, 0.5, 0.0] diff --git a/backend/tests/unit/services/test_title_generator.py b/backend/tests/unit/services/test_title_generator.py new file mode 100644 index 0000000..3ec7206 --- /dev/null +++ b/backend/tests/unit/services/test_title_generator.py @@ -0,0 +1,97 @@ +"""Unit tests for services.title_generator. + +The generator runs as a fire-and-forget background task on its own session +(outside the request's transaction), so it's covered here with a patched +OpenAI client and session context rather than at the integration tier. +""" + +import contextlib +from collections.abc import AsyncIterator +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from cortexdj.services.title_generator import _create_fallback_title, generate_thread_title + +pytestmark = pytest.mark.anyio + + +class TestCreateFallbackTitle: + def test_short_message_passes_through(self) -> None: + assert _create_fallback_title("Analyze my last session") == "Analyze my last session" + + def test_long_message_truncates_at_word_boundary(self) -> None: + message = "Please analyze my most recent relaxation session and compare it with last week" + title = _create_fallback_title(message) + assert title.endswith("...") + assert len(title) <= 43 # 40 chars + ellipsis + # No mid-word cut: everything before "..." is a whole-word prefix. + assert message.startswith(title.removesuffix("...")) + + def test_strips_surrounding_whitespace(self) -> None: + assert _create_fallback_title(" hi ") == "hi" + + +@contextlib.asynccontextmanager +async def _fake_session() -> AsyncIterator[MagicMock]: + yield MagicMock() + + +def _openai_client_returning(text: str) -> MagicMock: + client = MagicMock() + client.responses.create = AsyncMock(return_value=MagicMock(output_text=text)) + return client + + +class TestGenerateThreadTitle: + async def test_saves_llm_title_with_quotes_stripped(self) -> None: + with ( + patch( + "cortexdj.services.title_generator.AsyncOpenAI", + return_value=_openai_client_returning('"Neural Beats Recap"'), + ), + patch("cortexdj.services.title_generator.get_async_sqlalchemy_session", _fake_session), + patch("cortexdj.services.title_generator.Thread.update_title", new=AsyncMock()) as update_title, + ): + await generate_thread_title("t-1", "chat", "user msg", "assistant msg") + + assert update_title.call_args.args[1:] == ("t-1", "chat", "Neural Beats Recap") + + async def test_empty_llm_output_saves_nothing(self) -> None: + with ( + patch( + "cortexdj.services.title_generator.AsyncOpenAI", + return_value=_openai_client_returning(""), + ), + patch("cortexdj.services.title_generator.get_async_sqlalchemy_session", _fake_session), + patch("cortexdj.services.title_generator.Thread.update_title", new=AsyncMock()) as update_title, + ): + await generate_thread_title("t-1", "chat", "user msg", "assistant msg") + + update_title.assert_not_called() + + async def test_llm_failure_falls_back_to_truncated_user_message(self) -> None: + failing_client = MagicMock() + failing_client.responses.create = AsyncMock(side_effect=RuntimeError("api down")) + with ( + patch("cortexdj.services.title_generator.AsyncOpenAI", return_value=failing_client), + patch("cortexdj.services.title_generator.get_async_sqlalchemy_session", _fake_session), + patch("cortexdj.services.title_generator.Thread.update_title", new=AsyncMock()) as update_title, + ): + await generate_thread_title("t-1", "chat", "Compare my sessions", "resp") + + assert update_title.call_args.args[1:] == ("t-1", "chat", "Compare my sessions") + + async def test_fallback_save_failure_is_swallowed(self) -> None: + # A background task must never propagate — it has no request to fail. + failing_client = MagicMock() + failing_client.responses.create = AsyncMock(side_effect=RuntimeError("api down")) + with ( + patch("cortexdj.services.title_generator.AsyncOpenAI", return_value=failing_client), + patch("cortexdj.services.title_generator.get_async_sqlalchemy_session", _fake_session), + patch( + "cortexdj.services.title_generator.Thread.update_title", + new=AsyncMock(side_effect=RuntimeError("db down")), + ), + ): + await generate_thread_title("t-1", "chat", "user msg", "resp") diff --git a/backend/tests/test_trajectory.py b/backend/tests/unit/services/test_trajectory.py similarity index 100% rename from backend/tests/test_trajectory.py rename to backend/tests/unit/services/test_trajectory.py diff --git a/backend/tests/unit/utils/__init__.py b/backend/tests/unit/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/unit/utils/test_emotion.py b/backend/tests/unit/utils/test_emotion.py new file mode 100644 index 0000000..9b0814b --- /dev/null +++ b/backend/tests/unit/utils/test_emotion.py @@ -0,0 +1,27 @@ +"""Unit tests for utils.emotion quadrant lookups (agent-facing copy).""" + +from cortexdj.utils.emotion import ( + BRAIN_STATE_EXPLANATIONS, + QUADRANT_DESCRIPTIONS, + get_brain_state_explanation, + quadrant_to_mood_description, +) + + +class TestQuadrantToMoodDescription: + def test_known_quadrants(self) -> None: + for state in ("excited", "stressed", "relaxed", "calm"): + assert quadrant_to_mood_description(state) == QUADRANT_DESCRIPTIONS[state] + + def test_unknown_state_is_labeled_not_raised(self) -> None: + assert quadrant_to_mood_description("bored") == "Unknown state: bored" + + +class TestGetBrainStateExplanation: + def test_known_states_mention_a_frequency_band(self) -> None: + for state, explanation in BRAIN_STATE_EXPLANATIONS.items(): + assert get_brain_state_explanation(state) == explanation + assert any(band in explanation for band in ("alpha", "beta", "theta", "gamma")) + + def test_unknown_state_fallback(self) -> None: + assert get_brain_state_explanation("bored").startswith("No detailed explanation") diff --git a/backend/tests/unit/utils/test_message_serialization.py b/backend/tests/unit/utils/test_message_serialization.py new file mode 100644 index 0000000..c794fe4 --- /dev/null +++ b/backend/tests/unit/utils/test_message_serialization.py @@ -0,0 +1,54 @@ +"""Unit tests for utils.message_serialization round-trips.""" + +from pydantic_ai.messages import ModelRequest, ModelResponse, TextPart, UserPromptPart +from pydantic_ai.ui.vercel_ai.request_types import UIMessage + +from cortexdj.utils.message_serialization import ( + deserialize_messages, + dump_messages_for_frontend, + extract_latest_user_text, + prepare_messages_for_storage, +) + + +def _sample_messages() -> list[ModelRequest | ModelResponse]: + return [ + ModelRequest(parts=[UserPromptPart(content="How was my session?")]), + ModelResponse(parts=[TextPart(content="Mostly relaxed.")]), + ] + + +class TestStorageRoundTrip: + def test_dump_is_json_safe(self) -> None: + dumped = prepare_messages_for_storage(list(_sample_messages())) + assert all(isinstance(m, dict) for m in dumped) + + def test_deserialize_inverts_prepare(self) -> None: + original = list(_sample_messages()) + restored = deserialize_messages(prepare_messages_for_storage(original)) + assert len(restored) == 2 + assert isinstance(restored[0], ModelRequest) + assert isinstance(restored[1], ModelResponse) + assert restored[1].parts[0].content == "Mostly relaxed." # type: ignore[union-attr] + + +class TestDumpForFrontend: + def test_produces_ui_roles_and_text(self) -> None: + stored = prepare_messages_for_storage(list(_sample_messages())) + ui = dump_messages_for_frontend(stored) + assert [m["role"] for m in ui] == ["user", "assistant"] + assistant_parts = ui[1]["parts"] + assert any(p.get("type") == "text" and p.get("text") == "Mostly relaxed." for p in assistant_parts) + + +class TestExtractLatestUserText: + def test_returns_last_user_text(self) -> None: + messages = [ + UIMessage.model_validate({"id": "1", "role": "user", "parts": [{"type": "text", "text": "first"}]}), + UIMessage.model_validate({"id": "2", "role": "assistant", "parts": [{"type": "text", "text": "reply"}]}), + UIMessage.model_validate({"id": "3", "role": "user", "parts": [{"type": "text", "text": "second"}]}), + ] + assert extract_latest_user_text(messages) == "second" + + def test_empty_history_returns_empty_string(self) -> None: + assert extract_latest_user_text([]) == "" diff --git a/backend/uv.lock b/backend/uv.lock index 70caa72..a857f9f 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -526,6 +526,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "anyio" }, { name = "mypy" }, { name = "pre-commit" }, { name = "pydantic-evals" }, @@ -561,6 +562,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "anyio", specifier = ">=4.14.2" }, { name = "mypy", specifier = ">=2.3.0" }, { name = "pre-commit", specifier = ">=4.6.0" }, { name = "pydantic-evals", specifier = ">=2.11.0" }, From 5d997cafea8dd256768a2c93a79f4af1912aebac Mon Sep 17 00:00:00 2001 From: Luke Mainwaring Date: Thu, 16 Jul 2026 17:31:06 -0400 Subject: [PATCH 2/2] test: harden integration-tier guards per review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DB-name guard is now endswith("_test"), not a substring check — "cortexdj_latest" must never pass a guard that precedes downgrade base. - Guards fail loudly in CI (skip only locally) so a misconfigured tier can't silently deselect 28 tests while the job stays green. - _migrated starts pristine (downgrade base -> upgrade head) so empty-state assertions hold across runs; tier markers are applied structurally by path in the integration conftest instead of per-file pytestmark. - Move TestModel-backed prepare_tools tests from evals/ to unit/agents/ (they run in the default tier); agent-deps fakes move to tests/fakes.py. - Update the backend rules Testing section to describe the three tiers. Co-Authored-By: Claude Fable 5 --- .claude/rules/backend/code-conventions.md | 7 ++-- .github/workflows/ci.yml | 3 +- DEVELOPMENT.md | 2 +- backend/scripts/create-test-db.sh | 2 +- backend/tests/evals/test_brain_agent_evals.py | 2 +- backend/tests/{evals/conftest.py => fakes.py} | 9 +++-- backend/tests/integration/conftest.py | 33 ++++++++++++++++--- .../integration/models/test_models_crud.py | 2 -- .../integration/routers/test_retrieval_api.py | 3 -- .../integration/routers/test_sessions_api.py | 3 -- .../integration/routers/test_thread_api.py | 3 -- backend/tests/integration/test_health.py | 3 -- backend/tests/integration/test_migrations.py | 7 ++-- .../agents}/test_prepare_tools.py | 6 ++-- 14 files changed, 48 insertions(+), 37 deletions(-) rename backend/tests/{evals/conftest.py => fakes.py} (73%) rename backend/tests/{evals => unit/agents}/test_prepare_tools.py (96%) diff --git a/.claude/rules/backend/code-conventions.md b/.claude/rules/backend/code-conventions.md index 9c7ea71..1187b22 100644 --- a/.claude/rules/backend/code-conventions.md +++ b/.claude/rules/backend/code-conventions.md @@ -56,8 +56,11 @@ Python/FastAPI conventions for the CortexDJ backend. ## Testing -- Tests live flat under `backend/tests/` — fast, DB-free, provider-free. `conftest.py` sets a dummy `OPENAI_API_KEY` so importing `brain_agent` never needs a real key; tests that invoke the agent use `agent.override` with pydantic-ai `TestModel`. -- Real-LLM evals live in `tests/evals/`, marked `@pytest.mark.eval` and opt-in via `pytest -m eval`; the default run excludes them through `addopts = "-m 'not eval'"`. +- Three tiers under `backend/tests/`: `unit//` (fast, DB-free, provider-free — the default `pytest` run), `integration/` (real Postgres + HTTP, opt-in via `-m integration`), and `evals/` (real-LLM, `@pytest.mark.eval`, opt-in via `-m eval`). The default run excludes both opt-in tiers through `addopts = "-m 'not eval and not integration'"`; test dirs are packages so basenames can repeat across tiers. +- Async tests use anyio's pytest plugin: `@pytest.mark.anyio` plus the `anyio_backend` fixture in the root conftest. No pytest-asyncio, no raw `asyncio.run()` wrappers. +- The integration tier builds schema from the shipped Alembic migrations and rolls back an outer transaction per test. Isolation holds only because the app's sole commit point is the session dependency (`dependencies/db.py`) — services and model classmethods `flush()`, never `commit()`. The tier refuses database names not ending in `_test` and applies its markers structurally in `tests/integration/conftest.py` (don't add per-file `pytestmark`). Local setup: `backend/scripts/create-test-db.sh`, then `POSTGRES_DB=cortexdj_test uv run --directory backend pytest -m integration`. +- Shared builders live in `tests/factories.py` (`make_*` = unpersisted, `create_*` = persisted via flush); agent-deps mocks in `tests/fakes.py`. +- Root `conftest.py` sets a dummy `OPENAI_API_KEY` so importing `brain_agent` never needs a real key; tests that invoke the agent use `agent.override` with pydantic-ai `TestModel`. - CI's test job enforces the coverage floor; don't run coverage locally unless you're investigating a CI failure. ## Dependencies diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6ccede..c117bd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,8 @@ jobs: - name: Run tests + coverage working-directory: ./backend - # Coverage floor at 40% (baseline ~43%); ratchet up as coverage grows. + # Coverage floor at 40% (unit tier alone measures ~43%; combined with + # the integration tier, ~53%); ratchet up as coverage grows. # The explicit -m "not eval" OVERRIDES addopts' 'not integration' # (the last -m wins), which is what selects the integration tier # here — don't remove it as "redundant" or the tier silently diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 8969c8d..d07856d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -67,7 +67,7 @@ To run the integration tier locally, create the test database once, then point ` POSTGRES_DB=cortexdj_test uv run --directory backend pytest -m integration ``` -The tier refuses to run unless the database name contains `test`, because it runs `alembic upgrade`/`downgrade` against whatever database the env points at. Shared test-data builders live in `backend/tests/factories.py` (`make_*` = unpersisted, `create_*` = persisted via flush). +The tier refuses to run unless the database name ends with `_test`, because it runs `alembic upgrade`/`downgrade` against whatever database the env points at. Shared test-data builders live in `backend/tests/factories.py` (`make_*` = unpersisted, `create_*` = persisted via flush). ## Database migrations diff --git a/backend/scripts/create-test-db.sh b/backend/scripts/create-test-db.sh index eb16aad..efa266d 100755 --- a/backend/scripts/create-test-db.sh +++ b/backend/scripts/create-test-db.sh @@ -1,6 +1,6 @@ #!/bin/bash # Create the integration-test database (idempotent). The integration tier -# refuses to run unless POSTGRES_DB contains "test", because it runs +# refuses to run unless POSTGRES_DB ends with "_test", because it runs # `alembic upgrade`/`downgrade` against whatever database the env points at. set -euo pipefail diff --git a/backend/tests/evals/test_brain_agent_evals.py b/backend/tests/evals/test_brain_agent_evals.py index 5ebc708..1c1ca77 100644 --- a/backend/tests/evals/test_brain_agent_evals.py +++ b/backend/tests/evals/test_brain_agent_evals.py @@ -28,7 +28,7 @@ from pydantic_evals.evaluators import Evaluator, EvaluatorContext from cortexdj.agents.brain_agent import brain_agent -from tests.evals.conftest import fake_spotify_client, make_fake_deps +from tests.fakes import fake_spotify_client, make_fake_deps @dataclass diff --git a/backend/tests/evals/conftest.py b/backend/tests/fakes.py similarity index 73% rename from backend/tests/evals/conftest.py rename to backend/tests/fakes.py index f94c23a..ca137c3 100644 --- a/backend/tests/evals/conftest.py +++ b/backend/tests/fakes.py @@ -1,9 +1,8 @@ -"""Fixtures for brain_agent evals. +"""Fake dependency builders for agent tests (unit tier and evals). -Provides ``make_fake_deps`` — a constructor for ``AgentDeps`` instances -that don't hit the real database, Spotify, or EEG model. Used by both -the deterministic ``prepare_tools`` tests (TestModel-backed) and the -real-model ``@pytest.mark.eval`` tests. +``make_fake_deps`` constructs ``AgentDeps`` that never hit the real +database, Spotify, or EEG model. Kept separate from ``factories.py``: +these build mocks around agent wiring, not persistable ORM rows. """ from typing import Any diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py index b592563..1eb5ae6 100644 --- a/backend/tests/integration/conftest.py +++ b/backend/tests/integration/conftest.py @@ -20,6 +20,7 @@ ``POSTGRES_DB=cortexdj_test`` to override the ``.env`` database name. """ +import os import socket from collections.abc import AsyncGenerator @@ -44,6 +45,19 @@ _ALEMBIC_INI = "src/cortexdj/alembic.ini" # relative to backend/, where pytest runs +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Apply tier markers structurally, by path. + + A forgotten per-file ``pytestmark`` would otherwise promote a DB test + into the default (unit) tier — green in CI where Postgres exists, + confusing failures locally. + """ + for item in items: + if "tests/integration" in str(item.path): + item.add_marker(pytest.mark.integration) + item.add_marker(pytest.mark.anyio) + + def _db_reachable() -> bool: settings = get_settings() try: @@ -54,24 +68,35 @@ def _db_reachable() -> bool: def _is_test_db() -> bool: - return "test" in get_settings().POSTGRES_DB + # endswith, not substring: "cortexdj_latest" must not pass as a test DB — + # this tier runs `alembic downgrade base`, which drops every table. + return get_settings().POSTGRES_DB.endswith("_test") @pytest.fixture(scope="session", autouse=True) def _require_test_db() -> None: + # In CI a misconfigured tier must fail the job, not skip silently — + # the coverage floor alone wouldn't catch 28 skipped tests. + in_ci = bool(os.environ.get("CI")) if not _db_reachable(): - pytest.skip("integration tier needs Postgres (`docker compose up -d postgres`)") + message = "integration tier needs Postgres (`docker compose up -d postgres`)" + pytest.fail(f"{message} — refusing to skip in CI") if in_ci else pytest.skip(message) if not _is_test_db(): - pytest.skip( + message = ( "refusing to run against a non-test database — this tier runs alembic " "upgrade/downgrade; set POSTGRES_DB=cortexdj_test " "(see backend/scripts/create-test-db.sh)" ) + pytest.fail(message) if in_ci else pytest.skip(message) @pytest.fixture(scope="session") def _migrated(_require_test_db: None) -> None: - command.upgrade(Config(_ALEMBIC_INI), "head") + cfg = Config(_ALEMBIC_INI) + # Start pristine: wipes anything a stray seed run left in the test DB, so + # empty-state assertions (`total == 0`) hold across runs, not just within one. + command.downgrade(cfg, "base") + command.upgrade(cfg, "head") @pytest.fixture diff --git a/backend/tests/integration/models/test_models_crud.py b/backend/tests/integration/models/test_models_crud.py index 02e479a..9a28cb8 100644 --- a/backend/tests/integration/models/test_models_crud.py +++ b/backend/tests/integration/models/test_models_crud.py @@ -21,8 +21,6 @@ from cortexdj.schemas.thread import BrainContext from tests.factories import create_session, create_track_audio_embedding -pytestmark = [pytest.mark.integration, pytest.mark.anyio] - _CHAT = AgentType.CHAT.value diff --git a/backend/tests/integration/routers/test_retrieval_api.py b/backend/tests/integration/routers/test_retrieval_api.py index 33a7888..cb17032 100644 --- a/backend/tests/integration/routers/test_retrieval_api.py +++ b/backend/tests/integration/routers/test_retrieval_api.py @@ -7,11 +7,8 @@ ``tests/unit/agents/test_retrieval_tool.py``. """ -import pytest from httpx import AsyncClient -pytestmark = [pytest.mark.integration, pytest.mark.anyio] - async def test_similar_tracks_for_unknown_session_returns_404(client: AsyncClient) -> None: response = await client.get("/api/sessions/no-such-session/similar-tracks") diff --git a/backend/tests/integration/routers/test_sessions_api.py b/backend/tests/integration/routers/test_sessions_api.py index e7c6d29..2b855e2 100644 --- a/backend/tests/integration/routers/test_sessions_api.py +++ b/backend/tests/integration/routers/test_sessions_api.py @@ -2,14 +2,11 @@ from datetime import UTC, datetime -import pytest from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession from tests.factories import create_eeg_segment, create_session -pytestmark = [pytest.mark.integration, pytest.mark.anyio] - async def test_list_sessions_empty(client: AsyncClient) -> None: response = await client.get("/api/sessions") diff --git a/backend/tests/integration/routers/test_thread_api.py b/backend/tests/integration/routers/test_thread_api.py index 6332dcd..f8adc7d 100644 --- a/backend/tests/integration/routers/test_thread_api.py +++ b/backend/tests/integration/routers/test_thread_api.py @@ -1,6 +1,5 @@ """Route-level tests for the thread endpoints, asserted through HTTP.""" -import pytest from httpx import AsyncClient from pydantic_ai.messages import ModelRequest, ModelResponse, TextPart, UserPromptPart from sqlalchemy.ext.asyncio import AsyncSession @@ -10,8 +9,6 @@ from cortexdj.utils.message_serialization import prepare_messages_for_storage from tests.factories import create_thread -pytestmark = [pytest.mark.integration, pytest.mark.anyio] - _CHAT = AgentType.CHAT.value diff --git a/backend/tests/integration/test_health.py b/backend/tests/integration/test_health.py index da63504..034501e 100644 --- a/backend/tests/integration/test_health.py +++ b/backend/tests/integration/test_health.py @@ -1,8 +1,5 @@ -import pytest from httpx import AsyncClient -pytestmark = [pytest.mark.integration, pytest.mark.anyio] - async def test_db_health_check(client: AsyncClient) -> None: # Exercises the raw psycopg dependency against the real test database. diff --git a/backend/tests/integration/test_migrations.py b/backend/tests/integration/test_migrations.py index 1b40b9d..3f0ca3d 100644 --- a/backend/tests/integration/test_migrations.py +++ b/backend/tests/integration/test_migrations.py @@ -1,16 +1,13 @@ """Prove the shipped migrations round-trip cleanly (downgrade base → head). Safe only because the integration tier refuses to run against a database -whose name lacks "test" (see conftest ``_require_test_db``). +whose name doesn't end with "_test" (see conftest ``_require_test_db``). """ -import pytest from alembic import command from alembic.config import Config -pytestmark = pytest.mark.integration - -_ALEMBIC_INI = "src/cortexdj/alembic.ini" +from tests.integration.conftest import _ALEMBIC_INI def test_migrations_round_trip(_migrated: None) -> None: diff --git a/backend/tests/evals/test_prepare_tools.py b/backend/tests/unit/agents/test_prepare_tools.py similarity index 96% rename from backend/tests/evals/test_prepare_tools.py rename to backend/tests/unit/agents/test_prepare_tools.py index 3721668..06c617c 100644 --- a/backend/tests/evals/test_prepare_tools.py +++ b/backend/tests/unit/agents/test_prepare_tools.py @@ -18,7 +18,7 @@ from cortexdj.agents.brain_agent import brain_agent from cortexdj.ml.predict import EEGModel -from tests.evals.conftest import make_fake_deps +from tests.fakes import make_fake_deps def _offered_tool_names(model: TestModel) -> set[str]: @@ -72,7 +72,7 @@ async def test_eeg_tools_always_available_regardless_of_spotify(self) -> None: assert "build_mood_playlist" in offered async def test_shows_user_spotify_tools_when_connected(self) -> None: - from tests.evals.conftest import fake_spotify_client + from tests.fakes import fake_spotify_client model = await _run_agent_with_test_model(spotify_client=fake_spotify_client(), eeg_model=None) offered = _offered_tool_names(model) @@ -95,7 +95,7 @@ async def test_set_brain_context_always_available(self) -> None: assert "set_brain_context" in offered async def test_shows_model_tools_when_eeg_model_loaded(self) -> None: - from tests.evals.conftest import fake_eeg_model + from tests.fakes import fake_eeg_model model = await _run_agent_with_test_model(spotify_client=None, eeg_model=fake_eeg_model()) offered = _offered_tool_names(model)