diff --git a/README.md b/README.md index 1214d1f0..ae52a606 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ make migrate make dev ``` -Hit `http://localhost:8000` — you land on the public page. `/users/login` is the email+password login, `/dashboard` is the authenticated home, `/products` is a fully-working example module. +Hit `http://localhost:8000` — you land on the public page. `/users/login` is the email+password login, `/dashboard` is the authenticated home. ## Create a new module @@ -73,7 +73,7 @@ framework/ core/ # module system, discovery, events, diagnostics db/ # per-module Base, session, mixins, listeners hosting/ # app_builder, middleware, settings, Inertia glue -modules/ # plugin modules (auth, dashboard, products, ...) +modules/ # plugin modules (auth, dashboard, users, settings, ...) host/ main.py # FastAPI entry point routes.py # host-level routes (landing page) diff --git a/docs/e2e-testing.md b/docs/e2e-testing.md index f17bdb55..a2de1aef 100644 --- a/docs/e2e-testing.md +++ b/docs/e2e-testing.md @@ -1,22 +1,9 @@ # End-to-End Testing -The repo ships Playwright-driven smoke tests at -[tests/e2e/test_smoke.py](../tests/e2e/test_smoke.py). Four tests drive a real -Chromium browser through the core flows: - -* **`test_login_and_browse_smoke`** — landing → local email+password login → - dashboard → products browse → logout. Minimal regression guard. -* **`test_products_crud_smoke`** — same login + a full create / edit / delete - round-trip against the products module. -* **`test_password_reset_smoke`** — **skipped** (see inline comment in the - test file). `fastapi-users` `reset_password()` validates a password - fingerprint (`password_fgpt`) that is only available server-side. The - full HTTP-layer flow is covered by unit tests in - `modules/users/tests/test_api_auth.py`. -* **`test_admin_invite_smoke`** — admin invites a new user via the UI; the - invitee accepts the invite in a fresh browser context and lands on the - dashboard. Token is minted locally using the dev-default verify secret - (equivalent to what the ConsoleMailer logs). +Playwright-driven smoke tests live in [tests/e2e/](../tests/e2e/) — currently +just [`test_settings_ui.py`](../tests/e2e/test_settings_ui.py), which logs in, +navigates to `/settings/modules`, toggles a module setting, and verifies the +change hot-reloads into `app.state` without a server restart. End-to-end tests are gated behind the `e2e` pytest marker (declared in [pyproject.toml](../pyproject.toml)) and are **excluded from the default @@ -62,7 +49,7 @@ uv run pytest -m e2e tests/e2e ## Configuration -The tests read these environment variables (all optional): +When you write e2e tests, read these environment variables (all optional): | Variable | Default | Notes | | -------------- | ------------------------- | ------------------------------------------------------------ | @@ -71,38 +58,6 @@ The tests read these environment variables (all optional): | `E2E_PASSWORD` | `admin` | Password of the above admin user. | | `SM_USERS_VERIFICATION_TOKEN_SECRET` | `dev-verify-token-secret-change-me` | Must match the running server's value so locally-minted invite tokens are accepted. | -## What the smoke tests cover - -**`test_login_and_browse_smoke`** - -1. Landing page renders (`/`) with the "Get Started" CTA. -2. Local email+password login via `/users/login`. -3. Dashboard (`/dashboard/`) renders — proves session cookie + AuthMiddleware + - Inertia resolver + AuthenticatedLayout. -4. Products browse (`/products/`) renders — proves module pages resolve. -5. Logout returns the user to the public landing page. - -**`test_products_crud_smoke`** - -1. Login as admin. -2. Create a timestamped product via the Create form. -3. Edit its name and verify the new name appears in the list. -4. Delete it through the confirm dialog and verify the row disappears. - -The CRUD test relies on the admin user having the `admin` role (created -automatically by `sm-users create-admin` or the bootstrap env vars). - -**`test_admin_invite_smoke`** - -1. Admin logs in and submits the invite form at `/users/admin/invite`. -2. The test looks up the new user's UUID via the admin API. -3. A verify token is minted locally (same secret the server uses). -4. A fresh browser context navigates to `/users/invite/accept?token=…`, sets - a password, and verifies a redirect to `/dashboard`. - -These are **not** pixel-perfect regression tests — the goal is to catch broad -breakage in the auth + render + CRUD spine. - ## Debugging To see what the browser is doing, run headed with the Playwright trace diff --git a/docs/guide/project-structure.md b/docs/guide/project-structure.md index 9dbbdf6d..a13b9d55 100644 --- a/docs/guide/project-structure.md +++ b/docs/guide/project-structure.md @@ -12,11 +12,9 @@ simple_module_python/ │ ├── auth/ # session cookie, CSRF defences │ ├── background_tasks/ # Celery broker + worker integration │ ├── dashboard/ # authenticated landing page -│ ├── datasets/ # CSV / dataset uploads │ ├── feature_flags/ # admin UI for flag toggles │ ├── file_storage/ # pluggable storage backends (local, S3) │ ├── permissions/ # role/permission admin UI -│ ├── products/ # reference CRUD module (used in examples) │ ├── settings/ # DB-backed module settings + admin UI │ └── users/ # email+password auth, invites, bootstrap │ diff --git a/docs/guide/quickstart.md b/docs/guide/quickstart.md index d9c483ce..9e790cfb 100644 --- a/docs/guide/quickstart.md +++ b/docs/guide/quickstart.md @@ -29,7 +29,7 @@ The API and Vite dev servers start side by side. Visit: - `http://localhost:8000` — landing page - `http://localhost:8000/users/login` — sign-in screen -- `http://localhost:8000/products` — a fully-working example module (CRUD on a `products` table) +- `http://localhost:8000/dashboard` — the authenticated home (log in first) - `http://localhost:8000/settings/modules` — the admin settings UI (log in first) ## 4. Create an admin diff --git a/docs/release.md b/docs/release.md index e2777c42..56b98cf6 100644 --- a/docs/release.md +++ b/docs/release.md @@ -45,11 +45,9 @@ simple_module_test simple_module_auth simple_module_background_tasks simple_module_dashboard -simple_module_datasets simple_module_feature_flags simple_module_file_storage simple_module_permissions -simple_module_products simple_module_settings simple_module_users ``` @@ -223,11 +221,9 @@ Trusted Publishing is tied to the GitHub repo, not any personal account — so a | PyPI | `simple_module_auth` | [modules/auth/](../modules/auth/) | | PyPI | `simple_module_background_tasks` | [modules/background_tasks/](../modules/background_tasks/) | | PyPI | `simple_module_dashboard` | [modules/dashboard/](../modules/dashboard/) | -| PyPI | `simple_module_datasets` | [modules/datasets/](../modules/datasets/) | | PyPI | `simple_module_feature_flags` | [modules/feature_flags/](../modules/feature_flags/) | | PyPI | `simple_module_file_storage` | [modules/file_storage/](../modules/file_storage/) | | PyPI | `simple_module_permissions` | [modules/permissions/](../modules/permissions/) | -| PyPI | `simple_module_products` | [modules/products/](../modules/products/) — reference CRUD example | | PyPI | `simple_module_settings` | [modules/settings/](../modules/settings/) | | PyPI | `simple_module_users` | [modules/users/](../modules/users/) | | npm | `@simple-module-py/ui` | [packages/ui/](../packages/ui/) | diff --git a/framework/cli/simple_module_cli/catalog.py b/framework/cli/simple_module_cli/catalog.py index a2ef7d9a..43be0ba8 100644 --- a/framework/cli/simple_module_cli/catalog.py +++ b/framework/cli/simple_module_cli/catalog.py @@ -36,12 +36,11 @@ class ModuleEntry: "Permissions", requires=("auth", "users"), ), - "products": ModuleEntry("products", "simple_module_products", "Products"), "dashboard": ModuleEntry( "dashboard", "simple_module_dashboard", "Dashboard", - requires=("users", "products"), + requires=("users",), ), "settings": ModuleEntry("settings", "simple_module_settings", "Settings"), "feature_flags": ModuleEntry("feature_flags", "simple_module_feature_flags", "Feature Flags"), @@ -58,26 +57,13 @@ class ModuleEntry: requires=("users",), recipe="background_tasks", ), - "datasets": ModuleEntry( - "datasets", - "simple_module_datasets", - "Datasets", - requires=("file_storage", "background_tasks"), - ), } -# Example modules — `datasets` and `products` are intentionally excluded -# from every default preset because their module names collide with custom -# modules users typically want to register themselves. Pass them via -# `--with datasets,products` (or pick the `examples` preset) to opt in. -_EXAMPLE_MODULES: frozenset[str] = frozenset({"datasets", "products"}) - PRESETS: dict[str, tuple[str, ...]] = { "minimal": ("users",), "standard": ("users", "dashboard", "permissions"), - "full": tuple(name for name in CATALOG if name not in _EXAMPLE_MODULES), - "examples": tuple(CATALOG), + "full": tuple(CATALOG), } diff --git a/framework/cli/simple_module_cli/cli.py b/framework/cli/simple_module_cli/cli.py index 54272fbb..83d6ebea 100644 --- a/framework/cli/simple_module_cli/cli.py +++ b/framework/cli/simple_module_cli/cli.py @@ -45,7 +45,7 @@ def create_host( str, typer.Option( "--with", - help="Comma-separated module names to declare as deps (e.g. Auth,Products).", + help="Comma-separated module names to declare as deps (e.g. Auth,Dashboard).", ), ] = "", ) -> None: diff --git a/framework/cli/simple_module_cli/new.py b/framework/cli/simple_module_cli/new.py index 0423d533..c9295e92 100644 --- a/framework/cli/simple_module_cli/new.py +++ b/framework/cli/simple_module_cli/new.py @@ -25,7 +25,6 @@ class Preset(StrEnum): minimal = "minimal" standard = "standard" full = "full" - examples = "examples" def new_project( diff --git a/framework/cli/tests/test_cli_catalog.py b/framework/cli/tests/test_cli_catalog.py index dd8c0188..f3eb528f 100644 --- a/framework/cli/tests/test_cli_catalog.py +++ b/framework/cli/tests/test_cli_catalog.py @@ -41,17 +41,10 @@ def test_expand_deps_pulls_in_transitive_dep() -> None: def test_expand_deps_pulls_in_chain() -> None: - resolved, added = expand_deps(["datasets"]) - assert set(resolved) == { - "datasets", - "file_storage", - "settings", - "background_tasks", - "users", - "auth", - } + resolved, added = expand_deps(["permissions"]) + assert set(resolved) == {"permissions", "users", "auth"} added_names = {a for a, _ in added} - assert added_names == {"file_storage", "settings", "background_tasks", "users", "auth"} + assert added_names == {"users", "auth"} def test_expand_deps_idempotent_when_input_already_complete() -> None: diff --git a/framework/cli/tests/test_cli_wizard.py b/framework/cli/tests/test_cli_wizard.py index 41f78099..5da208fd 100644 --- a/framework/cli/tests/test_cli_wizard.py +++ b/framework/cli/tests/test_cli_wizard.py @@ -48,12 +48,11 @@ def test_wizard_minimal_preset() -> None: def test_wizard_full_preset_includes_background_tasks() -> None: _, _, selected, _ = _drive(["", "", "3", ""]) assert "background_tasks" in selected - assert "datasets" not in selected assert len(selected) >= 7 def test_wizard_custom_picks_only_yes_answers() -> None: - answers = ["", "", "4"] + ["n"] * 8 + ["y", "n", ""] + answers = ["", "", "4"] + ["n"] * 7 + ["y", ""] _, _, selected, out = _drive(answers) assert set(selected) == {"background_tasks", "users", "auth"} assert "Added users (required by background_tasks)" in out diff --git a/framework/cli/tests/test_scaffolding_host.py b/framework/cli/tests/test_scaffolding_host.py index c256f70e..60e35150 100644 --- a/framework/cli/tests/test_scaffolding_host.py +++ b/framework/cli/tests/test_scaffolding_host.py @@ -16,8 +16,8 @@ async def test_compute_returns_existing_page_dirs(self): modules = discover_modules() result = compute_module_pages(modules) - # Products + Dashboard ship pages/; Auth is API-only (no frontend pages). - assert {"Products", "Dashboard"}.issubset(result.keys()) + # Dashboard ships pages/; Auth is API-only (no frontend pages). + assert "Dashboard" in result assert "Auth" not in result for name, path in result.items(): assert path.is_dir(), f"{name} -> {path} should exist" @@ -53,12 +53,12 @@ async def test_write_manifest_emits_json_and_ts(self, tmp_path): assert written == {"manifest": manifest, "generated": generated, "css": css} data = json.loads(manifest.read_text(encoding="utf-8")) - assert "Products" in data - assert data["Products"].endswith("pages") or data["Products"].endswith("pages/") + assert "Dashboard" in data + assert data["Dashboard"].endswith("pages") or data["Dashboard"].endswith("pages/") ts = generated.read_text(encoding="utf-8") assert "import.meta.glob" in ts - assert "Products" in ts + assert "Dashboard" in ts assert "AUTO-GENERATED" in ts or "auto-generated" in ts.lower() # Glob patterns must be relative to output_dir — Vite treats # leading-slash paths as project-root-relative and silently matches @@ -78,7 +78,7 @@ async def test_creates_expected_backend_files(self, tmp_path): from simple_module_cli.scaffolding import create_host dest = tmp_path / "demo" - create_host(dest, name="demo-host", modules=["Products", "Auth"]) + create_host(dest, name="demo-host", modules=["Dashboard", "Auth"]) for relpath in [ "pyproject.toml", @@ -126,9 +126,9 @@ async def test_declares_selected_module_deps(self, tmp_path): from simple_module_cli.scaffolding import create_host dest = tmp_path / "demo" - create_host(dest, name="demo", modules=["Products", "Auth"]) + create_host(dest, name="demo", modules=["Dashboard", "Auth"]) pyproject = (dest / "pyproject.toml").read_text(encoding="utf-8") - assert "simple_module_products" in pyproject + assert "simple_module_dashboard" in pyproject assert "simple_module_auth" in pyproject async def test_refuses_existing_non_empty_dir(self, tmp_path): @@ -161,11 +161,11 @@ async def test_cli_create_host_runs_end_to_end(self, tmp_path): runner = CliRunner() result = runner.invoke( app, - ["create-host", "smoke-host", "--dest", str(tmp_path / "out"), "--with", "Products"], + ["create-host", "smoke-host", "--dest", str(tmp_path / "out"), "--with", "Dashboard"], ) assert result.exit_code == 0, result.output assert (tmp_path / "out" / "main.py").is_file() assert (tmp_path / "out" / "pyproject.toml").is_file() - assert "simple_module_products" in (tmp_path / "out" / "pyproject.toml").read_text( + assert "simple_module_dashboard" in (tmp_path / "out" / "pyproject.toml").read_text( encoding="utf-8" ) diff --git a/framework/core/tests/test_discovery.py b/framework/core/tests/test_discovery.py index 30403d1d..8f73c6d9 100644 --- a/framework/core/tests/test_discovery.py +++ b/framework/core/tests/test_discovery.py @@ -115,9 +115,9 @@ async def test_discover_finds_installed_modules(self): """discover_modules() should find modules registered via entry_points.""" modules = discover_modules() names = [m.meta.name for m in modules] - assert "Products" in names assert "Auth" in names assert "Dashboard" in names + assert "Users" in names class TestDiscoverModulesAdvanced: @@ -221,7 +221,7 @@ async def test_discover_with_none_loads_all(self): """Passing enabled=None keeps existing behaviour (load all installed modules).""" all_mods = discover_modules(enabled=None) names = {m.meta.name for m in all_mods} - assert {"Auth", "Products", "Dashboard"}.issubset(names) + assert {"Auth", "Users", "Dashboard"}.issubset(names) async def test_discover_with_allowlist_filters(self): """Passing enabled=['Auth'] loads only Auth, even if other modules are installed.""" @@ -234,9 +234,9 @@ async def test_discover_with_empty_list_loads_none(self): assert discover_modules(enabled=[]) == [] async def test_discover_allowlist_case_insensitive(self): - """Allowlist matching ignores case so 'products' and 'Products' both work.""" - names = [m.meta.name for m in discover_modules(enabled=["products"])] - assert names == ["Products"] + """Allowlist matching ignores case so 'dashboard' and 'Dashboard' both work.""" + names = [m.meta.name for m in discover_modules(enabled=["dashboard"])] + assert names == ["Dashboard"] async def test_discover_unknown_name_logged_and_ignored(self, caplog): """Names in enabled that don't match any installed module log a warning but don't raise.""" diff --git a/framework/db/tests/test_db_logging.py b/framework/db/tests/test_db_logging.py index 901e9098..ad1a4916 100644 --- a/framework/db/tests/test_db_logging.py +++ b/framework/db/tests/test_db_logging.py @@ -4,12 +4,10 @@ import contextlib import logging -from decimal import Decimal from unittest.mock import MagicMock from _models import _TenantBase, _TenantItem from simple_module_db.deps import get_db -from sqlalchemy.ext.asyncio import AsyncSession async def _drive_get_db(db_state, populate=None): @@ -82,47 +80,3 @@ async def test_read_only_skips_commit(self, db_state, caplog): read_only = [r for r in records if r.message == "db.session.read_only"] assert len(read_only) == 1 assert read_only[0].operation == "read_only_rollback" # type: ignore[attr-defined] - - -class TestEntityListenerLogging: - async def test_create_logs_entity_created(self, db_session: AsyncSession, caplog): - """Inserting a new entity should log db.entity.created.""" - from products.models import Product - - with caplog.at_level(logging.INFO, logger="simple_module.db"): - product = Product(name="Widget", price=Decimal("9.99")) - db_session.add(product) - await db_session.flush() - - created_msgs = [ - r - for r in caplog.records - if r.name == "simple_module.db" and r.message == "db.entity.created" - ] - assert len(created_msgs) == 1 - assert created_msgs[0].entity == "Product" # type: ignore[attr-defined] - assert created_msgs[0].operation == "create" # type: ignore[attr-defined] - - async def test_update_logs_entity_updated(self, db_session: AsyncSession, caplog): - """Modifying an entity should log db.entity.updated.""" - from products.models import Product - - product = Product(name="Widget", price=Decimal("9.99")) - db_session.add(product) - await db_session.flush() - - caplog.clear() - - product.name = "Updated Widget" - with caplog.at_level(logging.INFO, logger="simple_module.db"): - await db_session.flush() - - updated_msgs = [ - r - for r in caplog.records - if r.name == "simple_module.db" and r.message == "db.entity.updated" - ] - assert len(updated_msgs) == 1 - assert updated_msgs[0].entity == "Product" # type: ignore[attr-defined] - assert updated_msgs[0].operation == "update" # type: ignore[attr-defined] - assert updated_msgs[0].entity_id is not None # type: ignore[attr-defined] diff --git a/framework/db/tests/test_migrations.py b/framework/db/tests/test_migrations.py index 1295fdc0..d06753bf 100644 --- a/framework/db/tests/test_migrations.py +++ b/framework/db/tests/test_migrations.py @@ -17,10 +17,10 @@ async def test_combined_metadata_includes_installed_module_tables(self): metadata = build_module_metadata() table_names = set(metadata.tables.keys()) - # Products ships models and must contribute at least one table. + # Users ships models and must contribute at least one table. # (Dashboard is event-driven with no models; Auth's tables are # currently not part of this workspace's ORM surface.) - assert any("product" in name.lower() for name in table_names) + assert any("user" in name.lower() for name in table_names) assert len(table_names) >= 1 async def test_combined_metadata_only_returns_module_tables(self): diff --git a/framework/hosting/tests/test_app.py b/framework/hosting/tests/test_app.py index 33fe57cc..f9867e99 100644 --- a/framework/hosting/tests/test_app.py +++ b/framework/hosting/tests/test_app.py @@ -2,8 +2,6 @@ from __future__ import annotations -from collections import defaultdict - import httpx import pytest from fastapi import FastAPI @@ -29,12 +27,11 @@ async def test_app_state_has_registries(self, app: FastAPI): async def test_modules_enabled_limits_loaded_modules(self, settings: Settings): """Host respects settings.modules_enabled — only listed modules contribute routes.""" - # Only Auth should be loaded; Products + Dashboard routes must be absent. + # Only Auth should be loaded; Dashboard routes must be absent. restricted = settings.model_copy(update={"modules_enabled": ["Auth"]}) app = create_app(restricted) paths: set[str] = {str(r.path) for r in app.routes if hasattr(r, "path")} # Auth is now contracts-only, so it has no routes — only health remains. - assert not any(p.startswith("/api/products") for p in paths) assert "/dashboard" not in paths async def test_module_static_mounts_become_app_routes( @@ -117,9 +114,6 @@ async def test_expected_routes_registered(self, app: FastAPI): assert "/health/live" in route_paths assert "/health/ready" in route_paths - assert "/api/products/" in route_paths - assert "/api/products/{product_id}" in route_paths - # Users module owns login, register, etc. Auth module is contracts-only. assert "/users/login" in route_paths @@ -130,19 +124,6 @@ async def test_expected_routes_registered(self, app: FastAPI): # Bare-prefix alias — see wire_module_routes for the X-Inertia rationale. assert "/dashboard" in route_paths - async def test_products_api_methods(self, app: FastAPI): - """Products endpoints should support the correct HTTP methods.""" - routes_by_path: dict[str, set[str]] = defaultdict(set) - for route in app.routes: - if hasattr(route, "path") and hasattr(route, "methods"): - routes_by_path[route.path].update(route.methods) - - assert "GET" in routes_by_path.get("/api/products/", set()) - assert "POST" in routes_by_path.get("/api/products/", set()) - assert "GET" in routes_by_path.get("/api/products/{product_id}", set()) - assert "PUT" in routes_by_path.get("/api/products/{product_id}", set()) - assert "DELETE" in routes_by_path.get("/api/products/{product_id}", set()) - class TestProtectedPages: async def test_dashboard_redirects_unauthenticated(self, client: httpx.AsyncClient): @@ -150,11 +131,6 @@ async def test_dashboard_redirects_unauthenticated(self, client: httpx.AsyncClie assert resp.status_code == 302 assert "/users/login" in resp.headers["location"] - async def test_products_page_redirects_unauthenticated(self, client: httpx.AsyncClient): - resp = await client.get("/products/", follow_redirects=False) - assert resp.status_code == 302 - assert "/users/login" in resp.headers["location"] - class TestSecurityHeaders: async def test_security_headers_present(self, client: httpx.AsyncClient): diff --git a/host/migrations/versions/.gitkeep b/host/migrations/versions/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/host/migrations/versions/1fe7590fc594_add_settings_tables.py b/host/migrations/versions/1fe7590fc594_add_settings_tables.py deleted file mode 100644 index a0502504..00000000 --- a/host/migrations/versions/1fe7590fc594_add_settings_tables.py +++ /dev/null @@ -1,81 +0,0 @@ -"""add settings tables - -Revision ID: 1fe7590fc594 -Revises: b7e1af4c9d02 -Create Date: 2026-04-19 07:46:59.090748 -""" - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "1fe7590fc594" -down_revision: str | None = "b7e1af4c9d02" -branch_labels: str | Sequence[str] | None = ("settings",) -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # On PostgreSQL, create the `settings` schema before creating tables. - if op.get_context().dialect.name == "postgresql": - op.execute("CREATE SCHEMA IF NOT EXISTS settings") - - op.create_table( - "settings_setting", - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.Column("id", sa.Integer(), nullable=False), - sa.Column( - "scope", - sa.String(length=10), - nullable=False, - server_default=sa.text("'system'"), - ), - sa.Column( - "scope_id", - sa.String(length=255), - nullable=False, - server_default=sa.text("''"), - ), - sa.Column("key", sa.String(length=200), nullable=False), - sa.Column("value", sa.String(length=4000), nullable=False), - sa.Column( - "value_type", - sa.String(length=10), - nullable=False, - server_default=sa.text("'string'"), - ), - sa.Column("description", sa.String(length=2000), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_settings_setting")), - sa.UniqueConstraint( - "scope", "scope_id", "key", name="uq_settings_setting_scope_scope_id_key" - ), - ) - op.create_index(op.f("ix_settings_setting_scope"), "settings_setting", ["scope"], unique=False) - op.create_index( - op.f("ix_settings_setting_scope_id"), - "settings_setting", - ["scope_id"], - unique=False, - ) - op.create_index(op.f("ix_settings_setting_key"), "settings_setting", ["key"], unique=False) - - -def downgrade() -> None: - op.drop_index(op.f("ix_settings_setting_key"), table_name="settings_setting") - op.drop_index(op.f("ix_settings_setting_scope_id"), table_name="settings_setting") - op.drop_index(op.f("ix_settings_setting_scope"), table_name="settings_setting") - op.drop_table("settings_setting") - - # On PostgreSQL, drop the `settings` schema. - if op.get_context().dialect.name == "postgresql": - op.execute("DROP SCHEMA IF EXISTS settings") diff --git a/host/migrations/versions/2fdcd367b517_initial_schema.py b/host/migrations/versions/2fdcd367b517_initial_schema.py deleted file mode 100644 index c874e748..00000000 --- a/host/migrations/versions/2fdcd367b517_initial_schema.py +++ /dev/null @@ -1,33 +0,0 @@ -"""initial schema - -Revision ID: 2fdcd367b517 -Revises: 53819c12d603 -Create Date: 2026-04-15 12:06:38.726433 -""" - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "2fdcd367b517" -down_revision: str | None = "53819c12d603" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.add_column("products_product", sa.Column("is_deleted", sa.Boolean(), nullable=False)) - op.add_column("products_product", sa.Column("deleted_at", sa.DateTime(), nullable=True)) - op.add_column("products_product", sa.Column("deleted_by", sa.String(length=255), nullable=True)) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("products_product", "deleted_by") - op.drop_column("products_product", "deleted_at") - op.drop_column("products_product", "is_deleted") - # ### end Alembic commands ### diff --git a/host/migrations/versions/53819c12d603_initial_schema.py b/host/migrations/versions/53819c12d603_initial_schema.py deleted file mode 100644 index fa0dda4d..00000000 --- a/host/migrations/versions/53819c12d603_initial_schema.py +++ /dev/null @@ -1,46 +0,0 @@ -"""initial schema - -Revision ID: 53819c12d603 -Revises: -Create Date: 2026-04-13 20:30:12.888075 -""" - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "53819c12d603" -down_revision: str | None = None -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "products_product", - sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), - sa.Column("name", sa.String(length=200), nullable=False), - sa.Column("description", sa.String(length=2000), nullable=True), - sa.Column("price", sa.Numeric(precision=10, scale=2), nullable=False), - sa.Column("is_active", sa.Boolean(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_products_product")), - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("products_product") - # ### end Alembic commands ### diff --git a/host/migrations/versions/5d08d8587674_add_permissions_role_permission_table.py b/host/migrations/versions/5d08d8587674_add_permissions_role_permission_table.py deleted file mode 100644 index fdf16540..00000000 --- a/host/migrations/versions/5d08d8587674_add_permissions_role_permission_table.py +++ /dev/null @@ -1,54 +0,0 @@ -"""add permissions role permission table - -Revision ID: 5d08d8587674 -Revises: b7e1af4c9d02 -Create Date: 2026-04-19 07:58:55.443793 -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "5d08d8587674" -down_revision: str | None = "b7e1af4c9d02" -branch_labels: str | Sequence[str] | None = ("permissions",) -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # On PostgreSQL, create the `permissions` schema before creating tables. - if op.get_context().dialect.name == "postgresql": - op.execute("CREATE SCHEMA IF NOT EXISTS permissions") - - op.create_table( - "permissions_role_permission", - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.Column("role_name", sa.String(length=64), nullable=False), - sa.Column("permission_key", sa.String(length=128), nullable=False), - sa.Column("assigned_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("assigned_by", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint( - "role_name", - "permission_key", - name=op.f("pk_permissions_role_permission"), - ), - ) - - -def downgrade() -> None: - op.drop_table("permissions_role_permission") - - if op.get_context().dialect.name == "postgresql": - op.execute("DROP SCHEMA IF EXISTS permissions") diff --git a/host/migrations/versions/5d44218ee368_create_file_storage_tables.py b/host/migrations/versions/5d44218ee368_create_file_storage_tables.py deleted file mode 100644 index cff58dc1..00000000 --- a/host/migrations/versions/5d44218ee368_create_file_storage_tables.py +++ /dev/null @@ -1,77 +0,0 @@ -"""create file_storage tables - -Revision ID: 5d44218ee368 -Revises: b7e1af4c9d02 -Create Date: 2026-04-19 11:20:24.247357 -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "5d44218ee368" -down_revision: str | None = "b7e1af4c9d02" -branch_labels: str | Sequence[str] | None = ("file_storage",) -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - if op.get_context().dialect.name == "postgresql": - op.execute("CREATE SCHEMA IF NOT EXISTS file_storage") - - op.create_table( - "file_storage_stored_file", - sa.Column("is_deleted", sa.Boolean(), nullable=False), - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("deleted_by", sa.String(length=255), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("key", sa.String(length=512), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("content_type", sa.String(length=128), nullable=False), - sa.Column("size_bytes", sa.Integer(), nullable=False), - sa.Column("backend", sa.String(length=32), nullable=False), - sa.Column("checksum_sha256", sa.String(length=64), nullable=False), - sa.Column("extra_metadata", sa.JSON(), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_file_storage_stored_file")), - ) - op.create_index( - "ix_file_storage_stored_file_created_by", - "file_storage_stored_file", - ["created_by"], - unique=False, - ) - op.create_index( - "ix_file_storage_stored_file_is_deleted", - "file_storage_stored_file", - ["is_deleted"], - unique=False, - ) - op.create_index( - "ix_file_storage_stored_file_key", - "file_storage_stored_file", - ["key"], - unique=True, - ) - - -def downgrade() -> None: - op.drop_index("ix_file_storage_stored_file_key", table_name="file_storage_stored_file") - op.drop_index("ix_file_storage_stored_file_is_deleted", table_name="file_storage_stored_file") - op.drop_index("ix_file_storage_stored_file_created_by", table_name="file_storage_stored_file") - op.drop_table("file_storage_stored_file") - - if op.get_context().dialect.name == "postgresql": - op.execute("DROP SCHEMA IF EXISTS file_storage") diff --git a/host/migrations/versions/6df7645cc4d2_add_background_tasks_task_execution.py b/host/migrations/versions/6df7645cc4d2_add_background_tasks_task_execution.py deleted file mode 100644 index 920c9d68..00000000 --- a/host/migrations/versions/6df7645cc4d2_add_background_tasks_task_execution.py +++ /dev/null @@ -1,115 +0,0 @@ -"""add background_tasks_task_execution - -Revision ID: 6df7645cc4d2 -Revises: b7e1af4c9d02 -Create Date: 2026-04-19 07:44:06.824688 -""" - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "6df7645cc4d2" -down_revision: str | None = "b7e1af4c9d02" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "background_tasks_task_execution", - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column("celery_task_id", sa.String(length=64), nullable=True), - sa.Column("task_name", sa.String(length=255), nullable=False), - sa.Column("status", sa.String(length=20), nullable=False), - sa.Column("queue", sa.String(length=64), nullable=False), - sa.Column("args", sa.JSON(), nullable=True), - sa.Column("kwargs", sa.JSON(), nullable=True), - sa.Column("result", sa.JSON(), nullable=True), - sa.Column("traceback", sa.String(), nullable=True), - sa.Column("exception_type", sa.String(length=255), nullable=True), - sa.Column("worker", sa.String(length=255), nullable=True), - sa.Column("retries", sa.Integer(), nullable=False), - sa.Column("retried_from_id", sa.Uuid(), nullable=True), - sa.Column("queued_at", sa.DateTime(), nullable=True), - sa.Column("started_at", sa.DateTime(), nullable=True), - sa.Column("finished_at", sa.DateTime(), nullable=True), - sa.Column("heartbeat_at", sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint( - ["retried_from_id"], - ["background_tasks_task_execution.id"], - name=op.f( - "fk_background_tasks_task_execution_retried_from_id_background_tasks_task_execution" - ), - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_background_tasks_task_execution")), - ) - op.create_index( - op.f("ix_background_tasks_task_execution_celery_task_id"), - "background_tasks_task_execution", - ["celery_task_id"], - unique=False, - ) - op.create_index( - op.f("ix_background_tasks_task_execution_retried_from_id"), - "background_tasks_task_execution", - ["retried_from_id"], - unique=False, - ) - op.create_index( - op.f("ix_background_tasks_task_execution_status"), - "background_tasks_task_execution", - ["status"], - unique=False, - ) - op.create_index( - "ix_background_tasks_task_execution_status_queued", - "background_tasks_task_execution", - ["status", "queued_at"], - unique=False, - ) - op.create_index( - op.f("ix_background_tasks_task_execution_task_name"), - "background_tasks_task_execution", - ["task_name"], - unique=False, - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index( - op.f("ix_background_tasks_task_execution_task_name"), - table_name="background_tasks_task_execution", - ) - op.drop_index( - "ix_background_tasks_task_execution_status_queued", - table_name="background_tasks_task_execution", - ) - op.drop_index( - op.f("ix_background_tasks_task_execution_status"), - table_name="background_tasks_task_execution", - ) - op.drop_index( - op.f("ix_background_tasks_task_execution_retried_from_id"), - table_name="background_tasks_task_execution", - ) - op.drop_index( - op.f("ix_background_tasks_task_execution_celery_task_id"), - table_name="background_tasks_task_execution", - ) - op.drop_table("background_tasks_task_execution") - # ### end Alembic commands ### diff --git a/host/migrations/versions/77162e7b184b_initial_schema.py b/host/migrations/versions/77162e7b184b_initial_schema.py new file mode 100644 index 00000000..a705c2e8 --- /dev/null +++ b/host/migrations/versions/77162e7b184b_initial_schema.py @@ -0,0 +1,382 @@ +"""initial schema + +Revision ID: 77162e7b184b +Revises: +Create Date: 2026-04-30 23:59:42.399469 +""" + +from collections.abc import Sequence + +import fastapi_users_db_sqlalchemy.generics +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "77162e7b184b" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "background_tasks_task_execution", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("celery_task_id", sa.String(length=64), nullable=True), + sa.Column("task_name", sa.String(length=255), nullable=False), + sa.Column("status", sa.String(length=20), nullable=False), + sa.Column("queue", sa.String(length=64), nullable=False), + sa.Column("args", sa.JSON(), nullable=True), + sa.Column("kwargs", sa.JSON(), nullable=True), + sa.Column("result", sa.JSON(), nullable=True), + sa.Column("traceback", sa.String(), nullable=True), + sa.Column("exception_type", sa.String(length=255), nullable=True), + sa.Column("worker", sa.String(length=255), nullable=True), + sa.Column("retries", sa.Integer(), nullable=False), + sa.Column("retried_from_id", sa.Uuid(), nullable=True), + sa.Column("queued_at", sa.DateTime(), nullable=True), + sa.Column("started_at", sa.DateTime(), nullable=True), + sa.Column("finished_at", sa.DateTime(), nullable=True), + sa.Column("heartbeat_at", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint( + ["retried_from_id"], + ["background_tasks_task_execution.id"], + name=op.f( + "fk_background_tasks_task_execution_retried_from_id_background_tasks_task_execution" + ), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_background_tasks_task_execution")), + ) + op.create_index( + op.f("ix_background_tasks_task_execution_celery_task_id"), + "background_tasks_task_execution", + ["celery_task_id"], + unique=False, + ) + op.create_index( + op.f("ix_background_tasks_task_execution_retried_from_id"), + "background_tasks_task_execution", + ["retried_from_id"], + unique=False, + ) + op.create_index( + op.f("ix_background_tasks_task_execution_status"), + "background_tasks_task_execution", + ["status"], + unique=False, + ) + op.create_index( + "ix_background_tasks_task_execution_status_queued", + "background_tasks_task_execution", + ["status", "queued_at"], + unique=False, + ) + op.create_index( + op.f("ix_background_tasks_task_execution_task_name"), + "background_tasks_task_execution", + ["task_name"], + unique=False, + ) + op.create_table( + "feature_flags_override", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("scope", sa.String(length=10), nullable=False), + sa.Column("scope_id", sa.String(length=64), nullable=False), + sa.Column("name", sa.String(length=200), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_feature_flags_override")), + sa.UniqueConstraint( + "scope", "scope_id", "name", name="uq_feature_flags_override_scope_scope_id_name" + ), + ) + op.create_index( + op.f("ix_feature_flags_override_name"), "feature_flags_override", ["name"], unique=False + ) + op.create_index( + op.f("ix_feature_flags_override_scope"), "feature_flags_override", ["scope"], unique=False + ) + op.create_index( + op.f("ix_feature_flags_override_scope_id"), + "feature_flags_override", + ["scope_id"], + unique=False, + ) + op.create_table( + "file_storage_stored_file", + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("deleted_by", sa.String(length=255), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("key", sa.String(length=512), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("content_type", sa.String(length=128), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("backend", sa.String(length=32), nullable=False), + sa.Column("checksum_sha256", sa.String(length=64), nullable=False), + sa.Column("extra_metadata", sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_file_storage_stored_file")), + ) + op.create_index( + "ix_file_storage_stored_file_created_by", + "file_storage_stored_file", + ["created_by"], + unique=False, + ) + op.create_index( + "ix_file_storage_stored_file_is_deleted", + "file_storage_stored_file", + ["is_deleted"], + unique=False, + ) + op.create_index( + "ix_file_storage_stored_file_key", "file_storage_stored_file", ["key"], unique=True + ) + op.create_table( + "permissions_role_permission", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("role_name", sa.String(length=64), nullable=False), + sa.Column("permission_key", sa.String(length=128), nullable=False), + sa.Column("assigned_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("assigned_by", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint( + "role_name", "permission_key", name=op.f("pk_permissions_role_permission") + ), + ) + op.create_index( + "ix_permissions_role_permission_key", + "permissions_role_permission", + ["permission_key"], + unique=False, + ) + op.create_table( + "permissions_user_permission", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False), + sa.Column("permission_key", sa.String(length=128), nullable=False), + sa.Column("assigned_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("assigned_by", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint( + "user_id", "permission_key", name=op.f("pk_permissions_user_permission") + ), + ) + op.create_index( + "ix_permissions_user_permission_key", + "permissions_user_permission", + ["permission_key"], + unique=False, + ) + op.create_table( + "settings_setting", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("scope", sa.String(length=10), nullable=False), + sa.Column("scope_id", sa.String(length=255), nullable=False), + sa.Column("key", sa.String(length=200), nullable=False), + sa.Column("value", sa.String(length=4000), nullable=False), + sa.Column("value_type", sa.String(length=10), nullable=False), + sa.Column("description", sa.String(length=2000), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_settings_setting")), + sa.UniqueConstraint( + "scope", "scope_id", "key", name="uq_settings_setting_scope_scope_id_key" + ), + ) + op.create_index(op.f("ix_settings_setting_key"), "settings_setting", ["key"], unique=False) + op.create_index(op.f("ix_settings_setting_scope"), "settings_setting", ["scope"], unique=False) + op.create_index( + op.f("ix_settings_setting_scope_id"), "settings_setting", ["scope_id"], unique=False + ) + op.create_table( + "users_role", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False), + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("description", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_users_role")), + ) + op.create_index(op.f("ix_users_role_name"), "users_role", ["name"], unique=True) + op.create_table( + "users_user", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False), + sa.Column("email", sa.String(length=320), nullable=False), + sa.Column("hashed_password", sa.String(length=1024), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column("is_superuser", sa.Boolean(), nullable=False), + sa.Column("is_verified", sa.Boolean(), nullable=False), + sa.Column("full_name", sa.String(length=255), nullable=True), + sa.Column("tenant_id", sa.String(length=50), nullable=True), + sa.Column("disabled_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_users_user")), + ) + op.create_index(op.f("ix_users_user_email"), "users_user", ["email"], unique=True) + op.create_index( + op.f("ix_users_user_last_login_at"), "users_user", ["last_login_at"], unique=False + ) + op.create_index(op.f("ix_users_user_tenant_id"), "users_user", ["tenant_id"], unique=False) + op.create_table( + "users_access_token", + sa.Column("token", sa.String(length=43), nullable=False), + sa.Column( + "created_at", + fastapi_users_db_sqlalchemy.generics.TIMESTAMPAware(timezone=True), + nullable=False, + ), + sa.Column("user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False), + sa.ForeignKeyConstraint( + ["user_id"], + ["users_user.id"], + name=op.f("fk_users_access_token_user_id_users_user"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("token", name=op.f("pk_users_access_token")), + ) + op.create_index( + op.f("ix_users_access_token_created_at"), "users_access_token", ["created_at"], unique=False + ) + op.create_index( + op.f("ix_users_access_token_user_id"), "users_access_token", ["user_id"], unique=False + ) + op.create_table( + "users_user_role", + sa.Column("user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False), + sa.Column("role_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False), + sa.Column("assigned_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("assigned_by", sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["role_id"], + ["users_role.id"], + name=op.f("fk_users_user_role_role_id_users_role"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users_user.id"], + name=op.f("fk_users_user_role_user_id_users_user"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("user_id", "role_id", name=op.f("pk_users_user_role")), + ) + op.create_index("ix_users_user_role_role_id", "users_user_role", ["role_id"], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_users_user_role_role_id", table_name="users_user_role") + op.drop_table("users_user_role") + op.drop_index(op.f("ix_users_access_token_user_id"), table_name="users_access_token") + op.drop_index(op.f("ix_users_access_token_created_at"), table_name="users_access_token") + op.drop_table("users_access_token") + op.drop_index(op.f("ix_users_user_tenant_id"), table_name="users_user") + op.drop_index(op.f("ix_users_user_last_login_at"), table_name="users_user") + op.drop_index(op.f("ix_users_user_email"), table_name="users_user") + op.drop_table("users_user") + op.drop_index(op.f("ix_users_role_name"), table_name="users_role") + op.drop_table("users_role") + op.drop_index(op.f("ix_settings_setting_scope_id"), table_name="settings_setting") + op.drop_index(op.f("ix_settings_setting_scope"), table_name="settings_setting") + op.drop_index(op.f("ix_settings_setting_key"), table_name="settings_setting") + op.drop_table("settings_setting") + op.drop_index("ix_permissions_user_permission_key", table_name="permissions_user_permission") + op.drop_table("permissions_user_permission") + op.drop_index("ix_permissions_role_permission_key", table_name="permissions_role_permission") + op.drop_table("permissions_role_permission") + op.drop_index("ix_file_storage_stored_file_key", table_name="file_storage_stored_file") + op.drop_index("ix_file_storage_stored_file_is_deleted", table_name="file_storage_stored_file") + op.drop_index("ix_file_storage_stored_file_created_by", table_name="file_storage_stored_file") + op.drop_table("file_storage_stored_file") + op.drop_index(op.f("ix_feature_flags_override_scope_id"), table_name="feature_flags_override") + op.drop_index(op.f("ix_feature_flags_override_scope"), table_name="feature_flags_override") + op.drop_index(op.f("ix_feature_flags_override_name"), table_name="feature_flags_override") + op.drop_table("feature_flags_override") + op.drop_index( + op.f("ix_background_tasks_task_execution_task_name"), + table_name="background_tasks_task_execution", + ) + op.drop_index( + "ix_background_tasks_task_execution_status_queued", + table_name="background_tasks_task_execution", + ) + op.drop_index( + op.f("ix_background_tasks_task_execution_status"), + table_name="background_tasks_task_execution", + ) + op.drop_index( + op.f("ix_background_tasks_task_execution_retried_from_id"), + table_name="background_tasks_task_execution", + ) + op.drop_index( + op.f("ix_background_tasks_task_execution_celery_task_id"), + table_name="background_tasks_task_execution", + ) + op.drop_table("background_tasks_task_execution") + # ### end Alembic commands ### diff --git a/host/migrations/versions/8c12be982a27_create_users_tables.py b/host/migrations/versions/8c12be982a27_create_users_tables.py deleted file mode 100644 index 38dc4921..00000000 --- a/host/migrations/versions/8c12be982a27_create_users_tables.py +++ /dev/null @@ -1,131 +0,0 @@ -"""create users tables - -Revision ID: 8c12be982a27 -Revises: 2fdcd367b517 -Create Date: 2026-04-15 18:02:20.074558 -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op -from fastapi_users_db_sqlalchemy.generics import GUID, TIMESTAMPAware - -# revision identifiers, used by Alembic. -revision: str = "8c12be982a27" -down_revision: str | None = "2fdcd367b517" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # On PostgreSQL, create the `users` schema before creating tables. - if op.get_context().dialect.name == "postgresql": - op.execute("CREATE SCHEMA IF NOT EXISTS users") - - op.create_table( - "users_role", - sa.Column("id", GUID(), nullable=False), - sa.Column("name", sa.String(length=64), nullable=False), - sa.Column("description", sa.String(length=255), nullable=True), - sa.Column( - "created_at", - sa.DateTime(), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_users_role")), - ) - op.create_index(op.f("ix_users_role_name"), "users_role", ["name"], unique=True) - - op.create_table( - "users_user", - sa.Column("full_name", sa.String(length=255), nullable=True), - sa.Column("tenant_id", sa.String(length=50), nullable=True), - sa.Column("disabled_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("id", GUID(), nullable=False), - sa.Column("email", sa.String(length=320), nullable=False), - sa.Column("hashed_password", sa.String(length=1024), nullable=False), - sa.Column("is_active", sa.Boolean(), nullable=False), - sa.Column("is_superuser", sa.Boolean(), nullable=False), - sa.Column("is_verified", sa.Boolean(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_users_user")), - ) - op.create_index(op.f("ix_users_user_email"), "users_user", ["email"], unique=True) - op.create_index(op.f("ix_users_user_tenant_id"), "users_user", ["tenant_id"], unique=False) - - op.create_table( - "users_access_token", - sa.Column("user_id", GUID(), nullable=False), - sa.Column("token", sa.String(length=43), nullable=False), - sa.Column("created_at", TIMESTAMPAware(timezone=True), nullable=False), - sa.ForeignKeyConstraint( - ["user_id"], - ["users_user.id"], - name=op.f("fk_users_access_token_user_id_users_user"), - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("token", name=op.f("pk_users_access_token")), - ) - op.create_index( - op.f("ix_users_access_token_created_at"), - "users_access_token", - ["created_at"], - unique=False, - ) - - op.create_table( - "users_user_role", - sa.Column("user_id", GUID(), nullable=False), - sa.Column("role_id", GUID(), nullable=False), - sa.Column( - "assigned_at", - sa.DateTime(timezone=True), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("assigned_by", sa.String(length=255), nullable=True), - sa.ForeignKeyConstraint( - ["role_id"], - ["users_role.id"], - name=op.f("fk_users_user_role_role_id_users_role"), - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["user_id"], - ["users_user.id"], - name=op.f("fk_users_user_role_user_id_users_user"), - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("user_id", "role_id", name=op.f("pk_users_user_role")), - ) - - -def downgrade() -> None: - op.drop_table("users_user_role") - op.drop_index(op.f("ix_users_access_token_created_at"), table_name="users_access_token") - op.drop_table("users_access_token") - op.drop_index(op.f("ix_users_user_tenant_id"), table_name="users_user") - op.drop_index(op.f("ix_users_user_email"), table_name="users_user") - op.drop_table("users_user") - op.drop_index(op.f("ix_users_role_name"), table_name="users_role") - op.drop_table("users_role") - - # On PostgreSQL, drop the `users` schema. - if op.get_context().dialect.name == "postgresql": - op.execute("DROP SCHEMA IF EXISTS users") diff --git a/host/migrations/versions/9557a7c2e646_add_permissions_user_permission_table.py b/host/migrations/versions/9557a7c2e646_add_permissions_user_permission_table.py deleted file mode 100644 index b9b29111..00000000 --- a/host/migrations/versions/9557a7c2e646_add_permissions_user_permission_table.py +++ /dev/null @@ -1,58 +0,0 @@ -"""add permissions user permission table - -Revision ID: 9557a7c2e646 -Revises: 5d08d8587674 -Create Date: 2026-04-19 09:43:57.944179 -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op -from fastapi_users_db_sqlalchemy.generics import GUID - -# revision identifiers, used by Alembic. -revision: str = "9557a7c2e646" -down_revision: str | None = "5d08d8587674" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - op.create_table( - "permissions_user_permission", - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.Column("user_id", GUID(), nullable=False), - sa.Column("permission_key", sa.String(length=128), nullable=False), - sa.Column("assigned_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("assigned_by", sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint( - "user_id", - "permission_key", - name=op.f("pk_permissions_user_permission"), - ), - ) - op.create_index( - "ix_permissions_user_permission_key", - "permissions_user_permission", - ["permission_key"], - unique=False, - ) - - -def downgrade() -> None: - op.drop_index( - "ix_permissions_user_permission_key", - table_name="permissions_user_permission", - ) - op.drop_table("permissions_user_permission") diff --git a/host/migrations/versions/a01185374312_add_indexes_for_dashboard_queries.py b/host/migrations/versions/a01185374312_add_indexes_for_dashboard_queries.py deleted file mode 100644 index 24bc54c9..00000000 --- a/host/migrations/versions/a01185374312_add_indexes_for_dashboard_queries.py +++ /dev/null @@ -1,34 +0,0 @@ -"""add indexes for dashboard queries - -Revision ID: a01185374312 -Revises: e3ce9754e6dc -Create Date: 2026-04-15 23:53:08.569085 -""" - -from collections.abc import Sequence - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "a01185374312" -down_revision: str | None = "e3ce9754e6dc" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_index( - op.f("ix_products_product_is_active"), "products_product", ["is_active"], unique=False - ) - op.create_index( - op.f("ix_users_user_last_login_at"), "users_user", ["last_login_at"], unique=False - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f("ix_users_user_last_login_at"), table_name="users_user") - op.drop_index(op.f("ix_products_product_is_active"), table_name="products_product") - # ### end Alembic commands ### diff --git a/host/migrations/versions/a35930f574d8_index_permissions_role_permission_key.py b/host/migrations/versions/a35930f574d8_index_permissions_role_permission_key.py deleted file mode 100644 index ac162f5f..00000000 --- a/host/migrations/versions/a35930f574d8_index_permissions_role_permission_key.py +++ /dev/null @@ -1,33 +0,0 @@ -"""index permissions role permission key - -Revision ID: a35930f574d8 -Revises: 9557a7c2e646 -Create Date: 2026-04-19 11:52:35.013862 -""" - -from __future__ import annotations - -from collections.abc import Sequence - -from alembic import op - -revision: str = "a35930f574d8" -down_revision: str | None = "9557a7c2e646" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - op.create_index( - "ix_permissions_role_permission_key", - "permissions_role_permission", - ["permission_key"], - unique=False, - ) - - -def downgrade() -> None: - op.drop_index( - "ix_permissions_role_permission_key", - table_name="permissions_role_permission", - ) diff --git a/host/migrations/versions/b7e1af4c9d02_add_perf_indexes_and_fix_products.py b/host/migrations/versions/b7e1af4c9d02_add_perf_indexes_and_fix_products.py deleted file mode 100644 index 32a2ff8c..00000000 --- a/host/migrations/versions/b7e1af4c9d02_add_perf_indexes_and_fix_products.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Add perf indexes and drop low-cardinality boolean index. - -Creates: - * ``ix_users_user_email_lower`` — functional index on ``lower(email)`` so the - fastapi-users ``get_by_email`` query (which wraps email in ``lower()``) - can use an index instead of a seq-scan. - * ``ix_users_access_token_user_id`` — Postgres does not auto-index foreign - keys; this covers reverse lookups (e.g. revoking a user's sessions). - * ``ix_users_user_role_role_id`` — same reasoning for "who has role X?" - queries. The composite PK already covers ``user_id``-first lookups. - * ``ix_products_product_deleted`` — supports soft-delete filtering in the - product listing query. - -Drops: - * ``ix_products_product_is_active`` — a plain B-tree over a 2-value boolean - is almost never preferred by the planner over a seq-scan, yet it costs - writes on every insert/update. Replaced implicitly by the combined - filtering the ``products_product_deleted`` index covers. - -PostgreSQL path uses ``CREATE INDEX CONCURRENTLY`` via ``postgresql_concurrently`` -+ ``autocommit_block`` so index builds on large tables do not block writes. -SQLite ignores the flag (its ``CREATE INDEX`` is already fast and non-locking -for this workload). - -Revision ID: b7e1af4c9d02 -Revises: a01185374312 -Create Date: 2026-04-16 12:00:00.000000 -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "b7e1af4c9d02" -down_revision: str | None = "a01185374312" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -# Functional index: ``lower(email)``. Using a literal SQL expression keeps -# the syntax identical on Postgres and SQLite; SQLAlchemy's ``text()`` is -# escaped appropriately by the dialect in both cases. -_EMAIL_LOWER_EXPR = sa.text("lower(email)") - - -def upgrade() -> None: - is_postgres = op.get_context().dialect.name == "postgresql" - - # Each ``CREATE/DROP INDEX CONCURRENTLY`` auto-commits as soon as it finishes. - # ``if_exists`` / ``if_not_exists`` make the migration re-runnable after a - # partial failure mid-block: otherwise we'd have to manually clean up the - # already-committed indexes before retrying. - with op.get_context().autocommit_block(): - op.create_index( - "ix_users_user_email_lower", - "users_user", - [_EMAIL_LOWER_EXPR], - unique=False, - if_not_exists=True, - postgresql_concurrently=is_postgres, - ) - op.create_index( - "ix_users_access_token_user_id", - "users_access_token", - ["user_id"], - unique=False, - if_not_exists=True, - postgresql_concurrently=is_postgres, - ) - op.create_index( - "ix_users_user_role_role_id", - "users_user_role", - ["role_id"], - unique=False, - if_not_exists=True, - postgresql_concurrently=is_postgres, - ) - op.create_index( - "ix_products_product_is_deleted", - "products_product", - ["is_deleted"], - unique=False, - if_not_exists=True, - postgresql_concurrently=is_postgres, - ) - op.drop_index( - "ix_products_product_is_active", - table_name="products_product", - if_exists=True, - postgresql_concurrently=is_postgres, - ) - - -def downgrade() -> None: - is_postgres = op.get_context().dialect.name == "postgresql" - - with op.get_context().autocommit_block(): - op.create_index( - "ix_products_product_is_active", - "products_product", - ["is_active"], - unique=False, - if_not_exists=True, - postgresql_concurrently=is_postgres, - ) - op.drop_index( - "ix_products_product_is_deleted", - table_name="products_product", - if_exists=True, - postgresql_concurrently=is_postgres, - ) - op.drop_index( - "ix_users_user_role_role_id", - table_name="users_user_role", - if_exists=True, - postgresql_concurrently=is_postgres, - ) - op.drop_index( - "ix_users_access_token_user_id", - table_name="users_access_token", - if_exists=True, - postgresql_concurrently=is_postgres, - ) - op.drop_index( - "ix_users_user_email_lower", - table_name="users_user", - if_exists=True, - postgresql_concurrently=is_postgres, - ) diff --git a/host/migrations/versions/cf714c8bf117_add_feature_flags_override_table.py b/host/migrations/versions/cf714c8bf117_add_feature_flags_override_table.py deleted file mode 100644 index 53bb9d0d..00000000 --- a/host/migrations/versions/cf714c8bf117_add_feature_flags_override_table.py +++ /dev/null @@ -1,86 +0,0 @@ -"""add feature_flags override table - -Revision ID: cf714c8bf117 -Revises: 1fe7590fc594 -Create Date: 2026-04-19 13:58:41.316854 -""" - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "cf714c8bf117" -down_revision: str | None = "1fe7590fc594" -branch_labels: str | Sequence[str] | None = ("feature_flags",) -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # On PostgreSQL, create the `feature_flags` schema before creating tables. - if op.get_context().dialect.name == "postgresql": - op.execute("CREATE SCHEMA IF NOT EXISTS feature_flags") - - op.create_table( - "feature_flags_override", - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("scope", sa.String(length=10), nullable=False, server_default="system"), - sa.Column("scope_id", sa.String(length=64), nullable=False, server_default=""), - sa.Column("name", sa.String(length=200), nullable=False), - sa.Column("enabled", sa.Boolean(), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_feature_flags_override")), - sa.UniqueConstraint( - "scope", - "scope_id", - "name", - name="uq_feature_flags_override_scope_scope_id_name", - ), - ) - op.create_index( - op.f("ix_feature_flags_override_scope"), - "feature_flags_override", - ["scope"], - unique=False, - ) - op.create_index( - op.f("ix_feature_flags_override_scope_id"), - "feature_flags_override", - ["scope_id"], - unique=False, - ) - op.create_index( - op.f("ix_feature_flags_override_name"), - "feature_flags_override", - ["name"], - unique=False, - ) - - -def downgrade() -> None: - op.drop_index( - op.f("ix_feature_flags_override_name"), - table_name="feature_flags_override", - ) - op.drop_index( - op.f("ix_feature_flags_override_scope_id"), - table_name="feature_flags_override", - ) - op.drop_index( - op.f("ix_feature_flags_override_scope"), - table_name="feature_flags_override", - ) - op.drop_table("feature_flags_override") - - # On PostgreSQL, drop the `feature_flags` schema. - if op.get_context().dialect.name == "postgresql": - op.execute("DROP SCHEMA IF EXISTS feature_flags") diff --git a/host/migrations/versions/dad9a134290f_add_datasets_table.py b/host/migrations/versions/dad9a134290f_add_datasets_table.py deleted file mode 100644 index 8114b9bb..00000000 --- a/host/migrations/versions/dad9a134290f_add_datasets_table.py +++ /dev/null @@ -1,69 +0,0 @@ -"""add datasets table - -Revision ID: dad9a134290f -Revises: b7e1af4c9d02 -Create Date: 2026-04-19 11:51:19.145277 -""" - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "dad9a134290f" -down_revision: str | None = "b7e1af4c9d02" -branch_labels: str | Sequence[str] | None = ("datasets",) -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "datasets_dataset", - sa.Column("is_deleted", sa.Boolean(), nullable=False), - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("deleted_by", sa.String(length=255), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("(CURRENT_TIMESTAMP)"), - nullable=False, - ), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_by", sa.String(length=255), nullable=True), - sa.Column("updated_by", sa.String(length=255), nullable=True), - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=200), nullable=False), - sa.Column("slug", sa.String(length=200), nullable=False), - sa.Column("kind", sa.String(length=32), nullable=False), - sa.Column("description", sa.String(length=2000), nullable=True), - sa.Column("original_filename", sa.String(length=255), nullable=False), - sa.Column("mime_type", sa.String(length=127), nullable=True), - sa.Column("size_bytes", sa.Integer(), nullable=False), - sa.Column("storage_key", sa.String(length=512), nullable=False), - sa.Column("crs", sa.String(length=64), nullable=True), - sa.Column("bbox_min_x", sa.Float(), nullable=True), - sa.Column("bbox_min_y", sa.Float(), nullable=True), - sa.Column("bbox_max_x", sa.Float(), nullable=True), - sa.Column("bbox_max_y", sa.Float(), nullable=True), - sa.Column("feature_count", sa.Integer(), nullable=True), - sa.Column("band_count", sa.Integer(), nullable=True), - sa.Column("extraction_status", sa.String(length=16), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_datasets_dataset")), - ) - op.create_index( - "ix_datasets_dataset_is_deleted", "datasets_dataset", ["is_deleted"], unique=False - ) - op.create_index(op.f("ix_datasets_dataset_kind"), "datasets_dataset", ["kind"], unique=False) - op.create_index(op.f("ix_datasets_dataset_slug"), "datasets_dataset", ["slug"], unique=True) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f("ix_datasets_dataset_slug"), table_name="datasets_dataset") - op.drop_index(op.f("ix_datasets_dataset_kind"), table_name="datasets_dataset") - op.drop_index("ix_datasets_dataset_is_deleted", table_name="datasets_dataset") - op.drop_table("datasets_dataset") - # ### end Alembic commands ### diff --git a/host/migrations/versions/e3ce9754e6dc_seed_users_roles.py b/host/migrations/versions/e3ce9754e6dc_seed_users_roles.py deleted file mode 100644 index b7969965..00000000 --- a/host/migrations/versions/e3ce9754e6dc_seed_users_roles.py +++ /dev/null @@ -1,51 +0,0 @@ -"""seed users roles - -Revision ID: e3ce9754e6dc -Revises: 8c12be982a27 -Create Date: 2026-04-15 18:10:00.000000 -""" - -from __future__ import annotations - -import uuid -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op -from fastapi_users_db_sqlalchemy.generics import GUID - -revision: str = "e3ce9754e6dc" -down_revision: str | None = "8c12be982a27" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -# Duplicated from modules/users/users/constants.py — migrations are run from a -# context where module imports may not resolve (alembic discovers them via -# the simple_module entry point), and tying the migration to the module's -# import path is more fragile than hardcoding the stable UUIDs. -ADMIN_ROLE_ID = uuid.UUID("00000000-0000-0000-0000-000000000001") -USER_ROLE_ID = uuid.UUID("00000000-0000-0000-0000-000000000002") - - -def upgrade() -> None: - # Use GUID — same type used by the schema migration — so the stored - # representation matches on all backends (especially SQLite, where - # sa.Uuid() serializes without dashes but GUID keeps them, breaking JOINs). - roles_table = sa.table( - "users_role", - sa.column("id", GUID()), - sa.column("name", sa.String()), - sa.column("description", sa.String()), - ) - op.bulk_insert( - roles_table, - [ - {"id": ADMIN_ROLE_ID, "name": "admin", "description": "Administrator"}, - {"id": USER_ROLE_ID, "name": "user", "description": "Standard user"}, - ], - ) - - -def downgrade() -> None: - op.execute(f"DELETE FROM users_role WHERE id IN ('{ADMIN_ROLE_ID}', '{USER_ROLE_ID}')") diff --git a/host/pyproject.toml b/host/pyproject.toml index dbae2b21..35bce23f 100644 --- a/host/pyproject.toml +++ b/host/pyproject.toml @@ -7,8 +7,6 @@ dependencies = [ "simple_module_hosting", "simple_module_auth", "simple_module_dashboard", - "simple_module_products", - "simple_module_datasets", "simple_module_permissions", "simple_module_background_tasks", "simple_module_file_storage", @@ -20,9 +18,7 @@ dependencies = [ [tool.uv.sources] simple_module_hosting = { workspace = true } simple_module_auth = { workspace = true } -simple_module_products = { workspace = true } simple_module_dashboard = { workspace = true } -simple_module_datasets = { workspace = true } simple_module_permissions = { workspace = true } simple_module_background_tasks = { workspace = true } simple_module_file_storage = { workspace = true } diff --git a/modules/dashboard/README.md b/modules/dashboard/README.md index 610db8bc..0b7148e4 100644 --- a/modules/dashboard/README.md +++ b/modules/dashboard/README.md @@ -38,7 +38,7 @@ The dashboard sidebar picks it up automatically. ## Depends on - `simple_module_core`, `simple_module_db`, `simple_module_hosting` -- `simple_module_users`, `simple_module_products` (demo content used by the default layout) +- `simple_module_users` (user counts shown on the default layout) ## License diff --git a/modules/dashboard/dashboard/locales/en.json b/modules/dashboard/dashboard/locales/en.json index 5aa421af..c422fed4 100644 --- a/modules/dashboard/dashboard/locales/en.json +++ b/modules/dashboard/dashboard/locales/en.json @@ -5,7 +5,6 @@ "stats": { "total_users": "Total Users", "active_users": "Active Users (7d)", - "products": "Products", "modules": "Modules" }, "system_info_title": "System", diff --git a/modules/dashboard/dashboard/locales/es.json b/modules/dashboard/dashboard/locales/es.json index 207975f6..117b300c 100644 --- a/modules/dashboard/dashboard/locales/es.json +++ b/modules/dashboard/dashboard/locales/es.json @@ -5,7 +5,6 @@ "stats": { "total_users": "Usuarios Totales", "active_users": "Usuarios Activos (7d)", - "products": "Productos", "modules": "Módulos" }, "system_info_title": "Sistema", diff --git a/modules/dashboard/dashboard/module.py b/modules/dashboard/dashboard/module.py index 1596d490..9923975c 100644 --- a/modules/dashboard/dashboard/module.py +++ b/modules/dashboard/dashboard/module.py @@ -9,7 +9,6 @@ from simple_module_core.menu import MenuItem, MenuRegistry, MenuSection from simple_module_core.module import ModuleBase, ModuleMeta -_MODULE_PRODUCTS = "Products" _MODULE_USERS = "Users" _URL_DASHBOARD = "/dashboard/" _ICON_DASHBOARD = "home" @@ -20,7 +19,7 @@ class DashboardModule(ModuleBase): name="Dashboard", route_prefix="/api/dashboard", view_prefix="/dashboard", - depends_on=[_MODULE_PRODUCTS, _MODULE_USERS], + depends_on=[_MODULE_USERS], ) def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None: diff --git a/modules/dashboard/dashboard/pages/Home.tsx b/modules/dashboard/dashboard/pages/Home.tsx index 383bcacb..8bde7fba 100644 --- a/modules/dashboard/dashboard/pages/Home.tsx +++ b/modules/dashboard/dashboard/pages/Home.tsx @@ -4,9 +4,9 @@ import { PageShell } from '@simple-module-py/ui/components/PageShell'; import { Card, CardContent, CardHeader, CardTitle } from '@simple-module-py/ui/components/ui/card'; import { Table, TableBody, TableCell, TableRow } from '@simple-module-py/ui/components/ui/table'; import { AuthenticatedLayout } from '@simple-module-py/ui/layouts/AuthenticatedLayout'; -import { Activity, Box, Heart, Package, Server, Users } from 'lucide-react'; +import { Activity, Box, Heart, Server, Users } from 'lucide-react'; -type Accent = 'primary' | 'emerald' | 'violet' | 'amber'; +type Accent = 'emerald' | 'violet' | 'amber'; const HEALTH_STATUS_COLOR: Record = { healthy: 'bg-emerald-500', @@ -33,7 +33,6 @@ interface SystemInfo { interface Props { total_users: number; active_users_7d: number; - total_products: number; module_count: number; system_info: SystemInfo; } @@ -47,7 +46,7 @@ function Home() { title={t(keys.dashboard.home.title)} description={t(keys.dashboard.home.description)} > -
+
} accent="amber" /> - } - accent="primary" - /> = { - primary: { - card: 'border-primary-200 bg-gradient-to-br from-primary-50 to-card', - icon: 'text-primary-500 bg-primary-100', - value: 'text-primary-900', - }, emerald: { card: 'border-emerald-border bg-gradient-to-br from-emerald-bg to-card', icon: 'text-emerald-icon-fg bg-emerald-icon-bg', diff --git a/modules/dashboard/dashboard/stats.py b/modules/dashboard/dashboard/stats.py index 379a9736..31b2c768 100644 --- a/modules/dashboard/dashboard/stats.py +++ b/modules/dashboard/dashboard/stats.py @@ -8,7 +8,6 @@ from datetime import UTC, datetime, timedelta from fastapi import FastAPI -from products.models import Product from simple_module_core.health import HealthCheck, HealthStatus from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession @@ -42,14 +41,12 @@ async def fetch_dashboard_stats(db: AsyncSession, app: FastAPI) -> dict: total_users = await _count_users(db) active_users_7d = await _count_active_users(db, days=7) - total_products = await _count_products(db) modules_list = _get_module_info(app) health_checks = await _run_health_checks(app) result = { "total_users": total_users, "active_users_7d": active_users_7d, - "total_products": total_products, "module_count": len(modules_list), "system_info": { "modules": modules_list, @@ -85,13 +82,6 @@ async def _count_active_users(db: AsyncSession, *, days: int) -> int: return result.scalar_one() -async def _count_products(db: AsyncSession) -> int: - result = await db.execute( - select(func.count()).select_from(Product).where(Product.is_active.is_(True)) - ) - return result.scalar_one() - - def _get_module_info(app: FastAPI) -> list[dict[str, str]]: # Reads from the module list discovered once at startup, avoiding # expensive entry-point rescans on every request. diff --git a/modules/dashboard/pyproject.toml b/modules/dashboard/pyproject.toml index 5696ecaa..aba47446 100644 --- a/modules/dashboard/pyproject.toml +++ b/modules/dashboard/pyproject.toml @@ -24,7 +24,6 @@ dependencies = [ "simple_module_core==0.0.3", "simple_module_db==0.0.3", "simple_module_hosting==0.0.3", - "simple_module_products==0.0.3", "simple_module_users==0.0.3", ] @@ -53,5 +52,4 @@ packages = ["dashboard"] simple_module_core = { workspace = true } simple_module_db = { workspace = true } simple_module_hosting = { workspace = true } -simple_module_products = { workspace = true } simple_module_users = { workspace = true } diff --git a/modules/dashboard/tests/test_dashboard.py b/modules/dashboard/tests/test_dashboard.py index bd3ecd84..ff2e2a62 100644 --- a/modules/dashboard/tests/test_dashboard.py +++ b/modules/dashboard/tests/test_dashboard.py @@ -24,7 +24,6 @@ async def test_module_meta(self): mod = DashboardModule() assert mod.meta.name == "Dashboard" assert mod.meta.route_prefix == "/api/dashboard" - assert "Products" in mod.meta.depends_on assert "Users" in mod.meta.depends_on @@ -42,7 +41,6 @@ async def stats(self, app): async def test_returns_expected_keys(self, stats): assert "total_users" in stats assert "active_users_7d" in stats - assert "total_products" in stats assert "module_count" in stats assert "system_info" in stats @@ -79,7 +77,6 @@ async def test_stats_returns_all_fields(self, authenticated_client: httpx.AsyncC body = resp.json() assert "total_users" in body assert "active_users_7d" in body - assert "total_products" in body assert "module_count" in body assert "system_info" in body diff --git a/modules/datasets/LICENSE b/modules/datasets/LICENSE deleted file mode 100644 index a1e1d361..00000000 --- a/modules/datasets/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 Anto Subash - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/modules/datasets/README.md b/modules/datasets/README.md deleted file mode 100644 index 62a855a9..00000000 --- a/modules/datasets/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# simple_module_datasets - -Geospatial + tabular dataset upload module for [simple_module](https://github.com/antosubash/simple_module_python) apps. Users upload CSV/GeoJSON/Shapefile; the module parses, slugs a canonical name, and stores geometry using `shapely`. - -## Install - -```bash -pip install simple_module_datasets -``` - -Also needs `simple_module_file_storage` + `simple_module_background_tasks` (declared as deps). - -## What it provides - -- `POST /api/datasets` — multipart upload; the file is staged via `simple_module_file_storage`, then a Celery job parses it in the background. -- `Dataset` SQLModel record with `name`, `slug` (via `python-slugify`), `geometry_type`, `row_count`, `bbox`. -- Shapely-backed parsers for GeoJSON, CSV with lat/lon columns, and zipped Shapefiles. -- Admin UI for browsing + deleting datasets. - -## Usage - -Upload from a form: - -```bash -curl -X POST -F "file=@cities.geojson" http://localhost:8000/api/datasets -``` - -Query parsed datasets: - -```python -from datasets.service import DatasetService # type: ignore[import-not-found] - -async def list_by_bbox(svc: DatasetService = Depends(DatasetService), ...): - return await svc.intersects(bbox=(-74.1, 40.6, -73.8, 40.9)) -``` - -## Depends on - -- `simple_module_core`, `simple_module_db`, `simple_module_hosting` -- `simple_module_file_storage`, `simple_module_background_tasks` -- `shapely>=2.0`, `python-slugify>=8.0`, `celery>=5.4` - -## License - -MIT — see [LICENSE](https://github.com/antosubash/simple_module_python/blob/main/LICENSE). diff --git a/modules/datasets/datasets/__init__.py b/modules/datasets/datasets/__init__.py deleted file mode 100644 index fb7083d7..00000000 --- a/modules/datasets/datasets/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Datasets module.""" diff --git a/modules/datasets/datasets/constants.py b/modules/datasets/datasets/constants.py deleted file mode 100644 index 584604a4..00000000 --- a/modules/datasets/datasets/constants.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Stable identifiers for the datasets module. - -Every string that identifies a permission, role, page, module dependency, -event, route, or config key lives here. Everything else in the module -imports from this file, so ``rg`` can prove the module has no scattered -magic strings. -""" - -from __future__ import annotations - -from typing import Final - -# ── Module identity ────────────────────────────────────────────────── -MODULE_NAME: Final = "datasets" -MODULE_PASCAL: Final = "Datasets" -MODULE_DISPLAY_NAME: Final = "Datasets" - -# ── Configuration ──────────────────────────────────────────────────── -ENV_PREFIX: Final = "SM_DATASETS_" - -# ── Routing ────────────────────────────────────────────────────────── -ROUTE_PREFIX_API: Final = "/api/datasets" -ROUTE_PREFIX_VIEW: Final = "/datasets" - -# REST sub-paths (joined to ROUTE_PREFIX_API by the router) -PATH_DOWNLOAD: Final = "/{dataset_id}/download" -PATH_DATASET: Final = "/{dataset_id}" - -# View routes (relative to ROUTE_PREFIX_VIEW, used in redirects) -REDIRECT_BROWSE: Final = "/datasets/" - -# ── Module dependencies (for ModuleMeta.depends_on) ────────────────── -MODULE_FILE_STORAGE: Final = "FileStorage" -MODULE_BACKGROUND_TASKS: Final = "BackgroundTasks" - -# ── i18n ───────────────────────────────────────────────────────────── -LOCALE_NAMESPACE: Final = MODULE_NAME - -# ── Inertia page identifiers ───────────────────────────────────────── -PAGE_BROWSE: Final = "Datasets/Browse" -PAGE_CREATE: Final = "Datasets/Create" -PAGE_EDIT: Final = "Datasets/Edit" -PAGE_SHOW: Final = "Datasets/Show" - -# ── Permissions ────────────────────────────────────────────────────── -PERM_DATASETS_VIEW: Final = "datasets.view" -PERM_DATASETS_UPLOAD: Final = "datasets.upload" -PERM_DATASETS_EDIT: Final = "datasets.edit" -PERM_DATASETS_DELETE: Final = "datasets.delete" - -PERMISSION_GROUP: Final = "Datasets" -ALL_PERMISSIONS: Final = ( - PERM_DATASETS_VIEW, - PERM_DATASETS_UPLOAD, - PERM_DATASETS_EDIT, - PERM_DATASETS_DELETE, -) - -# ── Menu ───────────────────────────────────────────────────────────── -MENU_LABEL: Final = "Datasets" -MENU_ICON: Final = "layers" -MENU_ORDER: Final = 40 - -# ── Celery tasks ───────────────────────────────────────────────────── -TASK_EXTRACT_METADATA: Final = "datasets.extract_metadata" - -# ── Feature flags ──────────────────────────────────────────────────── -FLAG_AUTO_EXTRACT: Final = "datasets.auto_extract" -FLAG_ALLOW_RASTER_UPLOADS: Final = "datasets.allow_raster_uploads" - -# ── Settings keys (registered with the settings module) ────────────── -SETTING_MAX_UPLOAD_MB: Final = "datasets.max_upload_mb" -SETTING_DEFAULT_KIND: Final = "datasets.default_kind" - -# ── Roles ──────────────────────────────────────────────────────────── -ROLE_USER: Final = "user" -ROLE_ADMIN: Final = "admin" - -# Permissions granted to plain ``user`` role via ``map_role``. ``admin`` -# inherits everything through the framework's wildcard, so we don't list -# it explicitly here. -USER_ROLE_PERMISSIONS: Final = ( - PERM_DATASETS_VIEW, - PERM_DATASETS_UPLOAD, -) - -# ── Health check ───────────────────────────────────────────────────── -HEALTH_CHECK_STORAGE: Final = "datasets.storage" - -# ── Storage ────────────────────────────────────────────────────────── -STORAGE_KEY_PREFIX: Final = "datasets/" - -# ── Database ───────────────────────────────────────────────────────── -SCHEMA_NAME: Final = "datasets" -TABLE_DATASET: Final = "datasets_dataset" - -# ── Defaults ───────────────────────────────────────────────────────── -DEFAULT_MAX_UPLOAD_MB: Final = 256 -DEFAULT_UPLOAD_CHUNK_SIZE: Final = 1024 * 1024 # 1 MB -DEFAULT_PRESIGN_TTL_SECONDS: Final = 300 # 5 minutes -DEFAULT_FALLBACK_FILENAME: Final = "upload.bin" -DEFAULT_MIME_TYPE: Final = "application/octet-stream" -DEFAULT_GEOJSON_CRS: Final = "EPSG:4326" - - -class DatasetKind: - """Built-in dataset kinds. - - The ``kind`` column is a free-form string so new providers can register - their own labels, but these are the ones the shipped extractors and - the UI know about. - """ - - VECTOR_GEOJSON: Final = "vector_geojson" - VECTOR_SHAPEFILE: Final = "vector_shapefile" - VECTOR_KML: Final = "vector_kml" - RASTER_GEOTIFF: Final = "raster_geotiff" - TABULAR_CSV: Final = "tabular_csv" - OTHER: Final = "other" - - -ALL_KINDS: Final = ( - DatasetKind.VECTOR_GEOJSON, - DatasetKind.VECTOR_SHAPEFILE, - DatasetKind.VECTOR_KML, - DatasetKind.RASTER_GEOTIFF, - DatasetKind.TABULAR_CSV, - DatasetKind.OTHER, -) - - -class ExtractionStatus: - """Values stored in ``Dataset.extraction_status``. - - ``pending`` is set by the upload endpoint; ``ok``/``partial``/``failed`` - are set by the Celery worker once extraction completes. ``manual`` is - the fallback for kinds the extractor doesn't understand; ``not_found`` - is only used as a task-result marker (never stored). - """ - - PENDING: Final = "pending" - OK: Final = "ok" - PARTIAL: Final = "partial" - FAILED: Final = "failed" - MANUAL: Final = "manual" - NOT_FOUND: Final = "not_found" diff --git a/modules/datasets/datasets/contracts/__init__.py b/modules/datasets/datasets/contracts/__init__.py deleted file mode 100644 index 8889bec9..00000000 --- a/modules/datasets/datasets/contracts/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Datasets contracts — public interface for other modules. - -Downstream modules should import from here, not from -``datasets.models`` or ``datasets.service`` internals. The SM009 -diagnostic enforces framework→plugin purity; this package is the -supported surface for plugin→plugin coupling:: - - from datasets.contracts import ( - DatasetOut, # DTO returned by service lookups - DatasetFile, # handle for stored file access - DatasetUploaded, # event subscribers listen for - download_url, # URL helper for UIs - ) - -For the FastAPI dependency, prefer -``from datasets.deps import DatasetServiceDep``. Type-hint against the -concrete ``DatasetService`` — the module is single-impl, so there's no -Protocol abstraction (matching the framework's "ship a Protocol only -for real extension points" rule). -""" - -from datasets.contracts.events import DatasetDeleted, DatasetUploaded -from datasets.contracts.files import DatasetFile -from datasets.contracts.schemas import ( - KIND_VALUES, - DatasetKind, - DatasetOut, - DatasetUpdate, -) -from datasets.contracts.urls import ( - API_PREFIX, - VIEW_PREFIX, - detail_url, - download_url, - show_url, -) - -__all__ = [ - "API_PREFIX", - "KIND_VALUES", - "VIEW_PREFIX", - "DatasetDeleted", - "DatasetFile", - "DatasetKind", - "DatasetOut", - "DatasetUpdate", - "DatasetUploaded", - "detail_url", - "download_url", - "show_url", -] diff --git a/modules/datasets/datasets/contracts/events.py b/modules/datasets/datasets/contracts/events.py deleted file mode 100644 index 19f4c5c9..00000000 --- a/modules/datasets/datasets/contracts/events.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Domain events emitted by the Datasets module. - -Events carry the handful of fields subscribers most often route on -(``slug``, ``kind``) so downstream handlers don't have to make a round -trip to ``IDatasetService`` just to decide whether the event is for -them. Anything beyond this — CRS, bbox, feature counts — is still a -``get_by_id`` away. -""" - -from __future__ import annotations - -from dataclasses import dataclass - -from simple_module_core.events import Event - - -@dataclass -class DatasetUploaded(Event): - dataset_id: int - name: str - slug: str - kind: str - - -@dataclass -class DatasetDeleted(Event): - dataset_id: int - slug: str diff --git a/modules/datasets/datasets/contracts/files.py b/modules/datasets/datasets/contracts/files.py deleted file mode 100644 index 48e84609..00000000 --- a/modules/datasets/datasets/contracts/files.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Public-facing value types for dataset file access. - -Consumers that depend on the Datasets module import ``DatasetFile`` rather -than the private ``Dataset`` SQLModel table. It carries enough to stream -bytes, hand off to a parser library, or materialise to a local path -without a second round-trip. - -Because the datasets module delegates bytes storage to -``file_storage.StorageBackend``, a dataset's bytes may live on any -backend (local FS today, S3 or GCS tomorrow). ``DatasetFile`` hides that -difference: consumers call :meth:`stream` or :meth:`materialize_to` and -never touch backend-specific APIs. -""" - -from __future__ import annotations - -import tempfile -from collections.abc import AsyncIterator -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING - -from datasets.contracts.schemas import DatasetOut - -if TYPE_CHECKING: - from file_storage.contracts.service import StorageBackend - - -@dataclass(frozen=True) -class DatasetFile: - """Read-only handle to a stored dataset file. - - The handle is decoupled from any specific storage backend. Consumers - that need bytes use :meth:`stream` (async iterator) or :meth:`read` - (``bytes``). Consumers that need a filesystem path — e.g. to hand - the file off to ``fiona`` / ``rasterio`` / ``pandas`` that require - ``str(path)`` — use :meth:`materialize_to`. - """ - - metadata: DatasetOut - storage_key: str - original_filename: str - mime_type: str | None - # Backend injected by the service — not part of the printable repr. - _backend: StorageBackend - - async def stream(self) -> AsyncIterator[bytes]: - """Yield the file's bytes in chunks. Works on any backend.""" - return await self._backend.get(self.storage_key) - - async def read(self) -> bytes: - """Read the entire file into memory. Prefer :meth:`stream` for large files.""" - buf = bytearray() - async for chunk in await self._backend.get(self.storage_key): - buf.extend(chunk) - return bytes(buf) - - async def exists(self) -> bool: - return await self._backend.exists(self.storage_key) - - async def materialize_to(self, path: Path) -> Path: - """Download the file to ``path``. Returns the path. - - Use for libraries that require a filesystem path (``fiona``, - ``rasterio``, ``pandas.read_csv``). For the filesystem backend - this is almost free; for S3 it pulls bytes down once. Callers - are responsible for deleting the file. - """ - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("wb") as fp: - async for chunk in await self._backend.get(self.storage_key): - fp.write(chunk) - return path - - async def materialize_to_tempfile(self, suffix: str | None = None) -> Path: - """Download to a named temp file. Returns the path. - - Convenience wrapper around :meth:`materialize_to` for the common - "I need a Path for fiona/rasterio, then I'll delete it" pattern. - Caller must ``unlink`` the returned path. - """ - effective_suffix = suffix if suffix is not None else Path(self.original_filename).suffix - fd, tmp = tempfile.mkstemp(suffix=effective_suffix) - # We don't need the file descriptor — ``materialize_to`` re-opens the path. - import os - - os.close(fd) - return await self.materialize_to(Path(tmp)) diff --git a/modules/datasets/datasets/contracts/schemas.py b/modules/datasets/datasets/contracts/schemas.py deleted file mode 100644 index d0d902ad..00000000 --- a/modules/datasets/datasets/contracts/schemas.py +++ /dev/null @@ -1,64 +0,0 @@ -"""SQLModel DTOs for the Datasets module.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Literal - -from pydantic import ConfigDict -from sqlmodel import Field, SQLModel - -from datasets import constants - -DatasetKind = Literal[ - "vector_geojson", - "vector_shapefile", - "vector_kml", - "raster_geotiff", - "tabular_csv", - "other", -] - -# Duplicated from ``constants.ALL_KINDS`` because ``typing.Literal`` demands -# string literals at type-evaluation time — a tuple reference won't satisfy -# it. The runtime check against ``constants.ALL_KINDS`` is the one that -# matters; this Literal only narrows the ``DatasetUpdate.kind`` type. -KIND_VALUES: tuple[str, ...] = constants.ALL_KINDS - - -class DatasetOut(SQLModel): - """Dataset metadata returned by the API.""" - - model_config = ConfigDict(from_attributes=True) - - id: int - name: str - slug: str - kind: str - description: str | None = None - original_filename: str - mime_type: str | None = None - size_bytes: int - crs: str | None = None - bbox_min_x: float | None = None - bbox_min_y: float | None = None - bbox_max_x: float | None = None - bbox_max_y: float | None = None - feature_count: int | None = None - band_count: int | None = None - extraction_status: str - created_at: datetime | None = None - updated_at: datetime | None = None - - -class DatasetUpdate(SQLModel): - """Patchable metadata. Files are immutable after upload — re-upload to replace.""" - - name: str | None = Field(default=None, min_length=1, max_length=200) - description: str | None = Field(default=None, max_length=2000) - kind: DatasetKind | None = None - crs: str | None = Field(default=None, max_length=64) - bbox_min_x: float | None = None - bbox_min_y: float | None = None - bbox_max_x: float | None = None - bbox_max_y: float | None = None diff --git a/modules/datasets/datasets/contracts/urls.py b/modules/datasets/datasets/contracts/urls.py deleted file mode 100644 index 680424d9..00000000 --- a/modules/datasets/datasets/contracts/urls.py +++ /dev/null @@ -1,26 +0,0 @@ -"""URL builders consuming modules can use without hard-coding prefixes. - -If the Datasets module's ``route_prefix`` ever changes, downstream modules -don't have to audit every hard-coded ``/api/datasets/...`` string — they -just re-import these helpers. -""" - -from __future__ import annotations - -API_PREFIX = "/api/datasets" -VIEW_PREFIX = "/datasets" - - -def download_url(dataset_id: int) -> str: - """URL that streams the dataset's stored file back to the client.""" - return f"{API_PREFIX}/{dataset_id}/download" - - -def detail_url(dataset_id: int) -> str: - """URL of the dataset's JSON detail endpoint.""" - return f"{API_PREFIX}/{dataset_id}" - - -def show_url(dataset_id: int) -> str: - """URL of the Inertia ``Show`` page.""" - return f"{VIEW_PREFIX}/{dataset_id}" diff --git a/modules/datasets/datasets/deps.py b/modules/datasets/datasets/deps.py deleted file mode 100644 index 25270e4b..00000000 --- a/modules/datasets/datasets/deps.py +++ /dev/null @@ -1,97 +0,0 @@ -"""FastAPI dependencies for the Datasets module. - -Downstream modules that depend on ``Datasets`` import -``DatasetServiceDep`` directly:: - - from datasets.deps import DatasetServiceDep - - @router.get("/my-thing") - async def endpoint(datasets: DatasetServiceDep): - ds = await datasets.get_by_slug("world-borders") - ... - -The storage backend comes from the ``file_storage`` module's app-state -slot. That seam is what lets datasets work on local FS today and S3 -tomorrow without touching this module. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Annotated - -from fastapi import Depends, Request -from file_storage.contracts.service import StorageBackend -from simple_module_core.events import EventBus -from simple_module_db.deps import get_db -from sqlalchemy.ext.asyncio import AsyncSession - -from datasets.service import DatasetService - -if TYPE_CHECKING: - from celery import Celery - - -def get_storage_backend(request: Request) -> StorageBackend: - return request.app.state.file_storage.backend - - -async def get_dataset_service( - db: AsyncSession = Depends(get_db), - backend: StorageBackend = Depends(get_storage_backend), -) -> DatasetService: - return DatasetService(db, backend) - - -def get_event_bus(request: Request) -> EventBus: - return request.app.state.sm.event_bus - - -def get_celery(request: Request) -> Celery: - """Return the Celery app singleton owned by the background_tasks module. - - Depending on this means the datasets module won't boot unless - ``BackgroundTasks`` ran its ``on_startup`` first — which is enforced - via ``meta.depends_on``. - """ - return request.app.state.background_tasks.celery - - -def get_max_upload_bytes(request: Request) -> int: - """Resolve the max upload size. - - Prefers the runtime ``datasets.max_upload_mb`` value from the settings - module (so admins can tune it without a redeploy), falling back to the - env-var default on ``app.state.datasets.settings``. - """ - env_default = request.app.state.datasets.settings.max_upload_mb - override = _runtime_max_upload_mb(request, fallback=env_default) - return max(override, 1) * 1024 * 1024 - - -def _runtime_max_upload_mb(request: Request, *, fallback: int) -> int: - """Read the ``datasets.max_upload_mb`` SYSTEM scope setting. - - Returns ``fallback`` if the ``settings`` module isn't installed, the - registered accessor isn't available synchronously, or the stored value - can't be parsed as an int. Synchronous read happens through the - registry default when no DB row exists — we deliberately avoid doing - a DB query in a hot DI path. - """ - registry = getattr(getattr(request.app.state, "settings", None), "registry", None) - if registry is None: - return fallback - from datasets import constants - - definition = registry.get(constants.SETTING_MAX_UPLOAD_MB) - if definition is None: - return fallback - try: - return int(definition.default) - except (TypeError, ValueError): - return fallback - - -# Public type alias consumers can import directly — shortens -# ``service: DatasetService = Depends(get_dataset_service)`` to just -# ``datasets: DatasetServiceDep``. -DatasetServiceDep = Annotated[DatasetService, Depends(get_dataset_service)] diff --git a/modules/datasets/datasets/endpoints/__init__.py b/modules/datasets/datasets/endpoints/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/modules/datasets/datasets/endpoints/api.py b/modules/datasets/datasets/endpoints/api.py deleted file mode 100644 index 03833a99..00000000 --- a/modules/datasets/datasets/endpoints/api.py +++ /dev/null @@ -1,238 +0,0 @@ -"""REST API endpoints for the Datasets module.""" - -from __future__ import annotations - -import logging -import tempfile -from pathlib import Path - -from celery import Celery -from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile -from fastapi.responses import RedirectResponse, StreamingResponse -from file_storage.contracts.service import NotSupportedError, StorageNotFoundError -from simple_module_core.events import EventBus -from simple_module_core.feature_flags import is_flag_enabled -from simple_module_hosting.permissions import RequiresPermission - -from datasets import constants -from datasets.contracts.events import DatasetDeleted, DatasetUploaded -from datasets.contracts.schemas import DatasetOut, DatasetUpdate -from datasets.deps import ( - get_celery, - get_dataset_service, - get_event_bus, - get_max_upload_bytes, -) -from datasets.service import DatasetService, UploadInput - -logger = logging.getLogger(__name__) - -router = APIRouter() - - -def _enqueue_extraction(celery: Celery, dataset_id: int) -> None: - """Enqueue the extraction task, absorbing broker failures. - - A dead Redis shouldn't fail the upload — the Dataset row is already - persisted with ``extraction_status="pending"`` and can be re-extracted - later. We log the failure so the operator can see it and retry. - """ - try: - celery.send_task(constants.TASK_EXTRACT_METADATA, args=[dataset_id]) - except Exception: - logger.exception( - "Failed to enqueue %s for dataset %s — row will stay pending", - constants.TASK_EXTRACT_METADATA, - dataset_id, - ) - - -@router.get("/", response_model=list[DatasetOut]) -async def list_datasets( - service: DatasetService = Depends(get_dataset_service), -) -> list[DatasetOut]: - return await service.get_all() - - -@router.get(constants.PATH_DATASET, response_model=DatasetOut) -async def get_dataset( - dataset_id: int, - service: DatasetService = Depends(get_dataset_service), -) -> DatasetOut: - item = await service.get_by_id(dataset_id) - if item is None: - raise HTTPException(status_code=404, detail="Dataset not found") - return item - - -@router.get(constants.PATH_DOWNLOAD) -async def download_dataset( - dataset_id: int, - service: DatasetService = Depends(get_dataset_service), -): - handle = await service.get_file(dataset_id) - if handle is None: - raise HTTPException(status_code=404, detail="Dataset not found") - - # Presigned URLs let the client bypass the app entirely for S3-like - # backends — avoids proxying multi-GB files through the worker. - if service.backend.supports_presigned_url: - try: - url = await service.backend.presigned_get_url( - handle.storage_key, ttl_seconds=constants.DEFAULT_PRESIGN_TTL_SECONDS - ) - except NotSupportedError: - url = None - if url is not None: - return RedirectResponse(url, status_code=302) - - try: - stream = await handle.stream() - except StorageNotFoundError: - raise HTTPException( - status_code=410, detail="Dataset file is missing from storage" - ) from None - - disposition = f'attachment; filename="{handle.original_filename}"' - return StreamingResponse( - stream, - media_type=handle.mime_type or constants.DEFAULT_MIME_TYPE, - headers={"Content-Disposition": disposition}, - ) - - -async def perform_upload( - request: Request, - name: str, - description: str | None, - kind: str | None, - file: UploadFile, - service: DatasetService, - bus: EventBus, - celery: Celery, - max_upload_bytes: int, -) -> DatasetOut: - if kind is not None and kind not in constants.ALL_KINDS: - raise HTTPException(status_code=422, detail=f"Unknown kind: {kind}") - if kind == constants.DatasetKind.RASTER_GEOTIFF and not is_flag_enabled( - request, constants.FLAG_ALLOW_RASTER_UPLOADS - ): - raise HTTPException( - status_code=422, - detail="Raster uploads are disabled on this instance.", - ) - - original_filename = file.filename or constants.DEFAULT_FALLBACK_FILENAME - bytes_written = 0 - # Spool to a temp file first so size-validation can reject before we - # touch the storage backend, and so the service can hand a complete - # stream to backend.put(). - with tempfile.NamedTemporaryFile(delete=False) as tmp: - tmp_path = Path(tmp.name) - try: - while True: - chunk = await file.read(constants.DEFAULT_UPLOAD_CHUNK_SIZE) - if not chunk: - break - bytes_written += len(chunk) - if bytes_written > max_upload_bytes: - raise HTTPException( - status_code=413, - detail=f"Upload exceeds {max_upload_bytes} bytes", - ) - tmp.write(chunk) - except HTTPException: - tmp.close() - tmp_path.unlink(missing_ok=True) - raise - - try: - dataset = await service.register_upload( - UploadInput( - name=name, - original_filename=original_filename, - temp_path=tmp_path, - size_bytes=bytes_written, - mime_type=file.content_type, - description=description, - kind=kind, - ) - ) - finally: - tmp_path.unlink(missing_ok=True) - - # Hand metadata extraction off to a Celery worker. The Dataset row is - # already ``extraction_status="pending"`` — the worker flips it to - # ok / partial / failed once the parse completes. See - # ``datasets.tasks.extract_metadata_task``. Guarded by a feature - # flag so an admin can freeze auto-extraction during incidents - # without redeploying. - if is_flag_enabled(request, constants.FLAG_AUTO_EXTRACT): - _enqueue_extraction(celery, dataset.id) - - await bus.publish( - DatasetUploaded( - dataset_id=dataset.id, - name=dataset.name, - slug=dataset.slug, - kind=dataset.kind, - ) - ) - return dataset - - -@router.post( - "/", - response_model=DatasetOut, - status_code=201, - dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_UPLOAD))], -) -async def upload_dataset( - request: Request, - name: str = Form(..., min_length=1, max_length=200), - description: str | None = Form(default=None, max_length=2000), - kind: str | None = Form(default=None), - file: UploadFile = File(...), - service: DatasetService = Depends(get_dataset_service), - bus: EventBus = Depends(get_event_bus), - celery: Celery = Depends(get_celery), - max_upload_bytes: int = Depends(get_max_upload_bytes), -) -> DatasetOut: - return await perform_upload( - request, name, description, kind, file, service, bus, celery, max_upload_bytes - ) - - -@router.patch( - constants.PATH_DATASET, - response_model=DatasetOut, - dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_EDIT))], -) -async def update_dataset( - dataset_id: int, - data: DatasetUpdate, - service: DatasetService = Depends(get_dataset_service), -) -> DatasetOut: - item = await service.update(dataset_id, data) - if item is None: - raise HTTPException(status_code=404, detail="Dataset not found") - return item - - -@router.delete( - constants.PATH_DATASET, - status_code=204, - dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_DELETE))], -) -async def delete_dataset( - dataset_id: int, - service: DatasetService = Depends(get_dataset_service), - bus: EventBus = Depends(get_event_bus), -) -> None: - # Capture the slug before deletion so subscribers can index by slug - # without a post-delete lookup (which would always miss). - existing = await service.get_by_id(dataset_id) - if existing is None: - raise HTTPException(status_code=404, detail="Dataset not found") - await service.delete(dataset_id) - await bus.publish(DatasetDeleted(dataset_id=dataset_id, slug=existing.slug)) diff --git a/modules/datasets/datasets/endpoints/views.py b/modules/datasets/datasets/endpoints/views.py deleted file mode 100644 index 69a7b63d..00000000 --- a/modules/datasets/datasets/endpoints/views.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Inertia view endpoints for the Datasets module.""" - -from __future__ import annotations - -from celery import Celery -from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile -from inertia import InertiaResponse -from simple_module_core.events import EventBus -from simple_module_hosting.inertia_deps import InertiaDep -from simple_module_hosting.permissions import RequiresPermission -from starlette.responses import RedirectResponse - -from datasets import constants -from datasets.contracts.events import DatasetDeleted -from datasets.contracts.schemas import DatasetUpdate -from datasets.deps import ( - get_celery, - get_dataset_service, - get_event_bus, - get_max_upload_bytes, -) -from datasets.endpoints.api import perform_upload -from datasets.service import DatasetService - -# Module-local Inertia page identifiers. These must be Name-only literal -# assignments (not attribute access against ``constants``) so the SM003 -# orphan-page diagnostic can resolve them — see -# ``simple_module_core.diagnostics._module._iter_render_components``. -_PAGE_BROWSE = "Datasets/Browse" -_PAGE_CREATE = "Datasets/Create" -_PAGE_SHOW = "Datasets/Show" -_PAGE_EDIT = "Datasets/Edit" - -router = APIRouter() - - -@router.get("/", response_model=None) -async def browse( - inertia: InertiaDep, - service: DatasetService = Depends(get_dataset_service), -) -> InertiaResponse: - items = await service.get_all() - return await inertia.render( - _PAGE_BROWSE, - {"datasets": [item.model_dump(mode="json") for item in items]}, - ) - - -@router.get( - "/create", - response_model=None, - dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_UPLOAD))], -) -async def create_view(inertia: InertiaDep) -> InertiaResponse: - return await inertia.render(_PAGE_CREATE) - - -@router.post( - "/", - response_model=None, - dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_UPLOAD))], -) -async def upload_view( - request: Request, - name: str = Form(..., min_length=1, max_length=200), - description: str | None = Form(default=None, max_length=2000), - kind: str | None = Form(default=None), - file: UploadFile = File(...), - service: DatasetService = Depends(get_dataset_service), - bus: EventBus = Depends(get_event_bus), - celery: Celery = Depends(get_celery), - max_upload_bytes: int = Depends(get_max_upload_bytes), -) -> RedirectResponse: - # Inertia's client-side router expects a redirect after POST. Return 303 - # so it re-issues a GET against the browse page, which replies with a - # full Inertia response. - await perform_upload( - request, name, description, kind, file, service, bus, celery, max_upload_bytes - ) - return RedirectResponse(constants.REDIRECT_BROWSE, status_code=303) - - -@router.get("/{dataset_id}", response_model=None) -async def show_view( - dataset_id: int, - inertia: InertiaDep, - service: DatasetService = Depends(get_dataset_service), -) -> InertiaResponse: - item = await service.get_by_id(dataset_id) - if item is None: - return await inertia.render( - _PAGE_BROWSE, - {"datasets": [], "error": "Dataset not found"}, - ) - return await inertia.render(_PAGE_SHOW, {"dataset": item.model_dump(mode="json")}) - - -@router.get( - "/{dataset_id}/edit", - response_model=None, - dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_EDIT))], -) -async def edit_view( - dataset_id: int, - inertia: InertiaDep, - service: DatasetService = Depends(get_dataset_service), -) -> InertiaResponse: - item = await service.get_by_id(dataset_id) - if item is None: - return await inertia.render( - _PAGE_BROWSE, - {"datasets": [], "error": "Dataset not found"}, - ) - return await inertia.render(_PAGE_EDIT, {"dataset": item.model_dump(mode="json")}) - - -@router.patch( - "/{dataset_id}", - response_model=None, - dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_EDIT))], -) -async def update_view( - dataset_id: int, - data: DatasetUpdate, - service: DatasetService = Depends(get_dataset_service), -) -> RedirectResponse: - item = await service.update(dataset_id, data) - if item is None: - raise HTTPException(status_code=404, detail="Dataset not found") - return RedirectResponse(constants.REDIRECT_BROWSE, status_code=303) - - -@router.delete( - "/{dataset_id}", - response_model=None, - dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_DELETE))], -) -async def delete_view( - dataset_id: int, - service: DatasetService = Depends(get_dataset_service), - bus: EventBus = Depends(get_event_bus), -) -> RedirectResponse: - existing = await service.get_by_id(dataset_id) - if existing is None: - raise HTTPException(status_code=404, detail="Dataset not found") - await service.delete(dataset_id) - await bus.publish(DatasetDeleted(dataset_id=dataset_id, slug=existing.slug)) - return RedirectResponse(constants.REDIRECT_BROWSE, status_code=303) diff --git a/modules/datasets/datasets/extractors.py b/modules/datasets/datasets/extractors.py deleted file mode 100644 index bfb9a9a0..00000000 --- a/modules/datasets/datasets/extractors.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Best-effort metadata extraction for uploaded datasets. - -GeoJSON is handled with ``shapely`` (a standard geospatial dependency). -Optional ``fiona`` and ``rasterio`` extras enrich extraction for -shapefiles, KML, and rasters when available. Every extractor degrades -to ``ExtractionStatus.MANUAL`` so the catalog never blocks an upload -because a parser is missing. -""" - -from __future__ import annotations - -import json -from dataclasses import dataclass -from pathlib import Path - -from shapely.geometry import shape - -from datasets import constants -from datasets.constants import DatasetKind, ExtractionStatus - - -@dataclass -class ExtractedMeta: - crs: str | None = None - bbox_min_x: float | None = None - bbox_min_y: float | None = None - bbox_max_x: float | None = None - bbox_max_y: float | None = None - feature_count: int | None = None - band_count: int | None = None - status: str = ExtractionStatus.MANUAL - - -_EXTENSION_TO_KIND: dict[str, str] = { - ".geojson": DatasetKind.VECTOR_GEOJSON, - ".json": DatasetKind.VECTOR_GEOJSON, - ".shp": DatasetKind.VECTOR_SHAPEFILE, - ".zip": DatasetKind.VECTOR_SHAPEFILE, - ".kml": DatasetKind.VECTOR_KML, - ".kmz": DatasetKind.VECTOR_KML, - ".tif": DatasetKind.RASTER_GEOTIFF, - ".tiff": DatasetKind.RASTER_GEOTIFF, - ".csv": DatasetKind.TABULAR_CSV, -} - - -def kind_for_filename(filename: str) -> str: - """Map a filename to a coarse kind label. - - The mapping is intentionally cheap — anything we don't recognise becomes - ``DatasetKind.OTHER`` so the upload still lands and the user can correct - the kind via the edit form. - """ - suffix = Path(filename).suffix.lower() - return _EXTENSION_TO_KIND.get(suffix, DatasetKind.OTHER) - - -def extract_metadata(path: Path, kind: str) -> ExtractedMeta: - """Dispatch to a kind-specific extractor. - - Never raises — extraction failures collapse to - ``ExtractionStatus.FAILED`` so the upload itself succeeds and the user - can fill the metadata in. - """ - try: - if kind == DatasetKind.VECTOR_GEOJSON: - return _extract_geojson(path) - if kind == DatasetKind.RASTER_GEOTIFF: - return _extract_raster(path) - if kind in {DatasetKind.VECTOR_SHAPEFILE, DatasetKind.VECTOR_KML}: - return _extract_via_fiona(path) - except Exception: - return ExtractedMeta(status=ExtractionStatus.FAILED) - return ExtractedMeta(status=ExtractionStatus.MANUAL) - - -def _extract_geojson(path: Path) -> ExtractedMeta: - with path.open("rb") as fp: - doc = json.load(fp) - - features: list[dict] - if isinstance(doc, dict) and doc.get("type") == "FeatureCollection": - features = [f for f in doc.get("features", []) if isinstance(f, dict)] - elif isinstance(doc, dict) and doc.get("type") == "Feature": - features = [doc] - else: - features = [] - - bbox = doc.get("bbox") if isinstance(doc, dict) else None - if not bbox: - bounds = [ - shape(f["geometry"]).bounds for f in features if isinstance(f.get("geometry"), dict) - ] - if bounds: - bbox = ( - min(b[0] for b in bounds), - min(b[1] for b in bounds), - max(b[2] for b in bounds), - max(b[3] for b in bounds), - ) - - if not bbox: - return ExtractedMeta( - crs=constants.DEFAULT_GEOJSON_CRS, - feature_count=len(features), - status=ExtractionStatus.PARTIAL, - ) - return ExtractedMeta( - crs=constants.DEFAULT_GEOJSON_CRS, - bbox_min_x=float(bbox[0]), - bbox_min_y=float(bbox[1]), - bbox_max_x=float(bbox[2]), - bbox_max_y=float(bbox[3]), - feature_count=len(features), - status=ExtractionStatus.OK, - ) - - -def _extract_via_fiona(path: Path) -> ExtractedMeta: - try: - import fiona # type: ignore[import-not-found] # ty: ignore[unresolved-import] - except ImportError: - return ExtractedMeta(status=ExtractionStatus.MANUAL) - - with fiona.open(str(path)) as src: - bounds = src.bounds - crs = src.crs.get("init") if hasattr(src.crs, "get") else str(src.crs) if src.crs else None - return ExtractedMeta( - crs=crs, - bbox_min_x=float(bounds[0]), - bbox_min_y=float(bounds[1]), - bbox_max_x=float(bounds[2]), - bbox_max_y=float(bounds[3]), - feature_count=len(src), - status=ExtractionStatus.OK, - ) - - -def _extract_raster(path: Path) -> ExtractedMeta: - try: - import rasterio # type: ignore[import-not-found] # ty: ignore[unresolved-import] - except ImportError: - return ExtractedMeta(status=ExtractionStatus.MANUAL) - - with rasterio.open(str(path)) as src: - bounds = src.bounds - return ExtractedMeta( - crs=str(src.crs) if src.crs else None, - bbox_min_x=float(bounds.left), - bbox_min_y=float(bounds.bottom), - bbox_max_x=float(bounds.right), - bbox_max_y=float(bounds.top), - band_count=src.count, - status=ExtractionStatus.OK, - ) diff --git a/modules/datasets/datasets/locales/en.json b/modules/datasets/datasets/locales/en.json deleted file mode 100644 index b49266d1..00000000 --- a/modules/datasets/datasets/locales/en.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "browse": { - "title": "Datasets", - "description": "Upload and manage datasets", - "new_button": "Upload Dataset", - "empty_title": "No datasets yet", - "empty_description": "Upload your first dataset to get started.", - "create_button": "Upload Dataset" - }, - "table": { - "name": "Name", - "kind": "Kind", - "crs": "CRS", - "bbox": "Bounding Box", - "size": "Size", - "actions": "Actions" - }, - "form": { - "name_label": "Name", - "name_placeholder": "Descriptive name for the dataset", - "description_label": "Description", - "description_placeholder": "Optional notes about the dataset", - "kind_label": "Kind", - "kind_auto": "Auto-detect from filename", - "file_label": "File", - "crs_label": "CRS", - "cancel_button": "Cancel" - }, - "create": { - "title": "Upload Dataset", - "description": "Add a dataset to the catalog", - "submit_button": "Upload", - "submitting_button": "Uploading..." - }, - "edit": { - "title": "Edit: {name}", - "description": "Update dataset metadata", - "back_button": "Back to Datasets", - "submit_button": "Save Changes", - "submitting_button": "Saving..." - }, - "show": { - "download": "Download", - "edit": "Edit", - "kind": "Kind", - "original_file": "Original filename", - "size": "Size", - "mime_type": "MIME type", - "crs": "CRS", - "bbox": "Bounding box", - "features": "Feature count", - "bands": "Band count", - "extraction_status": "Extraction status" - }, - "toasts": { - "created": "Dataset uploaded", - "updated": "Dataset updated", - "deleted": "Dataset deleted" - }, - "validation": { - "name_required": "Name is required", - "file_required": "Please choose a file to upload" - }, - "errors": { - "not_found": "Dataset not found" - } -} diff --git a/modules/datasets/datasets/models.py b/modules/datasets/datasets/models.py deleted file mode 100644 index 8c19e5cd..00000000 --- a/modules/datasets/datasets/models.py +++ /dev/null @@ -1,49 +0,0 @@ -"""SQLModel tables for the Datasets module.""" - -from __future__ import annotations - -from simple_module_db.base import create_module_base -from simple_module_db.mixins import AuditMixin, SoftDeleteMixin -from sqlalchemy import Index -from sqlmodel import Field - -from datasets import constants - -# Provider is auto-detected from SM_DATABASE_URL (falls back to SQLite). -# On PostgreSQL this gives the module its own schema; on SQLite all modules -# share one schema, so __tablename__ is prefixed for isolation. -Base = create_module_base(constants.SCHEMA_NAME) - - -class Dataset(Base, AuditMixin, SoftDeleteMixin, table=True): # ty: ignore[unsupported-base] - """A dataset uploaded into the catalog. - - ``kind`` identifies the content type (vector GeoJSON, shapefile, - raster GeoTIFF, tabular CSV, ...) — see - :class:`datasets.constants.DatasetKind`. Geospatial fields (``crs``, - ``bbox_*``) stay null for non-geospatial kinds. - """ - - __tablename__ = constants.TABLE_DATASET - - id: int | None = Field(default=None, primary_key=True) - name: str = Field(max_length=200) - slug: str = Field(max_length=200, unique=True, index=True) - kind: str = Field(max_length=32, index=True) - description: str | None = Field(default=None, max_length=2000) - - original_filename: str = Field(max_length=255) - mime_type: str | None = Field(default=None, max_length=127) - size_bytes: int = Field(default=0) - storage_key: str = Field(max_length=512) - - crs: str | None = Field(default=None, max_length=64) - bbox_min_x: float | None = Field(default=None) - bbox_min_y: float | None = Field(default=None) - bbox_max_x: float | None = Field(default=None) - bbox_max_y: float | None = Field(default=None) - feature_count: int | None = Field(default=None) - band_count: int | None = Field(default=None) - extraction_status: str = Field(default=constants.ExtractionStatus.MANUAL, max_length=16) - - __table_args__ = (Index("ix_datasets_dataset_is_deleted", "is_deleted"),) diff --git a/modules/datasets/datasets/module.py b/modules/datasets/datasets/module.py deleted file mode 100644 index fc916d7b..00000000 --- a/modules/datasets/datasets/module.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Datasets module definition. - -Depends on ``FileStorage`` for bytes storage, ``BackgroundTasks`` for -the Celery pipeline the upload endpoint enqueues into, ``Permissions`` -for the permission system, ``FeatureFlags`` for runtime toggles, and -``Settings`` for admin-configurable limits. ``register_settings`` here -runs after those, so every ``app.state.*`` slot is populated before any -Datasets request fires. -""" - -from __future__ import annotations - -import importlib.resources -from pathlib import Path -from typing import TYPE_CHECKING - -from fastapi import APIRouter, FastAPI -from simple_module_core.feature_flags import FeatureFlagDefinition, FeatureFlagRegistry -from simple_module_core.menu import MenuItem, MenuRegistry, MenuSection -from simple_module_core.module import ModuleBase, ModuleMeta -from simple_module_core.permissions import PermissionRegistry - -from datasets import constants - -if TYPE_CHECKING: - from settings.contracts.registry import SettingDefinition - - -class DatasetsModule(ModuleBase): - meta = ModuleMeta( - name=constants.MODULE_PASCAL, - route_prefix=constants.ROUTE_PREFIX_API, - view_prefix=constants.ROUTE_PREFIX_VIEW, - depends_on=[constants.MODULE_FILE_STORAGE, constants.MODULE_BACKGROUND_TASKS], - ) - - def register_settings(self, app: FastAPI) -> None: - import importlib - - from datasets.services import DatasetsServices - from datasets.settings import DatasetsSettings - - # SM009 is AST-based: a static `from settings.registration import ...` - # from a module helper is fine (plugin→plugin), but we resolve via - # importlib here to match the convention used framework-side and to - # keep the dependency direction one-way explicit. - register_module_settings = importlib.import_module( - "settings.registration" - ).register_module_settings - - register_module_settings( - app, - "datasets", - DatasetsSettings, - lambda s: DatasetsServices(settings=s), - ) - - def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None: - from datasets.endpoints.api import router as api - from datasets.endpoints.views import router as views - - api_router.include_router(api) - view_router.include_router(views) - - def register_menu_items(self, registry: MenuRegistry) -> None: - registry.add( - MenuItem( - label=constants.MENU_LABEL, - url=constants.ROUTE_PREFIX_VIEW, - icon=constants.MENU_ICON, - order=constants.MENU_ORDER, - section=MenuSection.SIDEBAR, - ) - ) - - def register_permissions(self, registry: PermissionRegistry) -> None: - registry.add_group( - constants.PERMISSION_GROUP, - list(constants.ALL_PERMISSIONS), - ) - # Plain users can browse the catalog and upload their own datasets; - # edit/delete stay admin-only via the framework wildcard. - registry.map_role( - constants.ROLE_USER, - list(constants.USER_ROLE_PERMISSIONS), - ) - - def register_feature_flags(self, registry: FeatureFlagRegistry) -> None: - registry.add( - FeatureFlagDefinition( - name=constants.FLAG_AUTO_EXTRACT, - description=( - "Enqueue the Celery metadata-extraction task on upload. " - "Turn off to skip the worker hop and leave rows as " - "``extraction_status=pending`` for manual review." - ), - default_enabled=True, - ) - ) - registry.add( - FeatureFlagDefinition( - name=constants.FLAG_ALLOW_RASTER_UPLOADS, - description=( - "Accept raster (GeoTIFF) uploads. Disable on instances " - "without ``rasterio`` installed to give users a clear " - "422 instead of a silent failed extraction." - ), - default_enabled=True, - ) - ) - - def locale_dirs(self) -> dict[str, Path]: - base = Path(str(importlib.resources.files(__package__) / "locales")) - return {constants.LOCALE_NAMESPACE: base} - - async def on_startup(self, app: FastAPI) -> None: - """Register runtime-tunable settings with the ``settings`` module. - - Registered as ``on_startup`` (not in ``register_settings``) because - the settings module's ``app.state.settings`` slot is only populated - once its own ``register_settings`` has run — ``on_startup`` fires - after every module's registration is done. - """ - # The settings module may not be installed in every deployment — - # treat its absence as a warn, not a crash. - registry = getattr(getattr(app.state, "settings", None), "registry", None) - if registry is None: - return - for definition in _setting_definitions(): - if definition.key in registry: - continue - registry.add(definition) - - -def _setting_definitions() -> list[SettingDefinition]: - """Deferred import so the module still loads if ``settings`` is absent.""" - from settings.contracts.registry import SettingDefinition - from settings.contracts.schemas import SettingValueType - - return [ - SettingDefinition( - key=constants.SETTING_MAX_UPLOAD_MB, - default=str(constants.DEFAULT_MAX_UPLOAD_MB), - description=( - "Per-dataset upload size cap in megabytes. Overrides the " - "pydantic default on ``DatasetsSettings.max_upload_mb`` at " - "runtime." - ), - value_type=SettingValueType.INT, - ), - SettingDefinition( - key=constants.SETTING_DEFAULT_KIND, - default=constants.DatasetKind.OTHER, - description=( - "Dataset kind assigned when filename-based detection comes " - "back as ``other``. Useful for instances that know they only " - "ingest one kind (e.g. always ``vector_geojson``)." - ), - ), - ] diff --git a/modules/datasets/datasets/pages/Browse.tsx b/modules/datasets/datasets/pages/Browse.tsx deleted file mode 100644 index 1072f090..00000000 --- a/modules/datasets/datasets/pages/Browse.tsx +++ /dev/null @@ -1,165 +0,0 @@ -import { Link, router, usePage } from '@inertiajs/react'; -import { keys, useT } from '@simple-module-py/i18n'; -import { PageShell } from '@simple-module-py/ui/components/PageShell'; -import { Badge } from '@simple-module-py/ui/components/ui/badge'; -import { Button } from '@simple-module-py/ui/components/ui/button'; -import { Card } from '@simple-module-py/ui/components/ui/card'; -import { - Empty, - EmptyDescription, - EmptyMedia, - EmptyTitle, -} from '@simple-module-py/ui/components/ui/empty'; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from '@simple-module-py/ui/components/ui/table'; -import { usePermissions } from '@simple-module-py/ui/hooks/use-permissions'; -import { AuthenticatedLayout } from '@simple-module-py/ui/layouts/AuthenticatedLayout'; -import { Layers, Plus, Trash2 } from 'lucide-react'; - -interface Dataset { - id: number; - name: string; - slug: string; - kind: string; - size_bytes: number; - crs: string | null; - bbox_min_x: number | null; - bbox_min_y: number | null; - bbox_max_x: number | null; - bbox_max_y: number | null; - extraction_status: string; - created_at: string | null; -} - -interface Props { - datasets: Dataset[]; -} - -function formatBytes(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; - return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; -} - -function formatBbox(d: Dataset): string { - if ( - d.bbox_min_x === null || - d.bbox_min_y === null || - d.bbox_max_x === null || - d.bbox_max_y === null - ) { - return '—'; - } - return `[${d.bbox_min_x.toFixed(3)}, ${d.bbox_min_y.toFixed(3)}, ${d.bbox_max_x.toFixed(3)}, ${d.bbox_max_y.toFixed(3)}]`; -} - -function Browse() { - const { datasets } = usePage<{ props: Props }>().props as unknown as Props; - const { t } = useT(); - const { can } = usePermissions(); - const canUpload = can('datasets.upload'); - const canDelete = can('datasets.delete'); - - function handleDelete(dataset: Dataset) { - router.delete(`/datasets/${dataset.id}`, { preserveScroll: true }); - } - - return ( - - - - {t(keys.datasets.browse.new_button)} - - - ) : undefined - } - > - - - - - {t(keys.datasets.table.name)} - {t(keys.datasets.table.kind)} - - {t(keys.datasets.table.crs)} - - - {t(keys.datasets.table.bbox)} - - {t(keys.datasets.table.size)} - {t(keys.datasets.table.actions)} - - - - {datasets.map((dataset) => ( - - - - {dataset.name} - - - - {dataset.kind} - - - {dataset.crs || '—'} - - - {formatBbox(dataset)} - - - {formatBytes(dataset.size_bytes)} - - - {canDelete && ( - - )} - - - ))} - {datasets.length === 0 && ( - - - - - - - {t(keys.datasets.browse.empty_title)} - {t(keys.datasets.browse.empty_description)} - {canUpload && ( - - )} - - - - )} - -
- - - ); -} - -Browse.layout = (page: React.ReactNode) => {page}; -export default Browse; diff --git a/modules/datasets/datasets/pages/Create.tsx b/modules/datasets/datasets/pages/Create.tsx deleted file mode 100644 index 4be2b85d..00000000 --- a/modules/datasets/datasets/pages/Create.tsx +++ /dev/null @@ -1,144 +0,0 @@ -import { Link, router } from '@inertiajs/react'; -import { keys, useT } from '@simple-module-py/i18n'; -import { PageShell } from '@simple-module-py/ui/components/PageShell'; -import { Button } from '@simple-module-py/ui/components/ui/button'; -import { Card, CardContent } from '@simple-module-py/ui/components/ui/card'; -import { Input } from '@simple-module-py/ui/components/ui/input'; -import { Label } from '@simple-module-py/ui/components/ui/label'; -import { Textarea } from '@simple-module-py/ui/components/ui/textarea'; -import { AuthenticatedLayout } from '@simple-module-py/ui/layouts/AuthenticatedLayout'; -import { useState } from 'react'; -import { toast } from 'sonner'; - -const KINDS = [ - '', - 'vector_geojson', - 'vector_shapefile', - 'vector_kml', - 'raster_geotiff', - 'tabular_csv', - 'other', -]; - -function Create() { - const { t } = useT(); - const [name, setName] = useState(''); - const [description, setDescription] = useState(''); - const [kind, setKind] = useState(''); - const [file, setFile] = useState(null); - const [submitting, setSubmitting] = useState(false); - - function handleSubmit(e: React.FormEvent) { - e.preventDefault(); - if (!file) { - toast.error(t(keys.datasets.validation.file_required)); - return; - } - if (!name.trim()) { - toast.error(t(keys.datasets.validation.name_required)); - return; - } - const data = new FormData(); - data.append('name', name); - if (description) data.append('description', description); - if (kind) data.append('kind', kind); - data.append('file', file); - - setSubmitting(true); - router.post('/datasets/', data, { - forceFormData: true, - onSuccess: () => toast.success(t(keys.datasets.toasts.created)), - onError: (errs) => { - const first = Object.values(errs)[0]; - if (first) toast.error(String(first)); - }, - onFinish: () => setSubmitting(false), - }); - } - - return ( - - {t(keys.datasets.form.cancel_button)} - - } - > - - -
-
- - setName(e.target.value)} - placeholder={t(keys.datasets.form.name_placeholder)} - maxLength={200} - required - /> -
- -
- - -
- -
- - setFile(e.target.files?.[0] ?? null)} - required - /> -
- -
- -