diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..aa721049e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,40 @@ +# Monorepo-root build context (gateway + federation Dockerfiles). +# Keep gateway/ and federation/ and common/ available for COPY. + +**/.git +**/.venv +**/venv +**/node_modules +**/__pycache__ +**/.pytest_cache +**/.ruff_cache +**/dist +**/build +**/*.pyc + +# Secrets / local env +**/.envs/ +federation-shared.env + +# Unreadable / bulky OpenSearch host data +**/opensearch/data/ + +# Other monorepo trees not needed for gateway/federation images +sdk/ +jupyter/ +seaweedfs/ +docs/ +.cursor/ + +# Gateway local noise (mirrored from gateway/.dockerignore intent) +gateway/.editorconfig +gateway/.gitattributes +gateway/.github +gateway/.idea +gateway/.pre-commit-config.yaml +gateway/.readthedocs.yaml +gateway/.readthedocs.yml +gateway/.travis.yaml +gateway/.travis.yml +gateway/.gitlab-ci.yaml +gateway/.gitlab-ci.yml diff --git a/.github/workflows/fed-code-quality.yaml b/.github/workflows/fed-code-quality.yaml new file mode 100644 index 000000000..c7f821ad4 --- /dev/null +++ b/.github/workflows/fed-code-quality.yaml @@ -0,0 +1,105 @@ +# Workflow to run pre-commit checks and pytest for the Federation app +# GitHub Action Workflow validator: https://rhysd.github.io/actionlint/ +name: federation-checks + +on: + workflow_dispatch: + # To manually trigger the workflow + # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_dispatch + push: + paths: + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet + - federation/** + - common/** # Cover common/ changes too + - .pre-commit-config.yaml + - .github/workflows/fed-code-quality.yaml + branches: + - main + - master + pull_request: + paths: + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet + - federation/** + - common/** # Cover common/ changes too + - .pre-commit-config.yaml + - .github/workflows/fed-code-quality.yaml + branches: + - main + - master + types: + # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request + - ready_for_review + - synchronize + +env: + DOCKER_BUILDKIT: 1 + COMPOSE_DOCKER_CLI_BUILD: 1 + +jobs: + # Federation hooks live in the repo-root .pre-commit-config.yaml; + # prek is a federation --extra dev dependency. + fed-pre-commit: + runs-on: ubuntu-latest + env: + UV_LINK_MODE: copy + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + # https://github.com/marketplace/actions/astral-sh-setup-uv + + - name: Cache prek hooks + id: cache-prek + uses: actions/cache@v6 + # https://github.com/actions/cache/blob/main/examples.md#python---pip + with: + key: prek-federation-${{ hashFiles('.pre-commit-config.yaml') }} + path: ~/.cache/prek/ + + - name: Sync federation dependencies + working-directory: ./federation + # pyrefly-federation runs: cd federation && uv run --extra dev pyrefly check + run: uv sync --extra dev + + - name: Install hooks + working-directory: ./federation + run: uv run --extra dev prek install --install-hooks + + - name: Run federation prek hooks + working-directory: ./federation + # Only federation-scoped hooks (no biome / djLint / JS). + # prek walks up to the repo-root .pre-commit-config.yaml. + run: | + uv run --extra dev prek run ruff-check-federation --all-files + uv run --extra dev prek run ruff-format-federation --all-files + uv run --extra dev prek run pyrefly-federation --all-files + + # Run federation pytest (mocked deps — no Docker stack required) + fed-tests: + runs-on: ubuntu-latest + strategy: + matrix: + # uv will take care of installing other python versions, + # so we don't need a python-version matrix here. + platform: [ubuntu-latest] + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + # https://github.com/marketplace/actions/astral-sh-setup-uv + + - name: Install just on ubuntu + if: matrix.platform == 'ubuntu-latest' + working-directory: ./federation + run: | + npm install -g rust-just + + - name: Sync federation dependencies + working-directory: ./federation + run: uv sync --extra dev + + - name: Run tests + working-directory: ./federation + run: just test diff --git a/.github/workflows/gwy-code-quality.yaml b/.github/workflows/gwy-code-quality.yaml index 2bbf2b331..3cf525b66 100644 --- a/.github/workflows/gwy-code-quality.yaml +++ b/.github/workflows/gwy-code-quality.yaml @@ -144,7 +144,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v7 with: - context: gateway + context: . file: gateway/compose/production/django/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/common/pyproject.toml b/common/pyproject.toml new file mode 100644 index 000000000..bc431a5d8 --- /dev/null +++ b/common/pyproject.toml @@ -0,0 +1,18 @@ +[build-system] +requires = ["uv_build>=0.11.24,<0.12"] +build-backend = "uv_build" + +[tool.uv.build-backend] +module-name = "sds_opensearch_query" +module-root = "" + +[project] +name = "sds-opensearch-query" +version = "0.1.0" +description = "Shared OpenSearch and federation helpers for SDS gateway and sync" +requires-python = ">=3.13,<3.15" +dependencies = [ + "loguru>=0.7.2", + "opensearch-py>=2.7.1", + "requests>=2.32.0", +] diff --git a/common/sds_opensearch_query/__init__.py b/common/sds_opensearch_query/__init__.py new file mode 100644 index 000000000..1f48c1dc4 --- /dev/null +++ b/common/sds_opensearch_query/__init__.py @@ -0,0 +1,35 @@ +"""Framework-agnostic OpenSearch query helpers.""" + +from sds_opensearch_query.filters import build_metadata_filter_clauses +from sds_opensearch_query.filters import nested_query_clause +from sds_opensearch_query.index_write import FED_CAPTURES_INDEX +from sds_opensearch_query.index_write import FED_DATASETS_INDEX +from sds_opensearch_query.index_write import federated_doc_id +from sds_opensearch_query.index_write import index_federated_document +from sds_opensearch_query.mapping import flatten_property_paths +from sds_opensearch_query.query import bool_must_search_body +from sds_opensearch_query.query import federation_not_deleted_clause +from sds_opensearch_query.query import multi_match_clause +from sds_opensearch_query.query import run_search +from sds_opensearch_query.query import term_clause +from sds_opensearch_query.redis_channel import FEDERATION_EVENTS_CHANNEL_PREFIX +from sds_opensearch_query.redis_channel import federation_events_channel +from sds_opensearch_query.redis_channel import resolve_federation_events_channel + +__all__ = [ + "FEDERATION_EVENTS_CHANNEL_PREFIX", + "FED_CAPTURES_INDEX", + "FED_DATASETS_INDEX", + "bool_must_search_body", + "build_metadata_filter_clauses", + "federated_doc_id", + "federation_events_channel", + "federation_not_deleted_clause", + "flatten_property_paths", + "index_federated_document", + "multi_match_clause", + "nested_query_clause", + "resolve_federation_events_channel", + "run_search", + "term_clause", +] diff --git a/common/sds_opensearch_query/client.py b/common/sds_opensearch_query/client.py new file mode 100644 index 000000000..59caadf97 --- /dev/null +++ b/common/sds_opensearch_query/client.py @@ -0,0 +1,35 @@ +from loguru import logger as log +from opensearchpy import OpenSearch +from opensearchpy import RequestsHttpConnection +from requests.auth import HTTPBasicAuth + + +def build_opensearch_client( + *, + host: str, + port: int, + user: str = "", + password: str = "", + use_ssl: bool = False, + verify_certs: bool = False, + ca_certs: str | None = None, +) -> OpenSearch: + payload = { + "hosts": [{"host": host, "port": port}], + "use_ssl": use_ssl, + "verify_certs": verify_certs, + "ssl_show_warn": False, + "connection_class": RequestsHttpConnection, + } + if user: + payload["http_auth"] = HTTPBasicAuth(user, password) + if verify_certs: + if not ca_certs: + msg = ( + "OPENSEARCH_VERIFY_CERTS is True but OPENSEARCH_CA_CERTS is not set. " + "Provide a CA bundle path or set OPENSEARCH_VERIFY_CERTS to False." + ) + raise ValueError(msg) + log.info("OPENSEARCH_VERIFY_CERTS is True. Verifying certificates.") + payload["ca_certs"] = ca_certs + return OpenSearch(**payload) diff --git a/common/sds_opensearch_query/filters.py b/common/sds_opensearch_query/filters.py new file mode 100644 index 000000000..648cc9c81 --- /dev/null +++ b/common/sds_opensearch_query/filters.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any + +if TYPE_CHECKING: + from collections.abc import Callable + + +def nested_query_clause( + field_path: str, + query_type: str, + value: Any, + *, + levels_nested: int | None = None, + last_path: str | None = None, +) -> dict[str, Any]: + """Build a nested OpenSearch clause for a dotted field path.""" + if levels_nested is None: + levels_nested = field_path.count(".") + + if levels_nested == 0: + key = f"{last_path}.{field_path}" if last_path else field_path + return {query_type: {key: value}} + + path_parts = field_path.split(".") + current_path = path_parts[0] + if last_path is not None: + current_path = f"{last_path}.{current_path}" + + return { + "nested": { + "path": current_path, + "query": nested_query_clause( + field_path=".".join(path_parts[1:]), + query_type=query_type, + value=value, + levels_nested=levels_nested - 1, + last_path=current_path, + ), + }, + } + + +def build_metadata_filter_clauses( + metadata_filters: list[dict[str, Any]] | None, + *, + known_field_paths: frozenset[str] | None = None, + on_unknown_field: Callable[[str], None] | None = None, +) -> list[dict[str, Any]]: + """Turn API metadata filter dicts into OpenSearch query clauses.""" + if not metadata_filters: + return [] + + clauses: list[dict[str, Any]] = [] + for query in metadata_filters: + field_path: str = query["field_path"] + query_type: str = query["query_type"] + filter_value: Any = query["filter_value"] + + if known_field_paths is not None and field_path not in known_field_paths: + if on_unknown_field is not None: + on_unknown_field(field_path) + + levels_nested = field_path.count(".") + if levels_nested > 0: + clauses.append( + nested_query_clause( + field_path=field_path, + query_type=query_type, + value=filter_value, + levels_nested=levels_nested, + ), + ) + else: + clauses.append({query_type: {field_path: filter_value}}) + + return clauses diff --git a/common/sds_opensearch_query/index_write.py b/common/sds_opensearch_query/index_write.py new file mode 100644 index 000000000..c75e5c263 --- /dev/null +++ b/common/sds_opensearch_query/index_write.py @@ -0,0 +1,48 @@ +"""Shared fed-* OpenSearch document id and index write helpers.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any + +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from opensearchpy import OpenSearch + +FED_DATASETS_INDEX = "fed-datasets" +FED_CAPTURES_INDEX = "fed-captures" + + +def federated_doc_id(site_name: str, uuid: UUID | str) -> str: + """Stable OpenSearch ``_id`` for a site-owned federated asset.""" + return f"{site_name}:{uuid}" + + +def index_federated_document( + client: OpenSearch, + *, + index_name: str, + site_name: str, + uuid: UUID, + body: dict[str, Any], + event_at: datetime, + refresh: str | bool = "wait_for", +) -> str: + """Index a federated document and stamp ``federation_event_at``. + + Returns the document id written. + """ + doc_id = federated_doc_id(site_name, uuid) + doc = { + **body, + "federation_event_at": event_at.isoformat(), + } + client.index( + index=index_name, + id=doc_id, + body=doc, + refresh=refresh, + ) + return doc_id diff --git a/common/sds_opensearch_query/mapping.py b/common/sds_opensearch_query/mapping.py new file mode 100644 index 000000000..a9cdb7e4d --- /dev/null +++ b/common/sds_opensearch_query/mapping.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any + +if TYPE_CHECKING: + from collections.abc import Mapping + +FED_INDEX_SETTINGS: dict[str, Any] = { + "index": { + "number_of_shards": 1, + "number_of_replicas": 0, + }, +} + +# RFC §6 — fed-datasets search fields +RFC_FED_DATASET_PROPERTIES: dict[str, dict[str, Any]] = { + "uuid": {"type": "keyword"}, + "site_name": {"type": "keyword"}, # peer FQDN (federation.toml [site].fqdn) + "name": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, + "description": {"type": "text"}, + "abstract": {"type": "text"}, + "keywords": {"type": "keyword"}, + "owner_name": {"type": "keyword"}, + "created_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"}, + "updated_at": {"type": "date", "format": "strict_date_optional_time||epoch_millis"}, + "size": {"type": "long"}, + "capture_count": {"type": "integer"}, + "url": {"type": "keyword"}, +} + +# fed-captures: identity fields plus local capture OpenSearch prop dicts +# To see the full list of capture props and search props, see: +# gateway/sds_gateway/api_methods/utils/metadata_schemas.py +RFC_FED_CAPTURE_PROPERTIES: dict[str, dict[str, Any]] = { + "uuid": {"type": "keyword"}, + "site_name": {"type": "keyword"}, # peer FQDN (federation.toml [site].fqdn) + "capture_type": {"type": "keyword"}, + "channel": {"type": "keyword"}, + "capture_props": {"type": "nested", "dynamic": True}, + "search_props": {"type": "nested", "dynamic": True}, + "public_dataset_ids": {"type": "keyword"}, + "url": {"type": "keyword"}, +} + + +def _fed_mappings(properties: dict[str, dict[str, Any]]) -> dict[str, Any]: + return { + "dynamic": True, + "properties": properties, + } + + +def fed_dataset_mappings() -> dict[str, Any]: + return _fed_mappings(RFC_FED_DATASET_PROPERTIES) + + +def fed_capture_mappings() -> dict[str, Any]: + return _fed_mappings(RFC_FED_CAPTURE_PROPERTIES) + + +def flatten_property_paths( + properties: Mapping[str, Any], + *, + prefix: str = "", + separator: str = ".", +) -> frozenset[str]: + """Collect dotted field paths from an OpenSearch ``properties`` mapping.""" + paths: set[str] = set() + + for field, spec in properties.items(): + path = f"{prefix}{separator}{field}" if prefix else field + if not isinstance(spec, dict): + paths.add(path) + continue + + if spec.get("type") == "nested": + nested_props = spec.get("properties", {}) + if isinstance(nested_props, dict): + for nested_field in nested_props: + paths.add(f"{path}{separator}{nested_field}") + continue + + if "properties" in spec: + paths.update( + flatten_property_paths( + spec["properties"], + prefix=path, + separator=separator, + ), + ) + continue + + paths.add(path) + + return frozenset(paths) diff --git a/common/sds_opensearch_query/query.py b/common/sds_opensearch_query/query.py new file mode 100644 index 000000000..4dbb26ae5 --- /dev/null +++ b/common/sds_opensearch_query/query.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from typing import Any + +from opensearchpy import OpenSearch +from opensearchpy import exceptions as os_exceptions + +DEFAULT_SEARCH_SIZE = 10_000 + + +def term_clause(field: str, value: Any) -> dict[str, Any]: + return {"term": {field: value}} + + +def multi_match_clause( + query: str, + fields: list[str], + *, + match_type: str = "best_fields", +) -> dict[str, Any]: + return { + "multi_match": { + "query": query, + "fields": fields, + "type": match_type, + }, + } + + +def federation_not_deleted_clause() -> dict[str, Any]: + return {"term": {"is_deleted": False}} + + +def bool_must_search_body( + *must_clauses: dict[str, Any], + source_includes: list[str] | None = None, +) -> dict[str, Any]: + body: dict[str, Any] = { + "query": { + "bool": { + "must": list(must_clauses), + }, + }, + } + if source_includes is not None: + body["_source"] = {"includes": source_includes} + return body + + +def _request_error_message(err: os_exceptions.RequestError) -> str: + info = err.info + if isinstance(info, dict): + root_causes: list[dict[str, str]] = info.get("error", {}).get( + "root_cause", + [], + ) + root_cause_reason = root_causes[0].get("reason", "") if root_causes else "" + reason = str(root_cause_reason) if root_cause_reason else str(info) + else: + reason = str(info) + return f"Query error: {reason}" + + +def run_search( + client: OpenSearch, + *, + index: str, + body: dict[str, Any], + size: int = DEFAULT_SEARCH_SIZE, +) -> list[dict[str, Any]]: + """Run a search and return raw hit dicts. Raises ValueError for client errors.""" + try: + response = client.search( + index=index, + body=body, + size=size, # pyright: ignore[reportCallIssue] + ) + except os_exceptions.NotFoundError as err: + msg = f"Index '{index}' not found" + raise ValueError(msg) from err + except os_exceptions.ConnectionError: + raise + except os_exceptions.RequestError as err: + raise ValueError(_request_error_message(err)) from err + except os_exceptions.OpenSearchException: + raise + + return list(response["hits"]["hits"]) diff --git a/common/sds_opensearch_query/redis_channel.py b/common/sds_opensearch_query/redis_channel.py new file mode 100644 index 000000000..e19725f7e --- /dev/null +++ b/common/sds_opensearch_query/redis_channel.py @@ -0,0 +1,44 @@ +"""Shared Redis pub/sub channel naming for federation local change events.""" + +from __future__ import annotations + +FEDERATION_EVENTS_CHANNEL_PREFIX = "federation:events" + + +def federation_events_channel(site_name: str) -> str: + """Site-scoped channel ``federation:events:{site_name}``.""" + name = site_name.strip() + if not name: + msg = "site_name is required to build a federation events channel" + raise ValueError(msg) + return f"{FEDERATION_EVENTS_CHANNEL_PREFIX}:{name}" + + +def resolve_federation_events_channel( + *, + site_name: str = "", + channel_override: str | None = None, + env_override: str | None = None, + gateway_site_name: str | None = None, +) -> str: + """Resolve the Redis channel for local federation events. + + Precedence: + 1. ``channel_override`` / ``env_override`` (``FEDERATION_EVENTS_CHANNEL``) when set + 2. ``gateway_site_name`` (``FEDERATION_SITE_NAME``) when set + — matches gateway publish + 3. ``site_name`` (e.g. federation.toml ``[site].name``) when set + 4. empty string when nothing is configured (gateway settings default) + """ + override = ( + channel_override if channel_override is not None else env_override + ) or "" + if override.strip(): + return override.strip() + gateway = (gateway_site_name or "").strip() + if gateway: + return federation_events_channel(gateway) + site = (site_name or "").strip() + if site: + return federation_events_channel(site) + return "" diff --git a/common/uv.lock b/common/uv.lock new file mode 100644 index 000000000..912617164 --- /dev/null +++ b/common/uv.lock @@ -0,0 +1,221 @@ +version = 1 +revision = 3 +requires-python = ">=3.13, <3.15" + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "events" +version = "0.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/ed/e47dec0626edd468c84c04d97769e7ab4ea6457b7f54dcb3f72b17fcd876/Events-0.5-py3-none-any.whl", hash = "sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd", size = 6758, upload-time = "2023-07-31T08:23:13.645Z" }, +] + +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a1/121806ce69f23138dabe06aa595b0e5f1ae051a37e4c1954eed7d692c800/grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1", size = 12154419, upload-time = "2026-07-23T15:19:56.3Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" }, + { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" }, + { url = "https://files.pythonhosted.org/packages/93/b6/d70b69ae5c0cfc341b9ba474980e4ed99cbf05c0e4a14e9eee8cb73db0a5/grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf", size = 4393993, upload-time = "2026-07-23T15:20:11.017Z" }, + { url = "https://files.pythonhosted.org/packages/0f/13/45d4cccb555cf4c476226979bf3d2fd0b0254216f7564c3a053e35117efc/grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735", size = 5159650, upload-time = "2026-07-23T15:20:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, + { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" }, + { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "opensearch-protobufs" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/2f/e0cc165af7bb7b44cb00023b9fcaa01a28d1755a059ede28d0cd970c3cec/opensearch_protobufs-1.2.0-py3-none-any.whl", hash = "sha256:e806730894d0a0c8cdaa3cdbe07e4b7c46e1823f453777b36caf39e9cba28e2c", size = 54751, upload-time = "2026-01-22T18:51:56.805Z" }, +] + +[[package]] +name = "opensearch-py" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "events" }, + { name = "opensearch-protobufs" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/9e/e77844cb2d625ca32331bfdd28930113b3778399c01dd5f1c350ceb55e65/opensearch_py-3.2.0.tar.gz", hash = "sha256:f40fb3a295275422df2ad6d9459f667af94472d5a9e567072e9ecf163eb22613", size = 259927, upload-time = "2026-04-27T18:17:50.467Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/63/7abb96bf2e3619acbd27de99e60619bfacfb7c55b68c4792a258e6d92871/opensearch_py-3.2.0-py3-none-any.whl", hash = "sha256:721a0d3b13fbed9e82278aed748285cf63a1855354ab7e73e3d4992d1b93418b", size = 387286, upload-time = "2026-04-27T18:17:48.658Z" }, +] + +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "sds-opensearch-query" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "opensearch-py" }, + { name = "requests" }, +] + +[package.metadata] +requires-dist = [ + { name = "opensearch-py", specifier = ">=2.7.1" }, + { name = "requests", specifier = ">=2.32.0" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] diff --git a/federation/.gitignore b/federation/.gitignore index 8c7ad6af1..712f81586 100644 --- a/federation/.gitignore +++ b/federation/.gitignore @@ -3,6 +3,7 @@ __pycache__/ federation.toml +federation.peer.toml -.envs/* -!/.envs/example +# Local live-test secrets only (keep README + env.example tracked) +scripts/local_e2e/.env diff --git a/federation/Dockerfile b/federation/Dockerfile index 2f44a39c8..dacd1b9cf 100644 --- a/federation/Dockerfile +++ b/federation/Dockerfile @@ -2,14 +2,15 @@ FROM docker.io/python:3.14-slim-trixie COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ -WORKDIR /app +WORKDIR /app/federation ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 -COPY pyproject.toml uv.lock ./ -COPY sds_federation ./sds_federation +# Build context is repo root (compose: context: ..). +COPY common /app/common +COPY federation/pyproject.toml federation/uv.lock ./ +COPY federation/sds_federation ./sds_federation RUN uv sync --frozen --no-dev - EXPOSE 8000 CMD ["uv", "run", "uvicorn", "sds_federation.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/federation/README.md b/federation/README.md new file mode 100644 index 000000000..9ab8b9b7f --- /dev/null +++ b/federation/README.md @@ -0,0 +1,220 @@ +# SDS Federation Sync + +FastAPI service that indexes federated dataset/capture metadata into OpenSearch +(`fed-datasets`, `fed-captures`), receives peer webhooks, and bootstraps from the +local gateway export API plus peer sync list endpoints. + +On startup (unless `FEDERATION_BOOTSTRAP_ON_START=false`): + +1. Ensure `fed-*` indices exist +2. Mint or reuse a FederationSync export Api-Key +3. Pull local gateway `/api/v1/federation/export/{datasets,captures}/` +4. Pull each peer’s `/sync/api/v1/webhook/list-{datasets,captures}/` +5. Register with peers via `POST .../webhook/site-hello` +6. Subscribe to Redis `federation:events:{site}` for incremental updates + +Local HTTP (this machine only, not the peer path): host **8001** → container `:8000`. +Health: `http://localhost:8001/sync/health`. + +Peer sync is **HTTPS :443** via Traefik `PathPrefix(/sync)` (prefix is **not** stripped): + +- sds-dev: `https://sds-dev.crc.nd.edu/sync/health` +- sds-fed1: `https://sds-fed1.crc.nd.edu/sync/health` +- prod: `https://sds.crc.nd.edu/sync/health` + +Advertise that HTTPS URL in `[site].sync_service_url` / `FEDERATION_SYNC_SERVICE_URL` +(not `:8001`). Prod compose sets `FEDERATION_SYNC_SERVICE_URL=https://sds.crc.nd.edu/sync`. + +See also: + +- [scripts/local_e2e/README.md](scripts/local_e2e/README.md) — local live test (publish / seed / verify) +- [docs/peer-remote-test.md](docs/peer-remote-test.md) — two-server sync-only peer +- Repo-root `rfc_federation.md` — architecture RFC + +--- + +## Site identity (critical) + +| Setting | Matches | Used for | +|---------|---------|----------| +| `FEDERATION_SITE_NAME` | toml `[site].name` | Short peer id; Redis `federation:events:{name}` | +| `SDS_SITE_FQDN` | toml `[site].fqdn` | Export / OpenSearch **`site_name`**; sync `list-*` filters | + +Do **not** put the short name into OpenSearch docs. Gateway writes FQDN via `SDS_SITE_FQDN`. + +Local lab defaults: `name=crc`, `fqdn=sds.localhost`. Peer lab: `name=peer`, `fqdn=peer.local`. + +--- + +## Prerequisites + +- Docker / Compose, `just`, OpenSSL (for secrets) +- Gateway local stack (`sds-network-local`, app, Redis, OpenSearch) +- Python tooling for federation tests (`uv` in this directory) + +--- + +## Local initialization + +### 1. Generate shared secrets + +From the gateway directory (fills `gateway/.envs/local/*` and repo-root +`federation-shared.env`): + +```bash +cd gateway +./scripts/generate-secrets.sh local +``` + +Confirm the DRF token (40 characters). Leave `FEDERATION_SYNC_SERVER_API_KEY` empty +so sync mints an export key on start. + +### 2. Gateway federation env + +`gateway/.envs/local/django.env`: + +```env +FEDERATION_ENABLED=true +FEDERATION_SITE_NAME=crc +SDS_SITE_FQDN=sds.localhost +FEDERATION_SYNC_HEALTH_URL=http://sds-federation-local-sync:8000/sync/health +FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local +# Leave FEDERATION_EVENTS_CHANNEL unset → federation:events:crc +``` + +Optional first boot: `FEDERATION_SKIP_SYNC_HEALTH_PROBE=true` until sync is healthy. + +### 3. Federation config + +```bash +cd federation +cp federation.example.toml federation.toml +# [site] name/fqdn must match gateway FEDERATION_SITE_NAME / SDS_SITE_FQDN +``` + +Compose loads OpenSearch from `gateway/.envs//opensearch.env` and auth from +`federation-shared.env`. + +### 4. Start gateway + federation token + +```bash +cd gateway +just up +docker compose -f compose.local.yaml exec sds-gateway-local-app \ + python manage.py init_federation_sync_token +# or: prepare_gateway +``` + +### 5. Start sync + +**Solo (this site only):** + +```bash +cd federation +just build && just up +curl -s http://localhost:8001/sync/health | jq . +``` + +**Local peer mesh (recommended for p2p):** + +```bash +cd federation +just deploy-local-peer-2-peer # gateway check + build + local+peer up +just seed-peer # dummy peer.local docs on :9201 +docker restart sds-federation-local-sync # pull peer list / site-hello +``` + +Tear down both stacks: + +```bash +docker compose -f compose.local.yaml -f compose.peer.local.yaml down +# just down # only stops the primary compose file +``` + +### 6. Export Api-Key (manual checks) + +```bash +TOKEN=$(grep '^FEDERATION_SYNC_DRF_TOKEN=' ../federation-shared.env | cut -d= -f2-) +curl -sS http://localhost:8000/users/get-federation-sync-api-key/ \ + -H "Authorization: Token $TOKEN" | jq -r .api_key +``` + +Use header: `Authorization: Api-Key: ` (note the colon). + +--- + +## Auth model + +| Secret | Where | Role | +|--------|--------|------| +| `FEDERATION_SYNC_DRF_TOKEN` | `federation-shared.env` → DB via `init_federation_sync_token` | Mint endpoint auth | +| `FEDERATION_SYNC_SERVER_API_KEY` | Minted by sync (or set in shared env) | Gateway `/federation/export/*` | + +There is **no** `create_federation_sync_api_key` command. + +--- + +## Commands + +```bash +cd federation +just # list recipes +just deploy-local-peer-2-peer # local + peer stacks +just seed-peer # seed peer OpenSearch (:9201, site peer.local) +just verify-federation-live +just simulate-redis --event-type updated --item-type dataset --uuid +just test / just test-regression / just test-integration +``` + +--- + +## Publishing local data + +```bash +cd gateway +docker compose -f compose.local.yaml exec sds-gateway-local-app \ + python manage.py publish_for_federation \ + --dataset-uuid --capture-uuids +``` + +Confirm docs use the FQDN: + +```bash +curl -s 'http://localhost:9200/fed-datasets/_search' | jq '.hits.hits[]._source.site_name' +curl -s http://localhost:8001/sync/api/v1/webhook/list-datasets/ | jq 'length' +``` + +Live-test steps: [scripts/local_e2e/README.md](scripts/local_e2e/README.md). + +--- + +## Peer sync testing + +### Same machine + +`compose.peer.local.yaml` + `federation.peer.toml` (mutual `[[peers]]` with **container DNS** +URLs). Orchestrated by `just deploy-local-peer-2-peer`. + +- Peer has **no gateway** → `GATEWAY_INTERNAL_BASE_URL=http://127.0.0.1:9/...` failures in + peer logs are expected. +- Peer-owned docs: `just seed-peer` (writes `site_name=peer.local`). +- Then restart local sync so site-hello / list pull ingest peer docs. + +### Two servers (sync-only peer) + +See **[docs/peer-remote-test.md](docs/peer-remote-test.md)** and +`compose.peer.remote.yaml` / `federation.peer.remote.example.toml`. + +--- + +## Troubleshooting + +| Symptom | Check | +|---------|--------| +| Export `503` | Sync health URL, Api-Key, Redis, `FEDERATION_ENABLED` | +| Bootstrap mint fail | DRF token length 40; `init_federation_sync_token`; gateway URL | +| `list-*` empty but OS has docs | Doc `site_name` must be **FQDN** (`SDS_SITE_FQDN`), not short name | +| site-hello connect race | Retries are built in; ensure peer healthy before relying on backfill | +| Redis events missing | Leave `FEDERATION_EVENTS_CHANNEL` unset; channel = `federation:events:{FEDERATION_SITE_NAME}` | +| Gateway code changes not live | App image is not fully bind-mounted — rebuild gateway image | +| Docker build / `sds-opensearch-query` | Build context repo root; `COPY common/` | diff --git a/federation/compose.local.yaml b/federation/compose.local.yaml index 52d07c195..0c488d0ee 100644 --- a/federation/compose.local.yaml +++ b/federation/compose.local.yaml @@ -9,12 +9,11 @@ networks: services: sds-federation-local-sync: build: - context: . - dockerfile: Dockerfile + context: .. + dockerfile: federation/Dockerfile image: sds-federation-local-sync container_name: sds-federation-local-sync env_file: - - .envs/local/sync.env - ../gateway/.envs/local/opensearch.env - ../federation-shared.env environment: diff --git a/federation/compose.peer.local.yaml b/federation/compose.peer.local.yaml new file mode 100644 index 000000000..d5d1c2695 --- /dev/null +++ b/federation/compose.peer.local.yaml @@ -0,0 +1,65 @@ +networks: + sds-network-local: + external: true + name: sds-network-local + peer-internal: + driver: bridge + +services: + sds-federation-peer-opensearch: + image: opensearchproject/opensearch:2 + container_name: sds-federation-peer-opensearch + environment: + discovery.type: single-node + plugins.security.disabled: "true" + OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m + DISABLE_INSTALL_DEMO_CONFIG: "true" + ports: + - "9201:9200" + networks: [peer-internal] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health || exit 1"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 30s + + sds-federation-peer-redis: + image: redis:6 + container_name: sds-federation-peer-redis + ports: + - "6380:6379" + networks: [peer-internal] + + sds-federation-peer-sync: + build: + context: .. + dockerfile: federation/Dockerfile + image: sds-federation-local-sync + container_name: sds-federation-peer-sync + environment: + FEDERATION_CONFIG_PATH: /etc/sds/federation.toml + FEDERATION_SYNC_SERVICE_URL: http://sds-federation-peer-sync:8000/sync + # Dummy gateway — local export bootstrap will fail; that's OK + GATEWAY_INTERNAL_BASE_URL: http://127.0.0.1:9/api/v1 + REDIS_URL: redis://sds-federation-peer-redis:6379/0 + OPENSEARCH_HOST: sds-federation-peer-opensearch + OPENSEARCH_PORT: "9200" + OPENSEARCH_USER: "" + OPENSEARCH_PASSWORD: "" + OPENSEARCH_USE_SSL: "false" + OPENSEARCH_VERIFY_CERTS: "false" + # No mint: leave FEDERATION_SYNC_* unset + FEDERATION_SITE_NAME: peer + volumes: + - ./federation.peer.toml:/etc/sds/federation.toml:ro + ports: + - "8002:8000" + depends_on: + sds-federation-peer-opensearch: + condition: service_healthy + sds-federation-peer-redis: + condition: service_started + networks: + - sds-network-local + - peer-internal diff --git a/federation/compose.peer.remote.yaml b/federation/compose.peer.remote.yaml new file mode 100644 index 000000000..8b577a148 --- /dev/null +++ b/federation/compose.peer.remote.yaml @@ -0,0 +1,76 @@ +# Sync-only peer on a remote host (no gateway / Postgres). +# Pair with federation.peer.remote.toml and docs/peer-remote-test.md. +# +# On the peer machine (sds-fed1): +# cd ../network && just up # Traefik + sds-network-local +# cd ../federation +# cp federation.peer.remote.example.toml federation.peer.toml +# cp env.peer.example .env.peer +# docker compose -f compose.peer.remote.yaml --env-file .env.peer up -d --build +# +# Main site (sds-dev) must list this peer's public sync URL in federation.toml [[peers]]. + +networks: + peer-internal: + driver: bridge + sds-network-local: + external: true + name: sds-network-local + +services: + sds-federation-peer-opensearch: + image: opensearchproject/opensearch:2 + container_name: sds-federation-peer-opensearch + environment: + discovery.type: single-node + plugins.security.disabled: "true" + OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m + DISABLE_INSTALL_DEMO_CONFIG: "true" + ports: + # Host loopback only — seed_peer_opensearch.py. Not for peer sync. + - "127.0.0.1:${PEER_OPENSEARCH_PORT:-9200}:9200" + networks: [peer-internal] + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health || exit 1"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 40s + + sds-federation-peer-redis: + image: redis:6 + container_name: sds-federation-peer-redis + networks: [peer-internal] + + sds-federation-peer-sync: + build: + context: .. + dockerfile: federation/Dockerfile + image: sds-federation-peer-sync + container_name: sds-federation-peer-sync + environment: + FEDERATION_CONFIG_PATH: /etc/sds/federation.toml + # Must be reachable by the main site (set in toml sync_service_url too). + FEDERATION_SYNC_SERVICE_URL: ${PEER_SYNC_PUBLIC_URL:-https://sds-fed1.crc.nd.edu/sync} + # Intentional: sync-only peer has no gateway export. + GATEWAY_INTERNAL_BASE_URL: http://127.0.0.1:9/api/v1 + REDIS_URL: redis://sds-federation-peer-redis:6379/0 + OPENSEARCH_HOST: sds-federation-peer-opensearch + OPENSEARCH_PORT: "9200" + OPENSEARCH_USER: "" + OPENSEARCH_PASSWORD: "" + OPENSEARCH_USE_SSL: "false" + OPENSEARCH_VERIFY_CERTS: "false" + FEDERATION_SITE_NAME: ${PEER_SITE_NAME:-fed1} + FEDERATION_BOOTSTRAP_ON_START: ${FEDERATION_BOOTSTRAP_ON_START:-true} + volumes: + - ./federation.peer.toml:/etc/sds/federation.toml:ro + - ./certs:/etc/sds/certs:ro + depends_on: + sds-federation-peer-opensearch: + condition: service_healthy + sds-federation-peer-redis: + condition: service_started + networks: + - peer-internal + - sds-network-local diff --git a/federation/compose.production.yaml b/federation/compose.production.yaml index 439c85835..45d965848 100644 --- a/federation/compose.production.yaml +++ b/federation/compose.production.yaml @@ -9,17 +9,17 @@ networks: services: sds-federation-prod-sync: build: - context: . - dockerfile: Dockerfile + context: .. + dockerfile: federation/Dockerfile image: sds-federation-prod-sync container_name: sds-federation-prod-sync env_file: - - .envs/production/sync.env - ../gateway/.envs/production/opensearch.env - ../federation-shared.env environment: FEDERATION_CONFIG_PATH: /etc/sds/federation.toml - GATEWAY_INTERNAL_BASE_URL: http://sds-gateway-prod-app:8000/api/v1 + GATEWAY_INTERNAL_BASE_URL: http://sds-gateway-prod-app:18000/api/v1 + FEDERATION_SYNC_SERVICE_URL: https://sds.crc.nd.edu/sync REDIS_URL: redis://sds-gateway-prod-redis:6379/0 volumes: - ./federation.toml:/etc/sds/federation.toml:ro diff --git a/federation/docs/peer-remote-test.md b/federation/docs/peer-remote-test.md new file mode 100644 index 000000000..e45fd371c --- /dev/null +++ b/federation/docs/peer-remote-test.md @@ -0,0 +1,118 @@ +# Remote peer sync test (two servers) + +CRC lab pair: **sds-dev.crc.nd.edu** (full gateway + sync) ↔ **sds-fed1.crc.nd.edu** +(sync-only: OpenSearch + Redis + federation). Traefik `PathPrefix(/sync)` on both +hosts; do not strip the prefix. + +```text +[sds-dev] [sds-fed1] + gateway + Redis + OpenSearch + sync OpenSearch + Redis + sync + publishes real/public FINAL datasets seed_peer_opensearch.py + federation.toml [[peers]] → fed1 /sync federation.peer.toml [[peers]] → sds-dev /sync +``` + +FQDN must match the real hostname (`sds-fed1`, not `sds-fed-1`). + +## Identity + +| Field | sds-dev | sds-fed1 | +|-------|---------|----------| +| toml `[site].name` | `crc` (match `FEDERATION_SITE_NAME`) | `fed1` | +| toml `[site].fqdn` / OpenSearch `site_name` | `sds-dev.crc.nd.edu` | `sds-fed1.crc.nd.edu` | +| Public sync URL | `https://sds-dev.crc.nd.edu/sync` | `https://sds-fed1.crc.nd.edu/sync` | +| Redis channel | `federation:events:{name}` | `federation:events:fed1` | + +`list-*` and site-hello allowlists use **FQDN**, not the short name. Leave `ca_cert_path` +unset (Let's Encrypt). + +## sds-fed1 setup + +1. Deploy Traefik (creates `sds-network-local`, TLS for `sds-fed1.crc.nd.edu`): + + ```bash + cd network + just up + ``` + + DNS for `sds-fed1.crc.nd.edu` must point at this host (HTTP-01 on :80). + +2. Federation config: + + ```bash + cd federation + cp federation.peer.remote.example.toml federation.peer.toml + cp env.peer.example .env.peer + mkdir -p certs + ``` + +3. Start sync + Redis + OpenSearch only (`compose.peer.remote.yaml`, not `just up`): + + ```bash + docker compose -f compose.peer.remote.yaml --env-file .env.peer up -d --build + curl -sS https://sds-fed1.crc.nd.edu/sync/health | jq . + ``` + + Gateway export errors to `127.0.0.1:9` are **expected**. + +4. Seed peer-owned docs (`--site-name` = `[site].fqdn`): + + ```bash + uv sync --extra dev + uv run python scripts/seed_peer_opensearch.py \ + --opensearch-url http://localhost:9200 \ + --site-name sds-fed1.crc.nd.edu + curl -sS https://sds-fed1.crc.nd.edu/sync/api/v1/webhook/list-datasets/ \ + | jq '.[].site_name' + # → ["sds-fed1.crc.nd.edu"] + ``` + +## sds-dev setup + +1. Gateway + federation sync; Traefik toml with `Host(sds-dev.crc.nd.edu) && PathPrefix(/sync)`. +2. In `federation.toml`: + + ```toml + [site] + name = "crc" + fqdn = "sds-dev.crc.nd.edu" + display_name = "Notre Dame SDS Dev" + sync_service_url = "https://sds-dev.crc.nd.edu/sync" + + [[peers]] + name = "fed1" + fqdn = "sds-fed1.crc.nd.edu" + display_name = "Fed1 test peer" + gateway_api_base = "http://peer-gateway-unused:8000/api/v1" + sync_service_url = "https://sds-fed1.crc.nd.edu/sync" + ``` + +3. `SDS_SITE_FQDN=sds-dev.crc.nd.edu` (must match `[site].fqdn`). +4. Restart main sync after toml changes. + +## Bring-up order + +1. sds-dev gateway + sync healthy; `https://sds-dev.crc.nd.edu/sync/health` is 200. +2. sds-fed1 Traefik + peer stack healthy + seeded. +3. Restart **fed1** sync (site-hello → sds-dev backfill). +4. Restart **sds-dev** sync (bootstrap pull of fed1 `list-*`). + +## Verify + +```bash +# both public health +curl -sS https://sds-dev.crc.nd.edu/sync/health | jq . +curl -sS https://sds-fed1.crc.nd.edu/sync/health | jq . + +# fed1 docs on sds-dev OpenSearch +curl -sS 'http://localhost:9200/fed-datasets/_search?q=site_name:sds-fed1.crc.nd.edu' \ + | jq '.hits.total' +``` + +Fallback if Traefik/TLS is not ready on fed1: open host **:8000** and use +`http://sds-fed1.crc.nd.edu:8000/sync` in sds-dev `[[peers]]` and +`PEER_SYNC_PUBLIC_URL` (still use `https://sds-dev.crc.nd.edu/sync` the other way). + +## Local analogue + +Same idea on one machine: `just deploy-local-peer-2-peer` then `just seed-peer` +(see [../README.md](../README.md) and [../scripts/local_e2e/README.md](../scripts/local_e2e/README.md)). diff --git a/federation/env.peer.example b/federation/env.peer.example new file mode 100644 index 000000000..31baa2665 --- /dev/null +++ b/federation/env.peer.example @@ -0,0 +1,10 @@ +# Optional env for compose.peer.remote.yaml (copy to .env.peer; do not commit secrets). + +PEER_SITE_NAME=fed1 +PEER_OPENSEARCH_PORT=9200 + +# Public URL sds-dev puts in federation.toml [[peers]].sync_service_url +# Peers use Traefik :443 PathPrefix(/sync) — not a host-published sync port. +PEER_SYNC_PUBLIC_URL=https://sds-fed1.crc.nd.edu/sync + +FEDERATION_BOOTSTRAP_ON_START=true diff --git a/federation/federation.example.toml b/federation/federation.example.toml index dab5fd981..db687d174 100644 --- a/federation/federation.example.toml +++ b/federation/federation.example.toml @@ -2,6 +2,9 @@ name = "your-site" fqdn = "your-site.domain" display_name = "Your Site" +# Advertised in site-hello. Use the Traefik HTTPS /sync URL (not host :8001). +# Defaults to http://{fqdn}/sync/ if unset. +# sync_service_url = "https://your-site.domain/sync" [[peers]] # Notre Dame SDS site is the original site deployment diff --git a/federation/federation.peer.remote.example.toml b/federation/federation.peer.remote.example.toml new file mode 100644 index 000000000..e02b7a793 --- /dev/null +++ b/federation/federation.peer.remote.example.toml @@ -0,0 +1,16 @@ +# Copy to federation.peer.toml on sds-fed1 (gitignored if using that name). + +[site] +name = "fed1" +fqdn = "sds-fed1.crc.nd.edu" +display_name = "Fed1 test peer (sync-only)" +sync_service_url = "https://sds-fed1.crc.nd.edu/sync" + +[[peers]] +name = "crc" +fqdn = "sds-dev.crc.nd.edu" +display_name = "SDS Dev" +gateway_api_base = "https://sds-dev.crc.nd.edu/api/v1" +sync_service_url = "https://sds-dev.crc.nd.edu/sync" +# Leave unset so httpx uses the system CA (Let's Encrypt on sds-dev). +# ca_cert_path = "/etc/sds/certs/main-ca.pem" diff --git a/federation/justfile b/federation/justfile index 0685386f0..404b6509f 100644 --- a/federation/justfile +++ b/federation/justfile @@ -9,9 +9,9 @@ env_selection_script := "./scripts/env-selection.sh" compose_file := shell(env_selection_script + ' $1', "compose_file") env := shell(env_selection_script + ' $1', "env") -env_file := shell(env_selection_script + ' $1', "env_file") sync_container := shell(env_selection_script + ' $1', "sync_container") -docker_compose := "COMPOSE_FILE=" + compose_file + " docker compose --env-file " + env_file +# Service env comes from compose env_file (gateway opensearch.env + federation-shared.env). +docker_compose := "COMPOSE_FILE=" + compose_file + " docker compose" alias run := up @@ -24,7 +24,6 @@ env: #!/usr/bin/env bash echo -e "\nSelected env:\n" echo -e "\tEnvironment: \e[34m '{{ env }}'\e[0m" - echo -e "\tEnvironment file: \e[34m '{{ env_file }}'\e[0m" echo -e "\tCompose file: \e[34m '{{ compose_file }}'\e[0m" echo -e "\tSync container: \e[34m '{{ sync_container }}'\e[0m" echo -e "\tDocker compose command: \e[34m '{{ docker_compose }}'\e[0m" @@ -33,9 +32,11 @@ env: echo -e "\n\e[31mError:\e[0m Compose file '{{ compose_file }}' does not exist." exit 1 fi - if ! [ -f "{{ env_file }}" ]; then - echo -e "\n\e[33mWarning:\e[0m Env file '{{ env_file }}' does not exist." - echo -e "Copy from .envs/example/sync.env if needed." + if ! [ -f "../federation-shared.env" ]; then + echo -e "\n\e[33mWarning:\e[0m ../federation-shared.env missing — run gateway generate-secrets." + fi + if ! [ -f "../gateway/.envs/{{ env }}/opensearch.env" ]; then + echo -e "\n\e[33mWarning:\e[0m ../gateway/.envs/{{ env }}/opensearch.env missing." fi # install runtime + dev deps (pytest) @@ -59,12 +60,12 @@ test +args='': test-q: uv run pytest -q -# Run federation pre-commit hooks from repo root (requires gateway dev deps / pre-commit install) +# Run federation pre-commit hooks (prek walks up to repo-root .pre-commit-config.yaml) [group('qa')] pre-commit +args='': - cd .. && uv run --directory gateway --extra local pre-commit run ruff-check-federation --all-files {{ args }} - cd .. && uv run --directory gateway --extra local pre-commit run ruff-format-federation --all-files {{ args }} - cd .. && uv run --directory gateway --extra local pre-commit run pyrefly-federation --all-files {{ args }} + uv run --extra dev prek run ruff-check-federation --all-files {{ args }} + uv run --extra dev prek run ruff-format-federation --all-files {{ args }} + uv run --extra dev prek run pyrefly-federation --all-files {{ args }} # dev PKI for mTLS experiments (see docs/mtls-certificates.md) [group('setup')] @@ -80,6 +81,25 @@ gen-certs-ca: simulate-redis +args='': REDIS_URL="${REDIS_URL:-redis://localhost:6379/0}" uv run python scripts/simulate_redis_event.py {{ args }} +[group('dev')] +verify-federation-live +args='': + uv run python scripts/verify_federation_live.py {{ args }} + +# Deploy local + peer federation stacks (checks gateway, builds, waits for health) +[group('dev')] +deploy-local-peer-2-peer: + ./scripts/deploy-local-peer-2-peer.sh + +# Seed dummy peer.local docs into peer OpenSearch (:9201) +[group('dev')] +seed-peer +args='': + uv run python scripts/seed_peer_opensearch.py --opensearch-url http://localhost:9201 --site-name peer.local {{ args }} + +# Tear down local + peer compose stacks together +[group('dev')] +down-peer-2-peer +args='': + docker compose -f compose.local.yaml -f compose.peer.local.yaml down {{ args }} + [group('docker')] build +args='': {{ docker_compose }} build {{ args }} diff --git a/federation/pyproject.toml b/federation/pyproject.toml index 4fa9fb01e..0a52056b3 100644 --- a/federation/pyproject.toml +++ b/federation/pyproject.toml @@ -16,6 +16,7 @@ module-root = "" "httpx>=0.28.1", "loguru>=0.7.2", "opensearch-py>=2.8.0", + "sds-opensearch-query", "pydantic>=2.11.0", "redis>=5.2.1", "uvicorn[standard]>=0.34.0", @@ -24,6 +25,7 @@ module-root = "" [project.optional-dependencies] dev = [ "deptry>=0.24.0", + "prek>=0.3.4", "pyrefly>=0.42.1", "pytest>=8.3.0", "pytest-asyncio>=0.25.0", @@ -36,6 +38,7 @@ dev = [ [tool.deptry.per_rule_ignores] DEP002 = [ "deptry", + "prek", "pyrefly", "pytest", "pytest-asyncio", @@ -165,3 +168,6 @@ markers = [ "integration: tests that exercise FastAPI routes or multi-step pipeline", "regression: contract and indexer behavior guards", ] + +[tool.uv.sources] +sds-opensearch-query = { path = "../common" } diff --git a/federation/scripts/deploy-local-peer-2-peer.sh b/federation/scripts/deploy-local-peer-2-peer.sh new file mode 100755 index 000000000..05f8687da --- /dev/null +++ b/federation/scripts/deploy-local-peer-2-peer.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Bring up local + peer federation sync stacks for p2p testing. +# Prereq: gateway local stack can be started from ../gateway. +# Does not require Traefik; creates sds-network-local if missing. +set -euo pipefail + +FEDERATION_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GATEWAY_ROOT="${FEDERATION_ROOT}/../gateway" +LOCAL_COMPOSE="${FEDERATION_ROOT}/compose.local.yaml" +PEER_COMPOSE="${FEDERATION_ROOT}/compose.peer.local.yaml" +GATEWAY_COMPOSE="${GATEWAY_ROOT}/compose.local.yaml" + +GATEWAY_URL="${GATEWAY_URL:-http://localhost:8000}" +GATEWAY_HEALTH_PATH="${GATEWAY_HEALTH_PATH:-/}" +WAIT_SECS="${WAIT_SECS:-180}" +LOCAL_SYNC_URL="${LOCAL_SYNC_URL:-http://localhost:8001/sync/health}" +PEER_SYNC_URL="${PEER_SYNC_URL:-http://localhost:8002/sync/health}" +PEER_OS_URL="${PEER_OS_URL:-http://localhost:9201/_cluster/health}" + +cd "${FEDERATION_ROOT}" + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +info() { printf '==> %s\n' "$*"; } + +require_file() { + [[ -f "$1" ]] || die "missing $1" +} + +# Gateway/federation compose mark this external (Traefik creates it in real +# deploys). Local p2p does not need Traefik — create the net if missing. +ensure_docker_network() { + local name=$1 + if docker network inspect "${name}" >/dev/null 2>&1; then + return 0 + fi + info "Creating docker network ${name}" + docker network create --driver bridge "${name}" >/dev/null +} + +wait_http_ok() { + local url=$1 label=$2 + local deadline=$((SECONDS + WAIT_SECS)) + info "Waiting for ${label} (${url})" + while ((SECONDS < deadline)); do + if curl -fsS -o /dev/null --max-time 3 "${url}"; then + info "${label} is up" + return 0 + fi + sleep 2 + done + die "${label} not healthy after ${WAIT_SECS}s: ${url}" +} + +# Require /sync/health JSON status == "ok" (not merely HTTP 200/503). +wait_sync_operational() { + local url=$1 label=$2 + local deadline=$((SECONDS + WAIT_SECS)) + info "Waiting for ${label} operational (${url})" + while ((SECONDS < deadline)); do + body="$(curl -fsS --max-time 3 "${url}" 2>/dev/null || true)" + if [[ -n "${body}" ]] && printf '%s' "${body}" | grep -q '"status"[[:space:]]*:[[:space:]]*"ok"'; then + info "${label} operational" + return 0 + fi + sleep 2 + done + die "${label} not operational after ${WAIT_SECS}s: ${url}" +} + +restart_and_wait_sync() { + local container=$1 url=$2 label=$3 + info "Restarting ${container} so site-hello/bootstrap see a live peer" + docker restart "${container}" >/dev/null + wait_sync_operational "${url}" "${label}" +} + +# --- configs --- +info "Checking federation configs" +require_file "${LOCAL_COMPOSE}" +require_file "${PEER_COMPOSE}" +require_file "${FEDERATION_ROOT}/federation.toml" +require_file "${FEDERATION_ROOT}/federation.peer.toml" +require_file "${FEDERATION_ROOT}/../federation-shared.env" +require_file "${GATEWAY_ROOT}/.envs/local/opensearch.env" + +# Optional sanity: peer sync URL must be Docker DNS, not localhost +if ! grep -q 'sds-federation-peer-sync' "${FEDERATION_ROOT}/federation.toml"; then + die "federation.toml [[peers]] should use sync_service_url → sds-federation-peer-sync" +fi +if ! grep -q 'sds-federation-local-sync' "${FEDERATION_ROOT}/federation.peer.toml"; then + die "federation.peer.toml [[peers]] should use sync_service_url → sds-federation-local-sync" +fi + +# --- gateway --- +info "Ensuring gateway local stack is running" +require_file "${GATEWAY_COMPOSE}" +ensure_docker_network sds-network-local +( + cd "${GATEWAY_ROOT}" + # Start if missing; no-op if already up + COMPOSE_FILE=compose.local.yaml docker compose \ + --env-file .envs/local/opensearch.env \ + --env-file .envs/local/storage.env \ + up -d --remove-orphans +) + +docker network inspect sds-gateway-local-opensearch-net >/dev/null 2>&1 \ + || die "network sds-gateway-local-opensearch-net missing" + +wait_http_ok "${GATEWAY_URL}${GATEWAY_HEALTH_PATH}" "gateway" +docker exec sds-gateway-local-redis redis-cli ping | grep -q PONG \ + || die "gateway redis not responding" + +# --- federation images + up --- +info "Building federation sync image" +docker compose -f "${LOCAL_COMPOSE}" -f "${PEER_COMPOSE}" build + +info "Starting local + peer federation stacks" +docker compose -f "${LOCAL_COMPOSE}" -f "${PEER_COMPOSE}" up -d --remove-orphans + +# Peer OS is slow; assert host :9201 before trusting peer sync bootstrap +wait_http_ok "${PEER_OS_URL}" "peer opensearch" +wait_sync_operational "${LOCAL_SYNC_URL}" "local sync" +wait_sync_operational "${PEER_SYNC_URL}" "peer sync" + +# Local often site-hellos while peer is still starting; peer then pulls empty/missed +# lists. Restart both once peers are listening so mutual registration succeeds. +restart_and_wait_sync sds-federation-local-sync "${LOCAL_SYNC_URL}" "local sync" +restart_and_wait_sync sds-federation-peer-sync "${PEER_SYNC_URL}" "peer sync" + +info "Final health" +curl -sS "${LOCAL_SYNC_URL}" +echo +curl -sS "${PEER_SYNC_URL}" +echo +info "Done. Next: just seed-peer (peer-owned docs), then docker restart sds-federation-local-sync; publish local assets for crc→peer." diff --git a/federation/scripts/env-selection.sh b/federation/scripts/env-selection.sh index 3cb89bc2a..ea1dc330e 100755 --- a/federation/scripts/env-selection.sh +++ b/federation/scripts/env-selection.sh @@ -49,14 +49,8 @@ function get_target_value() { local) value='sds-federation-local-sync' ;; esac ;; - env_file) - case "${env_type}" in - production) value='.envs/production/sync.env' ;; - local) value='.envs/local/sync.env' ;; - esac - ;; *) - printf 'unsupported target: %s (use env, compose_file, sync_container, or env_file)\n' "${target}" >&2 + printf 'unsupported target: %s (use env, compose_file, or sync_container)\n' "${target}" >&2 exit 1 ;; esac @@ -71,7 +65,7 @@ function get_target_value() { function main() { if [[ $# -ne 1 ]]; then - printf 'usage: %s \n' "${0}" >&2 + printf 'usage: %s \n' "${0}" >&2 exit 1 fi diff --git a/federation/scripts/local_e2e/README.md b/federation/scripts/local_e2e/README.md new file mode 100644 index 000000000..04aed165a --- /dev/null +++ b/federation/scripts/local_e2e/README.md @@ -0,0 +1,102 @@ +# Local federation live test + +Exercise gateway → sync → OpenSearch without depending on any remote production site. + +**Never commit API tokens.** Copy `env.example` → `.env` in this directory (gitignored). + +## Identity (do not mix these up) + +| Setting | Role | +|---------|------| +| `FEDERATION_SITE_NAME` / toml `[site].name` | Short peer id; Redis channel `federation:events:{name}` | +| `SDS_SITE_FQDN` / toml `[site].fqdn` | OpenSearch + export `site_name`; sync `list-*` filters | + +Local compose defaults (see `federation.toml`): name `crc`, fqdn `sds.localhost`. + +## Prerequisites + +1. Gateway stack up (`gateway` compose local): app, Redis, OpenSearch, object store. +2. Federation sync up — either: + - Solo: `cd federation && just up` (port **8001**) + - Peer mesh: `just deploy-local-peer-2-peer` (local **8001**, peer **8002**) +3. Gateway `django.env`: + - `FEDERATION_ENABLED=true` + - `FEDERATION_SITE_NAME` / `SDS_SITE_FQDN` match `federation.toml` `[site]` +4. Mint export Api-Key (no `create_federation_sync_api_key` command): + + ```bash + TOKEN=$(grep '^FEDERATION_SYNC_DRF_TOKEN=' ../../../federation-shared.env | cut -d= -f2-) + curl -sS http://localhost:8000/users/get-federation-sync-api-key/ \ + -H "Authorization: Token $TOKEN" | jq -r .api_key + # save as FEDERATION_GATEWAY_API_KEY in .env + ``` + + Header form: `Authorization: Api-Key: ` (colon after `Api-Key`). +5. Gateway capture indices: `python manage.py init_indices` (inside app container). + +## Recommended: peer seed (no RF files) + +For peer→local backfill / `list-*` without a second gateway: + +```bash +cd federation +just seed-peer +docker restart sds-federation-local-sync +curl -s 'http://localhost:9200/fed-datasets/_search' \ + | jq '.hits.hits[]._source | {site: .site_name, name}' +``` + +## Local-owned data: publish any capture + +Create a public FINAL dataset in the UI or API (attach any local capture), then: + +```bash +cd gateway +docker compose -f compose.local.yaml exec sds-gateway-local-app \ + python manage.py publish_for_federation \ + --dataset-uuid \ + --capture-uuids +``` + +Optional Redis fan-out: + +```bash +cd federation +just simulate-redis --event-type updated --item-type dataset --uuid +``` + +Confirm OpenSearch `site_name` is the FQDN: + +```bash +curl -s 'http://localhost:9200/fed-datasets/_search' \ + | jq '.hits.hits[]._source.site_name' +# → matches SDS_SITE_FQDN / [site].fqdn +``` + +## Verify + +```bash +cd federation +set -a && source scripts/local_e2e/.env && set +a +just verify-federation-live +``` + +Or: + +```bash +curl -s http://localhost:8000/api/v1/federation/export/datasets/ \ + -H "Authorization: Api-Key: $FEDERATION_GATEWAY_API_KEY" | jq 'length' +curl -s http://localhost:8001/sync/api/v1/webhook/list-datasets/ | jq 'length' +``` + +## Optional: import a capture via SDK + +Federation does **not** ship remote download/upload helpers. Use the SpectrumX SDK from +`sdk/` against whatever host you choose (`SDS_HOST`, `SDS_SECRET_TOKEN`), then upload +to local gateway (`LOCAL_SDS_HOST` / `LOCAL_SDS_SECRET_TOKEN`). See `sdk/docs` for +`download_dataset` / `upload_capture`. Ensure channel layout and `drf_properties.h5` +match what the local gateway expects before `publish_for_federation`. + +## Remote peer (two servers) + +See [../../docs/peer-remote-test.md](../../docs/peer-remote-test.md). diff --git a/federation/scripts/local_e2e/env.example b/federation/scripts/local_e2e/env.example new file mode 100644 index 000000000..3ea78a539 --- /dev/null +++ b/federation/scripts/local_e2e/env.example @@ -0,0 +1,16 @@ +# Copy to federation/scripts/local_e2e/.env (never commit secrets). + +# Optional: remote SDS host for SDK imports (no default — set explicitly) +# SDS_HOST= +# SDS_SECRET_TOKEN= +# DATASET_UUID= +# TOP_LEVEL_DIR= + +# Local gateway (publish / upload target) +LOCAL_SDS_HOST=localhost:8000 +LOCAL_SDS_SECRET_TOKEN= + +# Federation sync live checks +FEDERATION_GATEWAY_API_KEY= +FEDERATION_SYNC_URL=http://localhost:8001 +LOCAL_SITE_FQDN=sds.localhost diff --git a/federation/scripts/seed_peer_opensearch.py b/federation/scripts/seed_peer_opensearch.py new file mode 100644 index 000000000..c6ea546ba --- /dev/null +++ b/federation/scripts/seed_peer_opensearch.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Seed dummy federated docs into a peer OpenSearch (no gateway). + +Use for peer→main backfill / list-* tests. ``site_name`` must be the peer FQDN +(matches federation.toml ``[site].fqdn``). + +Examples:: + + # Local peer stack (OpenSearch on host :9201) + uv run python scripts/seed_peer_opensearch.py \\ + --opensearch-url http://localhost:9201 \\ + --site-name peer.local + + # Remote peer after port-forward / public OS URL + uv run python scripts/seed_peer_opensearch.py \\ + --opensearch-url https://peer-os.example:9200 \\ + --site-name peer.example.com \\ + --user admin --password secret +""" + +from __future__ import annotations + +import argparse +import sys +from datetime import UTC +from datetime import datetime +from uuid import UUID + +from opensearchpy import OpenSearch +from sds_federation.schemas.webhooks import AssetTypeEnum +from sds_federation.services.fed_index import FederatedAssetIndexer +from sds_federation.services.fed_index import doc_id +from sds_federation.services.fed_index import ensure_fed_indices +from sds_federation.testing.sample_data import TEST_CAPTURE_UUID +from sds_federation.testing.sample_data import TEST_DATASET_UUID +from sds_federation.testing.sample_data import sample_federated_capture_doc +from sds_federation.testing.sample_data import sample_federated_dataset_doc + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--opensearch-url", + default="http://localhost:9201", + help="Peer OpenSearch base URL (local peer compose publishes :9201)", + ) + p.add_argument( + "--site-name", + default="peer.local", + help="Peer FQDN written into docs (must match toml [site].fqdn)", + ) + p.add_argument("--dataset-uuid", default=str(TEST_DATASET_UUID)) + p.add_argument("--capture-uuid", default=str(TEST_CAPTURE_UUID)) + p.add_argument("--user", default="", help="Optional basic-auth user") + p.add_argument("--password", default="", help="Optional basic-auth password") + p.add_argument( + "--use-ssl", + action="store_true", + help="Force SSL client (also inferred from https:// URL)", + ) + return p.parse_args() + + +def _client_from_url(args: argparse.Namespace) -> OpenSearch: + url = args.opensearch_url.rstrip("/") + use_ssl = args.use_ssl or url.startswith("https://") + # OpenSearch client wants host/port; parse simply. + without_scheme = url.split("://", 1)[-1] + host_port, _, _path = without_scheme.partition("/") + if ":" in host_port: + host, port_s = host_port.rsplit(":", 1) + port = int(port_s) + else: + host = host_port + port = 443 if use_ssl else 9200 + + kwargs: dict = { + "hosts": [{"host": host, "port": port}], + "use_ssl": use_ssl, + "verify_certs": False, + "ssl_show_warn": False, + } + if args.user: + kwargs["http_auth"] = (args.user, args.password) + return OpenSearch(**kwargs) + + +def main() -> int: + args = _parse_args() + site = args.site_name.strip() + if not site: + print("ERROR: --site-name is required", file=sys.stderr) + return 1 + + dataset_uuid = UUID(args.dataset_uuid) + capture_uuid = UUID(args.capture_uuid) + client = _client_from_url(args) + ensure_fed_indices(client) + indexer = FederatedAssetIndexer(client) + event_at = datetime.now(UTC) + + dataset = sample_federated_dataset_doc(uuid=dataset_uuid, site_name=site) + dataset = dataset.model_copy( + update={ + "name": f"Peer seed dataset ({site})", + "is_public": True, + "status": "final", + "status_display": "Final", + "created_at": event_at.isoformat(), + "updated_at": event_at.isoformat(), + }, + ) + capture = sample_federated_capture_doc(uuid=capture_uuid, site_name=site) + capture = capture.model_copy( + update={ + "name": f"Peer seed capture ({site})", + "channel": "chA", + "public_dataset_ids": [str(dataset_uuid)], + "created_at": event_at.isoformat(), + "updated_at": event_at.isoformat(), + }, + ) + + indexer.apply_asset_event( + event_at=event_at, + site_name=site, + asset=dataset, + asset_type=AssetTypeEnum.DATASET, + ) + indexer.apply_asset_event( + event_at=event_at, + site_name=site, + asset=capture, + asset_type=AssetTypeEnum.CAPTURE, + ) + + print(f"Seeded {AssetTypeEnum.DATASET.index_name} id={doc_id(site, dataset_uuid)}") + print(f"Seeded {AssetTypeEnum.CAPTURE.index_name} id={doc_id(site, capture_uuid)}") + print(f"site_name={site!r} opensearch={args.opensearch_url}") + print( + "Restart the *other* site's sync (or wait for site-hello) to pull these docs." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/federation/scripts/verify_federation_live.py b/federation/scripts/verify_federation_live.py new file mode 100644 index 000000000..92043dddf --- /dev/null +++ b/federation/scripts/verify_federation_live.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Smoke-test federation export + search against a running local sync service.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + +import httpx + +HTTP_OK = 200 +SYNC_PREFIX = "/sync" + + +def _normalize_sync_base(url: str) -> str: + """Ensure sync URLs include the mounted /sync prefix.""" + base = url.rstrip("/") + if not base.endswith(SYNC_PREFIX): + base = f"{base}{SYNC_PREFIX}" + return base + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--sync-base", + default=os.environ.get("FEDERATION_SYNC_URL", "http://localhost:8001"), + ) + parser.add_argument( + "--gateway-export", + default=os.environ.get( + "GATEWAY_EXPORT_URL", + "http://localhost:8000/api/v1/federation/export/datasets/", + ), + ) + parser.add_argument( + "--api-key", + default=os.environ.get("FEDERATION_GATEWAY_API_KEY", ""), + ) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + base = _normalize_sync_base(args.sync_base) + + with httpx.Client(timeout=30.0) as client: + health = client.get(f"{base}/health") + print(f"health {health.status_code}: {health.text[:200]}") + if health.status_code != HTTP_OK: + return 1 + + headers = {} + if args.api_key: + headers["Authorization"] = f"Api-Key: {args.api_key}" + export = client.get(args.gateway_export, headers=headers) + print(f"gateway export {export.status_code}") + if export.status_code == HTTP_OK: + data = export.json() + print(f" export datasets: {len(data)}") + if data: + print(f" first site_name: {data[0].get('site_name')}") + + listed = client.get( + f"{base}/api/v1/webhook/list-datasets/", + ) + print(f"listed datasets {listed.status_code}") + if listed.status_code == HTTP_OK: + body = listed.json() + print(json.dumps(body, indent=2)[:1500]) + else: + print(listed.text, file=sys.stderr) + return 2 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/federation/sds_federation/main.py b/federation/sds_federation/main.py index f8a19dc9e..6bc1e51cb 100644 --- a/federation/sds_federation/main.py +++ b/federation/sds_federation/main.py @@ -7,15 +7,17 @@ from fastapi import FastAPI from loguru import logger -from opensearchpy import OpenSearch +from redis.exceptions import TimeoutError as RedisTimeoutError +from sds_opensearch_query.client import build_opensearch_client from sds_federation.models import load_federation_config from sds_federation.routes.health import health_router from sds_federation.routes.webhooks import webhooks_router from sds_federation.services.bootstrap import run_bootstrap from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.local_events import build_gateway_http_client +from sds_federation.services.fed_index import ensure_fed_indices from sds_federation.services.local_events import run_federation_subscriber +from sds_federation.services.peer_http import build_peer_http_client from sds_federation.services.peer_registry import PeerRegistry API_PREFIX = "/api/v1" @@ -30,6 +32,14 @@ def _bootstrap_enabled() -> bool: ) +def get_setting(key: str, default: str = "") -> str: + return os.environ.get(key, default) + + +def _env_flag(key: str) -> bool: + return get_setting(key).strip().lower() in ("1", "true", "yes") + + sync_app = FastAPI(title="SDS Federation Sync") sync_app.include_router(health_router) sync_app.include_router(webhooks_router, prefix=API_PREFIX) @@ -38,12 +48,22 @@ def _bootstrap_enabled() -> bool: @asynccontextmanager async def lifespan(app: FastAPI): config = load_federation_config() - http = build_gateway_http_client() + http = build_peer_http_client() + + os_client = build_opensearch_client( + host=get_setting("OPENSEARCH_HOST", "opensearch"), + port=int(get_setting("OPENSEARCH_PORT", "9200")), + user=get_setting("OPENSEARCH_USER"), + password=get_setting("OPENSEARCH_PASSWORD"), + use_ssl=_env_flag("OPENSEARCH_USE_SSL"), + verify_certs=_env_flag("OPENSEARCH_VERIFY_CERTS"), + ca_certs=get_setting("OPENSEARCH_CA_CERTS") or None, + ) + try: + ensure_fed_indices(os_client) + except Exception as exc: # noqa: BLE001 + logger.error("Failed to ensure fed-* OpenSearch indices: {}", exc) - # TODO: Create a shared OpenSearch client for both apps in a later PR. - os_host = os.environ.get("OPENSEARCH_HOST", "opensearch") - os_port = os.environ.get("OPENSEARCH_PORT", "9200") - os_client = OpenSearch(hosts=[{"host": os_host, "port": int(os_port)}]) peer_registry = PeerRegistry() fed_indexer = FederatedAssetIndexer(os_client) @@ -90,7 +110,8 @@ async def lifespan(app: FastAPI): stop.set() sub_task.cancel() - with suppress(asyncio.CancelledError): + # Pubsub listen may surface TimeoutError while cancelling the blocked read. + with suppress(asyncio.CancelledError, TimeoutError, RedisTimeoutError): await sub_task await http.aclose() diff --git a/federation/sds_federation/models.py b/federation/sds_federation/models.py index 8d4165ab0..6cb1182eb 100644 --- a/federation/sds_federation/models.py +++ b/federation/sds_federation/models.py @@ -24,6 +24,11 @@ class PeerInfo(BaseModel): gateway_export_api_key: str = "" +def site_name_for_federation(site: SiteInfo | PeerInfo) -> str: + """FQDN stored in export docs and OpenSearch ``site_name`` (RFC §6).""" + return site.fqdn + + class FederationConfig(BaseModel): site: SiteInfo peers: list[PeerInfo] = Field(default_factory=list) @@ -31,6 +36,10 @@ class FederationConfig(BaseModel): sync_service_url: AnyHttpUrl +def allowed_federated_origin_fqdns(config: FederationConfig) -> set[str]: + return {peer.fqdn for peer in config.peers} | {config.site.fqdn} + + def load_federation_config() -> FederationConfig: path = Path(os.environ.get("FEDERATION_CONFIG_PATH", "federation.toml")) data = tomllib.loads(path.read_text(encoding="utf-8")) diff --git a/federation/sds_federation/routes/webhooks.py b/federation/sds_federation/routes/webhooks.py index 055bca0b4..729a10a67 100644 --- a/federation/sds_federation/routes/webhooks.py +++ b/federation/sds_federation/routes/webhooks.py @@ -4,17 +4,21 @@ import httpx from fastapi import APIRouter +from fastapi import BackgroundTasks from fastapi import HTTPException from fastapi import Request from loguru import logger +from sds_federation.models import PeerInfo +from sds_federation.models import allowed_federated_origin_fqdns +from sds_federation.models import site_name_for_federation from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import AssetUpdatedWebhook from sds_federation.schemas.webhooks import SiteHelloWebhook from sds_federation.services.bootstrap import backfill_peer_on_hello from sds_federation.services.bootstrap import peer_by_name from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_search import alist_federated_assets_for_site +from sds_federation.services.fed_index import alist_federated_assets_for_site from sds_federation.services.peer_registry import PeerRegistry from sds_federation.services.peer_sync import peer_for_outbound @@ -32,7 +36,7 @@ def _local_site_name(request: Request) -> str: config = getattr(request.app.state, "config", None) if config is None: raise HTTPException(status_code=503, detail="Config not ready") - return config.site.name + return site_name_for_federation(config.site) def _indexer(request: Request) -> FederatedAssetIndexer: @@ -55,27 +59,31 @@ def _http_client(request: Request) -> httpx.AsyncClient | None: def _allowed_origin_sites(request: Request, payload: AssetUpdatedWebhook) -> None: config = request.app.state.config - if payload.site_name == config.site.name: + if payload.site_name == site_name_for_federation(config.site): raise HTTPException( status_code=403, detail="Local site metadata is not accepted via peer webhooks", ) - allowed = {peer.name for peer in config.peers} + + allowed = allowed_federated_origin_fqdns(config) if payload.site_name not in allowed: raise HTTPException(status_code=403, detail="Unknown origin site") -@webhooks_router.post("/webhook/dataset-updated") -async def dataset_updated(payload: AssetUpdatedWebhook, request: Request) -> dict: - """ - Handle dataset-updated webhook from another site. - Index the dataset in the local site's OpenSearch. - """ +async def _apply_asset_updated( + payload: AssetUpdatedWebhook, + request: Request, + *, + expected_type: AssetTypeEnum, +) -> dict: _allowed_origin_sites(request, payload) - if payload.asset is None or payload.asset_type is not AssetTypeEnum.DATASET: + if payload.asset is None or payload.asset_type is not expected_type: raise HTTPException( status_code=422, - detail="Dataset body required for dataset-updated webhook.", + detail=( + f"{expected_type.value.capitalize()} body required for " + f"{expected_type.value}-updated webhook." + ), ) try: await asyncio.to_thread( @@ -90,66 +98,81 @@ async def dataset_updated(payload: AssetUpdatedWebhook, request: Request) -> dic return {"status": "accepted"} +@webhooks_router.post("/webhook/dataset-updated") +async def dataset_updated(payload: AssetUpdatedWebhook, request: Request) -> dict: + """Handle dataset-updated webhook from another site.""" + return await _apply_asset_updated( + payload, + request, + expected_type=AssetTypeEnum.DATASET, + ) + + @webhooks_router.post("/webhook/capture-updated") async def capture_updated(payload: AssetUpdatedWebhook, request: Request) -> dict: - """ - Handle capture-updated webhook from another site. - Index the capture in the local site's OpenSearch. - """ - _allowed_origin_sites(request, payload) - if payload.asset is None or payload.asset_type is not AssetTypeEnum.CAPTURE: - raise HTTPException( - status_code=422, - detail="Capture body required for capture-updated webhook.", - ) - try: - await asyncio.to_thread( - _indexer(request).apply_asset_event, - event_at=payload.timestamp, - site_name=payload.site_name, - asset=payload.asset, - asset_type=payload.asset_type, - ) - except ValueError as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc - return {"status": "accepted"} + """Handle capture-updated webhook from another site.""" + return await _apply_asset_updated( + payload, + request, + expected_type=AssetTypeEnum.CAPTURE, + ) -@webhooks_router.get("/webhook/list-datasets/") -async def list_datasets(request: Request) -> list[dict]: - """ - List all datasets for the local site to new peer on bootstrap. - """ +async def _list_local_assets( + request: Request, + asset_type: AssetTypeEnum, +) -> list[dict]: docs = await alist_federated_assets_for_site( _opensearch(request), site_name=_local_site_name(request), - asset_type=AssetTypeEnum.DATASET, + asset_type=asset_type, ) return [doc.model_dump(mode="json") for doc in docs] +@webhooks_router.get("/webhook/list-datasets/") +async def list_datasets(request: Request) -> list[dict]: + """List all datasets for the local site to new peer on bootstrap.""" + return await _list_local_assets(request, AssetTypeEnum.DATASET) + + @webhooks_router.get("/webhook/list-captures/") async def list_captures(request: Request) -> list[dict]: - """ - List all captures for the local site to new peer on bootstrap. - """ - docs = await alist_federated_assets_for_site( - _opensearch(request), - site_name=_local_site_name(request), - asset_type=AssetTypeEnum.CAPTURE, - ) - return [doc.model_dump(mode="json") for doc in docs] + """List all captures for the local site to new peer on bootstrap.""" + return await _list_local_assets(request, AssetTypeEnum.CAPTURE) + + +async def _run_site_hello_backfill( + http: httpx.AsyncClient, + peer: PeerInfo, + indexer: FederatedAssetIndexer, +) -> None: + """Background: pull registering peer's list-* exports (with retries).""" + try: + indexed = await backfill_peer_on_hello(http, peer, indexer) + logger.info( + "site-hello backfill indexed {} document(s) from {} ({})", + indexed, + peer.name, + peer.sync_service_url, + ) + except Exception as exc: # noqa: BLE001 + logger.error("site-hello backfill failed for {}: {}", peer.name, exc) @webhooks_router.post("/webhook/site-hello") -async def site_hello(payload: SiteHelloWebhook, request: Request) -> dict: +async def site_hello( + payload: SiteHelloWebhook, + request: Request, + background_tasks: BackgroundTasks, +) -> dict: config = request.app.state.config - if payload.site_name == config.site.name: + if payload.site_name == site_name_for_federation(config.site): raise HTTPException( status_code=422, detail="Cannot register self via site-hello", ) - allowed = {peer.name for peer in config.peers} + allowed = {peer.fqdn for peer in config.peers} if payload.site_name not in allowed: raise HTTPException(status_code=403, detail="Unknown registering site") @@ -173,19 +196,13 @@ async def site_hello(payload: SiteHelloWebhook, request: Request) -> dict: ) else: outbound = peer_for_outbound(peer, _peer_registry(request)) - try: - indexed = await backfill_peer_on_hello( - http, - outbound, - _indexer(request), - ) - logger.info( - "site-hello backfill indexed {} document(s) from {} ({})", - indexed, - outbound.name, - outbound.sync_service_url, - ) - except Exception as exc: # noqa: BLE001 - logger.error("site-hello backfill failed for {}: {}", outbound.name, exc) + # Return registered immediately; backfill retries while the peer finishes + # binding so mutual startup races do not fail the hello handshake. + background_tasks.add_task( + _run_site_hello_backfill, + http, + outbound, + _indexer(request), + ) return {"status": "registered", "site_name": hello.site_name} diff --git a/federation/sds_federation/schemas/opensearch_indices.py b/federation/sds_federation/schemas/opensearch_indices.py new file mode 100644 index 000000000..73000bc40 --- /dev/null +++ b/federation/sds_federation/schemas/opensearch_indices.py @@ -0,0 +1,49 @@ +"""OpenSearch index definitions for federated metadata (RFC fed-* indices). + +Explicit ``properties`` match the RFC search-facing field lists; export-only and +envelope fields (e.g. ``status``, ``capture_props``, ``is_deleted``) rely on +dynamic mapping. +""" + +from __future__ import annotations + +from typing import Any + +from sds_opensearch_query.mapping import FED_INDEX_SETTINGS +from sds_opensearch_query.mapping import RFC_FED_CAPTURE_PROPERTIES +from sds_opensearch_query.mapping import RFC_FED_DATASET_PROPERTIES +from sds_opensearch_query.mapping import fed_capture_mappings +from sds_opensearch_query.mapping import fed_dataset_mappings + +from sds_federation.schemas.webhooks import AssetTypeEnum + +__all__ = [ + "FED_INDEX_SETTINGS", + "RFC_FED_CAPTURE_PROPERTIES", + "RFC_FED_DATASET_PROPERTIES", + "fed_capture_mappings", + "fed_dataset_mappings", + "index_body_for_asset", + "index_body_for_index_name", +] + + +def index_body_for_asset(asset_type: AssetTypeEnum) -> dict[str, Any]: + mappings = ( + fed_dataset_mappings() + if asset_type == AssetTypeEnum.DATASET + else fed_capture_mappings() + ) + return { + "settings": FED_INDEX_SETTINGS, + "mappings": mappings, + } + + +def index_body_for_index_name(index_name: str) -> dict[str, Any]: + if index_name == AssetTypeEnum.DATASET.index_name: + return index_body_for_asset(AssetTypeEnum.DATASET) + if index_name == AssetTypeEnum.CAPTURE.index_name: + return index_body_for_asset(AssetTypeEnum.CAPTURE) + msg = f"unknown federated index: {index_name}" + raise ValueError(msg) diff --git a/federation/sds_federation/schemas/webhooks.py b/federation/sds_federation/schemas/webhooks.py index 24d236642..b9dadf6a2 100644 --- a/federation/sds_federation/schemas/webhooks.py +++ b/federation/sds_federation/schemas/webhooks.py @@ -33,7 +33,7 @@ def index_name(self) -> str: class FederatedDatasetDoc(BaseModel): """Must match gateway DatasetFederationSerializer output keys exactly.""" - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="allow") uuid: UUID name: str @@ -57,7 +57,7 @@ class FederatedDatasetDoc(BaseModel): is_public: bool = False owner_name: str = "" updated_at: str | None = None - site_name: str + site_name: str # peer site FQDN (RFC §6), not federation.toml [site].name size: int = 0 capture_count: int = 0 capture_file_count: int = 0 @@ -69,7 +69,7 @@ class FederatedDatasetDoc(BaseModel): class FederatedCaptureDoc(BaseModel): """Must match gateway CaptureFederationSerializer output keys exactly.""" - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="allow") uuid: UUID name: str = "" @@ -79,10 +79,11 @@ class FederatedCaptureDoc(BaseModel): top_level_dir: str = "" created_at: str | None = None updated_at: str | None = None - site_name: str + site_name: str # peer site FQDN (RFC §6), not federation.toml [site].name file_count: int = 0 size: int = 0 capture_props: dict[str, Any] = Field(default_factory=dict) + search_props: dict[str, Any] = Field(default_factory=dict) public_dataset_ids: list[str] = Field(default_factory=list) is_deleted: bool = False deleted_at: str | None = None @@ -108,7 +109,7 @@ class SiteHelloWebhook(BaseModel): model_config = ConfigDict(extra="forbid") - site_name: str + site_name: str # registering peer FQDN (same as fqdn) fqdn: str display_name: str = "" sync_service_url: AnyHttpUrl diff --git a/federation/sds_federation/services/bootstrap.py b/federation/sds_federation/services/bootstrap.py index ce0b64010..f04a7fc88 100644 --- a/federation/sds_federation/services/bootstrap.py +++ b/federation/sds_federation/services/bootstrap.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import os from datetime import UTC from datetime import datetime @@ -12,11 +13,13 @@ from sds_federation.models import FederationConfig from sds_federation.models import PeerInfo +from sds_federation.models import site_name_for_federation from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import FederatedCaptureDoc from sds_federation.schemas.webhooks import FederatedDatasetDoc from sds_federation.schemas.webhooks import SiteHelloWebhook from sds_federation.schemas.webhooks import asset_doc_class +from sds_federation.services.peer_http import peer_request from sds_federation.services.peer_sync import peer_webhook_url if TYPE_CHECKING: @@ -24,6 +27,10 @@ SITE_HELLO_PATH = "/webhook/site-hello" _MINT_PATH = "/users/get-federation-sync-api-key/" +# Peer sync often receives site-hello while the peer process is still binding. +_PEER_LIST_ATTEMPTS = 5 +_PEER_LIST_BACKOFF_SECS = 0.5 +_HTTP_INTERNAL_ERROR = 500 def _export_list_url(peer: PeerInfo, asset_type: AssetTypeEnum) -> str: @@ -129,14 +136,16 @@ async def _get_json( url: str, *, api_key: str, - verify: str | bool = True, + ca_cert_path: str = "", ) -> list | dict: headers = _gateway_auth_headers(api_key) - if verify is not True and verify: - async with httpx.AsyncClient(verify=verify, timeout=http.timeout) as client: - resp = await client.get(url, headers=headers) - else: - resp = await http.get(url, headers=headers) + resp = await peer_request( + http, + "GET", + url, + ca_cert_path=ca_cert_path, + headers=headers, + ) resp.raise_for_status() return resp.json() @@ -154,7 +163,7 @@ async def fetch_gateway_export_list( http, url, api_key=api_key, - verify=peer.ca_cert_path or True, + ca_cert_path=peer.ca_cert_path, ) if not isinstance(data, list): msg = f"expected list from {url}, got {type(data).__name__}" @@ -163,24 +172,70 @@ async def fetch_gateway_export_list( return [doc_class.model_validate(item) for item in data] +def _is_retryable_peer_list_error(exc: BaseException) -> bool: + """True for transient connect/read failures while a peer sync is starting.""" + if isinstance( + exc, + ( + httpx.ConnectError, + httpx.ConnectTimeout, + httpx.ReadTimeout, + httpx.WriteTimeout, + httpx.PoolTimeout, + httpx.RemoteProtocolError, + ), + ): + return True + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code >= _HTTP_INTERNAL_ERROR + return False + + async def fetch_peer_sync_list( http: httpx.AsyncClient, peer: PeerInfo, asset_type: AssetTypeEnum, + *, + attempts: int = _PEER_LIST_ATTEMPTS, + backoff_secs: float = _PEER_LIST_BACKOFF_SECS, ) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: - """Pull peer-owned docs from the peer sync service (fed-* OpenSearch export).""" + """Pull peer-owned docs from the peer sync service (fed-* OpenSearch export). + + Retries transient connection/5xx errors so site-hello backfill survives peer + startup races. + """ url = _webhook_list_url(peer, asset_type) - data = await _get_json( - http, - url, - api_key="", - verify=peer.ca_cert_path or True, - ) - if not isinstance(data, list): - msg = f"expected list from {url}, got {type(data).__name__}" - raise TypeError(msg) - doc_class = asset_doc_class(asset_type) - return [doc_class.model_validate(item) for item in data] + last_exc: httpx.HTTPError | None = None + tries = max(1, attempts) + for attempt in range(1, tries + 1): + try: + data = await _get_json( + http, + url, + api_key="", + ca_cert_path=peer.ca_cert_path, + ) + if not isinstance(data, list): + msg = f"expected list from {url}, got {type(data).__name__}" + raise TypeError(msg) + doc_class = asset_doc_class(asset_type) + return [doc_class.model_validate(item) for item in data] + except httpx.HTTPError as exc: + last_exc = exc + if not _is_retryable_peer_list_error(exc) or attempt >= tries: + raise + logger.warning( + "peer sync list {} {} attempt {}/{} failed ({}); retrying in {:.1f}s", + peer.name, + asset_type.value, + attempt, + tries, + exc, + backoff_secs * attempt, + ) + await asyncio.sleep(backoff_secs * attempt) + assert last_exc is not None + raise last_exc def _parse_doc_event_at( @@ -211,14 +266,14 @@ def _index_export_docs( ) -> int: indexed = 0 for doc in docs: - if doc.site_name != peer.name: + if doc.site_name != site_name_for_federation(peer): logger.error( - "bootstrap export failed for {} {}: site name mismatch " - "(doc.site_name={!r}, peer.name={!r})", + "bootstrap export failed for {} {}: site name (FQDN) mismatch " + "(doc.site_name={!r}, peer.fqdn={!r})", peer.name, asset_type.value, doc.site_name, - peer.name, + site_name_for_federation(peer), ) continue @@ -353,7 +408,7 @@ async def bootstrap_all_peers( def peer_by_name(config: FederationConfig, site_name: str) -> PeerInfo | None: for peer in config.peers: - if peer.name == site_name: + if site_name in (peer.name, peer.fqdn): return peer return None @@ -376,7 +431,7 @@ async def backfill_peer_on_hello( def _site_hello_payload(config: FederationConfig) -> SiteHelloWebhook: return SiteHelloWebhook( - site_name=config.site.name, + site_name=site_name_for_federation(config.site), fqdn=config.site.fqdn, display_name=config.site.display_name, sync_service_url=config.sync_service_url, @@ -391,14 +446,13 @@ async def push_site_hello_to_peer( ) -> dict: url = peer_webhook_url(peer, SITE_HELLO_PATH) body = _site_hello_payload(config).model_dump(mode="json") - if peer.ca_cert_path: - async with httpx.AsyncClient( - verify=peer.ca_cert_path, - timeout=http.timeout, - ) as tls_client: - resp = await tls_client.post(url, json=body) - else: - resp = await http.post(url, json=body) + resp = await peer_request( + http, + "POST", + url, + ca_cert_path=peer.ca_cert_path, + json=body, + ) resp.raise_for_status() data = resp.json() if not isinstance(data, dict): diff --git a/federation/sds_federation/services/fed_index.py b/federation/sds_federation/services/fed_index.py index c359cb727..4af0e49ab 100644 --- a/federation/sds_federation/services/fed_index.py +++ b/federation/sds_federation/services/fed_index.py @@ -1,16 +1,37 @@ +from __future__ import annotations + +import asyncio from datetime import datetime -from uuid import UUID +from typing import TYPE_CHECKING +from typing import Any from opensearchpy import OpenSearch from opensearchpy.exceptions import NotFoundError +from sds_opensearch_query.index_write import federated_doc_id +from sds_opensearch_query.index_write import index_federated_document +from sds_federation.schemas.opensearch_indices import index_body_for_asset from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import FederatedCaptureDoc from sds_federation.schemas.webhooks import FederatedDatasetDoc +from sds_federation.schemas.webhooks import asset_doc_class + +if TYPE_CHECKING: + from uuid import UUID +# Backwards-compatible alias used throughout federation tests/scripts. +doc_id = federated_doc_id -def doc_id(site_name: str, uuid: UUID) -> str: - return f"{site_name}:{uuid}" + +def ensure_fed_indices(client: OpenSearch) -> None: + for asset_type in AssetTypeEnum: + index_name = asset_type.index_name + if client.indices.exists(index=index_name): + continue + client.indices.create( + index=index_name, + body=index_body_for_asset(asset_type), + ) def _parse_event_at(value: object) -> datetime | None: @@ -21,6 +42,142 @@ def _parse_event_at(value: object) -> datetime | None: return None +_FEDERATION_META_KEYS = frozenset({"federation_event_at"}) + + +def _strip_federation_meta(source: dict) -> dict: + return { + key: value for key, value in source.items() if key not in _FEDERATION_META_KEYS + } + + +_LIST_PAGE_SIZE = 1000 + + +def _parse_hit( + source: dict, + asset_type: AssetTypeEnum, +) -> FederatedDatasetDoc | FederatedCaptureDoc | None: + if not isinstance(source, dict): + return None + doc_class = asset_doc_class(asset_type) + return doc_class.model_validate(_strip_federation_meta(source)) + + +def load_federated_asset( + client: OpenSearch, + *, + site_name: str, + uuid: UUID, + asset_type: AssetTypeEnum, +) -> FederatedDatasetDoc | FederatedCaptureDoc | None: + """Return the indexed document for a site asset, or None if missing.""" + + def _get() -> dict | None: + try: + response = client.get( + index=asset_type.index_name, + id=federated_doc_id(site_name, uuid), + ) + except NotFoundError: + return None + source = response.get("_source") + if not isinstance(source, dict): + return None + return source + + source = _get() + if source is None: + return None + return _parse_hit(source, asset_type) + + +async def aload_federated_asset( + client: OpenSearch, + *, + site_name: str, + uuid: UUID, + asset_type: AssetTypeEnum, +) -> FederatedDatasetDoc | FederatedCaptureDoc | None: + return await asyncio.to_thread( + load_federated_asset, + client, + site_name=site_name, + uuid=uuid, + asset_type=asset_type, + ) + + +def _site_owned_query(site_name: str) -> dict[str, Any]: + return { + "bool": { + "should": [ + {"term": {"site_name.keyword": site_name}}, + {"term": {"site_name": site_name}}, + ], + "minimum_should_match": 1, + } + } + + +def list_federated_assets_for_site( + client: OpenSearch, + *, + site_name: str, + asset_type: AssetTypeEnum, +) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: + """Return all fed-* docs owned by ``site_name`` (paginated search_after).""" + docs: list[FederatedDatasetDoc | FederatedCaptureDoc] = [] + search_after: list[Any] | None = None + + while True: + body: dict[str, Any] = { + "size": _LIST_PAGE_SIZE, + "sort": [{"_id": "asc"}], + "query": _site_owned_query(site_name), + } + if search_after is not None: + body["search_after"] = search_after + + response = client.search(index=asset_type.index_name, body=body) + hits = (response.get("hits") or {}).get("hits") or [] + if not hits: + break + + for hit in hits: + source = hit.get("_source") + if not isinstance(source, dict): + continue + if source.get("site_name") != site_name: + continue + parsed = _parse_hit(source, asset_type) + if parsed is not None: + docs.append(parsed) + + if len(hits) < _LIST_PAGE_SIZE: + break + last_sort = hits[-1].get("sort") + if not isinstance(last_sort, list) or not last_sort: + break + search_after = last_sort + + return docs + + +async def alist_federated_assets_for_site( + client: OpenSearch, + *, + site_name: str, + asset_type: AssetTypeEnum, +) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: + return await asyncio.to_thread( + list_federated_assets_for_site, + client, + site_name=site_name, + asset_type=asset_type, + ) + + class FederatedAssetIndexer: def __init__(self, client: OpenSearch) -> None: self._client = client @@ -43,7 +200,7 @@ def _is_stale( *, index_name: str, ) -> bool: - key = doc_id(site_name, uuid) + key = federated_doc_id(site_name, uuid) prev = self._last_event.get(key) if prev is None: prev = self._stored_event_at(index_name, key) @@ -52,7 +209,7 @@ def _is_stale( return bool(prev is not None and event_at <= prev) def _mark_applied(self, site_name: str, uuid: UUID, event_at: datetime) -> None: - self._last_event[doc_id(site_name, uuid)] = event_at + self._last_event[federated_doc_id(site_name, uuid)] = event_at def apply_asset_event( self, @@ -78,14 +235,13 @@ def apply_asset_event( ): return False - _id = doc_id(site_name, asset.uuid) - body = asset.model_dump(mode="json") - body["federation_event_at"] = event_at.isoformat() - self._client.index( - index=asset_type.index_name, - id=_id, - body=body, - refresh="wait_for", + index_federated_document( + self._client, + index_name=asset_type.index_name, + site_name=site_name, + uuid=asset.uuid, + body=asset.model_dump(mode="json"), + event_at=event_at, ) self._mark_applied(site_name, asset.uuid, event_at) diff --git a/federation/sds_federation/services/fed_search.py b/federation/sds_federation/services/fed_search.py deleted file mode 100644 index 65d6b4951..000000000 --- a/federation/sds_federation/services/fed_search.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Read federated documents from shared fed-* OpenSearch indices.""" - -from __future__ import annotations - -import asyncio -from typing import TYPE_CHECKING -from typing import Any - -from opensearchpy.exceptions import NotFoundError - -from sds_federation.schemas.webhooks import AssetTypeEnum -from sds_federation.schemas.webhooks import FederatedCaptureDoc -from sds_federation.schemas.webhooks import FederatedDatasetDoc -from sds_federation.schemas.webhooks import asset_doc_class -from sds_federation.services.fed_index import doc_id - -if TYPE_CHECKING: - from uuid import UUID - - from opensearchpy import OpenSearch - -_FEDERATION_META_KEYS = frozenset({"federation_event_at"}) -_LIST_PAGE_SIZE = 1000 - - -def _strip_federation_meta(source: dict) -> dict: - return { - key: value for key, value in source.items() if key not in _FEDERATION_META_KEYS - } - - -def _parse_hit( - source: dict, - asset_type: AssetTypeEnum, -) -> FederatedDatasetDoc | FederatedCaptureDoc | None: - if not isinstance(source, dict): - return None - doc_class = asset_doc_class(asset_type) - return doc_class.model_validate(_strip_federation_meta(source)) - - -def load_federated_asset( - client: OpenSearch, - *, - site_name: str, - uuid: UUID, - asset_type: AssetTypeEnum, -) -> FederatedDatasetDoc | FederatedCaptureDoc | None: - """Return the indexed document for a site asset, or None if missing.""" - - def _get() -> dict | None: - try: - response = client.get( - index=asset_type.index_name, id=doc_id(site_name, uuid) - ) - except NotFoundError: - return None - source = response.get("_source") - if not isinstance(source, dict): - return None - return source - - source = _get() - if source is None: - return None - return _parse_hit(source, asset_type) - - -def _site_owned_query(site_name: str) -> dict[str, Any]: - return { - "bool": { - "should": [ - {"term": {"site_name.keyword": site_name}}, - {"term": {"site_name": site_name}}, - ], - "minimum_should_match": 1, - } - } - - -def list_federated_assets_for_site( - client: OpenSearch, - *, - site_name: str, - asset_type: AssetTypeEnum, -) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: - """Return all fed-* docs owned by ``site_name`` (paginated search_after).""" - docs: list[FederatedDatasetDoc | FederatedCaptureDoc] = [] - search_after: list[Any] | None = None - - while True: - body: dict[str, Any] = { - "size": _LIST_PAGE_SIZE, - "sort": [{"_id": "asc"}], - "query": _site_owned_query(site_name), - } - if search_after is not None: - body["search_after"] = search_after - - response = client.search(index=asset_type.index_name, body=body) - hits = (response.get("hits") or {}).get("hits") or [] - if not hits: - break - - for hit in hits: - source = hit.get("_source") - if not isinstance(source, dict): - continue - if source.get("site_name") != site_name: - continue - parsed = _parse_hit(source, asset_type) - if parsed is not None: - docs.append(parsed) - - if len(hits) < _LIST_PAGE_SIZE: - break - last_sort = hits[-1].get("sort") - if not isinstance(last_sort, list) or not last_sort: - break - search_after = last_sort - - return docs - - -async def aload_federated_asset( - client: OpenSearch, - *, - site_name: str, - uuid: UUID, - asset_type: AssetTypeEnum, -) -> FederatedDatasetDoc | FederatedCaptureDoc | None: - return await asyncio.to_thread( - load_federated_asset, - client, - site_name=site_name, - uuid=uuid, - asset_type=asset_type, - ) - - -async def alist_federated_assets_for_site( - client: OpenSearch, - *, - site_name: str, - asset_type: AssetTypeEnum, -) -> list[FederatedDatasetDoc | FederatedCaptureDoc]: - return await asyncio.to_thread( - list_federated_assets_for_site, - client, - site_name=site_name, - asset_type=asset_type, - ) diff --git a/federation/sds_federation/services/local_events.py b/federation/sds_federation/services/local_events.py index 9f9abb94b..a37b94c8f 100644 --- a/federation/sds_federation/services/local_events.py +++ b/federation/sds_federation/services/local_events.py @@ -11,12 +11,14 @@ from opensearchpy import OpenSearch from sds_federation.models import FederationConfig +from sds_federation.models import site_name_for_federation from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.schemas.webhooks import AssetUpdatedWebhook from sds_federation.schemas.webhooks import FederatedCaptureDoc from sds_federation.schemas.webhooks import FederatedDatasetDoc from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_search import aload_federated_asset +from sds_federation.services.fed_index import aload_federated_asset +from sds_federation.services.peer_http import build_peer_http_client from sds_federation.services.peer_registry import PeerRegistry from sds_federation.services.peer_sync import push_asset_updated_to_peers from sds_federation.services.redis_channel import resolve_federation_events_channel @@ -26,6 +28,9 @@ Awaitable[FederatedDatasetDoc | FederatedCaptureDoc | None], ] +# Backwards-compatible alias for main.py / older call sites. +build_gateway_http_client = build_peer_http_client + def parse_redis_event_payload( data: dict, @@ -48,7 +53,7 @@ async def _default_load_asset( ) -> FederatedDatasetDoc | FederatedCaptureDoc | None: return await aload_federated_asset( os_client, - site_name=config.site.name, + site_name=site_name_for_federation(config.site), uuid=uuid, asset_type=asset_type, ) @@ -95,7 +100,7 @@ async def handle_redis_asset_event( payload = AssetUpdatedWebhook( timestamp=timestamp, - site_name=config.site.name, + site_name=site_name_for_federation(config.site), asset=asset, asset_type=asset_type, ) @@ -165,9 +170,16 @@ async def run_federation_subscriber( config.site.name, gateway_site, ) - client = aioredis.from_url(redis_url) + # socket_timeout=None: idle pubsub.listen() must not raise TimeoutError and + # kill the subscriber task (that fails /sync/health redis_subscriber check). + client = aioredis.from_url( + redis_url, + socket_timeout=None, + socket_connect_timeout=5.0, + ) pubsub = client.pubsub() await pubsub.subscribe(resolved_channel) + log.info(f"Subscribed to federation Redis channel {resolved_channel}") try: async for message in pubsub.listen(): if stop.is_set(): @@ -187,12 +199,3 @@ async def run_federation_subscriber( finally: await pubsub.unsubscribe(resolved_channel) await client.aclose() - - -def build_peer_http_client() -> httpx.AsyncClient: - return httpx.AsyncClient(timeout=30.0) - - -def build_gateway_http_client() -> httpx.AsyncClient: - """Deprecated alias for peer webhook HTTP client.""" - return build_peer_http_client() diff --git a/federation/sds_federation/services/peer_http.py b/federation/sds_federation/services/peer_http.py new file mode 100644 index 000000000..f399c068c --- /dev/null +++ b/federation/sds_federation/services/peer_http.py @@ -0,0 +1,46 @@ +"""HTTP helpers for peer sync and gateway export calls (optional mTLS verify).""" + +from __future__ import annotations + +from typing import Any + +import httpx + +DEFAULT_PEER_HTTP_TIMEOUT = 30.0 + + +def build_peer_http_client( + *, + timeout: float = DEFAULT_PEER_HTTP_TIMEOUT, +) -> httpx.AsyncClient: + """Default async client for peer webhooks and gateway export (no custom CA).""" + return httpx.AsyncClient(timeout=timeout) + + +async def peer_request( + http: httpx.AsyncClient, + method: str, + url: str, + *, + ca_cert_path: str = "", + headers: dict[str, str] | None = None, + json: Any | None = None, +) -> httpx.Response: + """GET/POST against a peer URL, optionally verifying with ``ca_cert_path``. + + When ``ca_cert_path`` is set, opens a short-lived client with that CA bundle + so the shared ``http`` client can stay verify=True for non-mTLS lab peers. + """ + kwargs: dict[str, Any] = {} + if headers is not None: + kwargs["headers"] = headers + if json is not None: + kwargs["json"] = json + + if ca_cert_path: + async with httpx.AsyncClient( + verify=ca_cert_path, + timeout=http.timeout, + ) as tls_client: + return await tls_client.request(method, url, **kwargs) + return await http.request(method, url, **kwargs) diff --git a/federation/sds_federation/services/peer_sync.py b/federation/sds_federation/services/peer_sync.py index 940b22517..a62f60b1b 100644 --- a/federation/sds_federation/services/peer_sync.py +++ b/federation/sds_federation/services/peer_sync.py @@ -5,6 +5,8 @@ import httpx from loguru import logger +from sds_federation.services.peer_http import peer_request + if TYPE_CHECKING: from sds_federation.models import FederationConfig from sds_federation.models import PeerInfo @@ -21,10 +23,14 @@ def peer_for_outbound( peer: PeerInfo, registry: PeerRegistry | None, ) -> PeerInfo: - """Return peer with sync_service_url overlaid from site-hello when present.""" + """Return peer with sync_service_url overlaid from site-hello when present. + + ``site-hello`` registers under RFC ``site_name`` (FQDN). Look up by fqdn first, + then short ``peer.name``, so toml name/fqdn mismatches still resolve. + """ if registry is None: return peer - hello = registry.get(peer.name) + hello = registry.get(peer.fqdn) or registry.get(peer.name) if hello is None: return peer return peer.model_copy(update={"sync_service_url": hello.sync_service_url}) @@ -42,14 +48,13 @@ async def push_asset_updated_to_peers( outbound = peer_for_outbound(peer, registry) url = peer_webhook_url(outbound, path) try: - if peer.ca_cert_path: - async with httpx.AsyncClient( - verify=peer.ca_cert_path, - timeout=http.timeout, - ) as tls_client: - resp = await tls_client.post(url, json=body) - else: - resp = await http.post(url, json=body) + resp = await peer_request( + http, + "POST", + url, + ca_cert_path=peer.ca_cert_path, + json=body, + ) resp.raise_for_status() except httpx.HTTPError as exc: logger.error("webhook to {} failed: {}", peer.name, exc) diff --git a/federation/sds_federation/services/redis_channel.py b/federation/sds_federation/services/redis_channel.py index 3e39d71ef..5814419d0 100644 --- a/federation/sds_federation/services/redis_channel.py +++ b/federation/sds_federation/services/redis_channel.py @@ -1,35 +1,16 @@ -"""Redis pub/sub channel naming for federation local change events.""" - -from __future__ import annotations - -FEDERATION_EVENTS_CHANNEL_PREFIX = "federation:events" +"""Redis pub/sub channel naming for federation local change events. +Re-exports the shared helpers from ``sds_opensearch_query``. +""" -def federation_events_channel(site_name: str) -> str: - """Site-scoped channel.""" - name = site_name.strip() - if not name: - msg = "site_name is required to build a federation events channel" - raise ValueError(msg) - return f"{FEDERATION_EVENTS_CHANNEL_PREFIX}:{name}" - +from __future__ import annotations -def resolve_federation_events_channel( - *, - site_name: str, - env_override: str | None = None, - gateway_site_name: str | None = None, -) -> str: - """Channel for subscribe/publish. +from sds_opensearch_query.redis_channel import FEDERATION_EVENTS_CHANNEL_PREFIX +from sds_opensearch_query.redis_channel import federation_events_channel +from sds_opensearch_query.redis_channel import resolve_federation_events_channel - Precedence: - 1. ``FEDERATION_EVENTS_CHANNEL`` (``env_override``) when set - 2. ``FEDERATION_SITE_NAME`` (``gateway_site_name``) when set — matches gateway - 3. ``site_name`` from federation.toml - """ - if env_override is not None and env_override.strip(): - return env_override.strip() - gateway = (gateway_site_name or "").strip() - if gateway: - return federation_events_channel(gateway) - return federation_events_channel(site_name) +__all__ = [ + "FEDERATION_EVENTS_CHANNEL_PREFIX", + "federation_events_channel", + "resolve_federation_events_channel", +] diff --git a/federation/sds_federation/testing/sample_data.py b/federation/sds_federation/testing/sample_data.py index d05a515b8..4d51ca792 100644 --- a/federation/sds_federation/testing/sample_data.py +++ b/federation/sds_federation/testing/sample_data.py @@ -31,7 +31,7 @@ def simulated_dataset_redis_payload( def sample_federated_dataset_doc( *, uuid: UUID = TEST_DATASET_UUID, - site_name: str = "testsite", + site_name: str = "localhost", ) -> FederatedDatasetDoc: return FederatedDatasetDoc( uuid=uuid, @@ -47,7 +47,7 @@ def sample_federated_dataset_doc( def sample_federated_capture_doc( *, uuid: UUID = TEST_CAPTURE_UUID, - site_name: str = "testsite", + site_name: str = "localhost", ) -> FederatedCaptureDoc: return FederatedCaptureDoc( uuid=uuid, diff --git a/federation/tests/conftest.py b/federation/tests/conftest.py index 035e82f39..0969b1c1d 100644 --- a/federation/tests/conftest.py +++ b/federation/tests/conftest.py @@ -8,6 +8,7 @@ import pytest from fastapi import FastAPI from sds_federation.models import FederationConfig +from sds_federation.models import site_name_for_federation from sds_federation.routes.webhooks import webhooks_router from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.services.fed_index import FederatedAssetIndexer @@ -97,7 +98,7 @@ def stub_dataset_resolver(test_site_config: FederationConfig): def _seed(recording_opensearch: RecordingOpenSearch) -> None: seed_federated_dataset_in_opensearch( recording_opensearch, - test_site_config.site.name, + site_name_for_federation(test_site_config.site), ) return _seed diff --git a/federation/tests/support/federation_mesh.py b/federation/tests/support/federation_mesh.py index f8da5741e..49b19fe83 100644 --- a/federation/tests/support/federation_mesh.py +++ b/federation/tests/support/federation_mesh.py @@ -20,6 +20,9 @@ API_PREFIX = "/api/v1" +TESTSITE_FQDN = "localhost" +PEER_ONE_FQDN = "peer.test" + TESTSITE_SYNC_ORIGIN = "http://testsite.test" PEER_ONE_SYNC_ORIGIN = "http://peer-one.test" TESTSITE_GATEWAY_HOST = "testsite-gateway.test" diff --git a/federation/tests/test_integration_bootstrap.py b/federation/tests/test_integration_bootstrap.py index de3c5bf0c..991f191d4 100644 --- a/federation/tests/test_integration_bootstrap.py +++ b/federation/tests/test_integration_bootstrap.py @@ -31,7 +31,7 @@ async def test_bootstrap_gateway_exports_indexes_documents( recording_opensearch: RecordingOpenSearch, ) -> None: - doc = sample_federated_dataset_doc(site_name="remote") + doc = sample_federated_dataset_doc(site_name="remote.test") export_body = json.dumps([doc.model_dump(mode="json")]) def handler(request: httpx.Request) -> httpx.Response: @@ -62,7 +62,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert count == 1 assert len(recording_opensearch.index_calls) == 1 assert recording_opensearch.index_calls[0]["id"] == doc_id( - "remote", + "remote.test", TEST_DATASET_UUID, ) @@ -106,7 +106,7 @@ async def test_push_site_hello_to_in_process_peer( result = await push_site_hello_to_peer(http, peer, caller_config) assert result["status"] == "registered" - assert app.state.peer_registry.get("testsite") is not None + assert app.state.peer_registry.get("localhost") is not None @pytest.mark.integration @@ -115,8 +115,8 @@ async def test_run_bootstrap_pulls_exports_then_registers( recording_opensearch: RecordingOpenSearch, monkeypatch: pytest.MonkeyPatch, ) -> None: - local_doc = sample_federated_dataset_doc(site_name="testsite") - peer_doc = sample_federated_dataset_doc(site_name="peer-one") + local_doc = sample_federated_dataset_doc(site_name="localhost") + peer_doc = sample_federated_dataset_doc(site_name="peer.test") hello_posts: list[str] = [] def handler(request: httpx.Request) -> httpx.Response: # noqa: PLR0911 @@ -186,7 +186,7 @@ def handler(request: httpx.Request) -> httpx.Response: # noqa: PLR0911 assert len(recording_opensearch.index_calls) == 2 indexed_ids = {call["id"] for call in recording_opensearch.index_calls} assert indexed_ids == { - doc_id("testsite", TEST_DATASET_UUID), - doc_id("peer-one", TEST_DATASET_UUID), + doc_id("localhost", TEST_DATASET_UUID), + doc_id("peer.test", TEST_DATASET_UUID), } assert len(hello_posts) == 1 diff --git a/federation/tests/test_integration_mesh.py b/federation/tests/test_integration_mesh.py index 182f289a8..64d86421b 100644 --- a/federation/tests/test_integration_mesh.py +++ b/federation/tests/test_integration_mesh.py @@ -20,6 +20,9 @@ from sds_federation.testing.sample_data import sample_federated_dataset_doc from sds_federation.testing.sample_data import simulated_dataset_redis_payload +from tests.support.federation_mesh import PEER_ONE_FQDN +from tests.support.federation_mesh import TESTSITE_FQDN + if TYPE_CHECKING: from tests.support.federation_mesh import FederationMesh @@ -49,13 +52,13 @@ async def test_mesh_dispatches_redis_event_to_peer_opensearch( peer = mesh.site("peer-one") assert len(peer.opensearch.index_calls) == 1 assert peer.opensearch.index_calls[0]["id"] == doc_id( - "testsite", + TESTSITE_FQDN, TEST_DATASET_UUID, ) assert len(mesh.recorded_webhooks) == 1 body = json.loads(mesh.recorded_webhooks[0].content.decode()) webhook = AssetUpdatedWebhook.model_validate(body) - assert webhook.site_name == "testsite" + assert webhook.site_name == TESTSITE_FQDN assert webhook.asset is not None @@ -73,7 +76,7 @@ async def test_mesh_site_hello_registers_on_peer_registry( caller.config, ) assert result["status"] == "registered" - assert peer.registry.get("testsite") is not None + assert peer.registry.get(TESTSITE_FQDN) is not None assert len(peer.registry.registration_events) == 1 @@ -83,12 +86,12 @@ async def test_mesh_site_hello_backfills_caller_exports( two_site_mesh: FederationMesh, ) -> None: mesh = two_site_mesh - doc = sample_federated_dataset_doc(site_name="testsite") + doc = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) caller = mesh.site("testsite") peer = mesh.site("peer-one") FederatedAssetIndexer(caller.opensearch).apply_asset_event( event_at=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), - site_name="testsite", + site_name=TESTSITE_FQDN, asset=doc, asset_type=AssetTypeEnum.DATASET, ) @@ -100,9 +103,9 @@ async def test_mesh_site_hello_backfills_caller_exports( ) assert result["status"] == "registered" - assert peer.registry.get("testsite") is not None + assert peer.registry.get(TESTSITE_FQDN) is not None assert any( - call["id"] == doc_id("testsite", TEST_DATASET_UUID) + call["id"] == doc_id(TESTSITE_FQDN, TEST_DATASET_UUID) for call in peer.opensearch.index_calls ) @@ -115,8 +118,8 @@ async def test_mesh_register_with_peers_both_directions( mesh = two_site_mesh await register_with_peers(mesh.http, mesh.site("testsite").config) await register_with_peers(mesh.http, mesh.site("peer-one").config) - assert mesh.site("peer-one").registry.get("testsite") is not None - assert mesh.site("testsite").registry.get("peer-one") is not None + assert mesh.site("peer-one").registry.get(TESTSITE_FQDN) is not None + assert mesh.site("testsite").registry.get(PEER_ONE_FQDN) is not None @pytest.mark.integration @@ -125,11 +128,11 @@ async def test_mesh_bootstrap_pulls_peer_sync_list( two_site_mesh: FederationMesh, ) -> None: mesh = two_site_mesh - doc = sample_federated_dataset_doc(site_name="testsite") + doc = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) caller = mesh.site("testsite") FederatedAssetIndexer(caller.opensearch).apply_asset_event( event_at=datetime(2026, 6, 11, 11, 0, 0, tzinfo=UTC), - site_name="testsite", + site_name=TESTSITE_FQDN, asset=doc, asset_type=AssetTypeEnum.DATASET, ) @@ -144,6 +147,6 @@ async def test_mesh_bootstrap_pulls_peer_sync_list( ) assert count == 1 assert peer.opensearch.index_calls[-1]["id"] == doc_id( - "testsite", + TESTSITE_FQDN, TEST_DATASET_UUID, ) diff --git a/federation/tests/test_integration_pipeline.py b/federation/tests/test_integration_pipeline.py index c229b4c56..22ab3b317 100644 --- a/federation/tests/test_integration_pipeline.py +++ b/federation/tests/test_integration_pipeline.py @@ -13,6 +13,7 @@ from sds_federation.testing.sample_data import simulated_dataset_redis_payload from tests.conftest import PEER_SYNC_BASE +from tests.support.federation_mesh import TESTSITE_FQDN from tests.support.mock_opensearch import RecordingOpenSearch @@ -45,7 +46,7 @@ async def test_redis_simulation_end_to_end_indexes_on_peer( assert len(local_opensearch.index_calls) == local_calls_before assert len(recording_opensearch.index_calls) == 1 assert recording_opensearch.index_calls[0]["id"] == doc_id( - "testsite", + TESTSITE_FQDN, TEST_DATASET_UUID, ) assert ( diff --git a/federation/tests/test_integration_webhooks.py b/federation/tests/test_integration_webhooks.py index a1599e81a..177fc39ff 100644 --- a/federation/tests/test_integration_webhooks.py +++ b/federation/tests/test_integration_webhooks.py @@ -22,8 +22,10 @@ if TYPE_CHECKING: from tests.support.mock_opensearch import RecordingOpenSearch +TESTSITE_FQDN = "localhost" -def _dataset_webhook_payload(*, site_name: str = "testsite") -> dict: + +def _dataset_webhook_payload(*, site_name: str = TESTSITE_FQDN) -> dict: asset = sample_federated_dataset_doc(site_name=site_name) webhook = AssetUpdatedWebhook( timestamp=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), @@ -48,7 +50,7 @@ async def test_dataset_webhook_indexes_via_http( ) as client: response = await client.post( f"{SYNC_API_PREFIX}/webhook/dataset-updated", - json=_dataset_webhook_payload(site_name="testsite"), + json=_dataset_webhook_payload(), ) assert response.status_code == 200 @@ -58,7 +60,7 @@ async def test_dataset_webhook_indexes_via_http( recording_opensearch.index_calls[0]["index"] == AssetTypeEnum.DATASET.index_name ) assert recording_opensearch.index_calls[0]["id"] == doc_id( - "testsite", + TESTSITE_FQDN, TEST_DATASET_UUID, ) @@ -112,7 +114,7 @@ async def test_webhook_rejects_site_name_mismatch_on_asset( ) -> None: config = make_peer_config() app = build_webhook_app(config, FederatedAssetIndexer(recording_opensearch)) - body = _dataset_webhook_payload(site_name="testsite") + body = _dataset_webhook_payload() body["asset"]["site_name"] = "mismatch" async with httpx.AsyncClient( @@ -136,8 +138,8 @@ async def test_site_hello_registers_known_peer( config = make_peer_config() app = build_webhook_app(config, FederatedAssetIndexer(recording_opensearch)) body = { - "site_name": "testsite", - "fqdn": "localhost", + "site_name": TESTSITE_FQDN, + "fqdn": TESTSITE_FQDN, "display_name": "Originating test site", "sync_service_url": "http://testsite.test/sync", } @@ -149,8 +151,8 @@ async def test_site_hello_registers_known_peer( response = await client.post(f"{SYNC_API_PREFIX}/webhook/site-hello", json=body) assert response.status_code == 200 - assert response.json() == {"status": "registered", "site_name": "testsite"} - assert app.state.peer_registry.get("testsite") is not None + assert response.json() == {"status": "registered", "site_name": TESTSITE_FQDN} + assert app.state.peer_registry.get(TESTSITE_FQDN) is not None @pytest.mark.integration @@ -159,7 +161,7 @@ async def test_site_hello_backfills_peer_exports( recording_opensearch: RecordingOpenSearch, ) -> None: config = make_peer_config() - doc = sample_federated_dataset_doc(site_name="testsite") + doc = sample_federated_dataset_doc(site_name="localhost") export_hits = {"datasets": 0, "captures": 0} # Intentionally differs from federation.toml peer sync URL. hello_sync_url = "http://dynamic-testsite.test/sync" @@ -194,7 +196,7 @@ def handler(request: httpx.Request) -> httpx.Response: response = await client.post( f"{SYNC_API_PREFIX}/webhook/site-hello", json={ - "site_name": "testsite", + "site_name": "localhost", "fqdn": "localhost", "display_name": "Originating test site", "sync_service_url": hello_sync_url, @@ -205,7 +207,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert export_hits == {"datasets": 1, "captures": 1} assert len(recording_opensearch.index_calls) == 1 assert recording_opensearch.index_calls[0]["id"] == doc_id( - "testsite", + "localhost", TEST_DATASET_UUID, ) @@ -240,7 +242,7 @@ async def test_site_hello_rejects_self_registration( config = make_peer_config() app = build_webhook_app(config, FederatedAssetIndexer(recording_opensearch)) body = { - "site_name": "peer-one", + "site_name": "peer.test", "fqdn": "peer.test", "sync_service_url": "http://peer-one.test/sync", } diff --git a/federation/tests/test_redis_event_pipeline.py b/federation/tests/test_redis_event_pipeline.py index 6c0a64e17..7fba7d8a1 100644 --- a/federation/tests/test_redis_event_pipeline.py +++ b/federation/tests/test_redis_event_pipeline.py @@ -20,6 +20,7 @@ from sds_federation.testing.sample_data import simulated_dataset_redis_payload from tests.conftest import seed_federated_dataset_in_opensearch +from tests.support.federation_mesh import TESTSITE_FQDN @pytest.mark.asyncio @@ -57,7 +58,7 @@ async def test_dispatch_reads_opensearch_and_posts_webhook_to_peer( recorded, transport = peer_webhook_recorder seed_federated_dataset_in_opensearch( recording_opensearch, - test_site_config.site.name, + TESTSITE_FQDN, ) calls_before = len(recording_opensearch.index_calls) indexer = FederatedAssetIndexer(recording_opensearch) @@ -82,7 +83,7 @@ async def test_dispatch_reads_opensearch_and_posts_webhook_to_peer( ) assert str(req.url) == expected_url body = json.loads(req.content.decode()) - assert body["site_name"] == "testsite" + assert body["site_name"] == TESTSITE_FQDN assert body["asset_type"] == "dataset" assert body["asset"]["uuid"] == str(TEST_DATASET_UUID) assert body["asset"]["name"] == "Simulated public dataset" @@ -97,11 +98,11 @@ async def test_dispatch_deleted_doc_from_opensearch( ) -> None: recorded, transport = peer_webhook_recorder doc = sample_federated_dataset_doc( - site_name=test_site_config.site.name, + site_name=TESTSITE_FQDN, ).model_copy(update={"is_deleted": True}) FederatedAssetIndexer(recording_opensearch).apply_asset_event( event_at=datetime.now(UTC), - site_name=test_site_config.site.name, + site_name=TESTSITE_FQDN, asset=doc, asset_type=AssetTypeEnum.DATASET, ) @@ -161,7 +162,7 @@ def handler(request: httpx.Request) -> httpx.Response: seed_federated_dataset_in_opensearch( recording_opensearch, - test_site_config.site.name, + TESTSITE_FQDN, ) peer = test_site_config.peers[0] registry = PeerRegistry() diff --git a/federation/tests/test_regression_common_shared.py b/federation/tests/test_regression_common_shared.py new file mode 100644 index 000000000..2fad017c5 --- /dev/null +++ b/federation/tests/test_regression_common_shared.py @@ -0,0 +1,104 @@ +"""Unit tests for shared federation Redis channel and index write helpers.""" + +from __future__ import annotations + +from datetime import UTC +from datetime import datetime +from uuid import UUID + +import pytest +from sds_opensearch_query.index_write import FED_DATASETS_INDEX +from sds_opensearch_query.index_write import federated_doc_id +from sds_opensearch_query.index_write import index_federated_document +from sds_opensearch_query.redis_channel import federation_events_channel +from sds_opensearch_query.redis_channel import resolve_federation_events_channel + + +def test_federation_events_channel_uses_site_name() -> None: + assert federation_events_channel("crc") == "federation:events:crc" + + +def test_federation_events_channel_rejects_blank_site() -> None: + with pytest.raises(ValueError, match="site_name"): + federation_events_channel(" ") + + +def test_resolve_prefers_channel_override() -> None: + assert ( + resolve_federation_events_channel( + site_name="crc", + channel_override="custom:channel", + gateway_site_name="other", + ) + == "custom:channel" + ) + + +def test_resolve_env_override_alias() -> None: + assert ( + resolve_federation_events_channel( + site_name="crc", + env_override="custom:channel", + ) + == "custom:channel" + ) + + +def test_resolve_prefers_gateway_site_name_over_toml() -> None: + assert ( + resolve_federation_events_channel( + site_name="toml-name", + gateway_site_name="gateway-name", + ) + == "federation:events:gateway-name" + ) + + +def test_resolve_derives_from_site_when_no_override() -> None: + assert ( + resolve_federation_events_channel(site_name="haystack") + == "federation:events:haystack" + ) + + +def test_resolve_empty_when_unconfigured() -> None: + assert resolve_federation_events_channel() == "" + + +def test_federated_doc_id_format() -> None: + uid = UUID("12345678-1234-5678-1234-567812345678") + assert federated_doc_id("sds.crc.nd.edu", uid) == ( + "sds.crc.nd.edu:12345678-1234-5678-1234-567812345678" + ) + + +class _RecordingClient: + def __init__(self) -> None: + self.calls: list[dict] = [] + + def index(self, **kwargs) -> None: + self.calls.append(kwargs) + + +def test_index_federated_document_stamps_event_at() -> None: + client = _RecordingClient() + uid = UUID("12345678-1234-5678-1234-567812345678") + event_at = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) + + doc_id = index_federated_document( + client, # type: ignore[arg-type] + index_name=FED_DATASETS_INDEX, + site_name="sds.crc.nd.edu", + uuid=uid, + body={"name": "demo", "site_name": "sds.crc.nd.edu"}, + event_at=event_at, + ) + + assert doc_id == federated_doc_id("sds.crc.nd.edu", uid) + assert len(client.calls) == 1 + call = client.calls[0] + assert call["index"] == FED_DATASETS_INDEX + assert call["id"] == doc_id + assert call["body"]["federation_event_at"] == event_at.isoformat() + assert call["body"]["name"] == "demo" + assert call["refresh"] == "wait_for" diff --git a/federation/tests/test_regression_fed_indices.py b/federation/tests/test_regression_fed_indices.py new file mode 100644 index 000000000..6ab2e2fb1 --- /dev/null +++ b/federation/tests/test_regression_fed_indices.py @@ -0,0 +1,39 @@ +"""Regression: ensure_fed_indices creates both fed-* indices.""" + +from __future__ import annotations + +from typing import Any + +import pytest +from sds_federation.schemas.webhooks import AssetTypeEnum +from sds_federation.services.fed_index import ensure_fed_indices + + +class _IndicesStub: + def __init__(self) -> None: + self.existing: set[str] = set() + self.create_calls: list[dict[str, Any]] = [] + + def exists(self, *, index: str, **kwargs: Any) -> bool: + return index in self.existing + + def create(self, *, index: str, body: dict[str, Any], **kwargs: Any) -> dict: + self.create_calls.append({"index": index, "body": body}) + self.existing.add(index) + return {"acknowledged": True} + + +class _ClientStub: + def __init__(self) -> None: + self.indices = _IndicesStub() + + +@pytest.mark.regression +def test_ensure_fed_indices_creates_missing_indices() -> None: + client = _ClientStub() + ensure_fed_indices(client) # type: ignore[arg-type] + names = {c["index"] for c in client.indices.create_calls} + assert names == { + AssetTypeEnum.DATASET.index_name, + AssetTypeEnum.CAPTURE.index_name, + } diff --git a/federation/tests/test_regression_fed_list.py b/federation/tests/test_regression_fed_list.py index 0fc52f74e..7d08603ac 100644 --- a/federation/tests/test_regression_fed_list.py +++ b/federation/tests/test_regression_fed_list.py @@ -12,9 +12,9 @@ from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.services.bootstrap import _index_export_docs from sds_federation.services.bootstrap import _parse_doc_event_at +from sds_federation.services.fed_index import _LIST_PAGE_SIZE from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_search import _LIST_PAGE_SIZE -from sds_federation.services.fed_search import list_federated_assets_for_site +from sds_federation.services.fed_index import list_federated_assets_for_site from sds_federation.testing.sample_data import sample_federated_dataset_doc from tests.support.federation_mesh import peer_one_config @@ -64,10 +64,10 @@ def test_bootstrap_index_uses_doc_updated_at_not_shared_now( ) -> None: updated = datetime(2026, 6, 1, 12, 0, 0, tzinfo=UTC) shared_now = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) - doc = sample_federated_dataset_doc(site_name="testsite").model_copy( + doc = sample_federated_dataset_doc(site_name="localhost").model_copy( update={"updated_at": updated.isoformat()}, ) - peer = peer_one_config().peers[0].model_copy(update={"name": "testsite"}) + peer = peer_one_config().peers[0] indexer = FederatedAssetIndexer(recording_opensearch) count = _index_export_docs( @@ -88,7 +88,7 @@ def test_bootstrap_index_uses_doc_updated_at_not_shared_now( assert ( indexer.apply_asset_event( event_at=updated + timedelta(seconds=1), - site_name="testsite", + site_name="localhost", asset=doc.model_copy(update={"name": "newer"}), asset_type=AssetTypeEnum.DATASET, ) diff --git a/federation/tests/test_regression_fed_search.py b/federation/tests/test_regression_fed_search.py index 7ab67330c..a5f1f5305 100644 --- a/federation/tests/test_regression_fed_search.py +++ b/federation/tests/test_regression_fed_search.py @@ -1,4 +1,4 @@ -"""Regression tests for fed_search OpenSearch reads.""" +"""Regression tests for fed_* OpenSearch document reads.""" from __future__ import annotations @@ -7,7 +7,7 @@ from sds_federation.schemas.webhooks import AssetTypeEnum from sds_federation.services.fed_index import FederatedAssetIndexer -from sds_federation.services.fed_search import load_federated_asset +from sds_federation.services.fed_index import load_federated_asset from sds_federation.testing.sample_data import TEST_DATASET_UUID from sds_federation.testing.sample_data import sample_federated_dataset_doc @@ -16,7 +16,7 @@ def test_load_federated_asset_returns_indexed_doc() -> None: opensearch = RecordingOpenSearch() - site = "testsite" + site = "localhost" doc = sample_federated_dataset_doc(site_name=site) FederatedAssetIndexer(opensearch).apply_asset_event( event_at=datetime.now(UTC), @@ -42,7 +42,7 @@ def test_load_federated_asset_missing_returns_none() -> None: assert ( load_federated_asset( opensearch, - site_name="testsite", + site_name="localhost", uuid=TEST_DATASET_UUID, asset_type=AssetTypeEnum.DATASET, ) diff --git a/federation/tests/test_regression_indexer.py b/federation/tests/test_regression_indexer.py index 08435ac11..1c6fe9f9b 100644 --- a/federation/tests/test_regression_indexer.py +++ b/federation/tests/test_regression_indexer.py @@ -14,13 +14,17 @@ from sds_federation.testing.sample_data import TEST_DATASET_UUID from sds_federation.testing.sample_data import sample_federated_dataset_doc +from tests.support.federation_mesh import TESTSITE_FQDN + if TYPE_CHECKING: from tests.support.mock_opensearch import RecordingOpenSearch @pytest.mark.regression def test_doc_id_format() -> None: - assert doc_id("crc", TEST_DATASET_UUID) == f"crc:{TEST_DATASET_UUID}" + assert doc_id("sds.crc.nd.edu", TEST_DATASET_UUID) == ( + f"sds.crc.nd.edu:{TEST_DATASET_UUID}" + ) @pytest.mark.regression @@ -29,11 +33,11 @@ def test_indexer_writes_dataset_document( ) -> None: indexer = FederatedAssetIndexer(recording_opensearch) event_at = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) - asset = sample_federated_dataset_doc(site_name="testsite") + asset = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) indexer.apply_asset_event( event_at=event_at, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -41,7 +45,7 @@ def test_indexer_writes_dataset_document( assert len(recording_opensearch.index_calls) == 1 call = recording_opensearch.index_calls[0] assert call["index"] == AssetTypeEnum.DATASET.index_name - assert call["id"] == doc_id("testsite", TEST_DATASET_UUID) + assert call["id"] == doc_id(TESTSITE_FQDN, TEST_DATASET_UUID) assert call["body"]["federation_event_at"] == event_at.isoformat() assert call["body"]["name"] == "Simulated public dataset" assert call["body"]["is_deleted"] is False @@ -53,13 +57,13 @@ def test_indexer_writes_deleted_flag_from_asset_body( ) -> None: indexer = FederatedAssetIndexer(recording_opensearch) event_at = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) - asset = sample_federated_dataset_doc(site_name="testsite").model_copy( + asset = sample_federated_dataset_doc(site_name=TESTSITE_FQDN).model_copy( update={"is_deleted": True}, ) indexer.apply_asset_event( event_at=event_at, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -72,19 +76,19 @@ def test_indexer_writes_deleted_flag_from_asset_body( @pytest.mark.regression def test_indexer_skips_stale_events(recording_opensearch: RecordingOpenSearch) -> None: indexer = FederatedAssetIndexer(recording_opensearch) - asset = sample_federated_dataset_doc(site_name="testsite") + asset = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) t1 = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) t0 = t1 - timedelta(seconds=1) indexer.apply_asset_event( event_at=t1, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) indexer.apply_asset_event( event_at=t0, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -96,20 +100,20 @@ def test_indexer_skips_stale_events(recording_opensearch: RecordingOpenSearch) - def test_indexer_skips_stale_events_after_restart( recording_opensearch: RecordingOpenSearch, ) -> None: - asset = sample_federated_dataset_doc(site_name="testsite") + asset = sample_federated_dataset_doc(site_name=TESTSITE_FQDN) t1 = datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) t0 = t1 - timedelta(seconds=1) FederatedAssetIndexer(recording_opensearch).apply_asset_event( event_at=t1, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) # New process: empty in-memory map; persisted federation_event_at must win. FederatedAssetIndexer(recording_opensearch).apply_asset_event( event_at=t0, - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -127,7 +131,7 @@ def test_indexer_rejects_site_name_mismatch( with pytest.raises(ValueError, match="site_name must match"): indexer.apply_asset_event( event_at=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), - site_name="testsite", + site_name=TESTSITE_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) diff --git a/federation/tests/test_regression_opensearch_indices.py b/federation/tests/test_regression_opensearch_indices.py new file mode 100644 index 000000000..458d2cb6e --- /dev/null +++ b/federation/tests/test_regression_opensearch_indices.py @@ -0,0 +1,61 @@ +"""Regression: OpenSearch fed-* mappings include identity fields plus +local capture ``capture_props`` / ``search_props`` (dynamic nested). +""" + +from __future__ import annotations + +import pytest +from sds_federation.schemas.opensearch_indices import RFC_FED_CAPTURE_PROPERTIES +from sds_federation.schemas.opensearch_indices import RFC_FED_DATASET_PROPERTIES +from sds_federation.schemas.opensearch_indices import fed_capture_mappings +from sds_federation.schemas.opensearch_indices import fed_dataset_mappings + +RFC_DATASET_FIELDS = frozenset( + { + "uuid", + "site_name", + "name", + "description", + "abstract", + "keywords", + "owner_name", + "created_at", + "updated_at", + "size", + "capture_count", + "url", + }, +) + +RFC_CAPTURE_FIELDS = frozenset( + { + "uuid", + "site_name", + "capture_type", + "channel", + "capture_props", + "search_props", + "public_dataset_ids", + "url", + }, +) + + +@pytest.mark.regression +def test_rfc_dataset_fields_explicit_in_mapping() -> None: + assert set(RFC_FED_DATASET_PROPERTIES) == RFC_DATASET_FIELDS + props = fed_dataset_mappings()["properties"] + assert set(props) == RFC_DATASET_FIELDS + + +@pytest.mark.regression +def test_rfc_capture_fields_explicit_in_mapping() -> None: + assert set(RFC_FED_CAPTURE_PROPERTIES) == RFC_CAPTURE_FIELDS + props = fed_capture_mappings()["properties"] + assert set(props) == RFC_CAPTURE_FIELDS + + +@pytest.mark.regression +def test_fed_mappings_allow_dynamic_extra_fields() -> None: + assert fed_dataset_mappings()["dynamic"] is True + assert fed_capture_mappings()["dynamic"] is True diff --git a/federation/tests/test_regression_opensearch_query.py b/federation/tests/test_regression_opensearch_query.py new file mode 100644 index 000000000..4f58cd4f1 --- /dev/null +++ b/federation/tests/test_regression_opensearch_query.py @@ -0,0 +1,76 @@ +"""Regression tests for shared sds_opensearch_query package.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest +from sds_opensearch_query import bool_must_search_body +from sds_opensearch_query import build_metadata_filter_clauses +from sds_opensearch_query import nested_query_clause +from sds_opensearch_query.client import build_opensearch_client + +pytest.importorskip("sds_opensearch_query") + + +@pytest.mark.regression +def test_nested_query_clause_single_level() -> None: + clause = nested_query_clause( + field_path="center_frequency", + query_type="term", + value=1.0, + levels_nested=0, + last_path="search_props", + ) + assert clause == {"term": {"search_props.center_frequency": 1.0}} + + +@pytest.mark.regression +def test_build_metadata_filter_clauses_term_and_nested() -> None: + clauses = build_metadata_filter_clauses( + [ + { + "field_path": "capture_type", + "query_type": "term", + "filter_value": "digital-rf", + }, + { + "field_path": "search_props.center_frequency", + "query_type": "range", + "filter_value": {"gte": 100}, + }, + ], + ) + assert len(clauses) == 2 + assert clauses[0] == {"term": {"capture_type": "digital-rf"}} + assert "nested" in clauses[1] + + +@pytest.mark.regression +def test_bool_must_search_body() -> None: + body = bool_must_search_body({"term": {"site_name": "crc"}}) + assert body["query"]["bool"]["must"] == [{"term": {"site_name": "crc"}}] + + +@pytest.mark.regression +def test_build_opensearch_client_omits_http_auth_when_user_blank() -> None: + with patch("sds_opensearch_query.client.OpenSearch") as mock_os: + build_opensearch_client(host="opensearch", port=9200, user="", password="") + kwargs = mock_os.call_args.kwargs + assert "http_auth" not in kwargs + + +@pytest.mark.regression +def test_build_opensearch_client_sets_http_auth_when_user_set() -> None: + user = "admin" + cred = "test-only-credential" + with patch("sds_opensearch_query.client.OpenSearch") as mock_os: + build_opensearch_client( + host="opensearch", + port=9200, + user=user, + password=cred, + ) + auth = mock_os.call_args.kwargs["http_auth"] + assert auth.username == user + assert auth.password == cred diff --git a/federation/tests/test_regression_peer_list_retry.py b/federation/tests/test_regression_peer_list_retry.py new file mode 100644 index 000000000..021b0c3eb --- /dev/null +++ b/federation/tests/test_regression_peer_list_retry.py @@ -0,0 +1,91 @@ +"""Regression: peer sync list fetch retries transient connection failures.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock +from unittest.mock import patch + +import httpx +import pytest +from sds_federation.models import PeerInfo +from sds_federation.schemas.webhooks import AssetTypeEnum +from sds_federation.services.bootstrap import _is_retryable_peer_list_error +from sds_federation.services.bootstrap import fetch_peer_sync_list +from sds_federation.testing.sample_data import sample_federated_dataset_doc + + +def _peer() -> PeerInfo: + return PeerInfo( + name="peer", + fqdn="peer.local", + display_name="Peer", + gateway_api_base="http://unused:8000/api/v1", + sync_service_url="http://sds-federation-peer-sync:8000/sync", + ) + + +def test_connect_error_is_retryable() -> None: + assert _is_retryable_peer_list_error(httpx.ConnectError("boom")) is True + assert _is_retryable_peer_list_error(httpx.ReadTimeout("slow")) is True + req = httpx.Request("GET", "http://example/sync/api/v1/webhook/list-datasets/") + resp = httpx.Response(503, request=req) + assert _is_retryable_peer_list_error( + httpx.HTTPStatusError("x", request=req, response=resp) + ) + resp_400 = httpx.Response(400, request=req) + assert ( + _is_retryable_peer_list_error( + httpx.HTTPStatusError("x", request=req, response=resp_400), + ) + is False + ) + + +@pytest.mark.asyncio +async def test_fetch_peer_sync_list_retries_then_succeeds() -> None: + doc = sample_federated_dataset_doc(site_name="peer.local") + calls = {"n": 0} + + async def flaky_get_json(*_args, **_kwargs): + calls["n"] += 1 + if calls["n"] < 3: + raise httpx.ConnectError("All connection attempts failed") + return [doc.model_dump(mode="json")] + + with ( + patch( + "sds_federation.services.bootstrap._get_json", + new=AsyncMock(side_effect=flaky_get_json), + ), + patch("sds_federation.services.bootstrap.asyncio.sleep", new=AsyncMock()), + ): + docs = await fetch_peer_sync_list( + httpx.AsyncClient(), + _peer(), + AssetTypeEnum.DATASET, + attempts=5, + backoff_secs=0.01, + ) + + assert calls["n"] == 3 + assert len(docs) == 1 + assert docs[0].site_name == "peer.local" + + +@pytest.mark.asyncio +async def test_fetch_peer_sync_list_exhausts_retries() -> None: + with ( + patch( + "sds_federation.services.bootstrap._get_json", + new=AsyncMock(side_effect=httpx.ConnectError("down")), + ), + patch("sds_federation.services.bootstrap.asyncio.sleep", new=AsyncMock()), + pytest.raises(httpx.ConnectError), + ): + await fetch_peer_sync_list( + httpx.AsyncClient(), + _peer(), + AssetTypeEnum.DATASET, + attempts=3, + backoff_secs=0.01, + ) diff --git a/federation/tests/test_regression_peer_sync.py b/federation/tests/test_regression_peer_sync.py index fd26107e1..1e21d261b 100644 --- a/federation/tests/test_regression_peer_sync.py +++ b/federation/tests/test_regression_peer_sync.py @@ -39,7 +39,7 @@ def test_peer_for_outbound_overlays_site_hello_url() -> None: registry = PeerRegistry() registry.register( SiteHelloWebhook( - site_name="peer-one", + site_name="peer.test", # RFC: FQDN fqdn="peer.test", display_name="Peer", sync_service_url="http://live-sync.test/sync", diff --git a/federation/tests/test_regression_schemas.py b/federation/tests/test_regression_schemas.py index 85ab1fef4..58813f1a9 100644 --- a/federation/tests/test_regression_schemas.py +++ b/federation/tests/test_regression_schemas.py @@ -18,19 +18,29 @@ @pytest.mark.regression -def test_federated_dataset_doc_rejects_unknown_fields() -> None: - base = sample_federated_dataset_doc().model_dump() - base["extra_gateway_field"] = "nope" - with pytest.raises(Exception): - FederatedDatasetDoc.model_validate(base) +def test_federated_docs_allow_unknown_fields() -> None: + dataset = sample_federated_dataset_doc().model_dump() + dataset["extra_gateway_field"] = "ok" + parsed_dataset = FederatedDatasetDoc.model_validate(dataset) + assert (parsed_dataset.model_extra or {})["extra_gateway_field"] == "ok" + + capture = sample_federated_capture_doc().model_dump() + capture["extra_gateway_field"] = "ok" + parsed_capture = FederatedCaptureDoc.model_validate(capture) + assert (parsed_capture.model_extra or {})["extra_gateway_field"] == "ok" + assert parsed_capture.capture_props == {} + assert parsed_capture.search_props == {} + + +CRC_FQDN = "sds.crc.nd.edu" @pytest.mark.regression def test_asset_updated_webhook_round_trip_dataset() -> None: - asset = sample_federated_dataset_doc(site_name="crc") + asset = sample_federated_dataset_doc(site_name=CRC_FQDN) payload = AssetUpdatedWebhook( timestamp=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), - site_name="crc", + site_name=CRC_FQDN, asset=asset, asset_type=AssetTypeEnum.DATASET, ) @@ -41,10 +51,10 @@ def test_asset_updated_webhook_round_trip_dataset() -> None: @pytest.mark.regression def test_asset_updated_webhook_round_trip_capture() -> None: - asset = sample_federated_capture_doc(site_name="crc") + asset = sample_federated_capture_doc(site_name=CRC_FQDN) payload = AssetUpdatedWebhook( timestamp=datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), - site_name="crc", + site_name=CRC_FQDN, asset=asset, asset_type=AssetTypeEnum.CAPTURE, ) diff --git a/federation/uv.lock b/federation/uv.lock index 3f2e50b0c..46d373b07 100644 --- a/federation/uv.lock +++ b/federation/uv.lock @@ -308,6 +308,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "prek" +version = "0.4.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/5c/cb6e63f7e5a58a5313ddb70409174f4dc004e4b0910b8a8d3f59b2225a95/prek-0.4.12.tar.gz", hash = "sha256:04beeba7f40437cd2f36804b84101bd7f3c9fb40b52da46a25604642ab2bfb09", size = 519080, upload-time = "2026-08-03T11:28:33.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/23/5811a3161e072e5f93e4da01af611ee30c32922507b8ab4d9873df6affd3/prek-0.4.12-py3-none-linux_armv6l.whl", hash = "sha256:cd92000b051e433f26340821cf1cc8e6e3960f1275f3d516ca01f05905abba64", size = 5793226, upload-time = "2026-08-03T11:28:09.534Z" }, + { url = "https://files.pythonhosted.org/packages/a3/88/8607845d94eb1482e1bd335dadf098618f077a15775f7e98de99669052b4/prek-0.4.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5904fe6c6ab26e7d8792a3c7f1e3fc8d94fcfb63ad33b247c35f004b62cb6275", size = 6132269, upload-time = "2026-08-03T11:28:11.147Z" }, + { url = "https://files.pythonhosted.org/packages/ac/28/571d79ba457fbd9ecf40ae879c91952e12f5fa475306218c91139b86db7a/prek-0.4.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:df3eff1db9c24dc293010a07bc7a0ae0c541d55af828f5586405dedc28c4920d", size = 5614964, upload-time = "2026-08-03T11:28:12.983Z" }, + { url = "https://files.pythonhosted.org/packages/b0/a9/3f5cb79a73c764a8ac38d5bcd51e0df57239856eca7949b09bdac4338bf3/prek-0.4.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:c7733b44ca772ea32ec6a8bee669d0358bdf45873e79767afed196065084f31c", size = 5941047, upload-time = "2026-08-03T11:28:14.45Z" }, + { url = "https://files.pythonhosted.org/packages/8c/00/1dfed0ef8af10c5c32aa903486dccd33d2df171f3d945a037c5692f10760/prek-0.4.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:87f170cf1ffd6e3a196f947b83dff1f6c2cd68635f8d49740278bebe7b682262", size = 5707994, upload-time = "2026-08-03T11:28:15.914Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bd/5f388f6cbdc0445b850e7c1a160d0be67fcef8bf221e3c8141a1feccef17/prek-0.4.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57dad513831f060cf73808df8edec29d46ec311435aa69f21c80edebf23dc5e1", size = 6133784, upload-time = "2026-08-03T11:28:17.184Z" }, + { url = "https://files.pythonhosted.org/packages/ba/47/342091a987bf68a74acec6d226a40ce7d51faf0019aa4126cc7bc952f8a7/prek-0.4.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b204844abc7ded983471f576ae8dc13b99e9b8d022e4d4b46176c6654769c9d8", size = 6901589, upload-time = "2026-08-03T11:28:18.545Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8a/3ef7bdc3c3441649ebc040b9e164a13163e1e5fabae23e7bbb901992f3de/prek-0.4.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43b0a5a9d3f2f77871fdcb7893bfc5c8fe7e44f4e603ce6e4712bfec96b2d6f2", size = 6342189, upload-time = "2026-08-03T11:28:20Z" }, + { url = "https://files.pythonhosted.org/packages/c4/da/6277908442301b1b92a2879f6b04aaa03accb900f80e42776fc28b8197ef/prek-0.4.12-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:0d188e572c306cc44b96e1bae5647e25b7bd311113f3f3f4a67320c257ee64a3", size = 5951250, upload-time = "2026-08-03T11:28:21.339Z" }, + { url = "https://files.pythonhosted.org/packages/a3/68/bff51a7332837edb1ecbe017325adb7fafd69b9c7828ddc81a1334b884af/prek-0.4.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:986f52d104b7066190f0f32aebe3467710356de265e9bfd892101ba99371db4d", size = 5804147, upload-time = "2026-08-03T11:28:22.656Z" }, + { url = "https://files.pythonhosted.org/packages/aa/de/b7f544971072ed7814125145dfeb1f7c15cce6b78ccea65a96298ff37838/prek-0.4.12-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:13e34d9e09bafcbf1f25a01cf86985e2c5e486591d3f45b2786ba3de82e5153a", size = 5680104, upload-time = "2026-08-03T11:28:24.271Z" }, + { url = "https://files.pythonhosted.org/packages/68/94/95942bcc20a6a91ec2989aa30fdeb00ad095be736ec48b4bbcf0376166b1/prek-0.4.12-py3-none-musllinux_1_1_i686.whl", hash = "sha256:3d0208370da73e8b5bc97f2492dc3975f8dd2c22f4bf6e1f2cf3342503764b52", size = 5975030, upload-time = "2026-08-03T11:28:25.683Z" }, + { url = "https://files.pythonhosted.org/packages/ef/6d/26e6497198d81cf9aa82495400aef46adea8df3e4a4efc5f00e3b6ab3292/prek-0.4.12-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b1005f42920111bec1403c25e8f2f12ec7af0be06686cc3b8dcf85429af908a8", size = 6458532, upload-time = "2026-08-03T11:28:27.121Z" }, + { url = "https://files.pythonhosted.org/packages/44/02/ee140c2eb4701bd194db429d84630733492be94897d5f72b61d6f11e6619/prek-0.4.12-py3-none-win32.whl", hash = "sha256:afee229488dcceaea282288e4d7096a93da5a8b85649d9ef506dbdbcd78f38a7", size = 5502213, upload-time = "2026-08-03T11:28:28.691Z" }, + { url = "https://files.pythonhosted.org/packages/e5/7b/744cff84def48c1ce38c0b4f643a3553c66976c5bb7869ab7317044870e4/prek-0.4.12-py3-none-win_amd64.whl", hash = "sha256:fdd27bad8adafea8fe77606950ca09200d59296a47ab131cfb88718d460949d7", size = 5868065, upload-time = "2026-08-03T11:28:30.377Z" }, + { url = "https://files.pythonhosted.org/packages/46/1d/e2c0fc222904ef73df1739b11a83edc29e38bc4bc61259f2ca6d2f15abb0/prek-0.4.12-py3-none-win_arm64.whl", hash = "sha256:45e34a24fba4a4e4568682477158591698efc2375b8d1d418ae424691c4bd01b", size = 5632819, upload-time = "2026-08-03T11:28:31.743Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -554,12 +578,14 @@ dependencies = [ { name = "opensearch-py" }, { name = "pydantic" }, { name = "redis" }, + { name = "sds-opensearch-query" }, { name = "uvicorn", extra = ["standard"] }, ] [package.optional-dependencies] dev = [ { name = "deptry" }, + { name = "prek" }, { name = "pyrefly" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -573,16 +599,35 @@ requires-dist = [ { name = "httpx", specifier = ">=0.28.1" }, { name = "loguru", specifier = ">=0.7.2" }, { name = "opensearch-py", specifier = ">=2.8.0" }, + { name = "prek", marker = "extra == 'dev'", specifier = ">=0.3.4" }, { name = "pydantic", specifier = ">=2.11.0" }, { name = "pyrefly", marker = "extra == 'dev'", specifier = ">=0.42.1" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25.0" }, { name = "redis", specifier = ">=5.2.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.0" }, + { name = "sds-opensearch-query", directory = "../common" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, ] provides-extras = ["dev"] +[[package]] +name = "sds-opensearch-query" +version = "0.1.0" +source = { directory = "../common" } +dependencies = [ + { name = "loguru" }, + { name = "opensearch-py" }, + { name = "requests" }, +] + +[package.metadata] +requires-dist = [ + { name = "loguru", specifier = ">=0.7.2" }, + { name = "opensearch-py", specifier = ">=2.7.1" }, + { name = "requests", specifier = ">=2.32.0" }, +] + [[package]] name = "six" version = "1.17.0" diff --git a/gateway/.envs/example/django.env b/gateway/.envs/example/django.env index 500d77454..d474ef87b 100644 --- a/gateway/.envs/example/django.env +++ b/gateway/.envs/example/django.env @@ -18,10 +18,8 @@ API_KEY= # FEDERATION_ENABLED=true # RFC [site].name (short peer id); set SDS_SITE_FQDN separately for [site].fqdn. # FEDERATION_SITE_NAME=crc -# Redis pub/sub channel federation-sync subscribes to. -# FEDERATION_EVENTS_CHANNEL=federation:events -# Health probe target (federation-sync service). -# FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health +# Health probe target (federation-sync service), replace {environment} with the environment you are deploying in. +# FEDERATION_SYNC_HEALTH_URL=http://sds-federation-{environment}-sync:8000/sync/health # FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local # Comma-separated CIDRs allowed to call export (default: private Docker ranges). # FEDERATION_EXPORT_ALLOWED_CIDRS= diff --git a/gateway/compose.ci.yaml b/gateway/compose.ci.yaml index 2a02b5e35..eaeb98825 100644 --- a/gateway/compose.ci.yaml +++ b/gateway/compose.ci.yaml @@ -33,8 +33,8 @@ networks: services: sds-gateway-ci-app: build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile image: sds-gateway-ci-app container_name: sds-gateway-ci-app tty: true # colored logs @@ -252,8 +252,8 @@ services: # Celery services for background tasks celery-worker: build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile image: sds-gateway-ci-app container_name: sds-gateway-ci-celery-worker tty: true @@ -310,8 +310,8 @@ services: celery-beat: # Celery Beat scheduler for periodic tasks build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile image: sds-gateway-ci-app container_name: sds-gateway-ci-celery-beat tty: true @@ -368,8 +368,8 @@ services: celery-flower: # Celery monitoring and administration tool build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile image: sds-gateway-ci-app container_name: sds-gateway-ci-celery-flower tty: true diff --git a/gateway/compose.local.yaml b/gateway/compose.local.yaml index c9d4d25aa..935bb15db 100644 --- a/gateway/compose.local.yaml +++ b/gateway/compose.local.yaml @@ -39,8 +39,8 @@ networks: services: sds-gateway-local-app: build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile args: GIT_COMMIT: ${GIT_COMMIT:-none} image: sds-gateway-local-app @@ -262,8 +262,8 @@ services: # Celery services for background tasks celery-worker: build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile args: GIT_COMMIT: ${GIT_COMMIT:-none} image: sds-gateway-local-app @@ -323,8 +323,8 @@ services: celery-beat: # Celery Beat scheduler for periodic tasks build: - context: . - dockerfile: ./compose/local/django/Dockerfile + context: .. + dockerfile: gateway/compose/local/django/Dockerfile args: GIT_COMMIT: ${GIT_COMMIT:-none} image: sds-gateway-local-app diff --git a/gateway/compose/local/django/Dockerfile b/gateway/compose/local/django/Dockerfile index 2692b8f90..c85a6542c 100644 --- a/gateway/compose/local/django/Dockerfile +++ b/gateway/compose/local/django/Dockerfile @@ -6,11 +6,12 @@ RUN pnpm runtime set node 24 -g && cp /pnpm/bin/node /usr/local/bin/node && rm / ARG APP_HOME=/app WORKDIR ${APP_HOME} -COPY ./package.json ${APP_HOME} -COPY ./pnpm-lock.yaml ${APP_HOME} -COPY ./pnpm-workspace.yaml ${APP_HOME} +# Build context is the monorepo root (compose: context: ..). +COPY gateway/package.json ${APP_HOME} +COPY gateway/pnpm-lock.yaml ${APP_HOME} +COPY gateway/pnpm-workspace.yaml ${APP_HOME} RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile -COPY . ${APP_HOME} +COPY gateway/ ${APP_HOME} RUN pnpm run build # https://hub.docker.com/_/python/tags?name=3.13 @@ -64,22 +65,25 @@ ENV UV_PROJECT_ENVIRONMENT=/opt/uv-venv/ RUN mkdir -p /opt/uv-cache/ /opt/uv-venv/ /opt/uv-venv-seed/ +# Path dep ../common from /app → /common +COPY common /common + # Bake third-party dependencies, then the project, into a seed virtualenv. -COPY ./pyproject.toml ./uv.lock ${APP_HOME}/ +COPY gateway/pyproject.toml gateway/uv.lock ${APP_HOME}/ RUN UV_PROJECT_ENVIRONMENT=${UV_PROJECT_SEED} uv sync --frozen --no-install-project --extra ${BUILD_ENVIRONMENT} # "entrypoint" only has a production version -COPY ./compose/production/django/entrypoint /entrypoint +COPY gateway/compose/production/django/entrypoint /entrypoint # copy command scripts and set exec flag -COPY ./compose/local/django/start /start -COPY ./compose/local/django/celery/flower-start /flower-start -COPY ./compose/local/django/celery/worker-start /worker-start -COPY ./compose/local/django/celery/beat-start /beat-start +COPY gateway/compose/local/django/start /start +COPY gateway/compose/local/django/celery/flower-start /flower-start +COPY gateway/compose/local/django/celery/worker-start /worker-start +COPY gateway/compose/local/django/celery/beat-start /beat-start RUN chmod +x /entrypoint /start /worker-start /flower-start /beat-start # copy application code to WORKDIR -COPY . ${APP_HOME} +COPY gateway/ ${APP_HOME} RUN UV_PROJECT_ENVIRONMENT=${UV_PROJECT_SEED} uv sync --frozen --extra ${BUILD_ENVIRONMENT} # Generate version.json with baked-in commit hash (passed as build ARG) diff --git a/gateway/compose/production/django/Dockerfile b/gateway/compose/production/django/Dockerfile index d1657be64..46abc171f 100644 --- a/gateway/compose/production/django/Dockerfile +++ b/gateway/compose/production/django/Dockerfile @@ -6,11 +6,12 @@ RUN pnpm runtime set node 24 -g && cp /pnpm/bin/node /usr/local/bin/node && rm / ARG APP_HOME=/app WORKDIR ${APP_HOME} -COPY ./package.json ${APP_HOME} -COPY ./pnpm-lock.yaml ${APP_HOME} -COPY ./pnpm-workspace.yaml ${APP_HOME} +# Build context is the monorepo root (compose/CI: context: .. / .). +COPY gateway/package.json ${APP_HOME} +COPY gateway/pnpm-lock.yaml ${APP_HOME} +COPY gateway/pnpm-workspace.yaml ${APP_HOME} RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile -COPY . ${APP_HOME} +COPY gateway/ ${APP_HOME} RUN pnpm run build # https://hub.docker.com/_/python/tags?name=3.13 @@ -67,16 +68,19 @@ RUN mkdir -p /opt/uv-cache/ /opt/uv-venv/ /opt/uv-venv-seed/ RUN addgroup --system django \ && adduser --system --ingroup django django +# Path dep ../common from /app → /common +COPY common /common + # Bake third-party dependencies first; install the project after app code is copied. -COPY ./pyproject.toml ./uv.lock ${APP_HOME}/ +COPY gateway/pyproject.toml gateway/uv.lock ${APP_HOME}/ RUN UV_PROJECT_ENVIRONMENT=${UV_PROJECT_SEED} uv sync --frozen --no-install-project --extra ${BUILD_ENVIRONMENT} # copy command scripts and set exec flag -COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint -COPY --chown=django:django ./compose/production/django/start /start -COPY --chown=django:django ./compose/production/django/celery/worker-start /worker-start -COPY --chown=django:django ./compose/production/django/celery/flower-start /flower-start -COPY --chown=django:django ./compose/production/django/celery/beat-start /beat-start +COPY --chown=django:django gateway/compose/production/django/entrypoint /entrypoint +COPY --chown=django:django gateway/compose/production/django/start /start +COPY --chown=django:django gateway/compose/production/django/celery/worker-start /worker-start +COPY --chown=django:django gateway/compose/production/django/celery/flower-start /flower-start +COPY --chown=django:django gateway/compose/production/django/celery/beat-start /beat-start RUN chmod +x /entrypoint /start /worker-start /flower-start /beat-start diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index dd0b98e90..90a007c66 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -753,7 +753,9 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: SDS_SITE_FQDN: str = env.str("SDS_SITE_FQDN", default="localhost") # Federation peer short name (RFC [site].name, e.g. crc, haystack); -# not SDS_PROGRAMMATIC_SITE_NAME. +# used for Redis channel ``federation:events:{name}``, not OpenSearch site_name. +# Export/OpenSearch ``site_name`` uses SDS_SITE_FQDN +# (matches federation.toml [site].fqdn). FEDERATION_SITE_NAME: str = env.str("FEDERATION_SITE_NAME", default="").strip() # Master switch: when False, federation export and Redis events are inactive. FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False) @@ -774,7 +776,7 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: FEDERATION_SYNC_DRF_TOKEN: str = env.str("FEDERATION_SYNC_DRF_TOKEN", default="") FEDERATION_SYNC_HEALTH_URL: str = env.str( "FEDERATION_SYNC_HEALTH_URL", - default="http://federation-sync:8000/sync/health", + default="http://sds-federation-local-sync:8000/sync/health", ) FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT: float = env.float( "FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT", diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index d35a667c7..841ea7ca4 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -56,6 +56,7 @@ "redis>=5.2.0", "requests>=2.32.5", "rich>=13.9.3", + "sds-opensearch-query", "scipy>=1.16.1", "sentry-sdk[django]>=2.25.1", "uvicorn>=0.32.0", @@ -180,6 +181,7 @@ "pytest-cov", # coverage plugin in tests "pytest-sugar", # better progress output for pytest "python-slugify", # provides the slugify import + "requests", # used by sds-opensearch-query OpenSearch client "ruff", # linter used in ci "uvicorn-worker", # uvicorn worker class for gunicorn "uvicorn", # asgi server used in dev @@ -662,3 +664,6 @@ [tool.uv] # https://docs.astral.sh/uv/concepts/resolution/#dependency-cooldowns exclude-newer = "1 week" + +[tool.uv.sources] +sds-opensearch-query = { path = "../common" } diff --git a/gateway/sds_gateway/api_methods/apps.py b/gateway/sds_gateway/api_methods/apps.py index db8856579..62cf738b8 100644 --- a/gateway/sds_gateway/api_methods/apps.py +++ b/gateway/sds_gateway/api_methods/apps.py @@ -1,3 +1,4 @@ +import asyncio import logging import sys @@ -19,6 +20,14 @@ def _skip_federation_init_in_ready() -> bool: } +def _in_async_context() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + class ApiMethodsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "sds_gateway.api_methods" @@ -41,7 +50,7 @@ def _init_federation_after_migrate(sender, **kwargs) -> None: dispatch_uid="api_methods_federation_operational_init", ) - if not _skip_federation_init_in_ready(): + if not _skip_federation_init_in_ready() and not _in_async_context(): initialize_federation_operational_state() silence_unwanted_logs() diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py index dcca8aada..4971b9653 100644 --- a/gateway/sds_gateway/api_methods/federation/availability.py +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -142,6 +142,10 @@ def evaluate_federation_operational() -> tuple[bool, str]: if not site_name: return False, "FEDERATION_SITE_NAME must be set when federation is enabled" + site_fqdn = (_setting("SDS_SITE_FQDN", default="") or "").strip() + if not site_fqdn: + return False, "SDS_SITE_FQDN must be set when federation is enabled" + for check in (_sync_api_key_present, _redis_ok, _sync_health_ok): ok, reason = check() if not ok: diff --git a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py index c49d46b59..046c71b81 100644 --- a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py +++ b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py @@ -31,7 +31,11 @@ def federation_site_name() -> str: - return getattr(settings, "FEDERATION_SITE_NAME", "").strip() + """FQDN written to export/OpenSearch ``site_name`` (RFC §6 / toml ``[site].fqdn``). + + Uses ``SDS_SITE_FQDN`. + """ + return str(getattr(settings, "SDS_SITE_FQDN", "") or "").strip() def capture_in_published_dataset(capture: Capture) -> bool: diff --git a/gateway/sds_gateway/api_methods/federation/fed_index.py b/gateway/sds_gateway/api_methods/federation/fed_index.py index 3a338a4da..2c4a42134 100644 --- a/gateway/sds_gateway/api_methods/federation/fed_index.py +++ b/gateway/sds_gateway/api_methods/federation/fed_index.py @@ -5,6 +5,11 @@ from typing import TYPE_CHECKING from typing import Any +from sds_opensearch_query.index_write import FED_CAPTURES_INDEX +from sds_opensearch_query.index_write import FED_DATASETS_INDEX +from sds_opensearch_query.index_write import federated_doc_id +from sds_opensearch_query.index_write import index_federated_document + from sds_gateway.api_methods.models import ItemType if TYPE_CHECKING: @@ -13,12 +18,13 @@ from opensearchpy import OpenSearch -FED_DATASETS_INDEX = "fed-datasets" -FED_CAPTURES_INDEX = "fed-captures" - - -def federated_doc_id(site_name: str, uuid: UUID) -> str: - return f"{site_name}:{uuid}" +__all__ = [ + "FED_CAPTURES_INDEX", + "FED_DATASETS_INDEX", + "LocalFederatedIndexer", + "federated_doc_id", + "index_for_item_type", +] def index_for_item_type(item_type: ItemType) -> str: @@ -45,15 +51,11 @@ def apply_local_event( uuid: UUID, body: dict[str, Any], ) -> None: - index_name = index_for_item_type(item_type) - doc_id = federated_doc_id(site_name, uuid) - doc = { - **body, - "federation_event_at": event_at.isoformat(), - } - self._client.index( - index=index_name, - id=doc_id, - body=doc, - refresh="wait_for", + index_federated_document( + self._client, + index_name=index_for_item_type(item_type), + site_name=site_name, + uuid=uuid, + body=body, + event_at=event_at, ) diff --git a/gateway/sds_gateway/api_methods/federation/redis_channel.py b/gateway/sds_gateway/api_methods/federation/redis_channel.py index 48abf17c0..914bbb6b3 100644 --- a/gateway/sds_gateway/api_methods/federation/redis_channel.py +++ b/gateway/sds_gateway/api_methods/federation/redis_channel.py @@ -1,22 +1,16 @@ -"""Redis pub/sub channel naming for federation change events (RFC §8).""" +"""Redis pub/sub channel naming for federation change events. -from __future__ import annotations +Re-exports the shared helpers from ``sds_opensearch_query``. +""" +from __future__ import annotations -def resolve_federation_events_channel( - *, - site_name: str = "", - channel_override: str = "", -) -> str: - """Return the Redis channel for local federation events. +from sds_opensearch_query.redis_channel import FEDERATION_EVENTS_CHANNEL_PREFIX +from sds_opensearch_query.redis_channel import federation_events_channel +from sds_opensearch_query.redis_channel import resolve_federation_events_channel - Override ``channel_override`` when set (``FEDERATION_EVENTS_CHANNEL`` env). - Otherwise use ``federation:events:{site_name}`` when ``site_name`` is set. - """ - override = (channel_override or "").strip() - if override: - return override - site = (site_name or "").strip() - if site: - return f"federation:events:{site}" - return "" +__all__ = [ + "FEDERATION_EVENTS_CHANNEL_PREFIX", + "federation_events_channel", + "resolve_federation_events_channel", +] diff --git a/gateway/sds_gateway/api_methods/federation/search_helpers.py b/gateway/sds_gateway/api_methods/federation/search_helpers.py new file mode 100644 index 000000000..c48ff8a48 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/search_helpers.py @@ -0,0 +1,152 @@ +"""Gateway-side OpenSearch search against shared fed-* indices.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any + +from sds_opensearch_query import bool_must_search_body +from sds_opensearch_query import build_metadata_filter_clauses +from sds_opensearch_query import federation_not_deleted_clause +from sds_opensearch_query import flatten_property_paths +from sds_opensearch_query import multi_match_clause +from sds_opensearch_query import run_search +from sds_opensearch_query import term_clause +from sds_opensearch_query.mapping import RFC_FED_CAPTURE_PROPERTIES +from sds_opensearch_query.mapping import RFC_FED_DATASET_PROPERTIES +from sds_opensearch_query.query import DEFAULT_SEARCH_SIZE + +from sds_gateway.api_methods.federation.fed_index import FED_CAPTURES_INDEX +from sds_gateway.api_methods.federation.fed_index import FED_DATASETS_INDEX + +if TYPE_CHECKING: + from collections.abc import Mapping + + from opensearchpy import OpenSearch + +FED_DATASET_TEXT_FIELDS = [ + "name^2", + "description", + "abstract", + "keywords", + "owner_name", +] + +FED_CAPTURE_TEXT_FIELDS = [ + "name", + "channel", + "capture_type", +] + +_WILDCARD_SITES = frozenset({"", "*", "all"}) + + +def _site_clause(site: str | None) -> dict[str, Any] | None: + if site is None or site.strip().lower() in _WILDCARD_SITES: + return None + return term_clause("site_name", site.strip()) + + +def _text_clause(q: str | None, fields: list[str]) -> dict[str, Any] | None: + if q is None or not q.strip(): + return None + return multi_match_clause(q.strip(), fields) + + +def _hits_to_response(hits: list[dict[str, Any]]) -> dict[str, Any]: + return { + "total": len(hits), + "hits": [ + { + "id": hit["_id"], + "score": hit.get("_score"), + "source": hit.get("_source", {}), + } + for hit in hits + ], + } + + +def _build_fed_must_clauses( + *, + q: str | None, + site: str | None, + metadata_filters: list[dict[str, Any]] | None, + rfc_properties: Mapping[str, dict[str, Any]], + text_fields: list[str], + extra_terms: list[tuple[str, Any]] | None = None, +) -> list[dict[str, Any]]: + known = flatten_property_paths(rfc_properties) + must: list[dict[str, Any]] = [federation_not_deleted_clause()] + + site_filter = _site_clause(site) + if site_filter is not None: + must.append(site_filter) + + for field, value in extra_terms or (): + if value is not None and value != "": + must.append(term_clause(field, value)) + + text = _text_clause(q, text_fields) + if text is not None: + must.append(text) + + must.extend( + build_metadata_filter_clauses( + metadata_filters, + known_field_paths=known, + ), + ) + return must + + +def search_federated_datasets( + client: OpenSearch, + *, + q: str | None = None, + site: str | None = None, + metadata_filters: list[dict[str, Any]] | None = None, + size: int = DEFAULT_SEARCH_SIZE, +) -> dict[str, Any]: + must = _build_fed_must_clauses( + q=q, + site=site, + metadata_filters=metadata_filters, + rfc_properties=RFC_FED_DATASET_PROPERTIES, + text_fields=FED_DATASET_TEXT_FIELDS, + ) + body = bool_must_search_body(*must) + hits = run_search( + client, + index=FED_DATASETS_INDEX, + body=body, + size=size, + ) + return _hits_to_response(hits) + + +def search_federated_captures( + client: OpenSearch, + *, + q: str | None = None, + site: str | None = None, + metadata_filters: list[dict[str, Any]] | None = None, + capture_type: str | None = None, + size: int = DEFAULT_SEARCH_SIZE, +) -> dict[str, Any]: + must = _build_fed_must_clauses( + q=q, + site=site, + metadata_filters=metadata_filters, + rfc_properties=RFC_FED_CAPTURE_PROPERTIES, + text_fields=FED_CAPTURE_TEXT_FIELDS, + extra_terms=[("capture_type", capture_type)], + ) + body = bool_must_search_body(*must) + hits = run_search( + client, + index=FED_CAPTURES_INDEX, + body=body, + size=size, + ) + return _hits_to_response(hits) diff --git a/gateway/sds_gateway/api_methods/helpers/index_handling.py b/gateway/sds_gateway/api_methods/helpers/index_handling.py index 8efaefdfc..98dfaa012 100644 --- a/gateway/sds_gateway/api_methods/helpers/index_handling.py +++ b/gateway/sds_gateway/api_methods/helpers/index_handling.py @@ -76,6 +76,35 @@ def index_capture_metadata(capture: Capture, capture_props: dict[str, Any]) -> N raise +def _props_from_source(source: dict[str, Any] | None) -> dict[str, dict[str, Any]]: + src = source or {} + capture_props = src.get("capture_props") or {} + search_props = src.get("search_props") or {} + return { + "capture_props": capture_props if isinstance(capture_props, dict) else {}, + "search_props": search_props if isinstance(search_props, dict) else {}, + } + + +def retrieve_indexed_prop_dicts(capture: Capture) -> dict[str, dict[str, Any]]: + """Return local-index ``capture_props`` and ``search_props`` for one capture.""" + empty = _props_from_source(None) + try: + os_client = get_opensearch_client() + response = os_client.get( + index=capture.index_name, + id=capture.uuid, + ) + return _props_from_source(response.get("_source")) + except os_exceptions.NotFoundError: + msg = "Document(s) or index not found in OpenSearch for metadata retrieval" + log.warning(msg) + return empty + except os_exceptions.OpenSearchException: + log.exception("Failed to retrieve capture OpenSearch props for federation") + return empty + + def retrieve_indexed_metadata( capture_or_captures: Capture | list[Capture], ) -> dict[str, Any]: @@ -98,7 +127,7 @@ def retrieve_indexed_metadata( index=capture_or_captures.index_name, id=capture_or_captures.uuid, ) - return response["_source"]["capture_props"] + return _props_from_source(response.get("_source"))["capture_props"] # we know it's a list here captures_list = capture_or_captures diff --git a/gateway/sds_gateway/api_methods/helpers/list_helpers.py b/gateway/sds_gateway/api_methods/helpers/list_helpers.py new file mode 100644 index 000000000..f91280231 --- /dev/null +++ b/gateway/sds_gateway/api_methods/helpers/list_helpers.py @@ -0,0 +1,320 @@ +"""Lightweight list-row adapters for local and federated dataset UI lists.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from typing import Any + +from django.conf import settings +from django.urls import reverse +from django.utils import dateparse +from django.utils import timezone + +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.models import ItemType +from sds_gateway.api_methods.models import PermissionLevel +from sds_gateway.api_methods.models import UserSharePermission + +if TYPE_CHECKING: + from collections.abc import Iterable + + from django.db.models import QuerySet + + from sds_gateway.api_methods.models import Dataset + from sds_gateway.users.models import User + + +def local_site_name() -> str: + """FQDN used as ``site_name`` for local rows (RFC peer identity).""" + return str(getattr(settings, "SDS_SITE_FQDN", "") or "").strip() + + +def _parse_datetime(value: Any) -> datetime | None: + if value is None or value == "": + return None + if isinstance(value, datetime): + parsed: datetime | None = value + elif isinstance(value, str): + parsed = dateparse.parse_datetime(value) + else: + return None + if parsed is None: + return None + if timezone.is_naive(parsed): + return timezone.make_aware(parsed, timezone.get_current_timezone()) + return parsed + + +def dataset_list_dropdown_menu_items(row: dict[str, Any]) -> list[dict[str, Any]]: + """Build dropdown_menu.html items for a serialized dataset list row.""" + if row.get("is_federated"): + return [] + + uuid = str(row.get("uuid") or "") + if not uuid: + return [] + + is_owner = row.get("is_owner") + permission_level = row.get("permission_level") + is_contributor = permission_level == PermissionLevel.CONTRIBUTOR + is_co_owner = permission_level == PermissionLevel.CO_OWNER + dataset_published = row.get("status") == DatasetStatus.FINAL and row.get( + "is_public" + ) + + items: list[dict[str, Any]] = [] + if is_owner or is_contributor or is_co_owner: + items.append( + { + "label": "Share", + "icon": "person-plus", + "type": "button", + "modal_toggle": True, + "modal_target": f"#shareModal-{uuid}", + "data_attrs": {}, + } + ) + + if not dataset_published: + items.append( + { + "label": "Edit", + "icon": "pencil", + "type": "link", + "href": f"{reverse('users:group_captures')}?dataset_uuid={uuid}", + "data_attrs": {}, + } + ) + + if is_owner or is_co_owner: + items.append( + { + "label": "Create New Version", + "icon": "folder-symlink", + "type": "button", + "modal_toggle": True, + "modal_target": f"#versioningModal-{uuid}", + "data_attrs": {}, + } + ) + if not dataset_published: + items.append( + { + "label": "Publish", + "icon": "globe", + "type": "button", + "modal_toggle": True, + "modal_target": f"#publish-dataset-modal-{uuid}", + "data_attrs": {"dataset-uuid": uuid}, + "extra_class": "publish-dataset-btn", + } + ) + + status = row.get("status") + is_public = bool(row.get("is_public")) + is_deletable_dataset = status == DatasetStatus.DRAFT and not is_public + if is_owner and is_deletable_dataset: + dataset_name = str(row.get("name") or "").strip() or "Dataset" + items.append( + { + "label": "Delete", + "icon": "trash", + "type": "button", + "extra_class": "delete-asset-btn", + "data_attrs": { + "asset-type": "dataset", + "asset-uuid": uuid, + "asset-name": dataset_name[:200], + **({"asset-shared": "true"} if row.get("is_shared") else {}), + }, + } + ) + + items.append( + { + "label": "Web Download", + "icon": "download", + "type": "button", + "modal_toggle": True, + "modal_target": f"#webDownloadModal-{uuid}", + "data_attrs": {}, + } + ) + items.append( + { + "label": "SDK Instructions", + "icon": "code-slash", + "type": "button", + "modal_toggle": True, + "modal_target": f"#sdkDownloadModal-{uuid}", + "data_attrs": {}, + } + ) + return items + + +def _keyword_names(dataset: Dataset) -> list[str]: + return [ + kw.name for kw in dataset.keywords.all() if not getattr(kw, "is_deleted", False) + ] + + +def _permission_maps_for_user( + datasets: list[Dataset], + user: User | None, +) -> tuple[dict[Any, str], set[Any], set[Any]]: + """Batch-load permission_level, is_shared, is_shared_with_me maps.""" + uuids = [ds.uuid for ds in datasets] + if not uuids: + return {}, set(), set() + + shared_uuids = set( + UserSharePermission.objects.filter( + item_uuid__in=uuids, + item_type=ItemType.DATASET, + is_deleted=False, + is_enabled=True, + ).values_list("item_uuid", flat=True) + ) + + perm_by_uuid: dict[Any, str] = {} + shared_with_me: set[Any] = set() + if user is not None and getattr(user, "is_authenticated", False): + for ds in datasets: + if ds.owner_id == user.id: + perm_by_uuid[ds.uuid] = PermissionLevel.OWNER + + user_perms = UserSharePermission.objects.filter( + item_uuid__in=uuids, + item_type=ItemType.DATASET, + shared_with=user, + is_deleted=False, + is_enabled=True, + ).values_list("item_uuid", "permission_level") + for item_uuid, level in user_perms: + shared_with_me.add(item_uuid) + if item_uuid not in perm_by_uuid: + perm_by_uuid[item_uuid] = level + + return perm_by_uuid, shared_uuids, shared_with_me + + +def serialize_local_dataset_row( + dataset: Dataset, + user: User | None = None, + *, + permission_level: str | None = None, + is_shared: bool = False, + is_shared_with_me: bool = False, + include_actions: bool = True, +) -> dict[str, Any]: + """Build a lightweight list-row dict for a local Dataset ORM instance.""" + is_owner = bool( + user is not None + and getattr(user, "is_authenticated", False) + and dataset.owner_id == user.id + ) + if permission_level is None and is_owner: + permission_level = PermissionLevel.OWNER + + row: dict[str, Any] = { + "uuid": dataset.uuid, + "name": dataset.name, + "version": dataset.version, + "authors": dataset.get_authors_display(), + "keywords": _keyword_names(dataset), + "created_at": dataset.created_at, + "updated_at": dataset.updated_at, + "site_name": local_site_name(), + "is_federated": False, + "is_public": dataset.is_public, + "status": dataset.status, + "status_display": dataset.get_status_display(), + "owner_name": dataset.owner.name if dataset.owner else "Owner", + "is_owner": is_owner, + "is_shared": is_shared, + "is_shared_with_me": is_shared_with_me and not is_owner, + "permission_level": permission_level, + # Keep ORM for existing list modals that still read ``row.dataset``. + "dataset": dataset, + } + row["dropdown_menu_items"] = ( + dataset_list_dropdown_menu_items(row) if include_actions else [] + ) + return row + + +def serialize_federated_dataset_row(doc: dict[str, Any]) -> dict[str, Any]: + """Normalize a fed-datasets OpenSearch ``_source`` into a list-row dict.""" + return { + "uuid": doc.get("uuid"), + "name": doc.get("name") or "", + "version": doc.get("version", 1), + "authors": doc.get("authors") or [], + "keywords": doc.get("keywords") or [], + "created_at": _parse_datetime(doc.get("created_at")), + "updated_at": _parse_datetime(doc.get("updated_at")), + "site_name": doc.get("site_name") or "", + "is_federated": True, + "is_public": bool(doc.get("is_public", True)), + "status": doc.get("status") or DatasetStatus.FINAL, + "status_display": doc.get("status_display") or "Final", + "owner_name": doc.get("owner_name") or "", + "abstract": doc.get("abstract") or "", + "description": doc.get("description") or "", + "is_owner": False, + "is_shared": False, + "is_shared_with_me": False, + "permission_level": None, + "can_edit": False, + "can_share": False, + "dropdown_menu_items": [], + } + + +def serialize_datasets_for_user( + datasets: QuerySet[Dataset] | Iterable[Dataset], + user: User | None, + *, + include_actions: bool = True, +) -> list[dict[str, Any]]: + """Serialize local datasets into list-row dicts (no heavy API serializer).""" + dataset_list = list(datasets) + perm_by_uuid, shared_uuids, shared_with_me = _permission_maps_for_user( + dataset_list, + user, + ) + return [ + serialize_local_dataset_row( + dataset, + user, + permission_level=perm_by_uuid.get(dataset.uuid), + is_shared=dataset.uuid in shared_uuids, + is_shared_with_me=dataset.uuid in shared_with_me, + include_actions=include_actions, + ) + for dataset in dataset_list + ] + + +def _sort_key_value(row: dict[str, Any], key: str) -> tuple[bool, Any]: + value = row.get(key) + # None sorts after real values when ascending; reverse flips that. + return (value is None, value) + + +def merge_dataset_list_rows( + local_rows: list[dict[str, Any]], + federated_rows: list[dict[str, Any]], + *, + sort_by: str = "created_at", + descending: bool = True, +) -> list[dict[str, Any]]: + """Merge local + federated list rows and sort by a shared field.""" + merged = [*local_rows, *federated_rows] + merged.sort( + key=lambda row: _sort_key_value(row, sort_by), + reverse=descending, + ) + return merged diff --git a/gateway/sds_gateway/api_methods/helpers/search_captures.py b/gateway/sds_gateway/api_methods/helpers/search_captures.py index 6cfa408bd..aa357ad13 100644 --- a/gateway/sds_gateway/api_methods/helpers/search_captures.py +++ b/gateway/sds_gateway/api_methods/helpers/search_captures.py @@ -8,6 +8,11 @@ from opensearchpy import exceptions as os_exceptions from rest_framework.request import Request from rich.pretty import pretty_repr +from sds_opensearch_query import bool_must_search_body +from sds_opensearch_query import build_metadata_filter_clauses +from sds_opensearch_query import flatten_property_paths +from sds_opensearch_query import run_search +from sds_opensearch_query.query import DEFAULT_SEARCH_SIZE from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import CaptureType @@ -20,10 +25,10 @@ from sds_gateway.api_methods.utils.asset_access_control import ( get_accessible_captures_queryset, ) -from sds_gateway.api_methods.utils.metadata_schemas import base_index_fields from sds_gateway.api_methods.utils.metadata_schemas import ( capture_index_mapping_by_type as md_props_by_type, ) +from sds_gateway.api_methods.utils.metadata_schemas import get_mapping_by_capture_type from sds_gateway.api_methods.utils.metadata_schemas import infer_index_name from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client from sds_gateway.api_methods.utils.relationship_utils import ( @@ -31,148 +36,38 @@ ) from sds_gateway.users.models import User -RangeValue = dict[str, int | float] UNKNOWN_CAPTURE_TYPE = "Unknown capture type" -# maximum size (doc count) of OpenSearch searches -MAX_OS_SIZE = 10_000 +MAX_OS_SIZE = DEFAULT_SEARCH_SIZE -def handle_nested_query( - field_path: str, - query_type: str, - value: Any, - levels_nested: int, - last_path: str | None = None, -) -> dict[str, Any]: - """Build a nested metadata query for a given field path and value. - - Args: - field_path: Full path to the field (e.g.'capture_props.metadata.fmax') - query_type: Type of query (e.g. 'match', 'term') - value: Value to match against - levels_nested: Number of nested levels to traverse - Returns: - Nested query dictionary for OpenSearch - """ - - if levels_nested == 0: - return {query_type: {f"{last_path}.{field_path}": value}} - - path_parts = field_path.split(".") - current_path = path_parts[0] - if last_path is not None: - current_path = f"{last_path}.{current_path}" - - return { - "nested": { - "path": current_path, - "query": handle_nested_query( - field_path=".".join(path_parts[1:]), - query_type=query_type, - value=value, - levels_nested=levels_nested - 1, - last_path=current_path, - ), - }, - } +def _warn_unknown_field(field_path: str) -> None: + msg = ( + f"Field '{field_path}' does not match an indexed field. " + "The filter may not be applied to the query accurately." + ) + log.warning(msg) def _build_os_metadata_query( capture_type: CaptureType | None = None, metadata_filters: list[dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: - """Build OpenSearch query for metadata fields. - - Args: - capture_type: Type of capture (e.g. 'drf') - metadata_filters: list of dicts with 'field', 'type', and 'value' keys - Returns: - List of OpenSearch query clauses for the metadata fields - """ - - metadata_queries: list[dict[str, Any]] = [] - if metadata_filters is None: - log.debug("No metadata filters provided to build the OpenSearch query.") - return metadata_queries - - index_fields = _flatten_index_mapping( - index_mapping=_get_index_mapping(capture_type=capture_type), - index_fields=base_index_fields.copy(), + known_paths = _known_field_paths_for_capture_type(capture_type) + clauses = build_metadata_filter_clauses( + metadata_filters, + known_field_paths=known_paths, + on_unknown_field=_warn_unknown_field, ) - - for query in metadata_filters: - field_path: str = query["field_path"] - query_type: str = query["query_type"] - filter_value: Any = query["filter_value"] - - # warn if the field is not in the index mapping - # but continue to build the query - if field_path not in index_fields: - msg = ( - f"Field '{field_path}' does not match an indexed field." - "The filter may not be applied to the query accurately." - ) - log.warning(msg) - - levels_nested = field_path.count(".") - if levels_nested > 0: - metadata_queries.append( - handle_nested_query( - field_path=field_path, - query_type=query_type, - value=filter_value, - levels_nested=levels_nested, - ), - ) - else: - metadata_queries.append({query_type: {field_path: filter_value}}) - log.debug( - f"Built {len(metadata_queries)} OpenSearch metadata " - f"queries: {metadata_queries}", + f"Built {len(clauses)} OpenSearch metadata queries: {clauses}", ) - return metadata_queries + return clauses -def _flatten_index_mapping( - index_mapping: Mapping[str, Any], - index_fields: list[str], - prefix: str = "capture_props", - separator: str = ".", -) -> list[str]: - """Flatten the index mapping to a list of fields. - Args: - index_mapping: The index mapping to flatten. - index_fields: The list of fields to flatten. - Returns: - A list of flattened fields. - """ - for field, field_type in index_mapping.items(): - if isinstance(field_type, dict) and field_type.get("type") == "nested": - index_fields.extend( - [ - f"{prefix}{separator}{field}{separator}{nested_field}" - for nested_field in field_type.get("properties", {}) - ], - ) - else: - index_fields.append(f"{prefix}{separator}{field}") - return index_fields - - -def _get_index_mapping(capture_type: CaptureType | None) -> dict[str, dict[str, Any]]: - """Retrieves the OpenSearch index mapping for a given capture type or all types. - - Args: - capture_type: The capture type for which the index mapping is requested. - If None, the function merges all capture type properties. - Raises: - ValueError: If the capture type is not None and is not recognized. - Returns: - A dict where the keys are property names and the values are dicts with - metadata for the specified capture type or all capture types combined. - """ +def _known_field_paths_for_capture_type( + capture_type: CaptureType | None, +) -> frozenset[str]: implemented_capture_types = set(md_props_by_type.keys()) assert implemented_capture_types, ( "No capture types are implemented. Please check the metadata properties." @@ -182,13 +77,14 @@ def _get_index_mapping(capture_type: CaptureType | None) -> dict[str, dict[str, if capture_type not in implemented_capture_types: msg = f"{UNKNOWN_CAPTURE_TYPE}: {capture_type}" raise ValueError(msg) - index_mapping = md_props_by_type.get(capture_type, {}) - else: - # merge all capture type properties into a single flat dictionary - index_mapping = {} - for ct in implemented_capture_types: - index_mapping.update(md_props_by_type[ct]) - return index_mapping + properties = get_mapping_by_capture_type(capture_type)["properties"] + return flatten_property_paths(properties) + + paths: set[str] = set() + for ct in implemented_capture_types: + properties = get_mapping_by_capture_type(ct)["properties"] + paths.update(flatten_property_paths(properties)) + return frozenset(paths) def get_capture_queryset( @@ -196,12 +92,9 @@ def get_capture_queryset( capture_type: CaptureType | None, ) -> QuerySet[Capture]: """Get the capture queryset based on the capture type.""" - # Get captures accessible to the user using database-level filtering capture_queryset = get_accessible_captures_queryset(request_user) - # filter by capture type if provided if capture_type: - # verify capture type exists before filtering if not md_props_by_type.get(capture_type): raise ValueError(UNKNOWN_CAPTURE_TYPE) capture_queryset = capture_queryset.filter(capture_type=capture_type) @@ -238,10 +131,14 @@ def search_captures( log.debug("No metadata queries provided. Returning all captures.") return capture_queryset - os_query = _build_os_query_for_captures( - capture_type=capture_type, - metadata_queries=metadata_queries, - ) + must_clauses: list[dict[str, Any]] = [] + if capture_type: + must_clauses.append({"term": {"capture_type": capture_type}}) + must_clauses.extend(metadata_queries) + + os_query = bool_must_search_body(*must_clauses) + log.debug("OpenSearch query:") + log.debug(pretty_repr(os_query, indent_size=4)) client = get_opensearch_client() index_name: str = ( @@ -249,50 +146,29 @@ def search_captures( ) try: - response = client.search( + hits = run_search( + client, index=index_name, body=os_query, - size=MAX_OS_SIZE, # pyright: ignore[reportCallIssue] + size=MAX_OS_SIZE, ) - except os_exceptions.NotFoundError as err: - msg = f"Index '{index_name}' not found" - log.exception(msg) - raise ValueError(msg) from err except os_exceptions.ConnectionError as err: msg = f"Failed to connect to OpenSearch: {err}" log.exception(msg) raise - except os_exceptions.RequestError as err: - # raise ValueError to trigger a 400 HTTP response - # and pass the information to the client about what - # went wrong with the query they provided - context_for_user = "Query error" - info = err.info - if isinstance(info, dict): - root_causes: list[dict[str, str]] = info.get("error", {}).get( - "root_cause", [] - ) - root_cause_reason: str = ( - root_causes[0].get("reason", "") if root_causes else "" - ) - reason = str(root_cause_reason) if root_cause_reason else str(info) - else: - reason = str(info) - msg = f"{context_for_user}: {reason}" - raise ValueError(msg) from err + except ValueError as err: + raise ValueError(str(err)) from err except os_exceptions.OpenSearchException as err: msg = f"OpenSearch generic error: {err}" log.exception(msg) raise - # Extract capture UUIDs from OpenSearch response - capture_uuids: list[str] = [hit["_id"] for hit in response["hits"]["hits"]] + capture_uuids: list[str] = [hit["_id"] for hit in hits] if not capture_uuids: log.debug("No captures found in OpenSearch.") return capture_queryset.none() - # Check if we're approaching the maximum size limit num_hits = len(capture_uuids) if num_hits > 0.9 * MAX_OS_SIZE: log.warning( @@ -301,9 +177,8 @@ def search_captures( ) log.debug(f"Found {len(capture_uuids)} matching captures.") - # Filter the queryset to only include captures found in OpenSearch filtered_queryset = capture_queryset.filter(uuid__in=capture_uuids).order_by( - "-updated_at" + "-updated_at", ) log.debug( @@ -314,43 +189,13 @@ def search_captures( return filtered_queryset -def _build_os_query_for_captures( - capture_type: CaptureType | None, - metadata_queries: list[dict[str, Any]], -) -> dict[str, Any]: - """Build the OpenSearch query for searching captures. - - Args: - capture_type: The type of capture to filter by, or None for all types. - metadata_queries: A list of metadata query clauses to include in the query. - - Returns: - A dictionary representing the OpenSearch query. - """ - must_clauses: list[dict[str, dict[str, Any]]] = [] - if capture_type: - must_clauses.append({"term": {"capture_type": capture_type}}) - must_clauses.extend(metadata_queries) - - query = { - "query": { - "bool": { - "must": must_clauses, - }, - }, - } - log.debug("OpenSearch query:") - log.debug(pretty_repr(query, indent_size=4)) - return query - - # TODO: add pagination before retrieval rather than after # Need to paginate/limit OpenSearch results list before grouping # and then paginate/limit the grouped captures def get_composite_captures( captures: QuerySet[Capture], request: Request | None = None, - bulk_metadata: dict[str, dict[str, Any]] | None = None, + bulk_metadata: dict[str, Mapping[str, Any]] | None = None, ) -> list[dict[str, Any]]: """Get captures as composite objects, grouping multi-channel captures. @@ -374,11 +219,9 @@ def get_composite_captures( for capture_list in grouped_captures.values(): if len(capture_list) > 1: - # Multiple captures with same top_level_dir - create composite composite_data = build_composite_capture_data(capture_list) composite_captures.append(composite_data) else: - # Single capture - serialize normally capture = capture_list[0] capture_data = serialize_capture_or_composite(capture, context=context) composite_captures.append(capture_data) diff --git a/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py b/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py new file mode 100644 index 000000000..8aee53f13 --- /dev/null +++ b/gateway/sds_gateway/api_methods/management/commands/publish_for_federation.py @@ -0,0 +1,71 @@ +"""Mark a dataset and its captures as public + FINAL for federation export.""" + +from __future__ import annotations + +from uuid import UUID + +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError +from django.db import transaction +from loguru import logger as log + +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import DatasetStatus + + +class Command(BaseCommand): + help = ( + "Set dataset status to FINAL and is_public=True, and mark linked captures " + "public (triggers federation Redis events when FEDERATION_ENABLED)." + ) + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--dataset-uuid", + required=True, + help="Dataset UUID to publish for federation", + ) + parser.add_argument( + "--capture-uuids", + nargs="*", + default=None, + help="Optional capture UUIDs to attach before publishing", + ) + + def handle(self, *args, **options) -> None: + dataset_uuid = UUID(str(options["dataset_uuid"])) + capture_uuids = options["capture_uuids"] or [] + + with transaction.atomic(): + dataset = Dataset.objects.select_for_update().get(uuid=dataset_uuid) + if capture_uuids: + captures = Capture.objects.filter( + uuid__in=[UUID(str(u)) for u in capture_uuids], + is_deleted=False, + ) + missing = set(capture_uuids) - {str(c.uuid) for c in captures} + if missing: + msg = f"Captures not found: {sorted(missing)}" + raise CommandError(msg) + dataset.captures.add(*captures) + + Capture.objects.filter( + datasets=dataset, + is_deleted=False, + ).update(is_public=True) + + dataset.status = DatasetStatus.FINAL + dataset.is_public = True + dataset.save(update_fields=["status", "is_public", "updated_at"]) + + log.info( + "Published dataset {} for federation export (FINAL, public)", + dataset_uuid, + ) + self.stdout.write( + self.style.SUCCESS( + f"Dataset {dataset_uuid} is FINAL and public. " + "Re-run federation bootstrap or save again to re-index if needed.", + ), + ) diff --git a/gateway/sds_gateway/api_methods/serializers/capture_serializers.py b/gateway/sds_gateway/api_methods/serializers/capture_serializers.py index 209763f0c..a23d01df3 100644 --- a/gateway/sds_gateway/api_methods/serializers/capture_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/capture_serializers.py @@ -12,6 +12,7 @@ from rest_framework.utils.serializer_helpers import ReturnList from sds_gateway.api_methods.helpers.index_handling import retrieve_indexed_metadata +from sds_gateway.api_methods.helpers.index_handling import retrieve_indexed_prop_dicts from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import CaptureType from sds_gateway.api_methods.models import DEPRECATEDPostProcessedData @@ -892,6 +893,7 @@ class CaptureFederationSerializer(serializers.ModelSerializer[Capture]): file_count = serializers.SerializerMethodField() size = serializers.SerializerMethodField() capture_props = serializers.SerializerMethodField() + search_props = serializers.SerializerMethodField() public_dataset_ids = serializers.SerializerMethodField() created_at = serializers.DateTimeField( format="%Y-%m-%d %H:%M:%S%z", @@ -919,6 +921,7 @@ class Meta: "file_count", "size", "capture_props", + "search_props", "public_dataset_ids", "created_at", "updated_at", @@ -935,8 +938,21 @@ def get_file_count(self, obj: Capture) -> int: def get_size(self, obj: Capture) -> int: return int(obj.get_files_summary()["total_size"]) + def _indexed_prop_dicts(self, obj: Capture) -> dict[str, dict[str, Any]]: + cache = getattr(self, "_indexed_prop_dicts_cache", None) + if cache is None: + cache = {} + self._indexed_prop_dicts_cache = cache + key = str(obj.uuid) + if key not in cache: + cache[key] = retrieve_indexed_prop_dicts(obj) + return cache[key] + def get_capture_props(self, obj: Capture) -> dict[str, Any]: - return obj.get_opensearch_metadata() or {} + return self._indexed_prop_dicts(obj)["capture_props"] + + def get_search_props(self, obj: Capture) -> dict[str, Any]: + return self._indexed_prop_dicts(obj)["search_props"] def get_public_dataset_ids(self, obj: Capture) -> list[str]: qs = obj.datasets.federation_exportable() diff --git a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py index a8deb619c..ce9c79438 100644 --- a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py @@ -289,6 +289,7 @@ class DatasetPublicSerializer(serializers.ModelSerializer[Dataset]): authors = serializers.SerializerMethodField() keywords = serializers.SerializerMethodField() + institutions = serializers.SerializerMethodField() created_at = serializers.DateTimeField( format=READABLE_ISO_DATE_TIME, read_only=True ) @@ -303,6 +304,13 @@ def get_keywords(self, obj): """Return a list of keyword names for the dataset.""" return [kw.name for kw in obj.keywords.filter(is_deleted=False)] + def get_institutions(self, obj): + if not obj.institutions: + return [] + if isinstance(obj.institutions, list): + return obj.institutions + return [] + def get_owner_name(self, obj): """Get the owner's display name.""" return obj.owner.name if obj.owner else "Owner" @@ -338,8 +346,12 @@ class DatasetFederationSerializer(DatasetPublicSerializer): """Serializer for dataset data for federation export.""" site_name = serializers.SerializerMethodField() + created_at = serializers.DateTimeField( + format="iso-8601", # ISO 8601 format for OpenSearch schema + read_only=True, + ) updated_at = serializers.DateTimeField( - format=READABLE_ISO_DATE_TIME, + format="iso-8601", # ISO 8601 format for OpenSearch schema read_only=True, ) size = serializers.SerializerMethodField() diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_events.py b/gateway/sds_gateway/api_methods/tests/test_federation_events.py index 363b1f199..aed52aeba 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_events.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_events.py @@ -99,6 +99,7 @@ class TestFederationSignals: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, FEDERATION_EVENTS_CHANNEL="federation:events:crc", ) @@ -127,7 +128,7 @@ def test_dataset_post_save_indexes_when_published( mock_indexer.apply_local_event.assert_called_once() call = mock_indexer.apply_local_event.call_args.kwargs - assert call["site_name"] == "crc" + assert call["site_name"] == "sds.localhost" assert call["item_type"] == ItemType.DATASET assert call["uuid"] == dataset.uuid mock_publish.assert_called_once_with( diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py index 4d0318bf1..39fdc002a 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -23,6 +23,7 @@ @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.crc.nd.edu", FEDERATION_OPERATIONAL_OVERRIDE=True, FEDERATION_EXPORT_ALLOWED_CIDRS=["0.0.0.0/0", "::/0"], ) @@ -96,7 +97,7 @@ def test_sync_key_can_retrieve_public_dataset(self) -> None: ) assert response.status_code == status.HTTP_200_OK assert response.json()["uuid"] == str(self.public_dataset.uuid) - assert response.json()["site_name"] == "crc" + assert response.json()["site_name"] == "sds.crc.nd.edu" def test_sync_key_dataset_detail_404_when_not_indexed(self) -> None: with patch( diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py index 33d8bde47..27e314994 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py @@ -49,7 +49,7 @@ def test_dataset_export_field_names_match_pydantic() -> None: ) serializer = DatasetFederationSerializer( dataset, - context={"site_name": "crc"}, + context={"site_name": "sds.crc.nd.edu"}, ) assert_field_names_match( serializer, @@ -64,7 +64,7 @@ def test_capture_export_field_names_match_pydantic() -> None: capture = CaptureFactory(owner=owner, is_public=True) serializer = CaptureFederationSerializer( capture, - context={"site_name": "crc"}, + context={"site_name": "sds.crc.nd.edu"}, ) assert_field_names_match( serializer, diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index f512459ab..f535018ba 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -52,6 +52,7 @@ def test_disabled_when_master_switch_off(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, @@ -63,6 +64,7 @@ def test_operational_when_probes_skipped(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, ) @@ -74,7 +76,9 @@ def test_fails_without_sync_api_key(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=False, FEDERATION_SYNC_HEALTH_URL="http://sync.test/health", FEDERATION_SKIP_REDIS_PROBE=True, ) @@ -93,7 +97,9 @@ def test_health_probe_success(self, mock_urlopen: MagicMock) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=False, FEDERATION_SYNC_HEALTH_URL="http://sync.test/health", FEDERATION_SKIP_REDIS_PROBE=True, ) @@ -134,6 +140,7 @@ def test_empty_cidr_override_falls_back_to_private_defaults(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, @@ -146,6 +153,20 @@ def test_fails_without_site_name(self) -> None: @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="", + FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, + FEDERATION_SKIP_REDIS_PROBE=True, + ) + def test_fails_without_site_fqdn(self) -> None: + ok, reason = evaluate_federation_operational() + assert ok is False + assert "SDS_SITE_FQDN" in reason + + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py index dc1935fe1..5544b352e 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py @@ -37,6 +37,7 @@ def _federation_on_commit(): @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestFederationDatasetSignals(TestCase): @@ -149,6 +150,7 @@ def test_deleted_dataset_skips_when_fed_doc_already_deleted( @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestFederationCaptureSignals(TestCase): @@ -230,6 +232,7 @@ def test_deleted_capture_reindexes_with_is_deleted( @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestDatasetDisconnectReindex(TestCase): @@ -293,6 +296,7 @@ def test_dataset_soft_delete_reindexes_orphan_capture( @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", + SDS_SITE_FQDN="sds.localhost", FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestFederationReindexOnCommit(TestCase): diff --git a/gateway/sds_gateway/api_methods/utils/opensearch_client.py b/gateway/sds_gateway/api_methods/utils/opensearch_client.py index 81380d8ca..1199c241e 100644 --- a/gateway/sds_gateway/api_methods/utils/opensearch_client.py +++ b/gateway/sds_gateway/api_methods/utils/opensearch_client.py @@ -1,30 +1,14 @@ from django.conf import settings -from loguru import logger as log -from opensearchpy import OpenSearch -from opensearchpy import RequestsHttpConnection -from requests.auth import HTTPBasicAuth +from sds_opensearch_query.client import build_opensearch_client def get_opensearch_client(): - payload = { - "hosts": [{"host": settings.OPENSEARCH_HOST, "port": settings.OPENSEARCH_PORT}], - "http_auth": HTTPBasicAuth( - settings.OPENSEARCH_USER, - settings.OPENSEARCH_PASSWORD, - ), - "use_ssl": settings.OPENSEARCH_USE_SSL, - "verify_certs": settings.OPENSEARCH_VERIFY_CERTS, - "ssl_show_warn": False, - "connection_class": RequestsHttpConnection, - } - if settings.OPENSEARCH_VERIFY_CERTS: - if not settings.OPENSEARCH_CA_CERTS: - msg = ( - "OPENSEARCH_VERIFY_CERTS is True but OPENSEARCH_CA_CERTS is not set." - "Not using SSL." - ) - log.warning(msg) - else: - log.info("OPENSEARCH_VERIFY_CERTS is True. Verifying certificates.") - payload["ca_certs"] = settings.OPENSEARCH_CA_CERTS - return OpenSearch(**payload) + return build_opensearch_client( + host=settings.OPENSEARCH_HOST, + port=settings.OPENSEARCH_PORT, + user=settings.OPENSEARCH_USER, + password=settings.OPENSEARCH_PASSWORD, + use_ssl=settings.OPENSEARCH_USE_SSL, + verify_certs=settings.OPENSEARCH_VERIFY_CERTS, + ca_certs=settings.OPENSEARCH_CA_CERTS, + ) diff --git a/gateway/sds_gateway/templates/pages/home.html b/gateway/sds_gateway/templates/pages/home.html index 8c552f007..b179d2022 100644 --- a/gateway/sds_gateway/templates/pages/home.html +++ b/gateway/sds_gateway/templates/pages/home.html @@ -64,12 +64,16 @@
data-item-uuid="{{ dataset.uuid }}" data-item-type="dataset" aria-label="View details for {{ dataset.name }}">{{ dataset.name }} - {% if dataset.is_public %}{% endif %} + {% if dataset.is_federated %} + {{ dataset.site_name }} + {% elif dataset.is_public %} + + {% endif %}
- {% if dataset.dataset.created_at %} - {{ dataset.dataset.created_at|date:"Y-m-d" }} + {% if dataset.created_at %} + {{ dataset.created_at|date:"Y-m-d" }} {% endif %}
diff --git a/gateway/sds_gateway/templates/users/components/dataset_list_modals.html b/gateway/sds_gateway/templates/users/components/dataset_list_modals.html index c9223469e..bfecb87f8 100644 --- a/gateway/sds_gateway/templates/users/components/dataset_list_modals.html +++ b/gateway/sds_gateway/templates/users/components/dataset_list_modals.html @@ -4,15 +4,17 @@ Context: page_obj (paginated dataset list) {% endcomment %} {% for dataset in page_obj %} - {% include "users/partials/web_download_modal.html" with item=dataset item_type="dataset" %} - {% include "users/partials/sdk_download_modal.html" with dataset=dataset %} - {% if dataset.is_owner or dataset.permission_level == 'co-owner' or dataset.permission_level == 'contributor' %} - {% include "users/partials/share_modal.html" with item=dataset item_type="dataset" %} - {% endif %} - {% if dataset.is_owner or dataset.permission_level == 'co-owner' %} - {% include "users/partials/dataset_version_control.html" with dataset=dataset %} - {% if not dataset.dataset.status == 'final' or not dataset.is_public %} - {% include "users/partials/publish_dataset_modal.html" with dataset=dataset %} + {% if not dataset.is_federated %} + {% include "users/partials/web_download_modal.html" with item=dataset item_type="dataset" %} + {% include "users/partials/sdk_download_modal.html" with dataset=dataset %} + {% if dataset.is_owner or dataset.permission_level == 'co-owner' or dataset.permission_level == 'contributor' %} + {% include "users/partials/share_modal.html" with item=dataset item_type="dataset" %} + {% endif %} + {% if dataset.is_owner or dataset.permission_level == 'co-owner' %} + {% include "users/partials/dataset_version_control.html" with dataset=dataset %} + {% if not dataset.status == 'final' or not dataset.is_public %} + {% include "users/partials/publish_dataset_modal.html" with dataset=dataset %} + {% endif %} {% endif %} {% endif %} {% endfor %} diff --git a/gateway/sds_gateway/templates/users/components/dataset_list_table_row.html b/gateway/sds_gateway/templates/users/components/dataset_list_table_row.html index 33cb6aae0..6775bf3e8 100644 --- a/gateway/sds_gateway/templates/users/components/dataset_list_table_row.html +++ b/gateway/sds_gateway/templates/users/components/dataset_list_table_row.html @@ -29,6 +29,9 @@ {{ dataset.status_display }} {% endif %} {% if dataset.is_public %}{% endif %} + {% if dataset.is_federated %} + {{ dataset.site_name }} + {% endif %} {% if dataset.is_shared_with_me %} - {% if dataset.dataset.created_at %} + {% if dataset.created_at %} {% localtime on %}
- {{ dataset.dataset.created_at|date:"Y-m-d" }} - {{ dataset.dataset.created_at|date:"H:i:s T" }} + {{ dataset.created_at|date:"Y-m-d" }} + {{ dataset.created_at|date:"H:i:s T" }}
{% endlocaltime %} {% else %} - {% endif %} -{% include "users/components/dropdown_menu.html" with items=dropdown_menu_items %} + + {% if dropdown_menu_items %} + {% include "users/components/dropdown_menu.html" with items=dropdown_menu_items %} + {% endif %} + diff --git a/gateway/sds_gateway/templates/users/partials/dataset_search_form.html b/gateway/sds_gateway/templates/users/partials/dataset_search_form.html index 64a0bfd6b..2b90dbc5c 100644 --- a/gateway/sds_gateway/templates/users/partials/dataset_search_form.html +++ b/gateway/sds_gateway/templates/users/partials/dataset_search_form.html @@ -9,14 +9,21 @@

Search Published Datasets

action="{% url 'users:search_datasets' %}" class="flex-grow-1 d-flex flex-column mt-5">
-
+
{{ search_form.query }} {% if search_form.query.help_text %} {{ search_form.query.help_text }} {% endif %}
-
+
+ + {{ search_form.site_name }} + {% if search_form.site_name.help_text %} + {{ search_form.site_name.help_text }} + {% endif %} +
+
diff --git a/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html b/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html index 91b1dd4b1..5270393a9 100644 --- a/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html +++ b/gateway/sds_gateway/templates/users/partials/search_published_datasets_tab.html @@ -19,7 +19,7 @@