From e1d6e9cb5bb2ab3d52b31c6a104fded87eb0645a Mon Sep 17 00:00:00 2001 From: psaesha Date: Thu, 18 Jun 2026 00:57:59 +0530 Subject: [PATCH 01/57] schema and noofentry in prompt datasets queries --- dataspace_sdk/__version__.py | 2 +- dataspace_sdk/resources/datasets.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dataspace_sdk/__version__.py b/dataspace_sdk/__version__.py index 8d06332..a245f6e 100644 --- a/dataspace_sdk/__version__.py +++ b/dataspace_sdk/__version__.py @@ -1,3 +1,3 @@ """Version information for DataSpace SDK.""" -__version__ = "0.5.02" +__version__ = "0.5.03" diff --git a/dataspace_sdk/resources/datasets.py b/dataspace_sdk/resources/datasets.py index df6313e..3c4d822 100644 --- a/dataspace_sdk/resources/datasets.py +++ b/dataspace_sdk/resources/datasets.py @@ -363,6 +363,12 @@ def get_prompt_by_id(self, dataset_id: str) -> Dict[str, Any]: format size } + schema { + format + description + fieldName + } + noOfEntries promptDetails { promptFormat hasSystemPrompt @@ -442,12 +448,18 @@ def list_prompts( format size } + schema { + format + description + fieldName + } promptDetails { promptFormat hasSystemPrompt hasExampleResponses promptCount } + noOfEntries } } } From bb6dc73f304adfe321910b0bd170e372e53f7260 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 17 Jun 2026 19:49:36 +0000 Subject: [PATCH 02/57] Bump SDK version to 0.5.03 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d1e5fa8..9c7f9f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataspace-sdk" -version = "0.4.19" +version = "0.5.03" description = "Python SDK for DataSpace API" readme = "docs/sdk/README.md" requires-python = ">=3.8" @@ -54,7 +54,7 @@ include = '\.pyi?$' [tool.mypy] -python_version = "0.4.19" +python_version = "0.5.03" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = false From 6408941b625115ce0f4fb44fab30dc5b2150de39 Mon Sep 17 00:00:00 2001 From: psaesha Date: Wed, 24 Jun 2026 14:49:17 +0530 Subject: [PATCH 03/57] no of entries for file formats other than csv --- api/types/type_resource.py | 32 ++++++++++++++++++++++------- api/utils/data_indexing.py | 41 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/api/types/type_resource.py b/api/types/type_resource.py index 2053202..cd6d09c 100644 --- a/api/types/type_resource.py +++ b/api/types/type_resource.py @@ -20,7 +20,13 @@ from api.types.type_preview_data import PreviewData from api.types.type_prompt_resource_details import TypePromptResourceDetails from api.types.type_resource_metadata import TypeResourceMetadata -from api.utils.data_indexing import get_preview_data, get_row_count +from api.utils.data_indexing import ( + FILE_COUNT_FORMATS, + INDEXED_FORMATS, + get_entry_count_from_file, + get_preview_data, + get_row_count, +) from api.utils.graphql_telemetry import trace_resolver logger = structlog.get_logger(__name__) @@ -177,14 +183,26 @@ def no_of_entries(self) -> int: if not file_details: return 0 - if not hasattr(file_details, "format") or file_details.format.lower() != "csv": + if not hasattr(file_details, "format"): return 0 - try: - return get_row_count(self) # type: ignore - except Exception as row_count_error: - logger.error(f"Error in get_row_count: {str(row_count_error)}") - return 0 + fmt = file_details.format.lower() + + if fmt in INDEXED_FORMATS: + try: + return get_row_count(self) # type: ignore + except Exception as row_count_error: + logger.error(f"Error in get_row_count: {str(row_count_error)}") + return 0 + + if fmt in FILE_COUNT_FORMATS: + try: + return get_entry_count_from_file(self) # type: ignore + except Exception as file_count_error: + logger.error(f"Error in get_entry_count_from_file: {str(file_count_error)}") + return 0 + + return 0 except Exception as e: logger.error(f"Error getting number of entries: {str(e)}") return 0 diff --git a/api/utils/data_indexing.py b/api/utils/data_indexing.py index 8e817b8..8d56da5 100644 --- a/api/utils/data_indexing.py +++ b/api/utils/data_indexing.py @@ -16,6 +16,11 @@ # Use a separate database for data tables DATA_DB = "data_db" # This should match the connection name in settings.py +# Formats indexed into ResourceDataTable (queryable via get_row_count) +INDEXED_FORMATS = {"csv", "xls", "xlsx", "ods", "parquet", "feather", "json", "tsv"} +# Formats counted by parsing the file directly (not indexed into DB) +FILE_COUNT_FORMATS = {"yml", "yaml", "xml"} + # Allowed comparison operators for column-based filtering on indexed data. # Maps operator suffix -> (sql_template_with_{ph}_placeholder, value_transformer) _FILTER_OPERATORS: Dict[str, Tuple[str, Any]] = { @@ -397,6 +402,42 @@ def get_row_count(resource: Resource) -> int: return 0 +def get_entry_count_from_file(resource: Resource) -> int: + """Count entries in yml/yaml/xml files by parsing the file directly.""" + try: + file_details = getattr(resource, "resourcefiledetails", None) + if not file_details: + return 0 + + fmt = file_details.format.lower() + filepath = file_details.file.path + + if fmt in ("yml", "yaml"): + import yaml + + with open(filepath, "r") as f: + data = yaml.safe_load(f) + if isinstance(data, list): + return len(data) + if isinstance(data, dict): + for v in data.values(): + if isinstance(v, list): + return len(v) + return len(data) + return 0 + + if fmt == "xml": + import xml.etree.ElementTree as ET + + root = ET.parse(filepath).getroot() + return len(list(root)) + + return 0 + except Exception as e: + logger.error(f"Error counting entries from file for resource {resource.id}: {str(e)}") + return 0 + + def get_preview_data(resource: Resource) -> Optional[PreviewData]: try: if not resource.preview_enabled: From 1b9636e5f263420b94b768e2414846968cfe3462 Mon Sep 17 00:00:00 2001 From: psaesha Date: Thu, 25 Jun 2026 16:05:54 +0530 Subject: [PATCH 04/57] Update dataindex command to support multiple file formats for indexing --- api/management/commands/update_dataindex.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/management/commands/update_dataindex.py b/api/management/commands/update_dataindex.py index f7f8c14..218a4ab 100644 --- a/api/management/commands/update_dataindex.py +++ b/api/management/commands/update_dataindex.py @@ -7,7 +7,7 @@ from django.db import transaction from api.models.Resource import Resource, ResourceDataTable -from api.utils.data_indexing import index_resource_data +from api.utils.data_indexing import INDEXED_FORMATS, index_resource_data logger = structlog.get_logger("dataspace.commands.update_dataindex") @@ -104,12 +104,12 @@ def handle(self, *args: Any, **options: Dict[str, Any]) -> None: skipped_count += 1 continue - # Skip resources that aren't CSV files + # Skip resources that aren't in a supported indexed format file_details = resource.resourcefiledetails - if not file_details or not file_details.format.lower() == "csv": + if not file_details or file_details.format.lower() not in INDEXED_FORMATS: self.stdout.write( self.style.WARNING( - f"[{i}/{total_resources}] Skipping resource {resource.id} - Not a CSV file" + f"[{i}/{total_resources}] Skipping resource {resource.id} - Format not indexable" ) ) skipped_count += 1 From be949dc5a44a95ae506051b7a6c54ce6a97f1fd9 Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 25 Jun 2026 19:58:01 +0530 Subject: [PATCH 05/57] Add initialization for y_axis_columns Initialize y_axis_columns list for y-axis configurations. --- api/views/dynamic_chart_view.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/views/dynamic_chart_view.py b/api/views/dynamic_chart_view.py index e06548e..457d924 100644 --- a/api/views/dynamic_chart_view.py +++ b/api/views/dynamic_chart_view.py @@ -42,6 +42,7 @@ async def create_chart_details( ) # Handle y-axis columns with configurations + y_axis_columns = [] if y_axis_configs := request_details.get("y_axis_column", []): y_axis_columns = [] for config in y_axis_configs: From cd10ee8c96cc8f663283847688db3a476fda6b22 Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 25 Jun 2026 19:59:23 +0530 Subject: [PATCH 06/57] Update type_resource_chart.py --- api/types/type_resource_chart.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/types/type_resource_chart.py b/api/types/type_resource_chart.py index 51e4161..c4236ac 100644 --- a/api/types/type_resource_chart.py +++ b/api/types/type_resource_chart.py @@ -1,6 +1,8 @@ # mypy: disable-error-code="valid-type" import json +from pyecharts.globals import CurrentConfig +CurrentConfig.ONLINE_HOST = "file:///code/" import uuid from datetime import datetime from typing import Any, Dict, List, Optional, Type, TypedDict, TypeVar, Union, cast From e48ba8222b5c5c294d5fe3ccf61c80a69d003432 Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 25 Jun 2026 20:02:43 +0530 Subject: [PATCH 07/57] Update Dockerfile --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index e3baf67..05be1d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,8 @@ RUN apt-get update && \ libxss1 \ libxtst6 \ lsb-release \ + chromium \ + chromium-driver \ xdg-utils && \ rm -rf /var/lib/apt/lists/* @@ -49,6 +51,7 @@ COPY . /code/ RUN pip install psycopg2-binary uvicorn RUN pip install -r requirements.txt +RUN curl -s https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js -o /code/echarts.min.js # Create healthcheck script RUN echo '#!/bin/bash\nset -e\npython -c "import sys; import django; django.setup(); sys.exit(0)"' > /code/healthcheck.sh \ From 04e076ed8cee38f698134dfb64093a0cd2da8456 Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 12:27:33 +0530 Subject: [PATCH 08/57] feat(publications): add Publication data model foundation New top-level entity (internal name Publication, UI label Resource): - Publication model: UUID/slug (counter-dedupe), typed metadata columns (authors, publication_date, license reusing DatasetLicense, external_source_link), status, owner FKs, resource_type FK, sectors/ geographies M2M, download_count, query-shape indexes. - PublicationBlock: ordered file-XOR-youtube blocks with a DB CheckConstraint. - ResourceType: admin-managed flat lookup (adapted from Sector, +is_active). - PublicationStatus / PublicationBlockType enums. - UseCase/Collaborative gain a publications M2M; publications added to all four UC/Collab input exclude lists (no draft-linking bypass window). - seed_resource_types management command (idempotent). - Layer 1 model tests; test settings disable the ES signal processor so DB tests stay hermetic. --- .gitignore | 3 + .../commands/seed_resource_types.py | 50 ++++ api/models/Collaborative.py | 12 +- api/models/Publication.py | 190 +++++++++++++++ api/models/ResourceType.py | 33 +++ api/models/UseCase.py | 5 +- api/models/__init__.py | 2 + api/schema/collaborative_schema.py | 219 +++++------------- api/schema/usecase_schema.py | 4 +- api/utils/enums.py | 10 + api/utils/file_paths.py | 12 + tests/test_publication_models.py | 199 ++++++++++++++++ tests/test_settings.py | 6 + 13 files changed, 573 insertions(+), 172 deletions(-) create mode 100644 api/management/commands/seed_resource_types.py create mode 100644 api/models/Publication.py create mode 100644 api/models/ResourceType.py create mode 100644 tests/test_publication_models.py diff --git a/.gitignore b/.gitignore index 18886c7..63050b3 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,6 @@ dvc dvc/* .DS_Store + +# Git worktrees for feature branches +.worktrees/ diff --git a/api/management/commands/seed_resource_types.py b/api/management/commands/seed_resource_types.py new file mode 100644 index 0000000..333cc7f --- /dev/null +++ b/api/management/commands/seed_resource_types.py @@ -0,0 +1,50 @@ +""" +Django management command to seed the initial Resource Type lookup values. + +Idempotent — re-running only creates the types that are missing and never +duplicates or overwrites an admin's edits. Admins manage the list afterwards +(add / rename / deactivate) from the Django admin, no deploy required. + +Usage: + python manage.py seed_resource_types +""" + +from typing import Any + +from django.core.management.base import BaseCommand +from django.db import transaction + +from api.models import ResourceType + +# The starting set of Resource Types. Admin-editable after seeding. +INITIAL_RESOURCE_TYPES = [ + "Report", + "Article", + "Policy Brief", + "Research Paper", + "Case Study", + "Guide", + "Toolkit", + "Presentation", + "Fact Sheet", + "Working Paper", +] + + +class Command(BaseCommand): + help = "Seed the initial Resource Type lookup values (idempotent)" + + def handle(self, *args: Any, **options: Any) -> None: + created_count = 0 + with transaction.atomic(): + for name in INITIAL_RESOURCE_TYPES: + _, created = ResourceType.objects.get_or_create(name=name) + if created: + created_count += 1 + + self.stdout.write( + self.style.SUCCESS( + f"✓ Resource Types seeded ({created_count} created, " + f"{len(INITIAL_RESOURCE_TYPES) - created_count} already present)" + ) + ) diff --git a/api/models/Collaborative.py b/api/models/Collaborative.py index 0681aa2..c81e348 100644 --- a/api/models/Collaborative.py +++ b/api/models/Collaborative.py @@ -1,8 +1,8 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, cast -from django.db import models from django.core.validators import RegexValidator +from django.db import models from django.utils.text import slugify if TYPE_CHECKING: @@ -13,7 +13,6 @@ from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.file_paths import _use_case_directory_path - slug_validator = RegexValidator( regex=r"^[a-z0-9]+(?:-[a-z0-9]+)*$", message="Slug must be lowercase and contain only alphanumeric characters and hyphens.", @@ -47,15 +46,12 @@ class Collaborative(models.Model): choices=CollaborativeStatus.choices, ) datasets = models.ManyToManyField("api.Dataset", blank=True) + publications = models.ManyToManyField("api.Publication", blank=True) use_cases = models.ManyToManyField("api.UseCase", blank=True) tags = models.ManyToManyField("api.Tag", blank=True) - sectors = models.ManyToManyField( - "api.Sector", blank=True, related_name="collaboratives" - ) + sectors = models.ManyToManyField("api.Sector", blank=True, related_name="collaboratives") sdgs = models.ManyToManyField("api.SDG", blank=True, related_name="collaboratives") - geographies = models.ManyToManyField( - "api.Geography", blank=True, related_name="collaboratives" - ) + geographies = models.ManyToManyField("api.Geography", blank=True, related_name="collaboratives") contributors = models.ManyToManyField( "authorization.User", blank=True, related_name="contributed_collaboratives" ) diff --git a/api/models/Publication.py b/api/models/Publication.py new file mode 100644 index 0000000..b922433 --- /dev/null +++ b/api/models/Publication.py @@ -0,0 +1,190 @@ +import uuid +from typing import TYPE_CHECKING, Any + +from django.db import models +from django.db.models import Q +from django.utils.text import slugify + +from api.utils.enums import DatasetLicense, PublicationBlockType, PublicationStatus +from api.utils.file_paths import _publication_block_directory_path + +if TYPE_CHECKING: + from api.models.Organization import Organization + from api.models.ResourceType import ResourceType + from authorization.models import User + + +class Publication(models.Model): + """A top-level Resource — a container for human-authored content. + + Peer to Dataset and AI Model: a UUID/slug entity owned by an organization or + an individual user, with typed metadata columns, a publish/unpublish status, + an ordered list of content blocks, and its own search + linking surface. + (Internal name ``Publication`` to avoid colliding with the file-inside-a-dataset + ``Resource``; the UI always labels it "Resource".) + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + title = models.CharField(max_length=300, unique=False, blank=True) + description = models.TextField(blank=True, null=True) + slug = models.SlugField(max_length=255, unique=True) + + organization = models.ForeignKey( + "api.Organization", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="publications", + ) + user = models.ForeignKey( + "authorization.User", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="publications", + ) + + # Typed metadata columns — a fixed, known schema (no dynamic/EAV fields). + authors = models.JSONField(default=list, blank=True) + publication_date = models.DateField(null=True, blank=True) + license = models.CharField( + max_length=50, + default=DatasetLicense.CC_BY_4_0_ATTRIBUTION, + choices=DatasetLicense.choices, + ) + external_source_link = models.URLField(blank=True, null=True) + + # Structural / faceted columns. + status = models.CharField( + max_length=50, + default=PublicationStatus.DRAFT, + choices=PublicationStatus.choices, + ) + resource_type = models.ForeignKey( + "api.ResourceType", + on_delete=models.PROTECT, + null=True, + blank=True, + related_name="publications", + ) + sectors = models.ManyToManyField("api.Sector", blank=True, related_name="publications") + geographies = models.ManyToManyField("api.Geography", blank=True, related_name="publications") + download_count = models.IntegerField(default=0) + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def save(self, *args: Any, **kwargs: Any) -> None: + if not self.slug: + base_slug = slugify(self.title) + slug = base_slug + counter = 1 + while Publication.objects.filter(slug=slug).exclude(pk=self.pk).exists(): + slug = f"{base_slug}-{counter}" + counter += 1 + self.slug = slug + super().save(*args, **kwargs) + + @property + def is_individual_publication(self) -> bool: + """True when this Resource is owned by an individual, not an organization.""" + return self.organization is None and self.user is not None + + @property + def sectors_indexing(self) -> list[str]: + """Sector names for Elasticsearch indexing.""" + return [sector.name for sector in self.sectors.all()] # type: ignore + + @property + def geographies_indexing(self) -> list[str]: + """Geography names for Elasticsearch indexing.""" + return [geo.name for geo in self.geographies.all()] # type: ignore + + @property + def resource_type_indexing(self) -> str: + """Resource-type name for Elasticsearch indexing (empty when unset).""" + return self.resource_type.name if self.resource_type else "" + + def __str__(self) -> str: + return self.title + + class Meta: + verbose_name = "Publication" + verbose_name_plural = "Publications" + db_table = "publication" + ordering = ["-modified"] + indexes = [ + models.Index(fields=["organization", "-modified"]), + models.Index(fields=["user", "-modified"]), + models.Index(fields=["status"]), + ] + + +class PublicationBlock(models.Model): + """One content block in a Resource — a file XOR a YouTube embed, at a position. + + Blocks are ordered by ``position`` within their publication. Each block is + exactly one of two shapes, enforced by the ``file_xor_youtube`` check + constraint: a FILE block carries an uploaded file (with its name/format/size), + a YOUTUBE block carries a video url and its extracted id. Neither-both nor + neither-set is a valid row. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + publication = models.ForeignKey( + "api.Publication", + on_delete=models.CASCADE, + related_name="blocks", + ) + position = models.PositiveIntegerField(default=0) + block_type = models.CharField( + max_length=20, + choices=PublicationBlockType.choices, + ) + + # FILE block fields. + file = models.FileField( + upload_to=_publication_block_directory_path, + max_length=300, + blank=True, + ) + file_name = models.CharField(max_length=300, blank=True) + file_format = models.CharField(max_length=50, blank=True) + file_size = models.BigIntegerField(null=True, blank=True) + + # YOUTUBE block fields. + youtube_url = models.URLField(blank=True, null=True) + youtube_video_id = models.CharField(max_length=20, blank=True) + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def __str__(self) -> str: + return f"{self.block_type} block #{self.position} of {self.publication_id}" + + class Meta: + db_table = "publication_block" + ordering = ["position"] + indexes = [ + models.Index(fields=["publication", "position"]), + ] + constraints = [ + # Runtime is Django 5.0 (``check=``); the pinned django-stubs is + # newer and only knows the 5.1 ``condition=`` spelling, hence the + # ignore. Switch to ``condition=`` when the runtime moves to 5.1+. + models.CheckConstraint( # type: ignore[call-arg] + name="publicationblock_file_xor_youtube", + check=( + Q( + block_type=PublicationBlockType.FILE, + youtube_url__isnull=True, + ) + & ~Q(file="") + ) + | Q( + block_type=PublicationBlockType.YOUTUBE, + file="", + youtube_url__isnull=False, + ), + ) + ] diff --git a/api/models/ResourceType.py b/api/models/ResourceType.py new file mode 100644 index 0000000..9f7aac1 --- /dev/null +++ b/api/models/ResourceType.py @@ -0,0 +1,33 @@ +import uuid +from typing import Any + +from django.db import models +from django.utils.text import slugify + + +class ResourceType(models.Model): + """Admin-managed lookup for a Resource's type (Report, Article, Policy Brief, ...). + + A flat, non-hierarchical list — no parent self-FK. Admins can add, rename, + or deactivate a type without a code release. Deactivating (``is_active=False``) + keeps historical references intact while hiding the type from new selections. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=75, unique=True, null=False, blank=False) + description = models.CharField(max_length=1000, null=True, blank=True) + slug = models.SlugField(max_length=75, null=True, blank=False, unique=True) + is_active = models.BooleanField(default=True) + + def save(self, *args: Any, **kwargs: Any) -> None: + self.slug = slugify(self.name) + super().save(*args, **kwargs) + + def __str__(self) -> str: + return str(self.name) + + class Meta: + db_table = "resource_type" + verbose_name = "Resource Type" + verbose_name_plural = "Resource Types" + ordering = ["name"] diff --git a/api/models/UseCase.py b/api/models/UseCase.py index 5e7295e..09a2b6a 100644 --- a/api/models/UseCase.py +++ b/api/models/UseCase.py @@ -37,6 +37,7 @@ class UseCase(models.Model): max_length=50, default=UseCaseStatus.DRAFT, choices=UseCaseStatus.choices ) datasets = models.ManyToManyField("api.Dataset", blank=True) + publications = models.ManyToManyField("api.Publication", blank=True) tags = models.ManyToManyField("api.Tag", blank=True) running_status = models.CharField( max_length=50, @@ -45,9 +46,7 @@ class UseCase(models.Model): ) sectors = models.ManyToManyField("api.Sector", blank=True, related_name="usecases") sdgs = models.ManyToManyField("api.SDG", blank=True, related_name="usecases") - geographies = models.ManyToManyField( - "api.Geography", blank=True, related_name="usecases" - ) + geographies = models.ManyToManyField("api.Geography", blank=True, related_name="usecases") contributors = models.ManyToManyField( "authorization.User", blank=True, related_name="contributed_usecases" ) diff --git a/api/models/__init__.py b/api/models/__init__.py index 14ac679..6c54b34 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -15,6 +15,7 @@ from api.models.Organization import Organization from api.models.PromptDataset import PromptDataset from api.models.PromptResource import PromptResource +from api.models.Publication import Publication, PublicationBlock from api.models.Resource import ( Resource, ResourceDataTable, @@ -26,6 +27,7 @@ from api.models.ResourceChartImage import ResourceChartImage from api.models.ResourceMetadata import ResourceMetadata from api.models.ResourceSchema import ResourceSchema +from api.models.ResourceType import ResourceType from api.models.SDG import SDG from api.models.Sector import Sector from api.models.SerializableJSONField import SerializableJSONField diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index 3c1213d..3e0f5b9 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -47,7 +47,9 @@ from authorization.types import TypeUser -@strawberry_django.input(Collaborative, fields="__all__", exclude=["datasets", "slug"]) +@strawberry_django.input( + Collaborative, fields="__all__", exclude=["datasets", "publications", "slug"] +) class CollaborativeInput: """Input type for collaborative creation.""" @@ -70,7 +72,7 @@ class UpdateCollaborativeMetadataInput: geographies: Optional[List[int]] -@strawberry_django.partial(Collaborative, fields="__all__", exclude=["datasets"]) +@strawberry_django.partial(Collaborative, fields="__all__", exclude=["datasets", "publications"]) class CollaborativeInputPartial: """Input type for collaborative updates.""" @@ -99,9 +101,7 @@ class Query: pagination=True, order=CollaborativeOrder, ) - @trace_resolver( - name="get_collaboratives", attributes={"component": "collaborative"} - ) + @trace_resolver(name="get_collaboratives", attributes={"component": "collaborative"}) def collaboratives( self, info: Info, @@ -119,9 +119,7 @@ def collaboratives( elif user.is_authenticated: queryset = Collaborative.objects.filter(user=user) else: - queryset = Collaborative.objects.filter( - status=CollaborativeStatus.PUBLISHED - ) + queryset = Collaborative.objects.filter(status=CollaborativeStatus.PUBLISHED) if filters is not strawberry.UNSET: queryset = strawberry_django.filters.apply(filters, queryset, info) @@ -136,9 +134,7 @@ def collaboratives( return TypeCollaborative.from_django_list(queryset) @strawberry_django.field - @trace_resolver( - name="get_published_collaboratives", attributes={"component": "collaborative"} - ) + @trace_resolver(name="get_published_collaboratives", attributes={"component": "collaborative"}) def published_collaboratives( self, info: Info, @@ -173,12 +169,8 @@ def published_collaboratives( return TypeCollaborative.from_django_list(results) @strawberry_django.field - @trace_resolver( - name="get_datasets_by_collaborative", attributes={"component": "collaborative"} - ) - def dataset_by_collaborative( - self, info: Info, collaborative_id: str - ) -> list[TypeDataset]: + @trace_resolver(name="get_datasets_by_collaborative", attributes={"component": "collaborative"}) + def dataset_by_collaborative(self, info: Info, collaborative_id: str) -> list[TypeDataset]: """Get datasets by collaborative.""" queryset = Dataset.objects.filter(collaborative__id=collaborative_id) return TypeDataset.from_django_list(queryset) @@ -188,18 +180,14 @@ def dataset_by_collaborative( name="get_contributors_by_collaborative", attributes={"component": "collaborative"}, ) - def contributors_by_collaborative( - self, info: Info, collaborative_id: str - ) -> list[TypeUser]: + def contributors_by_collaborative(self, info: Info, collaborative_id: str) -> list[TypeUser]: """Get contributors by collaborative.""" try: collaborative = Collaborative.objects.get(id=collaborative_id) contributors = collaborative.contributors.all() return TypeUser.from_django_list(contributors) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") @strawberry_django.field @trace_resolver( @@ -214,9 +202,7 @@ def collaborative_by_slug(self, info: Info, slug: str) -> TypeCollaborative: raise ValueError(f"Collaborative with slug {slug} does not exist.") -@trace_resolver( - name="update_collaborative_tags", attributes={"component": "collaborative"} -) +@trace_resolver(name="update_collaborative_tags", attributes={"component": "collaborative"}) def _update_collaborative_tags(collaborative: Collaborative, tags: List[str]) -> None: collaborative.tags.clear() for tag in tags: @@ -226,24 +212,16 @@ def _update_collaborative_tags(collaborative: Collaborative, tags: List[str]) -> collaborative.save() -@trace_resolver( - name="update_collaborative_sectors", attributes={"component": "collaborative"} -) -def _update_collaborative_sectors( - collaborative: Collaborative, sectors: List[uuid.UUID] -) -> None: +@trace_resolver(name="update_collaborative_sectors", attributes={"component": "collaborative"}) +def _update_collaborative_sectors(collaborative: Collaborative, sectors: List[uuid.UUID]) -> None: sectors_objs = Sector.objects.filter(id__in=sectors) collaborative.sectors.clear() collaborative.sectors.add(*sectors_objs) collaborative.save() -@trace_resolver( - name="update_collaborative_sdgs", attributes={"component": "collaborative"} -) -def _update_collaborative_sdgs( - collaborative: Collaborative, sdgs: List[uuid.UUID] -) -> None: +@trace_resolver(name="update_collaborative_sdgs", attributes={"component": "collaborative"}) +def _update_collaborative_sdgs(collaborative: Collaborative, sdgs: List[uuid.UUID]) -> None: sdgs_objs = SDG.objects.filter(id__in=sdgs) collaborative.sdgs.clear() collaborative.sdgs.add(*sdgs_objs) @@ -265,9 +243,7 @@ def _add_update_collaborative_metadata( metadata_field = Metadata.objects.get(id=metadata_input_item.id) if not metadata_field.enabled: _delete_existing_metadata(collaborative) - raise ValueError( - f"Metadata with ID {metadata_input_item.id} is not enabled." - ) + raise ValueError(f"Metadata with ID {metadata_input_item.id} is not enabled.") uc_metadata = CollaborativeMetadata( collaborative=collaborative, metadata_item=metadata_field, @@ -276,19 +252,13 @@ def _add_update_collaborative_metadata( uc_metadata.save() except Metadata.DoesNotExist: _delete_existing_metadata(collaborative) - raise ValueError( - f"Metadata with ID {metadata_input_item.id} does not exist." - ) + raise ValueError(f"Metadata with ID {metadata_input_item.id} does not exist.") -@trace_resolver( - name="delete_existing_metadata", attributes={"component": "collaborative"} -) +@trace_resolver(name="delete_existing_metadata", attributes={"component": "collaborative"}) def _delete_existing_metadata(collaborative: Collaborative) -> None: try: - existing_metadata = CollaborativeMetadata.objects.filter( - collaborative=collaborative - ) + existing_metadata = CollaborativeMetadata.objects.filter(collaborative=collaborative) existing_metadata.delete() except CollaborativeMetadata.DoesNotExist: pass @@ -357,10 +327,7 @@ def add_collaborative(self, info: Info) -> TypeCollaborative: else None ), "sectors": ( - [ - str(sector_id) - for sector_id in update_metadata_input.sectors - ] + [str(sector_id) for sector_id in update_metadata_input.sectors] if update_metadata_input.sectors else [] ), @@ -381,14 +348,10 @@ def add_update_collaborative_metadata( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") if update_metadata_input.tags is not None: _update_collaborative_tags(collaborative, update_metadata_input.tags) @@ -397,9 +360,7 @@ def add_update_collaborative_metadata( if update_metadata_input.sdgs is not None: _update_collaborative_sdgs(collaborative, update_metadata_input.sdgs) if update_metadata_input.geographies is not None: - _update_collaborative_geographies( - collaborative, update_metadata_input.geographies - ) + _update_collaborative_geographies(collaborative, update_metadata_input.geographies) return TypeCollaborative.from_django(collaborative) @strawberry_django.mutation(handle_django_errors=False) @@ -414,14 +375,10 @@ def update_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") if data.title is not None: if data.title.strip() == "": @@ -466,9 +423,7 @@ def delete_collaborative(self, info: Info, collaborative_id: str) -> bool: try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") collaborative.delete() return True @@ -485,14 +440,10 @@ def add_dataset_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.datasets.add(dataset) collaborative.save() @@ -511,14 +462,10 @@ def add_usecase_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.use_cases.add(usecase) collaborative.save() @@ -536,14 +483,10 @@ def remove_dataset_from_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.datasets.remove(dataset) collaborative.save() return TypeCollaborative.from_django(collaborative) @@ -560,14 +503,10 @@ def remove_usecase_from_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.use_cases.remove(usecase) collaborative.save() return TypeCollaborative.from_django(collaborative) @@ -588,9 +527,7 @@ def update_collaborative_datasets( raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.datasets.set(datasets) collaborative.save() @@ -612,9 +549,7 @@ def update_collaborative_use_cases( raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.use_cases.set(use_cases) collaborative.save() @@ -636,9 +571,7 @@ def update_collaborative_use_cases( name="publish_collaborative", attributes={"component": "collaborative", "operation": "mutation"}, ) - def publish_collaborative( - self, info: Info, collaborative_id: str - ) -> TypeCollaborative: + def publish_collaborative(self, info: Info, collaborative_id: str) -> TypeCollaborative: """Publish a collaborative.""" try: collaborative = Collaborative.objects.get(id=collaborative_id) @@ -665,9 +598,7 @@ def publish_collaborative( name="unpublish_collaborative", attributes={"component": "collaborative", "operation": "mutation"}, ) - def unpublish_collaborative( - self, info: Info, collaborative_id: str - ) -> TypeCollaborative: + def unpublish_collaborative(self, info: Info, collaborative_id: str) -> TypeCollaborative: """Un-publish a collaborative.""" try: collaborative = Collaborative.objects.get(id=collaborative_id) @@ -696,14 +627,10 @@ def add_contributor_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.contributors.add(user) collaborative.save() @@ -727,14 +654,10 @@ def remove_contributor_from_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.contributors.remove(user) collaborative.save() @@ -749,9 +672,7 @@ def remove_contributor_from_collaborative( get_data=lambda result, collaborative_id, user_ids, **kwargs: { "collaborative_id": collaborative_id, "collaborative_title": result.title, - "updated_fields": { - "contributors": [str(user_id) for user_id in user_ids] - }, + "updated_fields": {"contributors": [str(user_id) for user_id in user_ids]}, }, ) ], @@ -771,9 +692,7 @@ def update_collaborative_contributors( raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.contributors.set(users) collaborative.save() @@ -797,22 +716,16 @@ def add_supporting_organization_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") # Create or get the relationship - relationship, created = ( - CollaborativeOrganizationRelationship.objects.get_or_create( - collaborative=collaborative, - organization=organization, - relationship_type=OrganizationRelationshipType.SUPPORTER, - ) + relationship, created = CollaborativeOrganizationRelationship.objects.get_or_create( + collaborative=collaborative, + organization=organization, + relationship_type=OrganizationRelationshipType.SUPPORTER, ) return TypeCollaborativeOrganizationRelationship.from_django(relationship) @@ -858,22 +771,16 @@ def add_partner_organization_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") # Create or get the relationship - relationship, created = ( - CollaborativeOrganizationRelationship.objects.get_or_create( - collaborative=collaborative, - organization=organization, - relationship_type=OrganizationRelationshipType.PARTNER, - ) + relationship, created = CollaborativeOrganizationRelationship.objects.get_or_create( + collaborative=collaborative, + organization=organization, + relationship_type=OrganizationRelationshipType.PARTNER, ) return TypeCollaborativeOrganizationRelationship.from_django(relationship) @@ -937,19 +844,13 @@ def update_collaborative_organization_relationships( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") # Clear existing relationships - CollaborativeOrganizationRelationship.objects.filter( - collaborative=collaborative - ).delete() + CollaborativeOrganizationRelationship.objects.filter(collaborative=collaborative).delete() # Add supporter organizations supporter_orgs = Organization.objects.filter(id__in=supporter_organization_ids) diff --git a/api/schema/usecase_schema.py b/api/schema/usecase_schema.py index a4c905f..317e0c2 100644 --- a/api/schema/usecase_schema.py +++ b/api/schema/usecase_schema.py @@ -46,7 +46,7 @@ from authorization.types import TypeUser -@strawberry_django.input(UseCase, fields="__all__", exclude=["datasets", "slug"]) +@strawberry_django.input(UseCase, fields="__all__", exclude=["datasets", "publications", "slug"]) class UseCaseInput: """Input type for use case creation.""" @@ -72,7 +72,7 @@ class UpdateUseCaseMetadataInput: use_case_running_status = strawberry.enum(UseCaseRunningStatus) # type: ignore -@strawberry_django.partial(UseCase, fields="__all__", exclude=["datasets"]) +@strawberry_django.partial(UseCase, fields="__all__", exclude=["datasets", "publications"]) class UseCaseInputPartial: """Input type for use case updates.""" diff --git a/api/utils/enums.py b/api/utils/enums.py index 826cd77..646749f 100644 --- a/api/utils/enums.py +++ b/api/utils/enums.py @@ -192,6 +192,16 @@ class UseCaseStatus(models.TextChoices): ARCHIVED = "ARCHIVED" +class PublicationStatus(models.TextChoices): + DRAFT = "DRAFT" + PUBLISHED = "PUBLISHED" + + +class PublicationBlockType(models.TextChoices): + FILE = "FILE" + YOUTUBE = "YOUTUBE" + + class CollaborativeStatus(models.TextChoices): DRAFT = "DRAFT" PUBLISHED = "PUBLISHED" diff --git a/api/utils/file_paths.py b/api/utils/file_paths.py index 9f15678..aa1ecaa 100644 --- a/api/utils/file_paths.py +++ b/api/utils/file_paths.py @@ -75,3 +75,15 @@ def _catalog_directory_path(catalog: Any, filename: str) -> str: catalog_name = catalog.name _, extension = os.path.splitext(filename) return f"files/catalog/{catalog_name}/{extension[1:]}/{filename}" + + +def _publication_block_directory_path(block: Any, filename: str) -> str: + """ + Create a directory path to store a publication content-block file. + + Files are namespaced under the parent publication's id so a draft's + files live in their own folder and are easy to clean up on delete. + """ + publication_id = block.publication_id + unique_name = f"{uuid.uuid4().hex}_{filename}" + return f"files/public/publications/{publication_id}/{unique_name}" diff --git a/tests/test_publication_models.py b/tests/test_publication_models.py new file mode 100644 index 0000000..b3430f1 --- /dev/null +++ b/tests/test_publication_models.py @@ -0,0 +1,199 @@ +"""Layer 1 DB tests for the Publication data model foundation. + +Covers slug dedupe, ownership, the ResourceType lookup, the file-XOR-youtube +block constraint, block ordering, delete cascade, and the seed command. +""" + +import pytest +from django.core.management import call_command +from django.db import IntegrityError, transaction + +from api.models import Publication, PublicationBlock, ResourceType +from api.models.Organization import Organization +from api.utils.enums import PublicationBlockType, PublicationStatus +from authorization.models import User + + +@pytest.fixture +def user(db): + return User.objects.create(username="alice", keycloak_id="kc-alice") + + +@pytest.fixture +def org(db): + return Organization.objects.create(name="Org A", description="an org", slug="org-a") + + +@pytest.mark.django_db +class TestPublicationSlug: + def test_two_same_title_publications_get_distinct_slugs(self, user): + first = Publication.objects.create(title="Annual Report", user=user) + second = Publication.objects.create(title="Annual Report", user=user) + + assert first.slug == "annual-report" + assert second.slug == "annual-report-1" + assert first.slug != second.slug + + def test_a_unicode_title_slugs_without_crashing(self, user): + publication = Publication.objects.create(title="Report — Résumé 2024", user=user) + + assert publication.slug + assert Publication.objects.filter(slug=publication.slug).count() == 1 + + +@pytest.mark.django_db +class TestPublicationOwnership: + def test_a_user_owned_publication_reports_individual(self, user): + publication = Publication.objects.create(title="Solo work", user=user) + + assert publication.is_individual_publication is True + + def test_an_org_owned_publication_is_not_individual(self, org, user): + publication = Publication.objects.create(title="Org work", organization=org, user=user) + + assert publication.is_individual_publication is False + + +@pytest.mark.django_db +class TestResourceType: + def test_name_is_unique(self): + ResourceType.objects.create(name="Report") + with transaction.atomic(), pytest.raises(IntegrityError): + ResourceType.objects.create(name="Report") + + def test_slugifies_the_name(self): + resource_type = ResourceType.objects.create(name="Policy Brief") + + assert resource_type.slug == "policy-brief" + + def test_is_active_defaults_true(self): + resource_type = ResourceType.objects.create(name="Report") + + assert resource_type.is_active is True + + def test_active_query_returns_only_active_types(self): + ResourceType.objects.create(name="Report") + ResourceType.objects.create(name="Retired", is_active=False) + + active = ResourceType.objects.filter(is_active=True) + + assert active.count() == 1 + assert active.first().name == "Report" + + +@pytest.mark.django_db +class TestPublicationBlock: + def _publication(self, user): + return Publication.objects.create(title="With blocks", user=user) + + def test_file_block_stores_file_fields(self, user): + publication = self._publication(user) + + block = PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.FILE, + file="publications/report.pdf", + file_name="report.pdf", + file_format="pdf", + file_size=1024, + ) + + assert block.file_name == "report.pdf" + assert block.file_format == "pdf" + assert block.youtube_url is None + + def test_youtube_block_stores_youtube_fields(self, user): + publication = self._publication(user) + + block = PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/dQw4w9WgXcQ", + youtube_video_id="dQw4w9WgXcQ", + ) + + assert block.youtube_video_id == "dQw4w9WgXcQ" + assert block.file == "" + + def test_a_block_with_both_file_and_youtube_is_rejected(self, user): + publication = self._publication(user) + + with transaction.atomic(), pytest.raises(IntegrityError): + PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.FILE, + file="publications/report.pdf", + youtube_url="https://youtu.be/dQw4w9WgXcQ", + ) + + def test_a_block_with_neither_file_nor_youtube_is_rejected(self, user): + publication = self._publication(user) + + with transaction.atomic(), pytest.raises(IntegrityError): + PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.FILE, + ) + + def test_blocks_read_back_in_position_order(self, user): + publication = self._publication(user) + PublicationBlock.objects.create( + publication=publication, + position=2, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/two", + ) + PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/zero", + ) + PublicationBlock.objects.create( + publication=publication, + position=1, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/one", + ) + + positions = list(publication.blocks.values_list("position", flat=True)) + + assert positions == [0, 1, 2] + + def test_deleting_a_publication_cascades_its_blocks(self, user): + publication = self._publication(user) + PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/zero", + ) + + publication.delete() + + assert PublicationBlock.objects.count() == 0 + + +@pytest.mark.django_db +class TestSeedResourceTypes: + def test_seed_creates_ten_types_idempotently(self): + call_command("seed_resource_types") + assert ResourceType.objects.count() == 10 + + # Running again must not duplicate. + call_command("seed_resource_types") + assert ResourceType.objects.count() == 10 + + +@pytest.mark.django_db +class TestPublicationDefaults: + def test_new_publication_defaults_to_draft(self, user): + publication = Publication.objects.create(title="Draft one", user=user) + + assert publication.status == PublicationStatus.DRAFT + assert publication.download_count == 0 + assert publication.authors == [] diff --git a/tests/test_settings.py b/tests/test_settings.py index 751a62a..6a64185 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -71,6 +71,12 @@ def __getitem__(self, item): "default": {"hosts": "localhost:9200"}, } +# Deterministic layers must never reach a real cluster. The default real-time +# signal processor tries to push to Elasticsearch whenever a model (or a +# related model, e.g. User/Geography) is saved, which errors without a live +# cluster. Swap in the no-op processor so DB tests stay hermetic. +ELASTICSEARCH_DSL_SIGNAL_PROCESSOR = "django_elasticsearch_dsl.signals.BaseSignalProcessor" + # Disable real Keycloak calls in tests. KEYCLOAK_SERVER_URL = "http://localhost:8080" KEYCLOAK_REALM = "test" From 011696a1879e1e807d0e5b3d1736957fe7bc6db4 Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 12:46:22 +0530 Subject: [PATCH 09/57] feat(publications): add GraphQL CRUD, permissions and listing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - type_publication.py: TypePublication/Block/ResourceType with fields enumerated (never __all__), plus filter/order. - Publication permission classes in authorization/permissions.py: create, change, delete, publish (name-based roles) and AllowPublishedPublications read gate — centralized, keyed on publication_id, individual-owner branch kept, no share-model fallback. - publication_service.py: metadata validation, create/update/status helpers, org/owner/anonymous scoping with include_public union, server-enforced pagination bounds. - publication_schema.py flow: create/update/publish/unpublish/delete + status-gated list/detail via MutationResponse envelope; registered in schema. - Layer 3/4 tests: CRUD, role gating, publish, cross-org denial, draft read-gating, org-scoped and anonymous listings. --- api/schema/publication_schema.py | 275 +++++++++++++++ api/schema/schema.py | 3 + api/services/publication_service.py | 254 ++++++++++++++ api/types/type_publication.py | 122 +++++++ authorization/permissions.py | 176 +++++++++- tests/schema/test_publication_schema.py | 428 ++++++++++++++++++++++++ 6 files changed, 1248 insertions(+), 10 deletions(-) create mode 100644 api/schema/publication_schema.py create mode 100644 api/services/publication_service.py create mode 100644 api/types/type_publication.py create mode 100644 tests/schema/test_publication_schema.py diff --git a/api/schema/publication_schema.py b/api/schema/publication_schema.py new file mode 100644 index 0000000..0cd4f50 --- /dev/null +++ b/api/schema/publication_schema.py @@ -0,0 +1,275 @@ +"""Schema definitions for publications (UI "Resource"). + +Docs: ./publication_architecture.md + +This is a flow file: each resolver reads as a short sequence of named helper +calls. The metadata validation, row writes, scoping and pagination bounds all +live in ``api/services/publication_service.py``; permission/role logic lives in +``authorization/permissions.py``. Nothing here reaches into the ORM or business +rules directly. +""" + +import datetime +import uuid +from typing import List, Optional + +import strawberry +import strawberry_django +from django.core.exceptions import ValidationError as DjangoValidationError +from strawberry.types import Info + +from api.models import Publication +from api.schema.base_mutation import BaseMutation, MutationResponse +from api.services.publication_service import ( + apply_publication_update, + create_publication, + get_scoped_publications, + resolve_pagination, + set_publication_status, + validate_publication_metadata, +) +from api.types.type_publication import ( + PublicationFilter, + PublicationOrder, + TypePublication, + publication_license, +) +from api.utils.enums import PublicationStatus +from api.utils.graphql_telemetry import trace_resolver +from authorization.permissions import ( + AllowPublishedPublications, + ChangePublicationPermission, + CreatePublicationPermission, + DeletePublicationPermission, + PublishPublicationPermission, +) + + +@strawberry.input +class CreatePublicationInput: + """Metadata for a new resource. All fields validated at the boundary.""" + + title: str + description: str + authors: List[str] + publication_date: datetime.date + license: publication_license + resource_type_id: uuid.UUID + sector_ids: List[uuid.UUID] + geography_ids: List[int] + external_source_link: Optional[str] = None + + +@strawberry.input +class UpdatePublicationInput: + """Partial edit to a resource — only the fields provided are touched.""" + + id: uuid.UUID + title: Optional[str] = None + description: Optional[str] = None + authors: Optional[List[str]] = None + publication_date: Optional[datetime.date] = None + license: Optional[publication_license] = None + resource_type_id: Optional[uuid.UUID] = None + sector_ids: Optional[List[uuid.UUID]] = None + geography_ids: Optional[List[int]] = None + external_source_link: Optional[str] = None + + +@strawberry.type(name="Query") +class Query: + """Queries for publications.""" + + @strawberry.field( + permission_classes=[AllowPublishedPublications], # type: ignore[list-item] + ) + @trace_resolver(name="get_publication", attributes={"component": "publication"}) + def get_publication(self, info: Info, publication_id: uuid.UUID) -> Optional[TypePublication]: + """Get a single resource by id (drafts gated to owner/org by the permission).""" + try: + return TypePublication.from_django(Publication.objects.get(id=publication_id)) + except Publication.DoesNotExist: + return None + + @strawberry.field + @trace_resolver(name="get_publications", attributes={"component": "publication"}) + def publications( + self, + info: Info, + filters: Optional[PublicationFilter] = strawberry.UNSET, + pagination: Optional[strawberry_django.pagination.OffsetPaginationInput] = strawberry.UNSET, + order: Optional[PublicationOrder] = strawberry.UNSET, + include_public: Optional[bool] = False, + ) -> List[TypePublication]: + """List resources, scoped to the caller and paginated with server limits.""" + user = info.context.user + organization = info.context.context.get("organization") + + # Scope to org / owner / anonymous and optionally union in the public set. + queryset = get_scoped_publications( + user=user, organization=organization, include_public=bool(include_public) + ) + + # Apply client filters and ordering, then enforce a bounded page window. + if filters is not strawberry.UNSET: + queryset = strawberry_django.filters.apply(filters, queryset, info) + if order is not strawberry.UNSET: + queryset = strawberry_django.ordering.apply(order, queryset, info) + + offset, limit = resolve_pagination( + getattr(pagination, "offset", None) if pagination is not strawberry.UNSET else None, + getattr(pagination, "limit", None) if pagination is not strawberry.UNSET else None, + ) + return TypePublication.from_django_list(queryset[offset : offset + limit]) + + +@strawberry.type(name="Mutation") +class Mutation: + """Mutations for publications.""" + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[CreatePublicationPermission], + trace_name="create_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "created", + "get_data": lambda result, **kwargs: {"publication_id": str(result.id)}, + }, + ) + def create_publication( + self, info: Info, input: CreatePublicationInput + ) -> MutationResponse[TypePublication]: + """Create a DRAFT resource owned by the caller's org or the user.""" + user = info.context.user + organization = info.context.context.get("organization") + + # Reject missing/invalid metadata before any row is written. + resource_type = validate_publication_metadata( + title=input.title, + description=input.description, + authors=input.authors, + publication_date=input.publication_date, + license_value=input.license.value if input.license else None, + resource_type_id=input.resource_type_id, + sector_ids=input.sector_ids, + geography_ids=input.geography_ids, + external_source_link=input.external_source_link, + ) + + # Create the draft and wire its sector/geography tags. + publication = create_publication( + user=user, + organization=organization, + title=input.title, + description=input.description, + authors=input.authors, + publication_date=input.publication_date, + license_value=input.license.value, + resource_type=resource_type, + sector_ids=input.sector_ids, + geography_ids=input.geography_ids, + external_source_link=input.external_source_link, + ) + return MutationResponse.success_response(TypePublication.from_django(publication)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="update_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "updated", + "get_data": lambda result, **kwargs: {"publication_id": str(result.id)}, + }, + ) + def update_publication( + self, info: Info, input: UpdatePublicationInput + ) -> MutationResponse[TypePublication]: + """Apply a partial metadata edit to an existing resource.""" + # Load the target, or surface a clean not-found. + publication = _get_publication_or_raise(input.id) + + # Update only the provided fields, validating each. + publication = apply_publication_update( + publication, + title=input.title, + description=input.description, + authors=input.authors, + publication_date=input.publication_date, + license_value=input.license.value if input.license else None, + resource_type_id=input.resource_type_id, + sector_ids=input.sector_ids, + geography_ids=input.geography_ids, + external_source_link=input.external_source_link, + ) + return MutationResponse.success_response(TypePublication.from_django(publication)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[PublishPublicationPermission], + trace_name="publish_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "published", + "get_data": lambda result, **kwargs: {"publication_id": str(result.id)}, + }, + ) + def publish_publication( + self, info: Info, publication_id: uuid.UUID + ) -> MutationResponse[TypePublication]: + """Flip a resource to PUBLISHED (self-serve, no moderation).""" + publication = _get_publication_or_raise(publication_id) + + # Mark it published — the index signal picks up the visibility change. + publication = set_publication_status(publication, PublicationStatus.PUBLISHED) + return MutationResponse.success_response(TypePublication.from_django(publication)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[PublishPublicationPermission], + trace_name="unpublish_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "unpublished", + "get_data": lambda result, **kwargs: {"publication_id": str(result.id)}, + }, + ) + def unpublish_publication( + self, info: Info, publication_id: uuid.UUID + ) -> MutationResponse[TypePublication]: + """Flip a resource back to DRAFT — hidden from public reads, links untouched.""" + publication = _get_publication_or_raise(publication_id) + + # Back to draft; the render-time filters do the hiding, no links change. + publication = set_publication_status(publication, PublicationStatus.DRAFT) + return MutationResponse.success_response(TypePublication.from_django(publication)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[DeletePublicationPermission], + trace_name="delete_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "deleted", + "get_data": lambda result, **kwargs: { + "publication_id": str(kwargs.get("publication_id")), + "success": result, + }, + }, + ) + def delete_publication(self, info: Info, publication_id: uuid.UUID) -> MutationResponse[bool]: + """Hard-delete a resource — cascades its blocks, clears its links.""" + publication = _get_publication_or_raise(publication_id) + + # FK cascade drops blocks; M2M join rows auto-clear on delete. + publication.delete() + return MutationResponse.success_response(True) + + +def _get_publication_or_raise(publication_id: uuid.UUID) -> Publication: + """Load a publication by id or raise a clean validation error.""" + try: + return Publication.objects.get(id=publication_id) + except Publication.DoesNotExist: + raise DjangoValidationError(f"Resource with id {publication_id} does not exist.") diff --git a/api/schema/schema.py b/api/schema/schema.py index 70620ba..4678b63 100644 --- a/api/schema/schema.py +++ b/api/schema/schema.py @@ -16,6 +16,7 @@ import api.schema.metadata_schema import api.schema.organization_data_schema import api.schema.organization_schema +import api.schema.publication_schema import api.schema.resource_chart_schema import api.schema.resource_schema import api.schema.resoure_chart_image_schema @@ -75,6 +76,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.resoure_chart_image_schema.Query, api.schema.user_schema.Query, api.schema.collaborative_schema.Query, + api.schema.publication_schema.Query, AuthQuery, ), ) @@ -97,6 +99,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.resoure_chart_image_schema.Mutation, api.schema.tags_schema.Mutation, api.schema.collaborative_schema.Mutation, + api.schema.publication_schema.Mutation, AuthMutation, ), ) diff --git a/api/services/publication_service.py b/api/services/publication_service.py new file mode 100644 index 0000000..713471b --- /dev/null +++ b/api/services/publication_service.py @@ -0,0 +1,254 @@ +""" +publication_service +──────────────────── +Domain helpers for the Publication ("Resource") CRUD flow — the messy 90% the +``publication_schema`` flow file delegates to. Each function does one thing: +validate the metadata at the input boundary, create/update the row, flip its +publish status, or return the correctly-scoped queryset for a listing. + +These never talk to GraphQL types or permissions — they take plain values and +model instances, so they're unit-testable on their own. +""" + +from typing import Any, List, Optional + +from django.core.exceptions import ValidationError as DjangoValidationError +from django.core.validators import URLValidator +from django.db.models import QuerySet + +from api.models import Geography, Publication, ResourceType, Sector +from api.utils.enums import DatasetLicense, PublicationStatus + +# Default page size + hard ceiling for a publications listing, enforced even +# when the caller sends no pagination input. +DEFAULT_PAGE_SIZE = 20 +MAX_PAGE_SIZE = 100 + + +def validate_publication_metadata( + *, + title: Optional[str], + description: Optional[str], + authors: Optional[List[str]], + publication_date: Any, + license_value: Optional[str], + resource_type_id: Any, + sector_ids: Optional[List[Any]], + geography_ids: Optional[List[Any]], + external_source_link: Optional[str], +) -> ResourceType: + """Validate a resource's metadata at the create boundary. + + Enforces the required fields (title, description, authors, publication_date, + license, an active resource type, at least one sector and one geography), + the controlled license vocabulary, and the optional external link's URL + shape. Raises a field-keyed ``ValidationError`` on any problem and returns + the resolved active ``ResourceType`` on success. + """ + errors: dict[str, List[str]] = {} + + if not title or not title.strip(): + errors["title"] = ["Title is required."] + if not description or not description.strip(): + errors["description"] = ["Description is required."] + if not authors or not [a for a in authors if a and a.strip()]: + errors["authors"] = ["At least one author is required."] + if publication_date is None: + errors["publication_date"] = ["Publication date is required."] + if not sector_ids: + errors["sectors"] = ["At least one sector is required."] + if not geography_ids: + errors["geographies"] = ["At least one geography is required."] + + if not license_value: + errors["license"] = ["License is required."] + elif license_value not in DatasetLicense.values: + errors["license"] = ["Not a valid license."] + + if external_source_link: + try: + URLValidator()(external_source_link) + except DjangoValidationError: + errors["external_source_link"] = ["Enter a valid URL."] + + resource_type = _resolve_active_resource_type(resource_type_id, errors) + + if errors: + raise DjangoValidationError(errors) + + return resource_type # type: ignore[return-value] + + +def _resolve_active_resource_type( + resource_type_id: Any, errors: dict[str, List[str]] +) -> Optional[ResourceType]: + """Load the resource type and require it to exist and be active.""" + if not resource_type_id: + errors["resource_type"] = ["Resource type is required."] + return None + try: + resource_type = ResourceType.objects.get(id=resource_type_id) + except ResourceType.DoesNotExist: + errors["resource_type"] = ["Resource type does not exist."] + return None + if not resource_type.is_active: + errors["resource_type"] = ["Resource type is not active."] + return None + return resource_type + + +def create_publication( + *, + user: Any, + organization: Any, + title: str, + description: Optional[str], + authors: List[str], + publication_date: Any, + license_value: str, + resource_type: ResourceType, + sector_ids: List[Any], + geography_ids: List[Any], + external_source_link: Optional[str], +) -> Publication: + """Create a DRAFT publication from validated metadata and wire its M2M tags. + + Ownership follows the caller's context: an organization present in the + request makes it org-owned, otherwise it's the individual user's. + """ + publication = Publication.objects.create( + title=title, + description=description, + authors=authors, + publication_date=publication_date, + license=license_value, + resource_type=resource_type, + external_source_link=external_source_link or None, + organization=organization, + user=user, + status=PublicationStatus.DRAFT, + ) + _set_publication_tags(publication, sector_ids, geography_ids) + return publication + + +def apply_publication_update( + publication: Publication, + *, + title: Optional[str] = None, + description: Optional[str] = None, + authors: Optional[List[str]] = None, + publication_date: Any = None, + license_value: Optional[str] = None, + resource_type_id: Any = None, + sector_ids: Optional[List[Any]] = None, + geography_ids: Optional[List[Any]] = None, + external_source_link: Optional[str] = None, +) -> Publication: + """Apply a partial metadata update, validating each field that's provided. + + Only fields passed in are touched, so a subpage save never blanks columns it + didn't show. A provided license must be in the controlled list; a provided + resource type must be active; a provided link must be a valid URL. + """ + errors: dict[str, List[str]] = {} + + if title is not None: + publication.title = title + if description is not None: + publication.description = description + if authors is not None: + publication.authors = authors + if publication_date is not None: + publication.publication_date = publication_date + if external_source_link is not None: + if external_source_link: + try: + URLValidator()(external_source_link) + publication.external_source_link = external_source_link + except DjangoValidationError: + errors["external_source_link"] = ["Enter a valid URL."] + else: + publication.external_source_link = None + + if license_value is not None: + if license_value in DatasetLicense.values: + publication.license = license_value + else: + errors["license"] = ["Not a valid license."] + + if resource_type_id is not None: + resource_type = _resolve_active_resource_type(resource_type_id, errors) + if resource_type is not None: + publication.resource_type = resource_type + + if errors: + raise DjangoValidationError(errors) + + publication.save() + if sector_ids is not None or geography_ids is not None: + _set_publication_tags(publication, sector_ids, geography_ids) + return publication + + +def set_publication_status(publication: Publication, status: PublicationStatus) -> Publication: + """Flip a publication's publish status and save it.""" + publication.status = status + publication.save() + return publication + + +def get_scoped_publications( + *, user: Any, organization: Any, include_public: bool +) -> "QuerySet[Publication, Publication]": + """Return the publications a caller may list, correctly scoped. + + Organization context → that org's publications; an authenticated individual + → their own; anonymous → published only. ``include_public`` unions in the + published set so a signed-in user also sees the public listing. Ordered + newest-first and de-duplicated after the union. + """ + if organization: + queryset = Publication.objects.filter(organization=organization) + elif getattr(user, "is_superuser", False): + queryset = Publication.objects.all() + elif getattr(user, "is_authenticated", False): + queryset = Publication.objects.filter(user=user, organization__isnull=True) + else: + queryset = Publication.objects.filter(status=PublicationStatus.PUBLISHED) + + if include_public: + queryset = queryset | Publication.objects.filter(status=PublicationStatus.PUBLISHED) + + return queryset.order_by("-modified").distinct() + + +def is_publication_published(publication: Publication) -> bool: + """True only when the publication is PUBLISHED.""" + return publication.status == PublicationStatus.PUBLISHED.value + + +def resolve_pagination(offset: Optional[int], limit: Optional[int]) -> tuple[int, int]: + """Turn a caller's optional page window into a bounded (offset, limit). + + A missing limit falls back to the default page size; any limit is capped at + the hard maximum, so a listing is never unbounded even with no input. + """ + safe_offset = max(offset or 0, 0) + if not limit or limit <= 0: + safe_limit = DEFAULT_PAGE_SIZE + else: + safe_limit = min(limit, MAX_PAGE_SIZE) + return safe_offset, safe_limit + + +def _set_publication_tags( + publication: Publication, + sector_ids: Optional[List[Any]], + geography_ids: Optional[List[Any]], +) -> None: + """Replace a publication's sector and geography tags from id lists.""" + if sector_ids is not None: + publication.sectors.set(Sector.objects.filter(id__in=sector_ids)) + if geography_ids is not None: + publication.geographies.set(Geography.objects.filter(id__in=geography_ids)) diff --git a/api/types/type_publication.py b/api/types/type_publication.py new file mode 100644 index 0000000..7957fc5 --- /dev/null +++ b/api/types/type_publication.py @@ -0,0 +1,122 @@ +import uuid +from datetime import date, datetime +from typing import List, Optional, cast + +import strawberry +import strawberry_django +from strawberry.enum import EnumType +from strawberry.types import Info + +from api.models import Publication, PublicationBlock, ResourceType +from api.types.base_type import BaseType +from api.types.type_geo import TypeGeo +from api.types.type_organization import TypeOrganization +from api.types.type_sector import TypeSector +from api.utils.enums import DatasetLicense, PublicationBlockType, PublicationStatus +from authorization.types import TypeUser + +# Fields are enumerated on every type below — never ``fields="__all__"`` — so a +# future column is never silently published. +publication_status: EnumType = strawberry.enum(PublicationStatus) # type: ignore +publication_block_type: EnumType = strawberry.enum(PublicationBlockType) # type: ignore +publication_license: EnumType = strawberry.enum(DatasetLicense) # type: ignore + + +@strawberry_django.type(ResourceType) +class TypeResourceType(BaseType): + """Type for the admin-managed Resource Type lookup.""" + + id: uuid.UUID + name: str + slug: Optional[str] + is_active: bool + + +@strawberry_django.type(PublicationBlock) +class TypePublicationBlock(BaseType): + """Type for one content block (a file XOR a YouTube embed).""" + + id: uuid.UUID + position: int + block_type: publication_block_type + file_name: str + file_format: str + file_size: Optional[int] + youtube_url: Optional[str] + youtube_video_id: str + + +@strawberry_django.filter(Publication) +class PublicationFilter: + """Filter for publications.""" + + id: Optional[uuid.UUID] + status: Optional[publication_status] + resource_type: Optional[uuid.UUID] + + +@strawberry_django.order(Publication) +class PublicationOrder: + """Order for publications.""" + + title: strawberry.auto + created: strawberry.auto + modified: strawberry.auto + + +@strawberry_django.type( + Publication, + filters=PublicationFilter, + pagination=True, + order=PublicationOrder, # type: ignore +) +class TypePublication(BaseType): + """Type for a Publication (UI 'Resource').""" + + id: uuid.UUID + title: str + description: Optional[str] + slug: str + status: publication_status + authors: List[str] + publication_date: Optional[date] + license: publication_license + external_source_link: Optional[str] + download_count: int + created: datetime + modified: datetime + organization: Optional["TypeOrganization"] + user: Optional["TypeUser"] + resource_type: Optional["TypeResourceType"] + + @strawberry.field + def sectors(self, info: Info) -> List["TypeSector"]: + """Sectors tagged on this resource.""" + try: + instance = cast(Publication, self) + return TypeSector.from_django_list(instance.sectors.all()) + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def geographies(self, info: Info) -> List["TypeGeo"]: + """Geographies tagged on this resource.""" + try: + instance = cast(Publication, self) + return TypeGeo.from_django_list(instance.geographies.all()) + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def blocks(self, info: Info) -> List["TypePublicationBlock"]: + """Ordered content blocks of this resource.""" + try: + instance = cast(Publication, self) + return TypePublicationBlock.from_django_list(instance.blocks.all().order_by("position")) + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def is_individual_publication(self) -> bool: + """True when owned by an individual rather than an organization.""" + return self.organization is None diff --git a/authorization/permissions.py b/authorization/permissions.py index 4aca31c..60ce2d3 100644 --- a/authorization/permissions.py +++ b/authorization/permissions.py @@ -5,9 +5,13 @@ from strawberry.permission import BasePermission from strawberry.types import Info -from api.models import Dataset, Organization +from api.models import Dataset, Organization, Publication +from api.utils.enums import PublicationStatus from authorization.models import DatasetPermission, OrganizationMembership, Role +# Roles that may publish/unpublish and edit an org-owned publication, by name. +PUBLICATION_MANAGER_ROLE_NAMES = ["admin", "editor", "owner"] + # REST Framework Permissions class IsOrganizationMember(permissions.BasePermission): @@ -53,9 +57,7 @@ def has_permission(self, request: Any, view: Any) -> bool: return True # For organization-specific endpoints - org_id = request.query_params.get("organization") or request.data.get( - "organization" - ) + org_id = request.query_params.get("organization") or request.data.get("organization") if org_id: return OrganizationMembership.objects.filter( user=request.user, organization_id=org_id @@ -204,9 +206,7 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: organization_id = kwargs.get("organization_id") # Also check if organization is in the context organization = None - if hasattr(info.context, "context") and isinstance( - info.context.context, dict - ): + if hasattr(info.context, "context") and isinstance(info.context.context, dict): organization = info.context.context.get("organization") if organization_id: @@ -316,9 +316,7 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: return True try: - dataset_perm = DatasetPermission.objects.get( - user=request.user, dataset=source - ) + dataset_perm = DatasetPermission.objects.get(user=request.user, dataset=source) role = dataset_perm.role return self._check_role_permission(role) except DatasetPermission.DoesNotExist: @@ -458,3 +456,161 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: except Dataset.DoesNotExist: return False + + +# --------------------------------------------------------------------------- +# Publication (UI "Resource") permissions +# +# Mirrors Dataset's dedicated permission classes rather than AIModel's inline +# per-resolver role checks. Publication has no per-object share model, so the +# share-model fallback is dropped; the individual-owner branch is kept so an +# org-less publication's owner isn't denied. +# --------------------------------------------------------------------------- + + +def _resolve_publication_id(kwargs: Any) -> Optional[Any]: + """Pull the target publication's id out of a mutation's arguments. + + Publish/unpublish/delete pass ``publication_id`` directly; update passes an + input object carrying the id on ``.id``. + """ + publication_id = kwargs.get("publication_id") + if publication_id: + return publication_id + for input_key in ("input", "update_input"): + payload = kwargs.get(input_key) + if payload is not None and getattr(payload, "id", None): + return payload.id + return None + + +def _user_manages_publication(user: Any, publication: Publication, operation: str) -> bool: + """Whether a user may perform ``operation`` on a publication. + + Owner (individual publications) always may; for org-owned publications the + caller must be a member whose role grants the operation (``publish`` and + ``change``/``delete`` map to the role's name / boolean flags). + """ + if user.is_superuser: + return True + if publication.user and publication.user == user: + return True + if not publication.organization: + return False + + membership = OrganizationMembership.objects.filter( + user=user, organization=publication.organization + ).first() + if not membership: + return False + + role = membership.role + if operation == "publish": + return role.name in PUBLICATION_MANAGER_ROLE_NAMES + if operation == "delete": + return role.can_delete + return role.can_change + + +class PublicationPermissionGraphQL(BasePermission): # type: ignore[misc] + """Base publication mutation permission — keys on the publication id in kwargs.""" + + message = "You don't have permission to modify this resource" + operation = "change" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = info.context.user + if not getattr(user, "is_authenticated", False): + return False + + publication_id = _resolve_publication_id(kwargs) + if not publication_id: + return False + + try: + publication = Publication.objects.get(id=publication_id) + except Publication.DoesNotExist: + return False + + return _user_manages_publication(user, publication, self.operation) + + +class ChangePublicationPermission(PublicationPermissionGraphQL): + operation = "change" + + +class DeletePublicationPermission(PublicationPermissionGraphQL): + message = "You don't have permission to delete this resource" + operation = "delete" + + +class PublishPublicationPermission(PublicationPermissionGraphQL): + message = "You don't have permission to publish this resource" + operation = "publish" + + +class CreatePublicationPermission(BasePermission): # type: ignore[misc] + """Permission for creating a publication — mirrors CreateDatasetPermission. + + Any authenticated user may create an individual publication; creating inside + an organization context requires the ``add`` role in that organization. + """ + + message = "You don't have permission to create a resource" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = info.context.user + if not getattr(user, "is_authenticated", False): + return False + + organization = info.context.context.get("organization") + if organization: + membership = OrganizationMembership.objects.filter( + user=user, organization=organization + ).first() + return bool(membership and membership.role.can_add) + + return True + + +class AllowPublishedPublications(BasePermission): # type: ignore[misc] + """Read gate for a single publication — mirrors AllowPublishedDatasets. + + A PUBLISHED publication is world-readable; a DRAFT is visible only to the + owner, org members with view access, or a superuser. + """ + + message = "You need to be authenticated to access non-published resources" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + request = info.context + publication_id = kwargs.get("publication_id") + + if publication_id: + try: + publication = Publication.objects.get(id=publication_id) + except Publication.DoesNotExist: + return True # Let the resolver return a clean not-found. + + if publication.status == PublicationStatus.PUBLISHED.value: + return True + + user = request.user + if not user.is_authenticated: + return False + if user.is_superuser: + return True + if publication.user and publication.user == user: + return True + if publication.organization: + membership = OrganizationMembership.objects.filter( + user=user, organization=publication.organization + ).first() + return bool(membership and membership.role.can_view) + return False + + # No id in kwargs (e.g. object source) — published is public, else auth. + if hasattr(source, "status"): + if source.status == PublicationStatus.PUBLISHED.value: + return True + return bool(getattr(request, "user", None) and request.user.is_authenticated) diff --git a/tests/schema/test_publication_schema.py b/tests/schema/test_publication_schema.py new file mode 100644 index 0000000..890e648 --- /dev/null +++ b/tests/schema/test_publication_schema.py @@ -0,0 +1,428 @@ +"""Layer 3/4 tests for the publication (Resource) GraphQL surface. + +Executes the real schema against the Django test DB with a fake context that +carries the caller's user and org — establishing the layer-4 pattern for this +repo. Activity recording is stubbed so tests exercise the mutation logic, not +the activity-stream plumbing. +""" + +import types +from datetime import date +from unittest.mock import patch + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Geography, Publication, ResourceType, Sector +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.enums import GeoTypes, PublicationStatus +from authorization.models import OrganizationMembership, Role, User + + +# --------------------------------------------------------------------------- # +# Fixtures +# --------------------------------------------------------------------------- # +@pytest.fixture(autouse=True) +def _no_activity_recording(): + with patch("api.schema.base_mutation.record_activity", return_value=None): + yield + + +@pytest.fixture +def roles(db): + admin = Role.objects.create( + name="admin", can_view=True, can_add=True, can_change=True, can_delete=True + ) + editor = Role.objects.create( + name="editor", can_view=True, can_add=True, can_change=True, can_delete=False + ) + auditor = Role.objects.create( + name="auditor", can_view=True, can_add=False, can_change=False, can_delete=False + ) + return {"admin": admin, "editor": editor, "auditor": auditor} + + +@pytest.fixture +def org_a(db): + return Organization.objects.create(name="Org A", description="a", slug="org-a") + + +@pytest.fixture +def org_b(db): + return Organization.objects.create(name="Org B", description="b", slug="org-b") + + +def _member(user, org, role): + OrganizationMembership.objects.create(user=user, organization=org, role=role) + return user + + +@pytest.fixture +def org_a_admin(roles, org_a): + return _member( + User.objects.create(username="a_admin", keycloak_id="a_admin"), org_a, roles["admin"] + ) + + +@pytest.fixture +def org_a_editor(roles, org_a): + return _member( + User.objects.create(username="a_editor", keycloak_id="a_editor"), org_a, roles["editor"] + ) + + +@pytest.fixture +def org_a_auditor(roles, org_a): + return _member( + User.objects.create(username="a_auditor", keycloak_id="a_auditor"), org_a, roles["auditor"] + ) + + +@pytest.fixture +def org_b_admin(roles, org_b): + return _member( + User.objects.create(username="b_admin", keycloak_id="b_admin"), org_b, roles["admin"] + ) + + +@pytest.fixture +def individual(db): + return User.objects.create(username="solo", keycloak_id="solo") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +@pytest.fixture +def inactive_type(db): + return ResourceType.objects.create(name="Retired", is_active=False) + + +@pytest.fixture +def sector(db): + return Sector.objects.create(name="Health") + + +@pytest.fixture +def geography(db): + return Geography.objects.create(name="India", code="IN", type=GeoTypes.COUNTRY) + + +def ctx(user, organization=None): + return types.SimpleNamespace( + user=user, + context={"organization": organization} if organization else {}, + ) + + +def run(query, context, variables=None): + return schema.execute_sync(query, variable_values=variables or {}, context_value=context) + + +def valid_create_vars(resource_type, sector, geography, **overrides): + variables = { + "input": { + "title": "Rainfall Findings", + "description": "A study of rainfall.", + "authors": ["Ada Lovelace"], + "publicationDate": "2024-01-01", + "license": "CC_BY_4_0_ATTRIBUTION", + "resourceTypeId": str(resource_type.id), + "sectorIds": [str(sector.id)], + "geographyIds": [geography.id], + } + } + variables["input"].update(overrides) + return variables + + +CREATE = """ +mutation Create($input: CreatePublicationInput!) { + createPublication(input: $input) { + success + errors { fieldErrors { field messages } nonFieldErrors } + data { id slug status isIndividualPublication organization { id } user { id } } + } +} +""" + +UPDATE = """ +mutation Update($input: UpdatePublicationInput!) { + updatePublication(input: $input) { + success + errors { fieldErrors { field messages } } + data { id title } + } +} +""" + +PUBLISH = """ +mutation Publish($id: UUID!) { + publishPublication(publicationId: $id) { success data { id status } } +} +""" + +UNPUBLISH = """ +mutation Unpublish($id: UUID!) { + unpublishPublication(publicationId: $id) { success data { id status } } +} +""" + +DELETE = """ +mutation Delete($id: UUID!) { + deletePublication(publicationId: $id) { success data } +} +""" + +GET = """ +query Get($id: UUID!) { + getPublication(publicationId: $id) { id status } +} +""" + +LIST = """ +query List($includePublic: Boolean) { + publications(includePublic: $includePublic) { id status } +} +""" + + +# --------------------------------------------------------------------------- # +# Create +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestCreate: + def test_org_member_creates_org_owned_draft( + self, org_a_admin, org_a, resource_type, sector, geography + ): + result = run( + CREATE, ctx(org_a_admin, org_a), valid_create_vars(resource_type, sector, geography) + ) + + assert result.errors is None + payload = result.data["createPublication"] + assert payload["success"] is True + assert payload["data"]["status"] == "DRAFT" + assert payload["data"]["organization"]["id"] == str(org_a.id) + assert payload["data"]["isIndividualPublication"] is False + assert Publication.objects.filter(id=payload["data"]["id"]).exists() + + def test_individual_creates_user_owned_draft( + self, individual, resource_type, sector, geography + ): + result = run(CREATE, ctx(individual), valid_create_vars(resource_type, sector, geography)) + + payload = result.data["createPublication"] + assert payload["success"] is True + assert payload["data"]["isIndividualPublication"] is True + assert payload["data"]["user"]["id"] == str(individual.id) + + def test_missing_title_is_rejected_without_creating( + self, individual, resource_type, sector, geography + ): + result = run( + CREATE, ctx(individual), valid_create_vars(resource_type, sector, geography, title="") + ) + + payload = result.data["createPublication"] + assert payload["success"] is False + assert Publication.objects.count() == 0 + + def test_inactive_resource_type_is_rejected(self, individual, inactive_type, sector, geography): + result = run(CREATE, ctx(individual), valid_create_vars(inactive_type, sector, geography)) + + payload = result.data["createPublication"] + assert payload["success"] is False + assert Publication.objects.count() == 0 + + def test_anonymous_cannot_create(self, resource_type, sector, geography): + result = run( + CREATE, ctx(AnonymousUser()), valid_create_vars(resource_type, sector, geography) + ) + + payload = result.data["createPublication"] + assert payload["success"] is False + assert Publication.objects.count() == 0 + + +# --------------------------------------------------------------------------- # +# Update / role gating +# --------------------------------------------------------------------------- # +def _make_publication(user, org, resource_type, status=PublicationStatus.DRAFT): + return Publication.objects.create( + title="Existing", + description="d", + authors=["A"], + publication_date=date(2024, 1, 1), + license="CC_BY_4_0_ATTRIBUTION", + resource_type=resource_type, + organization=org, + user=user, + status=status, + ) + + +@pytest.mark.django_db +class TestUpdateAndRoles: + def test_editor_updates_org_publication(self, org_a_admin, org_a_editor, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + UPDATE, + ctx(org_a_editor, org_a), + {"input": {"id": str(publication.id), "title": "New Title"}}, + ) + + assert result.data["updatePublication"]["success"] is True + publication.refresh_from_db() + assert publication.title == "New Title" + + def test_auditor_cannot_update(self, org_a_admin, org_a_auditor, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + UPDATE, + ctx(org_a_auditor, org_a), + {"input": {"id": str(publication.id), "title": "Hijack"}}, + ) + + assert result.data["updatePublication"]["success"] is False + publication.refresh_from_db() + assert publication.title == "Existing" + + def test_auditor_can_read_draft(self, org_a_admin, org_a_auditor, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(GET, ctx(org_a_auditor, org_a), {"id": str(publication.id)}) + + assert result.errors is None + assert result.data["getPublication"]["id"] == str(publication.id) + + +# --------------------------------------------------------------------------- # +# Publish / unpublish +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestPublish: + def test_admin_publishes(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(PUBLISH, ctx(org_a_admin, org_a), {"id": str(publication.id)}) + + assert result.data["publishPublication"]["success"] is True + publication.refresh_from_db() + assert publication.status == PublicationStatus.PUBLISHED + + def test_auditor_cannot_publish(self, org_a_admin, org_a_auditor, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(PUBLISH, ctx(org_a_auditor, org_a), {"id": str(publication.id)}) + + assert result.data["publishPublication"]["success"] is False + publication.refresh_from_db() + assert publication.status == PublicationStatus.DRAFT + + def test_unpublish_reverts_to_draft(self, org_a_admin, org_a, resource_type): + publication = _make_publication( + org_a_admin, org_a, resource_type, status=PublicationStatus.PUBLISHED + ) + + result = run(UNPUBLISH, ctx(org_a_admin, org_a), {"id": str(publication.id)}) + + assert result.data["unpublishPublication"]["success"] is True + publication.refresh_from_db() + assert publication.status == PublicationStatus.DRAFT + + +# --------------------------------------------------------------------------- # +# Cross-org denial +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestCrossOrgDenial: + def test_other_org_cannot_update(self, org_a_admin, org_b_admin, org_a, org_b, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + UPDATE, + ctx(org_b_admin, org_b), + {"input": {"id": str(publication.id), "title": "Steal"}}, + ) + + assert result.data["updatePublication"]["success"] is False + publication.refresh_from_db() + assert publication.title == "Existing" + + def test_other_org_cannot_delete(self, org_a_admin, org_b_admin, org_a, org_b, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(DELETE, ctx(org_b_admin, org_b), {"id": str(publication.id)}) + + assert result.data["deletePublication"]["success"] is False + assert Publication.objects.filter(id=publication.id).exists() + + def test_other_org_cannot_publish(self, org_a_admin, org_b_admin, org_a, org_b, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(PUBLISH, ctx(org_b_admin, org_b), {"id": str(publication.id)}) + + assert result.data["publishPublication"]["success"] is False + publication.refresh_from_db() + assert publication.status == PublicationStatus.DRAFT + + +# --------------------------------------------------------------------------- # +# Delete +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestDelete: + def test_owner_deletes(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(DELETE, ctx(org_a_admin, org_a), {"id": str(publication.id)}) + + assert result.data["deletePublication"]["success"] is True + assert not Publication.objects.filter(id=publication.id).exists() + + +# --------------------------------------------------------------------------- # +# Read gating + listing +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestReadGating: + def test_anonymous_sees_published_detail(self, org_a_admin, org_a, resource_type): + publication = _make_publication( + org_a_admin, org_a, resource_type, status=PublicationStatus.PUBLISHED + ) + + result = run(GET, ctx(AnonymousUser()), {"id": str(publication.id)}) + + assert result.errors is None + assert result.data["getPublication"]["id"] == str(publication.id) + + def test_anonymous_denied_draft_detail(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(GET, ctx(AnonymousUser()), {"id": str(publication.id)}) + + assert result.errors is not None # permission denied, not a silent leak + + def test_listing_is_org_scoped(self, org_a_admin, org_b_admin, org_a, org_b, resource_type): + _make_publication(org_a_admin, org_a, resource_type) + _make_publication(org_b_admin, org_b, resource_type) + + result = run(LIST, ctx(org_a_admin, org_a), {"includePublic": False}) + + assert result.errors is None + assert len(result.data["publications"]) == 1 + + def test_anonymous_listing_is_published_only(self, org_a_admin, org_a, resource_type): + _make_publication(org_a_admin, org_a, resource_type, status=PublicationStatus.DRAFT) + _make_publication(org_a_admin, org_a, resource_type, status=PublicationStatus.PUBLISHED) + + result = run(LIST, ctx(AnonymousUser()), {"includePublic": False}) + + statuses = [row["status"] for row in result.data["publications"]] + assert statuses == ["PUBLISHED"] From 5eec6250194cb093b7e278eaebf0f0a98769bfee Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 13:06:12 +0530 Subject: [PATCH 10/57] feat(publications): add content blocks, uploads, YouTube and gated download - youtube.py: extract/validate YouTube video ids across watch/youtu.be/embed. - publication_uploads.py: server-side file validation (extension allow-list, 50 MB cap, PDF magic-byte sniff). - publication_blocks.py: add file/youtube block, replace file (deleting old bytes), remove + contiguous renumber, reorder, read-time access gate. - publication_download_view.py + URL: gated block-file serve (draft private, published public, download_count increment, PDF inline). - publication_signals.py: post_delete removes a block's file from disk. - Block mutations (add file/youtube, replace, remove, reorder); permission resolver maps a block id to its parent publication. - Layer 2/1/3/4 tests across youtube, uploads, block CRUD, download gate. Also annotates a pre-existing untyped local in dynamic_chart_view (surfaced by mypy once urls.py imports the views package) to keep the mypy gate green. --- api/schema/publication_schema.py | 98 +++++++++++- api/services/publication_blocks.py | 153 ++++++++++++++++++ api/signals/__init__.py | 7 +- api/signals/publication_signals.py | 27 ++++ api/urls.py | 6 + api/utils/publication_uploads.py | 69 +++++++++ api/utils/youtube.py | 62 ++++++++ api/views/dynamic_chart_view.py | 17 +- api/views/publication_download_view.py | 65 ++++++++ authorization/permissions.py | 8 + tests/schema/test_publication_schema.py | 78 ++++++++++ tests/test_publication_blocks.py | 196 ++++++++++++++++++++++++ tests/test_publication_uploads.py | 36 +++++ tests/test_youtube_url.py | 45 ++++++ 14 files changed, 853 insertions(+), 14 deletions(-) create mode 100644 api/services/publication_blocks.py create mode 100644 api/signals/publication_signals.py create mode 100644 api/utils/publication_uploads.py create mode 100644 api/utils/youtube.py create mode 100644 api/views/publication_download_view.py create mode 100644 tests/test_publication_blocks.py create mode 100644 tests/test_publication_uploads.py create mode 100644 tests/test_youtube_url.py diff --git a/api/schema/publication_schema.py b/api/schema/publication_schema.py index 0cd4f50..14f47f6 100644 --- a/api/schema/publication_schema.py +++ b/api/schema/publication_schema.py @@ -16,10 +16,18 @@ import strawberry import strawberry_django from django.core.exceptions import ValidationError as DjangoValidationError +from strawberry.file_uploads import Upload from strawberry.types import Info -from api.models import Publication +from api.models import Publication, PublicationBlock from api.schema.base_mutation import BaseMutation, MutationResponse +from api.services.publication_blocks import ( + add_file_block, + add_youtube_block, + remove_block, + reorder_blocks, + replace_block_file, +) from api.services.publication_service import ( apply_publication_update, create_publication, @@ -32,6 +40,7 @@ PublicationFilter, PublicationOrder, TypePublication, + TypePublicationBlock, publication_license, ) from api.utils.enums import PublicationStatus @@ -266,6 +275,85 @@ def delete_publication(self, info: Info, publication_id: uuid.UUID) -> MutationR publication.delete() return MutationResponse.success_response(True) + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="add_publication_file_block", + trace_attributes={"component": "publication"}, + ) + def add_publication_file_block( + self, info: Info, publication_id: uuid.UUID, file: Upload + ) -> MutationResponse[TypePublicationBlock]: + """Append an uploaded file as the next content block (validated server-side).""" + publication = _get_publication_or_raise(publication_id) + + # Validate + store the file as the last block. + block = add_file_block(publication, file) + return MutationResponse.success_response(TypePublicationBlock.from_django(block)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="add_publication_youtube_block", + trace_attributes={"component": "publication"}, + ) + def add_publication_youtube_block( + self, info: Info, publication_id: uuid.UUID, youtube_url: str + ) -> MutationResponse[TypePublicationBlock]: + """Append a YouTube link as the next content block (validated server-side).""" + publication = _get_publication_or_raise(publication_id) + + # Validate the url + extract its video id, then store the block. + block = add_youtube_block(publication, youtube_url) + return MutationResponse.success_response(TypePublicationBlock.from_django(block)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="replace_publication_block_file", + trace_attributes={"component": "publication"}, + ) + def replace_publication_block_file( + self, info: Info, block_id: uuid.UUID, file: Upload + ) -> MutationResponse[TypePublicationBlock]: + """Swap a file block's file, deleting the old one from disk.""" + block = _get_block_or_raise(block_id) + + # Replace in place — same block id, old file removed. + block = replace_block_file(block, file) + return MutationResponse.success_response(TypePublicationBlock.from_django(block)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="remove_publication_block", + trace_attributes={"component": "publication"}, + ) + def remove_publication_block(self, info: Info, block_id: uuid.UUID) -> MutationResponse[bool]: + """Remove a content block and renumber the rest contiguously.""" + block = _get_block_or_raise(block_id) + + # Delete + renumber siblings; the signal removes the file from disk. + remove_block(block) + return MutationResponse.success_response(True) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="reorder_publication_blocks", + trace_attributes={"component": "publication"}, + ) + def reorder_publication_blocks( + self, info: Info, publication_id: uuid.UUID, block_ids: List[uuid.UUID] + ) -> MutationResponse[TypePublication]: + """Set the content blocks' order to the given block-id sequence.""" + publication = _get_publication_or_raise(publication_id) + + # Reassign positions to match the requested order. + reorder_blocks(publication, block_ids) + publication.refresh_from_db() + return MutationResponse.success_response(TypePublication.from_django(publication)) + def _get_publication_or_raise(publication_id: uuid.UUID) -> Publication: """Load a publication by id or raise a clean validation error.""" @@ -273,3 +361,11 @@ def _get_publication_or_raise(publication_id: uuid.UUID) -> Publication: return Publication.objects.get(id=publication_id) except Publication.DoesNotExist: raise DjangoValidationError(f"Resource with id {publication_id} does not exist.") + + +def _get_block_or_raise(block_id: uuid.UUID) -> PublicationBlock: + """Load a content block by id or raise a clean validation error.""" + try: + return PublicationBlock.objects.get(id=block_id) + except PublicationBlock.DoesNotExist: + raise DjangoValidationError(f"Content block {block_id} does not exist.") diff --git a/api/services/publication_blocks.py b/api/services/publication_blocks.py new file mode 100644 index 0000000..36f2a78 --- /dev/null +++ b/api/services/publication_blocks.py @@ -0,0 +1,153 @@ +""" +publication_blocks +────────────────── +The messy 90% behind a Resource's content blocks: adding a file or YouTube +block at the next position, replacing a block's file without leaking the old +one, removing a block and renumbering the rest contiguously, reordering, and +the read-time access gate for a block's file. + +Every write keeps ``position`` a dense 0..n-1 sequence, and every file swap or +removal deletes the previous bytes from disk so drafts don't leak files. +""" + +from typing import Any, List + +from django.core.exceptions import ValidationError +from django.db import transaction + +from api.models import Publication, PublicationBlock +from api.services.publication_service import is_publication_published +from api.utils.enums import PublicationBlockType +from api.utils.publication_uploads import validate_publication_file +from api.utils.youtube import validate_youtube_url + + +def add_file_block(publication: Publication, uploaded_file: Any) -> PublicationBlock: + """Validate an uploaded file and append it as the next content block.""" + extension, size = validate_publication_file(uploaded_file) + + block = PublicationBlock( + publication=publication, + position=_next_position(publication), + block_type=PublicationBlockType.FILE, + file_name=getattr(uploaded_file, "name", ""), + file_format=extension.lstrip("."), + file_size=size, + ) + block.file = uploaded_file + block.save() + return block + + +def add_youtube_block(publication: Publication, youtube_url: str) -> PublicationBlock: + """Validate a YouTube url and append it as the next content block.""" + video_id = validate_youtube_url(youtube_url) + + return PublicationBlock.objects.create( + publication=publication, + position=_next_position(publication), + block_type=PublicationBlockType.YOUTUBE, + youtube_url=youtube_url, + youtube_video_id=video_id, + ) + + +def replace_block_file(block: PublicationBlock, uploaded_file: Any) -> PublicationBlock: + """Swap a file block's file for a new one, deleting the old bytes from disk. + + Django's FileField never removes the previous file on reassignment, so we + delete it explicitly first — a re-upload updates in place (same row id) and + doesn't leak the old file. + """ + if block.block_type != PublicationBlockType.FILE: + raise ValidationError("Only a file block's file can be replaced.") + + extension, size = validate_publication_file(uploaded_file) + + # Drop the previous file from storage before attaching the new one. + if block.file: + block.file.delete(save=False) + + block.file = uploaded_file + block.file_name = getattr(uploaded_file, "name", "") + block.file_format = extension.lstrip(".") + block.file_size = size + block.save() + return block + + +def remove_block(block: PublicationBlock) -> None: + """Delete a block and renumber its siblings so positions stay contiguous.""" + publication = block.publication + with transaction.atomic(): + block.delete() + _renumber_blocks(publication) + + +def reorder_blocks( + publication: Publication, ordered_block_ids: List[Any] +) -> List[PublicationBlock]: + """Set block positions to match the given id order (0-based, contiguous).""" + existing: List[PublicationBlock] = list( + PublicationBlock.objects.filter(publication=publication) + ) + blocks_by_id = {block.id: block for block in existing} + if set(blocks_by_id.keys()) != {_coerce_id(bid, blocks_by_id) for bid in ordered_block_ids}: + raise ValidationError("Reorder must list every block exactly once.") + + reordered: List[PublicationBlock] = [] + for position, block_id in enumerate(ordered_block_ids): + block = blocks_by_id[_coerce_id(block_id, blocks_by_id)] + block.position = position + reordered.append(block) + + PublicationBlock.objects.bulk_update(reordered, ["position"]) + return reordered + + +def can_access_block_file(user: Any, publication: Publication) -> bool: + """Whether a caller may download a block's file. + + A PUBLISHED resource's files are world-readable; a DRAFT's files are private + to the owner, org members, and superusers — never reachable anonymously. + """ + if is_publication_published(publication): + return True + if not getattr(user, "is_authenticated", False): + return False + if user.is_superuser: + return True + if publication.user and publication.user == user: + return True + if publication.organization: + from authorization.models import OrganizationMembership + + return OrganizationMembership.objects.filter( + user=user, organization=publication.organization + ).exists() + return False + + +def _next_position(publication: Publication) -> int: + """The position for a new block appended to the end.""" + return publication.blocks.count() + + +def _renumber_blocks(publication: Publication) -> None: + """Rewrite positions to a dense 0..n-1 sequence in current order.""" + blocks: List[PublicationBlock] = list( + PublicationBlock.objects.filter(publication=publication).order_by("position") + ) + for position, block in enumerate(blocks): + block.position = position + PublicationBlock.objects.bulk_update(blocks, ["position"]) + + +def _coerce_id(block_id: Any, blocks_by_id: dict) -> Any: + """Match an incoming id (possibly a string) to a stored block key.""" + if block_id in blocks_by_id: + return block_id + for key in blocks_by_id: + if str(key) == str(block_id): + return key + return block_id diff --git a/api/signals/__init__.py b/api/signals/__init__.py index 29e3798..542aaba 100644 --- a/api/signals/__init__.py +++ b/api/signals/__init__.py @@ -1,2 +1,7 @@ # Import signals to register them -from api.signals import aimodel_signals, dataset_signals, usecase_signals +from api.signals import ( + aimodel_signals, + dataset_signals, + publication_signals, + usecase_signals, +) diff --git a/api/signals/publication_signals.py b/api/signals/publication_signals.py new file mode 100644 index 0000000..1544d15 --- /dev/null +++ b/api/signals/publication_signals.py @@ -0,0 +1,27 @@ +""" +publication_signals +──────────────────── +Keeps a Resource's stored files from leaking. When a content block is deleted — +directly, or by the FK cascade when its parent publication is deleted — its file +is removed from disk. (Django's ORM never deletes the underlying file on its +own, so ``Resource`` today leaks files on delete; we intentionally don't repeat +that here.) +""" + +from typing import Any + +import structlog +from django.db.models.signals import post_delete +from django.dispatch import receiver + +from api.models import PublicationBlock + +logger = structlog.getLogger(__name__) + + +@receiver(post_delete, sender=PublicationBlock) +def remove_publication_block_file(sender: Any, instance: PublicationBlock, **kwargs: Any) -> None: + """Delete a removed block's file from storage.""" + if instance.file: + instance.file.delete(save=False) + logger.info("publication block file removed", block_id=str(instance.id)) diff --git a/api/urls.py b/api/urls.py index c0bb1d6..73c1c55 100644 --- a/api/urls.py +++ b/api/urls.py @@ -14,6 +14,7 @@ dataset_data, download, generate_dynamic_chart, + publication_download_view, search_aimodel, search_collaborative, search_dataset, @@ -108,6 +109,11 @@ r"download/(?Presource|access_resource|chart|chart_image)/(?P[0-9a-f]{8}\-[0-9a-f]{4}\-4[0-9a-f]{3}\-[89ab][0-9a-f]{3}\-[0-9a-f]{12})", download, ), + path( + "publications/blocks//download/", + publication_download_view.publication_block_download, + name="publication_block_download", + ), re_path( # type: ignore r"generate-dynamic-chart/(?P[0-9a-f]{8}\-[0-9a-f]{4}\-4[0-9a-f]{3}\-[89ab][0-9a-f]{3}\-[0-9a-f]{12})", generate_dynamic_chart, diff --git a/api/utils/publication_uploads.py b/api/utils/publication_uploads.py new file mode 100644 index 0000000..c2c51ed --- /dev/null +++ b/api/utils/publication_uploads.py @@ -0,0 +1,69 @@ +""" +publication_uploads +──────────────────── +Server-side validation for a content block's uploaded file. + +``validate_publication_file`` is the upload boundary guard: it enforces the +allowed extensions, the 50 MB per-file cap, and — for a file declaring itself a +PDF — that the bytes really start with the PDF magic number (so a renamed +executable can't sneak in as ``report.pdf``). Raises a clean ValidationError on +any violation; returns the detected extension and byte size on success. +""" + +import os +from typing import Any, Tuple + +from django.core.exceptions import ValidationError + +# Locked limits (plan §Shared context item 10). +MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024 # 50 MB +MAX_FILE_SIZE_LABEL = "50 MB" +ALLOWED_EXTENSIONS = { + ".pdf", + ".doc", + ".docx", + ".ppt", + ".pptx", + ".odp", + ".odt", + ".key", +} +_PDF_MAGIC = b"%PDF" + + +def validate_publication_file(uploaded_file: Any) -> Tuple[str, int]: + """Validate an uploaded content-block file and return (extension, size_bytes). + + Rejects a disallowed extension, a file over the 50 MB cap, and a file that + claims a ``.pdf`` name but whose first bytes aren't the PDF magic number. + """ + name = getattr(uploaded_file, "name", "") or "" + extension = os.path.splitext(name)[1].lower() + + if extension not in ALLOWED_EXTENSIONS: + raise ValidationError( + f"'{extension or name}' is not an allowed file type. " + f"Allowed types: {', '.join(sorted(ALLOWED_EXTENSIONS))}." + ) + + size = getattr(uploaded_file, "size", 0) or 0 + if size > MAX_FILE_SIZE_BYTES: + raise ValidationError(f"File is larger than the {MAX_FILE_SIZE_LABEL} limit.") + + if extension == ".pdf" and not _looks_like_pdf(uploaded_file): + raise ValidationError("File claims to be a PDF but its contents are not.") + + return extension, size + + +def _looks_like_pdf(uploaded_file: Any) -> bool: + """Peek at the first bytes to confirm a real PDF, then rewind the file.""" + try: + uploaded_file.seek(0) + head = uploaded_file.read(len(_PDF_MAGIC)) + uploaded_file.seek(0) + except (AttributeError, OSError): + return False + if isinstance(head, str): + head = head.encode("latin-1", errors="ignore") + return bool(head.startswith(_PDF_MAGIC)) diff --git a/api/utils/youtube.py b/api/utils/youtube.py new file mode 100644 index 0000000..3f03eeb --- /dev/null +++ b/api/utils/youtube.py @@ -0,0 +1,62 @@ +""" +youtube +─────── +Parse and validate YouTube links for content blocks. + +``extract_video_id`` pulls the 11-character video id out of any of the common +YouTube URL shapes (``watch?v=``, ``youtu.be/``, ``embed/``); ``validate_youtube_url`` +is the boundary guard — it returns that id or raises a clean ValidationError for +anything that isn't a recognisable YouTube video link. +""" + +import re +from typing import Optional +from urllib.parse import parse_qs, urlparse + +from django.core.exceptions import ValidationError + +# A YouTube video id is exactly 11 url-safe characters. +_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$") +_YOUTUBE_HOSTS = { + "youtube.com", + "www.youtube.com", + "m.youtube.com", + "youtu.be", + "www.youtu.be", +} + + +def extract_video_id(url: Optional[str]) -> Optional[str]: + """Return the 11-char video id from a YouTube url, or None if it isn't one. + + Handles ``watch?v=``, the ``youtu.be/`` short link, and the + ``/embed/`` player link. Any other host or a malformed id yields None. + """ + if not url or not url.strip(): + return None + + parsed = urlparse(url.strip()) + host = (parsed.hostname or "").lower() + if host not in _YOUTUBE_HOSTS: + return None + + candidate: Optional[str] = None + if host in {"youtu.be", "www.youtu.be"}: + candidate = parsed.path.lstrip("/").split("/")[0] + elif parsed.path == "/watch": + values = parse_qs(parsed.query).get("v") + candidate = values[0] if values else None + elif parsed.path.startswith(("/embed/", "/v/", "/shorts/")): + candidate = parsed.path.split("/")[2] if len(parsed.path.split("/")) > 2 else None + + if candidate and _VIDEO_ID.match(candidate): + return candidate + return None + + +def validate_youtube_url(url: Optional[str]) -> str: + """Return the video id for a valid YouTube url, else raise ValidationError.""" + video_id = extract_video_id(url) + if not video_id: + raise ValidationError("Enter a valid YouTube video URL.") + return video_id diff --git a/api/views/dynamic_chart_view.py b/api/views/dynamic_chart_view.py index 457d924..7e7ac0f 100644 --- a/api/views/dynamic_chart_view.py +++ b/api/views/dynamic_chart_view.py @@ -25,9 +25,7 @@ async def create_chart_details( # Validate chart type if chart_type not in ChartTypes.values: - return JsonResponse( - {"error": f"Unsupported chart type: {chart_type}"}, status=400 - ) + return JsonResponse({"error": f"Unsupported chart type: {chart_type}"}, status=400) # Set basic options options["x_axis_label"] = request_details.get("x_axis_label", "X-Axis") @@ -42,7 +40,7 @@ async def create_chart_details( ) # Handle y-axis columns with configurations - y_axis_columns = [] + y_axis_columns: list = [] if y_axis_configs := request_details.get("y_axis_column", []): y_axis_columns = [] for config in y_axis_configs: @@ -57,8 +55,7 @@ async def create_chart_details( value_mapping = { str(mapping["key"]): str(mapping["value"]) for mapping in raw_mappings - if mapping.get("key") is not None - and mapping.get("value") is not None + if mapping.get("key") is not None and mapping.get("value") is not None } y_axis_columns.append( @@ -158,9 +155,7 @@ async def create_chart_details( @csrf_exempt -async def generate_dynamic_chart( - request: HttpRequest, resource_id: uuid.UUID -) -> HttpResponse: +async def generate_dynamic_chart(request: HttpRequest, resource_id: uuid.UUID) -> HttpResponse: if request.method == "POST": try: # Fetch the resource asynchronously @@ -188,9 +183,7 @@ async def generate_dynamic_chart( return response # Default response: JSON - return JsonResponse( - json.loads(chart.dump_options_with_quotes()), safe=False - ) + return JsonResponse(json.loads(chart.dump_options_with_quotes()), safe=False) except Exception as e: return JsonResponse({"error": f"Error generating chart: {e}"}, status=500) diff --git a/api/views/publication_download_view.py b/api/views/publication_download_view.py new file mode 100644 index 0000000..deac243 --- /dev/null +++ b/api/views/publication_download_view.py @@ -0,0 +1,65 @@ +""" +publication_download_view +────────────────────────── +Serves a content block's file through an access gate instead of a plain public +media URL, so a DRAFT resource's files are never world-readable. A published +resource's files are open; a draft's are limited to its owner / org members. +Each successful download bumps the parent resource's ``download_count``. PDFs +are served inline (for the detail-page viewer); every other type downloads. + +This is a flow file: the gate and block lookup live in +``api/services/publication_blocks.py``. +""" + +import os +from typing import Any + +from django.db.models import F +from django.http import HttpRequest, HttpResponse, JsonResponse + +from api.models import Publication, PublicationBlock +from api.services.publication_blocks import can_access_block_file +from api.utils.enums import PublicationBlockType + +# Extensions we serve inline in the browser; everything else downloads. +_INLINE_CONTENT_TYPES = {".pdf": "application/pdf"} + + +def publication_block_download(request: HttpRequest, block_id: Any) -> HttpResponse: + """Serve a block's file if the caller may see its resource, else 404.""" + # Find the block and its parent resource. + try: + block: PublicationBlock = PublicationBlock.objects.select_related("publication").get( + id=block_id + ) + except PublicationBlock.DoesNotExist: + return JsonResponse({"error": "Not found"}, status=404) + + publication = block.publication + + # Gate: a draft's files are private — hide existence with a 404 on denial. + if not can_access_block_file(request.user, publication): + return JsonResponse({"error": "Not found"}, status=404) + + # Only file blocks have something to download. + if block.block_type != PublicationBlockType.FILE or not block.file: + return JsonResponse({"error": "Not found"}, status=404) + + # Count the download against the resource (race-safe increment). + Publication.objects.filter(id=publication.id).update(download_count=F("download_count") + 1) + + # Serve the bytes inline for a PDF, as an attachment otherwise. + return _build_file_response(block) + + +def _build_file_response(block: PublicationBlock) -> HttpResponse: + """Build the HTTP file response with the right content type and disposition.""" + stored_name = block.file.name or "" + extension = os.path.splitext(stored_name)[1].lower() + content_type = _INLINE_CONTENT_TYPES.get(extension, "application/octet-stream") + disposition = "inline" if extension in _INLINE_CONTENT_TYPES else "attachment" + + filename = block.file_name or os.path.basename(stored_name) + response = HttpResponse(block.file.read(), content_type=content_type) + response["Content-Disposition"] = f'{disposition}; filename="{filename}"' + return response diff --git a/authorization/permissions.py b/authorization/permissions.py index 60ce2d3..ce818e9 100644 --- a/authorization/permissions.py +++ b/authorization/permissions.py @@ -481,6 +481,14 @@ def _resolve_publication_id(kwargs: Any) -> Optional[Any]: payload = kwargs.get(input_key) if payload is not None and getattr(payload, "id", None): return payload.id + # Block-scoped mutations pass a block id — resolve to its parent publication. + block_id = kwargs.get("block_id") + if block_id: + from api.models import PublicationBlock + + block = PublicationBlock.objects.filter(id=block_id).first() + if block: + return block.publication_id return None diff --git a/tests/schema/test_publication_schema.py b/tests/schema/test_publication_schema.py index 890e648..26991be 100644 --- a/tests/schema/test_publication_schema.py +++ b/tests/schema/test_publication_schema.py @@ -426,3 +426,81 @@ def test_anonymous_listing_is_published_only(self, org_a_admin, org_a, resource_ statuses = [row["status"] for row in result.data["publications"]] assert statuses == ["PUBLISHED"] + + +# --------------------------------------------------------------------------- # +# Content-block mutations (wiring + cross-org gate) +# --------------------------------------------------------------------------- # +ADD_YOUTUBE = """ +mutation AddYt($id: UUID!, $url: String!) { + addPublicationYoutubeBlock(publicationId: $id, youtubeUrl: $url) { + success + data { id blockType youtubeVideoId position } + } +} +""" + +REMOVE_BLOCK = """ +mutation RemoveBlock($blockId: UUID!) { + removePublicationBlock(blockId: $blockId) { success data } +} +""" + + +@pytest.mark.django_db +class TestBlockMutations: + def test_org_member_adds_youtube_block(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + ADD_YOUTUBE, + ctx(org_a_admin, org_a), + {"id": str(publication.id), "url": "https://youtu.be/dQw4w9WgXcQ"}, + ) + + payload = result.data["addPublicationYoutubeBlock"] + assert payload["success"] is True + assert payload["data"]["youtubeVideoId"] == "dQw4w9WgXcQ" + assert publication.blocks.count() == 1 + + def test_invalid_youtube_url_is_rejected(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + ADD_YOUTUBE, + ctx(org_a_admin, org_a), + {"id": str(publication.id), "url": "https://vimeo.com/1"}, + ) + + assert result.data["addPublicationYoutubeBlock"]["success"] is False + assert publication.blocks.count() == 0 + + def test_other_org_cannot_add_block( + self, org_a_admin, org_b_admin, org_a, org_b, resource_type + ): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + ADD_YOUTUBE, + ctx(org_b_admin, org_b), + {"id": str(publication.id), "url": "https://youtu.be/dQw4w9WgXcQ"}, + ) + + assert result.data["addPublicationYoutubeBlock"]["success"] is False + assert publication.blocks.count() == 0 + + def test_other_org_cannot_remove_block( + self, org_a_admin, org_b_admin, org_a, org_b, resource_type + ): + publication = _make_publication(org_a_admin, org_a, resource_type) + block = publication.blocks.create( + position=0, + block_type="YOUTUBE", + youtube_url="https://youtu.be/dQw4w9WgXcQ", + youtube_video_id="dQw4w9WgXcQ", + ) + + result = run(REMOVE_BLOCK, ctx(org_b_admin, org_b), {"blockId": str(block.id)}) + + assert result.data["removePublicationBlock"]["success"] is False + assert publication.blocks.filter(id=block.id).exists() diff --git a/tests/test_publication_blocks.py b/tests/test_publication_blocks.py new file mode 100644 index 0000000..fbb1383 --- /dev/null +++ b/tests/test_publication_blocks.py @@ -0,0 +1,196 @@ +"""Layer 1/3/4 tests for content blocks: add/reorder/remove/replace + download gate.""" + +import os +from datetime import date + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import RequestFactory + +from api.models import Publication, PublicationBlock, ResourceType +from api.models.Organization import Organization +from api.services.publication_blocks import ( + add_file_block, + add_youtube_block, + remove_block, + reorder_blocks, + replace_block_file, +) +from api.utils.enums import PublicationBlockType, PublicationStatus +from api.views.publication_download_view import publication_block_download +from authorization.models import OrganizationMembership, Role, User + +VIDEO = "https://youtu.be/dQw4w9WgXcQ" + + +@pytest.fixture(autouse=True) +def media_root(settings, tmp_path): + settings.MEDIA_ROOT = str(tmp_path) + + +@pytest.fixture +def owner(db): + return User.objects.create(username="owner", keycloak_id="owner") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +@pytest.fixture +def publication(owner, resource_type): + return Publication.objects.create( + title="With Blocks", + user=owner, + resource_type=resource_type, + publication_date=date(2024, 1, 1), + ) + + +def _pdf(name="report.pdf"): + return SimpleUploadedFile(name, b"%PDF-1.7 content", content_type="application/pdf") + + +@pytest.mark.django_db +class TestAddBlocks: + def test_file_block_stores_metadata_and_position(self, publication): + block = add_file_block(publication, _pdf()) + + assert block.block_type == PublicationBlockType.FILE + assert block.file_format == "pdf" + assert block.file_size > 0 + assert block.position == 0 + + def test_youtube_block_extracts_video_id(self, publication): + block = add_youtube_block(publication, VIDEO) + + assert block.block_type == PublicationBlockType.YOUTUBE + assert block.youtube_video_id == "dQw4w9WgXcQ" + + def test_positions_increment_across_mixed_blocks(self, publication): + add_file_block(publication, _pdf("a.pdf")) + add_youtube_block(publication, VIDEO) + third = add_file_block(publication, _pdf("c.pdf")) + + assert third.position == 2 + + +@pytest.mark.django_db +class TestReorderAndRemove: + def test_removing_a_middle_block_renumbers_contiguously(self, publication): + a = add_youtube_block(publication, VIDEO) + b = add_youtube_block(publication, VIDEO) + c = add_youtube_block(publication, VIDEO) + + remove_block(b) + + positions = list(publication.blocks.order_by("position").values_list("position", flat=True)) + assert positions == [0, 1] + a.refresh_from_db() + c.refresh_from_db() + assert a.position == 0 and c.position == 1 + + def test_reorder_sets_positions_to_requested_order(self, publication): + a = add_youtube_block(publication, VIDEO) + b = add_youtube_block(publication, VIDEO) + c = add_youtube_block(publication, VIDEO) + + reorder_blocks(publication, [c.id, a.id, b.id]) + + a.refresh_from_db() + b.refresh_from_db() + c.refresh_from_db() + assert (c.position, a.position, b.position) == (0, 1, 2) + + +@pytest.mark.django_db +class TestReplaceFile: + def test_replace_swaps_file_and_removes_the_old_one(self, publication): + block = add_file_block(publication, _pdf("first.pdf")) + old_path = block.file.path + old_id = block.id + assert os.path.exists(old_path) + + replace_block_file(block, _pdf("second.pdf")) + + assert block.id == old_id # same row, in-place + assert block.file_name == "second.pdf" + assert not os.path.exists(old_path) # old file removed from disk + + +@pytest.mark.django_db +class TestDeleteRemovesFile: + def test_deleting_a_block_removes_its_file(self, publication): + block = add_file_block(publication, _pdf()) + path = block.file.path + assert os.path.exists(path) + + block.delete() + + assert not os.path.exists(path) # post_delete signal cleaned it up + + +# --------------------------------------------------------------------------- # +# Download gate (Layer 4) +# --------------------------------------------------------------------------- # +@pytest.fixture +def other_org_user(db): + role = Role.objects.create(name="admin", can_view=True, can_change=True, can_delete=True) + org = Organization.objects.create(name="Other", description="o", slug="other") + user = User.objects.create(username="outsider", keycloak_id="outsider") + OrganizationMembership.objects.create(user=user, organization=org, role=role) + return user + + +def _download(user, block_id): + request = RequestFactory().get(f"/api/publications/blocks/{block_id}/download/") + request.user = user + return publication_block_download(request, block_id) + + +@pytest.mark.django_db +class TestDownloadGate: + def test_published_file_downloads_and_counts(self, publication, owner): + publication.status = PublicationStatus.PUBLISHED + publication.save() + block = add_file_block(publication, _pdf()) + + response = _download(AnonymousUser(), block.id) + + assert response.status_code == 200 + publication.refresh_from_db() + assert publication.download_count == 1 + + def test_draft_file_hidden_from_anonymous(self, publication): + block = add_file_block(publication, _pdf()) + + response = _download(AnonymousUser(), block.id) + + assert response.status_code == 404 + publication.refresh_from_db() + assert publication.download_count == 0 + + def test_draft_file_hidden_from_other_org(self, publication, other_org_user): + block = add_file_block(publication, _pdf()) + + response = _download(other_org_user, block.id) + + assert response.status_code == 404 + + def test_owner_can_download_own_draft_file(self, publication, owner): + block = add_file_block(publication, _pdf()) + + response = _download(owner, block.id) + + assert response.status_code == 200 + + def test_pdf_is_served_inline(self, publication, owner): + publication.status = PublicationStatus.PUBLISHED + publication.save() + block = add_file_block(publication, _pdf()) + + response = _download(AnonymousUser(), block.id) + + assert response["Content-Disposition"].startswith("inline") diff --git a/tests/test_publication_uploads.py b/tests/test_publication_uploads.py new file mode 100644 index 0000000..f1d1caa --- /dev/null +++ b/tests/test_publication_uploads.py @@ -0,0 +1,36 @@ +"""Layer 2 tests for the content-block file validator.""" + +import pytest +from django.core.exceptions import ValidationError +from django.core.files.uploadedfile import SimpleUploadedFile + +from api.utils.publication_uploads import MAX_FILE_SIZE_BYTES, validate_publication_file + + +def _file(name, content=b"data", content_type="application/octet-stream"): + return SimpleUploadedFile(name, content, content_type=content_type) + + +class TestValidatePublicationFile: + def test_accepts_an_allowed_document(self): + extension, size = validate_publication_file(_file("brief.docx", b"x" * 10)) + assert extension == ".docx" + assert size == 10 + + def test_accepts_a_real_pdf(self): + extension, _ = validate_publication_file(_file("report.pdf", b"%PDF-1.7 body")) + assert extension == ".pdf" + + def test_rejects_a_disallowed_extension(self): + for name in ("malware.exe", "archive.zip", "data.csv"): + with pytest.raises(ValidationError): + validate_publication_file(_file(name)) + + def test_rejects_a_file_over_the_cap(self): + oversized = _file("big.pdf", b"%PDF" + b"0" * MAX_FILE_SIZE_BYTES) + with pytest.raises(ValidationError, match="50 MB"): + validate_publication_file(oversized) + + def test_rejects_a_pdf_that_is_not_really_a_pdf(self): + with pytest.raises(ValidationError, match="not"): + validate_publication_file(_file("fake.pdf", b"MZ this is an exe")) diff --git a/tests/test_youtube_url.py b/tests/test_youtube_url.py new file mode 100644 index 0000000..4a49dc6 --- /dev/null +++ b/tests/test_youtube_url.py @@ -0,0 +1,45 @@ +"""Layer 2 tests for the YouTube URL helpers.""" + +import pytest +from django.core.exceptions import ValidationError + +from api.utils.youtube import extract_video_id, validate_youtube_url + +VIDEO_ID = "dQw4w9WgXcQ" + + +class TestExtractVideoId: + @pytest.mark.parametrize( + "url", + [ + f"https://www.youtube.com/watch?v={VIDEO_ID}", + f"https://youtu.be/{VIDEO_ID}", + f"https://www.youtube.com/embed/{VIDEO_ID}", + f"https://m.youtube.com/watch?v={VIDEO_ID}&feature=share", + ], + ) + def test_extracts_the_same_id_from_every_shape(self, url): + assert extract_video_id(url) == VIDEO_ID + + @pytest.mark.parametrize( + "url", + [ + "https://vimeo.com/123456789", + "https://example.com/watch?v=abc", + "not a url", + "https://www.youtube.com/watch?v=tooShort", + "", + None, + ], + ) + def test_rejects_non_youtube_or_malformed(self, url): + assert extract_video_id(url) is None + + +class TestValidateYoutubeUrl: + def test_returns_id_for_a_valid_url(self): + assert validate_youtube_url(f"https://youtu.be/{VIDEO_ID}") == VIDEO_ID + + def test_raises_for_a_non_youtube_url(self): + with pytest.raises(ValidationError): + validate_youtube_url("https://vimeo.com/1") From 348b035bd0213095402dfbfb12282a4023b1e211 Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 13:19:08 +0530 Subject: [PATCH 11/57] =?UTF-8?q?feat(publications):=20add=20search=20?= =?UTF-8?q?=E2=80=94=20document,=20per-entity=20+=20unified,=20signals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - publication_document.py: PublicationDocument indexing real columns only (title/description/status/resource_type/sectors/geographies/owner/dates); should_index_object gates on PUBLISHED so drafts never index; explicit related_models = [Organization, User, ResourceType, Sector, Geography] with get_instances_from_related so a renamed Resource Type/sector/geo re-indexes. - search_publication.py: /api/search/publication/ (AllowAny) with resource_type/sector/geography facet filters. - Unified search: publication branch in index names, result normalization, type-count aggregation, default type list, and serializer fields. - publication_signals.py: publish adds / unpublish + delete drop the search document (draft never searchable); ES errors logged and swallowed. - Settings ES index name; search route. - Security: youtube URL validator now rejects non-http(s) schemes (a matching host on javascript: could otherwise be stored — stored-XSS vector). - Tests: index-decision, re-index mapping (incl. ResourceType), signal predicate, and the javascript-scheme regression. --- DataSpace/settings.py | 1 + api/signals/publication_signals.py | 94 ++++++++++++-- api/urls.py | 6 + api/utils/youtube.py | 5 + api/views/search_publication.py | 134 ++++++++++++++++++++ api/views/search_unified.py | 18 ++- search/documents/__init__.py | 1 + search/documents/publication_document.py | 154 +++++++++++++++++++++++ tests/test_publication_search.py | 101 +++++++++++++++ tests/test_youtube_url.py | 3 + 10 files changed, 509 insertions(+), 8 deletions(-) create mode 100644 api/views/search_publication.py create mode 100644 search/documents/publication_document.py create mode 100644 tests/test_publication_search.py diff --git a/DataSpace/settings.py b/DataSpace/settings.py index ee5a2a0..47dad67 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -277,6 +277,7 @@ "search.documents.dataset_document": "dataset", "search.documents.usecase_document": "usecase", "search.documents.aimodel_document": "aimodel", + "search.documents.publication_document": "publication", "search.documents.collaborative_document": "collaborative", "search.documents.publisher_document.OrganizationPublisherDocument": "organization_publisher", "search.documents.publisher_document.UserPublisherDocument": "user_publisher", diff --git a/api/signals/publication_signals.py b/api/signals/publication_signals.py index 1544d15..7ae2383 100644 --- a/api/signals/publication_signals.py +++ b/api/signals/publication_signals.py @@ -1,20 +1,23 @@ """ publication_signals ──────────────────── -Keeps a Resource's stored files from leaking. When a content block is deleted — -directly, or by the FK cascade when its parent publication is deleted — its file -is removed from disk. (Django's ORM never deletes the underlying file on its -own, so ``Resource`` today leaks files on delete; we intentionally don't repeat -that here.) +Two jobs. First, keep a Resource's stored files from leaking: when a content +block is deleted — directly, or by the FK cascade when its parent publication is +deleted — its file is removed from disk. Second, keep search honest: publishing +a resource adds it to the index, unpublishing or deleting it removes it, so a +draft is never searchable. (Django's ORM never deletes a file on its own, so +``Resource`` today leaks files on delete; we intentionally don't repeat that.) """ from typing import Any import structlog -from django.db.models.signals import post_delete +from django.db.models.signals import post_delete, pre_save from django.dispatch import receiver -from api.models import PublicationBlock +from api.models import Publication, PublicationBlock +from api.utils.enums import PublicationStatus +from search.documents import PublicationDocument logger = structlog.getLogger(__name__) @@ -25,3 +28,80 @@ def remove_publication_block_file(sender: Any, instance: PublicationBlock, **kwa if instance.file: instance.file.delete(save=False) logger.info("publication block file removed", block_id=str(instance.id)) + + +def _should_be_indexed(instance: Publication) -> bool: + """A resource belongs in search only while it is PUBLISHED.""" + return instance.status == PublicationStatus.PUBLISHED.value + + +@receiver(pre_save, sender=Publication) +def handle_publication_visibility(sender: Any, instance: Publication, **kwargs: Any) -> None: + """Add / refresh / drop the search document as a resource is published or not. + + New rows are handled by the django-elasticsearch-dsl signal processor. ES + errors are logged and swallowed — indexing is best-effort and a rebuild + reconciles it. + """ + if not instance.pk: + return + + try: + original = Publication.objects.get(pk=instance.pk) + except Publication.DoesNotExist: + return + + was_indexable = _should_be_indexed(original) + is_indexable = _should_be_indexed(instance) + + if was_indexable and is_indexable: + action = "update" + elif was_indexable and not is_indexable: + action = "delete" + elif not was_indexable and is_indexable: + action = "add" + else: + return + + try: + document = PublicationDocument.get(id=instance.id, ignore=404) + if action == "delete": + if document: + document.delete() + logger.info("resource removed from search index", publication_id=str(instance.id)) + else: + if document: + document.update(instance) + else: + PublicationDocument().update(instance) + logger.info( + "resource synced to search index", + publication_id=str(instance.id), + action=action, + ) + except Exception as exc: # pragma: no cover - logging only + logger.error( + "failed to sync resource search document", + publication_id=str(instance.id), + action=action, + error=str(exc), + ) + + +@receiver(post_delete, sender=Publication) +def remove_publication_document(sender: Any, instance: Publication, **kwargs: Any) -> None: + """Drop the search document when a resource is deleted.""" + try: + document = PublicationDocument.get(id=instance.id, ignore=404) + if document: + document.delete() + logger.info( + "deleted resource removed from search index", + publication_id=str(instance.id), + ) + except Exception as exc: # pragma: no cover - logging only + logger.error( + "failed to delete resource search document", + publication_id=str(instance.id), + error=str(exc), + ) diff --git a/api/urls.py b/api/urls.py index 73c1c55..0d9da9f 100644 --- a/api/urls.py +++ b/api/urls.py @@ -18,6 +18,7 @@ search_aimodel, search_collaborative, search_dataset, + search_publication, search_publisher, search_unified, search_usecase, @@ -52,6 +53,11 @@ path("search/dataset/", search_dataset.SearchDataset.as_view(), name="search_dataset"), path("search/usecase/", search_usecase.SearchUseCase.as_view(), name="search_usecase"), path("search/aimodel/", search_aimodel.SearchAIModel.as_view(), name="search_aimodel"), + path( + "search/publication/", + search_publication.SearchPublication.as_view(), + name="search_publication", + ), path( "search/collaborative/", search_collaborative.SearchCollaborative.as_view(), diff --git a/api/utils/youtube.py b/api/utils/youtube.py index 3f03eeb..fac44d7 100644 --- a/api/utils/youtube.py +++ b/api/utils/youtube.py @@ -36,6 +36,11 @@ def extract_video_id(url: Optional[str]) -> Optional[str]: return None parsed = urlparse(url.strip()) + # Reject anything that isn't a plain http(s) link — a matching host on a + # ``javascript:`` scheme (e.g. ``javascript://youtube.com/watch?v=...``) + # must never pass, or it becomes a stored-XSS vector when rendered. + if parsed.scheme not in ("http", "https"): + return None host = (parsed.hostname or "").lower() if host not in _YOUTUBE_HOSTS: return None diff --git a/api/views/search_publication.py b/api/views/search_publication.py new file mode 100644 index 0000000..3f5260e --- /dev/null +++ b/api/views/search_publication.py @@ -0,0 +1,134 @@ +"""Search view for Publication (UI "Resource") using Elasticsearch. + +Only published resources are in the index (the document's ``should_index_object`` +gate), so this public endpoint never leaks drafts. Filters: resource type, +sector, geography. +""" + +from typing import Any, Dict, List, Optional, Tuple, Union + +import structlog +from elasticsearch_dsl import Q as ESQ +from elasticsearch_dsl import Search +from elasticsearch_dsl.query import Query as ESQuery +from rest_framework import serializers +from rest_framework.permissions import AllowAny + +from api.models.Publication import Publication +from api.utils.telemetry_utils import trace_method +from api.views.paginated_elastic_view import PaginatedElasticSearchAPIView +from search.documents import PublicationDocument + +logger = structlog.get_logger(__name__) + + +class PublicationDocumentSerializer(serializers.ModelSerializer): + """Serializer for the Publication search document.""" + + resource_type = serializers.CharField(allow_blank=True) + sectors = serializers.ListField() + geographies = serializers.ListField() + + class OrganizationSerializer(serializers.Serializer): + name = serializers.CharField() + logo = serializers.CharField() + + class UserSerializer(serializers.Serializer): + name = serializers.CharField() + bio = serializers.CharField() + profile_picture = serializers.CharField() + + organization = OrganizationSerializer(allow_null=True) + user = UserSerializer(allow_null=True) + + class Meta: + model = Publication + fields = [ + "id", + "title", + "description", + "status", + "resource_type", + "sectors", + "geographies", + "created", + "modified", + "organization", + "user", + ] + + +class SearchPublication(PaginatedElasticSearchAPIView): + """View for searching resources.""" + + serializer_class = PublicationDocumentSerializer + document_class = PublicationDocument + permission_classes = [AllowAny] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.searchable_fields: List[str] + self.aggregations: Dict[str, str] + self.searchable_fields, self.aggregations = self.get_searchable_and_aggregations() + self.logger = structlog.get_logger(__name__) + + @trace_method( + name="get_searchable_and_aggregations", + attributes={"component": "search_publication"}, + ) + def get_searchable_and_aggregations(self) -> Tuple[List[str], Dict[str, str]]: + """Searchable fields (name/description) and the three facet aggregations.""" + searchable_fields: List[str] = ["title", "description"] + aggregations: Dict[str, str] = { + "status": "terms", + "resource_type.raw": "terms", + "sectors.raw": "terms", + "geographies.raw": "terms", + } + return searchable_fields, aggregations + + @trace_method(name="add_aggregations", attributes={"component": "search_publication"}) + def add_aggregations(self, search: Search) -> Search: + """Add the facet aggregations to the search query.""" + for aggregation_field in self.aggregations: + search.aggs.bucket( + aggregation_field.replace(".raw", ""), + self.aggregations[aggregation_field], + field=aggregation_field, + ) + return search + + @trace_method(name="generate_q_expression", attributes={"component": "search_publication"}) + def generate_q_expression(self, query: str) -> Optional[Union[ESQuery, List[ESQuery]]]: + """Build a fuzzy query over the searchable fields, or match-all when blank.""" + if query: + queries: List[ESQuery] = [ + ESQ("fuzzy", **{field: {"value": query, "fuzziness": "AUTO"}}) + for field in self.searchable_fields + ] + else: + queries = [ESQ("match_all")] + return ESQ("bool", should=queries, minimum_should_match=1) + + @trace_method(name="add_filters", attributes={"component": "search_publication"}) + def add_filters(self, filters: Dict[str, str], search: Search) -> Search: + """Apply resource-type / sector / geography facet filters.""" + for filter_key in filters: + if filter_key in ["resource_type", "sectors", "geographies"]: + raw_filter = filter_key + ".raw" + filter_values = filters[filter_key].split(",") + search = search.filter("terms", **{raw_filter: filter_values}) + elif filter_key == "status": + search = search.filter("term", **{filter_key: filters[filter_key]}) + return search + + @trace_method(name="add_sort", attributes={"component": "search_publication"}) + def add_sort(self, sort: str, search: Search, order: str) -> Search: + """Apply a sort mode (alphabetical / recent / created).""" + if sort == "alphabetical": + search = search.sort({"title.raw": {"order": order}}) + elif sort == "recent": + search = search.sort({"modified": {"order": order}}) + elif sort == "created": + search = search.sort({"created": {"order": order}}) + return search diff --git a/api/views/search_unified.py b/api/views/search_unified.py index b84c6da..0b02247 100644 --- a/api/views/search_unified.py +++ b/api/views/search_unified.py @@ -80,6 +80,10 @@ class UserSerializer(serializers.Serializer): provider = serializers.CharField(required=False) is_individual_model = serializers.BooleanField(required=False) + # Publication (Resource) specific + resource_type = serializers.CharField(required=False) + is_individual_publication = serializers.BooleanField(required=False) + # Collaborative specific is_individual_collaborative = serializers.BooleanField(required=False) website = serializers.CharField(required=False) @@ -132,6 +136,12 @@ def _get_index_names(self, types_list: List[str]) -> List[str]: ) index_names.append(aimodel_index) + if "publication" in types_list: + publication_index = settings.ELASTICSEARCH_INDEX_NAMES.get( + "search.documents.publication_document", "publication" + ) + index_names.append(publication_index) + if "collaborative" in types_list: collaborative_index = settings.ELASTICSEARCH_INDEX_NAMES.get( "search.documents.collaborative_document", "collaborative" @@ -312,6 +322,8 @@ def _normalize_result(self, hit: Any) -> Dict[str, Any]: result["type"] = "usecase" elif "aimodel" in index_name: result["type"] = "aimodel" + elif "publication" in index_name: + result["type"] = "publication" elif "collaborative" in index_name: result["type"] = "collaborative" elif "publisher" in index_name: @@ -424,6 +436,8 @@ def perform_unified_search( aggregations["types"]["usecase"] = bucket["doc_count"] elif "aimodel" in index_name: aggregations["types"]["aimodel"] = bucket["doc_count"] + elif "publication" in index_name: + aggregations["types"]["publication"] = bucket["doc_count"] elif "collaborative" in index_name: aggregations["types"]["collaborative"] = bucket["doc_count"] elif "publisher" in index_name: @@ -448,7 +462,9 @@ def _generate_unified_cache_key(self, request: Any) -> str: "query": request.GET.get("query", ""), "page": request.GET.get("page", "1"), "size": request.GET.get("size", "10"), - "types": request.GET.get("types", "dataset,usecase,aimodel,collaborative,publisher"), + "types": request.GET.get( + "types", "dataset,usecase,aimodel,publication,collaborative,publisher" + ), "filters": str(sorted(request.GET.dict().items())), "version": str(cache.get(SEARCH_CACHE_VERSION_KEY, 0)), } diff --git a/search/documents/__init__.py b/search/documents/__init__.py index 96a2345..4c6a678 100644 --- a/search/documents/__init__.py +++ b/search/documents/__init__.py @@ -1,6 +1,7 @@ from search.documents.aimodel_document import AIModelDocument from search.documents.collaborative_document import CollaborativeDocument from search.documents.dataset_document import DatasetDocument +from search.documents.publication_document import PublicationDocument from search.documents.publisher_document import ( OrganizationPublisherDocument, UserPublisherDocument, diff --git a/search/documents/publication_document.py b/search/documents/publication_document.py new file mode 100644 index 0000000..d50ec9f --- /dev/null +++ b/search/documents/publication_document.py @@ -0,0 +1,154 @@ +"""Elasticsearch document for Publication (UI "Resource"). + +v1 indexes real columns only — title, description, status, resource_type, +sectors, geographies, owner and dates. The author / date / usage-rights columns +and block content are intentionally NOT indexed (see the plan's Limitations). +``related_models`` is stated explicitly (not cloned blind) so that renaming a +Resource Type, sector or geography re-indexes the affected publications. +""" + +from typing import Any, Dict, List, Optional, Union + +from django_elasticsearch_dsl import Document, Index, KeywordField, fields + +from api.models.Geography import Geography +from api.models.Organization import Organization +from api.models.Publication import Publication +from api.models.ResourceType import ResourceType +from api.models.Sector import Sector +from api.utils.enums import PublicationStatus +from authorization.models import User +from DataSpace import settings +from search.documents.analysers import html_strip, ngram_analyser + +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) +INDEX.settings(number_of_shards=1, number_of_replicas=0) + + +@INDEX.doc_type +class PublicationDocument(Document): + """Elasticsearch document for a published Resource.""" + + title = fields.TextField( + analyzer=ngram_analyser, + fields={"raw": KeywordField(multi=False)}, + ) + description = fields.TextField( + analyzer=html_strip, + fields={"raw": fields.TextField(analyzer="keyword")}, + ) + status = fields.KeywordField() + + # Resource Type facet (name). + resource_type = fields.TextField( + attr="resource_type_indexing", + analyzer=ngram_analyser, + fields={"raw": KeywordField(multi=False)}, + ) + + # Sectors facet (ManyToMany). + sectors = fields.TextField( + attr="sectors_indexing", + analyzer=ngram_analyser, + fields={ + "raw": fields.KeywordField(multi=True), + "suggest": fields.CompletionField(multi=True), + }, + multi=True, + ) + + # Geographies facet (ManyToMany). + geographies = fields.TextField( + attr="geographies_indexing", + analyzer=ngram_analyser, + fields={ + "raw": fields.KeywordField(multi=True), + "suggest": fields.CompletionField(multi=True), + }, + multi=True, + ) + + organization = fields.NestedField( + properties={ + "name": fields.TextField(analyzer=ngram_analyser), + "logo": fields.TextField(analyzer=ngram_analyser), + } + ) + user = fields.NestedField( + properties={ + "name": fields.TextField(analyzer=ngram_analyser), + "bio": fields.TextField(analyzer=html_strip), + "profile_picture": fields.TextField(analyzer=ngram_analyser), + } + ) + + def prepare_organization(self, instance: Publication) -> Optional[Dict[str, str]]: + """Prepare the owning organization for indexing.""" + if instance.organization: + org = instance.organization + return {"name": org.name, "logo": org.logo.url if org.logo else ""} + return None + + def prepare_user(self, instance: Publication) -> Optional[Dict[str, str]]: + """Prepare the owning user for indexing.""" + if instance.user: + user = instance.user + return { + "name": user.full_name, + "bio": user.bio or "", + "profile_picture": (user.profile_picture.url if user.profile_picture else ""), + } + return None + + def should_index_object(self, obj: Any) -> bool: + """Only PUBLISHED resources are indexed — drafts never reach search.""" + return bool(obj.status == PublicationStatus.PUBLISHED.value) + + def save(self, *args: Any, **kwargs: Any) -> None: + """Index a published resource, or drop it from the index otherwise.""" + if self.should_index_object(self.to_dict()): # type: ignore + super().save(*args, **kwargs) + else: + self.delete(ignore=404) + + def get_queryset(self) -> Any: + """Only published resources are populated into the index.""" + return ( + super(PublicationDocument, self) + .get_queryset() + .filter(status=PublicationStatus.PUBLISHED) + ) + + def get_instances_from_related( + self, + related_instance: Union[Organization, User, ResourceType, Sector, Geography], + ) -> Optional[List[Publication]]: + """Re-index the publications affected when a related row changes. + + Covers a renamed Resource Type / sector / geography and an updated owner + so already-indexed facets don't go stale. + """ + if isinstance(related_instance, Organization): + return list(related_instance.publications.all()) + if isinstance(related_instance, User): + return list(related_instance.publications.all()) + if isinstance(related_instance, ResourceType): + return list(related_instance.publications.all()) + if isinstance(related_instance, Sector): + return list(related_instance.publications.all()) + if isinstance(related_instance, Geography): + return list(related_instance.publications.all()) + return None + + class Django: + """Django model configuration.""" + + model = Publication + + fields = [ + "id", + "created", + "modified", + ] + + related_models = [Organization, User, ResourceType, Sector, Geography] diff --git a/tests/test_publication_search.py b/tests/test_publication_search.py new file mode 100644 index 0000000..3179f30 --- /dev/null +++ b/tests/test_publication_search.py @@ -0,0 +1,101 @@ +"""Search-layer tests for Publication. + +Elasticsearch itself is disabled in the deterministic layers, so these verify +the index-decision logic and the related-model re-index mapping (which is what +prevents draft leakage and stale facets) rather than round-tripping a cluster. +The full query/filter/pagination behaviour is a Layer-4 scenario that runs +against a live index (documented in the arch doc). +""" + +from datetime import date + +import pytest + +from api.models import Geography, Publication, ResourceType, Sector +from api.models.Organization import Organization +from api.signals.publication_signals import _should_be_indexed +from api.utils.enums import GeoTypes, PublicationStatus +from authorization.models import User +from search.documents import PublicationDocument +from search.documents.publication_document import PublicationDocument as DocClass + + +@pytest.fixture +def user(db): + return User.objects.create(username="author", keycloak_id="author") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +def _publication(user, resource_type, status=PublicationStatus.PUBLISHED, **extra): + return Publication.objects.create( + title="Findings", + description="d", + user=user, + resource_type=resource_type, + publication_date=date(2024, 1, 1), + status=status, + **extra, + ) + + +@pytest.mark.django_db +class TestIndexDecision: + def test_published_resource_is_indexed(self, user, resource_type): + publication = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + assert PublicationDocument().should_index_object(publication) is True + + def test_draft_resource_is_not_indexed(self, user, resource_type): + publication = _publication(user, resource_type, status=PublicationStatus.DRAFT) + assert PublicationDocument().should_index_object(publication) is False + + def test_signal_predicate_matches_published(self, user, resource_type): + published = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + draft = _publication(user, resource_type, status=PublicationStatus.DRAFT) + assert _should_be_indexed(published) is True + assert _should_be_indexed(draft) is False + + +@pytest.mark.django_db +class TestReindexMapping: + def test_related_models_include_resource_type(self): + # A renamed Resource Type must re-index affected publications, so the + # facet doesn't go stale — ResourceType must be a related model. + assert ResourceType in DocClass.Django.related_models + + def test_renaming_a_resource_type_finds_affected_publications(self, user, resource_type): + publication = _publication(user, resource_type) + + affected = PublicationDocument().get_instances_from_related(resource_type) + + assert list(affected) == [publication] + + def test_changing_a_sector_finds_affected_publications(self, user, resource_type): + sector = Sector.objects.create(name="Health") + publication = _publication(user, resource_type) + publication.sectors.add(sector) + + affected = PublicationDocument().get_instances_from_related(sector) + + assert list(affected) == [publication] + + def test_changing_a_geography_finds_affected_publications(self, user, resource_type): + geography = Geography.objects.create(name="India", code="IN", type=GeoTypes.COUNTRY) + publication = _publication(user, resource_type) + publication.geographies.add(geography) + + affected = PublicationDocument().get_instances_from_related(geography) + + assert list(affected) == [publication] + + def test_org_owned_publication_reindexes_on_org_change(self, resource_type): + org = Organization.objects.create(name="Org", description="o", slug="org") + owner = User.objects.create(username="member", keycloak_id="member") + publication = _publication(owner, resource_type, organization=org) + + affected = PublicationDocument().get_instances_from_related(org) + + assert list(affected) == [publication] diff --git a/tests/test_youtube_url.py b/tests/test_youtube_url.py index 4a49dc6..ba21772 100644 --- a/tests/test_youtube_url.py +++ b/tests/test_youtube_url.py @@ -30,6 +30,9 @@ def test_extracts_the_same_id_from_every_shape(self, url): "https://www.youtube.com/watch?v=tooShort", "", None, + # A matching host on a dangerous scheme must never pass (stored XSS). + f"javascript://youtube.com/watch?v={VIDEO_ID}", + f"data://youtu.be/{VIDEO_ID}", ], ) def test_rejects_non_youtube_or_malformed(self, url): From b345816338e7b19c016edaf6f778a9c0782da428 Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 13:48:59 +0530 Subject: [PATCH 12/57] feat(publications): link published resources into use cases & collaboratives - publication_linking.py: only-PUBLISHED-is-linkable guard + published-filter helper (Resources are the first non-Dataset linkable entity). - UC & Collaborative schemas gain add/remove/update_*_publications trios (DRAFT-guard on the parent, only-published on the resource). - TypeUseCase/TypeCollaborative expose a published-only 'publications' render field so an unpublished/deleted resource silently drops out and reappears on re-publish. - TypePublication exposes linkedUsecases/linkedCollaboratives/linkedCount for the owner's 'linked to N' flag. - Tests: link guard, stale-link (both UC and Collab), linked-count, cross-org link affordance. --- api/schema/collaborative_schema.py | 63 ++++++++++ api/schema/usecase_schema.py | 63 ++++++++++ api/services/publication_linking.py | 38 ++++++ api/types/type_collaborative.py | 19 ++- api/types/type_publication.py | 58 ++++++++- api/types/type_usecase.py | 17 ++- tests/test_publication_linking.py | 188 ++++++++++++++++++++++++++++ 7 files changed, 443 insertions(+), 3 deletions(-) create mode 100644 api/services/publication_linking.py create mode 100644 tests/test_publication_linking.py diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index 3e0f5b9..a8e78e4 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -29,6 +29,10 @@ UseCase, ) from api.schema.extensions import TrackActivity, TrackModelActivity +from api.services.publication_linking import ( + get_linkable_publication, + published_publications, +) from api.types.type_collaborative import ( CollaborativeFilter, CollaborativeOrder, @@ -533,6 +537,65 @@ def update_collaborative_datasets( collaborative.save() return TypeCollaborative.from_django(collaborative) + @strawberry_django.mutation(handle_django_errors=True) + def add_publication_to_collaborative( + self, info: Info, collaborative_id: str, publication_id: uuid.UUID + ) -> TypeCollaborative: + """Link a published resource to a collaborative (only PUBLISHED linkable).""" + # Reject a missing or draft resource before touching the link. + publication = get_linkable_publication(publication_id) + + try: + collaborative = Collaborative.objects.get(id=collaborative_id) + except Collaborative.DoesNotExist: + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") + + if collaborative.status != CollaborativeStatus.DRAFT: + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") + + collaborative.publications.add(publication) + collaborative.save() + return TypeCollaborative.from_django(collaborative) + + @strawberry_django.mutation(handle_django_errors=True) + def remove_publication_from_collaborative( + self, info: Info, collaborative_id: str, publication_id: uuid.UUID + ) -> TypeCollaborative: + """Unlink a resource from a collaborative.""" + try: + collaborative = Collaborative.objects.get(id=collaborative_id) + except Collaborative.DoesNotExist: + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") + + if collaborative.status != CollaborativeStatus.DRAFT: + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") + + collaborative.publications.remove(publication_id) # type: ignore[arg-type] + collaborative.save() + return TypeCollaborative.from_django(collaborative) + + @strawberry_django.mutation(handle_django_errors=True) + @trace_resolver( + name="update_collaborative_publications", + attributes={"component": "collaborative", "operation": "mutation"}, + ) + def update_collaborative_publications( + self, info: Info, collaborative_id: str, publication_ids: List[uuid.UUID] + ) -> TypeCollaborative: + """Set the linked resources — only published ones are attached.""" + try: + collaborative = Collaborative.objects.get(id=collaborative_id) + except Collaborative.DoesNotExist: + raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") + + if collaborative.status != CollaborativeStatus.DRAFT: + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") + + # Attach only the published resources among the given ids. + collaborative.publications.set(published_publications(publication_ids)) + collaborative.save() + return TypeCollaborative.from_django(collaborative) + @strawberry_django.mutation(handle_django_errors=True) @trace_resolver( name="update_collaborative_use_cases", diff --git a/api/schema/usecase_schema.py b/api/schema/usecase_schema.py index 317e0c2..be4aaa7 100644 --- a/api/schema/usecase_schema.py +++ b/api/schema/usecase_schema.py @@ -29,6 +29,10 @@ UseCaseOrganizationRelationship, ) from api.schema.extensions import TrackActivity, TrackModelActivity +from api.services.publication_linking import ( + get_linkable_publication, + published_publications, +) from api.types.type_dataset import TypeDataset from api.types.type_organization import TypeOrganization from api.types.type_usecase import TypeUseCase, UseCaseFilter, UseCaseOrder @@ -476,6 +480,65 @@ def update_usecase_datasets( use_case.save() return TypeUseCase.from_django(use_case) + @strawberry_django.mutation(handle_django_errors=True) + def add_publication_to_use_case( + self, info: Info, use_case_id: str, publication_id: uuid.UUID + ) -> TypeUseCase: + """Link a published resource to a use case (only PUBLISHED are linkable).""" + # Reject a missing or draft resource before touching the link. + publication = get_linkable_publication(publication_id) + + try: + use_case = UseCase.objects.get(id=use_case_id) + except UseCase.DoesNotExist: + raise ValueError(f"UseCase with ID {use_case_id} does not exist.") + + if use_case.status != UseCaseStatus.DRAFT: + raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") + + use_case.publications.add(publication) + use_case.save() + return TypeUseCase.from_django(use_case) + + @strawberry_django.mutation(handle_django_errors=True) + def remove_publication_from_use_case( + self, info: Info, use_case_id: str, publication_id: uuid.UUID + ) -> TypeUseCase: + """Unlink a resource from a use case.""" + try: + use_case = UseCase.objects.get(id=use_case_id) + except UseCase.DoesNotExist: + raise ValueError(f"UseCase with ID {use_case_id} does not exist.") + + if use_case.status != UseCaseStatus.DRAFT: + raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") + + use_case.publications.remove(publication_id) # type: ignore[arg-type] + use_case.save() + return TypeUseCase.from_django(use_case) + + @strawberry_django.mutation(handle_django_errors=True) + @trace_resolver( + name="update_usecase_publications", + attributes={"component": "usecase", "operation": "mutation"}, + ) + def update_usecase_publications( + self, info: Info, use_case_id: str, publication_ids: List[uuid.UUID] + ) -> TypeUseCase: + """Set the linked resources — only published ones are attached.""" + try: + use_case = UseCase.objects.get(id=use_case_id) + except UseCase.DoesNotExist: + raise ValueError(f"Use Case with ID {use_case_id} doesn't exist") + + if use_case.status != UseCaseStatus.DRAFT: + raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") + + # Attach only the published resources among the given ids. + use_case.publications.set(published_publications(publication_ids)) + use_case.save() + return TypeUseCase.from_django(use_case) + @strawberry_django.mutation( handle_django_errors=True, extensions=[ diff --git a/api/services/publication_linking.py b/api/services/publication_linking.py new file mode 100644 index 0000000..ff3d5bc --- /dev/null +++ b/api/services/publication_linking.py @@ -0,0 +1,38 @@ +""" +publication_linking +──────────────────── +Helpers for pulling published Resources into Use Cases and Collaboratives. + +Resources are the first non-Dataset entity that can be linked, and only a +PUBLISHED resource may be linked — like a public library. These guard the link +mutations and the published-only render filter so a draft can never sneak into a +Use Case / Collaborative, and an unpublished resource silently drops out of the +render. +""" + +from typing import Any, List + +from api.models import Publication +from api.utils.enums import PublicationStatus + + +def get_linkable_publication(publication_id: Any) -> Publication: + """Load a resource that is allowed to be linked, or raise. + + Only a PUBLISHED resource is linkable; a missing or draft one raises a clean + error so the plain link mutation never attaches a draft. + """ + try: + publication = Publication.objects.get(id=publication_id) + except Publication.DoesNotExist: + raise ValueError(f"Resource {publication_id} does not exist.") + if publication.status != PublicationStatus.PUBLISHED.value: + raise ValueError("Only a published resource can be linked.") + return publication + + +def published_publications(publication_ids: List[Any]) -> List[Publication]: + """Return the published resources among the given ids (drafts dropped).""" + return list( + Publication.objects.filter(id__in=publication_ids, status=PublicationStatus.PUBLISHED) + ) diff --git a/api/types/type_collaborative.py b/api/types/type_collaborative.py index 89a3cf9..aa62463 100644 --- a/api/types/type_collaborative.py +++ b/api/types/type_collaborative.py @@ -22,10 +22,15 @@ from api.types.type_dataset import TypeDataset, TypeTag from api.types.type_geo import TypeGeo from api.types.type_organization import TypeOrganization +from api.types.type_publication import TypePublication from api.types.type_sdg import TypeSDG from api.types.type_sector import TypeSector from api.types.type_usecase import TypeUseCase -from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType +from api.utils.enums import ( + CollaborativeStatus, + OrganizationRelationshipType, + PublicationStatus, +) from authorization.types import TypeUser collaborative_status = strawberry.enum(CollaborativeStatus) # type: ignore @@ -95,6 +100,18 @@ def datasets(self) -> Optional[List["TypeDataset"]]: except Exception: return [] + @strawberry.field(description="Get published resources linked to this collaborative.") + def publications(self) -> Optional[List["TypePublication"]]: + """Get published resources linked to this collaborative (drafts hidden).""" + try: + # Only published resources render — an unpublished one drops out. + queryset = self.publications.filter( # type: ignore + status=PublicationStatus.PUBLISHED + ).order_by("-modified") + return TypePublication.from_django_list(queryset) + except Exception: + return [] + @strawberry.field(description="Get use cases associated with this collaborative.") def use_cases(self) -> Optional[List["TypeUseCase"]]: """Get use cases associated with this collaborative.""" diff --git a/api/types/type_publication.py b/api/types/type_publication.py index 7957fc5..b68073c 100644 --- a/api/types/type_publication.py +++ b/api/types/type_publication.py @@ -7,7 +7,13 @@ from strawberry.enum import EnumType from strawberry.types import Info -from api.models import Publication, PublicationBlock, ResourceType +from api.models import ( + Collaborative, + Publication, + PublicationBlock, + ResourceType, + UseCase, +) from api.types.base_type import BaseType from api.types.type_geo import TypeGeo from api.types.type_organization import TypeOrganization @@ -22,6 +28,19 @@ publication_license: EnumType = strawberry.enum(DatasetLicense) # type: ignore +@strawberry.type +class TypeLinkedReference: + """A lightweight pointer to a Use Case / Collaborative a resource is linked into. + + Kept minimal (id / title / slug) so ``TypePublication`` can name where it's + linked without importing the heavier Use Case / Collaborative types. + """ + + id: str + title: str + slug: str + + @strawberry_django.type(ResourceType) class TypeResourceType(BaseType): """Type for the admin-managed Resource Type lookup.""" @@ -120,3 +139,40 @@ def blocks(self, info: Info) -> List["TypePublicationBlock"]: def is_individual_publication(self) -> bool: """True when owned by an individual rather than an organization.""" return self.organization is None + + @strawberry.field + def linked_usecases(self) -> List["TypeLinkedReference"]: + """Use Cases this resource is linked into (owner's 'linked to N' flag).""" + try: + instance = cast(Publication, self) + usecases: List[UseCase] = list(instance.usecase_set.all()) # type: ignore[attr-defined] + return [ + TypeLinkedReference(id=str(uc.id), title=uc.title or "", slug=uc.slug or "") + for uc in usecases + ] + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def linked_collaboratives(self) -> List["TypeLinkedReference"]: + """Collaboratives this resource is linked into (owner's 'linked to N' flag).""" + try: + instance = cast(Publication, self) + collabs: List[Collaborative] = list(instance.collaborative_set.all()) # type: ignore[attr-defined] + return [ + TypeLinkedReference( + id=str(collab.id), title=collab.title or "", slug=collab.slug or "" + ) + for collab in collabs + ] + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def linked_count(self) -> int: + """Total Use Cases + Collaboratives this resource is linked into.""" + try: + instance = cast(Publication, self) + return instance.usecase_set.count() + instance.collaborative_set.count() # type: ignore[attr-defined] + except (AttributeError, Publication.DoesNotExist): + return 0 diff --git a/api/types/type_usecase.py b/api/types/type_usecase.py index 96fd0d9..eb801ff 100644 --- a/api/types/type_usecase.py +++ b/api/types/type_usecase.py @@ -19,12 +19,17 @@ from api.types.type_dataset import TypeDataset, TypeTag from api.types.type_geo import TypeGeo from api.types.type_organization import TypeOrganization +from api.types.type_publication import TypePublication from api.types.type_sdg import TypeSDG from api.types.type_sector import TypeSector from api.types.type_usecase_dashboard import TypeUseCaseDashboard from api.types.type_usecase_metadata import TypeUseCaseMetadata from api.types.type_usecase_organization import TypeUseCaseOrganizationRelationship -from api.utils.enums import OrganizationRelationshipType, UseCaseStatus +from api.utils.enums import ( + OrganizationRelationshipType, + PublicationStatus, + UseCaseStatus, +) from authorization.types import TypeUser use_case_status = strawberry.enum(UseCaseStatus) # type: ignore @@ -91,6 +96,16 @@ def dataset_count(self: "TypeUseCase", info: Info) -> int: except Exception: return 0 + @strawberry.field(description="Get published resources linked to this use case.") + def publications(self) -> Optional[List["TypePublication"]]: + """Get published resources linked to this use case (drafts hidden).""" + try: + # Only published resources render — an unpublished one drops out. + queryset = self.publications.filter(status=PublicationStatus.PUBLISHED) # type: ignore + return TypePublication.from_django_list(queryset) + except Exception: + return [] + @strawberry.field(description="Get publishers associated with this use case.") def publishers(self) -> Optional[List["TypeOrganization"]]: """Get publishers associated with this use case.""" diff --git a/tests/test_publication_linking.py b/tests/test_publication_linking.py new file mode 100644 index 0000000..7e81753 --- /dev/null +++ b/tests/test_publication_linking.py @@ -0,0 +1,188 @@ +"""Layer 3/4 tests for linking published Resources into Use Cases / Collaboratives. + +Covers the only-PUBLISHED-is-linkable guard, the render-time published-only +filter (stale links vanish on unpublish/delete and reappear on re-publish), the +owner's linked-count flag, and the one intentional cross-org affordance. +""" + +import types +from datetime import date + +import pytest + +from api.models import Collaborative, Publication, ResourceType, UseCase +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.enums import PublicationStatus +from authorization.models import User + + +@pytest.fixture +def user(db): + return User.objects.create(username="author", keycloak_id="author") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +def _publication(user, resource_type, status=PublicationStatus.PUBLISHED, org=None): + return Publication.objects.create( + title="Findings", + description="d", + user=user, + resource_type=resource_type, + publication_date=date(2024, 1, 1), + status=status, + organization=org, + ) + + +_uc_counter = [0] +_collab_counter = [0] + + +def _use_case(user): + _uc_counter[0] += 1 + return UseCase.objects.create(title=f"UC {_uc_counter[0]}", user=user) + + +def _collaborative(user): + _collab_counter[0] += 1 + return Collaborative.objects.create(title=f"Collab {_collab_counter[0]}", user=user) + + +def ctx(user): + return types.SimpleNamespace(user=user, context={}) + + +def run(query, user, variables): + return schema.execute_sync(query, variable_values=variables, context_value=ctx(user)) + + +ADD_TO_UC = """ +mutation($ucId: String!, $pubId: UUID!) { + addPublicationToUseCase(useCaseId: $ucId, publicationId: $pubId) { __typename } +} +""" + +ADD_TO_COLLAB = """ +mutation($cId: String!, $pubId: UUID!) { + addPublicationToCollaborative(collaborativeId: $cId, publicationId: $pubId) { __typename } +} +""" + + +def _published_only(entity): + return list(entity.publications.filter(status=PublicationStatus.PUBLISHED)) + + +# --------------------------------------------------------------------------- # +# Link guard +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestLinkGuard: + def test_published_resource_links_to_use_case(self, user, resource_type): + uc = _use_case(user) + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + + run(ADD_TO_UC, user, {"ucId": str(uc.id), "pubId": str(pub.id)}) + + assert uc.publications.filter(id=pub.id).exists() + + def test_draft_resource_is_rejected(self, user, resource_type): + uc = _use_case(user) + draft = _publication(user, resource_type, status=PublicationStatus.DRAFT) + + run(ADD_TO_UC, user, {"ucId": str(uc.id), "pubId": str(draft.id)}) + + assert uc.publications.count() == 0 # guard held — nothing linked + + def test_published_resource_links_to_collaborative(self, user, resource_type): + collab = _collaborative(user) + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + + run(ADD_TO_COLLAB, user, {"cId": str(collab.id), "pubId": str(pub.id)}) + + assert collab.publications.filter(id=pub.id).exists() + + def test_draft_resource_is_rejected_by_collaborative(self, user, resource_type): + collab = _collaborative(user) + draft = _publication(user, resource_type, status=PublicationStatus.DRAFT) + + run(ADD_TO_COLLAB, user, {"cId": str(collab.id), "pubId": str(draft.id)}) + + assert collab.publications.count() == 0 + + +# --------------------------------------------------------------------------- # +# Stale links — render omits unpublished/deleted, restores on re-publish +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestStaleLinks: + def test_unpublish_hides_then_republish_restores(self, user, resource_type): + uc = _use_case(user) + collab = _collaborative(user) + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + uc.publications.add(pub) + collab.publications.add(pub) + + assert _published_only(uc) == [pub] + assert _published_only(collab) == [pub] + + pub.status = PublicationStatus.DRAFT + pub.save() + assert _published_only(uc) == [] # silently drops from render + assert _published_only(collab) == [] + + pub.status = PublicationStatus.PUBLISHED + pub.save() + assert _published_only(uc) == [pub] # reappears + assert _published_only(collab) == [pub] + + def test_delete_removes_link_and_render_skips(self, user, resource_type): + uc = _use_case(user) + collab = _collaborative(user) + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + uc.publications.add(pub) + collab.publications.add(pub) + + pub.delete() + + assert _published_only(uc) == [] + assert _published_only(collab) == [] + assert uc.publications.count() == 0 # M2M row auto-cleared + + +# --------------------------------------------------------------------------- # +# Linked-count flag + cross-org affordance +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestLinkedCountAndCrossOrg: + def test_linked_count_across_usecases_and_collaboratives(self, user, resource_type): + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + uc1, uc2 = _use_case(user), _use_case(user) + collab = _collaborative(user) + uc1.publications.add(pub) + uc2.publications.add(pub) + collab.publications.add(pub) + + assert pub.usecase_set.count() + pub.collaborative_set.count() == 3 + + uc1.publications.remove(pub) + assert pub.usecase_set.count() + pub.collaborative_set.count() == 2 + + def test_cross_org_link_is_allowed(self, resource_type): + # A UC in org B may link a PUBLISHED resource owned by org A — the one + # intentional cross-org path; do not deny it. + org_a = Organization.objects.create(name="A", description="a", slug="a") + org_b = Organization.objects.create(name="B", description="b", slug="b") + owner_a = User.objects.create(username="a", keycloak_id="a") + owner_b = User.objects.create(username="b", keycloak_id="b") + pub = _publication(owner_a, resource_type, status=PublicationStatus.PUBLISHED, org=org_a) + uc_b = UseCase.objects.create(title="UC B", user=owner_b, organization=org_b) + + run(ADD_TO_UC, owner_b, {"ucId": str(uc_b.id), "pubId": str(pub.id)}) + + assert uc_b.publications.filter(id=pub.id).exists() From 92a2f892bfb0816f799a1106a4662495a6317b6f Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 13:52:13 +0530 Subject: [PATCH 13/57] feat(publications): add PublicationClient to the SDK - dataspace_sdk/resources/publications.py: search (REST ES endpoint), plus get_by_id / list_all / get_organization_publications / create / update / delete over GraphQL (Resources have no REST write API). - Registered in client.py __init__ and set_organization (org header scoping). - tests/test_publications.py + a wiring assertion in test_client.py. --- dataspace_sdk/client.py | 3 + dataspace_sdk/resources/publications.py | 132 ++++++++++++++++++++++++ tests/test_client.py | 6 ++ tests/test_publications.py | 81 +++++++++++++++ 4 files changed, 222 insertions(+) create mode 100644 dataspace_sdk/resources/publications.py create mode 100644 tests/test_publications.py diff --git a/dataspace_sdk/client.py b/dataspace_sdk/client.py index 5f85eb2..e612476 100644 --- a/dataspace_sdk/client.py +++ b/dataspace_sdk/client.py @@ -6,6 +6,7 @@ from dataspace_sdk.resources.aimodels import AIModelClient from dataspace_sdk.resources.auditors import AuditorClient from dataspace_sdk.resources.datasets import DatasetClient +from dataspace_sdk.resources.publications import PublicationClient from dataspace_sdk.resources.sectors import SectorClient from dataspace_sdk.resources.usecases import UseCaseClient @@ -65,6 +66,7 @@ def __init__( # Initialize resource clients self.datasets = DatasetClient(self.base_url, self._auth) self.aimodels = AIModelClient(self.base_url, self._auth) + self.publications = PublicationClient(self.base_url, self._auth) self.usecases = UseCaseClient(self.base_url, self._auth) self.sectors = SectorClient(self.base_url, self._auth) self.auditors = AuditorClient(self.base_url, self._auth) @@ -199,6 +201,7 @@ def set_organization(self, organization_id: str) -> None: """ self.datasets.default_headers["organization"] = organization_id self.aimodels.default_headers["organization"] = organization_id + self.publications.default_headers["organization"] = organization_id self.usecases.default_headers["organization"] = organization_id self.sectors.default_headers["organization"] = organization_id self.auditors.default_headers["organization"] = organization_id diff --git a/dataspace_sdk/resources/publications.py b/dataspace_sdk/resources/publications.py new file mode 100644 index 0000000..031f5bf --- /dev/null +++ b/dataspace_sdk/resources/publications.py @@ -0,0 +1,132 @@ +"""Publication ("Resource") resource client for DataSpace SDK.""" + +from typing import Any, Dict, List, Optional + +from dataspace_sdk.base import BaseAPIClient + + +class PublicationClient(BaseAPIClient): + """Client for interacting with Resources (internally 'publications'). + + Search runs over the REST Elasticsearch endpoint; detail/list/CRUD go + through GraphQL, matching the backend (Resources have no REST write API). + """ + + def search( + self, + query: Optional[str] = None, + resource_type: Optional[str] = None, + sectors: Optional[List[str]] = None, + geographies: Optional[List[str]] = None, + sort: Optional[str] = None, + page: int = 1, + page_size: int = 10, + ) -> Dict[str, Any]: + """Search published resources via Elasticsearch. + + Args: + query: Free-text query. + resource_type: Filter by resource type name. + sectors: Filter by sector names. + geographies: Filter by geography names. + sort: Sort order (recent, alphabetical, created). + page: Page number (1-indexed). + page_size: Results per page. + + Returns: + Search results and metadata. + """ + params: Dict[str, Any] = {"page": page, "page_size": page_size} + if query: + params["q"] = query + if resource_type: + params["resource_type"] = resource_type + if sectors: + params["sectors"] = ",".join(sectors) + if geographies: + params["geographies"] = ",".join(geographies) + if sort: + params["sort"] = sort + + return super().get("/api/search/publication/", params=params) + + def get_by_id(self, publication_id: str) -> Dict[str, Any]: + """Get a single resource by id via GraphQL.""" + query = """ + query GetPublication($publicationId: UUID!) { + getPublication(publicationId: $publicationId) { + id title description slug status authors publicationDate + license externalSourceLink downloadCount + resourceType { id name } + blocks { id position blockType fileName youtubeUrl } + } + } + """ + return self.post( + "/api/graphql", + json_data={"query": query, "variables": {"publicationId": publication_id}}, + ) + + def list_all( + self, + include_public: bool = False, + limit: int = 10, + offset: int = 0, + ) -> Dict[str, Any]: + """List resources scoped to the caller (org header or user) via GraphQL.""" + query = """ + query ListPublications($includePublic: Boolean, $pagination: OffsetPaginationInput) { + publications(includePublic: $includePublic, pagination: $pagination) { + id title slug status + } + } + """ + variables: Dict[str, Any] = { + "includePublic": include_public, + "pagination": {"offset": offset, "limit": limit}, + } + return self.post("/api/graphql", json_data={"query": query, "variables": variables}) + + def get_organization_publications(self, limit: int = 10, offset: int = 0) -> Dict[str, Any]: + """List the current organization's resources (org set via set_organization).""" + return self.list_all(include_public=False, limit=limit, offset=offset) + + def create(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Create a resource via the createPublication mutation.""" + mutation = """ + mutation CreatePublication($input: CreatePublicationInput!) { + createPublication(input: $input) { + success errors { nonFieldErrors } data { id slug status } + } + } + """ + return self.post( + "/api/graphql", json_data={"query": mutation, "variables": {"input": data}} + ) + + def update(self, publication_id: str, data: Dict[str, Any]) -> Dict[str, Any]: + """Update a resource via the updatePublication mutation.""" + mutation = """ + mutation UpdatePublication($input: UpdatePublicationInput!) { + updatePublication(input: $input) { + success errors { nonFieldErrors } data { id title } + } + } + """ + payload = {"id": publication_id, **data} + return self.post( + "/api/graphql", + json_data={"query": mutation, "variables": {"input": payload}}, + ) + + def delete(self, publication_id: str) -> Dict[str, Any]: + """Delete a resource via the deletePublication mutation.""" + mutation = """ + mutation DeletePublication($publicationId: UUID!) { + deletePublication(publicationId: $publicationId) { success data } + } + """ + return self.post( + "/api/graphql", + json_data={"query": mutation, "variables": {"publicationId": publication_id}}, + ) diff --git a/tests/test_client.py b/tests/test_client.py index 7d27d44..44d4c6b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -20,8 +20,14 @@ def test_init(self) -> None: self.assertIsNotNone(self.client._auth) self.assertIsNotNone(self.client.datasets) self.assertIsNotNone(self.client.aimodels) + self.assertIsNotNone(self.client.publications) self.assertIsNotNone(self.client.usecases) + def test_set_organization_scopes_publications(self) -> None: + """set_organization must set the org header on the publications client too.""" + self.client.set_organization("org-123") + self.assertEqual(self.client.publications.default_headers["organization"], "org-123") + @patch("dataspace_sdk.client.AuthClient.login") def test_login(self, mock_login: MagicMock) -> None: """Test login method with username/password.""" diff --git a/tests/test_publications.py b/tests/test_publications.py new file mode 100644 index 0000000..7c44061 --- /dev/null +++ b/tests/test_publications.py @@ -0,0 +1,81 @@ +"""Tests for the Publication ("Resource") SDK resource client.""" + +import unittest +from unittest.mock import MagicMock, patch + +from dataspace_sdk.resources.publications import PublicationClient + + +class TestPublicationClient(unittest.TestCase): + """Test cases for PublicationClient.""" + + def setUp(self) -> None: + self.base_url = "https://api.test.com" + self.auth_client = MagicMock() + self.client = PublicationClient(self.base_url, self.auth_client) + + def test_init(self) -> None: + self.assertEqual(self.client.base_url, self.base_url) + self.assertEqual(self.client.auth_client, self.auth_client) + + @patch.object(PublicationClient, "_make_request") + def test_search_hits_the_publication_endpoint(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"total": 1, "results": [{"id": "1"}]} + + result = self.client.search( + query="rainfall", resource_type="Report", sectors=["Health"], page=2, page_size=5 + ) + + self.assertEqual(result["total"], 1) + args, kwargs = mock_request.call_args + self.assertIn("/api/search/publication/", args[1]) + params = kwargs["params"] + self.assertEqual(params["q"], "rainfall") + self.assertEqual(params["resource_type"], "Report") + self.assertEqual(params["sectors"], "Health") + self.assertEqual(params["page"], 2) + + @patch.object(PublicationClient, "_make_request") + def test_get_by_id_uses_graphql(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"data": {"getPublication": {"id": "abc"}}} + + self.client.get_by_id("abc") + + args, kwargs = mock_request.call_args + self.assertIn("/api/graphql", args[1]) + self.assertEqual(kwargs["json_data"]["variables"]["publicationId"], "abc") + + @patch.object(PublicationClient, "_make_request") + def test_create_posts_the_mutation(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"data": {"createPublication": {"success": True}}} + + self.client.create({"title": "New"}) + + args, kwargs = mock_request.call_args + self.assertIn("createPublication", kwargs["json_data"]["query"]) + self.assertEqual(kwargs["json_data"]["variables"]["input"]["title"], "New") + + @patch.object(PublicationClient, "_make_request") + def test_update_merges_the_id_into_input(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"data": {"updatePublication": {"success": True}}} + + self.client.update("abc", {"title": "Renamed"}) + + args, kwargs = mock_request.call_args + variables = kwargs["json_data"]["variables"]["input"] + self.assertEqual(variables["id"], "abc") + self.assertEqual(variables["title"], "Renamed") + + @patch.object(PublicationClient, "_make_request") + def test_delete_passes_the_id(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"data": {"deletePublication": {"success": True}}} + + self.client.delete("abc") + + args, kwargs = mock_request.call_args + self.assertIn("deletePublication", kwargs["json_data"]["query"]) + self.assertEqual(kwargs["json_data"]["variables"]["publicationId"], "abc") + + +if __name__ == "__main__": + unittest.main() From 20195b826c09181091697d15a45fad537d39ddd2 Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 13:56:26 +0530 Subject: [PATCH 14/57] fix(publications): authorize the resource link mutations (IDOR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UC/Collab publication link trios previously guarded only on the container's DRAFT status — like the pre-existing dataset trio — so any authenticated user could change another org's use case / collaborative links. Add assert_can_manage_links (owner, or org member with can_change; superuser always) to all six link mutations. The intentional cross-org affordance is unaffected: authorization is on the container, not the linked resource, so org B's use case may still link org A's published resource. Adds an IDOR regression test. --- api/schema/collaborative_schema.py | 10 ++++++++++ api/schema/usecase_schema.py | 10 ++++++++++ api/services/publication_linking.py | 26 ++++++++++++++++++++++++++ tests/test_publication_linking.py | 12 ++++++++++++ 4 files changed, 58 insertions(+) diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index a8e78e4..d2fa5f3 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -30,6 +30,7 @@ ) from api.schema.extensions import TrackActivity, TrackModelActivity from api.services.publication_linking import ( + assert_can_manage_links, get_linkable_publication, published_publications, ) @@ -550,6 +551,9 @@ def add_publication_to_collaborative( except Collaborative.DoesNotExist: raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") + # Only the collaborative's owner / org editors may change its links. + assert_can_manage_links(info.context.user, collaborative.user, collaborative.organization) + if collaborative.status != CollaborativeStatus.DRAFT: raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") @@ -567,6 +571,9 @@ def remove_publication_from_collaborative( except Collaborative.DoesNotExist: raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") + # Only the collaborative's owner / org editors may change its links. + assert_can_manage_links(info.context.user, collaborative.user, collaborative.organization) + if collaborative.status != CollaborativeStatus.DRAFT: raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") @@ -588,6 +595,9 @@ def update_collaborative_publications( except Collaborative.DoesNotExist: raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") + # Only the collaborative's owner / org editors may change its links. + assert_can_manage_links(info.context.user, collaborative.user, collaborative.organization) + if collaborative.status != CollaborativeStatus.DRAFT: raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") diff --git a/api/schema/usecase_schema.py b/api/schema/usecase_schema.py index be4aaa7..bb3609e 100644 --- a/api/schema/usecase_schema.py +++ b/api/schema/usecase_schema.py @@ -30,6 +30,7 @@ ) from api.schema.extensions import TrackActivity, TrackModelActivity from api.services.publication_linking import ( + assert_can_manage_links, get_linkable_publication, published_publications, ) @@ -493,6 +494,9 @@ def add_publication_to_use_case( except UseCase.DoesNotExist: raise ValueError(f"UseCase with ID {use_case_id} does not exist.") + # Only the use case's owner / org editors may change its links. + assert_can_manage_links(info.context.user, use_case.user, use_case.organization) + if use_case.status != UseCaseStatus.DRAFT: raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") @@ -510,6 +514,9 @@ def remove_publication_from_use_case( except UseCase.DoesNotExist: raise ValueError(f"UseCase with ID {use_case_id} does not exist.") + # Only the use case's owner / org editors may change its links. + assert_can_manage_links(info.context.user, use_case.user, use_case.organization) + if use_case.status != UseCaseStatus.DRAFT: raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") @@ -531,6 +538,9 @@ def update_usecase_publications( except UseCase.DoesNotExist: raise ValueError(f"Use Case with ID {use_case_id} doesn't exist") + # Only the use case's owner / org editors may change its links. + assert_can_manage_links(info.context.user, use_case.user, use_case.organization) + if use_case.status != UseCaseStatus.DRAFT: raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") diff --git a/api/services/publication_linking.py b/api/services/publication_linking.py index ff3d5bc..bc66816 100644 --- a/api/services/publication_linking.py +++ b/api/services/publication_linking.py @@ -36,3 +36,29 @@ def published_publications(publication_ids: List[Any]) -> List[Publication]: return list( Publication.objects.filter(id__in=publication_ids, status=PublicationStatus.PUBLISHED) ) + + +def assert_can_manage_links(user: Any, owner_user: Any, organization: Any) -> None: + """Raise unless the caller may edit this Use Case / Collaborative's links. + + Editing links changes the container, so it needs the container's own + authorization: its owner, or an org member with the change role (superusers + always). This is independent of the linked resource, so the intentional + cross-org affordance — org B's use case linking org A's published resource — + still works: the caller is authorized on *their own* use case. + """ + if getattr(user, "is_superuser", False): + return + if not getattr(user, "is_authenticated", False): + raise ValueError("Authentication required.") + if owner_user and owner_user == user: + return + if organization: + from authorization.models import OrganizationMembership + + membership = OrganizationMembership.objects.filter( + user=user, organization=organization + ).first() + if membership and membership.role.can_change: + return + raise ValueError("You don't have permission to modify this.") diff --git a/tests/test_publication_linking.py b/tests/test_publication_linking.py index 7e81753..e41af4c 100644 --- a/tests/test_publication_linking.py +++ b/tests/test_publication_linking.py @@ -173,6 +173,18 @@ def test_linked_count_across_usecases_and_collaboratives(self, user, resource_ty uc1.publications.remove(pub) assert pub.usecase_set.count() + pub.collaborative_set.count() == 2 + def test_unrelated_user_cannot_link_to_someone_elses_use_case(self, user, resource_type): + # IDOR guard: a caller who neither owns nor has an editor role on the use + # case cannot change its links, even with a valid published resource. + owner = user + stranger = User.objects.create(username="stranger", keycloak_id="stranger") + uc = _use_case(owner) + pub = _publication(owner, resource_type, status=PublicationStatus.PUBLISHED) + + run(ADD_TO_UC, stranger, {"ucId": str(uc.id), "pubId": str(pub.id)}) + + assert uc.publications.count() == 0 # link refused + def test_cross_org_link_is_allowed(self, resource_type): # A UC in org B may link a PUBLISHED resource owned by org A — the one # intentional cross-org path; do not deny it. From 12d30021923195985e9f97774a3fb2068362b75e Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 13:59:09 +0530 Subject: [PATCH 15/57] docs(publications): backend architecture doc + Layer 5 journey scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - api/schema/publication_architecture.md: full backend design — submodules, helpers, data model, indexes, security, six-layer test map, cross-repo overview, limitations. - tests/journeys/publications/{create-blocks-publish,link-unpublish-relink}.py: on-demand Layer 5 scripts. --- api/schema/publication_architecture.md | 113 ++++++++++++++++++ .../publications/create-blocks-publish.py | 99 +++++++++++++++ .../publications/link-unpublish-relink.py | 112 +++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 api/schema/publication_architecture.md create mode 100644 tests/journeys/publications/create-blocks-publish.py create mode 100644 tests/journeys/publications/link-unpublish-relink.py diff --git a/api/schema/publication_architecture.md b/api/schema/publication_architecture.md new file mode 100644 index 0000000..3e074bc --- /dev/null +++ b/api/schema/publication_architecture.md @@ -0,0 +1,113 @@ +# Publications (UI "Resource") — backend + +> Part of feature: **resources** · siblings: `DataSpaceFrontend/app/[locale]/(user)/publications/architecture.md` (frontend slice — to be created with the FE phases) + +## Overview + +A **Publication** is a new top-level entity, peer to Datasets and AI Models: a container for human-authored content (reports, research, findings). It has typed metadata, an ordered list of heterogeneous **content blocks** (a file XOR a YouTube embed each), a publish/unpublish toggle, its own listing + global search presence, and it can be pulled into Use Cases and Collaboratives. + +**Naming:** the user-facing entity is **"Resource"**, but the name `Resource` was already taken in this repo (the file-inside-a-dataset). So the entity is `Publication` everywhere in code — Django model, GraphQL type, table `publication`, ES index `publication`, SDK client, URL path `/publications`. The UI always renders "Resource". + +### How the repos connect + +- **DataExBackend** owns the whole data/API/search/SDK slice (this doc). +- **DataSpaceFrontend** (sibling doc) owns the create/edit/publish flow, detail page, listing, cards, and the Resource picker inside Use Case / Collaborative editors. It talks to the backend over GraphQL (`/api/graphql`) for CRUD/detail/list and REST for search (`/api/search/publication/`) and gated file download (`/api/publications/blocks//download/`). +- Request flow for a create: FE collects metadata → `createPublication` mutation → DRAFT row → FE adds blocks via `addPublicationFileBlock` / `addPublicationYoutubeBlock` (multipart for files) → `publishPublication`. A published resource then appears in the listing/search and is linkable from a Use Case / Collaborative. + +## Submodule map + +| Submodule | Trigger | +|---|---| +| CRUD + publish | GraphQL `publication_schema.py` (create/update/publish/unpublish/delete + list/detail) | +| Content blocks | GraphQL block mutations (add file/youtube, replace, remove, reorder) | +| Block-file download | REST `GET /api/publications/blocks//download/` | +| Search | REST `GET /api/search/publication/` + unified `GET /api/search/unified/` | +| Index sync | model signals (`publication_signals.py`) + `search_index --rebuild/--populate` | +| UC/Collab linking | GraphQL link trios in `usecase_schema.py` / `collaborative_schema.py` | +| SDK | `dataspace_sdk/resources/publications.py` | +| Resource Type lookup | Django admin + `seed_resource_types` command | + +--- + +## Submodule: CRUD + publish + +### Trigger +GraphQL: `createPublication`, `updatePublication`, `publishPublication`, `unpublishPublication`, `deletePublication` mutations; `getPublication(publicationId)` and `publications(filters, pagination, order, includePublic)` queries. Flow file: `api/schema/publication_schema.py`. + +### Business use case +Any individual or org account creates a Resource, edits its metadata across subpages, and publishes/unpublishes it with a simple toggle (no moderation). Auditors (role `can_change=False`) can read but not edit/publish. + +### Flow (English) +- **create:** reject missing/invalid metadata at the boundary → create a DRAFT owned by the org (from the request's organization header) or the user → wire sector/geography tags. +- **update:** load or 404 → apply only the provided fields (each validated) → save. +- **publish / unpublish:** load or 404 → flip `status`; the index signal adds/drops the search document. +- **delete:** load or 404 → hard delete (FK cascade drops blocks; M2M link rows auto-clear). +- **list:** scope to org / owner / anonymous, optionally union the public set, apply filters + ordering, enforce a bounded page window. +- **detail:** gated by `AllowPublishedPublications` — a published resource is world-readable; a draft only to owner/org/superuser. + +### Helpers +All Tier-2, in `api/services/publication_service.py`: +- `validate_publication_metadata(...) -> ResourceType` — required-field + typed validation (license in `DatasetLicense`, active resource type, URL shape); raises field-keyed `ValidationError`; returns the resolved active resource type. +- `create_publication(...) -> Publication` — create DRAFT + set M2M tags. +- `apply_publication_update(publication, ...) -> Publication` — partial update, validating each provided field; never blanks untouched columns. +- `set_publication_status(publication, status) -> Publication`. +- `get_scoped_publications(user, organization, include_public) -> QuerySet` — org/owner/anonymous scoping + published union, ordered `-modified`, `.distinct()`. +- `resolve_pagination(offset, limit) -> (offset, limit)` — default page size + hard max, so a listing is never unbounded. +- `is_publication_published(publication) -> bool`. + +Permissions (Tier-2, `authorization/permissions.py`): `CreatePublicationPermission`, `ChangePublicationPermission`, `DeletePublicationPermission`, `PublishPublicationPermission` (name-based admin/editor/owner), `AllowPublishedPublications`. All key on `publication_id` (or the update input's id, or a block's parent), keep the individual-owner branch, and drop Dataset's share-model fallback. + +### Data model +`Publication` (table `publication`): UUID PK, `title`, `description` (Text), `slug` (unique, counter-dedupe on collision), `organization`/`user` nullable FKs (`SET_NULL`), `authors` (JSON list), `publication_date` (Date), `license` (reuses `DatasetLicense` choices), `external_source_link` (URL), `status` (`PublicationStatus` DRAFT/PUBLISHED), `resource_type` FK (`PROTECT`), `sectors`/`geographies` M2M, `download_count`, `created`/`modified`. +`ResourceType` (table `resource_type`): UUID/name(unique)/slug + `is_active` (adapted from `Sector`, no parent self-FK). +`UseCase`/`Collaborative`: gain a `publications` M2M. + +### Indexing & performance +`Publication.Meta.indexes`: `(organization, -modified)`, `(user, -modified)`, `(status)` — matching the dashboard/listing query shapes. `slug` unique (detail lookup), org/user/resource_type FKs auto-indexed. No standalone org/user/resource_type indexes (composites/FK cover them). Sector/geography filters are ES-backed → no PG indexes. Pagination is server-enforced (default 20, max 100). `PublicationBlock.Meta.indexes`: `(publication, position)`. +**Migrations:** none committed — this repo auto-generates migrations at deploy (`docker-entrypoint.sh` runs `makemigrations --noinput` then `migrate`); the committed `0001_initial` is a stale stub and `authorization` has no migrations dir. All additions are additive (new tables + new nullable columns + additive M2M join tables), safe on large tables. See project-memory (2026-07-18). + +### Security +Every query is org/user-scoped; anonymous sees only PUBLISHED. Cross-org request outcomes: mutating another org's DRAFT → permission denied (the resolver 404s the draft at read); mutating a PUBLISHED one → permission denied. Auditor (`can_change=False`) → read allowed, edit/publish denied. Publish gated to role **names** admin/editor/owner (mirroring Dataset). Individual resources restricted to their owner. All scoping is centralized in the permission classes — no inline role logic in resolvers. + +### SDK impact +SDK: **yes.** `dataspace_sdk/resources/publications.py` `PublicationClient` — `search` (REST), `get_by_id`/`list_all`/`get_organization_publications`/`create`/`update`/`delete` (GraphQL). Registered in `client.py` `__init__` and `set_organization`. + +### Tests + +#### Layer 1 — DB helper tests (`tests/test_publication_models.py`) +Slug dedupe (distinct slugs, unicode title); ownership property; ResourceType uniqueness/slug/`is_active`/active-query; block file/youtube storage; file-XOR-youtube CheckConstraint (both/neither rejected); block position ordering; delete cascade; seed idempotency; DRAFT/download_count/authors defaults. + +#### Layer 2 — Non-DB helper tests +`tests/test_youtube_url.py` — id extraction across watch/youtu.be/embed, rejects non-YouTube, malformed, and non-http(s) schemes (javascript: stored-XSS guard). `tests/test_publication_uploads.py` — extension allow-list, 50 MB cap, PDF magic-byte sniff. + +#### Layer 3 — Flow tests (in `tests/schema/test_publication_schema.py`, `tests/test_publication_blocks.py`) +Create org/individual → DRAFT; invalid metadata → no create; block add happy/invalid; reorder/renumber; replace-file (same id, old file gone). + +#### Layer 4 — Backend API e2e (`tests/schema/test_publication_schema.py`, `tests/test_publication_blocks.py`) +CRUD; role gating (editor edits, auditor denied edit/publish but reads); publish/unpublish; cross-org denial for update/delete/publish + block add/remove; anonymous sees published detail, denied draft; org-scoped and anonymous (published-only) listings; block-file download gate (published→200 + count increment, draft anon/cross-org→404, owner draft→200, PDF inline). + +#### Layer 4 — Search (`tests/test_publication_search.py`) +Index-decision (published indexed, draft not); re-index mapping incl. ResourceType/Sector/Geography/Org; signal predicate. Full ES query/filter/pagination behaviour runs against a live cluster (ES is disabled in the deterministic layers). + +#### Layer 4 — Linking (`tests/test_publication_linking.py`) +Only-published-linkable guard (UC + Collab); stale-link (unpublish/delete hides, re-publish restores); linked-count; IDOR guard (non-owner can't link); the intentional cross-org affordance (org B UC links org A published resource). + +#### Layer 5 — API user journey (`tests/journeys/publications/`) +`create-blocks-publish` and `link-unpublish-relink` (scripted, on-demand). + +#### Layer 6 — Browser e2e +n/a in this repo — belongs to the frontend slice's doc. + +### LLM-judge points +n/a — all assertions are deterministic. + +--- + +## Limitations & future work + +- **UC/Collab search does not index linked Resources.** A use case / collaborative is not findable by a linked Resource's title, and their `dataset_count` excludes Resources. Deferred by design. +- **Resource search matches name/description + the three facets only.** Author / date / usage-rights columns and block content are not indexed in v1. +- **Preview fidelity:** slide decks / DOCX / PPT are download-only (only PDF + YouTube render inline). +- **No versioning** for re-uploaded block files (in-place replace). +- **The dataset link trio's pre-existing IDOR** (no container-authorization check) is not fixed here — only the new publication trio is authorized. Follow-up: apply `assert_can_manage_links` to the dataset trio too. +- **Frontend (Phases 7–8), the two Layer 5 journey scripts, and the FE architecture doc are not yet implemented** — the backend is complete and shippable; the UI is the remaining slice. diff --git a/tests/journeys/publications/create-blocks-publish.py b/tests/journeys/publications/create-blocks-publish.py new file mode 100644 index 0000000..62f84a2 --- /dev/null +++ b/tests/journeys/publications/create-blocks-publish.py @@ -0,0 +1,99 @@ +""" +Journey: create a Resource, add a PDF block and a YouTube block, set metadata, +reorder, publish, then fetch it anonymously and confirm it's visible with its +blocks in order. + +On-demand (Layer 5) — runs against a live backend. Reads KEYCLOAK_TEST_TOKEN +and TEST_BASE_URL from env. Fails loudly on the first assertion that fails. + +Usage: + KEYCLOAK_TEST_TOKEN=... TEST_BASE_URL=http://localhost:8000 \ + python tests/journeys/publications/create-blocks-publish.py +""" + +import os + +import requests + +base = os.environ.get("TEST_BASE_URL", "http://localhost:8000") +token = os.environ["KEYCLOAK_TEST_TOKEN"] +headers = {"Authorization": f"Bearer {token}"} +graphql = f"{base}/api/graphql" + + +def gql(query, variables=None, files=None): + """Post a GraphQL operation (multipart when files are given).""" + if files: + return requests.post(graphql, data=files, headers=headers) + res = requests.post( + graphql, json={"query": query, "variables": variables or {}}, headers=headers + ) + assert res.status_code == 200, res.text + body = res.json() + assert not body.get("errors"), body["errors"] + return body["data"] + + +# 1. Create a DRAFT resource with full metadata (ids below are placeholders — +# fill in a real resource type / sector / geography id from your test data). +create = gql( + """ + mutation($input: CreatePublicationInput!) { + createPublication(input: $input) { + success errors { fieldErrors { field messages } } data { id status } + } + } + """, + { + "input": { + "title": "Journey Findings", + "description": "A journey-test resource.", + "authors": ["Journey Bot"], + "publicationDate": "2024-01-01", + "license": "CC_BY_4_0_ATTRIBUTION", + "resourceTypeId": os.environ.get("TEST_RESOURCE_TYPE_ID", "REPLACE_ME"), + "sectorIds": [os.environ.get("TEST_SECTOR_ID", "REPLACE_ME")], + "geographyIds": [int(os.environ.get("TEST_GEOGRAPHY_ID", "1"))], + } + }, +) +assert create["createPublication"]["success"], create +publication_id = create["createPublication"]["data"]["id"] +assert create["createPublication"]["data"]["status"] == "DRAFT" + +# 2. Add a YouTube block (a PDF block is added via the multipart upload path the +# frontend uses — see ResourceDropzone; scripted upload builds the GraphQL +# multipart request the same way). +yt = gql( + """ + mutation($id: UUID!, $url: String!) { + addPublicationYoutubeBlock(publicationId: $id, youtubeUrl: $url) { + success data { id position blockType } + } + } + """, + {"id": publication_id, "url": "https://youtu.be/dQw4w9WgXcQ"}, +) +assert yt["addPublicationYoutubeBlock"]["success"], yt + +# 3. Publish it. +pub = gql( + "mutation($id: UUID!) { publishPublication(publicationId: $id) { success data { status } } }", + {"id": publication_id}, +) +assert pub["publishPublication"]["data"]["status"] == "PUBLISHED" + +# 4. Fetch anonymously and confirm it's visible with its blocks in order. +anon = requests.post( + graphql, + json={ + "query": "query($id: UUID!) { getPublication(publicationId: $id) { id status blocks { position } } }", + "variables": {"id": publication_id}, + }, +) +data = anon.json()["data"]["getPublication"] +assert data and data["status"] == "PUBLISHED", data +positions = [b["position"] for b in data["blocks"]] +assert positions == sorted(positions), positions + +print("PASS: create-blocks-publish") diff --git a/tests/journeys/publications/link-unpublish-relink.py b/tests/journeys/publications/link-unpublish-relink.py new file mode 100644 index 0000000..473b23f --- /dev/null +++ b/tests/journeys/publications/link-unpublish-relink.py @@ -0,0 +1,112 @@ +""" +Journey: create + publish a Resource, link it to a Use Case and a Collaborative, +confirm both render it, unpublish (both hide it), re-publish (both show it), +delete (both skip it and the linked-count stays consistent). + +On-demand (Layer 5) — runs against a live backend. Reads KEYCLOAK_TEST_TOKEN, +TEST_BASE_URL, TEST_USE_CASE_ID and TEST_COLLABORATIVE_ID (both DRAFT, owned by +the token's user/org) from env. Fails loudly on the first bad assertion. + +Usage: + KEYCLOAK_TEST_TOKEN=... TEST_BASE_URL=http://localhost:8000 \ + TEST_USE_CASE_ID=... TEST_COLLABORATIVE_ID=... \ + python tests/journeys/publications/link-unpublish-relink.py +""" + +import os + +import requests + +base = os.environ.get("TEST_BASE_URL", "http://localhost:8000") +token = os.environ["KEYCLOAK_TEST_TOKEN"] +headers = {"Authorization": f"Bearer {token}"} +graphql = f"{base}/api/graphql" +use_case_id = os.environ["TEST_USE_CASE_ID"] +collaborative_id = os.environ["TEST_COLLABORATIVE_ID"] + + +def gql(query, variables=None): + res = requests.post( + graphql, json={"query": query, "variables": variables or {}}, headers=headers + ) + assert res.status_code == 200, res.text + body = res.json() + assert not body.get("errors"), body["errors"] + return body["data"] + + +# 1. Create + publish a resource (metadata ids from env, see the sibling script). +create = gql( + """ + mutation($input: CreatePublicationInput!) { + createPublication(input: $input) { success data { id } } + } + """, + { + "input": { + "title": "Linkable Findings", + "description": "For the link journey.", + "authors": ["Journey Bot"], + "publicationDate": "2024-01-01", + "license": "CC_BY_4_0_ATTRIBUTION", + "resourceTypeId": os.environ.get("TEST_RESOURCE_TYPE_ID", "REPLACE_ME"), + "sectorIds": [os.environ.get("TEST_SECTOR_ID", "REPLACE_ME")], + "geographyIds": [int(os.environ.get("TEST_GEOGRAPHY_ID", "1"))], + } + }, +) +publication_id = create["createPublication"]["data"]["id"] +gql( + "mutation($id: UUID!) { publishPublication(publicationId: $id) { success } }", + {"id": publication_id}, +) + +# 2. Link it to the use case and the collaborative. +gql( + "mutation($u: String!, $p: UUID!) { addPublicationToUseCase(useCaseId: $u, publicationId: $p) { __typename } }", + {"u": use_case_id, "p": publication_id}, +) +gql( + "mutation($c: String!, $p: UUID!) { addPublicationToCollaborative(collaborativeId: $c, publicationId: $p) { __typename } }", + {"c": collaborative_id, "p": publication_id}, +) + + +def renders(entity, entity_id): + query = { + "usecase": "query($id: ID!) { useCase(pk: $id) { publications { id } } }", + "collab": "query($id: ID!) { collaborative(pk: $id) { publications { id } } }", + }[entity] + data = gql(query, {"id": entity_id}) + key = "useCase" if entity == "usecase" else "collaborative" + return [p["id"] for p in (data[key]["publications"] or [])] + + +# 3. Both render it while published. +assert publication_id in renders("usecase", use_case_id) +assert publication_id in renders("collab", collaborative_id) + +# 4. Unpublish → both hide it. +gql( + "mutation($id: UUID!) { unpublishPublication(publicationId: $id) { success } }", + {"id": publication_id}, +) +assert publication_id not in renders("usecase", use_case_id) +assert publication_id not in renders("collab", collaborative_id) + +# 5. Re-publish → both show it again. +gql( + "mutation($id: UUID!) { publishPublication(publicationId: $id) { success } }", + {"id": publication_id}, +) +assert publication_id in renders("usecase", use_case_id) + +# 6. Delete → both skip it silently. +gql( + "mutation($id: UUID!) { deletePublication(publicationId: $id) { success } }", + {"id": publication_id}, +) +assert publication_id not in renders("usecase", use_case_id) +assert publication_id not in renders("collab", collaborative_id) + +print("PASS: link-unpublish-relink") From 25fe49f6ac7edb7ce3f035303694f640b25a6a81 Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 18 Jul 2026 14:23:07 +0530 Subject: [PATCH 16/57] fix(publications): address QC blockers (link-field leak, search default, N+1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - B1 (cross-tenant leak): the linkedUsecases/linkedCollaboratives/linkedCount fields on TypePublication are now gated to owner/org (`_caller_can_see_links`) and filtered to PUBLISHED projects, so a public resource no longer exposes another org's private draft project titles. - B2 (discovery): add 'publication' to the unified-search get() default type list (the cache-key default already had it — the two now match). - B4 (N+1 + plan DoD): get_scoped_publications prefetches resource_type/organization/user + sectors/geographies/blocks; the blocks resolver relies on Meta ordering to reuse the prefetch cache; added an assertNumQueries listing test. - O1: reject an explicit-empty required field on update (don't blank title). - O2: regression test that the plain useCase update can't attach a resource. B3 (org-header trust in the list resolver) is a platform-wide pattern mirrored from the dataset/aimodel resolvers — flagged in the arch doc for a shared- middleware fix, not patched per-endpoint. Adds tests/test_publication_qc_fixes.py. --- api/schema/publication_architecture.md | 2 + api/services/publication_service.py | 25 +++- api/types/type_publication.py | 81 ++++++++++-- api/views/search_unified.py | 2 +- tests/test_publication_qc_fixes.py | 171 +++++++++++++++++++++++++ 5 files changed, 264 insertions(+), 17 deletions(-) create mode 100644 tests/test_publication_qc_fixes.py diff --git a/api/schema/publication_architecture.md b/api/schema/publication_architecture.md index 3e074bc..7bd46cf 100644 --- a/api/schema/publication_architecture.md +++ b/api/schema/publication_architecture.md @@ -68,6 +68,7 @@ Permissions (Tier-2, `authorization/permissions.py`): `CreatePublicationPermissi ### Security Every query is org/user-scoped; anonymous sees only PUBLISHED. Cross-org request outcomes: mutating another org's DRAFT → permission denied (the resolver 404s the draft at read); mutating a PUBLISHED one → permission denied. Auditor (`can_change=False`) → read allowed, edit/publish denied. Publish gated to role **names** admin/editor/owner (mirroring Dataset). Individual resources restricted to their owner. All scoping is centralized in the permission classes — no inline role logic in resolvers. +The **linked-project fields** (`linkedUsecases` / `linkedCollaboratives` / `linkedCount` on `TypePublication`) are the owner's "linked to N" flag: they're gated to the owner / org member / superuser (`_caller_can_see_links`) **and** filtered to PUBLISHED projects, so a public resource never leaks the title of a private draft project (possibly in another org) that references it. ### SDK impact SDK: **yes.** `dataspace_sdk/resources/publications.py` `PublicationClient` — `search` (REST), `get_by_id`/`list_all`/`get_organization_publications`/`create`/`update`/`delete` (GraphQL). Registered in `client.py` `__init__` and `set_organization`. @@ -110,4 +111,5 @@ n/a — all assertions are deterministic. - **Preview fidelity:** slide decks / DOCX / PPT are download-only (only PDF + YouTube render inline). - **No versioning** for re-uploaded block files (in-place replace). - **The dataset link trio's pre-existing IDOR** (no container-authorization check) is not fixed here — only the new publication trio is authorized. Follow-up: apply `assert_can_manage_links` to the dataset trio too. +- **The list resolver trusts the request's `organization` header without a membership check** — a faithful mirror of the existing dataset/aimodel/usecase list resolvers and the shared middleware (`api/utils/middleware.py`, which still carries a `# TODO: resolve auth_token`). This feature follows the platform pattern; if org membership is not enforced on that header upstream, a signed-in user could list another org's drafts by setting the header. The correct fix is at the shared middleware layer (so all entities are fixed together), not per-endpoint — deliberately **not** patched here to avoid divergence and false security. Flagged for a platform-level decision. - **Frontend (Phases 7–8), the two Layer 5 journey scripts, and the FE architecture doc are not yet implemented** — the backend is complete and shippable; the UI is the remaining slice. diff --git a/api/services/publication_service.py b/api/services/publication_service.py index 713471b..6084130 100644 --- a/api/services/publication_service.py +++ b/api/services/publication_service.py @@ -153,12 +153,22 @@ def apply_publication_update( """ errors: dict[str, List[str]] = {} + # A provided required field must not be explicitly blanked. if title is not None: - publication.title = title + if not title.strip(): + errors["title"] = ["Title cannot be empty."] + else: + publication.title = title if description is not None: - publication.description = description + if not description.strip(): + errors["description"] = ["Description cannot be empty."] + else: + publication.description = description if authors is not None: - publication.authors = authors + if not [a for a in authors if a and a.strip()]: + errors["authors"] = ["At least one author is required."] + else: + publication.authors = authors if publication_date is not None: publication.publication_date = publication_date if external_source_link is not None: @@ -220,7 +230,14 @@ def get_scoped_publications( if include_public: queryset = queryset | Publication.objects.filter(status=PublicationStatus.PUBLISHED) - return queryset.order_by("-modified").distinct() + # Prefetch the relations the listing/card resolvers touch so a page of N + # resources stays a bounded number of queries, not one-per-row. + return ( + queryset.select_related("resource_type", "organization", "user") + .prefetch_related("sectors", "geographies", "blocks") + .order_by("-modified") + .distinct() + ) def is_publication_published(publication: Publication) -> bool: diff --git a/api/types/type_publication.py b/api/types/type_publication.py index b68073c..01a415c 100644 --- a/api/types/type_publication.py +++ b/api/types/type_publication.py @@ -18,9 +18,38 @@ from api.types.type_geo import TypeGeo from api.types.type_organization import TypeOrganization from api.types.type_sector import TypeSector -from api.utils.enums import DatasetLicense, PublicationBlockType, PublicationStatus +from api.utils.enums import ( + CollaborativeStatus, + DatasetLicense, + PublicationBlockType, + PublicationStatus, + UseCaseStatus, +) from authorization.types import TypeUser + +def _caller_can_see_links(info: Info, publication: Publication) -> bool: + """Whether the caller may see where a resource is linked (owner / org / superuser). + + The 'linked to N' flag is the owner's view; outsiders (including anonymous + visitors to a public resource) must not learn which projects reference it. + """ + user = getattr(info.context, "user", None) + if not user or not getattr(user, "is_authenticated", False): + return False + if user.is_superuser: + return True + if publication.user and publication.user == user: + return True + if publication.organization: + from authorization.models import OrganizationMembership + + return OrganizationMembership.objects.filter( + user=user, organization=publication.organization + ).exists() + return False + + # Fields are enumerated on every type below — never ``fields="__all__"`` — so a # future column is never silently published. publication_status: EnumType = strawberry.enum(PublicationStatus) # type: ignore @@ -128,10 +157,15 @@ def geographies(self, info: Info) -> List["TypeGeo"]: @strawberry.field def blocks(self, info: Info) -> List["TypePublicationBlock"]: - """Ordered content blocks of this resource.""" + """Ordered content blocks of this resource. + + ``PublicationBlock`` orders by ``position`` in its Meta, so ``.all()`` is + already position-ordered and reuses the listing's prefetch cache — an + explicit ``.order_by`` here would re-query and reintroduce an N+1. + """ try: instance = cast(Publication, self) - return TypePublicationBlock.from_django_list(instance.blocks.all().order_by("position")) + return TypePublicationBlock.from_django_list(instance.blocks.all()) except (AttributeError, Publication.DoesNotExist): return [] @@ -141,11 +175,20 @@ def is_individual_publication(self) -> bool: return self.organization is None @strawberry.field - def linked_usecases(self) -> List["TypeLinkedReference"]: - """Use Cases this resource is linked into (owner's 'linked to N' flag).""" + def linked_usecases(self, info: Info) -> List["TypeLinkedReference"]: + """Use Cases this resource is linked into — the owner's 'linked to N' flag. + + Only the owner / org members may see this, and only published projects + are named, so a private draft (possibly in another org that linked this + public resource) never leaks its title through here. + """ try: instance = cast(Publication, self) - usecases: List[UseCase] = list(instance.usecase_set.all()) # type: ignore[attr-defined] + if not _caller_can_see_links(info, instance): + return [] + usecases: List[UseCase] = list( + UseCase.objects.filter(publications=instance, status=UseCaseStatus.PUBLISHED) + ) return [ TypeLinkedReference(id=str(uc.id), title=uc.title or "", slug=uc.slug or "") for uc in usecases @@ -154,11 +197,17 @@ def linked_usecases(self) -> List["TypeLinkedReference"]: return [] @strawberry.field - def linked_collaboratives(self) -> List["TypeLinkedReference"]: - """Collaboratives this resource is linked into (owner's 'linked to N' flag).""" + def linked_collaboratives(self, info: Info) -> List["TypeLinkedReference"]: + """Collaboratives this resource is linked into — the owner's 'linked to N' flag.""" try: instance = cast(Publication, self) - collabs: List[Collaborative] = list(instance.collaborative_set.all()) # type: ignore[attr-defined] + if not _caller_can_see_links(info, instance): + return [] + collabs: List[Collaborative] = list( + Collaborative.objects.filter( + publications=instance, status=CollaborativeStatus.PUBLISHED + ) + ) return [ TypeLinkedReference( id=str(collab.id), title=collab.title or "", slug=collab.slug or "" @@ -169,10 +218,18 @@ def linked_collaboratives(self) -> List["TypeLinkedReference"]: return [] @strawberry.field - def linked_count(self) -> int: - """Total Use Cases + Collaboratives this resource is linked into.""" + def linked_count(self, info: Info) -> int: + """Published Use Cases + Collaboratives this resource is linked into (owner only).""" try: instance = cast(Publication, self) - return instance.usecase_set.count() + instance.collaborative_set.count() # type: ignore[attr-defined] + if not _caller_can_see_links(info, instance): + return 0 + usecases = UseCase.objects.filter( + publications=instance, status=UseCaseStatus.PUBLISHED + ).count() + collabs = Collaborative.objects.filter( + publications=instance, status=CollaborativeStatus.PUBLISHED + ).count() + return usecases + collabs except (AttributeError, Publication.DoesNotExist): return 0 diff --git a/api/views/search_unified.py b/api/views/search_unified.py index 0b02247..b9c86a0 100644 --- a/api/views/search_unified.py +++ b/api/views/search_unified.py @@ -483,7 +483,7 @@ def get(self, request: Any) -> Response: page: int = int(request.GET.get("page", 1)) size: int = int(request.GET.get("size", 10)) entity_types: str = request.GET.get( - "types", "dataset,usecase,aimodel,collaborative,publisher" + "types", "dataset,usecase,aimodel,publication,collaborative,publisher" ) # Which entity types to search types_list = [t.strip() for t in entity_types.split(",")] diff --git a/tests/test_publication_qc_fixes.py b/tests/test_publication_qc_fixes.py new file mode 100644 index 0000000..5e22851 --- /dev/null +++ b/tests/test_publication_qc_fixes.py @@ -0,0 +1,171 @@ +"""Regression tests for the QC findings on the Resources backend. + +B1 — linked-project fields are owner-gated and published-only (no draft-title leak). +B4 — the listing runs a bounded query count (no N+1). +O1 — an explicit-empty required field is rejected on update. +O2 — the plain use-case update mutation cannot attach a resource. +""" + +import types +from datetime import date + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Collaborative, Publication, ResourceType, UseCase +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.enums import PublicationStatus, UseCaseStatus +from authorization.models import OrganizationMembership, Role, User + + +@pytest.fixture(autouse=True) +def _no_activity(monkeypatch): + monkeypatch.setattr("api.schema.base_mutation.record_activity", lambda *a, **k: None) + + +@pytest.fixture +def owner(db): + return User.objects.create(username="owner", keycloak_id="owner") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +def _pub(owner, rt, status=PublicationStatus.PUBLISHED, org=None, title="Findings"): + return Publication.objects.create( + title=title, + description="d", + user=owner, + resource_type=rt, + publication_date=date(2024, 1, 1), + status=status, + organization=org, + ) + + +def ctx(user, organization=None): + return types.SimpleNamespace( + user=user, context={"organization": organization} if organization else {} + ) + + +def run(query, user, variables=None, organization=None): + return schema.execute_sync( + query, variable_values=variables or {}, context_value=ctx(user, organization) + ) + + +LINKS = """ +query($id: UUID!) { + getPublication(publicationId: $id) { + id linkedUsecases { id title } linkedCount + } +} +""" + + +@pytest.mark.django_db +class TestLinkedFieldsGate: + def _linked_setup(self, owner, rt): + pub = _pub(owner, rt, status=PublicationStatus.PUBLISHED) + published_uc = UseCase.objects.create(title="Public UC", user=owner) + published_uc.status = UseCaseStatus.PUBLISHED + published_uc.save() + draft_uc = UseCase.objects.create(title="Secret Draft UC", user=owner) + published_uc.publications.add(pub) + draft_uc.publications.add(pub) + return pub + + def test_anonymous_sees_no_linked_projects(self, owner, resource_type): + pub = self._linked_setup(owner, resource_type) + + result = run(LINKS, AnonymousUser(), {"id": str(pub.id)}) + + assert result.errors is None + data = result.data["getPublication"] + assert data["linkedUsecases"] == [] # no draft title leaked + assert data["linkedCount"] == 0 + + def test_owner_sees_only_published_links(self, owner, resource_type): + pub = self._linked_setup(owner, resource_type) + + result = run(LINKS, owner, {"id": str(pub.id)}) + + data = result.data["getPublication"] + titles = [uc["title"] for uc in data["linkedUsecases"]] + assert titles == ["Public UC"] # the draft UC is hidden even from the owner + assert data["linkedCount"] == 1 + + +@pytest.mark.django_db +class TestListingIsBounded: + def test_listing_query_count_is_bounded( + self, owner, resource_type, django_assert_max_num_queries + ): + for i in range(5): + pub = _pub(owner, resource_type, title=f"Doc {i}") + pub.sectors.set([]) + pub.blocks.create( + position=0, + block_type="YOUTUBE", + youtube_url="https://youtu.be/dQw4w9WgXcQ", + youtube_video_id="dQw4w9WgXcQ", + ) + + query = """ + query { + publications(includePublic: true) { + id title resourceType { name } sectors { id } geographies { id } blocks { id } + } + } + """ + # Bounded: a constant number of queries regardless of the 5 rows + relations. + with django_assert_max_num_queries(15): + result = run(query, owner) + assert result.errors is None + assert len(result.data["publications"]) == 5 + + +UPDATE = """ +mutation($input: UpdatePublicationInput!) { + updatePublication(input: $input) { success errors { fieldErrors { field } } } +} +""" + + +@pytest.mark.django_db +class TestUpdateEmptyRejected: + def test_explicit_empty_title_is_rejected(self, owner, resource_type): + pub = _pub(owner, resource_type, status=PublicationStatus.DRAFT) + + result = run(UPDATE, owner, {"input": {"id": str(pub.id), "title": " "}}) + + assert result.data["updatePublication"]["success"] is False + pub.refresh_from_db() + assert pub.title == "Findings" # not blanked + + +@pytest.mark.django_db +class TestPlainUpdateCannotAttachPublication: + def test_usecase_input_has_no_publications_field(self, owner, resource_type): + pub = _pub(owner, resource_type, status=PublicationStatus.PUBLISHED) + uc = UseCase.objects.create(title="UC", user=owner) + + # The UC update input excludes 'publications', so passing it is a schema + # error — the only way to attach a resource is the guarded link trio. + mutation = """ + mutation($input: UseCaseInputPartial!) { + updateUseCase(useCaseInputPartial: $input) { __typename } + } + """ + result = run( + mutation, + owner, + {"input": {"id": str(uc.id), "publications": [str(pub.id)]}}, + ) + + assert result.errors is not None # 'publications' is not a valid input field + assert uc.publications.count() == 0 From 850e7e62ccd1ef8741d95ab5e05bd9fb27e67e4c Mon Sep 17 00:00:00 2001 From: dc Date: Mon, 20 Jul 2026 12:53:41 +0530 Subject: [PATCH 17/57] feat(publications): add resourceTypes query for the create/edit form The frontend metadata form needs to populate a Resource Type dropdown; expose an active-only, name-sorted resourceTypes query. Adds a test. --- api/schema/publication_schema.py | 11 ++++++++++- tests/schema/test_publication_schema.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/api/schema/publication_schema.py b/api/schema/publication_schema.py index 14f47f6..6ac9148 100644 --- a/api/schema/publication_schema.py +++ b/api/schema/publication_schema.py @@ -19,7 +19,7 @@ from strawberry.file_uploads import Upload from strawberry.types import Info -from api.models import Publication, PublicationBlock +from api.models import Publication, PublicationBlock, ResourceType from api.schema.base_mutation import BaseMutation, MutationResponse from api.services.publication_blocks import ( add_file_block, @@ -41,6 +41,7 @@ PublicationOrder, TypePublication, TypePublicationBlock, + TypeResourceType, publication_license, ) from api.utils.enums import PublicationStatus @@ -89,6 +90,14 @@ class UpdatePublicationInput: class Query: """Queries for publications.""" + @strawberry.field + @trace_resolver(name="get_resource_types", attributes={"component": "publication"}) + def resource_types(self, info: Info) -> List[TypeResourceType]: + """List the active Resource Types for a create/edit form's dropdown.""" + return TypeResourceType.from_django_list( + ResourceType.objects.filter(is_active=True).order_by("name") + ) + @strawberry.field( permission_classes=[AllowPublishedPublications], # type: ignore[list-item] ) diff --git a/tests/schema/test_publication_schema.py b/tests/schema/test_publication_schema.py index 26991be..a0f21ef 100644 --- a/tests/schema/test_publication_schema.py +++ b/tests/schema/test_publication_schema.py @@ -504,3 +504,16 @@ def test_other_org_cannot_remove_block( assert result.data["removePublicationBlock"]["success"] is False assert publication.blocks.filter(id=block.id).exists() + + +RESOURCE_TYPES = "query { resourceTypes { id name isActive } }" + + +@pytest.mark.django_db +class TestResourceTypesQuery: + def test_lists_only_active_types_sorted(self, resource_type, inactive_type): + result = run(RESOURCE_TYPES, ctx(AnonymousUser())) + + assert result.errors is None + names = [t["name"] for t in result.data["resourceTypes"]] + assert names == ["Report"] # active only; "Retired" excluded From e54e74ca1bcceb54e4431602f15401e625ebec3f Mon Sep 17 00:00:00 2001 From: dc Date: Mon, 20 Jul 2026 12:56:09 +0530 Subject: [PATCH 18/57] docs(publications): mark frontend implemented in backend arch doc --- api/schema/publication_architecture.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/schema/publication_architecture.md b/api/schema/publication_architecture.md index 7bd46cf..ba4a52e 100644 --- a/api/schema/publication_architecture.md +++ b/api/schema/publication_architecture.md @@ -1,6 +1,6 @@ # Publications (UI "Resource") — backend -> Part of feature: **resources** · siblings: `DataSpaceFrontend/app/[locale]/(user)/publications/architecture.md` (frontend slice — to be created with the FE phases) +> Part of feature: **resources** · siblings: `DataSpaceFrontend/app/[locale]/(user)/publications/architecture.md` (frontend slice) ## Overview @@ -111,5 +111,6 @@ n/a — all assertions are deterministic. - **Preview fidelity:** slide decks / DOCX / PPT are download-only (only PDF + YouTube render inline). - **No versioning** for re-uploaded block files (in-place replace). - **The dataset link trio's pre-existing IDOR** (no container-authorization check) is not fixed here — only the new publication trio is authorized. Follow-up: apply `assert_can_manage_links` to the dataset trio too. +- **Frontend is implemented** (see the sibling FE doc). Its Layer 6 browser flows and the visual checklist are written but not yet walked (agents don't run a visual pass on their own) — run on request. - **The list resolver trusts the request's `organization` header without a membership check** — a faithful mirror of the existing dataset/aimodel/usecase list resolvers and the shared middleware (`api/utils/middleware.py`, which still carries a `# TODO: resolve auth_token`). This feature follows the platform pattern; if org membership is not enforced on that header upstream, a signed-in user could list another org's drafts by setting the header. The correct fix is at the shared middleware layer (so all entities are fixed together), not per-endpoint — deliberately **not** patched here to avoid divergence and false security. Flagged for a platform-level decision. - **Frontend (Phases 7–8), the two Layer 5 journey scripts, and the FE architecture doc are not yet implemented** — the backend is complete and shippable; the UI is the remaining slice. From 0ca3c0de22b52b748a68aef2020c4e85072efafa Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:31:53 +0530 Subject: [PATCH 19/57] fix: health check now returns 503 when a dependency is unhealthy health_check previously returned JsonResponse(data) unconditionally, so the ECS container healthcheck (curl -f) and any future smoke gate would false-green a container with a dead DB/ES/Redis/telemetry connection. Also exposes git_sha so a deploy can verify the running code matches what was just pushed. --- api/views/health.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/views/health.py b/api/views/health.py index 12d527d..41491a3 100644 --- a/api/views/health.py +++ b/api/views/health.py @@ -1,3 +1,4 @@ +import os from typing import Any, Dict import requests @@ -141,6 +142,7 @@ def health_check(request: HttpRequest) -> JsonResponse: data = { "status": "healthy" if overall_status else "unhealthy", "services": status, + "git_sha": os.environ.get("GIT_COMMIT_SHA", "unknown"), } - return JsonResponse(data) + return JsonResponse(data, status=200 if overall_status else 503) From ed4b1161fd38c08e34de3178655eaa91d3db7994 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:31:58 +0530 Subject: [PATCH 20/57] test: cover health check status codes and git_sha field --- tests/test_health.py | 99 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tests/test_health.py diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 0000000..93a6b28 --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,99 @@ +"""Tests for the /health/ endpoint's status-code and git_sha behavior.""" + +import os +import unittest +from contextlib import ExitStack +from unittest.mock import MagicMock, patch + +from django.test import Client, override_settings + + +class TestHealthCheck(unittest.TestCase): + """The endpoint must reflect actual dependency health in its status code.""" + + def setUp(self) -> None: + self.client = Client() + + def _mock_healthy_dependencies(self, stack: ExitStack) -> None: + """Patch ES/Redis/telemetry so the happy path doesn't need real services.""" + # tests/test_settings.py's ELASTICSEARCH_DSL omits http_auth (the real + # DataSpace/settings.py value always sets it) — health_check reads it + # unconditionally, so supply it here rather than touching test settings. + stack.enter_context( + override_settings( + ELASTICSEARCH_DSL={ + "default": {"hosts": "localhost:9200", "http_auth": ("user", "pass")} + } + ) + ) + mock_es_instance = MagicMock() + mock_es_instance.ping.return_value = True + stack.enter_context( + patch("api.views.health.Elasticsearch", return_value=mock_es_instance) + ) + # `cache` is Django's lazy DefaultConnectionProxy — patching .set/.get + # as attributes on it gets forwarded to the real backend instead of + # being intercepted, so replace the name binding in the health module + # wholesale instead. + cache_store: dict = {} + mock_cache = MagicMock() + mock_cache.set.side_effect = lambda k, v, timeout=None: cache_store.__setitem__(k, v) + mock_cache.get.side_effect = lambda k: cache_store.get(k) + stack.enter_context(patch("api.views.health.cache", mock_cache)) + mock_get = stack.enter_context(patch("api.views.health.requests.get")) + mock_get.return_value.status_code = 200 + + def test_returns_200_when_all_dependencies_healthy(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + response = self.client.get("/health/") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["status"], "healthy") + + def test_returns_503_when_elasticsearch_unhealthy(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + mock_es_instance = MagicMock() + mock_es_instance.ping.return_value = False + stack.enter_context( + patch("api.views.health.Elasticsearch", return_value=mock_es_instance) + ) + response = self.client.get("/health/") + + self.assertEqual(response.status_code, 503) + body = response.json() + self.assertEqual(body["status"], "unhealthy") + self.assertEqual(body["services"]["elasticsearch"]["status"], "unhealthy") + + def test_returns_503_when_redis_unhealthy(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + mock_cache = MagicMock() + mock_cache.set.side_effect = Exception("down") + stack.enter_context(patch("api.views.health.cache", mock_cache)) + response = self.client.get("/health/") + + self.assertEqual(response.status_code, 503) + self.assertEqual(response.json()["services"]["redis"]["status"], "unhealthy") + + def test_git_sha_defaults_to_unknown(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + stack.enter_context(patch.dict(os.environ, {}, clear=False)) + os.environ.pop("GIT_COMMIT_SHA", None) + response = self.client.get("/health/") + + self.assertEqual(response.json()["git_sha"], "unknown") + + def test_git_sha_reflects_env_var(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + stack.enter_context(patch.dict(os.environ, {"GIT_COMMIT_SHA": "abc1234"})) + response = self.client.get("/health/") + + self.assertEqual(response.json()["git_sha"], "abc1234") + + +if __name__ == "__main__": + unittest.main() From 029689ae7b68c43ddbacbdf091dfab726f3335d5 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:31:58 +0530 Subject: [PATCH 21/57] build: plumb GIT_COMMIT_SHA into the image Follows the same ARG->ENV pattern the container already uses for other build-time config. Feeds health_check's new git_sha field. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 05be1d8..1fb7c77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ FROM python:3.10 +ARG GIT_COMMIT_SHA=unknown +ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA} ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From 4d821392fe835999bfb9010ccb61c70f22f800b4 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:31:58 +0530 Subject: [PATCH 22/57] fix: drop makemigrations from the container boot path Generating migration files at deploy time instead of using committed ones is unsafe under a rolling deployment (briefly 2 tasks live) and means the schema that lands in prod was never reviewed. migrate itself stays here for now; moving it to an explicit one-off step is next. --- docker-entrypoint.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 10d7bf4..96033e9 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -45,10 +45,6 @@ mkdir -p /code/api/migrations chmod -R 777 /code/api/migrations touch /code/api/migrations/__init__.py -# Run makemigrations first to ensure migration files are created -echo "Running makemigrations..." -python manage.py makemigrations --noinput - # Run migrations echo "Running migrations..." python manage.py migrate --noinput From 2ada230e01ca42244e8e386b060614964ec63e39 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:32:58 +0530 Subject: [PATCH 23/57] fix: always run the CloudFormation deploy, don't guess from head_commit github.event.head_commit.modified is only populated for single-commit pushes -- a squash-merge touching aws/cloudformation would silently skip infra sync. The CFN deploy is already idempotent (--no-fail-on-empty-changeset), so simplest robust fix is to just run it every time; costs one extra ~10-20s idempotent call per deploy. --- .github/workflows/deploy-to-ecs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 4fe1415..73a5dbb 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -27,7 +27,6 @@ jobs: name: Deploy Infrastructure runs-on: ubuntu-latest environment: development - if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.modified, 'aws/cloudformation') steps: - name: Checkout From abeb9d16404a01ea7944918ce3f362b4eeb9f3eb Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:33:29 +0530 Subject: [PATCH 24/57] fix: stop deploy-app running on a cancelled workflow if: always() made this job run even if the workflow was cancelled before it started. Now that deploy-infrastructure always runs (prior commit), it can never legitimately be 'skipped' either, so the implicit needs: gating (success-only, false on cancellation) is exactly the behavior wanted -- no explicit if: needed. --- .github/workflows/deploy-to-ecs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 73a5dbb..5487b8f 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -61,7 +61,6 @@ jobs: runs-on: ubuntu-latest environment: development needs: deploy-infrastructure - if: always() # Run even if infrastructure deployment is skipped steps: - name: Checkout From 4761b415a2d06b559a799910f73ffbcc6f826f3a Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:37:45 +0530 Subject: [PATCH 25/57] feat: run migrations as an explicit ECS one-off task before deploy Bumps aws-actions/amazon-ecs-deploy-task-definition v1 -> v2 (purely additive per its changelog) to use its built-in run-task support: runs a standalone task on the new task definition, waits for it to stop, and fails the whole action on a non-zero container exit -- before the service update ever happens. Network config (subnets, security groups, public-IP assignment) is read from the currently running service via describe-services rather than hardcoded, so the migration task always runs in the same network context as the app. --- .github/workflows/deploy-to-ecs.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 5487b8f..bab2d4c 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -103,13 +103,33 @@ jobs: container-name: dataspace image: ${{ steps.build-image.outputs.image }} - - name: Deploy main application ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + - name: Get running service's network configuration + id: network-config + run: | + NETWORK_CONFIG=$(aws ecs describe-services \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" \ + --query 'services[0].networkConfiguration.awsvpcConfiguration' \ + --output json) + { + echo "subnets=$(echo "$NETWORK_CONFIG" | jq -r '.subnets | join(",")')" + echo "security_groups=$(echo "$NETWORK_CONFIG" | jq -r '.securityGroups | join(",")')" + echo "assign_public_ip=$(echo "$NETWORK_CONFIG" | jq -r '.assignPublicIp')" + } >> "$GITHUB_OUTPUT" + + - name: Run migrations, then deploy the ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def-app.outputs.task-definition }} service: ${{ secrets.ECS_SERVICE }} cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true + run-task: true + run-task-container-overrides: '[{"name":"dataspace","command":["python","manage.py","migrate","--noinput"]}]' + run-task-subnets: ${{ steps.network-config.outputs.subnets }} + run-task-security-groups: ${{ steps.network-config.outputs.security_groups }} + run-task-assign-public-IP: ${{ steps.network-config.outputs.assign_public_ip }} + wait-for-task-stopped: true deploy-otel: name: Deploy OpenTelemetry Collector From dd9a68c6245c68df4f8bbf732643057636323dbc Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:37:45 +0530 Subject: [PATCH 26/57] chore: drop the now-dead migrations-directory scaffolding mkdir/chmod/touch on api/migrations existed only to let the runtime makemigrations step (removed earlier) write new migration files. migrate doesn't need to write to that directory, just read committed migrations, so this block has been dead weight since makemigrations was dropped. --- docker-entrypoint.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 96033e9..a54011c 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -39,13 +39,12 @@ while True: time.sleep(2) END -# Ensure migrations directory exists with proper permissions -echo "Ensuring migrations directory exists..." -mkdir -p /code/api/migrations -chmod -R 777 /code/api/migrations -touch /code/api/migrations/__init__.py - -# Run migrations +# Run migrations. In the ECS pipeline this also runs earlier as an explicit +# one-off task before this service is deployed (see deploy-to-ecs.yml) so a +# broken migration fails the deploy loudly and visibly instead of surfacing +# only once containers are already shipping; this call is then a no-op +# (migrate is idempotent). Kept here unconditionally too, since this same +# entrypoint/image is what `docker compose up` uses for local dev. echo "Running migrations..." python manage.py migrate --noinput From 5234618e842154ffe0665644938ca32494e96166 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:44:17 +0530 Subject: [PATCH 27/57] feat: gate the ECS deploy on a real smoke-test run Adds a smoke-tests job calling CivicDataSpace-test's run-smoke.yml, passing deployed_sha (github.sha) so the gate verifies /health/'s git_sha field matches, and min_passed to catch a fully-skipped run looking green. Requires a new repo variable DEV_API_BASE_URL and the same secrets DataSpaceFrontend's own pipeline already passes to this workflow (HOME_URL_DEV, TEST_EMAIL_1/2, TEST_PASSWORD_1/2). Also captures the currently-running task definition ARN before the service update (deploy-app now has an output for it) -- needed by the rollback job that follows in the next commit. --- .github/workflows/deploy-to-ecs.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index bab2d4c..9051ae9 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -61,6 +61,8 @@ jobs: runs-on: ubuntu-latest environment: development needs: deploy-infrastructure + outputs: + previous_task_def_arn: ${{ steps.previous-task-def.outputs.arn }} steps: - name: Checkout @@ -103,6 +105,17 @@ jobs: container-name: dataspace image: ${{ steps.build-image.outputs.image }} + - name: Capture currently-running task definition (for rollback) + id: previous-task-def + run: | + ARN=$(aws ecs describe-services \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" \ + --query 'services[0].taskDefinition' \ + --output text) + echo "Currently running: $ARN" + echo "arn=$ARN" >> "$GITHUB_OUTPUT" + - name: Get running service's network configuration id: network-config run: | @@ -131,6 +144,21 @@ jobs: run-task-assign-public-IP: ${{ steps.network-config.outputs.assign_public_ip }} wait-for-task-stopped: true + smoke-tests: + name: Smoke Tests + needs: deploy-app + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + api_base_url: ${{ vars.DEV_API_BASE_URL }} + deployed_sha: ${{ github.sha }} + min_passed: 1 + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + deploy-otel: name: Deploy OpenTelemetry Collector runs-on: ubuntu-latest From ba349a653a2f1aca0a47129e33c56823e84e8217 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:44:45 +0530 Subject: [PATCH 28/57] feat: auto-rollback the ECS service when smoke tests fail Restores the task definition ARN captured before this deploy's service update (previous commit), waits for the rolled-back service to stabilize, then exits 1 -- the run stays red even after successful mitigation, matching the policy that a rollback is damage control, not a pass. Migrations applied by the bad deploy are never auto-reverted; the error message points at where to find what ran. OTel collector rollback is explicitly out of scope here -- separate service, independent risk, keeps this change's blast radius to the app service only. --- .github/workflows/deploy-to-ecs.yml | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 9051ae9..b18dcc1 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -159,6 +159,44 @@ jobs: TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + rollback-on-smoke-failure: + name: Rollback on Smoke Failure + runs-on: ubuntu-latest + environment: development + needs: [deploy-app, smoke-tests] + if: failure() && needs.deploy-app.result == 'success' + timeout-minutes: 15 + + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Restore the previously-running task definition + run: | + PREVIOUS_ARN="${{ needs.deploy-app.outputs.previous_task_def_arn }}" + if [ -z "$PREVIOUS_ARN" ]; then + echo "::error::No previous task definition was captured -- nothing to roll back to. This is expected on a service's very first deploy." + exit 1 + fi + echo "Rolling back to: $PREVIOUS_ARN" + aws ecs update-service \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --service "${{ secrets.ECS_SERVICE }}" \ + --task-definition "$PREVIOUS_ARN" \ + --force-new-deployment + aws ecs wait services-stable \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" + + - name: Mark this run as failed despite successful rollback + run: | + echo "::error::Smoke tests failed after deploy. Rolled back to the previous task definition -- migrations applied by this deploy were NOT reverted. Check what ran via the 'Run migrations, then deploy' step's logs before re-deploying." + exit 1 + deploy-otel: name: Deploy OpenTelemetry Collector runs-on: ubuntu-latest From 558211d6f37a5a477d3d78379f62b1b29bf68325 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 18 Aug 2026 14:38:09 +0530 Subject: [PATCH 29/57] fix: make backend deployable by image, not just local build backend had no image: reference and bind-mounted the working tree with uvicorn --reload -- fine for local dev, but means a CD pipeline has nothing to pull and, even if it did, the bind mount would shadow whatever image is actually running. Adds an image: tag (defaults to a local placeholder so plain local builds are unaffected) and drops the bind mount + reload command from the base file; hot-reload moves to a new opt-in overlay in the next commit. Also adds a release service for one-off management commands (starting with migrations) against the deployed image without touching the running backend container -- mirrors the pattern already proven for ParakhAPI's own EC2 pipeline. --- docker-compose.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e67e1ca..2b194cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,10 @@ services: backend: + image: ${DATASPACE_IMAGE:-dataspace-backend:local} build: . env_file: .env container_name: "DataSpace" - command: uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --reload - volumes: - - .:/code ports: - "8000:8000" depends_on: @@ -27,6 +25,21 @@ services: max-size: "10m" max-file: "3" + # One-off management commands (migrations, etc.) against the deployed + # image, without touching the running `backend` container. No + # container_name, so `docker compose run` can spin up a fresh instance + # even while `backend` is up. Never starts on a plain `up`. + release: + image: ${DATASPACE_IMAGE:-dataspace-backend:local} + build: . + env_file: .env + profiles: ["release"] + depends_on: + backend_db: + condition: service_healthy + entrypoint: ["python", "manage.py"] + command: ["migrate", "--noinput"] + backend_db: image: "postgres:14.4" env_file: .env From 1484ae16ac272381db0f0d6daafd29ffe5e39ef3 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 18 Aug 2026 14:38:09 +0530 Subject: [PATCH 30/57] feat: opt-in hot-reload overlay for local dev Compose merges (concatenates) volumes: lists across -f files rather than replacing them, so removing the bind mount from the base file required moving it here instead of an override that tries to remove it -- an override can only add, never subtract. Local dev usage: docker compose -f docker-compose.yml -f docker-compose.hotreload.yml up --build --- docker-compose.hotreload.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docker-compose.hotreload.yml diff --git a/docker-compose.hotreload.yml b/docker-compose.hotreload.yml new file mode 100644 index 0000000..d8fce17 --- /dev/null +++ b/docker-compose.hotreload.yml @@ -0,0 +1,9 @@ +# Opt-in overlay for local development: bind-mounts the working tree and +# runs uvicorn with --reload. Not used in any deployed environment. +# +# Usage: docker compose -f docker-compose.yml -f docker-compose.hotreload.yml up --build +services: + backend: + command: uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --reload + volumes: + - .:/code From 470b4cfe438189721c6ac49e17e9dd0c8457b8c4 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 18 Aug 2026 14:38:22 +0530 Subject: [PATCH 31/57] feat: add EC2 deploy pipeline for the real dev backend host build -> deploy -> smoke-tests -> rollback-on-smoke-failure, mirroring ParakhAPI's proven pattern. Deploys by immutable @sha256 digest to the existing EC2 host (not managed by this repo -- Postgres/Redis/ Elasticsearch/Keycloak already run there, untouched by this pipeline; only the backend container is pulled/swapped). Migrations run via the new release service as an explicit, blocking step before cutover. Two-tier rollback: an in-script health-check retry inside deploy for a container that never becomes healthy, and a separate job-level rollback for smoke-test failures that only surface after deploy already reported success -- same split Parakh's pipeline uses and for the same reason (different failure signals arrive at different times). otel-collector is deliberately left out -- it's present on the host but disabled today; this pipeline shouldn't silently re-enable it. --- .github/workflows/deploy-backend.yml | 193 +++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 .github/workflows/deploy-backend.yml diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml new file mode 100644 index 0000000..8e4cd49 --- /dev/null +++ b/.github/workflows/deploy-backend.yml @@ -0,0 +1,193 @@ +name: Deploy Backend to Dev EC2 + +on: + push: + branches: + - dev + workflow_dispatch: + inputs: + force_smoke_failure: + description: "Deliberately fail the smoke gate to prove rollback works" + type: boolean + default: false + +concurrency: + group: dataspace-backend-dev-deploy + cancel-in-progress: false + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + DEPLOY_PATH: ${{ vars.DEPLOY_PATH || 'DataSpaceBackend' }} + +jobs: + build: + name: Build & push image + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + image_ref: ${{ steps.push.outputs.image_ref }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=dev + type=sha,prefix=dev- + + - name: Build and push + id: build + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + GIT_COMMIT_SHA=${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Pin image by digest + id: push + run: | + echo "image_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" >> "$GITHUB_OUTPUT" + + - name: Sanity check the built image + run: | + docker run --rm \ + -e SECRET_KEY=build-check \ + -e URL_WHITELIST=http://localhost \ + -e DB_ENGINE=django.db.backends.sqlite3 \ + --entrypoint python \ + "${{ steps.push.outputs.image_ref }}" \ + manage.py check + + deploy: + name: Deploy to EC2 + needs: build + runs-on: ubuntu-latest + environment: development + timeout-minutes: 15 + outputs: + previous_image: ${{ steps.deploy.outputs.previous_image }} + + steps: + - name: Deploy over SSH + id: deploy + uses: appleboy/ssh-action@v1.0.3 + env: + DATASPACE_IMAGE: ${{ needs.build.outputs.image_ref }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + GHCR_ACTOR: ${{ github.actor }} + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + envs: DATASPACE_IMAGE,GHCR_TOKEN,GHCR_ACTOR + command_timeout: 12m + script: | + set -euo pipefail + cd "$HOME/${{ env.DEPLOY_PATH }}" + + echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_ACTOR" --password-stdin + + PREVIOUS_IMAGE=$(docker inspect --format='{{.Config.Image}}' DataSpace 2>/dev/null || echo "") + echo "Currently running: ${PREVIOUS_IMAGE:-}" + mkdir -p .deploy + echo "$PREVIOUS_IMAGE" > .deploy/previous_image + echo "previous_image=$PREVIOUS_IMAGE" >> "$GITHUB_OUTPUT" + + docker compose pull backend + + echo "Running migrations..." + docker compose --profile release run --rm release + + docker compose up -d --no-deps backend + + echo "Waiting for health..." + for i in $(seq 1 20); do + if curl -sf http://localhost:8000/health/ > /dev/null; then + echo "Healthy." + exit 0 + fi + sleep 6 + done + + echo "::error::backend did not become healthy after deploy -- rolling back in-place" + if [ -n "$PREVIOUS_IMAGE" ]; then + DATASPACE_IMAGE="$PREVIOUS_IMAGE" docker compose up -d --no-deps backend + fi + exit 1 + + smoke-tests: + name: Smoke Tests + needs: deploy + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + api_base_url: ${{ vars.DEV_API_BASE_URL }} + deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }} + min_passed: 1 + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + + rollback-on-smoke-failure: + name: Rollback on Smoke Failure + runs-on: ubuntu-latest + environment: development + needs: [deploy, smoke-tests] + if: failure() && needs.deploy.result == 'success' + timeout-minutes: 10 + + steps: + - name: Restore previous image over SSH + uses: appleboy/ssh-action@v1.0.3 + env: + PREVIOUS_IMAGE: ${{ needs.deploy.outputs.previous_image }} + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + envs: PREVIOUS_IMAGE + script: | + set -euo pipefail + cd "$HOME/${{ env.DEPLOY_PATH }}" + + if [ -z "${PREVIOUS_IMAGE:-}" ]; then + echo "::error::No previous image was captured -- nothing to roll back to. This is expected on the very first deploy." + exit 1 + fi + + echo "Rolling back to: $PREVIOUS_IMAGE" + DATASPACE_IMAGE="$PREVIOUS_IMAGE" docker compose up -d --no-deps backend + + for i in $(seq 1 20); do + if curl -sf http://localhost:8000/health/ > /dev/null; then + echo "Rollback healthy." + exit 0 + fi + sleep 6 + done + echo "::error::Rollback image also failed to become healthy -- needs manual intervention." + exit 1 + + - name: Mark this run as failed despite successful rollback + run: | + echo "::error::Smoke tests failed after deploy. Rolled back to the previous image -- migrations applied by this deploy were NOT reverted." + exit 1 From 3777b12ccc2bb6c5a144ca6ac748e4b7ea146942 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 1 Sep 2026 17:19:03 +0530 Subject: [PATCH 32/57] Fix dev EC2 deploy pipeline before its first real run The draft workflow (committed 2026-08-18, never pushed) copied ParakhAPI's pre-fix CD pipeline pattern -- inline multi-line appleboy/ssh-action script: blocks with an envs: input. That pattern reproducibly corrupts in transport for reasons never fully root-caused (five failed live iterations there before the fix). Applying the fix proactively here: every SSH step is now a single trivial invocation line, and the real deploy/rollback/finalize logic lives in scripts/ci-*.sh, shipped via the already-reliable scp-action. Also fixed, discovered by investigating the real target host directly: - docker/docker compose require sudo on this host (confirmed passwordless) -- the draft's bare docker commands would have failed immediately. Every ci-*.sh invocation is now prefixed with sudo. - The release service's entrypoint is already ["python", "manage.py"] (docker-compose.yml) -- the draft would have run "python manage.py python manage.py migrate", doubled. Scripts now pass bare manage.py subcommands. - DEPLOY_PATH is DataExchange/DataExBackend, not a standalone DataSpaceBackend checkout -- this host runs a git submodule inside a separate DataExchange superproject (confirmed via docker ps compose labels: project=dataexbackend, workdir matches). The backend's compose stack already runs self-contained from that directory; this pipeline never touches the superproject repo or its top-level orchestration file. - GHCR_TOKEN (a PAT) swapped for the run's own GITHUB_TOKEN with packages: read -- skip the manual-PAT requirement from the start. Added a finalize-deploy job (image pruning) for parity with ParakhAPI's pipeline; the draft didn't have one. All three scripts validated with shellcheck and bash -n, both locally and against the real target host's actual bash. Workflow validated with actionlint. EC2_HOST/EC2_USERNAME/EC2_PRIVATE_KEY set in the development environment, using a new deploy-only SSH key rather than the shared key that also opens several other boxes. --- .github/workflows/deploy-backend.yml | 266 ++++++++++++++++----------- .gitignore | 5 + scripts/ci-deploy.sh | 121 ++++++++++++ scripts/ci-finalize.sh | 42 +++++ scripts/ci-rollback.sh | 48 +++++ 5 files changed, 379 insertions(+), 103 deletions(-) create mode 100755 scripts/ci-deploy.sh create mode 100755 scripts/ci-finalize.sh create mode 100755 scripts/ci-rollback.sh diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 8e4cd49..f0e471a 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -1,34 +1,77 @@ +# Dev CD pipeline: build once in CI, deploy that exact artifact, verify it +# with real API tests, and roll the image back automatically if verification +# fails. +# +# build -> deploy -> smoke-tests -> { rollback-on-smoke-failure | finalize } +# +# Deploys are pinned to an immutable digest, never a tag: `dev` is a moving +# pointer, so "roll back to the previous dev tag" is not a thing you can +# express. The host records the digest it was running before each deploy in +# .deploy/previous_image, which is what makes cross-job rollback possible at +# all (rollback runs on a different runner, so no shell state survives). +# +# MIGRATIONS ARE NOT ROLLED BACK. A rollback restores the previous image and +# says so loudly; the schema stays forward. Migrations must therefore be +# written additively / backward-compatibly, so the previous image can still +# run against the newer schema. This is a policy constraint on how you write +# migrations, not something this pipeline can enforce for you. +# +# The target host is not a standalone checkout of this repo -- it's the +# DataExBackend submodule inside the separate CivicDataLab/DataExchange +# superproject, already running its own self-contained compose project +# there (confirmed via `docker ps` compose labels: project=dataexbackend, +# workdir=~/DataExchange/DataExBackend). This pipeline only ever touches +# that directory; it never touches the DataExchange repo or its top-level +# compose file. +# +# Several structural choices here mirror ParakhAPI's proven dev CD pipeline +# (deploy-parakh-api-dev.yml in CivicDataLab/ParakhAI-Backend) -- see the +# notes at each site before "simplifying" them. In particular: +# appleboy/ssh-action's inline multi-line `script:` input was found there to +# reproducibly fail with a spurious "syntax error near unexpected token ';'" +# for reasons never fully root-caused (confirmed the script text itself was +# valid bash both locally and on the real target host every time -- the +# corruption happened somewhere in the action's own transport). scp-action +# never had that problem. So no SSH step here ever carries more than a +# single trivial invocation line; all real logic lives in scripts/ci-*.sh, +# shipped as files. + name: Deploy Backend to Dev EC2 on: push: - branches: - - dev + branches: ['dev'] workflow_dispatch: inputs: force_smoke_failure: - description: "Deliberately fail the smoke gate to prove rollback works" + description: "Deliberately fail the smoke gate, to exercise the rollback path. Testing only." type: boolean + required: false default: false +# Queue overlapping deploys rather than cancelling: a cancelled run mid-deploy +# could leave .deploy/ state and the running containers disagreeing. concurrency: group: dataspace-backend-dev-deploy cancel-in-progress: false env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - DEPLOY_PATH: ${{ vars.DEPLOY_PATH || 'DataSpaceBackend' }} + IMAGE_NAME: civicdatalab/dataspacebackend + DEPLOY_PATH: ${{ vars.DEPLOY_PATH || 'DataExchange/DataExBackend' }} jobs: build: - name: Build & push image + name: Build and push image runs-on: ubuntu-latest timeout-minutes: 20 + permissions: + contents: read + packages: write outputs: - image_ref: ${{ steps.push.outputs.image_ref }} + image_ref: ${{ steps.ref.outputs.image_ref }} steps: - - name: Checkout + - name: Checkout code uses: actions/checkout@v4 - name: Log in to GHCR @@ -38,41 +81,35 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=dev - type=sha,prefix=dev- - - name: Build and push id: build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT_SHA=${{ github.sha }} + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max - - name: Pin image by digest - id: push + - name: Pin image reference by digest + id: ref run: | echo "image_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" >> "$GITHUB_OUTPUT" - - name: Sanity check the built image + - name: Sanity-check the built image + # Cheap, real gate: catches import errors and bad settings before + # anything touches the host. run: | docker run --rm \ - -e SECRET_KEY=build-check \ + -e SECRET_KEY=ci-sanity-check-not-a-real-key \ -e URL_WHITELIST=http://localhost \ -e DB_ENGINE=django.db.backends.sqlite3 \ --entrypoint python \ - "${{ steps.push.outputs.image_ref }}" \ + "${{ steps.ref.outputs.image_ref }}" \ manage.py check deploy: @@ -81,65 +118,58 @@ jobs: runs-on: ubuntu-latest environment: development timeout-minutes: 15 - outputs: - previous_image: ${{ steps.deploy.outputs.previous_image }} - + # packages: read -- GITHUB_TOKEN needs this explicitly granted to pull + # from GHCR; it isn't covered by the repo's default token permissions. + permissions: + contents: read + packages: read steps: - - name: Deploy over SSH - id: deploy + - name: Checkout code + uses: actions/checkout@v4 + + - name: Write GHCR token file + run: printf '%s' "${{ secrets.GITHUB_TOKEN }}" > .ghcr_token + + # Ship the compose files rather than relying on the DataExBackend + # submodule pointer inside the separate DataExchange repo -- bumping + # that pointer is a change to a different, shared repo and out of + # scope here. Shipping the file directly keeps this pipeline + # self-contained. + - name: Ship deploy files to host + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + source: docker-compose.yml,docker-compose.hotreload.yml,scripts/ci-deploy.sh,.ghcr_token + target: ${{ env.DEPLOY_PATH }} + + - name: Deploy uses: appleboy/ssh-action@v1.0.3 - env: - DATASPACE_IMAGE: ${{ needs.build.outputs.image_ref }} - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} - GHCR_ACTOR: ${{ github.actor }} with: host: ${{ vars.EC2_HOST }} username: ${{ secrets.EC2_USERNAME }} key: ${{ secrets.EC2_PRIVATE_KEY }} - envs: DATASPACE_IMAGE,GHCR_TOKEN,GHCR_ACTOR - command_timeout: 12m - script: | - set -euo pipefail - cd "$HOME/${{ env.DEPLOY_PATH }}" - - echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_ACTOR" --password-stdin - - PREVIOUS_IMAGE=$(docker inspect --format='{{.Config.Image}}' DataSpace 2>/dev/null || echo "") - echo "Currently running: ${PREVIOUS_IMAGE:-}" - mkdir -p .deploy - echo "$PREVIOUS_IMAGE" > .deploy/previous_image - echo "previous_image=$PREVIOUS_IMAGE" >> "$GITHUB_OUTPUT" - - docker compose pull backend - - echo "Running migrations..." - docker compose --profile release run --rm release - - docker compose up -d --no-deps backend - - echo "Waiting for health..." - for i in $(seq 1 20); do - if curl -sf http://localhost:8000/health/ > /dev/null; then - echo "Healthy." - exit 0 - fi - sleep 6 - done - - echo "::error::backend did not become healthy after deploy -- rolling back in-place" - if [ -n "$PREVIOUS_IMAGE" ]; then - DATASPACE_IMAGE="$PREVIOUS_IMAGE" docker compose up -d --no-deps backend - fi - exit 1 + script_stop: true + # sudo: docker/docker compose require it on this host (confirmed + # passwordless -- sudo -n succeeds non-interactively). + script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-deploy.sh "${{ needs.build.outputs.image_ref }}" "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" smoke-tests: name: Smoke Tests needs: deploy + # No `environment:` here -- GitHub rejects the entire workflow file at + # parse time if a `uses:` job declares one. Consequence: this job also + # cannot see environment-scoped vars, which is why api_base_url comes + # from a repo-level var. uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI with: api_base_url: ${{ vars.DEV_API_BASE_URL }} + # An obviously-wrong sentinel SHA fails the reusable workflow's own + # deployed-SHA-vs-live-/health/ assertion on purpose, for the + # force_smoke_failure test path. deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }} - min_passed: 1 + min_passed: ${{ inputs.force_smoke_failure && 999 || 1 }} secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} @@ -148,46 +178,76 @@ jobs: TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} rollback-on-smoke-failure: - name: Rollback on Smoke Failure - runs-on: ubuntu-latest - environment: development + name: Rollback (smoke tests failed) + # `deploy` must be in needs: for needs.deploy.result to resolve here. needs: [deploy, smoke-tests] + # failure()/success() builtins rather than needs.smoke-tests.result -- + # both are false on cancellation, which is the behaviour we want; + # if: always() would ignore cancellation entirely. if: failure() && needs.deploy.result == 'success' - timeout-minutes: 10 - + runs-on: ubuntu-latest + environment: development + timeout-minutes: 15 + permissions: + contents: read + packages: read steps: - - name: Restore previous image over SSH + - name: Checkout code + uses: actions/checkout@v4 + + - name: Write GHCR token file + run: printf '%s' "${{ secrets.GITHUB_TOKEN }}" > .ghcr_token + + - name: Ship rollback files to host + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + source: scripts/ci-rollback.sh,.ghcr_token + target: ${{ env.DEPLOY_PATH }} + + - name: Restore previous image uses: appleboy/ssh-action@v1.0.3 - env: - PREVIOUS_IMAGE: ${{ needs.deploy.outputs.previous_image }} with: host: ${{ vars.EC2_HOST }} username: ${{ secrets.EC2_USERNAME }} key: ${{ secrets.EC2_PRIVATE_KEY }} - envs: PREVIOUS_IMAGE - script: | - set -euo pipefail - cd "$HOME/${{ env.DEPLOY_PATH }}" - - if [ -z "${PREVIOUS_IMAGE:-}" ]; then - echo "::error::No previous image was captured -- nothing to roll back to. This is expected on the very first deploy." - exit 1 - fi - - echo "Rolling back to: $PREVIOUS_IMAGE" - DATASPACE_IMAGE="$PREVIOUS_IMAGE" docker compose up -d --no-deps backend - - for i in $(seq 1 20); do - if curl -sf http://localhost:8000/health/ > /dev/null; then - echo "Rollback healthy." - exit 0 - fi - sleep 6 - done - echo "::error::Rollback image also failed to become healthy -- needs manual intervention." - exit 1 - - - name: Mark this run as failed despite successful rollback + script_stop: true + script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-rollback.sh "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" + + - name: Mark this run as failed + # The mitigation succeeded, but the run must still read RED -- a bad + # deploy that silently self-heals is a bad deploy nobody investigates. run: | - echo "::error::Smoke tests failed after deploy. Rolled back to the previous image -- migrations applied by this deploy were NOT reverted." + echo "::error::Smoke tests failed after deploy; the image was rolled back. Migrations were NOT reverted -- see the rollback step's log." exit 1 + + finalize-deploy: + name: Finalize Deploy + needs: [deploy, smoke-tests] + if: success() + runs-on: ubuntu-latest + environment: development + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ship finalize script to host + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + source: scripts/ci-finalize.sh + target: ${{ env.DEPLOY_PATH }} + + - name: Prune to current + previous image + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + script_stop: true + script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-finalize.sh diff --git a/.gitignore b/.gitignore index 63050b3..847c7a0 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,8 @@ dvc/* # Git worktrees for feature branches .worktrees/ + +# CD/deploy runtime state (previous/current image refs, pending migrations +# list -- written by the deploy pipeline on the host, never committed) +.deploy/ +.ghcr_token diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh new file mode 100755 index 0000000..2118137 --- /dev/null +++ b/scripts/ci-deploy.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Deploy step logic, run on the dev host by the "Deploy" job in +# .github/workflows/deploy-backend.yml. Lives as a real file rather than an +# inline appleboy/ssh-action `script:` block -- ParakhAPI's equivalent +# pipeline hit a reproducible "syntax error near unexpected token ';'" from +# inline multi-line scripts sent through that action (confirmed the script +# text itself was valid bash both locally and on the target host every +# time; the corruption happened somewhere in the action's own transport, +# not in the script content). scp-action, which ships this file, has no +# such problem -- so complex logic never goes through the SSH action's +# inline `script:` here at all. +# +# Invoked as: sudo bash ci-deploy.sh "$IMAGE_REF" "$HEALTH_CHECK_URL" "$GHCR_USER" +# GHCR_TOKEN is read from .ghcr_token (shipped alongside this script, +# deleted immediately below) rather than passed as an argument, since +# argv is visible via ps aux for the process's lifetime. +# +# docker/docker compose need sudo on this host -- confirmed passwordless +# (sudo -n succeeds), so this script must itself be invoked with sudo. +set -euo pipefail + +IMAGE_REF="$1" +HEALTH_CHECK_URL="$2" +GHCR_USER="$3" + +mkdir -p .deploy +GHCR_TOKEN="$(cat .ghcr_token)" +rm -f .ghcr_token + +# --- preflight ------------------------------------------------- +AVAIL_MB=$(df -Pm . | awk "NR==2{print \$4}") +if [ "$AVAIL_MB" -lt 4096 ]; then + echo "::error::Only ${AVAIL_MB}MB free on the deploy volume; refusing to pull. Free space and re-run." + exit 1 +fi +docker compose version >/dev/null 2>&1 || { + echo "::error::Docker Compose V2 not available (V1 docker-compose is a different, incompatible tool)." + exit 1 +} + +# --- record rollback anchor BEFORE touching anything ----------- +PREV_REF="$(docker inspect --format "{{.Config.Image}}" DataSpace 2>/dev/null || true)" +if [ -z "$PREV_REF" ] && [ -f .deploy/current_image ]; then + PREV_REF="$(cat .deploy/current_image)" +fi +# Only a digest ref is safely rollback-able. Anything else (a tag, +# a locally-built image, or nothing) means the previous state was +# hand-managed -- record that honestly instead of writing a value +# a later rollback would deploy blindly. The very first image-based +# deploy will land here: the currently running container was built +# locally (`dataexbackend-backend`), not pulled by digest. +case "$PREV_REF" in + *@sha256:*) : ;; + *) + echo "::warning::No digest-pinned previous image found (got [${PREV_REF:-}]). Automatic rollback is UNAVAILABLE for this run." + PREV_REF="" + ;; +esac +printf "%s" "$PREV_REF" > .deploy/previous_image + +# --- pull the new image ---------------------------------------- +echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin +docker pull "$IMAGE_REF" +printf "DATASPACE_IMAGE=%s\n" "$IMAGE_REF" > .deploy/image.env + +COMPOSE="docker compose -f docker-compose.yml --env-file .env --env-file .deploy/image.env" +RELEASE="$COMPOSE --profile release run --rm --no-deps -T release" +# The release service's entrypoint is already ["python", "manage.py"] +# (see docker-compose.yml) -- args below are manage.py subcommands +# only, not full "python manage.py ..." invocations. + +# backend_db/elasticsearch/redis must be up before the release step below: +# it uses --no-deps (so compose never recreates them out from under a +# running deploy), which also means it will NOT wait for their +# depends_on healthchecks. In steady state these are already running, +# but a host that had the stack fully down would otherwise fail at +# migrate with a confusing connection error. up -d is idempotent -- +# no-ops when they are already healthy and their config is unchanged. +$COMPOSE up -d backend_db elasticsearch redis + +# --- release step, against the NEW image, before the swap ------ +# Recorded for the rollback message: a rollback restores the image +# but NOT the schema, so whoever reads that failure needs to know +# what was applied. +$RELEASE showmigrations --plan 2>/dev/null | grep "^\[ \]" > .deploy/migrations.txt || true +$RELEASE migrate --noinput + +# --- swap the running container --------------------------------- +# --no-deps so backend_db/elasticsearch/redis are never recreated +# out from under this. +$COMPOSE up -d --no-build --no-deps --force-recreate backend + +# --- health gate, with in-job rollback (tier 1) ------------------ +rollback_now() { + echo "::error::$1" + if [ -z "$PREV_REF" ]; then + echo "::error::No previous image recorded -- the NEW image is still live. Manual intervention required." + exit 1 + fi + echo "Restoring $PREV_REF" + printf "DATASPACE_IMAGE=%s\n" "$PREV_REF" > .deploy/image.env + $COMPOSE up -d --no-build --no-deps --force-recreate backend + exit 1 +} + +attempts=0 +until curl -fsS -o /dev/null --max-time 10 "$HEALTH_CHECK_URL"; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 20 ]; then + # $COMPOSE, not a bare docker compose -f ...: an explicit + # --env-file disables .env auto-discovery, and DATASPACE_IMAGE + # needs to keep resolving to the image just deployed for these + # logs to target the right container. + $COMPOSE logs --tail 50 backend || true + rollback_now "Deployed image did not become healthy after $attempts attempts." + fi + sleep 5 +done + +printf "%s" "$IMAGE_REF" > .deploy/current_image +echo "Deploy healthy: $IMAGE_REF" diff --git a/scripts/ci-finalize.sh b/scripts/ci-finalize.sh new file mode 100755 index 0000000..01691b1 --- /dev/null +++ b/scripts/ci-finalize.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Finalize-deploy image pruning logic, run on the dev host. See +# ci-deploy.sh's header comment for why this lives as a real file instead +# of an inline appleboy/ssh-action script. No secrets needed here. +set -euo pipefail + +CUR="$(cat .deploy/current_image 2>/dev/null || true)" +PREV="$(cat .deploy/previous_image 2>/dev/null || true)" + +# Keep the previous image: it is what makes rollback instant, and +# survivable even if GHCR is unreachable during an incident. +# Build the ref list explicitly rather than relying on unquoted +# expansion to drop an empty PREV -- PREV is legitimately empty on +# a first deploy, and passing "" to docker inspect is an error. +KEEP_REFS=() +[ -n "$CUR" ] && KEEP_REFS+=("$CUR") +[ -n "$PREV" ] && KEEP_REFS+=("$PREV") +KEEP="" +if [ ${#KEEP_REFS[@]} -gt 0 ]; then + KEEP="$(docker inspect --format "{{.Id}}" "${KEEP_REFS[@]}" 2>/dev/null | sort -u || true)" +fi + +# Guard against the degenerate case: if we somehow resolved +# nothing to keep, pruning by ID below would remove every image +# for this repo including the one currently running. Bail instead. +if [ -z "$KEEP" ]; then + echo "::warning::Could not resolve current/previous image IDs; skipping prune rather than risk removing the running image." + docker image prune -f + df -h . + exit 0 +fi + +for id in $(docker images --no-trunc --format "{{.ID}}" "ghcr.io/civicdatalab/dataspacebackend" 2>/dev/null); do + if ! printf "%s\n" "$KEEP" | grep -q "$id"; then + docker rmi "$id" 2>/dev/null || true + fi +done + +# Dangling layers only. NEVER `docker system prune -a` here -- it +# would remove the previous image and silently disable rollback. +docker image prune -f +df -h . diff --git a/scripts/ci-rollback.sh b/scripts/ci-rollback.sh new file mode 100755 index 0000000..cdc2963 --- /dev/null +++ b/scripts/ci-rollback.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Rollback-on-smoke-failure logic, run on the dev host. See ci-deploy.sh's +# header comment for why this lives as a real file instead of an inline +# appleboy/ssh-action script. +# +# Invoked as: sudo bash ci-rollback.sh "$HEALTH_CHECK_URL" "$GHCR_USER" +# GHCR_TOKEN is read from .ghcr_token (shipped alongside this script, +# deleted immediately below) rather than passed as an argument. +set -euo pipefail + +HEALTH_CHECK_URL="$1" +GHCR_USER="$2" + +GHCR_TOKEN="$(cat .ghcr_token)" +rm -f .ghcr_token + +PREV_REF="$(cat .deploy/previous_image 2>/dev/null || true)" +if [ -z "$PREV_REF" ]; then + echo "::error::Smoke tests failed but no previous image is recorded -- the NEW image is still live. Manual intervention required." + exit 1 +fi + +# finalize-deploy is the only pruner and it did not run, so the +# previous image should still be local. Pull is a safety net. +docker image inspect "$PREV_REF" >/dev/null 2>&1 || { + echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin + docker pull "$PREV_REF" +} + +printf "DATASPACE_IMAGE=%s\n" "$PREV_REF" > .deploy/image.env +COMPOSE="docker compose -f docker-compose.yml --env-file .env --env-file .deploy/image.env" + +$COMPOSE up -d --no-build --no-deps --force-recreate backend + +attempts=0 +until curl -fsS -o /dev/null --max-time 10 "$HEALTH_CHECK_URL"; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 20 ]; then + echo "::error::Rolled-back image did not become healthy after $attempts attempts." + exit 1 + fi + sleep 5 +done + +printf "%s" "$PREV_REF" > .deploy/current_image +echo "Rolled back to $PREV_REF" +echo "NOTE: migrations applied by the failed deploy were NOT reverted:" +cat .deploy/migrations.txt 2>/dev/null || echo " (none recorded)" From 068e4ea9f0577d6efeb82dcf78ca2e6c4bfb4318 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 1 Sep 2026 17:56:02 +0530 Subject: [PATCH 33/57] Add missing Buildx setup step with docker-container driver Build failed on the very first live run: "Cache export is not supported for the docker driver" -- the rewrite dropped the setup-buildx-action step entirely, so build-push-action fell back to the runner's ambient default builder (driver: docker, confirmed via the failed run's own builder-info log), which doesn't support cache-to: type=gha. --- .github/workflows/deploy-backend.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index f0e471a..3268598 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -74,6 +74,15 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + # driver: docker-container explicitly, not the ambient default -- + # this runner's default buildx context reports driver "docker", + # which does not support cache export (cache-to: type=gha below + # fails outright without this). + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: Log in to GHCR uses: docker/login-action@v3 with: From 782de94ae8d9adfb4b8ec6a3ca143287b9d548c6 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 1 Sep 2026 18:35:48 +0530 Subject: [PATCH 34/57] Fix slow cache export: drop mode=max, bump build timeout to 45m The first live build succeeded (image pushed fine, confirmed in the run log: step #16 DONE 651.7s) but then hung writing GHA cache layers and blew the build job's 20-minute timeout -- mode=max exports every intermediate stage's layers, which only pays off for multi-stage builds; this Dockerfile is single-stage (FROM python:3.10, no builder stage), so it was buying nothing but a slow, cold-cache export of one large layer (chromium + apt packages). Dropped to the default (mode=min). Bumped the timeout to 45m regardless, matching ParakhAPI's budget, as safety margin. --- .github/workflows/deploy-backend.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 3268598..3e20df3 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -64,7 +64,7 @@ jobs: build: name: Build and push image runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 45 permissions: contents: read packages: write @@ -101,8 +101,15 @@ jobs: tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }} + # mode=min (the default), not max: this Dockerfile is single-stage + # (no multi-stage FROM ... AS builder), so mode=max's extra + # intermediate-stage caching buys nothing here -- it only added a + # slow cache-export step that got stuck writing one large layer + # (chromium + apt packages) and blew the job's timeout on the + # first (cold-cache) run, even though the actual image build and + # push had already completed successfully by that point. cache-from: type=gha - cache-to: type=gha,mode=max + cache-to: type=gha - name: Pin image reference by digest id: ref From ba0280353e3f07fcca26382ce5b1e21e696c0af5 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 1 Sep 2026 23:30:42 +0530 Subject: [PATCH 35/57] Create logs/ dir in image: LOGGING writes there, dir never existed Sanity-check step failed on a genuinely fresh container (first time anyone's actually run manage.py check against this image without a pre-existing logs/ from some other source): FileNotFoundError on django.setup() itself, since Django's logging.config never creates a FileHandler's parent directory. Same bug would hit healthcheck.sh (also calls django.setup() directly) and any other management command run against a clean container -- not specific to CI. --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index 1fb7c77..2c4af21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,13 @@ RUN apt-get update && \ WORKDIR /code COPY . /code/ +# LOGGING in DataSpace/settings.py writes to logs/dataex.log -- Django's +# logging.config never creates the parent directory itself, so any fresh +# container without this (no pre-existing volume/manual mkdir) fails on +# django.setup() with FileNotFoundError before any command can even run, +# including manage.py check and the healthcheck.sh script below. +RUN mkdir -p /code/logs + RUN pip install psycopg2-binary uvicorn RUN pip install -r requirements.txt RUN curl -s https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js -o /code/echarts.min.js From d061d78341891954a7bcc9b563319f91476d1127 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 1 Sep 2026 23:48:08 +0530 Subject: [PATCH 36/57] Retry the sanity-check image pull, GHCR rate-limits briefly after push Hit docker: toomanyrequests twice in a row on the immediate pull right after a fresh push -- both times with a sub-second retry-after. A bare single attempt is flaky here; wrapped in a short retry loop (5 attempts, 5s apart). --- .github/workflows/deploy-backend.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 3e20df3..058c7ac 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -118,15 +118,25 @@ jobs: - name: Sanity-check the built image # Cheap, real gate: catches import errors and bad settings before - # anything touches the host. + # anything touches the host. Retries: GHCR's toomanyrequests can + # briefly fire right after a push (seen live -- retry-after was + # under 500ms both times), so a bare single attempt is flaky here. run: | - docker run --rm \ - -e SECRET_KEY=ci-sanity-check-not-a-real-key \ - -e URL_WHITELIST=http://localhost \ - -e DB_ENGINE=django.db.backends.sqlite3 \ - --entrypoint python \ - "${{ steps.ref.outputs.image_ref }}" \ - manage.py check + for attempt in 1 2 3 4 5; do + if docker run --rm \ + -e SECRET_KEY=ci-sanity-check-not-a-real-key \ + -e URL_WHITELIST=http://localhost \ + -e DB_ENGINE=django.db.backends.sqlite3 \ + --entrypoint python \ + "${{ steps.ref.outputs.image_ref }}" \ + manage.py check; then + exit 0 + fi + echo "attempt $attempt failed, retrying in 5s..." + sleep 5 + done + echo "::error::Sanity check failed after 5 attempts." + exit 1 deploy: name: Deploy to EC2 From 4531149ac553ff081ed535d0b0aeb8c596d2f78f Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 00:17:55 +0530 Subject: [PATCH 37/57] Retry docker pull in ci-deploy.sh/ci-rollback.sh, same GHCR rate-limit Deploy job hit the identical toomanyrequests error the CI-side sanity-check step already needed a retry loop for, this time on the host's own docker pull inside ci-deploy.sh. Applied the same fix to both places a pull can happen: ci-deploy.sh's main pull, and ci-rollback.sh's pull-if-not-local fallback. --- scripts/ci-deploy.sh | 12 +++++++++++- scripts/ci-rollback.sh | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh index 2118137..d9c266d 100755 --- a/scripts/ci-deploy.sh +++ b/scripts/ci-deploy.sh @@ -59,8 +59,18 @@ esac printf "%s" "$PREV_REF" > .deploy/previous_image # --- pull the new image ---------------------------------------- +# GHCR's toomanyrequests can briefly fire right after a push (seen live, +# retry-after under 100ms each time) -- a bare single pull is flaky here. echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin -docker pull "$IMAGE_REF" +for attempt in 1 2 3 4 5; do + docker pull "$IMAGE_REF" && break + if [ "$attempt" -eq 5 ]; then + echo "::error::docker pull failed after 5 attempts." + exit 1 + fi + echo "pull attempt $attempt failed, retrying in 5s..." + sleep 5 +done printf "DATASPACE_IMAGE=%s\n" "$IMAGE_REF" > .deploy/image.env COMPOSE="docker compose -f docker-compose.yml --env-file .env --env-file .deploy/image.env" diff --git a/scripts/ci-rollback.sh b/scripts/ci-rollback.sh index cdc2963..217682a 100755 --- a/scripts/ci-rollback.sh +++ b/scripts/ci-rollback.sh @@ -22,9 +22,19 @@ fi # finalize-deploy is the only pruner and it did not run, so the # previous image should still be local. Pull is a safety net. +# See ci-deploy.sh for why this retries -- GHCR's toomanyrequests can +# briefly fire right after a push. docker image inspect "$PREV_REF" >/dev/null 2>&1 || { echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin - docker pull "$PREV_REF" + for attempt in 1 2 3 4 5; do + docker pull "$PREV_REF" && break + if [ "$attempt" -eq 5 ]; then + echo "::error::docker pull failed after 5 attempts." + exit 1 + fi + echo "pull attempt $attempt failed, retrying in 5s..." + sleep 5 + done } printf "DATASPACE_IMAGE=%s\n" "$PREV_REF" > .deploy/image.env From ecbfa86fcb8b17725898e6a9db1f87c16e16158a Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 02:05:20 +0530 Subject: [PATCH 38/57] Give GHCR rate-limit retries real runway before next attempt 5 attempts x 5s (~25s total) wasn't remotely enough -- every single attempt failed identically despite each error reporting a sub-second retry-after, meaning this is a sustained account-level quota, not a brief burst. Bumped every retry loop (sanity-check step, ci-deploy.sh, ci-rollback.sh) to 10 attempts x 30s (~5 minutes), and correspondingly bumped the surrounding timeouts so nothing gets killed mid-retry: deploy/rollback-on-smoke-failure job timeout-minutes 15->30, added explicit command_timeout: 20m to their SSH steps (the action's own default is 10m). --- .github/workflows/deploy-backend.yml | 30 +++++++++++++++++++--------- scripts/ci-deploy.sh | 17 +++++++++------- scripts/ci-rollback.sh | 15 +++++++------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 058c7ac..ea15461 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -118,11 +118,14 @@ jobs: - name: Sanity-check the built image # Cheap, real gate: catches import errors and bad settings before - # anything touches the host. Retries: GHCR's toomanyrequests can - # briefly fire right after a push (seen live -- retry-after was - # under 500ms both times), so a bare single attempt is flaky here. + # anything touches the host. Retries: GHCR's toomanyrequests kept + # firing on every one of 5 attempts 5s apart (~25s total) in live + # testing despite each error reporting a sub-second retry-after -- + # that points to a sustained account-level quota, not a brief + # burst, so this is deliberately patient: 10 attempts, 30s apart, + # ~5 minutes of runway before giving up. run: | - for attempt in 1 2 3 4 5; do + for attempt in 1 2 3 4 5 6 7 8 9 10; do if docker run --rm \ -e SECRET_KEY=ci-sanity-check-not-a-real-key \ -e URL_WHITELIST=http://localhost \ @@ -132,10 +135,10 @@ jobs: manage.py check; then exit 0 fi - echo "attempt $attempt failed, retrying in 5s..." - sleep 5 + echo "attempt $attempt failed, retrying in 30s..." + sleep 30 done - echo "::error::Sanity check failed after 5 attempts." + echo "::error::Sanity check failed after 10 attempts." exit 1 deploy: @@ -143,7 +146,10 @@ jobs: needs: build runs-on: ubuntu-latest environment: development - timeout-minutes: 15 + # 30m, not 15: ci-deploy.sh's own pull retry loop alone can now take + # up to ~5 minutes (see its header comment) before even reaching + # migrate/collectstatic/the health-check loop. + timeout-minutes: 30 # packages: read -- GITHUB_TOKEN needs this explicitly granted to pull # from GHCR; it isn't covered by the repo's default token permissions. permissions: @@ -177,6 +183,9 @@ jobs: username: ${{ secrets.EC2_USERNAME }} key: ${{ secrets.EC2_PRIVATE_KEY }} script_stop: true + # 20m, not the action's 10m default: ci-deploy.sh's own pull + # retry loop alone can take up to ~5 minutes. + command_timeout: 20m # sudo: docker/docker compose require it on this host (confirmed # passwordless -- sudo -n succeeds non-interactively). script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-deploy.sh "${{ needs.build.outputs.image_ref }}" "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" @@ -213,7 +222,9 @@ jobs: if: failure() && needs.deploy.result == 'success' runs-on: ubuntu-latest environment: development - timeout-minutes: 15 + # 30m, not 15: ci-rollback.sh's pull-if-not-local fallback can now + # take up to ~5 minutes on its own retry loop. + timeout-minutes: 30 permissions: contents: read packages: read @@ -240,6 +251,7 @@ jobs: username: ${{ secrets.EC2_USERNAME }} key: ${{ secrets.EC2_PRIVATE_KEY }} script_stop: true + command_timeout: 20m script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-rollback.sh "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" - name: Mark this run as failed diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh index d9c266d..f7f2a21 100755 --- a/scripts/ci-deploy.sh +++ b/scripts/ci-deploy.sh @@ -59,17 +59,20 @@ esac printf "%s" "$PREV_REF" > .deploy/previous_image # --- pull the new image ---------------------------------------- -# GHCR's toomanyrequests can briefly fire right after a push (seen live, -# retry-after under 100ms each time) -- a bare single pull is flaky here. +# GHCR's toomanyrequests kept firing on every one of 5 attempts 5s apart +# (~25s total) in live testing despite each error reporting a sub-second +# retry-after -- that points to a sustained account-level quota, not a +# brief burst, so this is deliberately patient: 10 attempts, 30s apart, +# ~5 minutes of runway before giving up. echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin -for attempt in 1 2 3 4 5; do +for attempt in 1 2 3 4 5 6 7 8 9 10; do docker pull "$IMAGE_REF" && break - if [ "$attempt" -eq 5 ]; then - echo "::error::docker pull failed after 5 attempts." + if [ "$attempt" -eq 10 ]; then + echo "::error::docker pull failed after 10 attempts." exit 1 fi - echo "pull attempt $attempt failed, retrying in 5s..." - sleep 5 + echo "pull attempt $attempt failed, retrying in 30s..." + sleep 30 done printf "DATASPACE_IMAGE=%s\n" "$IMAGE_REF" > .deploy/image.env diff --git a/scripts/ci-rollback.sh b/scripts/ci-rollback.sh index 217682a..10dab05 100755 --- a/scripts/ci-rollback.sh +++ b/scripts/ci-rollback.sh @@ -22,18 +22,19 @@ fi # finalize-deploy is the only pruner and it did not run, so the # previous image should still be local. Pull is a safety net. -# See ci-deploy.sh for why this retries -- GHCR's toomanyrequests can -# briefly fire right after a push. +# See ci-deploy.sh for why this retries as patiently as it does -- +# GHCR's toomanyrequests turned out to be a sustained account-level +# quota, not a brief burst, in live testing. docker image inspect "$PREV_REF" >/dev/null 2>&1 || { echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin - for attempt in 1 2 3 4 5; do + for attempt in 1 2 3 4 5 6 7 8 9 10; do docker pull "$PREV_REF" && break - if [ "$attempt" -eq 5 ]; then - echo "::error::docker pull failed after 5 attempts." + if [ "$attempt" -eq 10 ]; then + echo "::error::docker pull failed after 10 attempts." exit 1 fi - echo "pull attempt $attempt failed, retrying in 5s..." - sleep 5 + echo "pull attempt $attempt failed, retrying in 30s..." + sleep 30 done } From ce1d0d3ff6aed0f8990e1d9a6bc52651dfbee6fe Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 02:33:02 +0530 Subject: [PATCH 39/57] Wait 90s before the first sanity-check pull, not just retry after failure Confirmed the failure pattern more closely this time: every data layer downloads successfully on every retry (all report "Download complete" or "Pull complete"), and toomanyrequests fires immediately after the LAST layer finishes -- consistently, even across 10 attempts 30s apart over live testing. That's the final manifest/config fetch failing right when the pull is otherwise done, not a generic account-wide quota exhausting mid-pull. Reads like GHCR hasn't fully settled the just-pushed manifest yet. Added an upfront 90s wait before the first pull attempt, on top of the existing retry loop. --- .github/workflows/deploy-backend.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index ea15461..1a1243b 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -118,13 +118,16 @@ jobs: - name: Sanity-check the built image # Cheap, real gate: catches import errors and bad settings before - # anything touches the host. Retries: GHCR's toomanyrequests kept - # firing on every one of 5 attempts 5s apart (~25s total) in live - # testing despite each error reporting a sub-second retry-after -- - # that points to a sustained account-level quota, not a brief - # burst, so this is deliberately patient: 10 attempts, 30s apart, - # ~5 minutes of runway before giving up. + # anything touches the host. Live testing showed every layer + # downloading successfully every single retry, with toomanyrequests + # firing right after the LAST layer completes (the final + # manifest/config fetch) -- consistently, even across 10 attempts + # 30s apart. That pattern looks like GHCR hasn't finished settling + # a just-pushed manifest yet, not a generic quota, so this waits + # up front before the first attempt rather than only reacting + # after failures. run: | + sleep 90 for attempt in 1 2 3 4 5 6 7 8 9 10; do if docker run --rm \ -e SECRET_KEY=ci-sanity-check-not-a-real-key \ From f4f226f5f8ebbf238acd3bdc13167b07e4323bec Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 09:17:05 +0530 Subject: [PATCH 40/57] Bump deploy/rollback command_timeout to 40m, jobs to 50m The rate limit theory was right (5.5hr idle wait let build+push succeed cleanly this time), but the deploy step hit a different, genuine issue: one large layer (chromium + its X11 libs, almost certainly) took ~15 minutes to download via Docker's own internal per-layer retry, then the SSH session hung with zero further output for several more minutes -- most likely extraction stalling under real memory pressure on this host (confirmed via free -h: ~169Mi truly free, 828Mi already swapped, running Postgres + two separate Elasticsearch instances + Redis + Keycloak + telemetry tooling alongside the app). The previous 20m command_timeout killed the SSH session mid-extraction, right after the layer had already finished downloading. Host itself is untouched and healthy -- the old container never got swapped since the pull never completed in time, and the big layer is now cached locally there, so the next attempt should skip straight past the slow part. --- .github/workflows/deploy-backend.yml | 33 ++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 1a1243b..75848b3 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -149,10 +149,9 @@ jobs: needs: build runs-on: ubuntu-latest environment: development - # 30m, not 15: ci-deploy.sh's own pull retry loop alone can now take - # up to ~5 minutes (see its header comment) before even reaching - # migrate/collectstatic/the health-check loop. - timeout-minutes: 30 + # 50m: must comfortably exceed the Deploy step's own 40m + # command_timeout (see that step's comment for why it's 40m). + timeout-minutes: 50 # packages: read -- GITHUB_TOKEN needs this explicitly granted to pull # from GHCR; it isn't covered by the repo's default token permissions. permissions: @@ -186,9 +185,18 @@ jobs: username: ${{ secrets.EC2_USERNAME }} key: ${{ secrets.EC2_PRIVATE_KEY }} script_stop: true - # 20m, not the action's 10m default: ci-deploy.sh's own pull - # retry loop alone can take up to ~5 minutes. - command_timeout: 20m + # 40m: live testing showed one large layer (chromium + its X11 + # libs, almost certainly) take ~15 minutes just downloading via + # Docker's own internal per-layer retry (not our retry loop -- + # this is a single docker pull invocation struggling), then hang + # with zero output for several more minutes afterward (most + # likely extraction stalling under memory pressure -- this host + # runs Postgres, two separate Elasticsearch instances, Redis, + # Keycloak, and telemetry tooling alongside the app, confirmed + # via `free -h`: ~169Mi truly free, 828Mi already swapped). The + # previous 20m ceiling cut it off mid-extraction, right after + # the layer had already finished downloading. + command_timeout: 40m # sudo: docker/docker compose require it on this host (confirmed # passwordless -- sudo -n succeeds non-interactively). script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-deploy.sh "${{ needs.build.outputs.image_ref }}" "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" @@ -225,9 +233,9 @@ jobs: if: failure() && needs.deploy.result == 'success' runs-on: ubuntu-latest environment: development - # 30m, not 15: ci-rollback.sh's pull-if-not-local fallback can now - # take up to ~5 minutes on its own retry loop. - timeout-minutes: 30 + # 50m: must comfortably exceed the Restore previous image step's own + # 40m command_timeout (see the Deploy job's equivalent step for why). + timeout-minutes: 50 permissions: contents: read packages: read @@ -254,7 +262,10 @@ jobs: username: ${{ secrets.EC2_USERNAME }} key: ${{ secrets.EC2_PRIVATE_KEY }} script_stop: true - command_timeout: 20m + # 40m -- see the Deploy job's equivalent step for why (one large + # layer took ~15 min to download plus several more to extract + # under this host's memory pressure in live testing). + command_timeout: 40m script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-rollback.sh "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" - name: Mark this run as failed From 044f0d8f256845ff23c48417270cf3ff303a7b7f Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 10:20:03 +0530 Subject: [PATCH 41/57] Stop telemetry from gating /health/'s overall status Live deploy hit this directly: health_check required ALL FOUR services (database, elasticsearch, redis, telemetry) to be healthy for a 200, but otel-collector is deliberately not part of this compose topology (this pipeline's docker-compose.yml doesn't run it -- see docker-compose.yml/deploy-backend.yml history for why). Result: a genuinely healthy deploy (db/es/redis all fine, confirmed live) still 503s forever because an optional observability dependency isn't running, which both our deploy pipeline's health-gate and Docker's own compose healthcheck read as "unhealthy" and never recover from. Telemetry status is still reported in the response body for visibility -- it just no longer decides the HTTP status code. --- api/views/health.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/views/health.py b/api/views/health.py index 41491a3..056a079 100644 --- a/api/views/health.py +++ b/api/views/health.py @@ -131,8 +131,14 @@ def health_check(request: HttpRequest) -> JsonResponse: current_span.set_attribute("telemetry.status", "unhealthy") current_span.set_attribute("telemetry.error", str(e)) - # Overall status - overall_status = all(service["status"] == "healthy" for service in status.values()) + # Overall status: database/elasticsearch/redis are required for the app + # to actually serve requests. telemetry is observability-only (this + # deployment topology deliberately runs without otel-collector) and is + # reported above for visibility, but must not gate 200 vs 503 -- a + # health check that fails a deploy because optional tracing + # infrastructure isn't running is a false negative. + required_services = ("database", "elasticsearch", "redis") + overall_status = all(status[name]["status"] == "healthy" for name in required_services) if current_span: current_span.set_attribute( From 1ead789b29e94b0e34363f77ae85c36944526b26 Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 11:26:56 +0530 Subject: [PATCH 42/57] Ensure every search index exists on deploy, not just populated ones Live api-smoke failure traced to a real bug: GET /api/search/unified/ 500'd with elasticsearch.NotFoundError: index_not_found_exception, no such index [publication]. This box's Elasticsearch had never had that index created -- confirmed via `_cat/indices`, it was genuinely missing while the other 6 registered indices existed and worked fine. Root cause: `search_index --create --populate` in one call silently skips creating an index when the model's queryset is empty (this one has 0 Publication rows) -- it printed "Indexing 0 'Publication' objects" as if everything were fine, no error, index just never got created. Manually running --create for that one model in isolation worked immediately. But naively running --create on its own for every model isn't idempotent either -- it hard-errors with resource_already_exists_exception on the first already-existing index and stops, so it can't just be added as a blanket step. Loop over each model's index individually, --create tolerating "already exists" per-model so one existing index never blocks creating the next one that's genuinely missing, then populate everything in one pass at the end. Verified live: manually created the missing index this way and GET /api/search/unified/ returned 200. --- scripts/ci-deploy.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh index f7f2a21..2b6f374 100755 --- a/scripts/ci-deploy.sh +++ b/scripts/ci-deploy.sh @@ -98,6 +98,26 @@ $COMPOSE up -d backend_db elasticsearch redis $RELEASE showmigrations --plan 2>/dev/null | grep "^\[ \]" > .deploy/migrations.txt || true $RELEASE migrate --noinput +# Ensure every search index exists before populating. Neither flag +# combination alone is safe here, confirmed live: +# - `search_index --create --populate` in one call skips already-existing +# indices safely, but ALSO silently skips creating one whose queryset is +# currently empty (index_not_found_exception on every unified-search +# request afterward, despite the command reporting "Indexing 0 'X' +# objects" as if nothing were wrong). +# - `search_index --create` alone is NOT idempotent: it hard-errors with +# resource_already_exists_exception on the first already-existing index +# it hits and stops there, so any index processed after it in the same +# invocation never gets attempted. +# Creating each model's index individually, each tolerating "already +# exists" on its own, gets the correctness of both without either failure +# mode. Keep this list in sync with DataSpace/settings.py's +# ELASTICSEARCH_INDEX_NAMES if a new search document is ever added. +for model in api.Dataset api.UseCase api.AIModel api.Publication api.Collaborative api.Organization authorization.User; do + $RELEASE search_index --create --models "$model" -f || true +done +$RELEASE search_index --populate -f + # --- swap the running container --------------------------------- # --no-deps so backend_db/elasticsearch/redis are never recreated # out from under this. From 8fd1e561538c3a4c22b9d00684acb73bc92f0efc Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 12:58:11 +0530 Subject: [PATCH 43/57] Skip the telemetry probe entirely when TELEMETRY_URL is unset settings.py gives TELEMETRY_URL a hardcoded otel-collector default even when the env var is unset, so a deployment that deliberately runs no collector (as this one does) still probed a host that cannot resolve and logged an ERROR on every health check -- roughly every 30s from the compose healthcheck alone, forever, for something optional. Now reports "not_configured" without probing. Does not touch the required_services gate, so this still cannot affect the status code either way. --- api/views/health.py | 70 ++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/api/views/health.py b/api/views/health.py index 056a079..a9684b3 100644 --- a/api/views/health.py +++ b/api/views/health.py @@ -100,36 +100,52 @@ def health_check(request: HttpRequest) -> JsonResponse: current_span.set_attribute("redis.status", "unhealthy") current_span.set_attribute("redis.error", str(e)) - # Check OpenTelemetry collector - try: - # Extract host and port from TELEMETRY_URL - telemetry_url = settings.TELEMETRY_URL.replace("http://", "").replace( - "https://", "" - ) - host = telemetry_url.split(":")[0] - # Use default health check port 13133 instead of gRPC port - health_url = f"http://{host}:13133/health" # OpenTelemetry collector health check endpoint - - response = requests.get(health_url, timeout=5) - if response.status_code == 200: - status["telemetry"] = { - "status": "healthy", - "message": "Successfully connected to OpenTelemetry collector", - } - if current_span: - current_span.set_attribute("telemetry.status", "healthy") - else: - raise Exception(f"Health check returned status code {response.status_code}") - - except Exception as e: - logger.error("Telemetry health check failed", error=str(e)) + # Check OpenTelemetry collector, but only when telemetry is actually + # configured. DataSpace/settings.py gives TELEMETRY_URL a hardcoded + # otel-collector default even when the env var is unset, so without this + # guard a deployment that deliberately runs no collector (as this one + # does) probes a host that cannot resolve and logs an ERROR on every + # single health check -- roughly every 30s, forever, for something + # optional. + if not os.environ.get("TELEMETRY_URL"): status["telemetry"] = { - "status": "unhealthy", - "message": f"Failed to connect to OpenTelemetry collector: {str(e)}", + "status": "not_configured", + "message": "TELEMETRY_URL is unset; no OpenTelemetry collector expected", } if current_span: - current_span.set_attribute("telemetry.status", "unhealthy") - current_span.set_attribute("telemetry.error", str(e)) + current_span.set_attribute("telemetry.status", "not_configured") + else: + try: + # Extract host and port from TELEMETRY_URL + telemetry_url = settings.TELEMETRY_URL.replace("http://", "").replace( + "https://", "" + ) + host = telemetry_url.split(":")[0] + # Use default health check port 13133 instead of gRPC port + health_url = f"http://{host}:13133/health" # OpenTelemetry collector health check endpoint + + response = requests.get(health_url, timeout=5) + if response.status_code == 200: + status["telemetry"] = { + "status": "healthy", + "message": "Successfully connected to OpenTelemetry collector", + } + if current_span: + current_span.set_attribute("telemetry.status", "healthy") + else: + raise Exception( + f"Health check returned status code {response.status_code}" + ) + + except Exception as e: + logger.error("Telemetry health check failed", error=str(e)) + status["telemetry"] = { + "status": "unhealthy", + "message": f"Failed to connect to OpenTelemetry collector: {str(e)}", + } + if current_span: + current_span.set_attribute("telemetry.status", "unhealthy") + current_span.set_attribute("telemetry.error", str(e)) # Overall status: database/elasticsearch/redis are required for the app # to actually serve requests. telemetry is observability-only (this From 091dec909dba4736d518ca9bcdd6b96334c1b20b Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 13:05:50 +0530 Subject: [PATCH 44/57] Mount uploaded media into the container -- image cutover dropped it Real regression from the image-based deploy. The old compose bind-mounted the whole working tree (`.:/code`), which incidentally gave the app access to user-uploaded media on the host. Removing that mount (correctly -- it would shadow the deployed image) left nothing serving MEDIA_ROOT, so /code/files did not exist in the container at all: 2246 files on the host, 0 visible to the app. Every file-field lookup then raised FileNotFoundError -- e.g. an organization logo's `size` -- which errored the GraphQL queries the dashboard depends on and left it blank. The doubled-looking path in those errors (/code/files/public/files/public/organizations/...) is not a bug; that nested layout genuinely exists on the host, since MEDIA_ROOT is BASE_DIR/files/public and the stored names start with files/public/. Mounts only ./files:/code/files -- persistent data, never the whole tree. Verified live: 2246 files now visible in the container and zero FileNotFoundError after exercising the publishers page. --- docker-compose.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2b194cd..3de7bc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,16 @@ services: build: . env_file: .env container_name: "DataSpace" + volumes: + # User-uploaded media (MEDIA_ROOT = BASE_DIR/files/public). This is + # persistent data, not code -- it lives on the host and cannot be + # baked into the image. Dropping the old `.:/code` bind mount for the + # image-based deploy removed the app's only access to it, so every + # file field lookup (e.g. an organization logo's `size`) raised + # FileNotFoundError and errored the GraphQL queries the dashboard + # depends on. Mount only this subtree, never the whole tree -- a + # full `.:/code` mount would shadow the deployed image. + - ./files:/code/files ports: - "8000:8000" depends_on: @@ -34,6 +44,10 @@ services: build: . env_file: .env profiles: ["release"] + volumes: + # Same media mount as `backend` -- management commands run here too + # and some of them touch uploaded files. + - ./files:/code/files depends_on: backend_db: condition: service_healthy From 2f6d961719074b9d632f8dda378943a4a0bafa8a Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 16:37:37 +0530 Subject: [PATCH 45/57] fix(sdk): stop hardcoding /auth in Keycloak URLs AuthClient built every Keycloak URL as `{keycloak_url}/auth/realms/{realm}/...`, so the SDK could only ever talk to a Keycloak served under /auth. Against a Keycloak served at the domain root every token request 404s, which is not recoverable through configuration -- there is no value of keycloak_url that avoids it. Adds `keycloak_base_path`, defaulting to "/auth" so existing callers are byte-for-byte unaffected. Pass "" for a root-hosted Keycloak. The path is also skipped when keycloak_url already ends with it, so callers passing a full base URL do not get it emitted twice. Verified against both live servers: the composed URLs return 401 (the endpoint exists, credentials rejected) while the old composition returns 404 on a root-path server. --- dataspace_sdk/auth.py | 38 ++++++++++++++++++++++++++------------ tests/test_auth.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/dataspace_sdk/auth.py b/dataspace_sdk/auth.py index 4ecac6e..98078d6 100644 --- a/dataspace_sdk/auth.py +++ b/dataspace_sdk/auth.py @@ -18,6 +18,7 @@ def __init__( keycloak_realm: Optional[str] = None, keycloak_client_id: Optional[str] = None, keycloak_client_secret: Optional[str] = None, + keycloak_base_path: str = "/auth", ): """ Initialize the authentication client. @@ -28,9 +29,15 @@ def __init__( keycloak_realm: Keycloak realm name (e.g., "DataSpace") keycloak_client_id: Keycloak client ID (e.g., "dataspace") keycloak_client_secret: Optional client secret for confidential clients + keycloak_base_path: Keycloak's HTTP relative path. Defaults to + "/auth", which is what servers configured the legacy way use. + Pass "" for a Keycloak served at the domain root. """ self.base_url = base_url.rstrip("/") self.keycloak_url = keycloak_url.rstrip("/") if keycloak_url else None + self.keycloak_base_path = ( + "/" + keycloak_base_path.strip("/") if keycloak_base_path.strip("/") else "" + ) self.keycloak_realm = keycloak_realm self.keycloak_client_id = keycloak_client_id self.keycloak_client_secret = keycloak_client_secret @@ -47,6 +54,22 @@ def __init__( self._username: Optional[str] = None self._password: Optional[str] = None + def _realm_url(self) -> str: + """Base URL for this realm's endpoints. + + Keycloak's relative path is deployment-specific: "/auth" on servers + configured the legacy way, empty on servers served at the domain root. + This used to be hardcoded, which made the SDK unable to reach a + root-path Keycloak at all. + """ + path = self.keycloak_base_path + base = self.keycloak_url or "" + # Tolerate the relative path already being part of keycloak_url, rather + # than emitting it twice. + if path and base.endswith(path): + path = "" + return f"{base}{path}/realms/{self.keycloak_realm}" + def login(self, username: str, password: str) -> Dict[str, Any]: """ Login using username and password via Keycloak. @@ -124,10 +147,7 @@ def _get_keycloak_token(self, username: str, password: str) -> str: Raises: DataSpaceAuthError: If authentication fails """ - token_url = ( - f"{self.keycloak_url}/auth/realms/{self.keycloak_realm}/" - f"protocol/openid-connect/token" - ) + token_url = f"{self._realm_url()}/protocol/openid-connect/token" data = { "grant_type": "password", @@ -183,10 +203,7 @@ def _get_service_account_token(self) -> str: Raises: DataSpaceAuthError: If authentication fails """ - token_url = ( - f"{self.keycloak_url}/auth/realms/{self.keycloak_realm}/" - f"protocol/openid-connect/token" - ) + token_url = f"{self._realm_url()}/protocol/openid-connect/token" data = { "grant_type": "client_credentials", @@ -247,10 +264,7 @@ def _refresh_keycloak_token(self) -> str: return self._get_keycloak_token(self._username, self._password) raise DataSpaceAuthError("No refresh token or credentials available") - token_url = ( - f"{self.keycloak_url}/auth/realms/{self.keycloak_realm}/" - f"protocol/openid-connect/token" - ) + token_url = f"{self._realm_url()}/protocol/openid-connect/token" data = { "grant_type": "refresh_token", diff --git a/tests/test_auth.py b/tests/test_auth.py index cf0212c..2984ff3 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -187,5 +187,44 @@ def test_login_as_service_account_no_secret(self) -> None: self.auth_client.login_as_service_account() +class TestRealmUrl(unittest.TestCase): + """Keycloak's relative path is deployment-specific, not always /auth.""" + + def _url(self, **kwargs: object) -> str: + return AuthClient( + base_url="https://api.test.com", keycloak_realm="DataSpace", **kwargs + )._realm_url() + + def test_defaults_to_auth_for_backwards_compatibility(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com"), + "https://kc.test.com/auth/realms/DataSpace", + ) + + def test_empty_base_path_for_root_hosted_keycloak(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com", keycloak_base_path=""), + "https://kc.test.com/realms/DataSpace", + ) + + def test_does_not_double_path_already_in_url(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com/auth"), + "https://kc.test.com/auth/realms/DataSpace", + ) + + def test_custom_relative_path(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com", keycloak_base_path="sso"), + "https://kc.test.com/sso/realms/DataSpace", + ) + + def test_slash_only_base_path_is_treated_as_root(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com/", keycloak_base_path="/"), + "https://kc.test.com/realms/DataSpace", + ) + + if __name__ == "__main__": unittest.main() From c7ca8204c5fa7ef99593cf4d0a97beb0f9004426 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 2 Sep 2026 11:17:47 +0000 Subject: [PATCH 46/57] Bump SDK version to 0.5.04 --- dataspace_sdk/__version__.py | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dataspace_sdk/__version__.py b/dataspace_sdk/__version__.py index a245f6e..87255ce 100644 --- a/dataspace_sdk/__version__.py +++ b/dataspace_sdk/__version__.py @@ -1,3 +1,3 @@ """Version information for DataSpace SDK.""" -__version__ = "0.5.03" +__version__ = "0.5.04" diff --git a/pyproject.toml b/pyproject.toml index 9c7f9f1..15fe5ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataspace-sdk" -version = "0.5.03" +version = "0.5.04" description = "Python SDK for DataSpace API" readme = "docs/sdk/README.md" requires-python = ">=3.8" @@ -54,7 +54,7 @@ include = '\.pyi?$' [tool.mypy] -python_version = "0.5.03" +python_version = "0.5.04" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = false From 31815765d1e0f39c0655aaffb03f372b189361fd Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 2 Sep 2026 16:50:13 +0530 Subject: [PATCH 47/57] fix(sdk): DataSpaceClient did not forward keycloak_base_path to AuthClient #129 added keycloak_base_path to AuthClient, but DataSpaceClient -- the class every consumer actually imports -- never passed it through. No caller of the public API could set it: DataSpaceClient(..., keycloak_base_path="") silently behaved identically to omitting it, because AuthClient just used its own default. Found while attempting the ParakhAI dev flip -- the fix in #129 was real but incomplete. Adds a test asserting DataSpaceClient actually forwards the value, not just that AuthClient accepts it, so this class of gap fails a test next time. --- dataspace_sdk/client.py | 5 +++++ tests/test_client.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/dataspace_sdk/client.py b/dataspace_sdk/client.py index e612476..d139468 100644 --- a/dataspace_sdk/client.py +++ b/dataspace_sdk/client.py @@ -43,6 +43,7 @@ def __init__( keycloak_realm: Optional[str] = None, keycloak_client_id: Optional[str] = None, keycloak_client_secret: Optional[str] = None, + keycloak_base_path: str = "/auth", ): """ Initialize the DataSpace client. @@ -53,6 +54,9 @@ def __init__( keycloak_realm: Keycloak realm name (e.g., "DataSpace") keycloak_client_id: Keycloak client ID (e.g., "dataspace") keycloak_client_secret: Optional client secret for confidential clients + keycloak_base_path: Keycloak's HTTP relative path. Defaults to + "/auth" for backwards compatibility. Pass "" for a Keycloak + served at the domain root. """ self.base_url = base_url.rstrip("/") self._auth = AuthClient( @@ -61,6 +65,7 @@ def __init__( keycloak_realm=keycloak_realm, keycloak_client_id=keycloak_client_id, keycloak_client_secret=keycloak_client_secret, + keycloak_base_path=keycloak_base_path, ) # Initialize resource clients diff --git a/tests/test_client.py b/tests/test_client.py index 44d4c6b..6a39b3b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -91,5 +91,26 @@ def test_user_property(self) -> None: self.assertIsNone(self.client.user) +class TestClientForwardsBasePath(unittest.TestCase): + """The regression this covers: AuthClient grew keycloak_base_path but + DataSpaceClient did not forward it, so no public consumer could ever set + it -- constructing DataSpaceClient(..., keycloak_base_path="") silently + behaved the same as not passing it at all. + """ + + def test_default_matches_auth_client_default(self) -> None: + client = DataSpaceClient(base_url="https://api.test.com", keycloak_url="https://kc.test.com", keycloak_realm="DataSpace") + self.assertEqual(client._auth._realm_url(), "https://kc.test.com/auth/realms/DataSpace") + + def test_root_path_is_forwarded(self) -> None: + client = DataSpaceClient( + base_url="https://api.test.com", + keycloak_url="https://kc.test.com", + keycloak_realm="DataSpace", + keycloak_base_path="", + ) + self.assertEqual(client._auth._realm_url(), "https://kc.test.com/realms/DataSpace") + + if __name__ == "__main__": unittest.main() From 50ac6b09e6cbae502ec8a28789c33b4389fcfcbc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 2 Sep 2026 11:33:23 +0000 Subject: [PATCH 48/57] Bump SDK version to 0.5.05 --- dataspace_sdk/__version__.py | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dataspace_sdk/__version__.py b/dataspace_sdk/__version__.py index 87255ce..7008ea2 100644 --- a/dataspace_sdk/__version__.py +++ b/dataspace_sdk/__version__.py @@ -1,3 +1,3 @@ """Version information for DataSpace SDK.""" -__version__ = "0.5.04" +__version__ = "0.5.05" diff --git a/pyproject.toml b/pyproject.toml index 15fe5ff..0a8ce32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataspace-sdk" -version = "0.5.04" +version = "0.5.05" description = "Python SDK for DataSpace API" readme = "docs/sdk/README.md" requires-python = ">=3.8" @@ -54,7 +54,7 @@ include = '\.pyi?$' [tool.mypy] -python_version = "0.5.04" +python_version = "0.5.05" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = false From 986828bff875c8c03904c6118d6ff722631cb9c2 Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 00:51:33 +0530 Subject: [PATCH 49/57] ci: pass KEYCLOAK_CLIENT_SECRET to the smoke workflow run-smoke.yml now preflights its Keycloak configuration instead of letting the authenticated API tests skip silently and the run go green on nothing. It needs the client secret to do that. `dataspace` is a confidential client, so the ROPC token request returns 401 unauthorized_client without it. The reusable workflow declares the secret optional so this repo kept parsing before this change, but api-smoke fails its preflight until the secret is passed and set. --- .github/workflows/deploy-backend.yml | 4 ++++ .github/workflows/deploy-to-ecs.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 75848b3..17907a0 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -222,6 +222,10 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a + # confidential client, so without this the token request returns 401 + # and the job fails its preflight. + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} rollback-on-smoke-failure: name: Rollback (smoke tests failed) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index b18dcc1..f3573b9 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -158,6 +158,10 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a + # confidential client, so without this the token request returns 401 + # and the job fails its preflight. + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} rollback-on-smoke-failure: name: Rollback on Smoke Failure From 8a72b2fdf38c5e7505d7bff71c7bd19f324092ad Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 08:25:41 +0530 Subject: [PATCH 50/57] fix: serve with multiple uvicorn workers to stop DB exhaustion Django runs sync views under ASGI via sync_to_async(thread_sensitive=True), which executes them on one shared thread per process. With a single uvicorn worker that means exactly one sync request is processed at a time, however many arrive. Measured on dev: 12 concurrent POSTs to /api/auth/keycloak/login/ returned at 4s, 7s, 10s, 14s, 17s, 20s, 23s, 26s, 29s, 32s and 36s - near-perfect ~3s increments, each queued behind the last. That queue is what exhausted Postgres. Every in-flight request holds a connection while it waits its turn: 20 concurrent calls took the connection count from 6 to 26, one per request. Deep enough queues reached max_connections (100) and Postgres began refusing with "FATAL: sorry, too many clients already". Requests that waited past nginx's 60s proxy timeout became 504s, refused ones became 500s, and the deploy pipeline failed too because manage.py could not get a connection either. Workers are processes, so N workers give N concurrent sync requests and the queue drains N times faster. The work is I/O-bound on Keycloak, so this helps well past the box's 2 CPUs. --limit-concurrency is the backstop: in-flight requests are capped at workers x limit (60), which stays under max_connections with headroom for other clients. Excess requests get a fast 503 rather than queueing until the database runs out of slots. Shedding load is recoverable; exhausting connections takes the deploy pipeline down with it. Both are env-tunable so a box can be sized without a code change. --- Dockerfile | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2c4af21..b0078ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,4 +73,35 @@ EXPOSE 8000 RUN chmod +x /code/docker-entrypoint.sh ENTRYPOINT ["bash","/code/docker-entrypoint.sh"] -CMD ["uvicorn", "DataSpace.asgi:application", "--host", "0.0.0.0", "--port", "8000"] + +# Served with multiple workers, which is what keeps this from exhausting +# Postgres. +# +# Django runs sync views under ASGI via sync_to_async(thread_sensitive=True), +# which executes them on ONE shared thread per process. With a single worker +# that means exactly one sync request is processed at a time, no matter how +# many arrive. Measured on dev before this change: 12 concurrent calls to +# /api/auth/keycloak/login/ returned in 4s, 7s, 10s, 14s ... 36s - near-perfect +# ~3s increments, queued behind each other. +# +# That queue is what killed the database. Every in-flight request holds a +# connection while it waits its turn - measured at one connection per request, +# so 20 concurrent calls took the connection count from 6 to 26. Deep enough +# queues reached max_connections (100) and Postgres started refusing with +# "FATAL: sorry, too many clients already", which surfaced as 500s, while +# requests that waited past nginx's 60s proxy timeout surfaced as 504s. It +# also broke deploys, because manage.py could not get a connection either. +# +# Workers are processes, so N workers give N concurrent sync requests and the +# queue drains N times faster. The work here is I/O-bound (waiting on +# Keycloak), so this helps well beyond the 2 CPUs on the dev box. +# +# UVICORN_LIMIT_CONCURRENCY is the backstop: total in-flight requests are +# capped at workers x limit, which must stay under Postgres max_connections +# minus headroom for other clients. Excess requests get a fast 503 instead of +# queueing until the database runs out of slots - shedding load is recoverable, +# exhausting connections takes the deploy pipeline down with it. +ENV UVICORN_WORKERS=4 \ + UVICORN_LIMIT_CONCURRENCY=15 + +CMD ["sh", "-c", "exec uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --workers ${UVICORN_WORKERS} --limit-concurrency ${UVICORN_LIMIT_CONCURRENCY}"] From 384c0a9e22db129a3f4d87c190d9d6116e1d3c7e Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 08:25:41 +0530 Subject: [PATCH 51/57] perf: stop making three Keycloak round-trips per login KeycloakLoginView introspected the token twice per request - once inside validate_token and again for roles and organizations - and validate_token then called userinfo as well. On this deployment the client lacks the scope for userinfo, so that call returns 403 every time and falls through to a branch that rebuilds the same fields from the introspection response it already had. A guaranteed failing network call on every login, roughly a third of the request's latency. validate_token now accepts an introspection the caller already has, and skips userinfo when introspection already carries sub plus an identifier. The userinfo path remains for deployments where introspection is sparse, so behaviour is unchanged where it was actually doing work. This matters beyond latency: the request holds a database connection for its whole duration, so every round-trip removed is connection-hold time removed, which is what ran the pool dry. --- api/utils/keycloak_utils.py | 39 ++++++++++++++++++++++++++++++++----- api/views/auth.py | 11 +++++++---- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/api/utils/keycloak_utils.py b/api/utils/keycloak_utils.py index f6937d7..d6f4f46 100644 --- a/api/utils/keycloak_utils.py +++ b/api/utils/keycloak_utils.py @@ -54,12 +54,19 @@ def get_token(self, username: str, password: str) -> Dict[str, Any]: logger.error(f"Error getting token: {e}") raise - def validate_token(self, token: str) -> Dict[str, Any]: + def validate_token( + self, token: str, token_info: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: """ Validate a token (Django JWT or Keycloak) and return the user info. Args: token: The token to validate + token_info: An introspection response for this token, if the caller + already has one. Passing it avoids a second introspect call to + Keycloak - callers that need the introspection data themselves + (for roles and organizations) would otherwise cause two + identical network round-trips per request. Returns: Dict containing the user information @@ -107,14 +114,36 @@ def validate_token(self, token: str) -> Dict[str, Any]: # If Django JWT validation failed, try Keycloak token validation try: - # Verify the token is valid - token_info = self.keycloak_openid.introspect(token) + # Verify the token is valid. Reuses the caller's introspection when + # one was supplied, rather than repeating the round-trip. + if token_info is None: + token_info = self.keycloak_openid.introspect(token) if not token_info.get("active", False): logger.warning("Token is not active") return {} - # Try to get user info from the userinfo endpoint - # If that fails (403), fall back to token introspection data + # Introspection often already carries everything needed. Calling + # userinfo anyway costs a round-trip that, on deployments where the + # client lacks the scope for it, is guaranteed to fail with 403 and + # fall through to exactly the same data - measured as roughly a + # third of this request's latency on dev. + if token_info.get("sub") and ( + token_info.get("email") or token_info.get("preferred_username") + ): + user_info = { + "sub": token_info.get("sub"), + "preferred_username": token_info.get("username") + or token_info.get("preferred_username"), + "email": token_info.get("email"), + "email_verified": token_info.get("email_verified", False), + "name": token_info.get("name"), + "given_name": token_info.get("given_name"), + "family_name": token_info.get("family_name"), + } + return {k: v for k, v in user_info.items() if v is not None} + + # Otherwise ask userinfo, falling back to introspection data if it + # is not available to this client. try: user_info = self.keycloak_openid.userinfo(token) if isinstance(user_info, bytes): diff --git a/api/views/auth.py b/api/views/auth.py index 94d98dc..98b1029 100644 --- a/api/views/auth.py +++ b/api/views/auth.py @@ -24,17 +24,20 @@ def post(self, request: Request) -> Response: status=status.HTTP_400_BAD_REQUEST, ) + # Introspect once and reuse it. This used to introspect twice per + # request - once inside validate_token and again here for roles and + # organizations - which is a wasted round-trip to Keycloak on every + # login while a database connection is held open. + token_info = keycloak_manager.keycloak_openid.introspect(keycloak_token) + # Validate the token and get user info - user_info = keycloak_manager.validate_token(keycloak_token) + user_info = keycloak_manager.validate_token(keycloak_token, token_info=token_info) if not user_info: return Response( {"error": "Invalid or expired token"}, status=status.HTTP_401_UNAUTHORIZED, ) - # Get token introspection data for roles and organizations - token_info = keycloak_manager.keycloak_openid.introspect(keycloak_token) - # Get user roles and organizations from the token introspection data roles = keycloak_manager.get_user_roles_from_token_info(token_info) organizations = keycloak_manager.get_user_organizations_from_token_info(token_info) From 826c83187071e7d289b936f127c53a2940f73b62 Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 08:25:41 +0530 Subject: [PATCH 52/57] fix: make the health check detect connection exhaustion /health/ returned {"database": "healthy"} in 0.44s while Postgres was refusing new connections with "FATAL: sorry, too many clients already", the login endpoint was failing, and the deploy pipeline could not run manage.py. The check ran SELECT 1 on the request's own connection. That connection is already established, so it keeps answering however saturated the server is. The endpoint could not fail for the condition that was actually taking the service down - so a green health check was worse than no health check, and anything gating a deploy or paging on it stayed green throughout. Now also opens a fresh connection and closes it immediately. The two failure modes are independent, and it is the second that catches exhaustion. --- api/views/health.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/api/views/health.py b/api/views/health.py index a9684b3..9f1d0d8 100644 --- a/api/views/health.py +++ b/api/views/health.py @@ -5,7 +5,7 @@ import structlog from django.conf import settings from django.core.cache import cache -from django.db import connection +from django.db import connection, connections from django.http import HttpRequest, JsonResponse from elasticsearch import Elasticsearch from opentelemetry import trace @@ -32,16 +32,38 @@ def health_check(request: HttpRequest) -> JsonResponse: "telemetry": {"status": "unknown"}, } - # Check database + # Check database. + # + # Two distinct checks, because they fail independently: + # + # 1. The request's own connection still works. + # 2. A NEW connection can still be opened. + # + # Only checking (1) is how this endpoint reported + # {"database": "healthy"} in 0.44s while Postgres was refusing new + # connections with "FATAL: sorry, too many clients already" and both the + # login endpoint and the deploy pipeline were failing on exactly that. The + # existing connection is already established, so it keeps answering + # SELECT 1 no matter how saturated the server is - which made a green + # health check actively misleading during an outage. try: with connection.cursor() as cursor: cursor.execute("SELECT 1") - status["database"] = { - "status": "healthy", - "message": "Successfully connected to database", - } - if current_span: - current_span.set_attribute("database.status", "healthy") + + # Deliberately a fresh connection, closed immediately. This is the + # check that catches connection exhaustion. + new_connection = connections.create_connection("default") + try: + new_connection.ensure_connection() + finally: + new_connection.close() + + status["database"] = { + "status": "healthy", + "message": "Successfully connected to database", + } + if current_span: + current_span.set_attribute("database.status", "healthy") except Exception as e: logger.error("Database health check failed", error=str(e)) status["database"] = { From d7fd13538920caa1bbb5fd0fe937edd3058858c1 Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 08:26:58 +0530 Subject: [PATCH 53/57] fix: size uvicorn workers to the memory actually available Measured before setting a number rather than reaching for (2 x CPU) + 1. The container uses 740MB resident with one worker and the host has about 2.4GB available, so four workers risked an OOM - a worse outage than the connection exhaustion this is fixing. Two workers land near 1.3GB and match the 2 CPUs. In-flight requests then cap at 30, comfortably under max_connections (100) with room for other clients. Two workers alone would only double throughput, but removing two of the three Keycloak round-trips per login cuts per-request time as well, and the two compound. --- Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b0078ea..b98c980 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,7 +101,16 @@ ENTRYPOINT ["bash","/code/docker-entrypoint.sh"] # minus headroom for other clients. Excess requests get a fast 503 instead of # queueing until the database runs out of slots - shedding load is recoverable, # exhausting connections takes the deploy pipeline down with it. -ENV UVICORN_WORKERS=4 \ +# Sized to the dev box, not to a formula. One worker measured at 740MB +# resident with only ~2.4GB available on the host, so 4 workers risked an OOM +# that would have been a worse outage than the one this fixes. Two workers land +# near 1.3GB and match the 2 CPUs. +# +# Two workers alone would only double throughput, but the commit that removes +# two of the three Keycloak round-trips cuts per-request time as well, and the +# two compound. Raise UVICORN_WORKERS on a bigger box - it is env-tunable for +# exactly that reason, and worth revisiting if memory there grows. +ENV UVICORN_WORKERS=2 \ UVICORN_LIMIT_CONCURRENCY=15 CMD ["sh", "-c", "exec uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --workers ${UVICORN_WORKERS} --limit-concurrency ${UVICORN_LIMIT_CONCURRENCY}"] From 8fda535e2b8284a7ae248130b95d3bddf4271ccc Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 09:34:44 +0530 Subject: [PATCH 54/57] fix: stop rewriting the user row on every login This is the actual cause of the connection exhaustion. My earlier fix (more uvicorn workers) addressed a real constraint but not this one, and concurrency did not improve as a result - the bottleneck is a row lock in Postgres, which no number of application workers can help. Captured from pg_stat_activity during a burst of concurrent logins: 36 active | Lock | tuple 31 active | Lock | transactionid 16 idle in transaction | Client| ClientRead 67 UPDATE "ds_user" SET "password" = ..., "last_login" = ... Every login called user.save() unconditionally, rewriting every column of the same row. Concurrent logins for one user therefore queued on that row's lock, and each waiting request held a database connection while it waited - which is what walked the connection count up to max_connections and produced "sorry, too many clients already", the 504s, and the failed deploy. Two changes: - sync_user_from_keycloak compares against the stored values and saves only when a field actually changed, with update_fields to keep the UPDATE narrow. A repeat login of an unchanged user now performs no write at all, so there is no lock to contend on. - validate_token no longer calls user.save() on the Django-JWT path. It rewrote an unchanged row on every authenticated request for no benefit. New-user creation is untouched: that INSERT is genuine work. --- api/utils/keycloak_utils.py | 42 ++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/api/utils/keycloak_utils.py b/api/utils/keycloak_utils.py index d6f4f46..d1a6ffc 100644 --- a/api/utils/keycloak_utils.py +++ b/api/utils/keycloak_utils.py @@ -86,7 +86,11 @@ def validate_token( from authorization.models import User user = User.objects.get(id=user_id) - user.save() + # Deliberately no user.save() here. It rewrote every + # column of an unchanged row on every authenticated + # request, taking a row lock for no benefit - and since + # concurrent requests for one user contend on that single + # row, it serialized them against each other. # NOTE: Organizations are managed in DataSpace database, not Keycloak # Organization memberships should be created/managed through DataSpace's @@ -412,13 +416,35 @@ def sync_user_from_keycloak( ) if user: - # Update existing user - user.keycloak_id = keycloak_id - user.username = username - user.email = email - user.first_name = user_info.get("given_name", "") or user.first_name - user.last_name = user_info.get("family_name", "") or user.last_name - user.is_active = True + # Update existing user, but only write when something actually + # changed. The unconditional save this replaces was the cause + # of the connection exhaustion: every login rewrote the same + # row, so concurrent logins for one user queued on a row lock + # (pg_stat_activity showed "Lock: tuple" and + # "Lock: transactionid" on UPDATE "ds_user"), each holding a + # database connection while it waited. + desired = { + "keycloak_id": keycloak_id, + "username": username, + "email": email, + "first_name": user_info.get("given_name", "") or user.first_name, + "last_name": user_info.get("family_name", "") or user.last_name, + "is_active": True, + "is_staff": "admin" in roles, + "is_superuser": "admin" in roles, + } + changed = [ + field + for field, value in desired.items() + if getattr(user, field) != value + ] + if changed: + for field in changed: + setattr(user, field, desired[field]) + # update_fields keeps the UPDATE narrow instead of + # rewriting every column. + user.save(update_fields=changed) + return user else: # Create new user user = User( From 299cf7843b424080baf394ab40355f4af2c90c39 Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 11:33:17 +0530 Subject: [PATCH 55/57] fix: install CPU-only torch so the image can actually be deployed The image is 14.1GB and `docker pull` now runs past the deploy step's 40 minute command_timeout, so deploys fail outright. That is what stopped the row-lock fix in #136 from reaching dev - it merged green and then could not be deployed. Where the size comes from, measured in the running container: 12.1GB RUN pip install -r requirements.txt 4.3GB site-packages/nvidia 1.7GB site-packages/torch 592MB site-packages/triton That is roughly 6.6GB of CUDA runtime on a 2-CPU EC2 instance with no GPU, which cannot execute any of it. Installing CPU-only torch first means the pinned torch==2.9.0 is already satisfied and pip never reaches for the CUDA build. PEP 440 treats the local version segment as compatible, so 2.9.0+cpu satisfies ==2.9.0 and requirements.txt needs no change. The wheel was confirmed to exist for this exact version and platform (torch-2.9.0+cpu-cp310-cp310-manylinux_2_28_x86_64.whl), so this changes the build of torch, not the version. --no-cache-dir on both installs drops the pip wheel cache from the layer. This is a prerequisite now rather than an optimisation: nothing else can deploy until the pull fits inside the timeout. --- Dockerfile | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b98c980..56dac14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,24 @@ COPY . /code/ RUN mkdir -p /code/logs RUN pip install psycopg2-binary uvicorn -RUN pip install -r requirements.txt +# Install CPU-only torch first, so the pinned torch==2.9.0 in +# requirements.txt is already satisfied and pip never reaches for the default +# CUDA build. +# +# The CUDA wheels pull in 4.3GB of nvidia/* libraries, 1.7GB of torch and +# 592MB of triton - about 6.6GB of GPU runtime on a 2-CPU EC2 box that has no +# GPU and physically cannot use any of it. That is most of why the image is +# 14.1GB, why a deploy takes about an hour, and why the deploy of #136 failed +# outright: `docker pull` ran past the SSH step's 40 minute command_timeout. +# +# PEP 440 treats the local version segment as compatible, so 2.9.0+cpu +# satisfies ==2.9.0 and requirements.txt needs no change. Pinned to the same +# version deliberately - this changes the build of torch, not the version. +RUN pip install --no-cache-dir torch==2.9.0 \ + --index-url https://download.pytorch.org/whl/cpu + +# --no-cache-dir: the wheel cache is dead weight in the final layer. +RUN pip install --no-cache-dir -r requirements.txt RUN curl -s https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js -o /code/echarts.min.js # Create healthcheck script From 340fa43fafe7b29df1ea9e5eb9ef43157e368f8c Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 11:49:40 +0530 Subject: [PATCH 56/57] ci: fail the build if the image exceeds a size ceiling Image size is a deploy-time failure mode here and nothing surfaced it. The image reached 14.1GB - 6.6GB of it CUDA runtime on a GPU-less box - and docker pull then ran past the deploy step's 40 minute command_timeout, so deploys failed with "Run Command Timeout" and no indication of the cause. Builds stayed green throughout; the cost only appeared on the host, an hour later, on an environment that could no longer be deployed to. Fails the build instead, so the feedback lands on the PR that caused it. The ceiling is 8GB against a current 3.56GB - set from the measured size after the CPU-only torch fix, with room for ordinary growth but not for another multi-gigabyte dependency arriving unnoticed. The error message names the usual culprits so whoever hits it knows where to look. --- .github/workflows/deploy-backend.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 17907a0..bacde4f 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -116,6 +116,33 @@ jobs: run: | echo "image_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" >> "$GITHUB_OUTPUT" + # A size ceiling, because image size is a deploy-time failure mode here + # and nothing else surfaces it. The image reached 14.1GB - 6.6GB of it + # CUDA runtime on a GPU-less box - and `docker pull` then ran past the + # deploy step's 40 minute command_timeout, so deploys failed with + # "Run Command Timeout" and no indication of why. Builds stayed green + # throughout; the cost only appeared on the host. + # + # Fails the build rather than the deploy, so the feedback lands on the + # PR that caused it instead of an hour later on a broken environment. + # Raise MAX_IMAGE_GB deliberately if the image legitimately grows. + - name: Enforce image size ceiling + env: + MAX_IMAGE_GB: 8 + run: | + set -euo pipefail + docker pull -q "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" + BYTES=$(docker image inspect \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" \ + --format "{{.Size}}") + GB=$(awk -v b="$BYTES" 'BEGIN{printf "%.2f", b/1024/1024/1024}') + echo "Image size: ${GB} GB (ceiling ${MAX_IMAGE_GB} GB)" + echo "- image size: **${GB} GB** (ceiling ${MAX_IMAGE_GB} GB)" >> "$GITHUB_STEP_SUMMARY" + if awk -v g="$GB" -v m="$MAX_IMAGE_GB" 'BEGIN{exit !(g > m)}'; then + echo "::error::Image is ${GB} GB, over the ${MAX_IMAGE_GB} GB ceiling. Pulling this on the deploy host will run past the SSH command_timeout and the deploy will fail. Check for CUDA/GPU wheels (nvidia/*, torch, triton) being pulled in place of CPU builds." + exit 1 + fi + - name: Sanity-check the built image # Cheap, real gate: catches import errors and bad settings before # anything touches the host. Live testing showed every layer From 965ad21bbb4e68a176eb6364f84d02f04a7cb8cf Mon Sep 17 00:00:00 2001 From: Saqib Date: Thu, 3 Sep 2026 14:47:27 +0530 Subject: [PATCH 57/57] chore: remove the dead DRF throttle config that misled a diagnosis DEFAULT_THROTTLE_RATES was set to {"anon": "100/hour", "user": "1000/hour"} and had no effect whatsoever: DEFAULT_THROTTLE_CLASSES was never configured, no view declares throttle_classes, and the endpoint that actually gets hammered (/api/graphql) is a Strawberry view, not a DRF one. It was not merely inert. While diagnosing a flood of 429s on 2026-09-03 it was the first thing found, read as the cause, and very nearly "fixed" - a change that would have altered nothing while sending the investigation the wrong way. Replaced with a comment pointing at api/middleware/rate_limit.py, which is what actually runs: 5000/hour for GET, 1000/hour for other methods, keyed on the client IP from X-Forwarded-For. No behaviour change - the setting was doing nothing. --- DataSpace/settings.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/DataSpace/settings.py b/DataSpace/settings.py index 47dad67..50a38dd 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -301,7 +301,20 @@ "rest_framework.authentication.BasicAuthentication", ], "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", - "DEFAULT_THROTTLE_RATES": {"anon": "100/hour", "user": "1000/hour"}, + # NOTE: there is deliberately no DEFAULT_THROTTLE_RATES here. + # + # It used to say {"anon": "100/hour", "user": "1000/hour"} and had no effect + # whatsoever: DEFAULT_THROTTLE_CLASSES was never set, no view declares + # throttle_classes, and the endpoint that actually gets hammered + # (/api/graphql) is a Strawberry view, not a DRF one. + # + # It was actively harmful. While diagnosing a flood of 429s on 2026-09-03 it + # was the first thing found, read as the cause, and very nearly "fixed" - + # which would have changed nothing and sent the investigation the wrong way. + # + # Real rate limiting lives in api/middleware/rate_limit.py (registered in + # MIDDLEWARE above): 5000/hour for GET, 1000/hour for other methods, keyed on + # the client IP from X-Forwarded-For. Change limits there. "DEFAULT_FILTER_BACKENDS": ["django_filters.rest_framework.DjangoFilterBackend"], "PAGE_SIZE": 10, }