diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml new file mode 100644 index 00000000..bacde4f9 --- /dev/null +++ b/.github/workflows/deploy-backend.yml @@ -0,0 +1,336 @@ +# Dev CD pipeline: build once in CI, deploy that exact artifact, verify it +# with real API tests, and roll the image back automatically if verification +# fails. +# +# build -> deploy -> smoke-tests -> { rollback-on-smoke-failure | finalize } +# +# Deploys are pinned to an immutable digest, never a tag: `dev` is a moving +# pointer, so "roll back to the previous dev tag" is not a thing you can +# express. The host records the digest it was running before each deploy in +# .deploy/previous_image, which is what makes cross-job rollback possible at +# all (rollback runs on a different runner, so no shell state survives). +# +# MIGRATIONS ARE NOT ROLLED BACK. A rollback restores the previous image and +# says so loudly; the schema stays forward. Migrations must therefore be +# written additively / backward-compatibly, so the previous image can still +# run against the newer schema. This is a policy constraint on how you write +# migrations, not something this pipeline can enforce for you. +# +# The target host is not a standalone checkout of this repo -- it's the +# DataExBackend submodule inside the separate CivicDataLab/DataExchange +# superproject, already running its own self-contained compose project +# there (confirmed via `docker ps` compose labels: project=dataexbackend, +# workdir=~/DataExchange/DataExBackend). This pipeline only ever touches +# that directory; it never touches the DataExchange repo or its top-level +# compose file. +# +# Several structural choices here mirror ParakhAPI's proven dev CD pipeline +# (deploy-parakh-api-dev.yml in CivicDataLab/ParakhAI-Backend) -- see the +# notes at each site before "simplifying" them. In particular: +# appleboy/ssh-action's inline multi-line `script:` input was found there to +# reproducibly fail with a spurious "syntax error near unexpected token ';'" +# for reasons never fully root-caused (confirmed the script text itself was +# valid bash both locally and on the real target host every time -- the +# corruption happened somewhere in the action's own transport). scp-action +# never had that problem. So no SSH step here ever carries more than a +# single trivial invocation line; all real logic lives in scripts/ci-*.sh, +# shipped as files. + +name: Deploy Backend to Dev EC2 + +on: + push: + branches: ['dev'] + workflow_dispatch: + inputs: + force_smoke_failure: + description: "Deliberately fail the smoke gate, to exercise the rollback path. Testing only." + type: boolean + required: false + default: false + +# Queue overlapping deploys rather than cancelling: a cancelled run mid-deploy +# could leave .deploy/ state and the running containers disagreeing. +concurrency: + group: dataspace-backend-dev-deploy + cancel-in-progress: false + +env: + REGISTRY: ghcr.io + IMAGE_NAME: civicdatalab/dataspacebackend + DEPLOY_PATH: ${{ vars.DEPLOY_PATH || 'DataExchange/DataExBackend' }} + +jobs: + build: + name: Build and push image + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + packages: write + outputs: + image_ref: ${{ steps.ref.outputs.image_ref }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + # driver: docker-container explicitly, not the ambient default -- + # this runner's default buildx context reports driver "docker", + # which does not support cache export (cache-to: type=gha below + # fails outright without this). + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + id: build + uses: docker/build-push-action@v6 + with: + context: . + push: true + build-args: | + GIT_COMMIT_SHA=${{ github.sha }} + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }} + # mode=min (the default), not max: this Dockerfile is single-stage + # (no multi-stage FROM ... AS builder), so mode=max's extra + # intermediate-stage caching buys nothing here -- it only added a + # slow cache-export step that got stuck writing one large layer + # (chromium + apt packages) and blew the job's timeout on the + # first (cold-cache) run, even though the actual image build and + # push had already completed successfully by that point. + cache-from: type=gha + cache-to: type=gha + + - name: Pin image reference by digest + id: ref + run: | + echo "image_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" >> "$GITHUB_OUTPUT" + + # A size ceiling, because image size is a deploy-time failure mode here + # and nothing else surfaces it. The image reached 14.1GB - 6.6GB of it + # CUDA runtime on a GPU-less box - and `docker pull` then ran past the + # deploy step's 40 minute command_timeout, so deploys failed with + # "Run Command Timeout" and no indication of why. Builds stayed green + # throughout; the cost only appeared on the host. + # + # Fails the build rather than the deploy, so the feedback lands on the + # PR that caused it instead of an hour later on a broken environment. + # Raise MAX_IMAGE_GB deliberately if the image legitimately grows. + - name: Enforce image size ceiling + env: + MAX_IMAGE_GB: 8 + run: | + set -euo pipefail + docker pull -q "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" + BYTES=$(docker image inspect \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" \ + --format "{{.Size}}") + GB=$(awk -v b="$BYTES" 'BEGIN{printf "%.2f", b/1024/1024/1024}') + echo "Image size: ${GB} GB (ceiling ${MAX_IMAGE_GB} GB)" + echo "- image size: **${GB} GB** (ceiling ${MAX_IMAGE_GB} GB)" >> "$GITHUB_STEP_SUMMARY" + if awk -v g="$GB" -v m="$MAX_IMAGE_GB" 'BEGIN{exit !(g > m)}'; then + echo "::error::Image is ${GB} GB, over the ${MAX_IMAGE_GB} GB ceiling. Pulling this on the deploy host will run past the SSH command_timeout and the deploy will fail. Check for CUDA/GPU wheels (nvidia/*, torch, triton) being pulled in place of CPU builds." + exit 1 + fi + + - name: Sanity-check the built image + # Cheap, real gate: catches import errors and bad settings before + # anything touches the host. Live testing showed every layer + # downloading successfully every single retry, with toomanyrequests + # firing right after the LAST layer completes (the final + # manifest/config fetch) -- consistently, even across 10 attempts + # 30s apart. That pattern looks like GHCR hasn't finished settling + # a just-pushed manifest yet, not a generic quota, so this waits + # up front before the first attempt rather than only reacting + # after failures. + run: | + sleep 90 + for attempt in 1 2 3 4 5 6 7 8 9 10; do + if docker run --rm \ + -e SECRET_KEY=ci-sanity-check-not-a-real-key \ + -e URL_WHITELIST=http://localhost \ + -e DB_ENGINE=django.db.backends.sqlite3 \ + --entrypoint python \ + "${{ steps.ref.outputs.image_ref }}" \ + manage.py check; then + exit 0 + fi + echo "attempt $attempt failed, retrying in 30s..." + sleep 30 + done + echo "::error::Sanity check failed after 10 attempts." + exit 1 + + deploy: + name: Deploy to EC2 + needs: build + runs-on: ubuntu-latest + environment: development + # 50m: must comfortably exceed the Deploy step's own 40m + # command_timeout (see that step's comment for why it's 40m). + timeout-minutes: 50 + # packages: read -- GITHUB_TOKEN needs this explicitly granted to pull + # from GHCR; it isn't covered by the repo's default token permissions. + permissions: + contents: read + packages: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Write GHCR token file + run: printf '%s' "${{ secrets.GITHUB_TOKEN }}" > .ghcr_token + + # Ship the compose files rather than relying on the DataExBackend + # submodule pointer inside the separate DataExchange repo -- bumping + # that pointer is a change to a different, shared repo and out of + # scope here. Shipping the file directly keeps this pipeline + # self-contained. + - name: Ship deploy files to host + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + source: docker-compose.yml,docker-compose.hotreload.yml,scripts/ci-deploy.sh,.ghcr_token + target: ${{ env.DEPLOY_PATH }} + + - name: Deploy + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + script_stop: true + # 40m: live testing showed one large layer (chromium + its X11 + # libs, almost certainly) take ~15 minutes just downloading via + # Docker's own internal per-layer retry (not our retry loop -- + # this is a single docker pull invocation struggling), then hang + # with zero output for several more minutes afterward (most + # likely extraction stalling under memory pressure -- this host + # runs Postgres, two separate Elasticsearch instances, Redis, + # Keycloak, and telemetry tooling alongside the app, confirmed + # via `free -h`: ~169Mi truly free, 828Mi already swapped). The + # previous 20m ceiling cut it off mid-extraction, right after + # the layer had already finished downloading. + command_timeout: 40m + # sudo: docker/docker compose require it on this host (confirmed + # passwordless -- sudo -n succeeds non-interactively). + script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-deploy.sh "${{ needs.build.outputs.image_ref }}" "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" + + smoke-tests: + name: Smoke Tests + needs: deploy + # No `environment:` here -- GitHub rejects the entire workflow file at + # parse time if a `uses:` job declares one. Consequence: this job also + # cannot see environment-scoped vars, which is why api_base_url comes + # from a repo-level var. + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + api_base_url: ${{ vars.DEV_API_BASE_URL }} + # An obviously-wrong sentinel SHA fails the reusable workflow's own + # deployed-SHA-vs-live-/health/ assertion on purpose, for the + # force_smoke_failure test path. + deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }} + min_passed: ${{ inputs.force_smoke_failure && 999 || 1 }} + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a + # confidential client, so without this the token request returns 401 + # and the job fails its preflight. + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + + rollback-on-smoke-failure: + name: Rollback (smoke tests failed) + # `deploy` must be in needs: for needs.deploy.result to resolve here. + needs: [deploy, smoke-tests] + # failure()/success() builtins rather than needs.smoke-tests.result -- + # both are false on cancellation, which is the behaviour we want; + # if: always() would ignore cancellation entirely. + if: failure() && needs.deploy.result == 'success' + runs-on: ubuntu-latest + environment: development + # 50m: must comfortably exceed the Restore previous image step's own + # 40m command_timeout (see the Deploy job's equivalent step for why). + timeout-minutes: 50 + permissions: + contents: read + packages: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Write GHCR token file + run: printf '%s' "${{ secrets.GITHUB_TOKEN }}" > .ghcr_token + + - name: Ship rollback files to host + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + source: scripts/ci-rollback.sh,.ghcr_token + target: ${{ env.DEPLOY_PATH }} + + - name: Restore previous image + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + script_stop: true + # 40m -- see the Deploy job's equivalent step for why (one large + # layer took ~15 min to download plus several more to extract + # under this host's memory pressure in live testing). + command_timeout: 40m + script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-rollback.sh "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" + + - name: Mark this run as failed + # The mitigation succeeded, but the run must still read RED -- a bad + # deploy that silently self-heals is a bad deploy nobody investigates. + run: | + echo "::error::Smoke tests failed after deploy; the image was rolled back. Migrations were NOT reverted -- see the rollback step's log." + exit 1 + + finalize-deploy: + name: Finalize Deploy + needs: [deploy, smoke-tests] + if: success() + runs-on: ubuntu-latest + environment: development + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ship finalize script to host + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + source: scripts/ci-finalize.sh + target: ${{ env.DEPLOY_PATH }} + + - name: Prune to current + previous image + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ vars.EC2_HOST }} + username: ${{ secrets.EC2_USERNAME }} + key: ${{ secrets.EC2_PRIVATE_KEY }} + script_stop: true + script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-finalize.sh diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 4fe14158..f3573b94 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -27,7 +27,6 @@ jobs: name: Deploy Infrastructure runs-on: ubuntu-latest environment: development - if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.modified, 'aws/cloudformation') steps: - name: Checkout @@ -62,7 +61,8 @@ jobs: runs-on: ubuntu-latest environment: development needs: deploy-infrastructure - if: always() # Run even if infrastructure deployment is skipped + outputs: + previous_task_def_arn: ${{ steps.previous-task-def.outputs.arn }} steps: - name: Checkout @@ -105,13 +105,101 @@ jobs: container-name: dataspace image: ${{ steps.build-image.outputs.image }} - - name: Deploy main application ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + - name: Capture currently-running task definition (for rollback) + id: previous-task-def + run: | + ARN=$(aws ecs describe-services \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" \ + --query 'services[0].taskDefinition' \ + --output text) + echo "Currently running: $ARN" + echo "arn=$ARN" >> "$GITHUB_OUTPUT" + + - name: Get running service's network configuration + id: network-config + run: | + NETWORK_CONFIG=$(aws ecs describe-services \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" \ + --query 'services[0].networkConfiguration.awsvpcConfiguration' \ + --output json) + { + echo "subnets=$(echo "$NETWORK_CONFIG" | jq -r '.subnets | join(",")')" + echo "security_groups=$(echo "$NETWORK_CONFIG" | jq -r '.securityGroups | join(",")')" + echo "assign_public_ip=$(echo "$NETWORK_CONFIG" | jq -r '.assignPublicIp')" + } >> "$GITHUB_OUTPUT" + + - name: Run migrations, then deploy the ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def-app.outputs.task-definition }} service: ${{ secrets.ECS_SERVICE }} cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true + run-task: true + run-task-container-overrides: '[{"name":"dataspace","command":["python","manage.py","migrate","--noinput"]}]' + run-task-subnets: ${{ steps.network-config.outputs.subnets }} + run-task-security-groups: ${{ steps.network-config.outputs.security_groups }} + run-task-assign-public-IP: ${{ steps.network-config.outputs.assign_public_ip }} + wait-for-task-stopped: true + + smoke-tests: + name: Smoke Tests + needs: deploy-app + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + api_base_url: ${{ vars.DEV_API_BASE_URL }} + deployed_sha: ${{ github.sha }} + min_passed: 1 + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a + # confidential client, so without this the token request returns 401 + # and the job fails its preflight. + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + + rollback-on-smoke-failure: + name: Rollback on Smoke Failure + runs-on: ubuntu-latest + environment: development + needs: [deploy-app, smoke-tests] + if: failure() && needs.deploy-app.result == 'success' + timeout-minutes: 15 + + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Restore the previously-running task definition + run: | + PREVIOUS_ARN="${{ needs.deploy-app.outputs.previous_task_def_arn }}" + if [ -z "$PREVIOUS_ARN" ]; then + echo "::error::No previous task definition was captured -- nothing to roll back to. This is expected on a service's very first deploy." + exit 1 + fi + echo "Rolling back to: $PREVIOUS_ARN" + aws ecs update-service \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --service "${{ secrets.ECS_SERVICE }}" \ + --task-definition "$PREVIOUS_ARN" \ + --force-new-deployment + aws ecs wait services-stable \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" + + - name: Mark this run as failed despite successful rollback + run: | + echo "::error::Smoke tests failed after deploy. Rolled back to the previous task definition -- migrations applied by this deploy were NOT reverted. Check what ran via the 'Run migrations, then deploy' step's logs before re-deploying." + exit 1 deploy-otel: name: Deploy OpenTelemetry Collector diff --git a/.gitignore b/.gitignore index 8fd0cefe..847c7a01 100644 --- a/.gitignore +++ b/.gitignore @@ -168,3 +168,13 @@ files/public/* #dvc files dvc dvc/* + +.DS_Store + +# Git worktrees for feature branches +.worktrees/ + +# CD/deploy runtime state (previous/current image refs, pending migrations +# list -- written by the deploy pipeline on the host, never committed) +.deploy/ +.ghcr_token diff --git a/DataSpace/settings.py b/DataSpace/settings.py index ee5a2a01..50a38ddb 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -277,6 +277,7 @@ "search.documents.dataset_document": "dataset", "search.documents.usecase_document": "usecase", "search.documents.aimodel_document": "aimodel", + "search.documents.publication_document": "publication", "search.documents.collaborative_document": "collaborative", "search.documents.publisher_document.OrganizationPublisherDocument": "organization_publisher", "search.documents.publisher_document.UserPublisherDocument": "user_publisher", @@ -300,7 +301,20 @@ "rest_framework.authentication.BasicAuthentication", ], "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", - "DEFAULT_THROTTLE_RATES": {"anon": "100/hour", "user": "1000/hour"}, + # NOTE: there is deliberately no DEFAULT_THROTTLE_RATES here. + # + # It used to say {"anon": "100/hour", "user": "1000/hour"} and had no effect + # whatsoever: DEFAULT_THROTTLE_CLASSES was never set, no view declares + # throttle_classes, and the endpoint that actually gets hammered + # (/api/graphql) is a Strawberry view, not a DRF one. + # + # It was actively harmful. While diagnosing a flood of 429s on 2026-09-03 it + # was the first thing found, read as the cause, and very nearly "fixed" - + # which would have changed nothing and sent the investigation the wrong way. + # + # Real rate limiting lives in api/middleware/rate_limit.py (registered in + # MIDDLEWARE above): 5000/hour for GET, 1000/hour for other methods, keyed on + # the client IP from X-Forwarded-For. Change limits there. "DEFAULT_FILTER_BACKENDS": ["django_filters.rest_framework.DjangoFilterBackend"], "PAGE_SIZE": 10, } diff --git a/Dockerfile b/Dockerfile index 05be1d8a..56dac148 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ FROM python:3.10 +ARG GIT_COMMIT_SHA=unknown +ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA} ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 @@ -49,8 +51,32 @@ RUN apt-get update && \ WORKDIR /code COPY . /code/ +# LOGGING in DataSpace/settings.py writes to logs/dataex.log -- Django's +# logging.config never creates the parent directory itself, so any fresh +# container without this (no pre-existing volume/manual mkdir) fails on +# django.setup() with FileNotFoundError before any command can even run, +# including manage.py check and the healthcheck.sh script below. +RUN mkdir -p /code/logs + RUN pip install psycopg2-binary uvicorn -RUN pip install -r requirements.txt +# Install CPU-only torch first, so the pinned torch==2.9.0 in +# requirements.txt is already satisfied and pip never reaches for the default +# CUDA build. +# +# The CUDA wheels pull in 4.3GB of nvidia/* libraries, 1.7GB of torch and +# 592MB of triton - about 6.6GB of GPU runtime on a 2-CPU EC2 box that has no +# GPU and physically cannot use any of it. That is most of why the image is +# 14.1GB, why a deploy takes about an hour, and why the deploy of #136 failed +# outright: `docker pull` ran past the SSH step's 40 minute command_timeout. +# +# PEP 440 treats the local version segment as compatible, so 2.9.0+cpu +# satisfies ==2.9.0 and requirements.txt needs no change. Pinned to the same +# version deliberately - this changes the build of torch, not the version. +RUN pip install --no-cache-dir torch==2.9.0 \ + --index-url https://download.pytorch.org/whl/cpu + +# --no-cache-dir: the wheel cache is dead weight in the final layer. +RUN pip install --no-cache-dir -r requirements.txt RUN curl -s https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js -o /code/echarts.min.js # Create healthcheck script @@ -64,4 +90,44 @@ EXPOSE 8000 RUN chmod +x /code/docker-entrypoint.sh ENTRYPOINT ["bash","/code/docker-entrypoint.sh"] -CMD ["uvicorn", "DataSpace.asgi:application", "--host", "0.0.0.0", "--port", "8000"] + +# Served with multiple workers, which is what keeps this from exhausting +# Postgres. +# +# Django runs sync views under ASGI via sync_to_async(thread_sensitive=True), +# which executes them on ONE shared thread per process. With a single worker +# that means exactly one sync request is processed at a time, no matter how +# many arrive. Measured on dev before this change: 12 concurrent calls to +# /api/auth/keycloak/login/ returned in 4s, 7s, 10s, 14s ... 36s - near-perfect +# ~3s increments, queued behind each other. +# +# That queue is what killed the database. Every in-flight request holds a +# connection while it waits its turn - measured at one connection per request, +# so 20 concurrent calls took the connection count from 6 to 26. Deep enough +# queues reached max_connections (100) and Postgres started refusing with +# "FATAL: sorry, too many clients already", which surfaced as 500s, while +# requests that waited past nginx's 60s proxy timeout surfaced as 504s. It +# also broke deploys, because manage.py could not get a connection either. +# +# Workers are processes, so N workers give N concurrent sync requests and the +# queue drains N times faster. The work here is I/O-bound (waiting on +# Keycloak), so this helps well beyond the 2 CPUs on the dev box. +# +# UVICORN_LIMIT_CONCURRENCY is the backstop: total in-flight requests are +# capped at workers x limit, which must stay under Postgres max_connections +# minus headroom for other clients. Excess requests get a fast 503 instead of +# queueing until the database runs out of slots - shedding load is recoverable, +# exhausting connections takes the deploy pipeline down with it. +# Sized to the dev box, not to a formula. One worker measured at 740MB +# resident with only ~2.4GB available on the host, so 4 workers risked an OOM +# that would have been a worse outage than the one this fixes. Two workers land +# near 1.3GB and match the 2 CPUs. +# +# Two workers alone would only double throughput, but the commit that removes +# two of the three Keycloak round-trips cuts per-request time as well, and the +# two compound. Raise UVICORN_WORKERS on a bigger box - it is env-tunable for +# exactly that reason, and worth revisiting if memory there grows. +ENV UVICORN_WORKERS=2 \ + UVICORN_LIMIT_CONCURRENCY=15 + +CMD ["sh", "-c", "exec uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --workers ${UVICORN_WORKERS} --limit-concurrency ${UVICORN_LIMIT_CONCURRENCY}"] diff --git a/api/management/commands/seed_resource_types.py b/api/management/commands/seed_resource_types.py new file mode 100644 index 00000000..333cc7ff --- /dev/null +++ b/api/management/commands/seed_resource_types.py @@ -0,0 +1,50 @@ +""" +Django management command to seed the initial Resource Type lookup values. + +Idempotent — re-running only creates the types that are missing and never +duplicates or overwrites an admin's edits. Admins manage the list afterwards +(add / rename / deactivate) from the Django admin, no deploy required. + +Usage: + python manage.py seed_resource_types +""" + +from typing import Any + +from django.core.management.base import BaseCommand +from django.db import transaction + +from api.models import ResourceType + +# The starting set of Resource Types. Admin-editable after seeding. +INITIAL_RESOURCE_TYPES = [ + "Report", + "Article", + "Policy Brief", + "Research Paper", + "Case Study", + "Guide", + "Toolkit", + "Presentation", + "Fact Sheet", + "Working Paper", +] + + +class Command(BaseCommand): + help = "Seed the initial Resource Type lookup values (idempotent)" + + def handle(self, *args: Any, **options: Any) -> None: + created_count = 0 + with transaction.atomic(): + for name in INITIAL_RESOURCE_TYPES: + _, created = ResourceType.objects.get_or_create(name=name) + if created: + created_count += 1 + + self.stdout.write( + self.style.SUCCESS( + f"✓ Resource Types seeded ({created_count} created, " + f"{len(INITIAL_RESOURCE_TYPES) - created_count} already present)" + ) + ) diff --git a/api/management/commands/update_dataindex.py b/api/management/commands/update_dataindex.py index 302fb58e..af1190a4 100644 --- a/api/management/commands/update_dataindex.py +++ b/api/management/commands/update_dataindex.py @@ -100,7 +100,7 @@ def handle(self, *args: Any, **options: Dict[str, Any]) -> None: skipped_count += 1 continue - # Skip resources that aren't CSV files + # Skip resources that aren't in a supported indexed format file_details = resource.resourcefiledetails if not file_details or file_details.format.lower() not in INDEXED_FORMATS: self.stdout.write( diff --git a/api/models/Collaborative.py b/api/models/Collaborative.py index 0681aa2b..c81e3489 100644 --- a/api/models/Collaborative.py +++ b/api/models/Collaborative.py @@ -1,8 +1,8 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, cast -from django.db import models from django.core.validators import RegexValidator +from django.db import models from django.utils.text import slugify if TYPE_CHECKING: @@ -13,7 +13,6 @@ from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.file_paths import _use_case_directory_path - slug_validator = RegexValidator( regex=r"^[a-z0-9]+(?:-[a-z0-9]+)*$", message="Slug must be lowercase and contain only alphanumeric characters and hyphens.", @@ -47,15 +46,12 @@ class Collaborative(models.Model): choices=CollaborativeStatus.choices, ) datasets = models.ManyToManyField("api.Dataset", blank=True) + publications = models.ManyToManyField("api.Publication", blank=True) use_cases = models.ManyToManyField("api.UseCase", blank=True) tags = models.ManyToManyField("api.Tag", blank=True) - sectors = models.ManyToManyField( - "api.Sector", blank=True, related_name="collaboratives" - ) + sectors = models.ManyToManyField("api.Sector", blank=True, related_name="collaboratives") sdgs = models.ManyToManyField("api.SDG", blank=True, related_name="collaboratives") - geographies = models.ManyToManyField( - "api.Geography", blank=True, related_name="collaboratives" - ) + geographies = models.ManyToManyField("api.Geography", blank=True, related_name="collaboratives") contributors = models.ManyToManyField( "authorization.User", blank=True, related_name="contributed_collaboratives" ) diff --git a/api/models/Publication.py b/api/models/Publication.py new file mode 100644 index 00000000..b922433f --- /dev/null +++ b/api/models/Publication.py @@ -0,0 +1,190 @@ +import uuid +from typing import TYPE_CHECKING, Any + +from django.db import models +from django.db.models import Q +from django.utils.text import slugify + +from api.utils.enums import DatasetLicense, PublicationBlockType, PublicationStatus +from api.utils.file_paths import _publication_block_directory_path + +if TYPE_CHECKING: + from api.models.Organization import Organization + from api.models.ResourceType import ResourceType + from authorization.models import User + + +class Publication(models.Model): + """A top-level Resource — a container for human-authored content. + + Peer to Dataset and AI Model: a UUID/slug entity owned by an organization or + an individual user, with typed metadata columns, a publish/unpublish status, + an ordered list of content blocks, and its own search + linking surface. + (Internal name ``Publication`` to avoid colliding with the file-inside-a-dataset + ``Resource``; the UI always labels it "Resource".) + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + title = models.CharField(max_length=300, unique=False, blank=True) + description = models.TextField(blank=True, null=True) + slug = models.SlugField(max_length=255, unique=True) + + organization = models.ForeignKey( + "api.Organization", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="publications", + ) + user = models.ForeignKey( + "authorization.User", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="publications", + ) + + # Typed metadata columns — a fixed, known schema (no dynamic/EAV fields). + authors = models.JSONField(default=list, blank=True) + publication_date = models.DateField(null=True, blank=True) + license = models.CharField( + max_length=50, + default=DatasetLicense.CC_BY_4_0_ATTRIBUTION, + choices=DatasetLicense.choices, + ) + external_source_link = models.URLField(blank=True, null=True) + + # Structural / faceted columns. + status = models.CharField( + max_length=50, + default=PublicationStatus.DRAFT, + choices=PublicationStatus.choices, + ) + resource_type = models.ForeignKey( + "api.ResourceType", + on_delete=models.PROTECT, + null=True, + blank=True, + related_name="publications", + ) + sectors = models.ManyToManyField("api.Sector", blank=True, related_name="publications") + geographies = models.ManyToManyField("api.Geography", blank=True, related_name="publications") + download_count = models.IntegerField(default=0) + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def save(self, *args: Any, **kwargs: Any) -> None: + if not self.slug: + base_slug = slugify(self.title) + slug = base_slug + counter = 1 + while Publication.objects.filter(slug=slug).exclude(pk=self.pk).exists(): + slug = f"{base_slug}-{counter}" + counter += 1 + self.slug = slug + super().save(*args, **kwargs) + + @property + def is_individual_publication(self) -> bool: + """True when this Resource is owned by an individual, not an organization.""" + return self.organization is None and self.user is not None + + @property + def sectors_indexing(self) -> list[str]: + """Sector names for Elasticsearch indexing.""" + return [sector.name for sector in self.sectors.all()] # type: ignore + + @property + def geographies_indexing(self) -> list[str]: + """Geography names for Elasticsearch indexing.""" + return [geo.name for geo in self.geographies.all()] # type: ignore + + @property + def resource_type_indexing(self) -> str: + """Resource-type name for Elasticsearch indexing (empty when unset).""" + return self.resource_type.name if self.resource_type else "" + + def __str__(self) -> str: + return self.title + + class Meta: + verbose_name = "Publication" + verbose_name_plural = "Publications" + db_table = "publication" + ordering = ["-modified"] + indexes = [ + models.Index(fields=["organization", "-modified"]), + models.Index(fields=["user", "-modified"]), + models.Index(fields=["status"]), + ] + + +class PublicationBlock(models.Model): + """One content block in a Resource — a file XOR a YouTube embed, at a position. + + Blocks are ordered by ``position`` within their publication. Each block is + exactly one of two shapes, enforced by the ``file_xor_youtube`` check + constraint: a FILE block carries an uploaded file (with its name/format/size), + a YOUTUBE block carries a video url and its extracted id. Neither-both nor + neither-set is a valid row. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + publication = models.ForeignKey( + "api.Publication", + on_delete=models.CASCADE, + related_name="blocks", + ) + position = models.PositiveIntegerField(default=0) + block_type = models.CharField( + max_length=20, + choices=PublicationBlockType.choices, + ) + + # FILE block fields. + file = models.FileField( + upload_to=_publication_block_directory_path, + max_length=300, + blank=True, + ) + file_name = models.CharField(max_length=300, blank=True) + file_format = models.CharField(max_length=50, blank=True) + file_size = models.BigIntegerField(null=True, blank=True) + + # YOUTUBE block fields. + youtube_url = models.URLField(blank=True, null=True) + youtube_video_id = models.CharField(max_length=20, blank=True) + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def __str__(self) -> str: + return f"{self.block_type} block #{self.position} of {self.publication_id}" + + class Meta: + db_table = "publication_block" + ordering = ["position"] + indexes = [ + models.Index(fields=["publication", "position"]), + ] + constraints = [ + # Runtime is Django 5.0 (``check=``); the pinned django-stubs is + # newer and only knows the 5.1 ``condition=`` spelling, hence the + # ignore. Switch to ``condition=`` when the runtime moves to 5.1+. + models.CheckConstraint( # type: ignore[call-arg] + name="publicationblock_file_xor_youtube", + check=( + Q( + block_type=PublicationBlockType.FILE, + youtube_url__isnull=True, + ) + & ~Q(file="") + ) + | Q( + block_type=PublicationBlockType.YOUTUBE, + file="", + youtube_url__isnull=False, + ), + ) + ] diff --git a/api/models/ResourceType.py b/api/models/ResourceType.py new file mode 100644 index 00000000..9f7aac16 --- /dev/null +++ b/api/models/ResourceType.py @@ -0,0 +1,33 @@ +import uuid +from typing import Any + +from django.db import models +from django.utils.text import slugify + + +class ResourceType(models.Model): + """Admin-managed lookup for a Resource's type (Report, Article, Policy Brief, ...). + + A flat, non-hierarchical list — no parent self-FK. Admins can add, rename, + or deactivate a type without a code release. Deactivating (``is_active=False``) + keeps historical references intact while hiding the type from new selections. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=75, unique=True, null=False, blank=False) + description = models.CharField(max_length=1000, null=True, blank=True) + slug = models.SlugField(max_length=75, null=True, blank=False, unique=True) + is_active = models.BooleanField(default=True) + + def save(self, *args: Any, **kwargs: Any) -> None: + self.slug = slugify(self.name) + super().save(*args, **kwargs) + + def __str__(self) -> str: + return str(self.name) + + class Meta: + db_table = "resource_type" + verbose_name = "Resource Type" + verbose_name_plural = "Resource Types" + ordering = ["name"] diff --git a/api/models/UseCase.py b/api/models/UseCase.py index 5e7295ea..09a2b6a8 100644 --- a/api/models/UseCase.py +++ b/api/models/UseCase.py @@ -37,6 +37,7 @@ class UseCase(models.Model): max_length=50, default=UseCaseStatus.DRAFT, choices=UseCaseStatus.choices ) datasets = models.ManyToManyField("api.Dataset", blank=True) + publications = models.ManyToManyField("api.Publication", blank=True) tags = models.ManyToManyField("api.Tag", blank=True) running_status = models.CharField( max_length=50, @@ -45,9 +46,7 @@ class UseCase(models.Model): ) sectors = models.ManyToManyField("api.Sector", blank=True, related_name="usecases") sdgs = models.ManyToManyField("api.SDG", blank=True, related_name="usecases") - geographies = models.ManyToManyField( - "api.Geography", blank=True, related_name="usecases" - ) + geographies = models.ManyToManyField("api.Geography", blank=True, related_name="usecases") contributors = models.ManyToManyField( "authorization.User", blank=True, related_name="contributed_usecases" ) diff --git a/api/models/__init__.py b/api/models/__init__.py index 14ac679f..6c54b34c 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -15,6 +15,7 @@ from api.models.Organization import Organization from api.models.PromptDataset import PromptDataset from api.models.PromptResource import PromptResource +from api.models.Publication import Publication, PublicationBlock from api.models.Resource import ( Resource, ResourceDataTable, @@ -26,6 +27,7 @@ from api.models.ResourceChartImage import ResourceChartImage from api.models.ResourceMetadata import ResourceMetadata from api.models.ResourceSchema import ResourceSchema +from api.models.ResourceType import ResourceType from api.models.SDG import SDG from api.models.Sector import Sector from api.models.SerializableJSONField import SerializableJSONField diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index 3c1213d7..d2fa5f34 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -29,6 +29,11 @@ UseCase, ) from api.schema.extensions import TrackActivity, TrackModelActivity +from api.services.publication_linking import ( + assert_can_manage_links, + get_linkable_publication, + published_publications, +) from api.types.type_collaborative import ( CollaborativeFilter, CollaborativeOrder, @@ -47,7 +52,9 @@ from authorization.types import TypeUser -@strawberry_django.input(Collaborative, fields="__all__", exclude=["datasets", "slug"]) +@strawberry_django.input( + Collaborative, fields="__all__", exclude=["datasets", "publications", "slug"] +) class CollaborativeInput: """Input type for collaborative creation.""" @@ -70,7 +77,7 @@ class UpdateCollaborativeMetadataInput: geographies: Optional[List[int]] -@strawberry_django.partial(Collaborative, fields="__all__", exclude=["datasets"]) +@strawberry_django.partial(Collaborative, fields="__all__", exclude=["datasets", "publications"]) class CollaborativeInputPartial: """Input type for collaborative updates.""" @@ -99,9 +106,7 @@ class Query: pagination=True, order=CollaborativeOrder, ) - @trace_resolver( - name="get_collaboratives", attributes={"component": "collaborative"} - ) + @trace_resolver(name="get_collaboratives", attributes={"component": "collaborative"}) def collaboratives( self, info: Info, @@ -119,9 +124,7 @@ def collaboratives( elif user.is_authenticated: queryset = Collaborative.objects.filter(user=user) else: - queryset = Collaborative.objects.filter( - status=CollaborativeStatus.PUBLISHED - ) + queryset = Collaborative.objects.filter(status=CollaborativeStatus.PUBLISHED) if filters is not strawberry.UNSET: queryset = strawberry_django.filters.apply(filters, queryset, info) @@ -136,9 +139,7 @@ def collaboratives( return TypeCollaborative.from_django_list(queryset) @strawberry_django.field - @trace_resolver( - name="get_published_collaboratives", attributes={"component": "collaborative"} - ) + @trace_resolver(name="get_published_collaboratives", attributes={"component": "collaborative"}) def published_collaboratives( self, info: Info, @@ -173,12 +174,8 @@ def published_collaboratives( return TypeCollaborative.from_django_list(results) @strawberry_django.field - @trace_resolver( - name="get_datasets_by_collaborative", attributes={"component": "collaborative"} - ) - def dataset_by_collaborative( - self, info: Info, collaborative_id: str - ) -> list[TypeDataset]: + @trace_resolver(name="get_datasets_by_collaborative", attributes={"component": "collaborative"}) + def dataset_by_collaborative(self, info: Info, collaborative_id: str) -> list[TypeDataset]: """Get datasets by collaborative.""" queryset = Dataset.objects.filter(collaborative__id=collaborative_id) return TypeDataset.from_django_list(queryset) @@ -188,18 +185,14 @@ def dataset_by_collaborative( name="get_contributors_by_collaborative", attributes={"component": "collaborative"}, ) - def contributors_by_collaborative( - self, info: Info, collaborative_id: str - ) -> list[TypeUser]: + def contributors_by_collaborative(self, info: Info, collaborative_id: str) -> list[TypeUser]: """Get contributors by collaborative.""" try: collaborative = Collaborative.objects.get(id=collaborative_id) contributors = collaborative.contributors.all() return TypeUser.from_django_list(contributors) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") @strawberry_django.field @trace_resolver( @@ -214,9 +207,7 @@ def collaborative_by_slug(self, info: Info, slug: str) -> TypeCollaborative: raise ValueError(f"Collaborative with slug {slug} does not exist.") -@trace_resolver( - name="update_collaborative_tags", attributes={"component": "collaborative"} -) +@trace_resolver(name="update_collaborative_tags", attributes={"component": "collaborative"}) def _update_collaborative_tags(collaborative: Collaborative, tags: List[str]) -> None: collaborative.tags.clear() for tag in tags: @@ -226,24 +217,16 @@ def _update_collaborative_tags(collaborative: Collaborative, tags: List[str]) -> collaborative.save() -@trace_resolver( - name="update_collaborative_sectors", attributes={"component": "collaborative"} -) -def _update_collaborative_sectors( - collaborative: Collaborative, sectors: List[uuid.UUID] -) -> None: +@trace_resolver(name="update_collaborative_sectors", attributes={"component": "collaborative"}) +def _update_collaborative_sectors(collaborative: Collaborative, sectors: List[uuid.UUID]) -> None: sectors_objs = Sector.objects.filter(id__in=sectors) collaborative.sectors.clear() collaborative.sectors.add(*sectors_objs) collaborative.save() -@trace_resolver( - name="update_collaborative_sdgs", attributes={"component": "collaborative"} -) -def _update_collaborative_sdgs( - collaborative: Collaborative, sdgs: List[uuid.UUID] -) -> None: +@trace_resolver(name="update_collaborative_sdgs", attributes={"component": "collaborative"}) +def _update_collaborative_sdgs(collaborative: Collaborative, sdgs: List[uuid.UUID]) -> None: sdgs_objs = SDG.objects.filter(id__in=sdgs) collaborative.sdgs.clear() collaborative.sdgs.add(*sdgs_objs) @@ -265,9 +248,7 @@ def _add_update_collaborative_metadata( metadata_field = Metadata.objects.get(id=metadata_input_item.id) if not metadata_field.enabled: _delete_existing_metadata(collaborative) - raise ValueError( - f"Metadata with ID {metadata_input_item.id} is not enabled." - ) + raise ValueError(f"Metadata with ID {metadata_input_item.id} is not enabled.") uc_metadata = CollaborativeMetadata( collaborative=collaborative, metadata_item=metadata_field, @@ -276,19 +257,13 @@ def _add_update_collaborative_metadata( uc_metadata.save() except Metadata.DoesNotExist: _delete_existing_metadata(collaborative) - raise ValueError( - f"Metadata with ID {metadata_input_item.id} does not exist." - ) + raise ValueError(f"Metadata with ID {metadata_input_item.id} does not exist.") -@trace_resolver( - name="delete_existing_metadata", attributes={"component": "collaborative"} -) +@trace_resolver(name="delete_existing_metadata", attributes={"component": "collaborative"}) def _delete_existing_metadata(collaborative: Collaborative) -> None: try: - existing_metadata = CollaborativeMetadata.objects.filter( - collaborative=collaborative - ) + existing_metadata = CollaborativeMetadata.objects.filter(collaborative=collaborative) existing_metadata.delete() except CollaborativeMetadata.DoesNotExist: pass @@ -357,10 +332,7 @@ def add_collaborative(self, info: Info) -> TypeCollaborative: else None ), "sectors": ( - [ - str(sector_id) - for sector_id in update_metadata_input.sectors - ] + [str(sector_id) for sector_id in update_metadata_input.sectors] if update_metadata_input.sectors else [] ), @@ -381,14 +353,10 @@ def add_update_collaborative_metadata( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") if update_metadata_input.tags is not None: _update_collaborative_tags(collaborative, update_metadata_input.tags) @@ -397,9 +365,7 @@ def add_update_collaborative_metadata( if update_metadata_input.sdgs is not None: _update_collaborative_sdgs(collaborative, update_metadata_input.sdgs) if update_metadata_input.geographies is not None: - _update_collaborative_geographies( - collaborative, update_metadata_input.geographies - ) + _update_collaborative_geographies(collaborative, update_metadata_input.geographies) return TypeCollaborative.from_django(collaborative) @strawberry_django.mutation(handle_django_errors=False) @@ -414,14 +380,10 @@ def update_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") if data.title is not None: if data.title.strip() == "": @@ -466,9 +428,7 @@ def delete_collaborative(self, info: Info, collaborative_id: str) -> bool: try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") collaborative.delete() return True @@ -485,14 +445,10 @@ def add_dataset_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.datasets.add(dataset) collaborative.save() @@ -511,14 +467,10 @@ def add_usecase_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.use_cases.add(usecase) collaborative.save() @@ -536,14 +488,10 @@ def remove_dataset_from_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.datasets.remove(dataset) collaborative.save() return TypeCollaborative.from_django(collaborative) @@ -560,14 +508,10 @@ def remove_usecase_from_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.use_cases.remove(usecase) collaborative.save() return TypeCollaborative.from_django(collaborative) @@ -588,14 +532,80 @@ def update_collaborative_datasets( raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.datasets.set(datasets) collaborative.save() return TypeCollaborative.from_django(collaborative) + @strawberry_django.mutation(handle_django_errors=True) + def add_publication_to_collaborative( + self, info: Info, collaborative_id: str, publication_id: uuid.UUID + ) -> TypeCollaborative: + """Link a published resource to a collaborative (only PUBLISHED linkable).""" + # Reject a missing or draft resource before touching the link. + publication = get_linkable_publication(publication_id) + + try: + collaborative = Collaborative.objects.get(id=collaborative_id) + except Collaborative.DoesNotExist: + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") + + # Only the collaborative's owner / org editors may change its links. + assert_can_manage_links(info.context.user, collaborative.user, collaborative.organization) + + if collaborative.status != CollaborativeStatus.DRAFT: + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") + + collaborative.publications.add(publication) + collaborative.save() + return TypeCollaborative.from_django(collaborative) + + @strawberry_django.mutation(handle_django_errors=True) + def remove_publication_from_collaborative( + self, info: Info, collaborative_id: str, publication_id: uuid.UUID + ) -> TypeCollaborative: + """Unlink a resource from a collaborative.""" + try: + collaborative = Collaborative.objects.get(id=collaborative_id) + except Collaborative.DoesNotExist: + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") + + # Only the collaborative's owner / org editors may change its links. + assert_can_manage_links(info.context.user, collaborative.user, collaborative.organization) + + if collaborative.status != CollaborativeStatus.DRAFT: + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") + + collaborative.publications.remove(publication_id) # type: ignore[arg-type] + collaborative.save() + return TypeCollaborative.from_django(collaborative) + + @strawberry_django.mutation(handle_django_errors=True) + @trace_resolver( + name="update_collaborative_publications", + attributes={"component": "collaborative", "operation": "mutation"}, + ) + def update_collaborative_publications( + self, info: Info, collaborative_id: str, publication_ids: List[uuid.UUID] + ) -> TypeCollaborative: + """Set the linked resources — only published ones are attached.""" + try: + collaborative = Collaborative.objects.get(id=collaborative_id) + except Collaborative.DoesNotExist: + raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") + + # Only the collaborative's owner / org editors may change its links. + assert_can_manage_links(info.context.user, collaborative.user, collaborative.organization) + + if collaborative.status != CollaborativeStatus.DRAFT: + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") + + # Attach only the published resources among the given ids. + collaborative.publications.set(published_publications(publication_ids)) + collaborative.save() + return TypeCollaborative.from_django(collaborative) + @strawberry_django.mutation(handle_django_errors=True) @trace_resolver( name="update_collaborative_use_cases", @@ -612,9 +622,7 @@ def update_collaborative_use_cases( raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.use_cases.set(use_cases) collaborative.save() @@ -636,9 +644,7 @@ def update_collaborative_use_cases( name="publish_collaborative", attributes={"component": "collaborative", "operation": "mutation"}, ) - def publish_collaborative( - self, info: Info, collaborative_id: str - ) -> TypeCollaborative: + def publish_collaborative(self, info: Info, collaborative_id: str) -> TypeCollaborative: """Publish a collaborative.""" try: collaborative = Collaborative.objects.get(id=collaborative_id) @@ -665,9 +671,7 @@ def publish_collaborative( name="unpublish_collaborative", attributes={"component": "collaborative", "operation": "mutation"}, ) - def unpublish_collaborative( - self, info: Info, collaborative_id: str - ) -> TypeCollaborative: + def unpublish_collaborative(self, info: Info, collaborative_id: str) -> TypeCollaborative: """Un-publish a collaborative.""" try: collaborative = Collaborative.objects.get(id=collaborative_id) @@ -696,14 +700,10 @@ def add_contributor_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.contributors.add(user) collaborative.save() @@ -727,14 +727,10 @@ def remove_contributor_from_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.contributors.remove(user) collaborative.save() @@ -749,9 +745,7 @@ def remove_contributor_from_collaborative( get_data=lambda result, collaborative_id, user_ids, **kwargs: { "collaborative_id": collaborative_id, "collaborative_title": result.title, - "updated_fields": { - "contributors": [str(user_id) for user_id in user_ids] - }, + "updated_fields": {"contributors": [str(user_id) for user_id in user_ids]}, }, ) ], @@ -771,9 +765,7 @@ def update_collaborative_contributors( raise ValueError(f"Collaborative with ID {collaborative_id} doesn't exist") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") collaborative.contributors.set(users) collaborative.save() @@ -797,22 +789,16 @@ def add_supporting_organization_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") # Create or get the relationship - relationship, created = ( - CollaborativeOrganizationRelationship.objects.get_or_create( - collaborative=collaborative, - organization=organization, - relationship_type=OrganizationRelationshipType.SUPPORTER, - ) + relationship, created = CollaborativeOrganizationRelationship.objects.get_or_create( + collaborative=collaborative, + organization=organization, + relationship_type=OrganizationRelationshipType.SUPPORTER, ) return TypeCollaborativeOrganizationRelationship.from_django(relationship) @@ -858,22 +844,16 @@ def add_partner_organization_to_collaborative( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") # Create or get the relationship - relationship, created = ( - CollaborativeOrganizationRelationship.objects.get_or_create( - collaborative=collaborative, - organization=organization, - relationship_type=OrganizationRelationshipType.PARTNER, - ) + relationship, created = CollaborativeOrganizationRelationship.objects.get_or_create( + collaborative=collaborative, + organization=organization, + relationship_type=OrganizationRelationshipType.PARTNER, ) return TypeCollaborativeOrganizationRelationship.from_django(relationship) @@ -937,19 +917,13 @@ def update_collaborative_organization_relationships( try: collaborative = Collaborative.objects.get(id=collaborative_id) except Collaborative.DoesNotExist: - raise ValueError( - f"Collaborative with ID {collaborative_id} does not exist." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} does not exist.") if collaborative.status != CollaborativeStatus.DRAFT: - raise ValueError( - f"Collaborative with ID {collaborative_id} is not in draft status." - ) + raise ValueError(f"Collaborative with ID {collaborative_id} is not in draft status.") # Clear existing relationships - CollaborativeOrganizationRelationship.objects.filter( - collaborative=collaborative - ).delete() + CollaborativeOrganizationRelationship.objects.filter(collaborative=collaborative).delete() # Add supporter organizations supporter_orgs = Organization.objects.filter(id__in=supporter_organization_ids) diff --git a/api/schema/publication_architecture.md b/api/schema/publication_architecture.md new file mode 100644 index 00000000..ba4a52ec --- /dev/null +++ b/api/schema/publication_architecture.md @@ -0,0 +1,116 @@ +# Publications (UI "Resource") — backend + +> Part of feature: **resources** · siblings: `DataSpaceFrontend/app/[locale]/(user)/publications/architecture.md` (frontend slice) + +## Overview + +A **Publication** is a new top-level entity, peer to Datasets and AI Models: a container for human-authored content (reports, research, findings). It has typed metadata, an ordered list of heterogeneous **content blocks** (a file XOR a YouTube embed each), a publish/unpublish toggle, its own listing + global search presence, and it can be pulled into Use Cases and Collaboratives. + +**Naming:** the user-facing entity is **"Resource"**, but the name `Resource` was already taken in this repo (the file-inside-a-dataset). So the entity is `Publication` everywhere in code — Django model, GraphQL type, table `publication`, ES index `publication`, SDK client, URL path `/publications`. The UI always renders "Resource". + +### How the repos connect + +- **DataExBackend** owns the whole data/API/search/SDK slice (this doc). +- **DataSpaceFrontend** (sibling doc) owns the create/edit/publish flow, detail page, listing, cards, and the Resource picker inside Use Case / Collaborative editors. It talks to the backend over GraphQL (`/api/graphql`) for CRUD/detail/list and REST for search (`/api/search/publication/`) and gated file download (`/api/publications/blocks//download/`). +- Request flow for a create: FE collects metadata → `createPublication` mutation → DRAFT row → FE adds blocks via `addPublicationFileBlock` / `addPublicationYoutubeBlock` (multipart for files) → `publishPublication`. A published resource then appears in the listing/search and is linkable from a Use Case / Collaborative. + +## Submodule map + +| Submodule | Trigger | +|---|---| +| CRUD + publish | GraphQL `publication_schema.py` (create/update/publish/unpublish/delete + list/detail) | +| Content blocks | GraphQL block mutations (add file/youtube, replace, remove, reorder) | +| Block-file download | REST `GET /api/publications/blocks//download/` | +| Search | REST `GET /api/search/publication/` + unified `GET /api/search/unified/` | +| Index sync | model signals (`publication_signals.py`) + `search_index --rebuild/--populate` | +| UC/Collab linking | GraphQL link trios in `usecase_schema.py` / `collaborative_schema.py` | +| SDK | `dataspace_sdk/resources/publications.py` | +| Resource Type lookup | Django admin + `seed_resource_types` command | + +--- + +## Submodule: CRUD + publish + +### Trigger +GraphQL: `createPublication`, `updatePublication`, `publishPublication`, `unpublishPublication`, `deletePublication` mutations; `getPublication(publicationId)` and `publications(filters, pagination, order, includePublic)` queries. Flow file: `api/schema/publication_schema.py`. + +### Business use case +Any individual or org account creates a Resource, edits its metadata across subpages, and publishes/unpublishes it with a simple toggle (no moderation). Auditors (role `can_change=False`) can read but not edit/publish. + +### Flow (English) +- **create:** reject missing/invalid metadata at the boundary → create a DRAFT owned by the org (from the request's organization header) or the user → wire sector/geography tags. +- **update:** load or 404 → apply only the provided fields (each validated) → save. +- **publish / unpublish:** load or 404 → flip `status`; the index signal adds/drops the search document. +- **delete:** load or 404 → hard delete (FK cascade drops blocks; M2M link rows auto-clear). +- **list:** scope to org / owner / anonymous, optionally union the public set, apply filters + ordering, enforce a bounded page window. +- **detail:** gated by `AllowPublishedPublications` — a published resource is world-readable; a draft only to owner/org/superuser. + +### Helpers +All Tier-2, in `api/services/publication_service.py`: +- `validate_publication_metadata(...) -> ResourceType` — required-field + typed validation (license in `DatasetLicense`, active resource type, URL shape); raises field-keyed `ValidationError`; returns the resolved active resource type. +- `create_publication(...) -> Publication` — create DRAFT + set M2M tags. +- `apply_publication_update(publication, ...) -> Publication` — partial update, validating each provided field; never blanks untouched columns. +- `set_publication_status(publication, status) -> Publication`. +- `get_scoped_publications(user, organization, include_public) -> QuerySet` — org/owner/anonymous scoping + published union, ordered `-modified`, `.distinct()`. +- `resolve_pagination(offset, limit) -> (offset, limit)` — default page size + hard max, so a listing is never unbounded. +- `is_publication_published(publication) -> bool`. + +Permissions (Tier-2, `authorization/permissions.py`): `CreatePublicationPermission`, `ChangePublicationPermission`, `DeletePublicationPermission`, `PublishPublicationPermission` (name-based admin/editor/owner), `AllowPublishedPublications`. All key on `publication_id` (or the update input's id, or a block's parent), keep the individual-owner branch, and drop Dataset's share-model fallback. + +### Data model +`Publication` (table `publication`): UUID PK, `title`, `description` (Text), `slug` (unique, counter-dedupe on collision), `organization`/`user` nullable FKs (`SET_NULL`), `authors` (JSON list), `publication_date` (Date), `license` (reuses `DatasetLicense` choices), `external_source_link` (URL), `status` (`PublicationStatus` DRAFT/PUBLISHED), `resource_type` FK (`PROTECT`), `sectors`/`geographies` M2M, `download_count`, `created`/`modified`. +`ResourceType` (table `resource_type`): UUID/name(unique)/slug + `is_active` (adapted from `Sector`, no parent self-FK). +`UseCase`/`Collaborative`: gain a `publications` M2M. + +### Indexing & performance +`Publication.Meta.indexes`: `(organization, -modified)`, `(user, -modified)`, `(status)` — matching the dashboard/listing query shapes. `slug` unique (detail lookup), org/user/resource_type FKs auto-indexed. No standalone org/user/resource_type indexes (composites/FK cover them). Sector/geography filters are ES-backed → no PG indexes. Pagination is server-enforced (default 20, max 100). `PublicationBlock.Meta.indexes`: `(publication, position)`. +**Migrations:** none committed — this repo auto-generates migrations at deploy (`docker-entrypoint.sh` runs `makemigrations --noinput` then `migrate`); the committed `0001_initial` is a stale stub and `authorization` has no migrations dir. All additions are additive (new tables + new nullable columns + additive M2M join tables), safe on large tables. See project-memory (2026-07-18). + +### Security +Every query is org/user-scoped; anonymous sees only PUBLISHED. Cross-org request outcomes: mutating another org's DRAFT → permission denied (the resolver 404s the draft at read); mutating a PUBLISHED one → permission denied. Auditor (`can_change=False`) → read allowed, edit/publish denied. Publish gated to role **names** admin/editor/owner (mirroring Dataset). Individual resources restricted to their owner. All scoping is centralized in the permission classes — no inline role logic in resolvers. +The **linked-project fields** (`linkedUsecases` / `linkedCollaboratives` / `linkedCount` on `TypePublication`) are the owner's "linked to N" flag: they're gated to the owner / org member / superuser (`_caller_can_see_links`) **and** filtered to PUBLISHED projects, so a public resource never leaks the title of a private draft project (possibly in another org) that references it. + +### SDK impact +SDK: **yes.** `dataspace_sdk/resources/publications.py` `PublicationClient` — `search` (REST), `get_by_id`/`list_all`/`get_organization_publications`/`create`/`update`/`delete` (GraphQL). Registered in `client.py` `__init__` and `set_organization`. + +### Tests + +#### Layer 1 — DB helper tests (`tests/test_publication_models.py`) +Slug dedupe (distinct slugs, unicode title); ownership property; ResourceType uniqueness/slug/`is_active`/active-query; block file/youtube storage; file-XOR-youtube CheckConstraint (both/neither rejected); block position ordering; delete cascade; seed idempotency; DRAFT/download_count/authors defaults. + +#### Layer 2 — Non-DB helper tests +`tests/test_youtube_url.py` — id extraction across watch/youtu.be/embed, rejects non-YouTube, malformed, and non-http(s) schemes (javascript: stored-XSS guard). `tests/test_publication_uploads.py` — extension allow-list, 50 MB cap, PDF magic-byte sniff. + +#### Layer 3 — Flow tests (in `tests/schema/test_publication_schema.py`, `tests/test_publication_blocks.py`) +Create org/individual → DRAFT; invalid metadata → no create; block add happy/invalid; reorder/renumber; replace-file (same id, old file gone). + +#### Layer 4 — Backend API e2e (`tests/schema/test_publication_schema.py`, `tests/test_publication_blocks.py`) +CRUD; role gating (editor edits, auditor denied edit/publish but reads); publish/unpublish; cross-org denial for update/delete/publish + block add/remove; anonymous sees published detail, denied draft; org-scoped and anonymous (published-only) listings; block-file download gate (published→200 + count increment, draft anon/cross-org→404, owner draft→200, PDF inline). + +#### Layer 4 — Search (`tests/test_publication_search.py`) +Index-decision (published indexed, draft not); re-index mapping incl. ResourceType/Sector/Geography/Org; signal predicate. Full ES query/filter/pagination behaviour runs against a live cluster (ES is disabled in the deterministic layers). + +#### Layer 4 — Linking (`tests/test_publication_linking.py`) +Only-published-linkable guard (UC + Collab); stale-link (unpublish/delete hides, re-publish restores); linked-count; IDOR guard (non-owner can't link); the intentional cross-org affordance (org B UC links org A published resource). + +#### Layer 5 — API user journey (`tests/journeys/publications/`) +`create-blocks-publish` and `link-unpublish-relink` (scripted, on-demand). + +#### Layer 6 — Browser e2e +n/a in this repo — belongs to the frontend slice's doc. + +### LLM-judge points +n/a — all assertions are deterministic. + +--- + +## Limitations & future work + +- **UC/Collab search does not index linked Resources.** A use case / collaborative is not findable by a linked Resource's title, and their `dataset_count` excludes Resources. Deferred by design. +- **Resource search matches name/description + the three facets only.** Author / date / usage-rights columns and block content are not indexed in v1. +- **Preview fidelity:** slide decks / DOCX / PPT are download-only (only PDF + YouTube render inline). +- **No versioning** for re-uploaded block files (in-place replace). +- **The dataset link trio's pre-existing IDOR** (no container-authorization check) is not fixed here — only the new publication trio is authorized. Follow-up: apply `assert_can_manage_links` to the dataset trio too. +- **Frontend is implemented** (see the sibling FE doc). Its Layer 6 browser flows and the visual checklist are written but not yet walked (agents don't run a visual pass on their own) — run on request. +- **The list resolver trusts the request's `organization` header without a membership check** — a faithful mirror of the existing dataset/aimodel/usecase list resolvers and the shared middleware (`api/utils/middleware.py`, which still carries a `# TODO: resolve auth_token`). This feature follows the platform pattern; if org membership is not enforced on that header upstream, a signed-in user could list another org's drafts by setting the header. The correct fix is at the shared middleware layer (so all entities are fixed together), not per-endpoint — deliberately **not** patched here to avoid divergence and false security. Flagged for a platform-level decision. +- **Frontend (Phases 7–8), the two Layer 5 journey scripts, and the FE architecture doc are not yet implemented** — the backend is complete and shippable; the UI is the remaining slice. diff --git a/api/schema/publication_schema.py b/api/schema/publication_schema.py new file mode 100644 index 00000000..6ac9148d --- /dev/null +++ b/api/schema/publication_schema.py @@ -0,0 +1,380 @@ +"""Schema definitions for publications (UI "Resource"). + +Docs: ./publication_architecture.md + +This is a flow file: each resolver reads as a short sequence of named helper +calls. The metadata validation, row writes, scoping and pagination bounds all +live in ``api/services/publication_service.py``; permission/role logic lives in +``authorization/permissions.py``. Nothing here reaches into the ORM or business +rules directly. +""" + +import datetime +import uuid +from typing import List, Optional + +import strawberry +import strawberry_django +from django.core.exceptions import ValidationError as DjangoValidationError +from strawberry.file_uploads import Upload +from strawberry.types import Info + +from api.models import Publication, PublicationBlock, ResourceType +from api.schema.base_mutation import BaseMutation, MutationResponse +from api.services.publication_blocks import ( + add_file_block, + add_youtube_block, + remove_block, + reorder_blocks, + replace_block_file, +) +from api.services.publication_service import ( + apply_publication_update, + create_publication, + get_scoped_publications, + resolve_pagination, + set_publication_status, + validate_publication_metadata, +) +from api.types.type_publication import ( + PublicationFilter, + PublicationOrder, + TypePublication, + TypePublicationBlock, + TypeResourceType, + publication_license, +) +from api.utils.enums import PublicationStatus +from api.utils.graphql_telemetry import trace_resolver +from authorization.permissions import ( + AllowPublishedPublications, + ChangePublicationPermission, + CreatePublicationPermission, + DeletePublicationPermission, + PublishPublicationPermission, +) + + +@strawberry.input +class CreatePublicationInput: + """Metadata for a new resource. All fields validated at the boundary.""" + + title: str + description: str + authors: List[str] + publication_date: datetime.date + license: publication_license + resource_type_id: uuid.UUID + sector_ids: List[uuid.UUID] + geography_ids: List[int] + external_source_link: Optional[str] = None + + +@strawberry.input +class UpdatePublicationInput: + """Partial edit to a resource — only the fields provided are touched.""" + + id: uuid.UUID + title: Optional[str] = None + description: Optional[str] = None + authors: Optional[List[str]] = None + publication_date: Optional[datetime.date] = None + license: Optional[publication_license] = None + resource_type_id: Optional[uuid.UUID] = None + sector_ids: Optional[List[uuid.UUID]] = None + geography_ids: Optional[List[int]] = None + external_source_link: Optional[str] = None + + +@strawberry.type(name="Query") +class Query: + """Queries for publications.""" + + @strawberry.field + @trace_resolver(name="get_resource_types", attributes={"component": "publication"}) + def resource_types(self, info: Info) -> List[TypeResourceType]: + """List the active Resource Types for a create/edit form's dropdown.""" + return TypeResourceType.from_django_list( + ResourceType.objects.filter(is_active=True).order_by("name") + ) + + @strawberry.field( + permission_classes=[AllowPublishedPublications], # type: ignore[list-item] + ) + @trace_resolver(name="get_publication", attributes={"component": "publication"}) + def get_publication(self, info: Info, publication_id: uuid.UUID) -> Optional[TypePublication]: + """Get a single resource by id (drafts gated to owner/org by the permission).""" + try: + return TypePublication.from_django(Publication.objects.get(id=publication_id)) + except Publication.DoesNotExist: + return None + + @strawberry.field + @trace_resolver(name="get_publications", attributes={"component": "publication"}) + def publications( + self, + info: Info, + filters: Optional[PublicationFilter] = strawberry.UNSET, + pagination: Optional[strawberry_django.pagination.OffsetPaginationInput] = strawberry.UNSET, + order: Optional[PublicationOrder] = strawberry.UNSET, + include_public: Optional[bool] = False, + ) -> List[TypePublication]: + """List resources, scoped to the caller and paginated with server limits.""" + user = info.context.user + organization = info.context.context.get("organization") + + # Scope to org / owner / anonymous and optionally union in the public set. + queryset = get_scoped_publications( + user=user, organization=organization, include_public=bool(include_public) + ) + + # Apply client filters and ordering, then enforce a bounded page window. + if filters is not strawberry.UNSET: + queryset = strawberry_django.filters.apply(filters, queryset, info) + if order is not strawberry.UNSET: + queryset = strawberry_django.ordering.apply(order, queryset, info) + + offset, limit = resolve_pagination( + getattr(pagination, "offset", None) if pagination is not strawberry.UNSET else None, + getattr(pagination, "limit", None) if pagination is not strawberry.UNSET else None, + ) + return TypePublication.from_django_list(queryset[offset : offset + limit]) + + +@strawberry.type(name="Mutation") +class Mutation: + """Mutations for publications.""" + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[CreatePublicationPermission], + trace_name="create_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "created", + "get_data": lambda result, **kwargs: {"publication_id": str(result.id)}, + }, + ) + def create_publication( + self, info: Info, input: CreatePublicationInput + ) -> MutationResponse[TypePublication]: + """Create a DRAFT resource owned by the caller's org or the user.""" + user = info.context.user + organization = info.context.context.get("organization") + + # Reject missing/invalid metadata before any row is written. + resource_type = validate_publication_metadata( + title=input.title, + description=input.description, + authors=input.authors, + publication_date=input.publication_date, + license_value=input.license.value if input.license else None, + resource_type_id=input.resource_type_id, + sector_ids=input.sector_ids, + geography_ids=input.geography_ids, + external_source_link=input.external_source_link, + ) + + # Create the draft and wire its sector/geography tags. + publication = create_publication( + user=user, + organization=organization, + title=input.title, + description=input.description, + authors=input.authors, + publication_date=input.publication_date, + license_value=input.license.value, + resource_type=resource_type, + sector_ids=input.sector_ids, + geography_ids=input.geography_ids, + external_source_link=input.external_source_link, + ) + return MutationResponse.success_response(TypePublication.from_django(publication)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="update_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "updated", + "get_data": lambda result, **kwargs: {"publication_id": str(result.id)}, + }, + ) + def update_publication( + self, info: Info, input: UpdatePublicationInput + ) -> MutationResponse[TypePublication]: + """Apply a partial metadata edit to an existing resource.""" + # Load the target, or surface a clean not-found. + publication = _get_publication_or_raise(input.id) + + # Update only the provided fields, validating each. + publication = apply_publication_update( + publication, + title=input.title, + description=input.description, + authors=input.authors, + publication_date=input.publication_date, + license_value=input.license.value if input.license else None, + resource_type_id=input.resource_type_id, + sector_ids=input.sector_ids, + geography_ids=input.geography_ids, + external_source_link=input.external_source_link, + ) + return MutationResponse.success_response(TypePublication.from_django(publication)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[PublishPublicationPermission], + trace_name="publish_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "published", + "get_data": lambda result, **kwargs: {"publication_id": str(result.id)}, + }, + ) + def publish_publication( + self, info: Info, publication_id: uuid.UUID + ) -> MutationResponse[TypePublication]: + """Flip a resource to PUBLISHED (self-serve, no moderation).""" + publication = _get_publication_or_raise(publication_id) + + # Mark it published — the index signal picks up the visibility change. + publication = set_publication_status(publication, PublicationStatus.PUBLISHED) + return MutationResponse.success_response(TypePublication.from_django(publication)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[PublishPublicationPermission], + trace_name="unpublish_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "unpublished", + "get_data": lambda result, **kwargs: {"publication_id": str(result.id)}, + }, + ) + def unpublish_publication( + self, info: Info, publication_id: uuid.UUID + ) -> MutationResponse[TypePublication]: + """Flip a resource back to DRAFT — hidden from public reads, links untouched.""" + publication = _get_publication_or_raise(publication_id) + + # Back to draft; the render-time filters do the hiding, no links change. + publication = set_publication_status(publication, PublicationStatus.DRAFT) + return MutationResponse.success_response(TypePublication.from_django(publication)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[DeletePublicationPermission], + trace_name="delete_publication", + trace_attributes={"component": "publication"}, + track_activity={ + "verb": "deleted", + "get_data": lambda result, **kwargs: { + "publication_id": str(kwargs.get("publication_id")), + "success": result, + }, + }, + ) + def delete_publication(self, info: Info, publication_id: uuid.UUID) -> MutationResponse[bool]: + """Hard-delete a resource — cascades its blocks, clears its links.""" + publication = _get_publication_or_raise(publication_id) + + # FK cascade drops blocks; M2M join rows auto-clear on delete. + publication.delete() + return MutationResponse.success_response(True) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="add_publication_file_block", + trace_attributes={"component": "publication"}, + ) + def add_publication_file_block( + self, info: Info, publication_id: uuid.UUID, file: Upload + ) -> MutationResponse[TypePublicationBlock]: + """Append an uploaded file as the next content block (validated server-side).""" + publication = _get_publication_or_raise(publication_id) + + # Validate + store the file as the last block. + block = add_file_block(publication, file) + return MutationResponse.success_response(TypePublicationBlock.from_django(block)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="add_publication_youtube_block", + trace_attributes={"component": "publication"}, + ) + def add_publication_youtube_block( + self, info: Info, publication_id: uuid.UUID, youtube_url: str + ) -> MutationResponse[TypePublicationBlock]: + """Append a YouTube link as the next content block (validated server-side).""" + publication = _get_publication_or_raise(publication_id) + + # Validate the url + extract its video id, then store the block. + block = add_youtube_block(publication, youtube_url) + return MutationResponse.success_response(TypePublicationBlock.from_django(block)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="replace_publication_block_file", + trace_attributes={"component": "publication"}, + ) + def replace_publication_block_file( + self, info: Info, block_id: uuid.UUID, file: Upload + ) -> MutationResponse[TypePublicationBlock]: + """Swap a file block's file, deleting the old one from disk.""" + block = _get_block_or_raise(block_id) + + # Replace in place — same block id, old file removed. + block = replace_block_file(block, file) + return MutationResponse.success_response(TypePublicationBlock.from_django(block)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="remove_publication_block", + trace_attributes={"component": "publication"}, + ) + def remove_publication_block(self, info: Info, block_id: uuid.UUID) -> MutationResponse[bool]: + """Remove a content block and renumber the rest contiguously.""" + block = _get_block_or_raise(block_id) + + # Delete + renumber siblings; the signal removes the file from disk. + remove_block(block) + return MutationResponse.success_response(True) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangePublicationPermission], + trace_name="reorder_publication_blocks", + trace_attributes={"component": "publication"}, + ) + def reorder_publication_blocks( + self, info: Info, publication_id: uuid.UUID, block_ids: List[uuid.UUID] + ) -> MutationResponse[TypePublication]: + """Set the content blocks' order to the given block-id sequence.""" + publication = _get_publication_or_raise(publication_id) + + # Reassign positions to match the requested order. + reorder_blocks(publication, block_ids) + publication.refresh_from_db() + return MutationResponse.success_response(TypePublication.from_django(publication)) + + +def _get_publication_or_raise(publication_id: uuid.UUID) -> Publication: + """Load a publication by id or raise a clean validation error.""" + try: + return Publication.objects.get(id=publication_id) + except Publication.DoesNotExist: + raise DjangoValidationError(f"Resource with id {publication_id} does not exist.") + + +def _get_block_or_raise(block_id: uuid.UUID) -> PublicationBlock: + """Load a content block by id or raise a clean validation error.""" + try: + return PublicationBlock.objects.get(id=block_id) + except PublicationBlock.DoesNotExist: + raise DjangoValidationError(f"Content block {block_id} does not exist.") diff --git a/api/schema/schema.py b/api/schema/schema.py index 70620ba9..4678b63d 100644 --- a/api/schema/schema.py +++ b/api/schema/schema.py @@ -16,6 +16,7 @@ import api.schema.metadata_schema import api.schema.organization_data_schema import api.schema.organization_schema +import api.schema.publication_schema import api.schema.resource_chart_schema import api.schema.resource_schema import api.schema.resoure_chart_image_schema @@ -75,6 +76,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.resoure_chart_image_schema.Query, api.schema.user_schema.Query, api.schema.collaborative_schema.Query, + api.schema.publication_schema.Query, AuthQuery, ), ) @@ -97,6 +99,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.resoure_chart_image_schema.Mutation, api.schema.tags_schema.Mutation, api.schema.collaborative_schema.Mutation, + api.schema.publication_schema.Mutation, AuthMutation, ), ) diff --git a/api/schema/usecase_schema.py b/api/schema/usecase_schema.py index e49b829b..e99fe9e4 100644 --- a/api/schema/usecase_schema.py +++ b/api/schema/usecase_schema.py @@ -28,6 +28,11 @@ UseCaseOrganizationRelationship, ) from api.schema.extensions import TrackActivity, TrackModelActivity +from api.services.publication_linking import ( + assert_can_manage_links, + get_linkable_publication, + published_publications, +) from api.types.type_dataset import TypeDataset from api.types.type_organization import TypeOrganization from api.types.type_usecase import TypeUseCase, UseCaseFilter, UseCaseOrder @@ -45,7 +50,7 @@ from authorization.types import TypeUser -@strawberry_django.input(UseCase, fields="__all__", exclude=["datasets", "slug"]) +@strawberry_django.input(UseCase, fields="__all__", exclude=["datasets", "publications", "slug"]) class UseCaseInput: """Input type for use case creation.""" @@ -71,7 +76,7 @@ class UpdateUseCaseMetadataInput: use_case_running_status = strawberry.enum(UseCaseRunningStatus) # type: ignore -@strawberry_django.partial(UseCase, fields="__all__", exclude=["datasets"]) +@strawberry_django.partial(UseCase, fields="__all__", exclude=["datasets", "publications"]) class UseCaseInputPartial: """Input type for use case updates.""" @@ -493,6 +498,74 @@ def update_usecase_datasets( use_case.save() return TypeUseCase.from_django(use_case) + @strawberry_django.mutation(handle_django_errors=True) + def add_publication_to_use_case( + self, info: Info, use_case_id: str, publication_id: uuid.UUID + ) -> TypeUseCase: + """Link a published resource to a use case (only PUBLISHED are linkable).""" + # Reject a missing or draft resource before touching the link. + publication = get_linkable_publication(publication_id) + + try: + use_case = UseCase.objects.get(id=use_case_id) + except UseCase.DoesNotExist: + raise ValueError(f"UseCase with ID {use_case_id} does not exist.") + + # Only the use case's owner / org editors may change its links. + assert_can_manage_links(info.context.user, use_case.user, use_case.organization) + + if use_case.status != UseCaseStatus.DRAFT: + raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") + + use_case.publications.add(publication) + use_case.save() + return TypeUseCase.from_django(use_case) + + @strawberry_django.mutation(handle_django_errors=True) + def remove_publication_from_use_case( + self, info: Info, use_case_id: str, publication_id: uuid.UUID + ) -> TypeUseCase: + """Unlink a resource from a use case.""" + try: + use_case = UseCase.objects.get(id=use_case_id) + except UseCase.DoesNotExist: + raise ValueError(f"UseCase with ID {use_case_id} does not exist.") + + # Only the use case's owner / org editors may change its links. + assert_can_manage_links(info.context.user, use_case.user, use_case.organization) + + if use_case.status != UseCaseStatus.DRAFT: + raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") + + use_case.publications.remove(publication_id) # type: ignore[arg-type] + use_case.save() + return TypeUseCase.from_django(use_case) + + @strawberry_django.mutation(handle_django_errors=True) + @trace_resolver( + name="update_usecase_publications", + attributes={"component": "usecase", "operation": "mutation"}, + ) + def update_usecase_publications( + self, info: Info, use_case_id: str, publication_ids: List[uuid.UUID] + ) -> TypeUseCase: + """Set the linked resources — only published ones are attached.""" + try: + use_case = UseCase.objects.get(id=use_case_id) + except UseCase.DoesNotExist: + raise ValueError(f"Use Case with ID {use_case_id} doesn't exist") + + # Only the use case's owner / org editors may change its links. + assert_can_manage_links(info.context.user, use_case.user, use_case.organization) + + if use_case.status != UseCaseStatus.DRAFT: + raise ValueError(f"UseCase with ID {use_case_id} is not in draft status.") + + # Attach only the published resources among the given ids. + use_case.publications.set(published_publications(publication_ids)) + use_case.save() + return TypeUseCase.from_django(use_case) + @strawberry_django.mutation( handle_django_errors=True, extensions=[ diff --git a/api/services/publication_blocks.py b/api/services/publication_blocks.py new file mode 100644 index 00000000..36f2a78c --- /dev/null +++ b/api/services/publication_blocks.py @@ -0,0 +1,153 @@ +""" +publication_blocks +────────────────── +The messy 90% behind a Resource's content blocks: adding a file or YouTube +block at the next position, replacing a block's file without leaking the old +one, removing a block and renumbering the rest contiguously, reordering, and +the read-time access gate for a block's file. + +Every write keeps ``position`` a dense 0..n-1 sequence, and every file swap or +removal deletes the previous bytes from disk so drafts don't leak files. +""" + +from typing import Any, List + +from django.core.exceptions import ValidationError +from django.db import transaction + +from api.models import Publication, PublicationBlock +from api.services.publication_service import is_publication_published +from api.utils.enums import PublicationBlockType +from api.utils.publication_uploads import validate_publication_file +from api.utils.youtube import validate_youtube_url + + +def add_file_block(publication: Publication, uploaded_file: Any) -> PublicationBlock: + """Validate an uploaded file and append it as the next content block.""" + extension, size = validate_publication_file(uploaded_file) + + block = PublicationBlock( + publication=publication, + position=_next_position(publication), + block_type=PublicationBlockType.FILE, + file_name=getattr(uploaded_file, "name", ""), + file_format=extension.lstrip("."), + file_size=size, + ) + block.file = uploaded_file + block.save() + return block + + +def add_youtube_block(publication: Publication, youtube_url: str) -> PublicationBlock: + """Validate a YouTube url and append it as the next content block.""" + video_id = validate_youtube_url(youtube_url) + + return PublicationBlock.objects.create( + publication=publication, + position=_next_position(publication), + block_type=PublicationBlockType.YOUTUBE, + youtube_url=youtube_url, + youtube_video_id=video_id, + ) + + +def replace_block_file(block: PublicationBlock, uploaded_file: Any) -> PublicationBlock: + """Swap a file block's file for a new one, deleting the old bytes from disk. + + Django's FileField never removes the previous file on reassignment, so we + delete it explicitly first — a re-upload updates in place (same row id) and + doesn't leak the old file. + """ + if block.block_type != PublicationBlockType.FILE: + raise ValidationError("Only a file block's file can be replaced.") + + extension, size = validate_publication_file(uploaded_file) + + # Drop the previous file from storage before attaching the new one. + if block.file: + block.file.delete(save=False) + + block.file = uploaded_file + block.file_name = getattr(uploaded_file, "name", "") + block.file_format = extension.lstrip(".") + block.file_size = size + block.save() + return block + + +def remove_block(block: PublicationBlock) -> None: + """Delete a block and renumber its siblings so positions stay contiguous.""" + publication = block.publication + with transaction.atomic(): + block.delete() + _renumber_blocks(publication) + + +def reorder_blocks( + publication: Publication, ordered_block_ids: List[Any] +) -> List[PublicationBlock]: + """Set block positions to match the given id order (0-based, contiguous).""" + existing: List[PublicationBlock] = list( + PublicationBlock.objects.filter(publication=publication) + ) + blocks_by_id = {block.id: block for block in existing} + if set(blocks_by_id.keys()) != {_coerce_id(bid, blocks_by_id) for bid in ordered_block_ids}: + raise ValidationError("Reorder must list every block exactly once.") + + reordered: List[PublicationBlock] = [] + for position, block_id in enumerate(ordered_block_ids): + block = blocks_by_id[_coerce_id(block_id, blocks_by_id)] + block.position = position + reordered.append(block) + + PublicationBlock.objects.bulk_update(reordered, ["position"]) + return reordered + + +def can_access_block_file(user: Any, publication: Publication) -> bool: + """Whether a caller may download a block's file. + + A PUBLISHED resource's files are world-readable; a DRAFT's files are private + to the owner, org members, and superusers — never reachable anonymously. + """ + if is_publication_published(publication): + return True + if not getattr(user, "is_authenticated", False): + return False + if user.is_superuser: + return True + if publication.user and publication.user == user: + return True + if publication.organization: + from authorization.models import OrganizationMembership + + return OrganizationMembership.objects.filter( + user=user, organization=publication.organization + ).exists() + return False + + +def _next_position(publication: Publication) -> int: + """The position for a new block appended to the end.""" + return publication.blocks.count() + + +def _renumber_blocks(publication: Publication) -> None: + """Rewrite positions to a dense 0..n-1 sequence in current order.""" + blocks: List[PublicationBlock] = list( + PublicationBlock.objects.filter(publication=publication).order_by("position") + ) + for position, block in enumerate(blocks): + block.position = position + PublicationBlock.objects.bulk_update(blocks, ["position"]) + + +def _coerce_id(block_id: Any, blocks_by_id: dict) -> Any: + """Match an incoming id (possibly a string) to a stored block key.""" + if block_id in blocks_by_id: + return block_id + for key in blocks_by_id: + if str(key) == str(block_id): + return key + return block_id diff --git a/api/services/publication_linking.py b/api/services/publication_linking.py new file mode 100644 index 00000000..bc668167 --- /dev/null +++ b/api/services/publication_linking.py @@ -0,0 +1,64 @@ +""" +publication_linking +──────────────────── +Helpers for pulling published Resources into Use Cases and Collaboratives. + +Resources are the first non-Dataset entity that can be linked, and only a +PUBLISHED resource may be linked — like a public library. These guard the link +mutations and the published-only render filter so a draft can never sneak into a +Use Case / Collaborative, and an unpublished resource silently drops out of the +render. +""" + +from typing import Any, List + +from api.models import Publication +from api.utils.enums import PublicationStatus + + +def get_linkable_publication(publication_id: Any) -> Publication: + """Load a resource that is allowed to be linked, or raise. + + Only a PUBLISHED resource is linkable; a missing or draft one raises a clean + error so the plain link mutation never attaches a draft. + """ + try: + publication = Publication.objects.get(id=publication_id) + except Publication.DoesNotExist: + raise ValueError(f"Resource {publication_id} does not exist.") + if publication.status != PublicationStatus.PUBLISHED.value: + raise ValueError("Only a published resource can be linked.") + return publication + + +def published_publications(publication_ids: List[Any]) -> List[Publication]: + """Return the published resources among the given ids (drafts dropped).""" + return list( + Publication.objects.filter(id__in=publication_ids, status=PublicationStatus.PUBLISHED) + ) + + +def assert_can_manage_links(user: Any, owner_user: Any, organization: Any) -> None: + """Raise unless the caller may edit this Use Case / Collaborative's links. + + Editing links changes the container, so it needs the container's own + authorization: its owner, or an org member with the change role (superusers + always). This is independent of the linked resource, so the intentional + cross-org affordance — org B's use case linking org A's published resource — + still works: the caller is authorized on *their own* use case. + """ + if getattr(user, "is_superuser", False): + return + if not getattr(user, "is_authenticated", False): + raise ValueError("Authentication required.") + if owner_user and owner_user == user: + return + if organization: + from authorization.models import OrganizationMembership + + membership = OrganizationMembership.objects.filter( + user=user, organization=organization + ).first() + if membership and membership.role.can_change: + return + raise ValueError("You don't have permission to modify this.") diff --git a/api/services/publication_service.py b/api/services/publication_service.py new file mode 100644 index 00000000..6084130f --- /dev/null +++ b/api/services/publication_service.py @@ -0,0 +1,271 @@ +""" +publication_service +──────────────────── +Domain helpers for the Publication ("Resource") CRUD flow — the messy 90% the +``publication_schema`` flow file delegates to. Each function does one thing: +validate the metadata at the input boundary, create/update the row, flip its +publish status, or return the correctly-scoped queryset for a listing. + +These never talk to GraphQL types or permissions — they take plain values and +model instances, so they're unit-testable on their own. +""" + +from typing import Any, List, Optional + +from django.core.exceptions import ValidationError as DjangoValidationError +from django.core.validators import URLValidator +from django.db.models import QuerySet + +from api.models import Geography, Publication, ResourceType, Sector +from api.utils.enums import DatasetLicense, PublicationStatus + +# Default page size + hard ceiling for a publications listing, enforced even +# when the caller sends no pagination input. +DEFAULT_PAGE_SIZE = 20 +MAX_PAGE_SIZE = 100 + + +def validate_publication_metadata( + *, + title: Optional[str], + description: Optional[str], + authors: Optional[List[str]], + publication_date: Any, + license_value: Optional[str], + resource_type_id: Any, + sector_ids: Optional[List[Any]], + geography_ids: Optional[List[Any]], + external_source_link: Optional[str], +) -> ResourceType: + """Validate a resource's metadata at the create boundary. + + Enforces the required fields (title, description, authors, publication_date, + license, an active resource type, at least one sector and one geography), + the controlled license vocabulary, and the optional external link's URL + shape. Raises a field-keyed ``ValidationError`` on any problem and returns + the resolved active ``ResourceType`` on success. + """ + errors: dict[str, List[str]] = {} + + if not title or not title.strip(): + errors["title"] = ["Title is required."] + if not description or not description.strip(): + errors["description"] = ["Description is required."] + if not authors or not [a for a in authors if a and a.strip()]: + errors["authors"] = ["At least one author is required."] + if publication_date is None: + errors["publication_date"] = ["Publication date is required."] + if not sector_ids: + errors["sectors"] = ["At least one sector is required."] + if not geography_ids: + errors["geographies"] = ["At least one geography is required."] + + if not license_value: + errors["license"] = ["License is required."] + elif license_value not in DatasetLicense.values: + errors["license"] = ["Not a valid license."] + + if external_source_link: + try: + URLValidator()(external_source_link) + except DjangoValidationError: + errors["external_source_link"] = ["Enter a valid URL."] + + resource_type = _resolve_active_resource_type(resource_type_id, errors) + + if errors: + raise DjangoValidationError(errors) + + return resource_type # type: ignore[return-value] + + +def _resolve_active_resource_type( + resource_type_id: Any, errors: dict[str, List[str]] +) -> Optional[ResourceType]: + """Load the resource type and require it to exist and be active.""" + if not resource_type_id: + errors["resource_type"] = ["Resource type is required."] + return None + try: + resource_type = ResourceType.objects.get(id=resource_type_id) + except ResourceType.DoesNotExist: + errors["resource_type"] = ["Resource type does not exist."] + return None + if not resource_type.is_active: + errors["resource_type"] = ["Resource type is not active."] + return None + return resource_type + + +def create_publication( + *, + user: Any, + organization: Any, + title: str, + description: Optional[str], + authors: List[str], + publication_date: Any, + license_value: str, + resource_type: ResourceType, + sector_ids: List[Any], + geography_ids: List[Any], + external_source_link: Optional[str], +) -> Publication: + """Create a DRAFT publication from validated metadata and wire its M2M tags. + + Ownership follows the caller's context: an organization present in the + request makes it org-owned, otherwise it's the individual user's. + """ + publication = Publication.objects.create( + title=title, + description=description, + authors=authors, + publication_date=publication_date, + license=license_value, + resource_type=resource_type, + external_source_link=external_source_link or None, + organization=organization, + user=user, + status=PublicationStatus.DRAFT, + ) + _set_publication_tags(publication, sector_ids, geography_ids) + return publication + + +def apply_publication_update( + publication: Publication, + *, + title: Optional[str] = None, + description: Optional[str] = None, + authors: Optional[List[str]] = None, + publication_date: Any = None, + license_value: Optional[str] = None, + resource_type_id: Any = None, + sector_ids: Optional[List[Any]] = None, + geography_ids: Optional[List[Any]] = None, + external_source_link: Optional[str] = None, +) -> Publication: + """Apply a partial metadata update, validating each field that's provided. + + Only fields passed in are touched, so a subpage save never blanks columns it + didn't show. A provided license must be in the controlled list; a provided + resource type must be active; a provided link must be a valid URL. + """ + errors: dict[str, List[str]] = {} + + # A provided required field must not be explicitly blanked. + if title is not None: + if not title.strip(): + errors["title"] = ["Title cannot be empty."] + else: + publication.title = title + if description is not None: + if not description.strip(): + errors["description"] = ["Description cannot be empty."] + else: + publication.description = description + if authors is not None: + if not [a for a in authors if a and a.strip()]: + errors["authors"] = ["At least one author is required."] + else: + publication.authors = authors + if publication_date is not None: + publication.publication_date = publication_date + if external_source_link is not None: + if external_source_link: + try: + URLValidator()(external_source_link) + publication.external_source_link = external_source_link + except DjangoValidationError: + errors["external_source_link"] = ["Enter a valid URL."] + else: + publication.external_source_link = None + + if license_value is not None: + if license_value in DatasetLicense.values: + publication.license = license_value + else: + errors["license"] = ["Not a valid license."] + + if resource_type_id is not None: + resource_type = _resolve_active_resource_type(resource_type_id, errors) + if resource_type is not None: + publication.resource_type = resource_type + + if errors: + raise DjangoValidationError(errors) + + publication.save() + if sector_ids is not None or geography_ids is not None: + _set_publication_tags(publication, sector_ids, geography_ids) + return publication + + +def set_publication_status(publication: Publication, status: PublicationStatus) -> Publication: + """Flip a publication's publish status and save it.""" + publication.status = status + publication.save() + return publication + + +def get_scoped_publications( + *, user: Any, organization: Any, include_public: bool +) -> "QuerySet[Publication, Publication]": + """Return the publications a caller may list, correctly scoped. + + Organization context → that org's publications; an authenticated individual + → their own; anonymous → published only. ``include_public`` unions in the + published set so a signed-in user also sees the public listing. Ordered + newest-first and de-duplicated after the union. + """ + if organization: + queryset = Publication.objects.filter(organization=organization) + elif getattr(user, "is_superuser", False): + queryset = Publication.objects.all() + elif getattr(user, "is_authenticated", False): + queryset = Publication.objects.filter(user=user, organization__isnull=True) + else: + queryset = Publication.objects.filter(status=PublicationStatus.PUBLISHED) + + if include_public: + queryset = queryset | Publication.objects.filter(status=PublicationStatus.PUBLISHED) + + # Prefetch the relations the listing/card resolvers touch so a page of N + # resources stays a bounded number of queries, not one-per-row. + return ( + queryset.select_related("resource_type", "organization", "user") + .prefetch_related("sectors", "geographies", "blocks") + .order_by("-modified") + .distinct() + ) + + +def is_publication_published(publication: Publication) -> bool: + """True only when the publication is PUBLISHED.""" + return publication.status == PublicationStatus.PUBLISHED.value + + +def resolve_pagination(offset: Optional[int], limit: Optional[int]) -> tuple[int, int]: + """Turn a caller's optional page window into a bounded (offset, limit). + + A missing limit falls back to the default page size; any limit is capped at + the hard maximum, so a listing is never unbounded even with no input. + """ + safe_offset = max(offset or 0, 0) + if not limit or limit <= 0: + safe_limit = DEFAULT_PAGE_SIZE + else: + safe_limit = min(limit, MAX_PAGE_SIZE) + return safe_offset, safe_limit + + +def _set_publication_tags( + publication: Publication, + sector_ids: Optional[List[Any]], + geography_ids: Optional[List[Any]], +) -> None: + """Replace a publication's sector and geography tags from id lists.""" + if sector_ids is not None: + publication.sectors.set(Sector.objects.filter(id__in=sector_ids)) + if geography_ids is not None: + publication.geographies.set(Geography.objects.filter(id__in=geography_ids)) diff --git a/api/signals/__init__.py b/api/signals/__init__.py index 29e37986..542aabaf 100644 --- a/api/signals/__init__.py +++ b/api/signals/__init__.py @@ -1,2 +1,7 @@ # Import signals to register them -from api.signals import aimodel_signals, dataset_signals, usecase_signals +from api.signals import ( + aimodel_signals, + dataset_signals, + publication_signals, + usecase_signals, +) diff --git a/api/signals/publication_signals.py b/api/signals/publication_signals.py new file mode 100644 index 00000000..7ae23833 --- /dev/null +++ b/api/signals/publication_signals.py @@ -0,0 +1,107 @@ +""" +publication_signals +──────────────────── +Two jobs. First, keep a Resource's stored files from leaking: when a content +block is deleted — directly, or by the FK cascade when its parent publication is +deleted — its file is removed from disk. Second, keep search honest: publishing +a resource adds it to the index, unpublishing or deleting it removes it, so a +draft is never searchable. (Django's ORM never deletes a file on its own, so +``Resource`` today leaks files on delete; we intentionally don't repeat that.) +""" + +from typing import Any + +import structlog +from django.db.models.signals import post_delete, pre_save +from django.dispatch import receiver + +from api.models import Publication, PublicationBlock +from api.utils.enums import PublicationStatus +from search.documents import PublicationDocument + +logger = structlog.getLogger(__name__) + + +@receiver(post_delete, sender=PublicationBlock) +def remove_publication_block_file(sender: Any, instance: PublicationBlock, **kwargs: Any) -> None: + """Delete a removed block's file from storage.""" + if instance.file: + instance.file.delete(save=False) + logger.info("publication block file removed", block_id=str(instance.id)) + + +def _should_be_indexed(instance: Publication) -> bool: + """A resource belongs in search only while it is PUBLISHED.""" + return instance.status == PublicationStatus.PUBLISHED.value + + +@receiver(pre_save, sender=Publication) +def handle_publication_visibility(sender: Any, instance: Publication, **kwargs: Any) -> None: + """Add / refresh / drop the search document as a resource is published or not. + + New rows are handled by the django-elasticsearch-dsl signal processor. ES + errors are logged and swallowed — indexing is best-effort and a rebuild + reconciles it. + """ + if not instance.pk: + return + + try: + original = Publication.objects.get(pk=instance.pk) + except Publication.DoesNotExist: + return + + was_indexable = _should_be_indexed(original) + is_indexable = _should_be_indexed(instance) + + if was_indexable and is_indexable: + action = "update" + elif was_indexable and not is_indexable: + action = "delete" + elif not was_indexable and is_indexable: + action = "add" + else: + return + + try: + document = PublicationDocument.get(id=instance.id, ignore=404) + if action == "delete": + if document: + document.delete() + logger.info("resource removed from search index", publication_id=str(instance.id)) + else: + if document: + document.update(instance) + else: + PublicationDocument().update(instance) + logger.info( + "resource synced to search index", + publication_id=str(instance.id), + action=action, + ) + except Exception as exc: # pragma: no cover - logging only + logger.error( + "failed to sync resource search document", + publication_id=str(instance.id), + action=action, + error=str(exc), + ) + + +@receiver(post_delete, sender=Publication) +def remove_publication_document(sender: Any, instance: Publication, **kwargs: Any) -> None: + """Drop the search document when a resource is deleted.""" + try: + document = PublicationDocument.get(id=instance.id, ignore=404) + if document: + document.delete() + logger.info( + "deleted resource removed from search index", + publication_id=str(instance.id), + ) + except Exception as exc: # pragma: no cover - logging only + logger.error( + "failed to delete resource search document", + publication_id=str(instance.id), + error=str(exc), + ) diff --git a/api/types/type_collaborative.py b/api/types/type_collaborative.py index 11805ad4..5fa74803 100644 --- a/api/types/type_collaborative.py +++ b/api/types/type_collaborative.py @@ -20,10 +20,15 @@ from api.types.type_dataset import TypeDataset, TypeTag from api.types.type_geo import TypeGeo from api.types.type_organization import TypeOrganization +from api.types.type_publication import TypePublication from api.types.type_sdg import TypeSDG from api.types.type_sector import TypeSector from api.types.type_usecase import TypeUseCase -from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType +from api.utils.enums import ( + CollaborativeStatus, + OrganizationRelationshipType, + PublicationStatus, +) from authorization.types import TypeUser collaborative_status = strawberry.enum(CollaborativeStatus) # type: ignore @@ -95,6 +100,18 @@ def datasets(self) -> Optional[List["TypeDataset"]]: except Exception: return [] + @strawberry.field(description="Get published resources linked to this collaborative.") + def publications(self) -> Optional[List["TypePublication"]]: + """Get published resources linked to this collaborative (drafts hidden).""" + try: + # Only published resources render — an unpublished one drops out. + queryset = self.publications.filter( # type: ignore + status=PublicationStatus.PUBLISHED + ).order_by("-modified") + return TypePublication.from_django_list(queryset) + except Exception: + return [] + @strawberry.field(description="Get use cases associated with this collaborative.") def use_cases(self) -> Optional[List["TypeUseCase"]]: """Get use cases associated with this collaborative.""" diff --git a/api/types/type_publication.py b/api/types/type_publication.py new file mode 100644 index 00000000..01a415c4 --- /dev/null +++ b/api/types/type_publication.py @@ -0,0 +1,235 @@ +import uuid +from datetime import date, datetime +from typing import List, Optional, cast + +import strawberry +import strawberry_django +from strawberry.enum import EnumType +from strawberry.types import Info + +from api.models import ( + Collaborative, + Publication, + PublicationBlock, + ResourceType, + UseCase, +) +from api.types.base_type import BaseType +from api.types.type_geo import TypeGeo +from api.types.type_organization import TypeOrganization +from api.types.type_sector import TypeSector +from api.utils.enums import ( + CollaborativeStatus, + DatasetLicense, + PublicationBlockType, + PublicationStatus, + UseCaseStatus, +) +from authorization.types import TypeUser + + +def _caller_can_see_links(info: Info, publication: Publication) -> bool: + """Whether the caller may see where a resource is linked (owner / org / superuser). + + The 'linked to N' flag is the owner's view; outsiders (including anonymous + visitors to a public resource) must not learn which projects reference it. + """ + user = getattr(info.context, "user", None) + if not user or not getattr(user, "is_authenticated", False): + return False + if user.is_superuser: + return True + if publication.user and publication.user == user: + return True + if publication.organization: + from authorization.models import OrganizationMembership + + return OrganizationMembership.objects.filter( + user=user, organization=publication.organization + ).exists() + return False + + +# Fields are enumerated on every type below — never ``fields="__all__"`` — so a +# future column is never silently published. +publication_status: EnumType = strawberry.enum(PublicationStatus) # type: ignore +publication_block_type: EnumType = strawberry.enum(PublicationBlockType) # type: ignore +publication_license: EnumType = strawberry.enum(DatasetLicense) # type: ignore + + +@strawberry.type +class TypeLinkedReference: + """A lightweight pointer to a Use Case / Collaborative a resource is linked into. + + Kept minimal (id / title / slug) so ``TypePublication`` can name where it's + linked without importing the heavier Use Case / Collaborative types. + """ + + id: str + title: str + slug: str + + +@strawberry_django.type(ResourceType) +class TypeResourceType(BaseType): + """Type for the admin-managed Resource Type lookup.""" + + id: uuid.UUID + name: str + slug: Optional[str] + is_active: bool + + +@strawberry_django.type(PublicationBlock) +class TypePublicationBlock(BaseType): + """Type for one content block (a file XOR a YouTube embed).""" + + id: uuid.UUID + position: int + block_type: publication_block_type + file_name: str + file_format: str + file_size: Optional[int] + youtube_url: Optional[str] + youtube_video_id: str + + +@strawberry_django.filter(Publication) +class PublicationFilter: + """Filter for publications.""" + + id: Optional[uuid.UUID] + status: Optional[publication_status] + resource_type: Optional[uuid.UUID] + + +@strawberry_django.order(Publication) +class PublicationOrder: + """Order for publications.""" + + title: strawberry.auto + created: strawberry.auto + modified: strawberry.auto + + +@strawberry_django.type( + Publication, + filters=PublicationFilter, + pagination=True, + order=PublicationOrder, # type: ignore +) +class TypePublication(BaseType): + """Type for a Publication (UI 'Resource').""" + + id: uuid.UUID + title: str + description: Optional[str] + slug: str + status: publication_status + authors: List[str] + publication_date: Optional[date] + license: publication_license + external_source_link: Optional[str] + download_count: int + created: datetime + modified: datetime + organization: Optional["TypeOrganization"] + user: Optional["TypeUser"] + resource_type: Optional["TypeResourceType"] + + @strawberry.field + def sectors(self, info: Info) -> List["TypeSector"]: + """Sectors tagged on this resource.""" + try: + instance = cast(Publication, self) + return TypeSector.from_django_list(instance.sectors.all()) + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def geographies(self, info: Info) -> List["TypeGeo"]: + """Geographies tagged on this resource.""" + try: + instance = cast(Publication, self) + return TypeGeo.from_django_list(instance.geographies.all()) + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def blocks(self, info: Info) -> List["TypePublicationBlock"]: + """Ordered content blocks of this resource. + + ``PublicationBlock`` orders by ``position`` in its Meta, so ``.all()`` is + already position-ordered and reuses the listing's prefetch cache — an + explicit ``.order_by`` here would re-query and reintroduce an N+1. + """ + try: + instance = cast(Publication, self) + return TypePublicationBlock.from_django_list(instance.blocks.all()) + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def is_individual_publication(self) -> bool: + """True when owned by an individual rather than an organization.""" + return self.organization is None + + @strawberry.field + def linked_usecases(self, info: Info) -> List["TypeLinkedReference"]: + """Use Cases this resource is linked into — the owner's 'linked to N' flag. + + Only the owner / org members may see this, and only published projects + are named, so a private draft (possibly in another org that linked this + public resource) never leaks its title through here. + """ + try: + instance = cast(Publication, self) + if not _caller_can_see_links(info, instance): + return [] + usecases: List[UseCase] = list( + UseCase.objects.filter(publications=instance, status=UseCaseStatus.PUBLISHED) + ) + return [ + TypeLinkedReference(id=str(uc.id), title=uc.title or "", slug=uc.slug or "") + for uc in usecases + ] + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def linked_collaboratives(self, info: Info) -> List["TypeLinkedReference"]: + """Collaboratives this resource is linked into — the owner's 'linked to N' flag.""" + try: + instance = cast(Publication, self) + if not _caller_can_see_links(info, instance): + return [] + collabs: List[Collaborative] = list( + Collaborative.objects.filter( + publications=instance, status=CollaborativeStatus.PUBLISHED + ) + ) + return [ + TypeLinkedReference( + id=str(collab.id), title=collab.title or "", slug=collab.slug or "" + ) + for collab in collabs + ] + except (AttributeError, Publication.DoesNotExist): + return [] + + @strawberry.field + def linked_count(self, info: Info) -> int: + """Published Use Cases + Collaboratives this resource is linked into (owner only).""" + try: + instance = cast(Publication, self) + if not _caller_can_see_links(info, instance): + return 0 + usecases = UseCase.objects.filter( + publications=instance, status=UseCaseStatus.PUBLISHED + ).count() + collabs = Collaborative.objects.filter( + publications=instance, status=CollaborativeStatus.PUBLISHED + ).count() + return usecases + collabs + except (AttributeError, Publication.DoesNotExist): + return 0 diff --git a/api/types/type_usecase.py b/api/types/type_usecase.py index 821379ea..48d16400 100644 --- a/api/types/type_usecase.py +++ b/api/types/type_usecase.py @@ -17,12 +17,17 @@ from api.types.type_dataset import TypeDataset, TypeTag from api.types.type_geo import TypeGeo from api.types.type_organization import TypeOrganization +from api.types.type_publication import TypePublication from api.types.type_sdg import TypeSDG from api.types.type_sector import TypeSector from api.types.type_usecase_dashboard import TypeUseCaseDashboard from api.types.type_usecase_metadata import TypeUseCaseMetadata from api.types.type_usecase_organization import TypeUseCaseOrganizationRelationship -from api.utils.enums import OrganizationRelationshipType, UseCaseStatus +from api.utils.enums import ( + OrganizationRelationshipType, + PublicationStatus, + UseCaseStatus, +) from authorization.types import TypeUser use_case_status = strawberry.enum(UseCaseStatus) # type: ignore @@ -93,6 +98,16 @@ def dataset_count(self: "TypeUseCase", info: Info) -> int: except Exception: return 0 + @strawberry.field(description="Get published resources linked to this use case.") + def publications(self) -> Optional[List["TypePublication"]]: + """Get published resources linked to this use case (drafts hidden).""" + try: + # Only published resources render — an unpublished one drops out. + queryset = self.publications.filter(status=PublicationStatus.PUBLISHED) # type: ignore + return TypePublication.from_django_list(queryset) + except Exception: + return [] + @strawberry.field(description="Get publishers associated with this use case.") def publishers(self) -> Optional[List["TypeOrganization"]]: """Get publishers associated with this use case.""" diff --git a/api/urls.py b/api/urls.py index 6d67c407..d339e452 100644 --- a/api/urls.py +++ b/api/urls.py @@ -13,9 +13,11 @@ auth, download, generate_dynamic_chart, + publication_download_view, search_aimodel, search_collaborative, search_dataset, + search_publication, search_publisher, search_unified, search_usecase, @@ -50,6 +52,11 @@ path("search/dataset/", search_dataset.SearchDataset.as_view(), name="search_dataset"), path("search/usecase/", search_usecase.SearchUseCase.as_view(), name="search_usecase"), path("search/aimodel/", search_aimodel.SearchAIModel.as_view(), name="search_aimodel"), + path( + "search/publication/", + search_publication.SearchPublication.as_view(), + name="search_publication", + ), path( "search/collaborative/", search_collaborative.SearchCollaborative.as_view(), @@ -92,6 +99,11 @@ r"download/(?Presource|access_resource|chart|chart_image)/(?P[0-9a-f]{8}\-[0-9a-f]{4}\-4[0-9a-f]{3}\-[89ab][0-9a-f]{3}\-[0-9a-f]{12})", download, ), + path( + "publications/blocks//download/", + publication_download_view.publication_block_download, + name="publication_block_download", + ), re_path( # type: ignore r"generate-dynamic-chart/(?P[0-9a-f]{8}\-[0-9a-f]{4}\-4[0-9a-f]{3}\-[89ab][0-9a-f]{3}\-[0-9a-f]{12})", generate_dynamic_chart, diff --git a/api/utils/data_indexing.py b/api/utils/data_indexing.py index 76459620..e93ac56f 100644 --- a/api/utils/data_indexing.py +++ b/api/utils/data_indexing.py @@ -15,7 +15,6 @@ # Use a separate database for data tables DATA_DB = "data_db" # This should match the connection name in settings.py - # Formats indexed into ResourceDataTable (queryable via get_row_count) INDEXED_FORMATS = {"csv", "xls", "xlsx", "ods", "parquet", "feather", "json", "tsv"} # Formats counted by parsing the file directly (not indexed into DB) @@ -474,3 +473,4 @@ def get_preview_data(resource: Resource) -> Optional[PreviewData]: f"Error getting preview data for resource {resource.id}: {str(e)}, traceback: {traceback.format_exc()}" ) return None + diff --git a/api/utils/enums.py b/api/utils/enums.py index 826cd77e..646749ff 100644 --- a/api/utils/enums.py +++ b/api/utils/enums.py @@ -192,6 +192,16 @@ class UseCaseStatus(models.TextChoices): ARCHIVED = "ARCHIVED" +class PublicationStatus(models.TextChoices): + DRAFT = "DRAFT" + PUBLISHED = "PUBLISHED" + + +class PublicationBlockType(models.TextChoices): + FILE = "FILE" + YOUTUBE = "YOUTUBE" + + class CollaborativeStatus(models.TextChoices): DRAFT = "DRAFT" PUBLISHED = "PUBLISHED" diff --git a/api/utils/file_paths.py b/api/utils/file_paths.py index 9f156780..aa1ecaa2 100644 --- a/api/utils/file_paths.py +++ b/api/utils/file_paths.py @@ -75,3 +75,15 @@ def _catalog_directory_path(catalog: Any, filename: str) -> str: catalog_name = catalog.name _, extension = os.path.splitext(filename) return f"files/catalog/{catalog_name}/{extension[1:]}/{filename}" + + +def _publication_block_directory_path(block: Any, filename: str) -> str: + """ + Create a directory path to store a publication content-block file. + + Files are namespaced under the parent publication's id so a draft's + files live in their own folder and are easy to clean up on delete. + """ + publication_id = block.publication_id + unique_name = f"{uuid.uuid4().hex}_{filename}" + return f"files/public/publications/{publication_id}/{unique_name}" diff --git a/api/utils/keycloak_utils.py b/api/utils/keycloak_utils.py index 15cd6fda..6f02af53 100644 --- a/api/utils/keycloak_utils.py +++ b/api/utils/keycloak_utils.py @@ -54,12 +54,19 @@ def get_token(self, username: str, password: str) -> Dict[str, Any]: logger.error(f"Error getting token: {e}") raise - def validate_token(self, token: str) -> Dict[str, Any]: + def validate_token( + self, token: str, token_info: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: """ Validate a token (Django JWT or Keycloak) and return the user info. Args: token: The token to validate + token_info: An introspection response for this token, if the caller + already has one. Passing it avoids a second introspect call to + Keycloak - callers that need the introspection data themselves + (for roles and organizations) would otherwise cause two + identical network round-trips per request. Returns: Dict containing the user information @@ -79,7 +86,11 @@ def validate_token(self, token: str) -> Dict[str, Any]: from authorization.models import User user = User.objects.get(id=user_id) - user.save() + # Deliberately no user.save() here. It rewrote every + # column of an unchanged row on every authenticated + # request, taking a row lock for no benefit - and since + # concurrent requests for one user contend on that single + # row, it serialized them against each other. # NOTE: Organizations are managed in DataSpace database, not Keycloak # Organization memberships should be created/managed through DataSpace's @@ -107,14 +118,36 @@ def validate_token(self, token: str) -> Dict[str, Any]: # If Django JWT validation failed, try Keycloak token validation try: - # Verify the token is valid - token_info = self.keycloak_openid.introspect(token) + # Verify the token is valid. Reuses the caller's introspection when + # one was supplied, rather than repeating the round-trip. + if token_info is None: + token_info = self.keycloak_openid.introspect(token) if not token_info.get("active", False): logger.warning("Token is not active") return {} - # Try to get user info from the userinfo endpoint - # If that fails (403), fall back to token introspection data + # Introspection often already carries everything needed. Calling + # userinfo anyway costs a round-trip that, on deployments where the + # client lacks the scope for it, is guaranteed to fail with 403 and + # fall through to exactly the same data - measured as roughly a + # third of this request's latency on dev. + if token_info.get("sub") and ( + token_info.get("email") or token_info.get("preferred_username") + ): + user_info = { + "sub": token_info.get("sub"), + "preferred_username": token_info.get("username") + or token_info.get("preferred_username"), + "email": token_info.get("email"), + "email_verified": token_info.get("email_verified", False), + "name": token_info.get("name"), + "given_name": token_info.get("given_name"), + "family_name": token_info.get("family_name"), + } + return {k: v for k, v in user_info.items() if v is not None} + + # Otherwise ask userinfo, falling back to introspection data if it + # is not available to this client. try: user_info = self.keycloak_openid.userinfo(token) return user_info @@ -379,13 +412,35 @@ def sync_user_from_keycloak( ) if user: - # Update existing user - user.keycloak_id = keycloak_id - user.username = username - user.email = email - user.first_name = user_info.get("given_name", "") or user.first_name - user.last_name = user_info.get("family_name", "") or user.last_name - user.is_active = True + # Update existing user, but only write when something actually + # changed. The unconditional save this replaces was the cause + # of the connection exhaustion: every login rewrote the same + # row, so concurrent logins for one user queued on a row lock + # (pg_stat_activity showed "Lock: tuple" and + # "Lock: transactionid" on UPDATE "ds_user"), each holding a + # database connection while it waited. + desired = { + "keycloak_id": keycloak_id, + "username": username, + "email": email, + "first_name": user_info.get("given_name", "") or user.first_name, + "last_name": user_info.get("family_name", "") or user.last_name, + "is_active": True, + "is_staff": "admin" in roles, + "is_superuser": "admin" in roles, + } + changed = [ + field + for field, value in desired.items() + if getattr(user, field) != value + ] + if changed: + for field in changed: + setattr(user, field, desired[field]) + # update_fields keeps the UPDATE narrow instead of + # rewriting every column. + user.save(update_fields=changed) + return user else: # Create new user user = User( diff --git a/api/utils/publication_uploads.py b/api/utils/publication_uploads.py new file mode 100644 index 00000000..c2c51ede --- /dev/null +++ b/api/utils/publication_uploads.py @@ -0,0 +1,69 @@ +""" +publication_uploads +──────────────────── +Server-side validation for a content block's uploaded file. + +``validate_publication_file`` is the upload boundary guard: it enforces the +allowed extensions, the 50 MB per-file cap, and — for a file declaring itself a +PDF — that the bytes really start with the PDF magic number (so a renamed +executable can't sneak in as ``report.pdf``). Raises a clean ValidationError on +any violation; returns the detected extension and byte size on success. +""" + +import os +from typing import Any, Tuple + +from django.core.exceptions import ValidationError + +# Locked limits (plan §Shared context item 10). +MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024 # 50 MB +MAX_FILE_SIZE_LABEL = "50 MB" +ALLOWED_EXTENSIONS = { + ".pdf", + ".doc", + ".docx", + ".ppt", + ".pptx", + ".odp", + ".odt", + ".key", +} +_PDF_MAGIC = b"%PDF" + + +def validate_publication_file(uploaded_file: Any) -> Tuple[str, int]: + """Validate an uploaded content-block file and return (extension, size_bytes). + + Rejects a disallowed extension, a file over the 50 MB cap, and a file that + claims a ``.pdf`` name but whose first bytes aren't the PDF magic number. + """ + name = getattr(uploaded_file, "name", "") or "" + extension = os.path.splitext(name)[1].lower() + + if extension not in ALLOWED_EXTENSIONS: + raise ValidationError( + f"'{extension or name}' is not an allowed file type. " + f"Allowed types: {', '.join(sorted(ALLOWED_EXTENSIONS))}." + ) + + size = getattr(uploaded_file, "size", 0) or 0 + if size > MAX_FILE_SIZE_BYTES: + raise ValidationError(f"File is larger than the {MAX_FILE_SIZE_LABEL} limit.") + + if extension == ".pdf" and not _looks_like_pdf(uploaded_file): + raise ValidationError("File claims to be a PDF but its contents are not.") + + return extension, size + + +def _looks_like_pdf(uploaded_file: Any) -> bool: + """Peek at the first bytes to confirm a real PDF, then rewind the file.""" + try: + uploaded_file.seek(0) + head = uploaded_file.read(len(_PDF_MAGIC)) + uploaded_file.seek(0) + except (AttributeError, OSError): + return False + if isinstance(head, str): + head = head.encode("latin-1", errors="ignore") + return bool(head.startswith(_PDF_MAGIC)) diff --git a/api/utils/youtube.py b/api/utils/youtube.py new file mode 100644 index 00000000..fac44d70 --- /dev/null +++ b/api/utils/youtube.py @@ -0,0 +1,67 @@ +""" +youtube +─────── +Parse and validate YouTube links for content blocks. + +``extract_video_id`` pulls the 11-character video id out of any of the common +YouTube URL shapes (``watch?v=``, ``youtu.be/``, ``embed/``); ``validate_youtube_url`` +is the boundary guard — it returns that id or raises a clean ValidationError for +anything that isn't a recognisable YouTube video link. +""" + +import re +from typing import Optional +from urllib.parse import parse_qs, urlparse + +from django.core.exceptions import ValidationError + +# A YouTube video id is exactly 11 url-safe characters. +_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$") +_YOUTUBE_HOSTS = { + "youtube.com", + "www.youtube.com", + "m.youtube.com", + "youtu.be", + "www.youtu.be", +} + + +def extract_video_id(url: Optional[str]) -> Optional[str]: + """Return the 11-char video id from a YouTube url, or None if it isn't one. + + Handles ``watch?v=``, the ``youtu.be/`` short link, and the + ``/embed/`` player link. Any other host or a malformed id yields None. + """ + if not url or not url.strip(): + return None + + parsed = urlparse(url.strip()) + # Reject anything that isn't a plain http(s) link — a matching host on a + # ``javascript:`` scheme (e.g. ``javascript://youtube.com/watch?v=...``) + # must never pass, or it becomes a stored-XSS vector when rendered. + if parsed.scheme not in ("http", "https"): + return None + host = (parsed.hostname or "").lower() + if host not in _YOUTUBE_HOSTS: + return None + + candidate: Optional[str] = None + if host in {"youtu.be", "www.youtu.be"}: + candidate = parsed.path.lstrip("/").split("/")[0] + elif parsed.path == "/watch": + values = parse_qs(parsed.query).get("v") + candidate = values[0] if values else None + elif parsed.path.startswith(("/embed/", "/v/", "/shorts/")): + candidate = parsed.path.split("/")[2] if len(parsed.path.split("/")) > 2 else None + + if candidate and _VIDEO_ID.match(candidate): + return candidate + return None + + +def validate_youtube_url(url: Optional[str]) -> str: + """Return the video id for a valid YouTube url, else raise ValidationError.""" + video_id = extract_video_id(url) + if not video_id: + raise ValidationError("Enter a valid YouTube video URL.") + return video_id diff --git a/api/views/auth.py b/api/views/auth.py index 94d98dc7..98b10295 100644 --- a/api/views/auth.py +++ b/api/views/auth.py @@ -24,17 +24,20 @@ def post(self, request: Request) -> Response: status=status.HTTP_400_BAD_REQUEST, ) + # Introspect once and reuse it. This used to introspect twice per + # request - once inside validate_token and again here for roles and + # organizations - which is a wasted round-trip to Keycloak on every + # login while a database connection is held open. + token_info = keycloak_manager.keycloak_openid.introspect(keycloak_token) + # Validate the token and get user info - user_info = keycloak_manager.validate_token(keycloak_token) + user_info = keycloak_manager.validate_token(keycloak_token, token_info=token_info) if not user_info: return Response( {"error": "Invalid or expired token"}, status=status.HTTP_401_UNAUTHORIZED, ) - # Get token introspection data for roles and organizations - token_info = keycloak_manager.keycloak_openid.introspect(keycloak_token) - # Get user roles and organizations from the token introspection data roles = keycloak_manager.get_user_roles_from_token_info(token_info) organizations = keycloak_manager.get_user_organizations_from_token_info(token_info) diff --git a/api/views/dynamic_chart_view.py b/api/views/dynamic_chart_view.py index 457d9249..7e7ac0f0 100644 --- a/api/views/dynamic_chart_view.py +++ b/api/views/dynamic_chart_view.py @@ -25,9 +25,7 @@ async def create_chart_details( # Validate chart type if chart_type not in ChartTypes.values: - return JsonResponse( - {"error": f"Unsupported chart type: {chart_type}"}, status=400 - ) + return JsonResponse({"error": f"Unsupported chart type: {chart_type}"}, status=400) # Set basic options options["x_axis_label"] = request_details.get("x_axis_label", "X-Axis") @@ -42,7 +40,7 @@ async def create_chart_details( ) # Handle y-axis columns with configurations - y_axis_columns = [] + y_axis_columns: list = [] if y_axis_configs := request_details.get("y_axis_column", []): y_axis_columns = [] for config in y_axis_configs: @@ -57,8 +55,7 @@ async def create_chart_details( value_mapping = { str(mapping["key"]): str(mapping["value"]) for mapping in raw_mappings - if mapping.get("key") is not None - and mapping.get("value") is not None + if mapping.get("key") is not None and mapping.get("value") is not None } y_axis_columns.append( @@ -158,9 +155,7 @@ async def create_chart_details( @csrf_exempt -async def generate_dynamic_chart( - request: HttpRequest, resource_id: uuid.UUID -) -> HttpResponse: +async def generate_dynamic_chart(request: HttpRequest, resource_id: uuid.UUID) -> HttpResponse: if request.method == "POST": try: # Fetch the resource asynchronously @@ -188,9 +183,7 @@ async def generate_dynamic_chart( return response # Default response: JSON - return JsonResponse( - json.loads(chart.dump_options_with_quotes()), safe=False - ) + return JsonResponse(json.loads(chart.dump_options_with_quotes()), safe=False) except Exception as e: return JsonResponse({"error": f"Error generating chart: {e}"}, status=500) diff --git a/api/views/health.py b/api/views/health.py index 12d527d9..9f1d0d84 100644 --- a/api/views/health.py +++ b/api/views/health.py @@ -1,10 +1,11 @@ +import os from typing import Any, Dict import requests import structlog from django.conf import settings from django.core.cache import cache -from django.db import connection +from django.db import connection, connections from django.http import HttpRequest, JsonResponse from elasticsearch import Elasticsearch from opentelemetry import trace @@ -31,16 +32,38 @@ def health_check(request: HttpRequest) -> JsonResponse: "telemetry": {"status": "unknown"}, } - # Check database + # Check database. + # + # Two distinct checks, because they fail independently: + # + # 1. The request's own connection still works. + # 2. A NEW connection can still be opened. + # + # Only checking (1) is how this endpoint reported + # {"database": "healthy"} in 0.44s while Postgres was refusing new + # connections with "FATAL: sorry, too many clients already" and both the + # login endpoint and the deploy pipeline were failing on exactly that. The + # existing connection is already established, so it keeps answering + # SELECT 1 no matter how saturated the server is - which made a green + # health check actively misleading during an outage. try: with connection.cursor() as cursor: cursor.execute("SELECT 1") - status["database"] = { - "status": "healthy", - "message": "Successfully connected to database", - } - if current_span: - current_span.set_attribute("database.status", "healthy") + + # Deliberately a fresh connection, closed immediately. This is the + # check that catches connection exhaustion. + new_connection = connections.create_connection("default") + try: + new_connection.ensure_connection() + finally: + new_connection.close() + + status["database"] = { + "status": "healthy", + "message": "Successfully connected to database", + } + if current_span: + current_span.set_attribute("database.status", "healthy") except Exception as e: logger.error("Database health check failed", error=str(e)) status["database"] = { @@ -99,39 +122,61 @@ def health_check(request: HttpRequest) -> JsonResponse: current_span.set_attribute("redis.status", "unhealthy") current_span.set_attribute("redis.error", str(e)) - # Check OpenTelemetry collector - try: - # Extract host and port from TELEMETRY_URL - telemetry_url = settings.TELEMETRY_URL.replace("http://", "").replace( - "https://", "" - ) - host = telemetry_url.split(":")[0] - # Use default health check port 13133 instead of gRPC port - health_url = f"http://{host}:13133/health" # OpenTelemetry collector health check endpoint - - response = requests.get(health_url, timeout=5) - if response.status_code == 200: - status["telemetry"] = { - "status": "healthy", - "message": "Successfully connected to OpenTelemetry collector", - } - if current_span: - current_span.set_attribute("telemetry.status", "healthy") - else: - raise Exception(f"Health check returned status code {response.status_code}") - - except Exception as e: - logger.error("Telemetry health check failed", error=str(e)) + # Check OpenTelemetry collector, but only when telemetry is actually + # configured. DataSpace/settings.py gives TELEMETRY_URL a hardcoded + # otel-collector default even when the env var is unset, so without this + # guard a deployment that deliberately runs no collector (as this one + # does) probes a host that cannot resolve and logs an ERROR on every + # single health check -- roughly every 30s, forever, for something + # optional. + if not os.environ.get("TELEMETRY_URL"): status["telemetry"] = { - "status": "unhealthy", - "message": f"Failed to connect to OpenTelemetry collector: {str(e)}", + "status": "not_configured", + "message": "TELEMETRY_URL is unset; no OpenTelemetry collector expected", } if current_span: - current_span.set_attribute("telemetry.status", "unhealthy") - current_span.set_attribute("telemetry.error", str(e)) - - # Overall status - overall_status = all(service["status"] == "healthy" for service in status.values()) + current_span.set_attribute("telemetry.status", "not_configured") + else: + try: + # Extract host and port from TELEMETRY_URL + telemetry_url = settings.TELEMETRY_URL.replace("http://", "").replace( + "https://", "" + ) + host = telemetry_url.split(":")[0] + # Use default health check port 13133 instead of gRPC port + health_url = f"http://{host}:13133/health" # OpenTelemetry collector health check endpoint + + response = requests.get(health_url, timeout=5) + if response.status_code == 200: + status["telemetry"] = { + "status": "healthy", + "message": "Successfully connected to OpenTelemetry collector", + } + if current_span: + current_span.set_attribute("telemetry.status", "healthy") + else: + raise Exception( + f"Health check returned status code {response.status_code}" + ) + + except Exception as e: + logger.error("Telemetry health check failed", error=str(e)) + status["telemetry"] = { + "status": "unhealthy", + "message": f"Failed to connect to OpenTelemetry collector: {str(e)}", + } + if current_span: + current_span.set_attribute("telemetry.status", "unhealthy") + current_span.set_attribute("telemetry.error", str(e)) + + # Overall status: database/elasticsearch/redis are required for the app + # to actually serve requests. telemetry is observability-only (this + # deployment topology deliberately runs without otel-collector) and is + # reported above for visibility, but must not gate 200 vs 503 -- a + # health check that fails a deploy because optional tracing + # infrastructure isn't running is a false negative. + required_services = ("database", "elasticsearch", "redis") + overall_status = all(status[name]["status"] == "healthy" for name in required_services) if current_span: current_span.set_attribute( @@ -141,6 +186,7 @@ def health_check(request: HttpRequest) -> JsonResponse: data = { "status": "healthy" if overall_status else "unhealthy", "services": status, + "git_sha": os.environ.get("GIT_COMMIT_SHA", "unknown"), } - return JsonResponse(data) + return JsonResponse(data, status=200 if overall_status else 503) diff --git a/api/views/publication_download_view.py b/api/views/publication_download_view.py new file mode 100644 index 00000000..deac2435 --- /dev/null +++ b/api/views/publication_download_view.py @@ -0,0 +1,65 @@ +""" +publication_download_view +────────────────────────── +Serves a content block's file through an access gate instead of a plain public +media URL, so a DRAFT resource's files are never world-readable. A published +resource's files are open; a draft's are limited to its owner / org members. +Each successful download bumps the parent resource's ``download_count``. PDFs +are served inline (for the detail-page viewer); every other type downloads. + +This is a flow file: the gate and block lookup live in +``api/services/publication_blocks.py``. +""" + +import os +from typing import Any + +from django.db.models import F +from django.http import HttpRequest, HttpResponse, JsonResponse + +from api.models import Publication, PublicationBlock +from api.services.publication_blocks import can_access_block_file +from api.utils.enums import PublicationBlockType + +# Extensions we serve inline in the browser; everything else downloads. +_INLINE_CONTENT_TYPES = {".pdf": "application/pdf"} + + +def publication_block_download(request: HttpRequest, block_id: Any) -> HttpResponse: + """Serve a block's file if the caller may see its resource, else 404.""" + # Find the block and its parent resource. + try: + block: PublicationBlock = PublicationBlock.objects.select_related("publication").get( + id=block_id + ) + except PublicationBlock.DoesNotExist: + return JsonResponse({"error": "Not found"}, status=404) + + publication = block.publication + + # Gate: a draft's files are private — hide existence with a 404 on denial. + if not can_access_block_file(request.user, publication): + return JsonResponse({"error": "Not found"}, status=404) + + # Only file blocks have something to download. + if block.block_type != PublicationBlockType.FILE or not block.file: + return JsonResponse({"error": "Not found"}, status=404) + + # Count the download against the resource (race-safe increment). + Publication.objects.filter(id=publication.id).update(download_count=F("download_count") + 1) + + # Serve the bytes inline for a PDF, as an attachment otherwise. + return _build_file_response(block) + + +def _build_file_response(block: PublicationBlock) -> HttpResponse: + """Build the HTTP file response with the right content type and disposition.""" + stored_name = block.file.name or "" + extension = os.path.splitext(stored_name)[1].lower() + content_type = _INLINE_CONTENT_TYPES.get(extension, "application/octet-stream") + disposition = "inline" if extension in _INLINE_CONTENT_TYPES else "attachment" + + filename = block.file_name or os.path.basename(stored_name) + response = HttpResponse(block.file.read(), content_type=content_type) + response["Content-Disposition"] = f'{disposition}; filename="{filename}"' + return response diff --git a/api/views/search_publication.py b/api/views/search_publication.py new file mode 100644 index 00000000..3f5260e6 --- /dev/null +++ b/api/views/search_publication.py @@ -0,0 +1,134 @@ +"""Search view for Publication (UI "Resource") using Elasticsearch. + +Only published resources are in the index (the document's ``should_index_object`` +gate), so this public endpoint never leaks drafts. Filters: resource type, +sector, geography. +""" + +from typing import Any, Dict, List, Optional, Tuple, Union + +import structlog +from elasticsearch_dsl import Q as ESQ +from elasticsearch_dsl import Search +from elasticsearch_dsl.query import Query as ESQuery +from rest_framework import serializers +from rest_framework.permissions import AllowAny + +from api.models.Publication import Publication +from api.utils.telemetry_utils import trace_method +from api.views.paginated_elastic_view import PaginatedElasticSearchAPIView +from search.documents import PublicationDocument + +logger = structlog.get_logger(__name__) + + +class PublicationDocumentSerializer(serializers.ModelSerializer): + """Serializer for the Publication search document.""" + + resource_type = serializers.CharField(allow_blank=True) + sectors = serializers.ListField() + geographies = serializers.ListField() + + class OrganizationSerializer(serializers.Serializer): + name = serializers.CharField() + logo = serializers.CharField() + + class UserSerializer(serializers.Serializer): + name = serializers.CharField() + bio = serializers.CharField() + profile_picture = serializers.CharField() + + organization = OrganizationSerializer(allow_null=True) + user = UserSerializer(allow_null=True) + + class Meta: + model = Publication + fields = [ + "id", + "title", + "description", + "status", + "resource_type", + "sectors", + "geographies", + "created", + "modified", + "organization", + "user", + ] + + +class SearchPublication(PaginatedElasticSearchAPIView): + """View for searching resources.""" + + serializer_class = PublicationDocumentSerializer + document_class = PublicationDocument + permission_classes = [AllowAny] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.searchable_fields: List[str] + self.aggregations: Dict[str, str] + self.searchable_fields, self.aggregations = self.get_searchable_and_aggregations() + self.logger = structlog.get_logger(__name__) + + @trace_method( + name="get_searchable_and_aggregations", + attributes={"component": "search_publication"}, + ) + def get_searchable_and_aggregations(self) -> Tuple[List[str], Dict[str, str]]: + """Searchable fields (name/description) and the three facet aggregations.""" + searchable_fields: List[str] = ["title", "description"] + aggregations: Dict[str, str] = { + "status": "terms", + "resource_type.raw": "terms", + "sectors.raw": "terms", + "geographies.raw": "terms", + } + return searchable_fields, aggregations + + @trace_method(name="add_aggregations", attributes={"component": "search_publication"}) + def add_aggregations(self, search: Search) -> Search: + """Add the facet aggregations to the search query.""" + for aggregation_field in self.aggregations: + search.aggs.bucket( + aggregation_field.replace(".raw", ""), + self.aggregations[aggregation_field], + field=aggregation_field, + ) + return search + + @trace_method(name="generate_q_expression", attributes={"component": "search_publication"}) + def generate_q_expression(self, query: str) -> Optional[Union[ESQuery, List[ESQuery]]]: + """Build a fuzzy query over the searchable fields, or match-all when blank.""" + if query: + queries: List[ESQuery] = [ + ESQ("fuzzy", **{field: {"value": query, "fuzziness": "AUTO"}}) + for field in self.searchable_fields + ] + else: + queries = [ESQ("match_all")] + return ESQ("bool", should=queries, minimum_should_match=1) + + @trace_method(name="add_filters", attributes={"component": "search_publication"}) + def add_filters(self, filters: Dict[str, str], search: Search) -> Search: + """Apply resource-type / sector / geography facet filters.""" + for filter_key in filters: + if filter_key in ["resource_type", "sectors", "geographies"]: + raw_filter = filter_key + ".raw" + filter_values = filters[filter_key].split(",") + search = search.filter("terms", **{raw_filter: filter_values}) + elif filter_key == "status": + search = search.filter("term", **{filter_key: filters[filter_key]}) + return search + + @trace_method(name="add_sort", attributes={"component": "search_publication"}) + def add_sort(self, sort: str, search: Search, order: str) -> Search: + """Apply a sort mode (alphabetical / recent / created).""" + if sort == "alphabetical": + search = search.sort({"title.raw": {"order": order}}) + elif sort == "recent": + search = search.sort({"modified": {"order": order}}) + elif sort == "created": + search = search.sort({"created": {"order": order}}) + return search diff --git a/api/views/search_unified.py b/api/views/search_unified.py index b84c6daf..b9c86a0b 100644 --- a/api/views/search_unified.py +++ b/api/views/search_unified.py @@ -80,6 +80,10 @@ class UserSerializer(serializers.Serializer): provider = serializers.CharField(required=False) is_individual_model = serializers.BooleanField(required=False) + # Publication (Resource) specific + resource_type = serializers.CharField(required=False) + is_individual_publication = serializers.BooleanField(required=False) + # Collaborative specific is_individual_collaborative = serializers.BooleanField(required=False) website = serializers.CharField(required=False) @@ -132,6 +136,12 @@ def _get_index_names(self, types_list: List[str]) -> List[str]: ) index_names.append(aimodel_index) + if "publication" in types_list: + publication_index = settings.ELASTICSEARCH_INDEX_NAMES.get( + "search.documents.publication_document", "publication" + ) + index_names.append(publication_index) + if "collaborative" in types_list: collaborative_index = settings.ELASTICSEARCH_INDEX_NAMES.get( "search.documents.collaborative_document", "collaborative" @@ -312,6 +322,8 @@ def _normalize_result(self, hit: Any) -> Dict[str, Any]: result["type"] = "usecase" elif "aimodel" in index_name: result["type"] = "aimodel" + elif "publication" in index_name: + result["type"] = "publication" elif "collaborative" in index_name: result["type"] = "collaborative" elif "publisher" in index_name: @@ -424,6 +436,8 @@ def perform_unified_search( aggregations["types"]["usecase"] = bucket["doc_count"] elif "aimodel" in index_name: aggregations["types"]["aimodel"] = bucket["doc_count"] + elif "publication" in index_name: + aggregations["types"]["publication"] = bucket["doc_count"] elif "collaborative" in index_name: aggregations["types"]["collaborative"] = bucket["doc_count"] elif "publisher" in index_name: @@ -448,7 +462,9 @@ def _generate_unified_cache_key(self, request: Any) -> str: "query": request.GET.get("query", ""), "page": request.GET.get("page", "1"), "size": request.GET.get("size", "10"), - "types": request.GET.get("types", "dataset,usecase,aimodel,collaborative,publisher"), + "types": request.GET.get( + "types", "dataset,usecase,aimodel,publication,collaborative,publisher" + ), "filters": str(sorted(request.GET.dict().items())), "version": str(cache.get(SEARCH_CACHE_VERSION_KEY, 0)), } @@ -467,7 +483,7 @@ def get(self, request: Any) -> Response: page: int = int(request.GET.get("page", 1)) size: int = int(request.GET.get("size", 10)) entity_types: str = request.GET.get( - "types", "dataset,usecase,aimodel,collaborative,publisher" + "types", "dataset,usecase,aimodel,publication,collaborative,publisher" ) # Which entity types to search types_list = [t.strip() for t in entity_types.split(",")] diff --git a/authorization/permissions.py b/authorization/permissions.py index 4aca31c4..ce818e9b 100644 --- a/authorization/permissions.py +++ b/authorization/permissions.py @@ -5,9 +5,13 @@ from strawberry.permission import BasePermission from strawberry.types import Info -from api.models import Dataset, Organization +from api.models import Dataset, Organization, Publication +from api.utils.enums import PublicationStatus from authorization.models import DatasetPermission, OrganizationMembership, Role +# Roles that may publish/unpublish and edit an org-owned publication, by name. +PUBLICATION_MANAGER_ROLE_NAMES = ["admin", "editor", "owner"] + # REST Framework Permissions class IsOrganizationMember(permissions.BasePermission): @@ -53,9 +57,7 @@ def has_permission(self, request: Any, view: Any) -> bool: return True # For organization-specific endpoints - org_id = request.query_params.get("organization") or request.data.get( - "organization" - ) + org_id = request.query_params.get("organization") or request.data.get("organization") if org_id: return OrganizationMembership.objects.filter( user=request.user, organization_id=org_id @@ -204,9 +206,7 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: organization_id = kwargs.get("organization_id") # Also check if organization is in the context organization = None - if hasattr(info.context, "context") and isinstance( - info.context.context, dict - ): + if hasattr(info.context, "context") and isinstance(info.context.context, dict): organization = info.context.context.get("organization") if organization_id: @@ -316,9 +316,7 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: return True try: - dataset_perm = DatasetPermission.objects.get( - user=request.user, dataset=source - ) + dataset_perm = DatasetPermission.objects.get(user=request.user, dataset=source) role = dataset_perm.role return self._check_role_permission(role) except DatasetPermission.DoesNotExist: @@ -458,3 +456,169 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: except Dataset.DoesNotExist: return False + + +# --------------------------------------------------------------------------- +# Publication (UI "Resource") permissions +# +# Mirrors Dataset's dedicated permission classes rather than AIModel's inline +# per-resolver role checks. Publication has no per-object share model, so the +# share-model fallback is dropped; the individual-owner branch is kept so an +# org-less publication's owner isn't denied. +# --------------------------------------------------------------------------- + + +def _resolve_publication_id(kwargs: Any) -> Optional[Any]: + """Pull the target publication's id out of a mutation's arguments. + + Publish/unpublish/delete pass ``publication_id`` directly; update passes an + input object carrying the id on ``.id``. + """ + publication_id = kwargs.get("publication_id") + if publication_id: + return publication_id + for input_key in ("input", "update_input"): + payload = kwargs.get(input_key) + if payload is not None and getattr(payload, "id", None): + return payload.id + # Block-scoped mutations pass a block id — resolve to its parent publication. + block_id = kwargs.get("block_id") + if block_id: + from api.models import PublicationBlock + + block = PublicationBlock.objects.filter(id=block_id).first() + if block: + return block.publication_id + return None + + +def _user_manages_publication(user: Any, publication: Publication, operation: str) -> bool: + """Whether a user may perform ``operation`` on a publication. + + Owner (individual publications) always may; for org-owned publications the + caller must be a member whose role grants the operation (``publish`` and + ``change``/``delete`` map to the role's name / boolean flags). + """ + if user.is_superuser: + return True + if publication.user and publication.user == user: + return True + if not publication.organization: + return False + + membership = OrganizationMembership.objects.filter( + user=user, organization=publication.organization + ).first() + if not membership: + return False + + role = membership.role + if operation == "publish": + return role.name in PUBLICATION_MANAGER_ROLE_NAMES + if operation == "delete": + return role.can_delete + return role.can_change + + +class PublicationPermissionGraphQL(BasePermission): # type: ignore[misc] + """Base publication mutation permission — keys on the publication id in kwargs.""" + + message = "You don't have permission to modify this resource" + operation = "change" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = info.context.user + if not getattr(user, "is_authenticated", False): + return False + + publication_id = _resolve_publication_id(kwargs) + if not publication_id: + return False + + try: + publication = Publication.objects.get(id=publication_id) + except Publication.DoesNotExist: + return False + + return _user_manages_publication(user, publication, self.operation) + + +class ChangePublicationPermission(PublicationPermissionGraphQL): + operation = "change" + + +class DeletePublicationPermission(PublicationPermissionGraphQL): + message = "You don't have permission to delete this resource" + operation = "delete" + + +class PublishPublicationPermission(PublicationPermissionGraphQL): + message = "You don't have permission to publish this resource" + operation = "publish" + + +class CreatePublicationPermission(BasePermission): # type: ignore[misc] + """Permission for creating a publication — mirrors CreateDatasetPermission. + + Any authenticated user may create an individual publication; creating inside + an organization context requires the ``add`` role in that organization. + """ + + message = "You don't have permission to create a resource" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = info.context.user + if not getattr(user, "is_authenticated", False): + return False + + organization = info.context.context.get("organization") + if organization: + membership = OrganizationMembership.objects.filter( + user=user, organization=organization + ).first() + return bool(membership and membership.role.can_add) + + return True + + +class AllowPublishedPublications(BasePermission): # type: ignore[misc] + """Read gate for a single publication — mirrors AllowPublishedDatasets. + + A PUBLISHED publication is world-readable; a DRAFT is visible only to the + owner, org members with view access, or a superuser. + """ + + message = "You need to be authenticated to access non-published resources" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + request = info.context + publication_id = kwargs.get("publication_id") + + if publication_id: + try: + publication = Publication.objects.get(id=publication_id) + except Publication.DoesNotExist: + return True # Let the resolver return a clean not-found. + + if publication.status == PublicationStatus.PUBLISHED.value: + return True + + user = request.user + if not user.is_authenticated: + return False + if user.is_superuser: + return True + if publication.user and publication.user == user: + return True + if publication.organization: + membership = OrganizationMembership.objects.filter( + user=user, organization=publication.organization + ).first() + return bool(membership and membership.role.can_view) + return False + + # No id in kwargs (e.g. object source) — published is public, else auth. + if hasattr(source, "status"): + if source.status == PublicationStatus.PUBLISHED.value: + return True + return bool(getattr(request, "user", None) and request.user.is_authenticated) diff --git a/dataspace_sdk/__version__.py b/dataspace_sdk/__version__.py index cff36390..7008ea21 100644 --- a/dataspace_sdk/__version__.py +++ b/dataspace_sdk/__version__.py @@ -1,3 +1,3 @@ """Version information for DataSpace SDK.""" -__version__ = "0.4.19" +__version__ = "0.5.05" diff --git a/dataspace_sdk/auth.py b/dataspace_sdk/auth.py index 4ecac6e2..98078d67 100644 --- a/dataspace_sdk/auth.py +++ b/dataspace_sdk/auth.py @@ -18,6 +18,7 @@ def __init__( keycloak_realm: Optional[str] = None, keycloak_client_id: Optional[str] = None, keycloak_client_secret: Optional[str] = None, + keycloak_base_path: str = "/auth", ): """ Initialize the authentication client. @@ -28,9 +29,15 @@ def __init__( keycloak_realm: Keycloak realm name (e.g., "DataSpace") keycloak_client_id: Keycloak client ID (e.g., "dataspace") keycloak_client_secret: Optional client secret for confidential clients + keycloak_base_path: Keycloak's HTTP relative path. Defaults to + "/auth", which is what servers configured the legacy way use. + Pass "" for a Keycloak served at the domain root. """ self.base_url = base_url.rstrip("/") self.keycloak_url = keycloak_url.rstrip("/") if keycloak_url else None + self.keycloak_base_path = ( + "/" + keycloak_base_path.strip("/") if keycloak_base_path.strip("/") else "" + ) self.keycloak_realm = keycloak_realm self.keycloak_client_id = keycloak_client_id self.keycloak_client_secret = keycloak_client_secret @@ -47,6 +54,22 @@ def __init__( self._username: Optional[str] = None self._password: Optional[str] = None + def _realm_url(self) -> str: + """Base URL for this realm's endpoints. + + Keycloak's relative path is deployment-specific: "/auth" on servers + configured the legacy way, empty on servers served at the domain root. + This used to be hardcoded, which made the SDK unable to reach a + root-path Keycloak at all. + """ + path = self.keycloak_base_path + base = self.keycloak_url or "" + # Tolerate the relative path already being part of keycloak_url, rather + # than emitting it twice. + if path and base.endswith(path): + path = "" + return f"{base}{path}/realms/{self.keycloak_realm}" + def login(self, username: str, password: str) -> Dict[str, Any]: """ Login using username and password via Keycloak. @@ -124,10 +147,7 @@ def _get_keycloak_token(self, username: str, password: str) -> str: Raises: DataSpaceAuthError: If authentication fails """ - token_url = ( - f"{self.keycloak_url}/auth/realms/{self.keycloak_realm}/" - f"protocol/openid-connect/token" - ) + token_url = f"{self._realm_url()}/protocol/openid-connect/token" data = { "grant_type": "password", @@ -183,10 +203,7 @@ def _get_service_account_token(self) -> str: Raises: DataSpaceAuthError: If authentication fails """ - token_url = ( - f"{self.keycloak_url}/auth/realms/{self.keycloak_realm}/" - f"protocol/openid-connect/token" - ) + token_url = f"{self._realm_url()}/protocol/openid-connect/token" data = { "grant_type": "client_credentials", @@ -247,10 +264,7 @@ def _refresh_keycloak_token(self) -> str: return self._get_keycloak_token(self._username, self._password) raise DataSpaceAuthError("No refresh token or credentials available") - token_url = ( - f"{self.keycloak_url}/auth/realms/{self.keycloak_realm}/" - f"protocol/openid-connect/token" - ) + token_url = f"{self._realm_url()}/protocol/openid-connect/token" data = { "grant_type": "refresh_token", diff --git a/dataspace_sdk/client.py b/dataspace_sdk/client.py index 5f85eb28..d1394680 100644 --- a/dataspace_sdk/client.py +++ b/dataspace_sdk/client.py @@ -6,6 +6,7 @@ from dataspace_sdk.resources.aimodels import AIModelClient from dataspace_sdk.resources.auditors import AuditorClient from dataspace_sdk.resources.datasets import DatasetClient +from dataspace_sdk.resources.publications import PublicationClient from dataspace_sdk.resources.sectors import SectorClient from dataspace_sdk.resources.usecases import UseCaseClient @@ -42,6 +43,7 @@ def __init__( keycloak_realm: Optional[str] = None, keycloak_client_id: Optional[str] = None, keycloak_client_secret: Optional[str] = None, + keycloak_base_path: str = "/auth", ): """ Initialize the DataSpace client. @@ -52,6 +54,9 @@ def __init__( keycloak_realm: Keycloak realm name (e.g., "DataSpace") keycloak_client_id: Keycloak client ID (e.g., "dataspace") keycloak_client_secret: Optional client secret for confidential clients + keycloak_base_path: Keycloak's HTTP relative path. Defaults to + "/auth" for backwards compatibility. Pass "" for a Keycloak + served at the domain root. """ self.base_url = base_url.rstrip("/") self._auth = AuthClient( @@ -60,11 +65,13 @@ def __init__( keycloak_realm=keycloak_realm, keycloak_client_id=keycloak_client_id, keycloak_client_secret=keycloak_client_secret, + keycloak_base_path=keycloak_base_path, ) # Initialize resource clients self.datasets = DatasetClient(self.base_url, self._auth) self.aimodels = AIModelClient(self.base_url, self._auth) + self.publications = PublicationClient(self.base_url, self._auth) self.usecases = UseCaseClient(self.base_url, self._auth) self.sectors = SectorClient(self.base_url, self._auth) self.auditors = AuditorClient(self.base_url, self._auth) @@ -199,6 +206,7 @@ def set_organization(self, organization_id: str) -> None: """ self.datasets.default_headers["organization"] = organization_id self.aimodels.default_headers["organization"] = organization_id + self.publications.default_headers["organization"] = organization_id self.usecases.default_headers["organization"] = organization_id self.sectors.default_headers["organization"] = organization_id self.auditors.default_headers["organization"] = organization_id diff --git a/dataspace_sdk/resources/datasets.py b/dataspace_sdk/resources/datasets.py index 24dee57a..c3ae3a62 100644 --- a/dataspace_sdk/resources/datasets.py +++ b/dataspace_sdk/resources/datasets.py @@ -337,6 +337,12 @@ def get_prompt_by_id(self, dataset_id: str) -> Dict[str, Any]: format size } + schema { + format + description + fieldName + } + noOfEntries promptDetails { promptFormat hasSystemPrompt @@ -416,12 +422,18 @@ def list_prompts( format size } + schema { + format + description + fieldName + } promptDetails { promptFormat hasSystemPrompt hasExampleResponses promptCount } + noOfEntries } } } diff --git a/dataspace_sdk/resources/publications.py b/dataspace_sdk/resources/publications.py new file mode 100644 index 00000000..031f5bfa --- /dev/null +++ b/dataspace_sdk/resources/publications.py @@ -0,0 +1,132 @@ +"""Publication ("Resource") resource client for DataSpace SDK.""" + +from typing import Any, Dict, List, Optional + +from dataspace_sdk.base import BaseAPIClient + + +class PublicationClient(BaseAPIClient): + """Client for interacting with Resources (internally 'publications'). + + Search runs over the REST Elasticsearch endpoint; detail/list/CRUD go + through GraphQL, matching the backend (Resources have no REST write API). + """ + + def search( + self, + query: Optional[str] = None, + resource_type: Optional[str] = None, + sectors: Optional[List[str]] = None, + geographies: Optional[List[str]] = None, + sort: Optional[str] = None, + page: int = 1, + page_size: int = 10, + ) -> Dict[str, Any]: + """Search published resources via Elasticsearch. + + Args: + query: Free-text query. + resource_type: Filter by resource type name. + sectors: Filter by sector names. + geographies: Filter by geography names. + sort: Sort order (recent, alphabetical, created). + page: Page number (1-indexed). + page_size: Results per page. + + Returns: + Search results and metadata. + """ + params: Dict[str, Any] = {"page": page, "page_size": page_size} + if query: + params["q"] = query + if resource_type: + params["resource_type"] = resource_type + if sectors: + params["sectors"] = ",".join(sectors) + if geographies: + params["geographies"] = ",".join(geographies) + if sort: + params["sort"] = sort + + return super().get("/api/search/publication/", params=params) + + def get_by_id(self, publication_id: str) -> Dict[str, Any]: + """Get a single resource by id via GraphQL.""" + query = """ + query GetPublication($publicationId: UUID!) { + getPublication(publicationId: $publicationId) { + id title description slug status authors publicationDate + license externalSourceLink downloadCount + resourceType { id name } + blocks { id position blockType fileName youtubeUrl } + } + } + """ + return self.post( + "/api/graphql", + json_data={"query": query, "variables": {"publicationId": publication_id}}, + ) + + def list_all( + self, + include_public: bool = False, + limit: int = 10, + offset: int = 0, + ) -> Dict[str, Any]: + """List resources scoped to the caller (org header or user) via GraphQL.""" + query = """ + query ListPublications($includePublic: Boolean, $pagination: OffsetPaginationInput) { + publications(includePublic: $includePublic, pagination: $pagination) { + id title slug status + } + } + """ + variables: Dict[str, Any] = { + "includePublic": include_public, + "pagination": {"offset": offset, "limit": limit}, + } + return self.post("/api/graphql", json_data={"query": query, "variables": variables}) + + def get_organization_publications(self, limit: int = 10, offset: int = 0) -> Dict[str, Any]: + """List the current organization's resources (org set via set_organization).""" + return self.list_all(include_public=False, limit=limit, offset=offset) + + def create(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Create a resource via the createPublication mutation.""" + mutation = """ + mutation CreatePublication($input: CreatePublicationInput!) { + createPublication(input: $input) { + success errors { nonFieldErrors } data { id slug status } + } + } + """ + return self.post( + "/api/graphql", json_data={"query": mutation, "variables": {"input": data}} + ) + + def update(self, publication_id: str, data: Dict[str, Any]) -> Dict[str, Any]: + """Update a resource via the updatePublication mutation.""" + mutation = """ + mutation UpdatePublication($input: UpdatePublicationInput!) { + updatePublication(input: $input) { + success errors { nonFieldErrors } data { id title } + } + } + """ + payload = {"id": publication_id, **data} + return self.post( + "/api/graphql", + json_data={"query": mutation, "variables": {"input": payload}}, + ) + + def delete(self, publication_id: str) -> Dict[str, Any]: + """Delete a resource via the deletePublication mutation.""" + mutation = """ + mutation DeletePublication($publicationId: UUID!) { + deletePublication(publicationId: $publicationId) { success data } + } + """ + return self.post( + "/api/graphql", + json_data={"query": mutation, "variables": {"publicationId": publication_id}}, + ) diff --git a/docker-compose.hotreload.yml b/docker-compose.hotreload.yml new file mode 100644 index 00000000..d8fce17a --- /dev/null +++ b/docker-compose.hotreload.yml @@ -0,0 +1,9 @@ +# Opt-in overlay for local development: bind-mounts the working tree and +# runs uvicorn with --reload. Not used in any deployed environment. +# +# Usage: docker compose -f docker-compose.yml -f docker-compose.hotreload.yml up --build +services: + backend: + command: uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --reload + volumes: + - .:/code diff --git a/docker-compose.yml b/docker-compose.yml index e67e1ca1..3de7bc83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,20 @@ services: backend: + image: ${DATASPACE_IMAGE:-dataspace-backend:local} build: . env_file: .env container_name: "DataSpace" - command: uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --reload volumes: - - .:/code + # User-uploaded media (MEDIA_ROOT = BASE_DIR/files/public). This is + # persistent data, not code -- it lives on the host and cannot be + # baked into the image. Dropping the old `.:/code` bind mount for the + # image-based deploy removed the app's only access to it, so every + # file field lookup (e.g. an organization logo's `size`) raised + # FileNotFoundError and errored the GraphQL queries the dashboard + # depends on. Mount only this subtree, never the whole tree -- a + # full `.:/code` mount would shadow the deployed image. + - ./files:/code/files ports: - "8000:8000" depends_on: @@ -27,6 +35,25 @@ services: max-size: "10m" max-file: "3" + # One-off management commands (migrations, etc.) against the deployed + # image, without touching the running `backend` container. No + # container_name, so `docker compose run` can spin up a fresh instance + # even while `backend` is up. Never starts on a plain `up`. + release: + image: ${DATASPACE_IMAGE:-dataspace-backend:local} + build: . + env_file: .env + profiles: ["release"] + volumes: + # Same media mount as `backend` -- management commands run here too + # and some of them touch uploaded files. + - ./files:/code/files + depends_on: + backend_db: + condition: service_healthy + entrypoint: ["python", "manage.py"] + command: ["migrate", "--noinput"] + backend_db: image: "postgres:14.4" env_file: .env diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 10d7bf4d..a54011c6 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -39,17 +39,12 @@ while True: time.sleep(2) END -# Ensure migrations directory exists with proper permissions -echo "Ensuring migrations directory exists..." -mkdir -p /code/api/migrations -chmod -R 777 /code/api/migrations -touch /code/api/migrations/__init__.py - -# Run makemigrations first to ensure migration files are created -echo "Running makemigrations..." -python manage.py makemigrations --noinput - -# Run migrations +# Run migrations. In the ECS pipeline this also runs earlier as an explicit +# one-off task before this service is deployed (see deploy-to-ecs.yml) so a +# broken migration fails the deploy loudly and visibly instead of surfacing +# only once containers are already shipping; this call is then a no-op +# (migrate is idempotent). Kept here unconditionally too, since this same +# entrypoint/image is what `docker compose up` uses for local dev. echo "Running migrations..." python manage.py migrate --noinput diff --git a/pyproject.toml b/pyproject.toml index 93dcb1d7..0a8ce32c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dataspace-sdk" -version = "0.5.02" +version = "0.5.05" description = "Python SDK for DataSpace API" readme = "docs/sdk/README.md" requires-python = ">=3.8" @@ -54,7 +54,7 @@ include = '\.pyi?$' [tool.mypy] -python_version = "0.5.02" +python_version = "0.5.05" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = false diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh new file mode 100755 index 00000000..2b6f374f --- /dev/null +++ b/scripts/ci-deploy.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Deploy step logic, run on the dev host by the "Deploy" job in +# .github/workflows/deploy-backend.yml. Lives as a real file rather than an +# inline appleboy/ssh-action `script:` block -- ParakhAPI's equivalent +# pipeline hit a reproducible "syntax error near unexpected token ';'" from +# inline multi-line scripts sent through that action (confirmed the script +# text itself was valid bash both locally and on the target host every +# time; the corruption happened somewhere in the action's own transport, +# not in the script content). scp-action, which ships this file, has no +# such problem -- so complex logic never goes through the SSH action's +# inline `script:` here at all. +# +# Invoked as: sudo bash ci-deploy.sh "$IMAGE_REF" "$HEALTH_CHECK_URL" "$GHCR_USER" +# GHCR_TOKEN is read from .ghcr_token (shipped alongside this script, +# deleted immediately below) rather than passed as an argument, since +# argv is visible via ps aux for the process's lifetime. +# +# docker/docker compose need sudo on this host -- confirmed passwordless +# (sudo -n succeeds), so this script must itself be invoked with sudo. +set -euo pipefail + +IMAGE_REF="$1" +HEALTH_CHECK_URL="$2" +GHCR_USER="$3" + +mkdir -p .deploy +GHCR_TOKEN="$(cat .ghcr_token)" +rm -f .ghcr_token + +# --- preflight ------------------------------------------------- +AVAIL_MB=$(df -Pm . | awk "NR==2{print \$4}") +if [ "$AVAIL_MB" -lt 4096 ]; then + echo "::error::Only ${AVAIL_MB}MB free on the deploy volume; refusing to pull. Free space and re-run." + exit 1 +fi +docker compose version >/dev/null 2>&1 || { + echo "::error::Docker Compose V2 not available (V1 docker-compose is a different, incompatible tool)." + exit 1 +} + +# --- record rollback anchor BEFORE touching anything ----------- +PREV_REF="$(docker inspect --format "{{.Config.Image}}" DataSpace 2>/dev/null || true)" +if [ -z "$PREV_REF" ] && [ -f .deploy/current_image ]; then + PREV_REF="$(cat .deploy/current_image)" +fi +# Only a digest ref is safely rollback-able. Anything else (a tag, +# a locally-built image, or nothing) means the previous state was +# hand-managed -- record that honestly instead of writing a value +# a later rollback would deploy blindly. The very first image-based +# deploy will land here: the currently running container was built +# locally (`dataexbackend-backend`), not pulled by digest. +case "$PREV_REF" in + *@sha256:*) : ;; + *) + echo "::warning::No digest-pinned previous image found (got [${PREV_REF:-}]). Automatic rollback is UNAVAILABLE for this run." + PREV_REF="" + ;; +esac +printf "%s" "$PREV_REF" > .deploy/previous_image + +# --- pull the new image ---------------------------------------- +# GHCR's toomanyrequests kept firing on every one of 5 attempts 5s apart +# (~25s total) in live testing despite each error reporting a sub-second +# retry-after -- that points to a sustained account-level quota, not a +# brief burst, so this is deliberately patient: 10 attempts, 30s apart, +# ~5 minutes of runway before giving up. +echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin +for attempt in 1 2 3 4 5 6 7 8 9 10; do + docker pull "$IMAGE_REF" && break + if [ "$attempt" -eq 10 ]; then + echo "::error::docker pull failed after 10 attempts." + exit 1 + fi + echo "pull attempt $attempt failed, retrying in 30s..." + sleep 30 +done +printf "DATASPACE_IMAGE=%s\n" "$IMAGE_REF" > .deploy/image.env + +COMPOSE="docker compose -f docker-compose.yml --env-file .env --env-file .deploy/image.env" +RELEASE="$COMPOSE --profile release run --rm --no-deps -T release" +# The release service's entrypoint is already ["python", "manage.py"] +# (see docker-compose.yml) -- args below are manage.py subcommands +# only, not full "python manage.py ..." invocations. + +# backend_db/elasticsearch/redis must be up before the release step below: +# it uses --no-deps (so compose never recreates them out from under a +# running deploy), which also means it will NOT wait for their +# depends_on healthchecks. In steady state these are already running, +# but a host that had the stack fully down would otherwise fail at +# migrate with a confusing connection error. up -d is idempotent -- +# no-ops when they are already healthy and their config is unchanged. +$COMPOSE up -d backend_db elasticsearch redis + +# --- release step, against the NEW image, before the swap ------ +# Recorded for the rollback message: a rollback restores the image +# but NOT the schema, so whoever reads that failure needs to know +# what was applied. +$RELEASE showmigrations --plan 2>/dev/null | grep "^\[ \]" > .deploy/migrations.txt || true +$RELEASE migrate --noinput + +# Ensure every search index exists before populating. Neither flag +# combination alone is safe here, confirmed live: +# - `search_index --create --populate` in one call skips already-existing +# indices safely, but ALSO silently skips creating one whose queryset is +# currently empty (index_not_found_exception on every unified-search +# request afterward, despite the command reporting "Indexing 0 'X' +# objects" as if nothing were wrong). +# - `search_index --create` alone is NOT idempotent: it hard-errors with +# resource_already_exists_exception on the first already-existing index +# it hits and stops there, so any index processed after it in the same +# invocation never gets attempted. +# Creating each model's index individually, each tolerating "already +# exists" on its own, gets the correctness of both without either failure +# mode. Keep this list in sync with DataSpace/settings.py's +# ELASTICSEARCH_INDEX_NAMES if a new search document is ever added. +for model in api.Dataset api.UseCase api.AIModel api.Publication api.Collaborative api.Organization authorization.User; do + $RELEASE search_index --create --models "$model" -f || true +done +$RELEASE search_index --populate -f + +# --- swap the running container --------------------------------- +# --no-deps so backend_db/elasticsearch/redis are never recreated +# out from under this. +$COMPOSE up -d --no-build --no-deps --force-recreate backend + +# --- health gate, with in-job rollback (tier 1) ------------------ +rollback_now() { + echo "::error::$1" + if [ -z "$PREV_REF" ]; then + echo "::error::No previous image recorded -- the NEW image is still live. Manual intervention required." + exit 1 + fi + echo "Restoring $PREV_REF" + printf "DATASPACE_IMAGE=%s\n" "$PREV_REF" > .deploy/image.env + $COMPOSE up -d --no-build --no-deps --force-recreate backend + exit 1 +} + +attempts=0 +until curl -fsS -o /dev/null --max-time 10 "$HEALTH_CHECK_URL"; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 20 ]; then + # $COMPOSE, not a bare docker compose -f ...: an explicit + # --env-file disables .env auto-discovery, and DATASPACE_IMAGE + # needs to keep resolving to the image just deployed for these + # logs to target the right container. + $COMPOSE logs --tail 50 backend || true + rollback_now "Deployed image did not become healthy after $attempts attempts." + fi + sleep 5 +done + +printf "%s" "$IMAGE_REF" > .deploy/current_image +echo "Deploy healthy: $IMAGE_REF" diff --git a/scripts/ci-finalize.sh b/scripts/ci-finalize.sh new file mode 100755 index 00000000..01691b15 --- /dev/null +++ b/scripts/ci-finalize.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Finalize-deploy image pruning logic, run on the dev host. See +# ci-deploy.sh's header comment for why this lives as a real file instead +# of an inline appleboy/ssh-action script. No secrets needed here. +set -euo pipefail + +CUR="$(cat .deploy/current_image 2>/dev/null || true)" +PREV="$(cat .deploy/previous_image 2>/dev/null || true)" + +# Keep the previous image: it is what makes rollback instant, and +# survivable even if GHCR is unreachable during an incident. +# Build the ref list explicitly rather than relying on unquoted +# expansion to drop an empty PREV -- PREV is legitimately empty on +# a first deploy, and passing "" to docker inspect is an error. +KEEP_REFS=() +[ -n "$CUR" ] && KEEP_REFS+=("$CUR") +[ -n "$PREV" ] && KEEP_REFS+=("$PREV") +KEEP="" +if [ ${#KEEP_REFS[@]} -gt 0 ]; then + KEEP="$(docker inspect --format "{{.Id}}" "${KEEP_REFS[@]}" 2>/dev/null | sort -u || true)" +fi + +# Guard against the degenerate case: if we somehow resolved +# nothing to keep, pruning by ID below would remove every image +# for this repo including the one currently running. Bail instead. +if [ -z "$KEEP" ]; then + echo "::warning::Could not resolve current/previous image IDs; skipping prune rather than risk removing the running image." + docker image prune -f + df -h . + exit 0 +fi + +for id in $(docker images --no-trunc --format "{{.ID}}" "ghcr.io/civicdatalab/dataspacebackend" 2>/dev/null); do + if ! printf "%s\n" "$KEEP" | grep -q "$id"; then + docker rmi "$id" 2>/dev/null || true + fi +done + +# Dangling layers only. NEVER `docker system prune -a` here -- it +# would remove the previous image and silently disable rollback. +docker image prune -f +df -h . diff --git a/scripts/ci-rollback.sh b/scripts/ci-rollback.sh new file mode 100755 index 00000000..10dab053 --- /dev/null +++ b/scripts/ci-rollback.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Rollback-on-smoke-failure logic, run on the dev host. See ci-deploy.sh's +# header comment for why this lives as a real file instead of an inline +# appleboy/ssh-action script. +# +# Invoked as: sudo bash ci-rollback.sh "$HEALTH_CHECK_URL" "$GHCR_USER" +# GHCR_TOKEN is read from .ghcr_token (shipped alongside this script, +# deleted immediately below) rather than passed as an argument. +set -euo pipefail + +HEALTH_CHECK_URL="$1" +GHCR_USER="$2" + +GHCR_TOKEN="$(cat .ghcr_token)" +rm -f .ghcr_token + +PREV_REF="$(cat .deploy/previous_image 2>/dev/null || true)" +if [ -z "$PREV_REF" ]; then + echo "::error::Smoke tests failed but no previous image is recorded -- the NEW image is still live. Manual intervention required." + exit 1 +fi + +# finalize-deploy is the only pruner and it did not run, so the +# previous image should still be local. Pull is a safety net. +# See ci-deploy.sh for why this retries as patiently as it does -- +# GHCR's toomanyrequests turned out to be a sustained account-level +# quota, not a brief burst, in live testing. +docker image inspect "$PREV_REF" >/dev/null 2>&1 || { + echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin + for attempt in 1 2 3 4 5 6 7 8 9 10; do + docker pull "$PREV_REF" && break + if [ "$attempt" -eq 10 ]; then + echo "::error::docker pull failed after 10 attempts." + exit 1 + fi + echo "pull attempt $attempt failed, retrying in 30s..." + sleep 30 + done +} + +printf "DATASPACE_IMAGE=%s\n" "$PREV_REF" > .deploy/image.env +COMPOSE="docker compose -f docker-compose.yml --env-file .env --env-file .deploy/image.env" + +$COMPOSE up -d --no-build --no-deps --force-recreate backend + +attempts=0 +until curl -fsS -o /dev/null --max-time 10 "$HEALTH_CHECK_URL"; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 20 ]; then + echo "::error::Rolled-back image did not become healthy after $attempts attempts." + exit 1 + fi + sleep 5 +done + +printf "%s" "$PREV_REF" > .deploy/current_image +echo "Rolled back to $PREV_REF" +echo "NOTE: migrations applied by the failed deploy were NOT reverted:" +cat .deploy/migrations.txt 2>/dev/null || echo " (none recorded)" diff --git a/search/documents/__init__.py b/search/documents/__init__.py index 96a2345c..4c6a678a 100644 --- a/search/documents/__init__.py +++ b/search/documents/__init__.py @@ -1,6 +1,7 @@ from search.documents.aimodel_document import AIModelDocument from search.documents.collaborative_document import CollaborativeDocument from search.documents.dataset_document import DatasetDocument +from search.documents.publication_document import PublicationDocument from search.documents.publisher_document import ( OrganizationPublisherDocument, UserPublisherDocument, diff --git a/search/documents/publication_document.py b/search/documents/publication_document.py new file mode 100644 index 00000000..d50ec9f4 --- /dev/null +++ b/search/documents/publication_document.py @@ -0,0 +1,154 @@ +"""Elasticsearch document for Publication (UI "Resource"). + +v1 indexes real columns only — title, description, status, resource_type, +sectors, geographies, owner and dates. The author / date / usage-rights columns +and block content are intentionally NOT indexed (see the plan's Limitations). +``related_models`` is stated explicitly (not cloned blind) so that renaming a +Resource Type, sector or geography re-indexes the affected publications. +""" + +from typing import Any, Dict, List, Optional, Union + +from django_elasticsearch_dsl import Document, Index, KeywordField, fields + +from api.models.Geography import Geography +from api.models.Organization import Organization +from api.models.Publication import Publication +from api.models.ResourceType import ResourceType +from api.models.Sector import Sector +from api.utils.enums import PublicationStatus +from authorization.models import User +from DataSpace import settings +from search.documents.analysers import html_strip, ngram_analyser + +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) +INDEX.settings(number_of_shards=1, number_of_replicas=0) + + +@INDEX.doc_type +class PublicationDocument(Document): + """Elasticsearch document for a published Resource.""" + + title = fields.TextField( + analyzer=ngram_analyser, + fields={"raw": KeywordField(multi=False)}, + ) + description = fields.TextField( + analyzer=html_strip, + fields={"raw": fields.TextField(analyzer="keyword")}, + ) + status = fields.KeywordField() + + # Resource Type facet (name). + resource_type = fields.TextField( + attr="resource_type_indexing", + analyzer=ngram_analyser, + fields={"raw": KeywordField(multi=False)}, + ) + + # Sectors facet (ManyToMany). + sectors = fields.TextField( + attr="sectors_indexing", + analyzer=ngram_analyser, + fields={ + "raw": fields.KeywordField(multi=True), + "suggest": fields.CompletionField(multi=True), + }, + multi=True, + ) + + # Geographies facet (ManyToMany). + geographies = fields.TextField( + attr="geographies_indexing", + analyzer=ngram_analyser, + fields={ + "raw": fields.KeywordField(multi=True), + "suggest": fields.CompletionField(multi=True), + }, + multi=True, + ) + + organization = fields.NestedField( + properties={ + "name": fields.TextField(analyzer=ngram_analyser), + "logo": fields.TextField(analyzer=ngram_analyser), + } + ) + user = fields.NestedField( + properties={ + "name": fields.TextField(analyzer=ngram_analyser), + "bio": fields.TextField(analyzer=html_strip), + "profile_picture": fields.TextField(analyzer=ngram_analyser), + } + ) + + def prepare_organization(self, instance: Publication) -> Optional[Dict[str, str]]: + """Prepare the owning organization for indexing.""" + if instance.organization: + org = instance.organization + return {"name": org.name, "logo": org.logo.url if org.logo else ""} + return None + + def prepare_user(self, instance: Publication) -> Optional[Dict[str, str]]: + """Prepare the owning user for indexing.""" + if instance.user: + user = instance.user + return { + "name": user.full_name, + "bio": user.bio or "", + "profile_picture": (user.profile_picture.url if user.profile_picture else ""), + } + return None + + def should_index_object(self, obj: Any) -> bool: + """Only PUBLISHED resources are indexed — drafts never reach search.""" + return bool(obj.status == PublicationStatus.PUBLISHED.value) + + def save(self, *args: Any, **kwargs: Any) -> None: + """Index a published resource, or drop it from the index otherwise.""" + if self.should_index_object(self.to_dict()): # type: ignore + super().save(*args, **kwargs) + else: + self.delete(ignore=404) + + def get_queryset(self) -> Any: + """Only published resources are populated into the index.""" + return ( + super(PublicationDocument, self) + .get_queryset() + .filter(status=PublicationStatus.PUBLISHED) + ) + + def get_instances_from_related( + self, + related_instance: Union[Organization, User, ResourceType, Sector, Geography], + ) -> Optional[List[Publication]]: + """Re-index the publications affected when a related row changes. + + Covers a renamed Resource Type / sector / geography and an updated owner + so already-indexed facets don't go stale. + """ + if isinstance(related_instance, Organization): + return list(related_instance.publications.all()) + if isinstance(related_instance, User): + return list(related_instance.publications.all()) + if isinstance(related_instance, ResourceType): + return list(related_instance.publications.all()) + if isinstance(related_instance, Sector): + return list(related_instance.publications.all()) + if isinstance(related_instance, Geography): + return list(related_instance.publications.all()) + return None + + class Django: + """Django model configuration.""" + + model = Publication + + fields = [ + "id", + "created", + "modified", + ] + + related_models = [Organization, User, ResourceType, Sector, Geography] diff --git a/tests/journeys/publications/create-blocks-publish.py b/tests/journeys/publications/create-blocks-publish.py new file mode 100644 index 00000000..62f84a20 --- /dev/null +++ b/tests/journeys/publications/create-blocks-publish.py @@ -0,0 +1,99 @@ +""" +Journey: create a Resource, add a PDF block and a YouTube block, set metadata, +reorder, publish, then fetch it anonymously and confirm it's visible with its +blocks in order. + +On-demand (Layer 5) — runs against a live backend. Reads KEYCLOAK_TEST_TOKEN +and TEST_BASE_URL from env. Fails loudly on the first assertion that fails. + +Usage: + KEYCLOAK_TEST_TOKEN=... TEST_BASE_URL=http://localhost:8000 \ + python tests/journeys/publications/create-blocks-publish.py +""" + +import os + +import requests + +base = os.environ.get("TEST_BASE_URL", "http://localhost:8000") +token = os.environ["KEYCLOAK_TEST_TOKEN"] +headers = {"Authorization": f"Bearer {token}"} +graphql = f"{base}/api/graphql" + + +def gql(query, variables=None, files=None): + """Post a GraphQL operation (multipart when files are given).""" + if files: + return requests.post(graphql, data=files, headers=headers) + res = requests.post( + graphql, json={"query": query, "variables": variables or {}}, headers=headers + ) + assert res.status_code == 200, res.text + body = res.json() + assert not body.get("errors"), body["errors"] + return body["data"] + + +# 1. Create a DRAFT resource with full metadata (ids below are placeholders — +# fill in a real resource type / sector / geography id from your test data). +create = gql( + """ + mutation($input: CreatePublicationInput!) { + createPublication(input: $input) { + success errors { fieldErrors { field messages } } data { id status } + } + } + """, + { + "input": { + "title": "Journey Findings", + "description": "A journey-test resource.", + "authors": ["Journey Bot"], + "publicationDate": "2024-01-01", + "license": "CC_BY_4_0_ATTRIBUTION", + "resourceTypeId": os.environ.get("TEST_RESOURCE_TYPE_ID", "REPLACE_ME"), + "sectorIds": [os.environ.get("TEST_SECTOR_ID", "REPLACE_ME")], + "geographyIds": [int(os.environ.get("TEST_GEOGRAPHY_ID", "1"))], + } + }, +) +assert create["createPublication"]["success"], create +publication_id = create["createPublication"]["data"]["id"] +assert create["createPublication"]["data"]["status"] == "DRAFT" + +# 2. Add a YouTube block (a PDF block is added via the multipart upload path the +# frontend uses — see ResourceDropzone; scripted upload builds the GraphQL +# multipart request the same way). +yt = gql( + """ + mutation($id: UUID!, $url: String!) { + addPublicationYoutubeBlock(publicationId: $id, youtubeUrl: $url) { + success data { id position blockType } + } + } + """, + {"id": publication_id, "url": "https://youtu.be/dQw4w9WgXcQ"}, +) +assert yt["addPublicationYoutubeBlock"]["success"], yt + +# 3. Publish it. +pub = gql( + "mutation($id: UUID!) { publishPublication(publicationId: $id) { success data { status } } }", + {"id": publication_id}, +) +assert pub["publishPublication"]["data"]["status"] == "PUBLISHED" + +# 4. Fetch anonymously and confirm it's visible with its blocks in order. +anon = requests.post( + graphql, + json={ + "query": "query($id: UUID!) { getPublication(publicationId: $id) { id status blocks { position } } }", + "variables": {"id": publication_id}, + }, +) +data = anon.json()["data"]["getPublication"] +assert data and data["status"] == "PUBLISHED", data +positions = [b["position"] for b in data["blocks"]] +assert positions == sorted(positions), positions + +print("PASS: create-blocks-publish") diff --git a/tests/journeys/publications/link-unpublish-relink.py b/tests/journeys/publications/link-unpublish-relink.py new file mode 100644 index 00000000..473b23f4 --- /dev/null +++ b/tests/journeys/publications/link-unpublish-relink.py @@ -0,0 +1,112 @@ +""" +Journey: create + publish a Resource, link it to a Use Case and a Collaborative, +confirm both render it, unpublish (both hide it), re-publish (both show it), +delete (both skip it and the linked-count stays consistent). + +On-demand (Layer 5) — runs against a live backend. Reads KEYCLOAK_TEST_TOKEN, +TEST_BASE_URL, TEST_USE_CASE_ID and TEST_COLLABORATIVE_ID (both DRAFT, owned by +the token's user/org) from env. Fails loudly on the first bad assertion. + +Usage: + KEYCLOAK_TEST_TOKEN=... TEST_BASE_URL=http://localhost:8000 \ + TEST_USE_CASE_ID=... TEST_COLLABORATIVE_ID=... \ + python tests/journeys/publications/link-unpublish-relink.py +""" + +import os + +import requests + +base = os.environ.get("TEST_BASE_URL", "http://localhost:8000") +token = os.environ["KEYCLOAK_TEST_TOKEN"] +headers = {"Authorization": f"Bearer {token}"} +graphql = f"{base}/api/graphql" +use_case_id = os.environ["TEST_USE_CASE_ID"] +collaborative_id = os.environ["TEST_COLLABORATIVE_ID"] + + +def gql(query, variables=None): + res = requests.post( + graphql, json={"query": query, "variables": variables or {}}, headers=headers + ) + assert res.status_code == 200, res.text + body = res.json() + assert not body.get("errors"), body["errors"] + return body["data"] + + +# 1. Create + publish a resource (metadata ids from env, see the sibling script). +create = gql( + """ + mutation($input: CreatePublicationInput!) { + createPublication(input: $input) { success data { id } } + } + """, + { + "input": { + "title": "Linkable Findings", + "description": "For the link journey.", + "authors": ["Journey Bot"], + "publicationDate": "2024-01-01", + "license": "CC_BY_4_0_ATTRIBUTION", + "resourceTypeId": os.environ.get("TEST_RESOURCE_TYPE_ID", "REPLACE_ME"), + "sectorIds": [os.environ.get("TEST_SECTOR_ID", "REPLACE_ME")], + "geographyIds": [int(os.environ.get("TEST_GEOGRAPHY_ID", "1"))], + } + }, +) +publication_id = create["createPublication"]["data"]["id"] +gql( + "mutation($id: UUID!) { publishPublication(publicationId: $id) { success } }", + {"id": publication_id}, +) + +# 2. Link it to the use case and the collaborative. +gql( + "mutation($u: String!, $p: UUID!) { addPublicationToUseCase(useCaseId: $u, publicationId: $p) { __typename } }", + {"u": use_case_id, "p": publication_id}, +) +gql( + "mutation($c: String!, $p: UUID!) { addPublicationToCollaborative(collaborativeId: $c, publicationId: $p) { __typename } }", + {"c": collaborative_id, "p": publication_id}, +) + + +def renders(entity, entity_id): + query = { + "usecase": "query($id: ID!) { useCase(pk: $id) { publications { id } } }", + "collab": "query($id: ID!) { collaborative(pk: $id) { publications { id } } }", + }[entity] + data = gql(query, {"id": entity_id}) + key = "useCase" if entity == "usecase" else "collaborative" + return [p["id"] for p in (data[key]["publications"] or [])] + + +# 3. Both render it while published. +assert publication_id in renders("usecase", use_case_id) +assert publication_id in renders("collab", collaborative_id) + +# 4. Unpublish → both hide it. +gql( + "mutation($id: UUID!) { unpublishPublication(publicationId: $id) { success } }", + {"id": publication_id}, +) +assert publication_id not in renders("usecase", use_case_id) +assert publication_id not in renders("collab", collaborative_id) + +# 5. Re-publish → both show it again. +gql( + "mutation($id: UUID!) { publishPublication(publicationId: $id) { success } }", + {"id": publication_id}, +) +assert publication_id in renders("usecase", use_case_id) + +# 6. Delete → both skip it silently. +gql( + "mutation($id: UUID!) { deletePublication(publicationId: $id) { success } }", + {"id": publication_id}, +) +assert publication_id not in renders("usecase", use_case_id) +assert publication_id not in renders("collab", collaborative_id) + +print("PASS: link-unpublish-relink") diff --git a/tests/schema/test_publication_schema.py b/tests/schema/test_publication_schema.py new file mode 100644 index 00000000..a0f21ef1 --- /dev/null +++ b/tests/schema/test_publication_schema.py @@ -0,0 +1,519 @@ +"""Layer 3/4 tests for the publication (Resource) GraphQL surface. + +Executes the real schema against the Django test DB with a fake context that +carries the caller's user and org — establishing the layer-4 pattern for this +repo. Activity recording is stubbed so tests exercise the mutation logic, not +the activity-stream plumbing. +""" + +import types +from datetime import date +from unittest.mock import patch + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Geography, Publication, ResourceType, Sector +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.enums import GeoTypes, PublicationStatus +from authorization.models import OrganizationMembership, Role, User + + +# --------------------------------------------------------------------------- # +# Fixtures +# --------------------------------------------------------------------------- # +@pytest.fixture(autouse=True) +def _no_activity_recording(): + with patch("api.schema.base_mutation.record_activity", return_value=None): + yield + + +@pytest.fixture +def roles(db): + admin = Role.objects.create( + name="admin", can_view=True, can_add=True, can_change=True, can_delete=True + ) + editor = Role.objects.create( + name="editor", can_view=True, can_add=True, can_change=True, can_delete=False + ) + auditor = Role.objects.create( + name="auditor", can_view=True, can_add=False, can_change=False, can_delete=False + ) + return {"admin": admin, "editor": editor, "auditor": auditor} + + +@pytest.fixture +def org_a(db): + return Organization.objects.create(name="Org A", description="a", slug="org-a") + + +@pytest.fixture +def org_b(db): + return Organization.objects.create(name="Org B", description="b", slug="org-b") + + +def _member(user, org, role): + OrganizationMembership.objects.create(user=user, organization=org, role=role) + return user + + +@pytest.fixture +def org_a_admin(roles, org_a): + return _member( + User.objects.create(username="a_admin", keycloak_id="a_admin"), org_a, roles["admin"] + ) + + +@pytest.fixture +def org_a_editor(roles, org_a): + return _member( + User.objects.create(username="a_editor", keycloak_id="a_editor"), org_a, roles["editor"] + ) + + +@pytest.fixture +def org_a_auditor(roles, org_a): + return _member( + User.objects.create(username="a_auditor", keycloak_id="a_auditor"), org_a, roles["auditor"] + ) + + +@pytest.fixture +def org_b_admin(roles, org_b): + return _member( + User.objects.create(username="b_admin", keycloak_id="b_admin"), org_b, roles["admin"] + ) + + +@pytest.fixture +def individual(db): + return User.objects.create(username="solo", keycloak_id="solo") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +@pytest.fixture +def inactive_type(db): + return ResourceType.objects.create(name="Retired", is_active=False) + + +@pytest.fixture +def sector(db): + return Sector.objects.create(name="Health") + + +@pytest.fixture +def geography(db): + return Geography.objects.create(name="India", code="IN", type=GeoTypes.COUNTRY) + + +def ctx(user, organization=None): + return types.SimpleNamespace( + user=user, + context={"organization": organization} if organization else {}, + ) + + +def run(query, context, variables=None): + return schema.execute_sync(query, variable_values=variables or {}, context_value=context) + + +def valid_create_vars(resource_type, sector, geography, **overrides): + variables = { + "input": { + "title": "Rainfall Findings", + "description": "A study of rainfall.", + "authors": ["Ada Lovelace"], + "publicationDate": "2024-01-01", + "license": "CC_BY_4_0_ATTRIBUTION", + "resourceTypeId": str(resource_type.id), + "sectorIds": [str(sector.id)], + "geographyIds": [geography.id], + } + } + variables["input"].update(overrides) + return variables + + +CREATE = """ +mutation Create($input: CreatePublicationInput!) { + createPublication(input: $input) { + success + errors { fieldErrors { field messages } nonFieldErrors } + data { id slug status isIndividualPublication organization { id } user { id } } + } +} +""" + +UPDATE = """ +mutation Update($input: UpdatePublicationInput!) { + updatePublication(input: $input) { + success + errors { fieldErrors { field messages } } + data { id title } + } +} +""" + +PUBLISH = """ +mutation Publish($id: UUID!) { + publishPublication(publicationId: $id) { success data { id status } } +} +""" + +UNPUBLISH = """ +mutation Unpublish($id: UUID!) { + unpublishPublication(publicationId: $id) { success data { id status } } +} +""" + +DELETE = """ +mutation Delete($id: UUID!) { + deletePublication(publicationId: $id) { success data } +} +""" + +GET = """ +query Get($id: UUID!) { + getPublication(publicationId: $id) { id status } +} +""" + +LIST = """ +query List($includePublic: Boolean) { + publications(includePublic: $includePublic) { id status } +} +""" + + +# --------------------------------------------------------------------------- # +# Create +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestCreate: + def test_org_member_creates_org_owned_draft( + self, org_a_admin, org_a, resource_type, sector, geography + ): + result = run( + CREATE, ctx(org_a_admin, org_a), valid_create_vars(resource_type, sector, geography) + ) + + assert result.errors is None + payload = result.data["createPublication"] + assert payload["success"] is True + assert payload["data"]["status"] == "DRAFT" + assert payload["data"]["organization"]["id"] == str(org_a.id) + assert payload["data"]["isIndividualPublication"] is False + assert Publication.objects.filter(id=payload["data"]["id"]).exists() + + def test_individual_creates_user_owned_draft( + self, individual, resource_type, sector, geography + ): + result = run(CREATE, ctx(individual), valid_create_vars(resource_type, sector, geography)) + + payload = result.data["createPublication"] + assert payload["success"] is True + assert payload["data"]["isIndividualPublication"] is True + assert payload["data"]["user"]["id"] == str(individual.id) + + def test_missing_title_is_rejected_without_creating( + self, individual, resource_type, sector, geography + ): + result = run( + CREATE, ctx(individual), valid_create_vars(resource_type, sector, geography, title="") + ) + + payload = result.data["createPublication"] + assert payload["success"] is False + assert Publication.objects.count() == 0 + + def test_inactive_resource_type_is_rejected(self, individual, inactive_type, sector, geography): + result = run(CREATE, ctx(individual), valid_create_vars(inactive_type, sector, geography)) + + payload = result.data["createPublication"] + assert payload["success"] is False + assert Publication.objects.count() == 0 + + def test_anonymous_cannot_create(self, resource_type, sector, geography): + result = run( + CREATE, ctx(AnonymousUser()), valid_create_vars(resource_type, sector, geography) + ) + + payload = result.data["createPublication"] + assert payload["success"] is False + assert Publication.objects.count() == 0 + + +# --------------------------------------------------------------------------- # +# Update / role gating +# --------------------------------------------------------------------------- # +def _make_publication(user, org, resource_type, status=PublicationStatus.DRAFT): + return Publication.objects.create( + title="Existing", + description="d", + authors=["A"], + publication_date=date(2024, 1, 1), + license="CC_BY_4_0_ATTRIBUTION", + resource_type=resource_type, + organization=org, + user=user, + status=status, + ) + + +@pytest.mark.django_db +class TestUpdateAndRoles: + def test_editor_updates_org_publication(self, org_a_admin, org_a_editor, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + UPDATE, + ctx(org_a_editor, org_a), + {"input": {"id": str(publication.id), "title": "New Title"}}, + ) + + assert result.data["updatePublication"]["success"] is True + publication.refresh_from_db() + assert publication.title == "New Title" + + def test_auditor_cannot_update(self, org_a_admin, org_a_auditor, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + UPDATE, + ctx(org_a_auditor, org_a), + {"input": {"id": str(publication.id), "title": "Hijack"}}, + ) + + assert result.data["updatePublication"]["success"] is False + publication.refresh_from_db() + assert publication.title == "Existing" + + def test_auditor_can_read_draft(self, org_a_admin, org_a_auditor, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(GET, ctx(org_a_auditor, org_a), {"id": str(publication.id)}) + + assert result.errors is None + assert result.data["getPublication"]["id"] == str(publication.id) + + +# --------------------------------------------------------------------------- # +# Publish / unpublish +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestPublish: + def test_admin_publishes(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(PUBLISH, ctx(org_a_admin, org_a), {"id": str(publication.id)}) + + assert result.data["publishPublication"]["success"] is True + publication.refresh_from_db() + assert publication.status == PublicationStatus.PUBLISHED + + def test_auditor_cannot_publish(self, org_a_admin, org_a_auditor, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(PUBLISH, ctx(org_a_auditor, org_a), {"id": str(publication.id)}) + + assert result.data["publishPublication"]["success"] is False + publication.refresh_from_db() + assert publication.status == PublicationStatus.DRAFT + + def test_unpublish_reverts_to_draft(self, org_a_admin, org_a, resource_type): + publication = _make_publication( + org_a_admin, org_a, resource_type, status=PublicationStatus.PUBLISHED + ) + + result = run(UNPUBLISH, ctx(org_a_admin, org_a), {"id": str(publication.id)}) + + assert result.data["unpublishPublication"]["success"] is True + publication.refresh_from_db() + assert publication.status == PublicationStatus.DRAFT + + +# --------------------------------------------------------------------------- # +# Cross-org denial +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestCrossOrgDenial: + def test_other_org_cannot_update(self, org_a_admin, org_b_admin, org_a, org_b, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + UPDATE, + ctx(org_b_admin, org_b), + {"input": {"id": str(publication.id), "title": "Steal"}}, + ) + + assert result.data["updatePublication"]["success"] is False + publication.refresh_from_db() + assert publication.title == "Existing" + + def test_other_org_cannot_delete(self, org_a_admin, org_b_admin, org_a, org_b, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(DELETE, ctx(org_b_admin, org_b), {"id": str(publication.id)}) + + assert result.data["deletePublication"]["success"] is False + assert Publication.objects.filter(id=publication.id).exists() + + def test_other_org_cannot_publish(self, org_a_admin, org_b_admin, org_a, org_b, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(PUBLISH, ctx(org_b_admin, org_b), {"id": str(publication.id)}) + + assert result.data["publishPublication"]["success"] is False + publication.refresh_from_db() + assert publication.status == PublicationStatus.DRAFT + + +# --------------------------------------------------------------------------- # +# Delete +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestDelete: + def test_owner_deletes(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(DELETE, ctx(org_a_admin, org_a), {"id": str(publication.id)}) + + assert result.data["deletePublication"]["success"] is True + assert not Publication.objects.filter(id=publication.id).exists() + + +# --------------------------------------------------------------------------- # +# Read gating + listing +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestReadGating: + def test_anonymous_sees_published_detail(self, org_a_admin, org_a, resource_type): + publication = _make_publication( + org_a_admin, org_a, resource_type, status=PublicationStatus.PUBLISHED + ) + + result = run(GET, ctx(AnonymousUser()), {"id": str(publication.id)}) + + assert result.errors is None + assert result.data["getPublication"]["id"] == str(publication.id) + + def test_anonymous_denied_draft_detail(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run(GET, ctx(AnonymousUser()), {"id": str(publication.id)}) + + assert result.errors is not None # permission denied, not a silent leak + + def test_listing_is_org_scoped(self, org_a_admin, org_b_admin, org_a, org_b, resource_type): + _make_publication(org_a_admin, org_a, resource_type) + _make_publication(org_b_admin, org_b, resource_type) + + result = run(LIST, ctx(org_a_admin, org_a), {"includePublic": False}) + + assert result.errors is None + assert len(result.data["publications"]) == 1 + + def test_anonymous_listing_is_published_only(self, org_a_admin, org_a, resource_type): + _make_publication(org_a_admin, org_a, resource_type, status=PublicationStatus.DRAFT) + _make_publication(org_a_admin, org_a, resource_type, status=PublicationStatus.PUBLISHED) + + result = run(LIST, ctx(AnonymousUser()), {"includePublic": False}) + + statuses = [row["status"] for row in result.data["publications"]] + assert statuses == ["PUBLISHED"] + + +# --------------------------------------------------------------------------- # +# Content-block mutations (wiring + cross-org gate) +# --------------------------------------------------------------------------- # +ADD_YOUTUBE = """ +mutation AddYt($id: UUID!, $url: String!) { + addPublicationYoutubeBlock(publicationId: $id, youtubeUrl: $url) { + success + data { id blockType youtubeVideoId position } + } +} +""" + +REMOVE_BLOCK = """ +mutation RemoveBlock($blockId: UUID!) { + removePublicationBlock(blockId: $blockId) { success data } +} +""" + + +@pytest.mark.django_db +class TestBlockMutations: + def test_org_member_adds_youtube_block(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + ADD_YOUTUBE, + ctx(org_a_admin, org_a), + {"id": str(publication.id), "url": "https://youtu.be/dQw4w9WgXcQ"}, + ) + + payload = result.data["addPublicationYoutubeBlock"] + assert payload["success"] is True + assert payload["data"]["youtubeVideoId"] == "dQw4w9WgXcQ" + assert publication.blocks.count() == 1 + + def test_invalid_youtube_url_is_rejected(self, org_a_admin, org_a, resource_type): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + ADD_YOUTUBE, + ctx(org_a_admin, org_a), + {"id": str(publication.id), "url": "https://vimeo.com/1"}, + ) + + assert result.data["addPublicationYoutubeBlock"]["success"] is False + assert publication.blocks.count() == 0 + + def test_other_org_cannot_add_block( + self, org_a_admin, org_b_admin, org_a, org_b, resource_type + ): + publication = _make_publication(org_a_admin, org_a, resource_type) + + result = run( + ADD_YOUTUBE, + ctx(org_b_admin, org_b), + {"id": str(publication.id), "url": "https://youtu.be/dQw4w9WgXcQ"}, + ) + + assert result.data["addPublicationYoutubeBlock"]["success"] is False + assert publication.blocks.count() == 0 + + def test_other_org_cannot_remove_block( + self, org_a_admin, org_b_admin, org_a, org_b, resource_type + ): + publication = _make_publication(org_a_admin, org_a, resource_type) + block = publication.blocks.create( + position=0, + block_type="YOUTUBE", + youtube_url="https://youtu.be/dQw4w9WgXcQ", + youtube_video_id="dQw4w9WgXcQ", + ) + + result = run(REMOVE_BLOCK, ctx(org_b_admin, org_b), {"blockId": str(block.id)}) + + assert result.data["removePublicationBlock"]["success"] is False + assert publication.blocks.filter(id=block.id).exists() + + +RESOURCE_TYPES = "query { resourceTypes { id name isActive } }" + + +@pytest.mark.django_db +class TestResourceTypesQuery: + def test_lists_only_active_types_sorted(self, resource_type, inactive_type): + result = run(RESOURCE_TYPES, ctx(AnonymousUser())) + + assert result.errors is None + names = [t["name"] for t in result.data["resourceTypes"]] + assert names == ["Report"] # active only; "Retired" excluded diff --git a/tests/test_auth.py b/tests/test_auth.py index cf0212c4..2984ff39 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -187,5 +187,44 @@ def test_login_as_service_account_no_secret(self) -> None: self.auth_client.login_as_service_account() +class TestRealmUrl(unittest.TestCase): + """Keycloak's relative path is deployment-specific, not always /auth.""" + + def _url(self, **kwargs: object) -> str: + return AuthClient( + base_url="https://api.test.com", keycloak_realm="DataSpace", **kwargs + )._realm_url() + + def test_defaults_to_auth_for_backwards_compatibility(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com"), + "https://kc.test.com/auth/realms/DataSpace", + ) + + def test_empty_base_path_for_root_hosted_keycloak(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com", keycloak_base_path=""), + "https://kc.test.com/realms/DataSpace", + ) + + def test_does_not_double_path_already_in_url(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com/auth"), + "https://kc.test.com/auth/realms/DataSpace", + ) + + def test_custom_relative_path(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com", keycloak_base_path="sso"), + "https://kc.test.com/sso/realms/DataSpace", + ) + + def test_slash_only_base_path_is_treated_as_root(self) -> None: + self.assertEqual( + self._url(keycloak_url="https://kc.test.com/", keycloak_base_path="/"), + "https://kc.test.com/realms/DataSpace", + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_client.py b/tests/test_client.py index 7d27d44d..6a39b3b6 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -20,8 +20,14 @@ def test_init(self) -> None: self.assertIsNotNone(self.client._auth) self.assertIsNotNone(self.client.datasets) self.assertIsNotNone(self.client.aimodels) + self.assertIsNotNone(self.client.publications) self.assertIsNotNone(self.client.usecases) + def test_set_organization_scopes_publications(self) -> None: + """set_organization must set the org header on the publications client too.""" + self.client.set_organization("org-123") + self.assertEqual(self.client.publications.default_headers["organization"], "org-123") + @patch("dataspace_sdk.client.AuthClient.login") def test_login(self, mock_login: MagicMock) -> None: """Test login method with username/password.""" @@ -85,5 +91,26 @@ def test_user_property(self) -> None: self.assertIsNone(self.client.user) +class TestClientForwardsBasePath(unittest.TestCase): + """The regression this covers: AuthClient grew keycloak_base_path but + DataSpaceClient did not forward it, so no public consumer could ever set + it -- constructing DataSpaceClient(..., keycloak_base_path="") silently + behaved the same as not passing it at all. + """ + + def test_default_matches_auth_client_default(self) -> None: + client = DataSpaceClient(base_url="https://api.test.com", keycloak_url="https://kc.test.com", keycloak_realm="DataSpace") + self.assertEqual(client._auth._realm_url(), "https://kc.test.com/auth/realms/DataSpace") + + def test_root_path_is_forwarded(self) -> None: + client = DataSpaceClient( + base_url="https://api.test.com", + keycloak_url="https://kc.test.com", + keycloak_realm="DataSpace", + keycloak_base_path="", + ) + self.assertEqual(client._auth._realm_url(), "https://kc.test.com/realms/DataSpace") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 00000000..93a6b28c --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,99 @@ +"""Tests for the /health/ endpoint's status-code and git_sha behavior.""" + +import os +import unittest +from contextlib import ExitStack +from unittest.mock import MagicMock, patch + +from django.test import Client, override_settings + + +class TestHealthCheck(unittest.TestCase): + """The endpoint must reflect actual dependency health in its status code.""" + + def setUp(self) -> None: + self.client = Client() + + def _mock_healthy_dependencies(self, stack: ExitStack) -> None: + """Patch ES/Redis/telemetry so the happy path doesn't need real services.""" + # tests/test_settings.py's ELASTICSEARCH_DSL omits http_auth (the real + # DataSpace/settings.py value always sets it) — health_check reads it + # unconditionally, so supply it here rather than touching test settings. + stack.enter_context( + override_settings( + ELASTICSEARCH_DSL={ + "default": {"hosts": "localhost:9200", "http_auth": ("user", "pass")} + } + ) + ) + mock_es_instance = MagicMock() + mock_es_instance.ping.return_value = True + stack.enter_context( + patch("api.views.health.Elasticsearch", return_value=mock_es_instance) + ) + # `cache` is Django's lazy DefaultConnectionProxy — patching .set/.get + # as attributes on it gets forwarded to the real backend instead of + # being intercepted, so replace the name binding in the health module + # wholesale instead. + cache_store: dict = {} + mock_cache = MagicMock() + mock_cache.set.side_effect = lambda k, v, timeout=None: cache_store.__setitem__(k, v) + mock_cache.get.side_effect = lambda k: cache_store.get(k) + stack.enter_context(patch("api.views.health.cache", mock_cache)) + mock_get = stack.enter_context(patch("api.views.health.requests.get")) + mock_get.return_value.status_code = 200 + + def test_returns_200_when_all_dependencies_healthy(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + response = self.client.get("/health/") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["status"], "healthy") + + def test_returns_503_when_elasticsearch_unhealthy(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + mock_es_instance = MagicMock() + mock_es_instance.ping.return_value = False + stack.enter_context( + patch("api.views.health.Elasticsearch", return_value=mock_es_instance) + ) + response = self.client.get("/health/") + + self.assertEqual(response.status_code, 503) + body = response.json() + self.assertEqual(body["status"], "unhealthy") + self.assertEqual(body["services"]["elasticsearch"]["status"], "unhealthy") + + def test_returns_503_when_redis_unhealthy(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + mock_cache = MagicMock() + mock_cache.set.side_effect = Exception("down") + stack.enter_context(patch("api.views.health.cache", mock_cache)) + response = self.client.get("/health/") + + self.assertEqual(response.status_code, 503) + self.assertEqual(response.json()["services"]["redis"]["status"], "unhealthy") + + def test_git_sha_defaults_to_unknown(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + stack.enter_context(patch.dict(os.environ, {}, clear=False)) + os.environ.pop("GIT_COMMIT_SHA", None) + response = self.client.get("/health/") + + self.assertEqual(response.json()["git_sha"], "unknown") + + def test_git_sha_reflects_env_var(self) -> None: + with ExitStack() as stack: + self._mock_healthy_dependencies(stack) + stack.enter_context(patch.dict(os.environ, {"GIT_COMMIT_SHA": "abc1234"})) + response = self.client.get("/health/") + + self.assertEqual(response.json()["git_sha"], "abc1234") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_publication_blocks.py b/tests/test_publication_blocks.py new file mode 100644 index 00000000..fbb1383a --- /dev/null +++ b/tests/test_publication_blocks.py @@ -0,0 +1,196 @@ +"""Layer 1/3/4 tests for content blocks: add/reorder/remove/replace + download gate.""" + +import os +from datetime import date + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import RequestFactory + +from api.models import Publication, PublicationBlock, ResourceType +from api.models.Organization import Organization +from api.services.publication_blocks import ( + add_file_block, + add_youtube_block, + remove_block, + reorder_blocks, + replace_block_file, +) +from api.utils.enums import PublicationBlockType, PublicationStatus +from api.views.publication_download_view import publication_block_download +from authorization.models import OrganizationMembership, Role, User + +VIDEO = "https://youtu.be/dQw4w9WgXcQ" + + +@pytest.fixture(autouse=True) +def media_root(settings, tmp_path): + settings.MEDIA_ROOT = str(tmp_path) + + +@pytest.fixture +def owner(db): + return User.objects.create(username="owner", keycloak_id="owner") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +@pytest.fixture +def publication(owner, resource_type): + return Publication.objects.create( + title="With Blocks", + user=owner, + resource_type=resource_type, + publication_date=date(2024, 1, 1), + ) + + +def _pdf(name="report.pdf"): + return SimpleUploadedFile(name, b"%PDF-1.7 content", content_type="application/pdf") + + +@pytest.mark.django_db +class TestAddBlocks: + def test_file_block_stores_metadata_and_position(self, publication): + block = add_file_block(publication, _pdf()) + + assert block.block_type == PublicationBlockType.FILE + assert block.file_format == "pdf" + assert block.file_size > 0 + assert block.position == 0 + + def test_youtube_block_extracts_video_id(self, publication): + block = add_youtube_block(publication, VIDEO) + + assert block.block_type == PublicationBlockType.YOUTUBE + assert block.youtube_video_id == "dQw4w9WgXcQ" + + def test_positions_increment_across_mixed_blocks(self, publication): + add_file_block(publication, _pdf("a.pdf")) + add_youtube_block(publication, VIDEO) + third = add_file_block(publication, _pdf("c.pdf")) + + assert third.position == 2 + + +@pytest.mark.django_db +class TestReorderAndRemove: + def test_removing_a_middle_block_renumbers_contiguously(self, publication): + a = add_youtube_block(publication, VIDEO) + b = add_youtube_block(publication, VIDEO) + c = add_youtube_block(publication, VIDEO) + + remove_block(b) + + positions = list(publication.blocks.order_by("position").values_list("position", flat=True)) + assert positions == [0, 1] + a.refresh_from_db() + c.refresh_from_db() + assert a.position == 0 and c.position == 1 + + def test_reorder_sets_positions_to_requested_order(self, publication): + a = add_youtube_block(publication, VIDEO) + b = add_youtube_block(publication, VIDEO) + c = add_youtube_block(publication, VIDEO) + + reorder_blocks(publication, [c.id, a.id, b.id]) + + a.refresh_from_db() + b.refresh_from_db() + c.refresh_from_db() + assert (c.position, a.position, b.position) == (0, 1, 2) + + +@pytest.mark.django_db +class TestReplaceFile: + def test_replace_swaps_file_and_removes_the_old_one(self, publication): + block = add_file_block(publication, _pdf("first.pdf")) + old_path = block.file.path + old_id = block.id + assert os.path.exists(old_path) + + replace_block_file(block, _pdf("second.pdf")) + + assert block.id == old_id # same row, in-place + assert block.file_name == "second.pdf" + assert not os.path.exists(old_path) # old file removed from disk + + +@pytest.mark.django_db +class TestDeleteRemovesFile: + def test_deleting_a_block_removes_its_file(self, publication): + block = add_file_block(publication, _pdf()) + path = block.file.path + assert os.path.exists(path) + + block.delete() + + assert not os.path.exists(path) # post_delete signal cleaned it up + + +# --------------------------------------------------------------------------- # +# Download gate (Layer 4) +# --------------------------------------------------------------------------- # +@pytest.fixture +def other_org_user(db): + role = Role.objects.create(name="admin", can_view=True, can_change=True, can_delete=True) + org = Organization.objects.create(name="Other", description="o", slug="other") + user = User.objects.create(username="outsider", keycloak_id="outsider") + OrganizationMembership.objects.create(user=user, organization=org, role=role) + return user + + +def _download(user, block_id): + request = RequestFactory().get(f"/api/publications/blocks/{block_id}/download/") + request.user = user + return publication_block_download(request, block_id) + + +@pytest.mark.django_db +class TestDownloadGate: + def test_published_file_downloads_and_counts(self, publication, owner): + publication.status = PublicationStatus.PUBLISHED + publication.save() + block = add_file_block(publication, _pdf()) + + response = _download(AnonymousUser(), block.id) + + assert response.status_code == 200 + publication.refresh_from_db() + assert publication.download_count == 1 + + def test_draft_file_hidden_from_anonymous(self, publication): + block = add_file_block(publication, _pdf()) + + response = _download(AnonymousUser(), block.id) + + assert response.status_code == 404 + publication.refresh_from_db() + assert publication.download_count == 0 + + def test_draft_file_hidden_from_other_org(self, publication, other_org_user): + block = add_file_block(publication, _pdf()) + + response = _download(other_org_user, block.id) + + assert response.status_code == 404 + + def test_owner_can_download_own_draft_file(self, publication, owner): + block = add_file_block(publication, _pdf()) + + response = _download(owner, block.id) + + assert response.status_code == 200 + + def test_pdf_is_served_inline(self, publication, owner): + publication.status = PublicationStatus.PUBLISHED + publication.save() + block = add_file_block(publication, _pdf()) + + response = _download(AnonymousUser(), block.id) + + assert response["Content-Disposition"].startswith("inline") diff --git a/tests/test_publication_linking.py b/tests/test_publication_linking.py new file mode 100644 index 00000000..e41af4c4 --- /dev/null +++ b/tests/test_publication_linking.py @@ -0,0 +1,200 @@ +"""Layer 3/4 tests for linking published Resources into Use Cases / Collaboratives. + +Covers the only-PUBLISHED-is-linkable guard, the render-time published-only +filter (stale links vanish on unpublish/delete and reappear on re-publish), the +owner's linked-count flag, and the one intentional cross-org affordance. +""" + +import types +from datetime import date + +import pytest + +from api.models import Collaborative, Publication, ResourceType, UseCase +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.enums import PublicationStatus +from authorization.models import User + + +@pytest.fixture +def user(db): + return User.objects.create(username="author", keycloak_id="author") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +def _publication(user, resource_type, status=PublicationStatus.PUBLISHED, org=None): + return Publication.objects.create( + title="Findings", + description="d", + user=user, + resource_type=resource_type, + publication_date=date(2024, 1, 1), + status=status, + organization=org, + ) + + +_uc_counter = [0] +_collab_counter = [0] + + +def _use_case(user): + _uc_counter[0] += 1 + return UseCase.objects.create(title=f"UC {_uc_counter[0]}", user=user) + + +def _collaborative(user): + _collab_counter[0] += 1 + return Collaborative.objects.create(title=f"Collab {_collab_counter[0]}", user=user) + + +def ctx(user): + return types.SimpleNamespace(user=user, context={}) + + +def run(query, user, variables): + return schema.execute_sync(query, variable_values=variables, context_value=ctx(user)) + + +ADD_TO_UC = """ +mutation($ucId: String!, $pubId: UUID!) { + addPublicationToUseCase(useCaseId: $ucId, publicationId: $pubId) { __typename } +} +""" + +ADD_TO_COLLAB = """ +mutation($cId: String!, $pubId: UUID!) { + addPublicationToCollaborative(collaborativeId: $cId, publicationId: $pubId) { __typename } +} +""" + + +def _published_only(entity): + return list(entity.publications.filter(status=PublicationStatus.PUBLISHED)) + + +# --------------------------------------------------------------------------- # +# Link guard +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestLinkGuard: + def test_published_resource_links_to_use_case(self, user, resource_type): + uc = _use_case(user) + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + + run(ADD_TO_UC, user, {"ucId": str(uc.id), "pubId": str(pub.id)}) + + assert uc.publications.filter(id=pub.id).exists() + + def test_draft_resource_is_rejected(self, user, resource_type): + uc = _use_case(user) + draft = _publication(user, resource_type, status=PublicationStatus.DRAFT) + + run(ADD_TO_UC, user, {"ucId": str(uc.id), "pubId": str(draft.id)}) + + assert uc.publications.count() == 0 # guard held — nothing linked + + def test_published_resource_links_to_collaborative(self, user, resource_type): + collab = _collaborative(user) + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + + run(ADD_TO_COLLAB, user, {"cId": str(collab.id), "pubId": str(pub.id)}) + + assert collab.publications.filter(id=pub.id).exists() + + def test_draft_resource_is_rejected_by_collaborative(self, user, resource_type): + collab = _collaborative(user) + draft = _publication(user, resource_type, status=PublicationStatus.DRAFT) + + run(ADD_TO_COLLAB, user, {"cId": str(collab.id), "pubId": str(draft.id)}) + + assert collab.publications.count() == 0 + + +# --------------------------------------------------------------------------- # +# Stale links — render omits unpublished/deleted, restores on re-publish +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestStaleLinks: + def test_unpublish_hides_then_republish_restores(self, user, resource_type): + uc = _use_case(user) + collab = _collaborative(user) + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + uc.publications.add(pub) + collab.publications.add(pub) + + assert _published_only(uc) == [pub] + assert _published_only(collab) == [pub] + + pub.status = PublicationStatus.DRAFT + pub.save() + assert _published_only(uc) == [] # silently drops from render + assert _published_only(collab) == [] + + pub.status = PublicationStatus.PUBLISHED + pub.save() + assert _published_only(uc) == [pub] # reappears + assert _published_only(collab) == [pub] + + def test_delete_removes_link_and_render_skips(self, user, resource_type): + uc = _use_case(user) + collab = _collaborative(user) + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + uc.publications.add(pub) + collab.publications.add(pub) + + pub.delete() + + assert _published_only(uc) == [] + assert _published_only(collab) == [] + assert uc.publications.count() == 0 # M2M row auto-cleared + + +# --------------------------------------------------------------------------- # +# Linked-count flag + cross-org affordance +# --------------------------------------------------------------------------- # +@pytest.mark.django_db +class TestLinkedCountAndCrossOrg: + def test_linked_count_across_usecases_and_collaboratives(self, user, resource_type): + pub = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + uc1, uc2 = _use_case(user), _use_case(user) + collab = _collaborative(user) + uc1.publications.add(pub) + uc2.publications.add(pub) + collab.publications.add(pub) + + assert pub.usecase_set.count() + pub.collaborative_set.count() == 3 + + uc1.publications.remove(pub) + assert pub.usecase_set.count() + pub.collaborative_set.count() == 2 + + def test_unrelated_user_cannot_link_to_someone_elses_use_case(self, user, resource_type): + # IDOR guard: a caller who neither owns nor has an editor role on the use + # case cannot change its links, even with a valid published resource. + owner = user + stranger = User.objects.create(username="stranger", keycloak_id="stranger") + uc = _use_case(owner) + pub = _publication(owner, resource_type, status=PublicationStatus.PUBLISHED) + + run(ADD_TO_UC, stranger, {"ucId": str(uc.id), "pubId": str(pub.id)}) + + assert uc.publications.count() == 0 # link refused + + def test_cross_org_link_is_allowed(self, resource_type): + # A UC in org B may link a PUBLISHED resource owned by org A — the one + # intentional cross-org path; do not deny it. + org_a = Organization.objects.create(name="A", description="a", slug="a") + org_b = Organization.objects.create(name="B", description="b", slug="b") + owner_a = User.objects.create(username="a", keycloak_id="a") + owner_b = User.objects.create(username="b", keycloak_id="b") + pub = _publication(owner_a, resource_type, status=PublicationStatus.PUBLISHED, org=org_a) + uc_b = UseCase.objects.create(title="UC B", user=owner_b, organization=org_b) + + run(ADD_TO_UC, owner_b, {"ucId": str(uc_b.id), "pubId": str(pub.id)}) + + assert uc_b.publications.filter(id=pub.id).exists() diff --git a/tests/test_publication_models.py b/tests/test_publication_models.py new file mode 100644 index 00000000..b3430f1b --- /dev/null +++ b/tests/test_publication_models.py @@ -0,0 +1,199 @@ +"""Layer 1 DB tests for the Publication data model foundation. + +Covers slug dedupe, ownership, the ResourceType lookup, the file-XOR-youtube +block constraint, block ordering, delete cascade, and the seed command. +""" + +import pytest +from django.core.management import call_command +from django.db import IntegrityError, transaction + +from api.models import Publication, PublicationBlock, ResourceType +from api.models.Organization import Organization +from api.utils.enums import PublicationBlockType, PublicationStatus +from authorization.models import User + + +@pytest.fixture +def user(db): + return User.objects.create(username="alice", keycloak_id="kc-alice") + + +@pytest.fixture +def org(db): + return Organization.objects.create(name="Org A", description="an org", slug="org-a") + + +@pytest.mark.django_db +class TestPublicationSlug: + def test_two_same_title_publications_get_distinct_slugs(self, user): + first = Publication.objects.create(title="Annual Report", user=user) + second = Publication.objects.create(title="Annual Report", user=user) + + assert first.slug == "annual-report" + assert second.slug == "annual-report-1" + assert first.slug != second.slug + + def test_a_unicode_title_slugs_without_crashing(self, user): + publication = Publication.objects.create(title="Report — Résumé 2024", user=user) + + assert publication.slug + assert Publication.objects.filter(slug=publication.slug).count() == 1 + + +@pytest.mark.django_db +class TestPublicationOwnership: + def test_a_user_owned_publication_reports_individual(self, user): + publication = Publication.objects.create(title="Solo work", user=user) + + assert publication.is_individual_publication is True + + def test_an_org_owned_publication_is_not_individual(self, org, user): + publication = Publication.objects.create(title="Org work", organization=org, user=user) + + assert publication.is_individual_publication is False + + +@pytest.mark.django_db +class TestResourceType: + def test_name_is_unique(self): + ResourceType.objects.create(name="Report") + with transaction.atomic(), pytest.raises(IntegrityError): + ResourceType.objects.create(name="Report") + + def test_slugifies_the_name(self): + resource_type = ResourceType.objects.create(name="Policy Brief") + + assert resource_type.slug == "policy-brief" + + def test_is_active_defaults_true(self): + resource_type = ResourceType.objects.create(name="Report") + + assert resource_type.is_active is True + + def test_active_query_returns_only_active_types(self): + ResourceType.objects.create(name="Report") + ResourceType.objects.create(name="Retired", is_active=False) + + active = ResourceType.objects.filter(is_active=True) + + assert active.count() == 1 + assert active.first().name == "Report" + + +@pytest.mark.django_db +class TestPublicationBlock: + def _publication(self, user): + return Publication.objects.create(title="With blocks", user=user) + + def test_file_block_stores_file_fields(self, user): + publication = self._publication(user) + + block = PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.FILE, + file="publications/report.pdf", + file_name="report.pdf", + file_format="pdf", + file_size=1024, + ) + + assert block.file_name == "report.pdf" + assert block.file_format == "pdf" + assert block.youtube_url is None + + def test_youtube_block_stores_youtube_fields(self, user): + publication = self._publication(user) + + block = PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/dQw4w9WgXcQ", + youtube_video_id="dQw4w9WgXcQ", + ) + + assert block.youtube_video_id == "dQw4w9WgXcQ" + assert block.file == "" + + def test_a_block_with_both_file_and_youtube_is_rejected(self, user): + publication = self._publication(user) + + with transaction.atomic(), pytest.raises(IntegrityError): + PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.FILE, + file="publications/report.pdf", + youtube_url="https://youtu.be/dQw4w9WgXcQ", + ) + + def test_a_block_with_neither_file_nor_youtube_is_rejected(self, user): + publication = self._publication(user) + + with transaction.atomic(), pytest.raises(IntegrityError): + PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.FILE, + ) + + def test_blocks_read_back_in_position_order(self, user): + publication = self._publication(user) + PublicationBlock.objects.create( + publication=publication, + position=2, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/two", + ) + PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/zero", + ) + PublicationBlock.objects.create( + publication=publication, + position=1, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/one", + ) + + positions = list(publication.blocks.values_list("position", flat=True)) + + assert positions == [0, 1, 2] + + def test_deleting_a_publication_cascades_its_blocks(self, user): + publication = self._publication(user) + PublicationBlock.objects.create( + publication=publication, + position=0, + block_type=PublicationBlockType.YOUTUBE, + youtube_url="https://youtu.be/zero", + ) + + publication.delete() + + assert PublicationBlock.objects.count() == 0 + + +@pytest.mark.django_db +class TestSeedResourceTypes: + def test_seed_creates_ten_types_idempotently(self): + call_command("seed_resource_types") + assert ResourceType.objects.count() == 10 + + # Running again must not duplicate. + call_command("seed_resource_types") + assert ResourceType.objects.count() == 10 + + +@pytest.mark.django_db +class TestPublicationDefaults: + def test_new_publication_defaults_to_draft(self, user): + publication = Publication.objects.create(title="Draft one", user=user) + + assert publication.status == PublicationStatus.DRAFT + assert publication.download_count == 0 + assert publication.authors == [] diff --git a/tests/test_publication_qc_fixes.py b/tests/test_publication_qc_fixes.py new file mode 100644 index 00000000..5e22851e --- /dev/null +++ b/tests/test_publication_qc_fixes.py @@ -0,0 +1,171 @@ +"""Regression tests for the QC findings on the Resources backend. + +B1 — linked-project fields are owner-gated and published-only (no draft-title leak). +B4 — the listing runs a bounded query count (no N+1). +O1 — an explicit-empty required field is rejected on update. +O2 — the plain use-case update mutation cannot attach a resource. +""" + +import types +from datetime import date + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Collaborative, Publication, ResourceType, UseCase +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.enums import PublicationStatus, UseCaseStatus +from authorization.models import OrganizationMembership, Role, User + + +@pytest.fixture(autouse=True) +def _no_activity(monkeypatch): + monkeypatch.setattr("api.schema.base_mutation.record_activity", lambda *a, **k: None) + + +@pytest.fixture +def owner(db): + return User.objects.create(username="owner", keycloak_id="owner") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +def _pub(owner, rt, status=PublicationStatus.PUBLISHED, org=None, title="Findings"): + return Publication.objects.create( + title=title, + description="d", + user=owner, + resource_type=rt, + publication_date=date(2024, 1, 1), + status=status, + organization=org, + ) + + +def ctx(user, organization=None): + return types.SimpleNamespace( + user=user, context={"organization": organization} if organization else {} + ) + + +def run(query, user, variables=None, organization=None): + return schema.execute_sync( + query, variable_values=variables or {}, context_value=ctx(user, organization) + ) + + +LINKS = """ +query($id: UUID!) { + getPublication(publicationId: $id) { + id linkedUsecases { id title } linkedCount + } +} +""" + + +@pytest.mark.django_db +class TestLinkedFieldsGate: + def _linked_setup(self, owner, rt): + pub = _pub(owner, rt, status=PublicationStatus.PUBLISHED) + published_uc = UseCase.objects.create(title="Public UC", user=owner) + published_uc.status = UseCaseStatus.PUBLISHED + published_uc.save() + draft_uc = UseCase.objects.create(title="Secret Draft UC", user=owner) + published_uc.publications.add(pub) + draft_uc.publications.add(pub) + return pub + + def test_anonymous_sees_no_linked_projects(self, owner, resource_type): + pub = self._linked_setup(owner, resource_type) + + result = run(LINKS, AnonymousUser(), {"id": str(pub.id)}) + + assert result.errors is None + data = result.data["getPublication"] + assert data["linkedUsecases"] == [] # no draft title leaked + assert data["linkedCount"] == 0 + + def test_owner_sees_only_published_links(self, owner, resource_type): + pub = self._linked_setup(owner, resource_type) + + result = run(LINKS, owner, {"id": str(pub.id)}) + + data = result.data["getPublication"] + titles = [uc["title"] for uc in data["linkedUsecases"]] + assert titles == ["Public UC"] # the draft UC is hidden even from the owner + assert data["linkedCount"] == 1 + + +@pytest.mark.django_db +class TestListingIsBounded: + def test_listing_query_count_is_bounded( + self, owner, resource_type, django_assert_max_num_queries + ): + for i in range(5): + pub = _pub(owner, resource_type, title=f"Doc {i}") + pub.sectors.set([]) + pub.blocks.create( + position=0, + block_type="YOUTUBE", + youtube_url="https://youtu.be/dQw4w9WgXcQ", + youtube_video_id="dQw4w9WgXcQ", + ) + + query = """ + query { + publications(includePublic: true) { + id title resourceType { name } sectors { id } geographies { id } blocks { id } + } + } + """ + # Bounded: a constant number of queries regardless of the 5 rows + relations. + with django_assert_max_num_queries(15): + result = run(query, owner) + assert result.errors is None + assert len(result.data["publications"]) == 5 + + +UPDATE = """ +mutation($input: UpdatePublicationInput!) { + updatePublication(input: $input) { success errors { fieldErrors { field } } } +} +""" + + +@pytest.mark.django_db +class TestUpdateEmptyRejected: + def test_explicit_empty_title_is_rejected(self, owner, resource_type): + pub = _pub(owner, resource_type, status=PublicationStatus.DRAFT) + + result = run(UPDATE, owner, {"input": {"id": str(pub.id), "title": " "}}) + + assert result.data["updatePublication"]["success"] is False + pub.refresh_from_db() + assert pub.title == "Findings" # not blanked + + +@pytest.mark.django_db +class TestPlainUpdateCannotAttachPublication: + def test_usecase_input_has_no_publications_field(self, owner, resource_type): + pub = _pub(owner, resource_type, status=PublicationStatus.PUBLISHED) + uc = UseCase.objects.create(title="UC", user=owner) + + # The UC update input excludes 'publications', so passing it is a schema + # error — the only way to attach a resource is the guarded link trio. + mutation = """ + mutation($input: UseCaseInputPartial!) { + updateUseCase(useCaseInputPartial: $input) { __typename } + } + """ + result = run( + mutation, + owner, + {"input": {"id": str(uc.id), "publications": [str(pub.id)]}}, + ) + + assert result.errors is not None # 'publications' is not a valid input field + assert uc.publications.count() == 0 diff --git a/tests/test_publication_search.py b/tests/test_publication_search.py new file mode 100644 index 00000000..3179f30f --- /dev/null +++ b/tests/test_publication_search.py @@ -0,0 +1,101 @@ +"""Search-layer tests for Publication. + +Elasticsearch itself is disabled in the deterministic layers, so these verify +the index-decision logic and the related-model re-index mapping (which is what +prevents draft leakage and stale facets) rather than round-tripping a cluster. +The full query/filter/pagination behaviour is a Layer-4 scenario that runs +against a live index (documented in the arch doc). +""" + +from datetime import date + +import pytest + +from api.models import Geography, Publication, ResourceType, Sector +from api.models.Organization import Organization +from api.signals.publication_signals import _should_be_indexed +from api.utils.enums import GeoTypes, PublicationStatus +from authorization.models import User +from search.documents import PublicationDocument +from search.documents.publication_document import PublicationDocument as DocClass + + +@pytest.fixture +def user(db): + return User.objects.create(username="author", keycloak_id="author") + + +@pytest.fixture +def resource_type(db): + return ResourceType.objects.create(name="Report") + + +def _publication(user, resource_type, status=PublicationStatus.PUBLISHED, **extra): + return Publication.objects.create( + title="Findings", + description="d", + user=user, + resource_type=resource_type, + publication_date=date(2024, 1, 1), + status=status, + **extra, + ) + + +@pytest.mark.django_db +class TestIndexDecision: + def test_published_resource_is_indexed(self, user, resource_type): + publication = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + assert PublicationDocument().should_index_object(publication) is True + + def test_draft_resource_is_not_indexed(self, user, resource_type): + publication = _publication(user, resource_type, status=PublicationStatus.DRAFT) + assert PublicationDocument().should_index_object(publication) is False + + def test_signal_predicate_matches_published(self, user, resource_type): + published = _publication(user, resource_type, status=PublicationStatus.PUBLISHED) + draft = _publication(user, resource_type, status=PublicationStatus.DRAFT) + assert _should_be_indexed(published) is True + assert _should_be_indexed(draft) is False + + +@pytest.mark.django_db +class TestReindexMapping: + def test_related_models_include_resource_type(self): + # A renamed Resource Type must re-index affected publications, so the + # facet doesn't go stale — ResourceType must be a related model. + assert ResourceType in DocClass.Django.related_models + + def test_renaming_a_resource_type_finds_affected_publications(self, user, resource_type): + publication = _publication(user, resource_type) + + affected = PublicationDocument().get_instances_from_related(resource_type) + + assert list(affected) == [publication] + + def test_changing_a_sector_finds_affected_publications(self, user, resource_type): + sector = Sector.objects.create(name="Health") + publication = _publication(user, resource_type) + publication.sectors.add(sector) + + affected = PublicationDocument().get_instances_from_related(sector) + + assert list(affected) == [publication] + + def test_changing_a_geography_finds_affected_publications(self, user, resource_type): + geography = Geography.objects.create(name="India", code="IN", type=GeoTypes.COUNTRY) + publication = _publication(user, resource_type) + publication.geographies.add(geography) + + affected = PublicationDocument().get_instances_from_related(geography) + + assert list(affected) == [publication] + + def test_org_owned_publication_reindexes_on_org_change(self, resource_type): + org = Organization.objects.create(name="Org", description="o", slug="org") + owner = User.objects.create(username="member", keycloak_id="member") + publication = _publication(owner, resource_type, organization=org) + + affected = PublicationDocument().get_instances_from_related(org) + + assert list(affected) == [publication] diff --git a/tests/test_publication_uploads.py b/tests/test_publication_uploads.py new file mode 100644 index 00000000..f1d1caae --- /dev/null +++ b/tests/test_publication_uploads.py @@ -0,0 +1,36 @@ +"""Layer 2 tests for the content-block file validator.""" + +import pytest +from django.core.exceptions import ValidationError +from django.core.files.uploadedfile import SimpleUploadedFile + +from api.utils.publication_uploads import MAX_FILE_SIZE_BYTES, validate_publication_file + + +def _file(name, content=b"data", content_type="application/octet-stream"): + return SimpleUploadedFile(name, content, content_type=content_type) + + +class TestValidatePublicationFile: + def test_accepts_an_allowed_document(self): + extension, size = validate_publication_file(_file("brief.docx", b"x" * 10)) + assert extension == ".docx" + assert size == 10 + + def test_accepts_a_real_pdf(self): + extension, _ = validate_publication_file(_file("report.pdf", b"%PDF-1.7 body")) + assert extension == ".pdf" + + def test_rejects_a_disallowed_extension(self): + for name in ("malware.exe", "archive.zip", "data.csv"): + with pytest.raises(ValidationError): + validate_publication_file(_file(name)) + + def test_rejects_a_file_over_the_cap(self): + oversized = _file("big.pdf", b"%PDF" + b"0" * MAX_FILE_SIZE_BYTES) + with pytest.raises(ValidationError, match="50 MB"): + validate_publication_file(oversized) + + def test_rejects_a_pdf_that_is_not_really_a_pdf(self): + with pytest.raises(ValidationError, match="not"): + validate_publication_file(_file("fake.pdf", b"MZ this is an exe")) diff --git a/tests/test_publications.py b/tests/test_publications.py new file mode 100644 index 00000000..7c440615 --- /dev/null +++ b/tests/test_publications.py @@ -0,0 +1,81 @@ +"""Tests for the Publication ("Resource") SDK resource client.""" + +import unittest +from unittest.mock import MagicMock, patch + +from dataspace_sdk.resources.publications import PublicationClient + + +class TestPublicationClient(unittest.TestCase): + """Test cases for PublicationClient.""" + + def setUp(self) -> None: + self.base_url = "https://api.test.com" + self.auth_client = MagicMock() + self.client = PublicationClient(self.base_url, self.auth_client) + + def test_init(self) -> None: + self.assertEqual(self.client.base_url, self.base_url) + self.assertEqual(self.client.auth_client, self.auth_client) + + @patch.object(PublicationClient, "_make_request") + def test_search_hits_the_publication_endpoint(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"total": 1, "results": [{"id": "1"}]} + + result = self.client.search( + query="rainfall", resource_type="Report", sectors=["Health"], page=2, page_size=5 + ) + + self.assertEqual(result["total"], 1) + args, kwargs = mock_request.call_args + self.assertIn("/api/search/publication/", args[1]) + params = kwargs["params"] + self.assertEqual(params["q"], "rainfall") + self.assertEqual(params["resource_type"], "Report") + self.assertEqual(params["sectors"], "Health") + self.assertEqual(params["page"], 2) + + @patch.object(PublicationClient, "_make_request") + def test_get_by_id_uses_graphql(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"data": {"getPublication": {"id": "abc"}}} + + self.client.get_by_id("abc") + + args, kwargs = mock_request.call_args + self.assertIn("/api/graphql", args[1]) + self.assertEqual(kwargs["json_data"]["variables"]["publicationId"], "abc") + + @patch.object(PublicationClient, "_make_request") + def test_create_posts_the_mutation(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"data": {"createPublication": {"success": True}}} + + self.client.create({"title": "New"}) + + args, kwargs = mock_request.call_args + self.assertIn("createPublication", kwargs["json_data"]["query"]) + self.assertEqual(kwargs["json_data"]["variables"]["input"]["title"], "New") + + @patch.object(PublicationClient, "_make_request") + def test_update_merges_the_id_into_input(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"data": {"updatePublication": {"success": True}}} + + self.client.update("abc", {"title": "Renamed"}) + + args, kwargs = mock_request.call_args + variables = kwargs["json_data"]["variables"]["input"] + self.assertEqual(variables["id"], "abc") + self.assertEqual(variables["title"], "Renamed") + + @patch.object(PublicationClient, "_make_request") + def test_delete_passes_the_id(self, mock_request: MagicMock) -> None: + mock_request.return_value = {"data": {"deletePublication": {"success": True}}} + + self.client.delete("abc") + + args, kwargs = mock_request.call_args + self.assertIn("deletePublication", kwargs["json_data"]["query"]) + self.assertEqual(kwargs["json_data"]["variables"]["publicationId"], "abc") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_settings.py b/tests/test_settings.py index 2fee6f51..6a641855 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -6,16 +6,16 @@ import sys # Add the project root directory to Python path -project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, project_root) from DataSpace.settings import * # Use an in-memory SQLite database for testing DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", } } @@ -24,9 +24,10 @@ # Use a faster password hasher during tests PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', + "django.contrib.auth.hashers.MD5PasswordHasher", ] + # Disable migrations during tests class DisableMigrations: def __contains__(self, item): @@ -35,19 +36,49 @@ def __contains__(self, item): def __getitem__(self, item): return None + MIGRATION_MODULES = DisableMigrations() # Disable celery tasks during tests CELERY_ALWAYS_EAGER = True CELERY_EAGER_PROPAGATES_EXCEPTIONS = True -# Required apps for testing -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'api', +# NOTE: We intentionally do NOT override INSTALLED_APPS — the real settings +# already define AUTH_USER_MODEL = "authorization.User", so the +# ``authorization`` app must be present for Django to bootstrap. Trimming the +# list to a "minimal" set previously broke every test with +# ``ImproperlyConfigured: AUTH_USER_MODEL refers to model 'authorization.User' +# that has not been installed``. Use ``MIGRATION_MODULES`` above to keep +# tests fast instead of stripping apps. + +# Drop middleware that requires live external services (Keycloak / rate +# limiter / activity stream) so unit tests can boot without network access. +MIDDLEWARE = [ + m + for m in MIDDLEWARE # noqa: F405 — imported via ``from DataSpace.settings import *`` + if m + not in { + "authorization.middleware.KeycloakAuthenticationMiddleware", + "authorization.middleware.activity_consent.ActivityConsentMiddleware", + "api.middleware.rate_limit.rate_limit_middleware", + "api.middleware.request_validator.RequestValidationMiddleware", + } ] + +# Elasticsearch is optional during unit tests — point the DSL at a dummy host +# so module import doesn't try to connect. +ELASTICSEARCH_DSL = { + "default": {"hosts": "localhost:9200"}, +} + +# Deterministic layers must never reach a real cluster. The default real-time +# signal processor tries to push to Elasticsearch whenever a model (or a +# related model, e.g. User/Geography) is saved, which errors without a live +# cluster. Swap in the no-op processor so DB tests stay hermetic. +ELASTICSEARCH_DSL_SIGNAL_PROCESSOR = "django_elasticsearch_dsl.signals.BaseSignalProcessor" + +# Disable real Keycloak calls in tests. +KEYCLOAK_SERVER_URL = "http://localhost:8080" +KEYCLOAK_REALM = "test" +KEYCLOAK_CLIENT_ID = "test-client" +KEYCLOAK_CLIENT_SECRET = "test-secret" diff --git a/tests/test_youtube_url.py b/tests/test_youtube_url.py new file mode 100644 index 00000000..ba217729 --- /dev/null +++ b/tests/test_youtube_url.py @@ -0,0 +1,48 @@ +"""Layer 2 tests for the YouTube URL helpers.""" + +import pytest +from django.core.exceptions import ValidationError + +from api.utils.youtube import extract_video_id, validate_youtube_url + +VIDEO_ID = "dQw4w9WgXcQ" + + +class TestExtractVideoId: + @pytest.mark.parametrize( + "url", + [ + f"https://www.youtube.com/watch?v={VIDEO_ID}", + f"https://youtu.be/{VIDEO_ID}", + f"https://www.youtube.com/embed/{VIDEO_ID}", + f"https://m.youtube.com/watch?v={VIDEO_ID}&feature=share", + ], + ) + def test_extracts_the_same_id_from_every_shape(self, url): + assert extract_video_id(url) == VIDEO_ID + + @pytest.mark.parametrize( + "url", + [ + "https://vimeo.com/123456789", + "https://example.com/watch?v=abc", + "not a url", + "https://www.youtube.com/watch?v=tooShort", + "", + None, + # A matching host on a dangerous scheme must never pass (stored XSS). + f"javascript://youtube.com/watch?v={VIDEO_ID}", + f"data://youtu.be/{VIDEO_ID}", + ], + ) + def test_rejects_non_youtube_or_malformed(self, url): + assert extract_video_id(url) is None + + +class TestValidateYoutubeUrl: + def test_returns_id_for_a_valid_url(self): + assert validate_youtube_url(f"https://youtu.be/{VIDEO_ID}") == VIDEO_ID + + def test_raises_for_a_non_youtube_url(self): + with pytest.raises(ValidationError): + validate_youtube_url("https://vimeo.com/1")