From 14b882ff3864c710b2b0e8b94f19edced61d156d Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 08:30:59 +0530 Subject: [PATCH 01/20] ci: generalize deploy-backend.yml to cover dev and prod Renamed the workflow's display name from "Deploy Backend to Dev EC2" to "Deploy Backend to EC2". One workflow now handles both: push to dev deploys to development (unchanged behavior), push to main deploys to production -- environment and image tag derive from github.ref_name. Same shape DataSpaceFrontend/ParakhAPI/ParakhAI-frontend's pipelines already use. No script changes needed: ci-deploy.sh/ci-rollback.sh/ci-finalize.sh are already fully environment-agnostic (single docker-compose.yml, no dev/prod split the way ParakhAPI needed ENV_SUFFIX for). prod-cds also already uses the same ~/DataExchange/DataExBackend submodule layout as dev-cds (confirmed via a recent read-only survey), so no host migration script is needed either -- this is the simplest of the four sibling pipelines. New repo var PROD_API_BASE_URL (https://api.datakeep.civicdays.in -- confirmed via a prior survey, NOT api.civicdataspace.in as the naming pattern would suggest) for the smoke-tests job, which is a `uses:` job and can't see environment-scoped vars. Needs a `production` GH environment with EC2_HOST/EC2_USERNAME/ EC2_PRIVATE_KEY before the first push to main can deploy -- tracked separately, not part of this change (touches prod-cds, not mine to set up). Also flagged inline: prod-cds currently has 12 uncommitted files in its checkout that an image-based deploy will silently discard -- diff those before the first real prod run. actionlint: zero findings. --- .github/workflows/deploy-backend.yml | 38 ++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index bacde4f..300e584 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -35,12 +35,25 @@ # never had that problem. So no SSH step here ever carries more than a # single trivial invocation line; all real logic lives in scripts/ci-*.sh, # shipped as files. +# +# Covers both dev (push to `dev`) and prod (push to `main`) -- environment +# and image tag derive from github.ref_name, same shape the sibling repos' +# pipelines (DataSpaceFrontend, ParakhAPI, ParakhAI-frontend) already use. +# scripts/ci-*.sh need no ENV_SUFFIX-style parameterization here: there's a +# single docker-compose.yml (no dev/prod split, unlike ParakhAPI), and +# prod-cds already uses the same ~/DataExchange/DataExBackend submodule +# layout as dev-cds (confirmed via a recent read-only survey) -- no host +# migration needed either. First push to `main` through this pipeline is +# still a real cutover though: prod is currently on a locally-built image +# with `--reload` (stuck at 1 worker) and has uncommitted files in its +# checkout that an image-based deploy will silently discard -- diff those +# before relying on this for a real prod deploy. -name: Deploy Backend to Dev EC2 +name: Deploy Backend to EC2 on: push: - branches: ['dev'] + branches: ['dev', 'main'] workflow_dispatch: inputs: force_smoke_failure: @@ -49,16 +62,19 @@ on: required: false default: false -# Queue overlapping deploys rather than cancelling: a cancelled run mid-deploy -# could leave .deploy/ state and the running containers disagreeing. +# Queue overlapping deploys to the SAME environment rather than cancelling (a +# cancelled run mid-deploy could leave .deploy/ state and the running +# containers disagreeing) -- keyed by ref so a dev push and a main push never +# block each other. concurrency: - group: dataspace-backend-dev-deploy + group: dataspace-backend-deploy-${{ github.ref_name }} cancel-in-progress: false env: REGISTRY: ghcr.io IMAGE_NAME: civicdatalab/dataspacebackend DEPLOY_PATH: ${{ vars.DEPLOY_PATH || 'DataExchange/DataExBackend' }} + GH_ENVIRONMENT: ${{ github.ref_name == 'main' && 'production' || 'development' }} jobs: build: @@ -99,7 +115,7 @@ jobs: build-args: | GIT_COMMIT_SHA=${{ github.sha }} tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }} # mode=min (the default), not max: this Dockerfile is single-stage # (no multi-stage FROM ... AS builder), so mode=max's extra @@ -175,7 +191,7 @@ jobs: name: Deploy to EC2 needs: build runs-on: ubuntu-latest - environment: development + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} # 50m: must comfortably exceed the Deploy step's own 40m # command_timeout (see that step's comment for why it's 40m). timeout-minutes: 50 @@ -234,10 +250,10 @@ jobs: # No `environment:` here -- GitHub rejects the entire workflow file at # parse time if a `uses:` job declares one. Consequence: this job also # cannot see environment-scoped vars, which is why api_base_url comes - # from a repo-level var. + # from repo-level vars, picked by ref rather than by environment. uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI with: - api_base_url: ${{ vars.DEV_API_BASE_URL }} + api_base_url: ${{ github.ref_name == 'main' && vars.PROD_API_BASE_URL || vars.DEV_API_BASE_URL }} # An obviously-wrong sentinel SHA fails the reusable workflow's own # deployed-SHA-vs-live-/health/ assertion on purpose, for the # force_smoke_failure test path. @@ -263,7 +279,7 @@ jobs: # if: always() would ignore cancellation entirely. if: failure() && needs.deploy.result == 'success' runs-on: ubuntu-latest - environment: development + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} # 50m: must comfortably exceed the Restore previous image step's own # 40m command_timeout (see the Deploy job's equivalent step for why). timeout-minutes: 50 @@ -311,7 +327,7 @@ jobs: needs: [deploy, smoke-tests] if: success() runs-on: ubuntu-latest - environment: development + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} timeout-minutes: 10 steps: - name: Checkout code From 3ecac4ef039e5d11e72c9233e142fd40d62ac375 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 08:50:44 +0530 Subject: [PATCH 02/20] fix: recover 3 real hotfixes made directly on prod-cds, never committed prod-cds's checkout had 12 uncommitted changes (found while auditing it before trusting an image-based deploy there, which would have silently discarded all of them). Sorted into what to keep vs. discard; these 3 are real, confirmed-needed fixes: - ALLOWED_HOSTS: add prod-parakh's IP (13.235.189.143). ParakhAPI/ ParakhAI-frontend (both hosted there) make server-side calls to this backend; without this Django 400s them as a disallowed host. - rate_limit middleware: exempt health checks and keycloak login/refresh from rate limiting. Also fixed the exempt path list itself while restoring it: the original prod hotfix exempted "/api/auth/keycloak/refresh/", but the real route (DataSpace/urls.py -> api/urls.py) is "/api/auth/token/refresh/" -- that exemption never actually matched anything. Added the /auth/-mounted equivalents too (authorization.urls exposes its own keycloak/login + token/refresh under /auth/, separate from api.urls's copy under /api/) since it wasn't obvious which one is actually in current use -- exempting both is safe, missing the real one risks locking users out under load. - keycloak_utils: log Elasticsearch bulk errors on a user-sync failure, not just the top-level exception message. Deliberately NOT carried forward: a Dockerfile change reordering `chmod +x docker-entrypoint.sh` earlier in the build. Both the original and moved position chmod the file before ENTRYPOINT ever uses it, so there's no behavioral difference either way, and nobody could recall why it was made -- dropped rather than guess at an undocumented rationale. Also on the host, confirmed safe to leave un-committed (backups, stray build/debug artifacts, not source): rate_limit.py.bak.* files, a stray `db.sqlite3;`, logs/, static/, staticfiles/, render.html, snapshot.html. --- DataSpace/settings.py | 1 + api/middleware/rate_limit.py | 23 +++++++++++++++++++++++ api/utils/keycloak_utils.py | 2 ++ 3 files changed, 26 insertions(+) diff --git a/DataSpace/settings.py b/DataSpace/settings.py index 50a38dd..24b3682 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -59,6 +59,7 @@ whitelisted_urls = env("URL_WHITELIST").split(",") ALLOWED_HOSTS = [ + "13.235.189.143", # Allow parakh IP "localhost", # Allow localhost without port "127.0.0.1", # Allow IPv4 loopback "::1", # Allow IPv6 loopback diff --git a/api/middleware/rate_limit.py b/api/middleware/rate_limit.py index e53bf83..d8c8d8d 100644 --- a/api/middleware/rate_limit.py +++ b/api/middleware/rate_limit.py @@ -103,6 +103,29 @@ def middleware(request: HttpRequest) -> HttpResponse: client_ip = get_client_ip(request) whitelisted_ips = get_whitelisted_ips() + # Skip rate limiting for selected paths. Both `api.urls` (mounted at + # /api/) and `authorization.urls` (mounted at /auth/) expose their + # own keycloak login/token-refresh views -- exempt both sets rather + # than guess which one a given client actually calls. Rate-limiting + # /health/ would make health checks themselves start failing under + # load; rate-limiting login/refresh risks locking users out. + exempt_paths = { + "/api/auth/keycloak/login/", + "/api/auth/keycloak/login", + "/api/auth/token/refresh/", + "/api/auth/token/refresh", + "/auth/keycloak/login/", + "/auth/keycloak/login", + "/auth/token/refresh/", + "/auth/token/refresh", + "/health/", + "/health", + } + + if request.path in exempt_paths: + logger.debug(f"Skipping rate limit for exempt path: {request.path}") + return get_response(request) + # Skip rate limiting for whitelisted IPs if client_ip in whitelisted_ips: logger.debug(f"Skipping rate limit for whitelisted IP: {client_ip}") diff --git a/api/utils/keycloak_utils.py b/api/utils/keycloak_utils.py index d1a6ffc..b088fbb 100644 --- a/api/utils/keycloak_utils.py +++ b/api/utils/keycloak_utils.py @@ -469,6 +469,8 @@ def sync_user_from_keycloak( return user except Exception as e: logger.error(f"Error synchronizing user from Keycloak: {e}") + if hasattr(e, "errors"): + logger.error(f"Elasticsearch bulk errors: {e.errors}") return None From 5b608727ed82a0e95e90eed845248339ea5d6b30 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 08:55:07 +0530 Subject: [PATCH 03/20] docs: fix stale Keycloak domain and /auth prefix across SDK docs opub-kc.civicdatalab.in doesn't match the confirmed current production Keycloak domain (auth.civicdatalab.in) -- stale across 6 files (SDK docstrings, README, QUICKSTART, AUTHENTICATION.md, one example script). Swept all 16 occurrences. Also fixed AUTHENTICATION.md's troubleshooting section, which hedged between with/without a /auth path prefix: current Keycloak versions drop that context path by default (confirmed: auth.civicdatalab.in's real realm URLs have no /auth segment), so state that plainly instead of "try both", with a one-line note for anyone still on an older standalone Keycloak that kept the legacy path. --- dataspace_sdk/auth.py | 2 +- dataspace_sdk/client.py | 6 +++--- docs/sdk/AUTHENTICATION.md | 26 +++++++++++++------------- docs/sdk/QUICKSTART.md | 2 +- docs/sdk/README.md | 4 ++-- examples/username_password_login.py | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/dataspace_sdk/auth.py b/dataspace_sdk/auth.py index 98078d6..e217a3f 100644 --- a/dataspace_sdk/auth.py +++ b/dataspace_sdk/auth.py @@ -25,7 +25,7 @@ def __init__( Args: base_url: Base URL of the DataSpace API - keycloak_url: Keycloak server URL (e.g., "https://opub-kc.civicdatalab.in") + keycloak_url: Keycloak server URL (e.g., "https://auth.civicdatalab.in") keycloak_realm: Keycloak realm name (e.g., "DataSpace") keycloak_client_id: Keycloak client ID (e.g., "dataspace") keycloak_client_secret: Optional client secret for confidential clients diff --git a/dataspace_sdk/client.py b/dataspace_sdk/client.py index d139468..14dcefe 100644 --- a/dataspace_sdk/client.py +++ b/dataspace_sdk/client.py @@ -50,7 +50,7 @@ def __init__( Args: base_url: Base URL of the DataSpace API (e.g., "https://api.dataspace.example.com") - keycloak_url: Keycloak server URL (e.g., "https://opub-kc.civicdatalab.in") + keycloak_url: Keycloak server URL (e.g., "https://auth.civicdatalab.in") keycloak_realm: Keycloak realm name (e.g., "DataSpace") keycloak_client_id: Keycloak client ID (e.g., "dataspace") keycloak_client_secret: Optional client secret for confidential clients @@ -93,7 +93,7 @@ def login(self, username: str, password: str) -> dict: Example: >>> client = DataSpaceClient( ... base_url="https://api.dataspace.example.com", - ... keycloak_url="https://opub-kc.civicdatalab.in", + ... keycloak_url="https://auth.civicdatalab.in", ... keycloak_realm="DataSpace", ... keycloak_client_id="dataspace" ... ) @@ -120,7 +120,7 @@ def login_as_service_account(self) -> dict: Example: >>> client = DataSpaceClient( ... base_url="https://api.dataspace.example.com", - ... keycloak_url="https://opub-kc.civicdatalab.in", + ... keycloak_url="https://auth.civicdatalab.in", ... keycloak_realm="DataSpace", ... keycloak_client_id="dataspace", ... keycloak_client_secret="your-secret" diff --git a/docs/sdk/AUTHENTICATION.md b/docs/sdk/AUTHENTICATION.md index 65aeb13..e85f81e 100644 --- a/docs/sdk/AUTHENTICATION.md +++ b/docs/sdk/AUTHENTICATION.md @@ -21,7 +21,7 @@ from dataspace_sdk import DataSpaceClient # Initialize with Keycloak configuration client = DataSpaceClient( base_url="https://dataspace.civicdatalab.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace" ) @@ -48,7 +48,7 @@ Best for: ```python client = DataSpaceClient( base_url="https://dataspace.civicdatalab.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace" ) @@ -100,7 +100,7 @@ from dataspace_sdk import DataSpaceClient client = DataSpaceClient( base_url="https://dataspace.civicdatalab.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace" ) @@ -129,7 +129,7 @@ You need these details from your Keycloak setup: ```python client = DataSpaceClient( base_url="https://dataspace.civicdatalab.in", # DataSpace API URL - keycloak_url="https://opub-kc.civicdatalab.in", # Keycloak server URL + keycloak_url="https://auth.civicdatalab.in", # Keycloak server URL keycloak_realm="DataSpace", # Realm name keycloak_client_id="dataspace", # Client ID keycloak_client_secret="optional-secret" # Only for confidential clients @@ -169,7 +169,7 @@ Create a `.env` file: ```bash DATASPACE_API_URL=https://dataspace.civicdatalab.in -KEYCLOAK_URL=https://opub-kc.civicdatalab.in +KEYCLOAK_URL=https://auth.civicdatalab.in KEYCLOAK_REALM=DataSpace KEYCLOAK_CLIENT_ID=dataspace DATASPACE_USERNAME=your-email@example.com @@ -186,7 +186,7 @@ from dataspace_sdk.exceptions import DataSpaceAuthError client = DataSpaceClient( base_url="https://dataspace.civicdatalab.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace" ) @@ -277,7 +277,7 @@ For production applications, use a confidential client with client_secret: ```python client = DataSpaceClient( base_url="https://dataspace.civicdatalab.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace-prod", keycloak_client_secret=os.getenv("KEYCLOAK_CLIENT_SECRET") @@ -342,16 +342,16 @@ If you're experiencing frequent token expiration: This usually means wrong Keycloak URL: ```python -# Try with /auth prefix -keycloak_url="https://opub-kc.civicdatalab.in/auth" - -# Or without -keycloak_url="https://opub-kc.civicdatalab.in" +keycloak_url="https://auth.civicdatalab.in" # Test in browser: -# https://opub-kc.civicdatalab.in/auth/realms/DataSpace/.well-known/openid-configuration +# https://auth.civicdatalab.in/realms/DataSpace/.well-known/openid-configuration ``` +No `/auth` path prefix -- current Keycloak versions drop it by default. If +you're pointed at an older standalone Keycloak instance that still uses the +legacy context path, try `https:///auth/realms/...` instead. + ### "Client requires user consent" Error In Keycloak admin console: diff --git a/docs/sdk/QUICKSTART.md b/docs/sdk/QUICKSTART.md index 6138293..f081f49 100644 --- a/docs/sdk/QUICKSTART.md +++ b/docs/sdk/QUICKSTART.md @@ -24,7 +24,7 @@ from dataspace_sdk import DataSpaceClient # 1. Initialize the client with Keycloak configuration client = DataSpaceClient( base_url="https://dataspace.civicdatalab.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace" ) diff --git a/docs/sdk/README.md b/docs/sdk/README.md index 37b7246..bfbe1fb 100644 --- a/docs/sdk/README.md +++ b/docs/sdk/README.md @@ -32,7 +32,7 @@ from dataspace_sdk import DataSpaceClient # Initialize the client with Keycloak configuration client = DataSpaceClient( base_url="https://dev.api.civicdataspace.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace", keycloak_client_secret="your_client_secret" @@ -85,7 +85,7 @@ from dataspace_sdk import DataSpaceClient client = DataSpaceClient( base_url="https://dev.api.civicdataspace.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace", keycloak_client_secret="your_client_secret" diff --git a/examples/username_password_login.py b/examples/username_password_login.py index c477228..627e93c 100644 --- a/examples/username_password_login.py +++ b/examples/username_password_login.py @@ -14,7 +14,7 @@ # Initialize client with Keycloak configuration client = DataSpaceClient( base_url="https://dataspace.civicdatalab.in", - keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_url="https://auth.civicdatalab.in", keycloak_realm="DataSpace", keycloak_client_id="dataspace", # keycloak_client_secret="your-secret" # Only if using confidential client From 7c4542d80bcdd290c43465a81f7408cbf8d3c768 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 09:45:31 +0530 Subject: [PATCH 04/20] fix: register debug_toolbar in INSTALLED_APPS when DEBUG is on MIDDLEWARE conditionally added debug_toolbar's middleware when DEBUG is True, but INSTALLED_APPS never registered the app itself -- git history shows this pairing existed before and the INSTALLED_APPS half got dropped somewhere along this repo's messy merge/revert history. Broke every build since (confirmed via 3 failed CI runs today, all at the same step): the sanity-check step's `manage.py check` runs with DEBUG defaulting to True (no DEBUG env var set), and any request/check path that touches debug_toolbar's models crashes with "Model class debug_toolbar.models.HistoryEntry doesn't declare an explicit app_label and isn't in an application in INSTALLED_APPS" -- the middleware imports the app's models without the app being registered. --- DataSpace/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DataSpace/settings.py b/DataSpace/settings.py index 50a38dd..e1389d4 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -154,8 +154,12 @@ "api.middleware.logging.StructuredLoggingMiddleware", ] -# Add debug toolbar middleware if in debug mode +# Add debug toolbar app and middleware if in debug mode. Both are required -- +# the middleware alone crashes on any request/check that touches +# debug_toolbar's models: "HistoryEntry doesn't declare an explicit +# app_label and isn't in an application in INSTALLED_APPS". if DEBUG: + INSTALLED_APPS.append("debug_toolbar") MIDDLEWARE.insert(1, "debug_toolbar.middleware.DebugToolbarMiddleware") MIDDLEWARE += [ From ce7e5bc3a6f438f205d8cad3342c90d596e03743 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 10:24:28 +0530 Subject: [PATCH 05/20] fix: ci-deploy.sh no longer fights externally-owned dependency containers Blocked the first real prod deploy attempt today: `$COMPOSE up -d backend_db elasticsearch redis` failed with "Conflict. The container name '/DataExBackendElastic' is already in use" -- confirmed live via `docker ps` compose-project labels that on prod-cds these three are long-lived containers (DataExBackendDb up 2 months, DataExBackendElastic up 6 months, DataExBackendRedis up 14 months) owned by the `dataexchange` project (the top-level superproject), not this pipeline's own `dataexbackend` project. Docker container names are host-global, not project-scoped, so compose doesn't recognize an externally-owned same-named container as already satisfying the service and tries (and fails) to create a new one. The script aborted immediately on this (set -euo pipefail) before ever touching the running `backend` app container -- confirmed live, prod stayed healthy and untouched throughout. Fix: check each dependency container by name first, only run `up -d` for whichever is genuinely absent (a fresh host with nothing running yet). Verified live against the real prod-cds container state (with sudo, matching how the script actually runs there): correctly detects all three as already running and would skip the `up -d` call entirely. --- scripts/ci-deploy.sh | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh index 2b6f374..3f3aab6 100755 --- a/scripts/ci-deploy.sh +++ b/scripts/ci-deploy.sh @@ -82,14 +82,35 @@ RELEASE="$COMPOSE --profile release run --rm --no-deps -T release" # (see docker-compose.yml) -- args below are manage.py subcommands # only, not full "python manage.py ..." invocations. -# backend_db/elasticsearch/redis must be up before the release step below: -# it uses --no-deps (so compose never recreates them out from under a -# running deploy), which also means it will NOT wait for their -# depends_on healthchecks. In steady state these are already running, -# but a host that had the stack fully down would otherwise fail at -# migrate with a confusing connection error. up -d is idempotent -- -# no-ops when they are already healthy and their config is unchanged. -$COMPOSE up -d backend_db elasticsearch redis +# backend_db/elasticsearch/redis must be reachable before the release step +# below (it uses --no-deps and won't wait for their depends_on +# healthchecks). On prod-cds these are long-lived shared containers already +# owned by a DIFFERENT compose project (dataexchange, the top-level +# superproject) than this pipeline's own (dataexbackend) -- confirmed live +# via `docker ps` compose-project labels (DataExBackendDb up 2 months, +# DataExBackendElastic up 6 months, DataExBackendRedis up 14 months). +# Docker container names are host-global, not project-scoped, so blindly +# running `$COMPOSE up -d` against them hits a naming conflict: compose +# doesn't recognize a same-named container owned by another project as +# already satisfying the service, and tries (and fails) to create a new +# one. Bring up only whichever of the three is genuinely absent (a fresh +# host with nothing running yet) -- trust any that already exist, under any +# project, rather than fighting over ownership. +declare -A SVC_CONTAINER=( + [backend_db]=DataExBackendDb + [elasticsearch]=DataExBackendElastic + [redis]=DataExBackendRedis +) +MISSING=() +for svc in "${!SVC_CONTAINER[@]}"; do + docker ps --format '{{.Names}}' | grep -qx "${SVC_CONTAINER[$svc]}" || MISSING+=("$svc") +done +if [ "${#MISSING[@]}" -gt 0 ]; then + echo "Starting missing dependency containers: ${MISSING[*]}" + $COMPOSE up -d "${MISSING[@]}" +else + echo "backend_db/elasticsearch/redis already running (externally managed) -- skipping" +fi # --- release step, against the NEW image, before the swap ------ # Recorded for the rollback message: a rollback restores the image From 2b8eb61bbb2ce5c10003c22d896cd2708c1afaf5 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 11:02:04 +0530 Subject: [PATCH 06/20] fix: attach backend/release to dataexchange_default network Blocked the DataSpaceBackend production deploy: the `release` step's migrate command failed with `could not translate host name "backend_db" to address: Temporary failure in name resolution`. Root cause: this compose file's `backend_db`/`elasticsearch`/`redis` services are, on both dev-cds and prod-cds, deliberately never started by this pipeline (ci-deploy.sh's earlier fix, PR #163, correctly skips them since same-named containers already exist under the separate `dataexchange` superproject). Those real containers live on `dataexchange_default`, a network this project never declared -- `backend`/`release` only ever joined the implicit default network for this project (`dataexbackend_default` on the real hosts), which has no route to them. The currently-running backend container only works today because of an undocumented `docker network connect dataexchange_default DataSpace` done by hand on the host at some point -- confirmed live via `docker inspect`. That connection is not captured anywhere in code, doesn't apply to the `release` container (a fresh container per run), and would NOT survive a real `--force-recreate` of `backend` either. Fix: declare `dataexchange_default` as an external network and attach both `backend` and `release` to it alongside the project's own default network -- covers both a genuinely fresh host (where this file's own backend_db/elasticsearch/redis come up on `default`) and dev-cds/prod-cds (where the real ones live on dataexchange_default). Verified both networks exist on both real hosts via `docker network ls`, and that the rendered config (`docker compose config` / `--profile release config`) resolves both services onto both networks correctly. Caveat, not addressed here: `external: true` means `docker compose up` would fail outright on a machine that has neither network (e.g. a fresh local dev setup never connected to the DataExchange superproject) -- `docker compose config` itself still validates fine without it existing, only `up` needs it. Not fixing speculatively since both confirmed-real deploy targets (dev-cds, prod-cds) already have it. --- docker-compose.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 3de7bc8..b0a35a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,25 @@ services: options: max-size: "10m" max-file: "3" + # On prod-cds, backend_db/elasticsearch/redis below are NOT what's + # actually running -- ci-deploy.sh deliberately skips starting them + # when same-named containers already exist under the `dataexchange` + # superproject (see its own comment), which lives on a DIFFERENT + # default network (dataexchange_default) than this project's own + # (dataexbackend_default, this compose file's implicit default). + # Confirmed live: the currently-running DataSpace container only + # reaches them because of an undocumented `docker network connect + # dataexchange_default DataSpace` done by hand on the host at some + # point -- never captured here, and would NOT survive a fresh + # --force-recreate (confirmed: a real deploy's `release` container hit + # "could not translate host name backend_db" on a fresh network-less + # container). Joining both networks here covers both cases: a genuinely + # fresh host (where this file's own backend_db/elasticsearch/redis come + # up on `default`) and prod-cds (where the real ones live on + # dataexchange_default). + networks: + - default + - dataexchange_default # One-off management commands (migrations, etc.) against the deployed # image, without touching the running `backend` container. No @@ -53,6 +72,10 @@ services: condition: service_healthy entrypoint: ["python", "manage.py"] command: ["migrate", "--noinput"] + # See `backend`'s identical networks: block above for why both are needed. + networks: + - default + - dataexchange_default backend_db: image: "postgres:14.4" @@ -253,3 +276,10 @@ volumes: driver: local redis-data: driver: local + +networks: + # The dataexchange superproject's default network on prod-cds -- external + # because this compose project doesn't own it. See the comment on + # `backend`'s networks: block above. + dataexchange_default: + external: true From 8f5ff05c23ce689796b66450df02b34ddfd21e1a Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 11:28:55 +0530 Subject: [PATCH 07/20] fix: commit 46 missing api/authorization migrations, was gitignored Blocked the production deploy's release step: `search_index --populate` crashed with `relation "publication" does not exist`. Root cause traced much further than expected. .gitignore had `api/migrations/*` and `authorization/migrations/*` -- every migration generated for either app since whenever that was added has been silently invisible to git, no matter how many times someone ran `git add`. This repo's `api/migrations/` only ever had `0001_initial.py` committed (likely force-added once, or predates the ignore rule); `authorization/migrations/` had nothing at all, not even `__init__.py`. Confirmed via git log this wasn't always broken: `30480c6 "reset migrations"` deliberately squashed history down to one clean `0001_initial.py` at some point (a legitimate, intentional cleanup, probably around a Postgres migration). But nothing was ever committed again afterward -- every migration generated since (via normal `makemigrations` + `git add`, silently swallowed by the ignore rule) only ever existed on whichever host it was generated on. dev-cds's live checkout had the full, real, working set: 46 files for `api` (0002-0047, ending in the Publication/ResourceType migration this deploy needed) and all 5 for `authorization` (0001-0004 + __init__.py, defining `authorization.User` itself -- AUTH_USER_MODEL). Pulled all 51 files directly from dev-cds (read-only), verified: - Every file compiles clean (`py_compile`). - The full api dependency chain is linear and unbroken, 0001 -> 0047, no gaps, no branches. - authorization's 4 files chain cleanly from 0001, which itself correctly depends on api/0023 (a real cross-app FK relationship) -- confirms the two apps' histories are genuinely interleaved and consistent, not independently drifted. - No overlap with the deliberately-excluded fetch_resource_data feature from the earlier dev/main reconciliation (PR #153) -- the two incidental substring matches found were unrelated migration filenames. Removed both ignore rules. Django migrations must always be committed; there is no legitimate reason to exclude them. This also means any genuinely fresh database (a new contributor's local setup, a CI job with --create-db, a disaster-recovery rebuild) has been silently unable to reach current schema this whole time -- would have stopped dead at whatever 0001_initial.py defines. That risk is now closed. --- .gitignore | 2 - .../0002_dataspace_dataset_dataspace.py | 34 + api/migrations/0003_usecase.py | 29 + api/migrations/0004_usecase_status.py | 18 + api/migrations/0005_alter_usecase_slug.py | 18 + ...usecase_description_alter_usecase_title.py | 23 + api/migrations/0007_resourcechartimage.py | 24 + ...ter_resourcechartdetails_aggregate_type.py | 18 + ...ter_resourcechartdetails_aggregate_type.py | 18 + ..._resourcechartdetails_modified_and_more.py | 29 + .../0011_alter_resourceschema_format.py | 18 + .../0012_resourcechartdetails_filters.py | 18 + ...rcechartdetails_aggregate_type_and_more.py | 61 ++ ..._options_alter_dataset_options_and_more.py | 263 ++++++ .../0015_dataset_slug_resource_slug.py | 68 ++ ..._alter_dataset_slug_alter_resource_slug.py | 23 + api/migrations/0017_resourcedatatable.py | 29 + .../0018_resource_download_count.py | 18 + ...ry_sector_alter_sector_options_and_more.py | 30 + ...eated_datasetmetadata_modified_and_more.py | 120 +++ ...me_description_usecase_summary_and_more.py | 33 + .../0022_resource_version_resourceversion.py | 49 ++ api/migrations/0023_alter_usecase_summary.py | 18 + ...4_alter_organization_organization_types.py | 42 + ...usecase_completed_on_usecase_started_on.py | 23 + api/migrations/0026_usecase_contributors.py | 24 + ...secaseorganizationrelationship_and_more.py | 66 ++ ...028_dataset_access_type_dataset_license.py | 47 ++ api/migrations/0029_dataset_user.py | 27 + ...30_organization_github_profile_and_more.py | 33 + ...r_alter_resourcechartdetails_chart_type.py | 46 ++ ...32_resourcechartdetails_status_and_more.py | 39 + api/migrations/0033_usecase_organization.py | 24 + api/migrations/0034_usecasedashboard.py | 36 + api/migrations/0035_usecase_platform_url.py | 18 + ...eographies_usecase_geographies_and_more.py | 746 ++++++++++++++++++ ...l_status_aimodelversion_versionprovider.py | 233 ++++++ .../0038_aimodelversion_lifecycle_stage.py | 31 + ...nprovider_api_auth_header_name_and_more.py | 143 ++++ ...0040_promptdataset_dataset_dataset_type.py | 145 ++++ ...041_alter_promptdataset_domain_and_more.py | 67 ++ .../0042_promptresource_and_more.py | 106 +++ api/migrations/0043_promptdataset_purpose.py | 29 + ..._schema_alter_aimodel_metadata_and_more.py | 50 ++ api/migrations/0045_aimodel_domain.py | 39 + .../0046_alter_collaborative_slug.py | 30 + ...ion_collaborative_publications_and_more.py | 226 ++++++ authorization/migrations/0001_initial.py | 261 ++++++ .../migrations/0002_alter_user_table.py | 17 + ..._profile_user_linkedin_profile_and_more.py | 43 + authorization/migrations/0004_userconsent.py | 50 ++ authorization/migrations/__init__.py | 0 52 files changed, 3600 insertions(+), 2 deletions(-) create mode 100755 api/migrations/0002_dataspace_dataset_dataspace.py create mode 100755 api/migrations/0003_usecase.py create mode 100755 api/migrations/0004_usecase_status.py create mode 100755 api/migrations/0005_alter_usecase_slug.py create mode 100755 api/migrations/0006_alter_usecase_description_alter_usecase_title.py create mode 100755 api/migrations/0007_resourcechartimage.py create mode 100755 api/migrations/0008_alter_resourcechartdetails_aggregate_type.py create mode 100755 api/migrations/0009_alter_resourcechartdetails_aggregate_type.py create mode 100755 api/migrations/0010_resourcechartdetails_modified_and_more.py create mode 100755 api/migrations/0011_alter_resourceschema_format.py create mode 100755 api/migrations/0012_resourcechartdetails_filters.py create mode 100755 api/migrations/0013_remove_resourcechartdetails_aggregate_type_and_more.py create mode 100755 api/migrations/0014_alter_category_options_alter_dataset_options_and_more.py create mode 100755 api/migrations/0015_dataset_slug_resource_slug.py create mode 100755 api/migrations/0016_alter_dataset_slug_alter_resource_slug.py create mode 100755 api/migrations/0017_resourcedatatable.py create mode 100755 api/migrations/0018_resource_download_count.py create mode 100755 api/migrations/0019_rename_category_sector_alter_sector_options_and_more.py create mode 100755 api/migrations/0020_datasetmetadata_created_datasetmetadata_modified_and_more.py create mode 100755 api/migrations/0021_rename_description_usecase_summary_and_more.py create mode 100755 api/migrations/0022_resource_version_resourceversion.py create mode 100755 api/migrations/0023_alter_usecase_summary.py create mode 100755 api/migrations/0024_alter_organization_organization_types.py create mode 100755 api/migrations/0025_usecase_completed_on_usecase_started_on.py create mode 100755 api/migrations/0026_usecase_contributors.py create mode 100755 api/migrations/0027_usecaseorganizationrelationship_and_more.py create mode 100755 api/migrations/0028_dataset_access_type_dataset_license.py create mode 100755 api/migrations/0029_dataset_user.py create mode 100755 api/migrations/0030_organization_github_profile_and_more.py create mode 100755 api/migrations/0031_usecase_user_alter_resourcechartdetails_chart_type.py create mode 100755 api/migrations/0032_resourcechartdetails_status_and_more.py create mode 100755 api/migrations/0033_usecase_organization.py create mode 100755 api/migrations/0034_usecasedashboard.py create mode 100755 api/migrations/0035_usecase_platform_url.py create mode 100755 api/migrations/0036_sdg_dataset_geographies_usecase_geographies_and_more.py create mode 100755 api/migrations/0037_alter_aimodel_status_aimodelversion_versionprovider.py create mode 100755 api/migrations/0038_aimodelversion_lifecycle_stage.py create mode 100755 api/migrations/0039_versionprovider_api_auth_header_name_and_more.py create mode 100755 api/migrations/0040_promptdataset_dataset_dataset_type.py create mode 100755 api/migrations/0041_alter_promptdataset_domain_and_more.py create mode 100755 api/migrations/0042_promptresource_and_more.py create mode 100755 api/migrations/0043_promptdataset_purpose.py create mode 100755 api/migrations/0044_alter_aimodel_input_schema_alter_aimodel_metadata_and_more.py create mode 100755 api/migrations/0045_aimodel_domain.py create mode 100755 api/migrations/0046_alter_collaborative_slug.py create mode 100644 api/migrations/0047_resourcetype_publication_collaborative_publications_and_more.py create mode 100644 authorization/migrations/0001_initial.py create mode 100644 authorization/migrations/0002_alter_user_table.py create mode 100644 authorization/migrations/0003_user_github_profile_user_linkedin_profile_and_more.py create mode 100644 authorization/migrations/0004_userconsent.py create mode 100644 authorization/migrations/__init__.py diff --git a/.gitignore b/.gitignore index 847c7a0..68a4bce 100644 --- a/.gitignore +++ b/.gitignore @@ -155,8 +155,6 @@ cython_debug/ #local env file .env -api/migrations/* -authorization/migrations/* # AWS files diff --git a/api/migrations/0002_dataspace_dataset_dataspace.py b/api/migrations/0002_dataspace_dataset_dataspace.py new file mode 100755 index 0000000..f631925 --- /dev/null +++ b/api/migrations/0002_dataspace_dataset_dataspace.py @@ -0,0 +1,34 @@ +# Generated by Django 5.0.4 on 2024-10-17 11:47 + +import api.utils.file_paths +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='DataSpace', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200)), + ('description', models.CharField(max_length=1000)), + ('logo', models.ImageField(blank=True, null=True, upload_to=api.utils.file_paths._dataspace_directory_path)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('homepage', models.URLField(blank=True)), + ('contact_email', models.EmailField(blank=True, max_length=254, null=True)), + ('slug', models.SlugField(max_length=75, null=True, unique=True)), + ], + ), + migrations.AddField( + model_name='dataset', + name='dataspace', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.dataspace'), + ), + ] diff --git a/api/migrations/0003_usecase.py b/api/migrations/0003_usecase.py new file mode 100755 index 0000000..db9434c --- /dev/null +++ b/api/migrations/0003_usecase.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.4 on 2024-10-17 18:09 + +import api.utils.file_paths +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0002_dataspace_dataset_dataspace'), + ] + + operations = [ + migrations.CreateModel( + name='UseCase', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=200, unique=True)), + ('description', models.CharField(max_length=1000)), + ('logo', models.ImageField(blank=True, null=True, upload_to=api.utils.file_paths._use_case_directory_path)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('website', models.URLField(blank=True)), + ('contact_email', models.EmailField(blank=True, max_length=254, null=True)), + ('slug', models.SlugField(max_length=75, null=True, unique=True)), + ('datasets', models.ManyToManyField(blank=True, to='api.dataset')), + ], + ), + ] diff --git a/api/migrations/0004_usecase_status.py b/api/migrations/0004_usecase_status.py new file mode 100755 index 0000000..7b5735d --- /dev/null +++ b/api/migrations/0004_usecase_status.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2024-10-17 18:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0003_usecase'), + ] + + operations = [ + migrations.AddField( + model_name='usecase', + name='status', + field=models.CharField(choices=[('DRAFT', 'Draft'), ('PUBLISHED', 'Published'), ('ARCHIVED', 'Archived')], default='DRAFT', max_length=50), + ), + ] diff --git a/api/migrations/0005_alter_usecase_slug.py b/api/migrations/0005_alter_usecase_slug.py new file mode 100755 index 0000000..7cff027 --- /dev/null +++ b/api/migrations/0005_alter_usecase_slug.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2024-10-18 09:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0004_usecase_status'), + ] + + operations = [ + migrations.AlterField( + model_name='usecase', + name='slug', + field=models.SlugField(blank=True, max_length=75, null=True, unique=True), + ), + ] diff --git a/api/migrations/0006_alter_usecase_description_alter_usecase_title.py b/api/migrations/0006_alter_usecase_description_alter_usecase_title.py new file mode 100755 index 0000000..5a21279 --- /dev/null +++ b/api/migrations/0006_alter_usecase_description_alter_usecase_title.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.4 on 2024-10-21 05:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0005_alter_usecase_slug'), + ] + + operations = [ + migrations.AlterField( + model_name='usecase', + name='description', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + migrations.AlterField( + model_name='usecase', + name='title', + field=models.CharField(blank=True, max_length=200, null=True, unique=True), + ), + ] diff --git a/api/migrations/0007_resourcechartimage.py b/api/migrations/0007_resourcechartimage.py new file mode 100755 index 0000000..cdc31de --- /dev/null +++ b/api/migrations/0007_resourcechartimage.py @@ -0,0 +1,24 @@ +# Generated by Django 5.0.4 on 2024-10-22 12:41 + +import api.utils.file_paths +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0006_alter_usecase_description_alter_usecase_title'), + ] + + operations = [ + migrations.CreateModel( + name='ResourceChartImage', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(blank=True, max_length=50)), + ('description', models.CharField(blank=True, max_length=1000)), + ('image', models.ImageField(blank=True, null=True, upload_to=api.utils.file_paths._chart_image_directory_path)), + ], + ), + ] diff --git a/api/migrations/0008_alter_resourcechartdetails_aggregate_type.py b/api/migrations/0008_alter_resourcechartdetails_aggregate_type.py new file mode 100755 index 0000000..9771abd --- /dev/null +++ b/api/migrations/0008_alter_resourcechartdetails_aggregate_type.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2024-10-23 09:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0007_resourcechartimage'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcechartdetails', + name='aggregate_type', + field=models.CharField(choices=[('NONE', 'None'), ('SUM', 'Sum'), ('AVG', 'Average'), ('COUNT', 'Count')], default='NONE', max_length=50), + ), + ] diff --git a/api/migrations/0009_alter_resourcechartdetails_aggregate_type.py b/api/migrations/0009_alter_resourcechartdetails_aggregate_type.py new file mode 100755 index 0000000..9b0a2e4 --- /dev/null +++ b/api/migrations/0009_alter_resourcechartdetails_aggregate_type.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2024-10-23 10:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0008_alter_resourcechartdetails_aggregate_type'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcechartdetails', + name='aggregate_type', + field=models.CharField(choices=[('NONE', 'None'), ('SUM', 'Sum'), ('MEAN', 'Average'), ('COUNT', 'Count')], default='NONE', max_length=50), + ), + ] diff --git a/api/migrations/0010_resourcechartdetails_modified_and_more.py b/api/migrations/0010_resourcechartdetails_modified_and_more.py new file mode 100755 index 0000000..f3b9353 --- /dev/null +++ b/api/migrations/0010_resourcechartdetails_modified_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.4 on 2024-10-23 11:00 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0009_alter_resourcechartdetails_aggregate_type'), + ] + + operations = [ + migrations.AddField( + model_name='resourcechartdetails', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='resourcechartimage', + name='dataset', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='chart_images', to='api.dataset'), + ), + migrations.AddField( + model_name='resourcechartimage', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/api/migrations/0011_alter_resourceschema_format.py b/api/migrations/0011_alter_resourceschema_format.py new file mode 100755 index 0000000..6e4f46e --- /dev/null +++ b/api/migrations/0011_alter_resourceschema_format.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2024-10-24 06:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0010_resourcechartdetails_modified_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='resourceschema', + name='format', + field=models.CharField(choices=[('STRING', 'String'), ('NUMBER', 'Number'), ('INTEGER', 'Integer'), ('DATE', 'Date'), ('BOOLEAN', 'Boolean')], max_length=255), + ), + ] diff --git a/api/migrations/0012_resourcechartdetails_filters.py b/api/migrations/0012_resourcechartdetails_filters.py new file mode 100755 index 0000000..d8580d9 --- /dev/null +++ b/api/migrations/0012_resourcechartdetails_filters.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2024-12-14 16:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0011_alter_resourceschema_format'), + ] + + operations = [ + migrations.AddField( + model_name='resourcechartdetails', + name='filters', + field=models.JSONField(blank=True, default=list), + ), + ] diff --git a/api/migrations/0013_remove_resourcechartdetails_aggregate_type_and_more.py b/api/migrations/0013_remove_resourcechartdetails_aggregate_type_and_more.py new file mode 100755 index 0000000..50d150c --- /dev/null +++ b/api/migrations/0013_remove_resourcechartdetails_aggregate_type_and_more.py @@ -0,0 +1,61 @@ +# Generated by Django 5.0.4 on 2025-02-26 14:07 + +import api.models.SerializableJSONField +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0012_resourcechartdetails_filters'), + ] + + operations = [ + migrations.RemoveField( + model_name='resourcechartdetails', + name='aggregate_type', + ), + migrations.RemoveField( + model_name='resourcechartdetails', + name='region_column', + ), + migrations.RemoveField( + model_name='resourcechartdetails', + name='show_legend', + ), + migrations.RemoveField( + model_name='resourcechartdetails', + name='value_column', + ), + migrations.RemoveField( + model_name='resourcechartdetails', + name='x_axis_column', + ), + migrations.RemoveField( + model_name='resourcechartdetails', + name='x_axis_label', + ), + migrations.RemoveField( + model_name='resourcechartdetails', + name='y_axis_column', + ), + migrations.RemoveField( + model_name='resourcechartdetails', + name='y_axis_label', + ), + migrations.AddField( + model_name='resourcechartdetails', + name='options', + field=api.models.SerializableJSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='resourcechartdetails', + name='chart_type', + field=models.CharField(choices=[('BAR_VERTICAL', 'Bar Vertical'), ('BAR_HORIZONTAL', 'Bar Horizontal'), ('GROUPED_BAR_VERTICAL', 'Grouped Bar Vertical'), ('GROUPED_BAR_HORIZONTAL', 'Grouped Bar Horizontal'), ('LINE', 'Line'), ('MULTILINE', 'Multiline'), ('ASSAM_DISTRICT', 'Assam District'), ('ASSAM_RC', 'Assam Rc')], default='BAR_HORIZONTAL', max_length=50), + ), + migrations.AlterField( + model_name='resourcechartdetails', + name='filters', + field=api.models.SerializableJSONField(blank=True, default=list, null=True), + ), + ] diff --git a/api/migrations/0014_alter_category_options_alter_dataset_options_and_more.py b/api/migrations/0014_alter_category_options_alter_dataset_options_and_more.py new file mode 100755 index 0000000..1057b48 --- /dev/null +++ b/api/migrations/0014_alter_category_options_alter_dataset_options_and_more.py @@ -0,0 +1,263 @@ +# Generated by Django 5.0.4 on 2025-03-04 10:33 + +import django.db.models.deletion +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0013_remove_resourcechartdetails_aggregate_type_and_more'), + ] + + operations = [ + migrations.AlterModelOptions( + name='category', + options={'verbose_name_plural': 'categories'}, + ), + migrations.AlterModelOptions( + name='dataset', + options={'ordering': ['-modified'], 'verbose_name': 'Dataset', 'verbose_name_plural': 'Datasets'}, + ), + migrations.AlterModelOptions( + name='datasetmetadata', + options={'ordering': ['metadata_item__label']}, + ), + migrations.AlterModelOptions( + name='geography', + options={'ordering': ['name'], 'verbose_name_plural': 'geographies'}, + ), + migrations.AlterModelOptions( + name='metadata', + options={'ordering': ['label']}, + ), + migrations.AlterModelOptions( + name='resourcechartdetails', + options={'ordering': ['-created']}, + ), + migrations.AlterModelOptions( + name='resourcemetadata', + options={'ordering': ['metadata_item__label']}, + ), + migrations.AddField( + model_name='accessmodelresource', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='accessmodelresource', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='metadata', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='metadata', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='resource', + name='is_active', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='resource', + name='url', + field=models.URLField(default='', max_length=500), + preserve_default=False, + ), + migrations.AddField( + model_name='resourcechartdetails', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='resourcemetadata', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='resourcemetadata', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='resourceschema', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='resourceschema', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='dataset', + name='categories', + field=models.ManyToManyField(blank=True, related_name='datasets', to='api.category'), + ), + migrations.AlterField( + model_name='dataset', + name='dataspace', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='datasets', to='api.dataspace'), + ), + migrations.AlterField( + model_name='dataset', + name='description', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + migrations.AlterField( + model_name='dataset', + name='organization', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='datasets', to='api.organization'), + ), + migrations.AlterField( + model_name='datasetmetadata', + name='id', + field=models.AutoField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='geography', + name='id', + field=models.AutoField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='geography', + name='parent_id', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.geography'), + ), + migrations.AlterField( + model_name='metadata', + name='data_standard', + field=models.CharField(blank=True, choices=[('DCATV3', 'Dcatv3'), ('OCDS', 'Ocds'), ('OBDS', 'Obds'), ('NA', 'Na')], max_length=50, null=True), + ), + migrations.AlterField( + model_name='metadata', + name='id', + field=models.AutoField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='resource', + name='dataset', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='resources', to='api.dataset'), + preserve_default=False, + ), + migrations.AlterField( + model_name='resource', + name='description', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='resource', + name='name', + field=models.CharField(max_length=200), + ), + migrations.AlterField( + model_name='resourcechartdetails', + name='resource', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.resource'), + ), + migrations.AlterField( + model_name='resourcemetadata', + name='id', + field=models.AutoField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='resourcemetadata', + name='metadata_item', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='resource_metadata', to='api.metadata'), + ), + migrations.AlterField( + model_name='resourcemetadata', + name='resource', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='metadata_items', to='api.resource'), + ), + migrations.AlterField( + model_name='resourcemetadata', + name='value', + field=models.JSONField(), + ), + migrations.AlterField( + model_name='usecase', + name='id', + field=models.AutoField(primary_key=True, serialize=False), + ), + migrations.AlterUniqueTogether( + name='datasetmetadata', + unique_together={('dataset', 'metadata_item')}, + ), + migrations.AlterUniqueTogether( + name='resourcemetadata', + unique_together={('resource', 'metadata_item')}, + ), + migrations.AlterModelTable( + name='accessmodel', + table='access_model', + ), + migrations.AlterModelTable( + name='accessmodelresource', + table='access_model_resource', + ), + migrations.AlterModelTable( + name='category', + table='category', + ), + migrations.AlterModelTable( + name='dataset', + table='dataset', + ), + migrations.AlterModelTable( + name='datasetmetadata', + table='dataset_metadata', + ), + migrations.AlterModelTable( + name='dataspace', + table='dataspace', + ), + migrations.AlterModelTable( + name='geography', + table='geography', + ), + migrations.AlterModelTable( + name='metadata', + table='metadata', + ), + migrations.AlterModelTable( + name='organization', + table='organization', + ), + migrations.AlterModelTable( + name='resourcechartdetails', + table='resource_chart_details', + ), + migrations.AlterModelTable( + name='resourcechartimage', + table='resource_chart_image', + ), + migrations.AlterModelTable( + name='resourcemetadata', + table='resource_metadata', + ), + migrations.AlterModelTable( + name='resourceschema', + table='resource_schema', + ), + migrations.AlterModelTable( + name='tag', + table='tag', + ), + migrations.AlterModelTable( + name='usecase', + table='use_case', + ), + ] diff --git a/api/migrations/0015_dataset_slug_resource_slug.py b/api/migrations/0015_dataset_slug_resource_slug.py new file mode 100755 index 0000000..ecbdccc --- /dev/null +++ b/api/migrations/0015_dataset_slug_resource_slug.py @@ -0,0 +1,68 @@ +# Generated by Django 5.0.4 on 2025-03-04 10:37 + +from django.db import migrations, models + +import django.db.models.deletion +import django.utils.timezone +from django.db import migrations, models +from django.utils.text import slugify + +def populate_slugs(apps, schema_editor): + # Handle Dataset slugs + Dataset = apps.get_model('api', 'Dataset') + for dataset in Dataset.objects.all(): + base_slug = slugify(dataset.title) if dataset.title else 'dataset' + slug = base_slug + counter = 1 + while Dataset.objects.filter(slug=slug).exists(): + slug = f"{base_slug}-{counter}" + counter += 1 + dataset.slug = slug + dataset.save() + + # Handle Resource slugs + Resource = apps.get_model('api', 'Resource') + for resource in Resource.objects.all(): + base_slug = slugify(resource.name) if resource.name else 'resource' + slug = base_slug + counter = 1 + while Resource.objects.filter(slug=slug).exists(): + slug = f"{base_slug}-{counter}" + counter += 1 + resource.slug = slug + resource.save() +def reverse_populate_slugs(apps, schema_editor): + Dataset = apps.get_model('api', 'Dataset') + Resource = apps.get_model('api', 'Resource') + Dataset.objects.all().update(slug="") + Resource.objects.all().update(slug="") + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0014_alter_category_options_alter_dataset_options_and_more'), + ] + + operations = [ + migrations.RunSQL( + # Forward SQL - drop existing indexes if they exist + """ + DROP INDEX IF EXISTS api_dataset_slug_ad6fedeb_like; + DROP INDEX IF EXISTS api_resource_slug_ad6fedeb_like; + """, + # Reverse SQL - do nothing + "SELECT 1;" + ), + migrations.AddField( + model_name='dataset', + name='slug', + field=models.SlugField(default='2025-03-04 10:37:40.235409+00:00', max_length=255 ), + preserve_default=False, + ), + migrations.AddField( + model_name='resource', + name='slug', + field=models.SlugField(default='2025-03-04 10:37:50.554696+00:00', max_length=255), + preserve_default=False, + ), + migrations.RunPython(populate_slugs,reverse_populate_slugs,),] diff --git a/api/migrations/0016_alter_dataset_slug_alter_resource_slug.py b/api/migrations/0016_alter_dataset_slug_alter_resource_slug.py new file mode 100755 index 0000000..064287f --- /dev/null +++ b/api/migrations/0016_alter_dataset_slug_alter_resource_slug.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.4 on 2025-03-04 10:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0015_dataset_slug_resource_slug'), + ] + + operations = [ + migrations.AlterField( + model_name='dataset', + name='slug', + field=models.SlugField(max_length=255, unique=True), + ), + migrations.AlterField( + model_name='resource', + name='slug', + field=models.SlugField(max_length=255, unique=True), + ), + ] diff --git a/api/migrations/0017_resourcedatatable.py b/api/migrations/0017_resourcedatatable.py new file mode 100755 index 0000000..bbf0d59 --- /dev/null +++ b/api/migrations/0017_resourcedatatable.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.4 on 2025-03-13 11:49 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0016_alter_dataset_slug_alter_resource_slug'), + ] + + operations = [ + migrations.CreateModel( + name='ResourceDataTable', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('table_name', models.CharField(max_length=255, unique=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('resource', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='api.resource')), + ], + options={ + 'db_table': 'resource_data_table', + 'ordering': ['-modified'], + }, + ), + ] diff --git a/api/migrations/0018_resource_download_count.py b/api/migrations/0018_resource_download_count.py new file mode 100755 index 0000000..73b3d6d --- /dev/null +++ b/api/migrations/0018_resource_download_count.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2025-03-26 12:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0017_resourcedatatable"), + ] + + operations = [ + migrations.AddField( + model_name="resource", + name="download_count", + field=models.IntegerField(default=0), + ), + ] diff --git a/api/migrations/0019_rename_category_sector_alter_sector_options_and_more.py b/api/migrations/0019_rename_category_sector_alter_sector_options_and_more.py new file mode 100755 index 0000000..0b23bef --- /dev/null +++ b/api/migrations/0019_rename_category_sector_alter_sector_options_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 5.0.4 on 2025-03-27 10:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0018_resource_download_count"), + ] + + operations = [ + migrations.RenameModel( + old_name="Category", + new_name="Sector", + ), + migrations.AlterModelOptions( + name="sector", + options={"verbose_name_plural": "sectors"}, + ), + migrations.RenameField( + model_name="dataset", + old_name="categories", + new_name="sectors", + ), + migrations.AlterModelTable( + name="sector", + table="sector", + ), + ] diff --git a/api/migrations/0020_datasetmetadata_created_datasetmetadata_modified_and_more.py b/api/migrations/0020_datasetmetadata_created_datasetmetadata_modified_and_more.py new file mode 100755 index 0000000..a2a0e53 --- /dev/null +++ b/api/migrations/0020_datasetmetadata_created_datasetmetadata_modified_and_more.py @@ -0,0 +1,120 @@ +# Generated by Django 5.0.4 on 2025-04-10 07:27 + +import django.db.models.deletion +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0019_rename_category_sector_alter_sector_options_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="datasetmetadata", + name="created", + field=models.DateTimeField( + auto_now_add=True, default=django.utils.timezone.now + ), + preserve_default=False, + ), + migrations.AddField( + model_name="datasetmetadata", + name="modified", + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name="usecase", + name="running_status", + field=models.CharField( + choices=[ + ("DRAFT", "Draft"), + ("PUBLISHED", "Published"), + ("ARCHIVED", "Archived"), + ], + default="DRAFT", + max_length=50, + ), + ), + migrations.AddField( + model_name="usecase", + name="sectors", + field=models.ManyToManyField( + blank=True, related_name="usecases", to="api.sector" + ), + ), + migrations.AddField( + model_name="usecase", + name="tags", + field=models.ManyToManyField(blank=True, to="api.tag"), + ), + migrations.AlterField( + model_name="metadata", + name="model", + field=models.CharField( + choices=[ + ("DATASET", "Dataset"), + ("RESOURCE", "Reseource"), + ("USECASE", "Usecase"), + ], + max_length=50, + ), + ), + migrations.AlterField( + model_name="resourcechartdetails", + name="chart_type", + field=models.CharField( + choices=[ + ("BAR_VERTICAL", "Bar Vertical"), + ("BAR_HORIZONTAL", "Bar Horizontal"), + ("GROUPED_BAR_VERTICAL", "Grouped Bar Vertical"), + ("GROUPED_BAR_HORIZONTAL", "Grouped Bar Horizontal"), + ("LINE", "Line"), + ("MULTILINE", "Multiline"), + ("ASSAM_DISTRICT", "Assam District"), + ("ASSAM_RC", "Assam Rc"), + ("TREEMAP", "Treemap"), + ], + default="BAR_HORIZONTAL", + max_length=50, + ), + ), + migrations.AlterField( + model_name="resourcemetadata", + name="metadata_item", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.metadata" + ), + ), + migrations.CreateModel( + name="UseCaseMetadata", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("value", models.CharField(max_length=1000)), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "metadata_item", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.metadata" + ), + ), + ( + "usecase", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="metadata", + to="api.usecase", + ), + ), + ], + options={ + "db_table": "usecase_metadata", + "ordering": ["metadata_item__label"], + "abstract": False, + "unique_together": {("usecase", "metadata_item")}, + }, + ), + ] diff --git a/api/migrations/0021_rename_description_usecase_summary_and_more.py b/api/migrations/0021_rename_description_usecase_summary_and_more.py new file mode 100755 index 0000000..a6a2427 --- /dev/null +++ b/api/migrations/0021_rename_description_usecase_summary_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.0.4 on 2025-04-10 10:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0020_datasetmetadata_created_datasetmetadata_modified_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="usecase", + old_name="description", + new_name="summary", + ), + migrations.AlterField( + model_name="usecase", + name="running_status", + field=models.CharField( + choices=[ + ("INITIATED", "Initiated"), + ("ON_GOING", "On Going"), + ("COMPLETED", "Completed"), + ("ON_HOLD", "On Hold"), + ("CANCELLED", "Cancelled"), + ], + default="INITIATED", + max_length=50, + ), + ), + ] diff --git a/api/migrations/0022_resource_version_resourceversion.py b/api/migrations/0022_resource_version_resourceversion.py new file mode 100755 index 0000000..bc56e73 --- /dev/null +++ b/api/migrations/0022_resource_version_resourceversion.py @@ -0,0 +1,49 @@ +# Generated by Django 5.0.4 on 2025-04-10 11:30 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0021_rename_description_usecase_summary_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="resource", + name="version", + field=models.CharField(default="v1.0", max_length=50), + ), + migrations.CreateModel( + name="ResourceVersion", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("version_number", models.CharField(max_length=50)), + ("commit_hash", models.CharField(max_length=64, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("change_description", models.TextField(blank=True)), + ( + "resource", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="versions", + to="api.resource", + ), + ), + ], + options={ + "db_table": "resource_version", + "unique_together": {("resource", "version_number")}, + }, + ), + ] diff --git a/api/migrations/0023_alter_usecase_summary.py b/api/migrations/0023_alter_usecase_summary.py new file mode 100755 index 0000000..522c3c2 --- /dev/null +++ b/api/migrations/0023_alter_usecase_summary.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2025-04-11 06:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0022_resource_version_resourceversion"), + ] + + operations = [ + migrations.AlterField( + model_name="usecase", + name="summary", + field=models.CharField(blank=True, max_length=10000, null=True), + ), + ] diff --git a/api/migrations/0024_alter_organization_organization_types.py b/api/migrations/0024_alter_organization_organization_types.py new file mode 100755 index 0000000..0d52260 --- /dev/null +++ b/api/migrations/0024_alter_organization_organization_types.py @@ -0,0 +1,42 @@ +# Generated by Django 5.0.4 on 2025-04-29 08:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0023_alter_usecase_summary"), + ] + + operations = [ + migrations.AlterField( + model_name="organization", + name="organization_types", + field=models.CharField( + choices=[ + ("STATE GOVERNMENT", "State Government"), + ("UNION TERRITORY GOVERNMENT", "Union Territory Government"), + ("URBAN LOCAL BODY", "Urban Local Body"), + ("ACADEMIC INSTITUTION", "Academic Institution"), + ("CENTRAL GOVERNMENT", "Central Government"), + ("CITIZENS GROUP", "Citizens Group"), + ("CIVIL SOCIETY ORGANISATION", "Civil Society Organisation"), + ("INDUSTRY BODY", "Industry Body"), + ("MEDIA ORGANISATION", "Media Organisation"), + ( + "OPEN DATA/TECHNOLOGY COMMUNITY", + "Open Data Technology Community", + ), + ("PRIVATE COMPANY", "Private Company"), + ("PUBLIC SECTOR COMPANY", "Public Sector Company"), + ("OTHERS", "Others"), + ("STARTUP", "Startup"), + ("GOVERNMENT", "Government"), + ("CORPORATIONS", "Corporations"), + ("NGO", "Ngo"), + ], + max_length=100, + ), + ), + ] diff --git a/api/migrations/0025_usecase_completed_on_usecase_started_on.py b/api/migrations/0025_usecase_completed_on_usecase_started_on.py new file mode 100755 index 0000000..e4c2f3c --- /dev/null +++ b/api/migrations/0025_usecase_completed_on_usecase_started_on.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.4 on 2025-04-30 06:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0024_alter_organization_organization_types"), + ] + + operations = [ + migrations.AddField( + model_name="usecase", + name="completed_on", + field=models.DateField(blank=True, null=True), + ), + migrations.AddField( + model_name="usecase", + name="started_on", + field=models.DateField(blank=True, null=True), + ), + ] diff --git a/api/migrations/0026_usecase_contributors.py b/api/migrations/0026_usecase_contributors.py new file mode 100755 index 0000000..949765b --- /dev/null +++ b/api/migrations/0026_usecase_contributors.py @@ -0,0 +1,24 @@ +# Generated by Django 5.0.4 on 2025-04-30 07:15 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0025_usecase_completed_on_usecase_started_on"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="usecase", + name="contributors", + field=models.ManyToManyField( + blank=True, + related_name="contributed_usecases", + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/api/migrations/0027_usecaseorganizationrelationship_and_more.py b/api/migrations/0027_usecaseorganizationrelationship_and_more.py new file mode 100755 index 0000000..2dd68d3 --- /dev/null +++ b/api/migrations/0027_usecaseorganizationrelationship_and_more.py @@ -0,0 +1,66 @@ +# Generated by Django 5.0.4 on 2025-04-30 12:34 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0026_usecase_contributors"), + ] + + operations = [ + migrations.CreateModel( + name="UseCaseOrganizationRelationship", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "relationship_type", + models.CharField( + choices=[("SUPPORTER", "Supporter"), ("PARTNER", "Partner")], + max_length=50, + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "organization", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="api.organization", + ), + ), + ( + "usecase", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.usecase" + ), + ), + ], + options={ + "verbose_name": "UseCase Organization Relationship", + "verbose_name_plural": "UseCase Organization Relationships", + "db_table": "usecase_organization_relationship", + "unique_together": {("usecase", "organization", "relationship_type")}, + }, + ), + migrations.AddField( + model_name="usecase", + name="organizations", + field=models.ManyToManyField( + blank=True, + related_name="related_usecases", + through="api.UseCaseOrganizationRelationship", + to="api.organization", + ), + ), + ] diff --git a/api/migrations/0028_dataset_access_type_dataset_license.py b/api/migrations/0028_dataset_access_type_dataset_license.py new file mode 100755 index 0000000..96c12c4 --- /dev/null +++ b/api/migrations/0028_dataset_access_type_dataset_license.py @@ -0,0 +1,47 @@ +# Generated by Django 5.0.4 on 2025-05-01 12:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0027_usecaseorganizationrelationship_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="dataset", + name="access_type", + field=models.CharField( + choices=[ + ("PUBLIC", "Public"), + ("PRIVATE", "Private"), + ("RESTRICTED", "Restricted"), + ], + default="PUBLIC", + max_length=50, + ), + ), + migrations.AddField( + model_name="dataset", + name="license", + field=models.CharField( + choices=[ + ("GOVERNMENT_OPEN_DATA_LICENSE", "Government Open Data License"), + ("CC_BY_4_0_ATTRIBUTION", "Cc By 4 0 Attribution"), + ( + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "Cc By Sa 4 0 Attribution Share Alike", + ), + ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "Open Data Commons By Attribution", + ), + ("OPEN_DATABASE_LICENSE", "Open Database License"), + ], + default="CC_BY_4_0_ATTRIBUTION", + max_length=50, + ), + ), + ] diff --git a/api/migrations/0029_dataset_user.py b/api/migrations/0029_dataset_user.py new file mode 100755 index 0000000..ea6c3a1 --- /dev/null +++ b/api/migrations/0029_dataset_user.py @@ -0,0 +1,27 @@ +# Generated by Django 5.0.4 on 2025-05-07 13:15 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0028_dataset_access_type_dataset_license"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="dataset", + name="user", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="datasets", + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/api/migrations/0030_organization_github_profile_and_more.py b/api/migrations/0030_organization_github_profile_and_more.py new file mode 100755 index 0000000..985ff76 --- /dev/null +++ b/api/migrations/0030_organization_github_profile_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.0.4 on 2025-05-09 14:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0029_dataset_user"), + ] + + operations = [ + migrations.AddField( + model_name="organization", + name="github_profile", + field=models.URLField(blank=True, null=True), + ), + migrations.AddField( + model_name="organization", + name="linkedin_profile", + field=models.URLField(blank=True, null=True), + ), + migrations.AddField( + model_name="organization", + name="location", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="organization", + name="twitter_profile", + field=models.URLField(blank=True, null=True), + ), + ] diff --git a/api/migrations/0031_usecase_user_alter_resourcechartdetails_chart_type.py b/api/migrations/0031_usecase_user_alter_resourcechartdetails_chart_type.py new file mode 100755 index 0000000..210c00a --- /dev/null +++ b/api/migrations/0031_usecase_user_alter_resourcechartdetails_chart_type.py @@ -0,0 +1,46 @@ +# Generated by Django 5.0.4 on 2025-05-21 09:06 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0030_organization_github_profile_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="usecase", + name="user", + field=models.ForeignKey( + default=1, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="resourcechartdetails", + name="chart_type", + field=models.CharField( + choices=[ + ("BAR", "Bar"), + ("LINE", "Line"), + ("BAR_VERTICAL", "Bar Vertical"), + ("BAR_HORIZONTAL", "Bar Horizontal"), + ("GROUPED_BAR_VERTICAL", "Grouped Bar Vertical"), + ("GROUPED_BAR_HORIZONTAL", "Grouped Bar Horizontal"), + ("MULTILINE", "Multiline"), + ("ASSAM_DISTRICT", "Assam District"), + ("ASSAM_RC", "Assam Rc"), + ("TREEMAP", "Treemap"), + ], + default="BAR_HORIZONTAL", + max_length=50, + ), + ), + ] diff --git a/api/migrations/0032_resourcechartdetails_status_and_more.py b/api/migrations/0032_resourcechartdetails_status_and_more.py new file mode 100755 index 0000000..b89f1fc --- /dev/null +++ b/api/migrations/0032_resourcechartdetails_status_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 5.0.4 on 2025-05-21 13:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0031_usecase_user_alter_resourcechartdetails_chart_type"), + ] + + operations = [ + migrations.AddField( + model_name="resourcechartdetails", + name="status", + field=models.CharField( + choices=[ + ("DRAFT", "Draft"), + ("PUBLISHED", "Published"), + ("ARCHIVED", "Archived"), + ], + default="DRAFT", + max_length=50, + ), + ), + migrations.AddField( + model_name="resourcechartimage", + name="status", + field=models.CharField( + choices=[ + ("DRAFT", "Draft"), + ("PUBLISHED", "Published"), + ("ARCHIVED", "Archived"), + ], + default="DRAFT", + max_length=50, + ), + ), + ] diff --git a/api/migrations/0033_usecase_organization.py b/api/migrations/0033_usecase_organization.py new file mode 100755 index 0000000..af2821d --- /dev/null +++ b/api/migrations/0033_usecase_organization.py @@ -0,0 +1,24 @@ +# Generated by Django 5.0.4 on 2025-05-27 08:43 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0032_resourcechartdetails_status_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="usecase", + name="organization", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="api.organization", + ), + ), + ] diff --git a/api/migrations/0034_usecasedashboard.py b/api/migrations/0034_usecasedashboard.py new file mode 100755 index 0000000..c358818 --- /dev/null +++ b/api/migrations/0034_usecasedashboard.py @@ -0,0 +1,36 @@ +# Generated by Django 5.0.4 on 2025-06-13 06:47 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0033_usecase_organization"), + ] + + operations = [ + migrations.CreateModel( + name="UseCaseDashboard", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("name", models.CharField(max_length=200)), + ("link", models.URLField()), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "usecase", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="dashboards", + to="api.usecase", + ), + ), + ], + options={ + "db_table": "usecase_dashboard", + "ordering": ["created"], + }, + ), + ] diff --git a/api/migrations/0035_usecase_platform_url.py b/api/migrations/0035_usecase_platform_url.py new file mode 100755 index 0000000..17bbeab --- /dev/null +++ b/api/migrations/0035_usecase_platform_url.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2025-06-26 10:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0034_usecasedashboard"), + ] + + operations = [ + migrations.AddField( + model_name="usecase", + name="platform_url", + field=models.URLField(blank=True, null=True), + ), + ] diff --git a/api/migrations/0036_sdg_dataset_geographies_usecase_geographies_and_more.py b/api/migrations/0036_sdg_dataset_geographies_usecase_geographies_and_more.py new file mode 100755 index 0000000..78d7b9b --- /dev/null +++ b/api/migrations/0036_sdg_dataset_geographies_usecase_geographies_and_more.py @@ -0,0 +1,746 @@ +# Generated by Django 5.0.4 on 2025-12-23 08:38 + +import api.utils.file_paths +import django.core.validators +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0035_usecase_platform_url"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="SDG", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + ("name", models.CharField(max_length=100, unique=True)), + ("code", models.CharField(max_length=10, unique=True)), + ("number", models.IntegerField(blank=True, null=True)), + ("description", models.TextField(blank=True, null=True)), + ("slug", models.SlugField(max_length=100, null=True, unique=True)), + ], + options={ + "verbose_name": "SDG", + "verbose_name_plural": "SDGs", + "db_table": "sdg", + "ordering": ["number"], + }, + ), + migrations.AddField( + model_name="dataset", + name="geographies", + field=models.ManyToManyField(blank=True, related_name="datasets", to="api.geography"), + ), + migrations.AddField( + model_name="usecase", + name="geographies", + field=models.ManyToManyField(blank=True, related_name="usecases", to="api.geography"), + ), + migrations.AlterField( + model_name="dataset", + name="user", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="datasets", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="geography", + name="type", + field=models.CharField( + choices=[ + ("REGION", "Region"), + ("COUNTRY", "Country"), + ("STATE", "State"), + ("DISTRICT", "District"), + ("UT", "Ut"), + ], + max_length=20, + ), + ), + migrations.AlterField( + model_name="metadata", + name="model", + field=models.CharField( + choices=[ + ("DATASET", "Dataset"), + ("RESOURCE", "Reseource"), + ("USECASE", "Usecase"), + ("COLLABORATIVE", "Collaborative"), + ], + max_length=50, + ), + ), + migrations.AlterField( + model_name="organization", + name="logo", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + upload_to=api.utils.file_paths._organization_directory_path, + ), + ), + migrations.AlterField( + model_name="resourcechartdetails", + name="chart_type", + field=models.CharField( + choices=[ + ("BAR", "Bar"), + ("LINE", "Line"), + ("BIG_NUMBER", "Big Number"), + ("ASSAM_DISTRICT", "Assam District"), + ("ASSAM_RC", "Assam Rc"), + ("TREEMAP", "Treemap"), + ("POLYGON_MAP", "Polygon Map"), + ("POINT_MAP", "Point Map"), + ("GEOSPATIAL_MAP", "Geospatial Map"), + ], + default="BAR", + max_length=50, + ), + ), + migrations.AlterField( + model_name="resourcechartimage", + name="image", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + upload_to=api.utils.file_paths._chart_image_directory_path, + ), + ), + migrations.AlterField( + model_name="resourcefiledetails", + name="file", + field=models.FileField(max_length=300, upload_to="resources/"), + ), + migrations.AlterField( + model_name="usecase", + name="logo", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + migrations.CreateModel( + name="AIModel", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(help_text="Model name or identifier", max_length=255)), + ("display_name", models.CharField(help_text="Human-readable name", max_length=255)), + ("version", models.CharField(blank=True, help_text="Model version", max_length=50)), + ( + "description", + models.TextField(help_text="Description of the model's capabilities"), + ), + ( + "model_type", + models.CharField( + choices=[ + ("TRANSLATION", "Translation"), + ("TEXT_GENERATION", "Text Generation"), + ("SUMMARIZATION", "Summarization"), + ("QUESTION_ANSWERING", "Question Answering"), + ("SENTIMENT_ANALYSIS", "Sentiment Analysis"), + ("TEXT_CLASSIFICATION", "Text Classification"), + ("NAMED_ENTITY_RECOGNITION", "Named Entity Recognition"), + ("TEXT_TO_SPEECH", "Text To Speech"), + ("SPEECH_TO_TEXT", "Speech To Text"), + ("OTHER", "Other"), + ], + max_length=50, + ), + ), + ( + "provider", + models.CharField( + choices=[ + ("OPENAI", "Openai"), + ("LLAMA_OLLAMA", "Llama Ollama"), + ("LLAMA_TOGETHER", "Llama Together"), + ("LLAMA_REPLICATE", "Llama Replicate"), + ("LLAMA_CUSTOM", "Llama Custom"), + ("CUSTOM", "Custom"), + ("HUGGINGFACE", "Huggingface"), + ], + max_length=50, + ), + ), + ( + "provider_model_id", + models.CharField( + blank=True, + help_text="Provider's model identifier (e.g., gpt-4, claude-3-opus)", + max_length=255, + ), + ), + ( + "hf_use_pipeline", + models.BooleanField(default=False, help_text="Use Pipeline inference API"), + ), + ( + "hf_auth_token", + models.CharField( + blank=True, + help_text="Huggingface Auth Token for gated models", + max_length=255, + null=True, + ), + ), + ( + "hf_model_class", + models.CharField( + blank=True, + choices=[ + ("AutoModelForCausalLM", "Causal Language Modeling Head"), + ( + "AutoModelForSeq2SeqLM", + "Sequence-To-Sequence Language Modeling Head", + ), + ("AutoModelForSequenceClassification", "Sequence Classification Head"), + ("AutoModelForNextSentencePrediction", "Next Sentence Prediction Head"), + ("AutoModelForMultipleChoice", "Multiple Choice Head"), + ("AutoModelForTokenClassification", "Token Classification Head"), + ("AutoModelForQuestionAnswering", "Question Answering Head"), + ("AutoModelForMaskedLM", "Masked Language Modeling Head"), + ], + help_text="Specify model head to use", + max_length=100, + null=True, + ), + ), + ( + "hf_attn_implementation", + models.CharField( + blank=True, + default="flash_attention_2", + help_text="Attention Function", + max_length=255, + ), + ), + ( + "framework", + models.CharField( + blank=True, + choices=[("pt", "PyTorch"), ("tf", "TensorFlow")], + help_text="Framework (PyTorch or TensorFlow)", + max_length=10, + null=True, + ), + ), + ("supports_streaming", models.BooleanField(default=False)), + ( + "max_tokens", + models.IntegerField( + blank=True, help_text="Maximum tokens supported by the model", null=True + ), + ), + ( + "supported_languages", + models.JSONField( + default=list, + help_text="List of supported language codes (e.g., ['en', 'es', 'fr'])", + ), + ), + ( + "input_schema", + models.JSONField( + default=dict, help_text="Expected input format and parameters" + ), + ), + ( + "output_schema", + models.JSONField(default=dict, help_text="Expected output format"), + ), + ( + "metadata", + models.JSONField( + default=dict, + help_text="Additional metadata (training data info, limitations, etc.)", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("REGISTERED", "Registered"), + ("VALIDATING", "Validating"), + ("ACTIVE", "Active"), + ("AUDITING", "Auditing"), + ("APPROVED", "Approved"), + ("FLAGGED", "Flagged"), + ("DEPRECATED", "Deprecated"), + ], + default="REGISTERED", + max_length=20, + ), + ), + ( + "is_public", + models.BooleanField( + default=False, help_text="Whether this model is publicly visible" + ), + ), + ( + "is_active", + models.BooleanField( + default=True, help_text="Whether this model is currently active" + ), + ), + ("average_latency_ms", models.FloatField(blank=True, null=True)), + ("success_rate", models.FloatField(blank=True, null=True)), + ("last_audit_score", models.FloatField(blank=True, null=True)), + ("audit_count", models.IntegerField(default=0)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("last_tested_at", models.DateTimeField(blank=True, null=True)), + ( + "geographies", + models.ManyToManyField( + blank=True, related_name="ai_models", to="api.geography" + ), + ), + ( + "organization", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="ai_models", + to="api.organization", + ), + ), + ( + "sectors", + models.ManyToManyField(blank=True, related_name="ai_models", to="api.sector"), + ), + ("tags", models.ManyToManyField(blank=True, to="api.tag")), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="ai_models", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["-created_at"], + }, + ), + migrations.CreateModel( + name="Catalog", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(max_length=255)), + ( + "logo", + models.ImageField( + blank=True, + null=True, + upload_to=api.utils.file_paths._catalog_directory_path, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ("homepage", models.URLField(blank=True)), + ("slug", models.SlugField(max_length=75, null=True, unique=True)), + ("datasets", models.ManyToManyField(related_name="catalogs", to="api.dataset")), + ], + options={ + "verbose_name": "catalog", + "verbose_name_plural": "catalogs", + }, + ), + migrations.CreateModel( + name="Collaborative", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("title", models.CharField(blank=True, max_length=200, null=True, unique=True)), + ("summary", models.CharField(blank=True, max_length=10000, null=True)), + ( + "logo", + models.ImageField( + blank=True, + max_length=300, + null=True, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + ( + "cover_image", + models.ImageField( + blank=True, + max_length=300, + null=True, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ("website", models.URLField(blank=True)), + ("contact_email", models.EmailField(blank=True, max_length=254, null=True)), + ("slug", models.SlugField(blank=True, max_length=75, null=True, unique=True)), + ( + "status", + models.CharField( + choices=[ + ("DRAFT", "Draft"), + ("PUBLISHED", "Published"), + ("ARCHIVED", "Archived"), + ], + default="DRAFT", + max_length=50, + ), + ), + ("started_on", models.DateField(blank=True, null=True)), + ("completed_on", models.DateField(blank=True, null=True)), + ("platform_url", models.URLField(blank=True, null=True)), + ( + "contributors", + models.ManyToManyField( + blank=True, + related_name="contributed_collaboratives", + to=settings.AUTH_USER_MODEL, + ), + ), + ("datasets", models.ManyToManyField(blank=True, to="api.dataset")), + ( + "geographies", + models.ManyToManyField( + blank=True, related_name="collaboratives", to="api.geography" + ), + ), + ( + "organization", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="api.organization", + ), + ), + ( + "sectors", + models.ManyToManyField( + blank=True, related_name="collaboratives", to="api.sector" + ), + ), + ("tags", models.ManyToManyField(blank=True, to="api.tag")), + ("use_cases", models.ManyToManyField(blank=True, to="api.usecase")), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), + ], + options={ + "db_table": "collaborative", + }, + ), + migrations.CreateModel( + name="CollaborativeMetadata", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("value", models.CharField(max_length=1000)), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "collaborative", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="metadata", + to="api.collaborative", + ), + ), + ( + "metadata_item", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.metadata" + ), + ), + ], + options={ + "db_table": "collaborative_metadata", + "ordering": ["metadata_item__label"], + "abstract": False, + }, + ), + migrations.CreateModel( + name="CollaborativeOrganizationRelationship", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "relationship_type", + models.CharField( + choices=[("SUPPORTER", "Supporter"), ("PARTNER", "Partner")], max_length=50 + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "collaborative", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.collaborative" + ), + ), + ( + "organization", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.organization" + ), + ), + ], + options={ + "verbose_name": "Collaborative Organization Relationship", + "verbose_name_plural": "Collaborative Organization Relationships", + "db_table": "collaborative_organization_relationship", + }, + ), + migrations.AddField( + model_name="collaborative", + name="organizations", + field=models.ManyToManyField( + blank=True, + related_name="related_collaboratives", + through="api.CollaborativeOrganizationRelationship", + to="api.organization", + ), + ), + migrations.CreateModel( + name="ModelAPIKey", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "name", + models.CharField(help_text="Friendly name for this API key", max_length=100), + ), + ("encrypted_key", models.BinaryField(help_text="Encrypted API key or token")), + ( + "key_type", + models.CharField( + default="api_key", + help_text="Type of credential (api_key, bearer_token, oauth_token, etc.)", + max_length=50, + ), + ), + ("is_active", models.BooleanField(default=True)), + ( + "expires_at", + models.DateTimeField( + blank=True, help_text="Expiration date for the key", null=True + ), + ), + ("last_used_at", models.DateTimeField(blank=True, null=True)), + ("usage_count", models.IntegerField(default=0)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "model", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="api_keys", + to="api.aimodel", + ), + ), + ], + options={ + "verbose_name": "Model API Key", + "verbose_name_plural": "Model API Keys", + "ordering": ["-created_at"], + }, + ), + migrations.CreateModel( + name="ModelEndpoint", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "url", + models.URLField( + help_text="API endpoint URL", + max_length=500, + validators=[django.core.validators.URLValidator()], + ), + ), + ( + "http_method", + models.CharField( + choices=[("GET", "Get"), ("POST", "Post"), ("PUT", "Put")], + default="POST", + max_length=10, + ), + ), + ( + "auth_type", + models.CharField( + choices=[ + ("BEARER", "Bearer"), + ("API_KEY", "Api Key"), + ("BASIC", "Basic"), + ("OAUTH2", "Oauth2"), + ("CUSTOM", "Custom"), + ("NONE", "None"), + ], + default="BEARER", + max_length=20, + ), + ), + ( + "auth_header_name", + models.CharField( + default="Authorization", + help_text="Header name for authentication (e.g., 'Authorization', 'X-API-Key')", + max_length=100, + ), + ), + ( + "headers", + models.JSONField( + default=dict, help_text="Additional headers to include in requests" + ), + ), + ( + "request_template", + models.JSONField( + default=dict, help_text="Template for request body with placeholders" + ), + ), + ( + "response_path", + models.CharField( + blank=True, + help_text="JSON path to extract response (e.g., 'choices[0].message.content')", + max_length=255, + ), + ), + ("timeout_seconds", models.IntegerField(default=30)), + ("max_retries", models.IntegerField(default=3)), + ( + "is_primary", + models.BooleanField(default=True, help_text="Primary endpoint to use"), + ), + ("is_active", models.BooleanField(default=True)), + ( + "rate_limit_per_minute", + models.IntegerField( + blank=True, help_text="Rate limit for this endpoint", null=True + ), + ), + ("last_success_at", models.DateTimeField(blank=True, null=True)), + ("last_failure_at", models.DateTimeField(blank=True, null=True)), + ("total_requests", models.IntegerField(default=0)), + ("failed_requests", models.IntegerField(default=0)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "model", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="endpoints", + to="api.aimodel", + ), + ), + ], + options={ + "ordering": ["-is_primary", "-created_at"], + }, + ), + migrations.AddField( + model_name="collaborative", + name="sdgs", + field=models.ManyToManyField(blank=True, related_name="collaboratives", to="api.sdg"), + ), + migrations.AddField( + model_name="usecase", + name="sdgs", + field=models.ManyToManyField(blank=True, related_name="usecases", to="api.sdg"), + ), + migrations.AddIndex( + model_name="aimodel", + index=models.Index( + fields=["user", "-created_at"], name="api_aimodel_user_id_7a7462_idx" + ), + ), + migrations.AddIndex( + model_name="aimodel", + index=models.Index( + fields=["status", "is_active"], name="api_aimodel_status_f93998_idx" + ), + ), + migrations.AddIndex( + model_name="aimodel", + index=models.Index( + fields=["model_type", "provider"], name="api_aimodel_model_t_0b746e_idx" + ), + ), + migrations.AddIndex( + model_name="aimodel", + index=models.Index( + fields=["provider", "provider_model_id"], name="api_aimodel_provide_f67293_idx" + ), + ), + migrations.AlterUniqueTogether( + name="aimodel", + unique_together={("user", "name", "version")}, + ), + migrations.AlterUniqueTogether( + name="collaborativemetadata", + unique_together={("collaborative", "metadata_item")}, + ), + migrations.AlterUniqueTogether( + name="collaborativeorganizationrelationship", + unique_together={("collaborative", "organization", "relationship_type")}, + ), + migrations.AddIndex( + model_name="modelendpoint", + index=models.Index( + fields=["model", "is_primary", "is_active"], name="api_modelen_model_i_0c0857_idx" + ), + ), + ] diff --git a/api/migrations/0037_alter_aimodel_status_aimodelversion_versionprovider.py b/api/migrations/0037_alter_aimodel_status_aimodelversion_versionprovider.py new file mode 100755 index 0000000..f2bef6c --- /dev/null +++ b/api/migrations/0037_alter_aimodel_status_aimodelversion_versionprovider.py @@ -0,0 +1,233 @@ +# Generated by Django 5.0.4 on 2025-12-24 10:34 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0036_sdg_dataset_geographies_usecase_geographies_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="aimodel", + name="status", + field=models.CharField( + choices=[ + ("DRAFT", "Draft"), + ("REGISTERED", "Registered"), + ("VALIDATING", "Validating"), + ("ACTIVE", "Active"), + ("AUDITING", "Auditing"), + ("APPROVED", "Approved"), + ("FLAGGED", "Flagged"), + ("DEPRECATED", "Deprecated"), + ], + default="REGISTERED", + max_length=20, + ), + ), + migrations.CreateModel( + name="AIModelVersion", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "version", + models.CharField(help_text="Version number (e.g., 1.0.0)", max_length=50), + ), + ( + "version_notes", + models.TextField(blank=True, help_text="Changelog/notes for this version"), + ), + ("supports_streaming", models.BooleanField(default=False)), + ( + "max_tokens", + models.IntegerField( + blank=True, help_text="Maximum tokens supported", null=True + ), + ), + ( + "supported_languages", + models.JSONField(default=list, help_text="List of supported language codes"), + ), + ( + "input_schema", + models.JSONField( + default=dict, help_text="Expected input format and parameters" + ), + ), + ( + "output_schema", + models.JSONField(default=dict, help_text="Expected output format"), + ), + ( + "metadata", + models.JSONField( + default=dict, help_text="Additional version-specific metadata" + ), + ), + ( + "status", + models.CharField( + choices=[ + ("DRAFT", "Draft"), + ("REGISTERED", "Registered"), + ("VALIDATING", "Validating"), + ("ACTIVE", "Active"), + ("AUDITING", "Auditing"), + ("APPROVED", "Approved"), + ("FLAGGED", "Flagged"), + ("DEPRECATED", "Deprecated"), + ], + default="REGISTERED", + max_length=20, + ), + ), + ( + "is_latest", + models.BooleanField( + default=False, help_text="Whether this is the latest version" + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("published_at", models.DateTimeField(blank=True, null=True)), + ( + "ai_model", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="versions", + to="api.aimodel", + ), + ), + ], + options={ + "verbose_name": "AI Model Version", + "verbose_name_plural": "AI Model Versions", + "ordering": ["-created_at"], + "unique_together": {("ai_model", "version")}, + }, + ), + migrations.CreateModel( + name="VersionProvider", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "provider", + models.CharField( + choices=[ + ("OPENAI", "Openai"), + ("LLAMA_OLLAMA", "Llama Ollama"), + ("LLAMA_TOGETHER", "Llama Together"), + ("LLAMA_REPLICATE", "Llama Replicate"), + ("LLAMA_CUSTOM", "Llama Custom"), + ("CUSTOM", "Custom"), + ("HUGGINGFACE", "Huggingface"), + ], + max_length=50, + ), + ), + ( + "provider_model_id", + models.CharField( + blank=True, + help_text="Provider's model identifier (e.g., gpt-4, claude-3-opus)", + max_length=255, + ), + ), + ( + "is_primary", + models.BooleanField( + default=False, + help_text="Whether this is the primary provider for the version", + ), + ), + ("is_active", models.BooleanField(default=True)), + ( + "hf_use_pipeline", + models.BooleanField(default=False, help_text="Use Pipeline inference API"), + ), + ( + "hf_auth_token", + models.CharField( + blank=True, + help_text="Huggingface Auth Token for gated models", + max_length=255, + null=True, + ), + ), + ( + "hf_model_class", + models.CharField( + blank=True, + choices=[ + ("AutoModelForCausalLM", "Causal Language Modeling Head"), + ( + "AutoModelForSeq2SeqLM", + "Sequence-To-Sequence Language Modeling Head", + ), + ("AutoModelForSequenceClassification", "Sequence Classification Head"), + ("AutoModelForNextSentencePrediction", "Next Sentence Prediction Head"), + ("AutoModelForMultipleChoice", "Multiple Choice Head"), + ("AutoModelForTokenClassification", "Token Classification Head"), + ("AutoModelForQuestionAnswering", "Question Answering Head"), + ("AutoModelForMaskedLM", "Masked Language Modeling Head"), + ], + help_text="Specify model head to use", + max_length=100, + null=True, + ), + ), + ( + "hf_attn_implementation", + models.CharField( + blank=True, + default="flash_attention_2", + help_text="Attention Function", + max_length=255, + ), + ), + ( + "framework", + models.CharField( + blank=True, + choices=[("pt", "PyTorch"), ("tf", "TensorFlow")], + help_text="Framework (PyTorch or TensorFlow)", + max_length=10, + null=True, + ), + ), + ( + "config", + models.JSONField(default=dict, help_text="Provider-specific configuration"), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "version", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="providers", + to="api.aimodelversion", + ), + ), + ], + options={ + "verbose_name": "Version Provider", + "verbose_name_plural": "Version Providers", + "ordering": ["-is_primary", "-created_at"], + }, + ), + ] diff --git a/api/migrations/0038_aimodelversion_lifecycle_stage.py b/api/migrations/0038_aimodelversion_lifecycle_stage.py new file mode 100755 index 0000000..414dfa3 --- /dev/null +++ b/api/migrations/0038_aimodelversion_lifecycle_stage.py @@ -0,0 +1,31 @@ +# Generated by Django 5.0.4 on 2025-12-26 08:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0037_alter_aimodel_status_aimodelversion_versionprovider"), + ] + + operations = [ + migrations.AddField( + model_name="aimodelversion", + name="lifecycle_stage", + field=models.CharField( + choices=[ + ("DEVELOPMENT", "Development"), + ("TESTING", "Testing"), + ("BETA", "Beta Testing"), + ("STAGING", "Staging"), + ("PRODUCTION", "Production"), + ("DEPRECATED", "Deprecated"), + ("RETIRED", "Retired"), + ], + default="DEVELOPMENT", + help_text="Current lifecycle stage of this version", + max_length=20, + ), + ), + ] diff --git a/api/migrations/0039_versionprovider_api_auth_header_name_and_more.py b/api/migrations/0039_versionprovider_api_auth_header_name_and_more.py new file mode 100755 index 0000000..bc12fd0 --- /dev/null +++ b/api/migrations/0039_versionprovider_api_auth_header_name_and_more.py @@ -0,0 +1,143 @@ +# Generated by Django 5.0.4 on 2025-12-30 15:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0038_aimodelversion_lifecycle_stage"), + ] + + operations = [ + migrations.AddField( + model_name="versionprovider", + name="api_auth_header_name", + field=models.CharField( + default="Authorization", + help_text="Header name for authentication (e.g., Authorization, X-API-Key)", + max_length=100, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_auth_type", + field=models.CharField( + choices=[ + ("BEARER", "Bearer"), + ("API_KEY", "Api Key"), + ("BASIC", "Basic"), + ("OAUTH2", "Oauth2"), + ("CUSTOM", "Custom"), + ("NONE", "None"), + ], + default="BEARER", + help_text="Authentication type for API calls", + max_length=20, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_endpoint_url", + field=models.URLField( + blank=True, + help_text="API endpoint URL (e.g., https://api.openai.com/v1/chat/completions)", + max_length=500, + null=True, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_headers", + field=models.JSONField( + blank=True, default=dict, help_text="Additional headers to include in requests" + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_http_method", + field=models.CharField( + choices=[("GET", "Get"), ("POST", "Post"), ("PUT", "Put")], + default="POST", + help_text="HTTP method for API calls", + max_length=10, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_key", + field=models.CharField( + blank=True, + help_text="API key or token (encrypted at rest)", + max_length=500, + null=True, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_key_prefix", + field=models.CharField( + blank=True, + default="Bearer", + help_text="Prefix for API key (e.g., Bearer, Token)", + max_length=50, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_request_template", + field=models.JSONField( + blank=True, + default=dict, + help_text="Request body template with placeholders like {input}, {prompt}", + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_response_path", + field=models.CharField( + blank=True, + help_text="JSON path to extract response (e.g., choices[0].message.content)", + max_length=255, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="api_timeout_seconds", + field=models.IntegerField(default=60, help_text="Request timeout in seconds"), + ), + migrations.AddField( + model_name="versionprovider", + name="hf_device_map", + field=models.CharField( + blank=True, + default="auto", + help_text="Device map for model loading (auto, cpu, cuda)", + max_length=50, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="hf_torch_dtype", + field=models.CharField( + blank=True, + default="auto", + help_text="Torch dtype (auto, float16, float32, bfloat16)", + max_length=50, + ), + ), + migrations.AddField( + model_name="versionprovider", + name="hf_trust_remote_code", + field=models.BooleanField( + default=True, help_text="Trust remote code when loading model" + ), + ), + migrations.AlterField( + model_name="versionprovider", + name="config", + field=models.JSONField( + default=dict, help_text="Additional provider-specific configuration" + ), + ), + ] diff --git a/api/migrations/0040_promptdataset_dataset_dataset_type.py b/api/migrations/0040_promptdataset_dataset_dataset_type.py new file mode 100755 index 0000000..a636b90 --- /dev/null +++ b/api/migrations/0040_promptdataset_dataset_dataset_type.py @@ -0,0 +1,145 @@ +# Generated by Django 5.0.4 on 2026-01-12 12:55 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0039_versionprovider_api_auth_header_name_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="PromptDataset", + fields=[ + ( + "dataset_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="api.dataset", + ), + ), + ( + "task_type", + models.CharField( + blank=True, + choices=[ + ("TEXT_GENERATION", "Text Generation"), + ("TEXT_CLASSIFICATION", "Text Classification"), + ("QUESTION_ANSWERING", "Question Answering"), + ("SUMMARIZATION", "Summarization"), + ("TRANSLATION", "Translation"), + ("SENTIMENT_ANALYSIS", "Sentiment Analysis"), + ("NAMED_ENTITY_RECOGNITION", "Named Entity Recognition"), + ("CODE_GENERATION", "Code Generation"), + ("CONVERSATION", "Conversation"), + ("INSTRUCTION_FOLLOWING", "Instruction Following"), + ("REASONING", "Reasoning"), + ("CREATIVE_WRITING", "Creative Writing"), + ("DATA_EXTRACTION", "Data Extraction"), + ("OTHER", "Other"), + ], + max_length=100, + null=True, + ), + ), + ( + "target_languages", + models.JSONField( + blank=True, + help_text="List of target languages for the prompts (e.g., ['en', 'hi', 'ta'])", + null=True, + ), + ), + ( + "domain", + models.CharField( + blank=True, + help_text="Domain or category (e.g., healthcare, education, legal)", + max_length=200, + null=True, + ), + ), + ( + "target_model_types", + models.JSONField( + blank=True, + help_text="List of AI model types these prompts are designed for", + null=True, + ), + ), + ( + "prompt_format", + models.CharField( + blank=True, + help_text="Format of prompts (e.g., instruction, chat, completion)", + max_length=100, + null=True, + ), + ), + ( + "has_system_prompt", + models.BooleanField( + default=False, + help_text="Whether the prompts include system-level instructions", + ), + ), + ( + "has_example_responses", + models.BooleanField( + default=False, + help_text="Whether the prompts include example/expected responses", + ), + ), + ( + "avg_prompt_length", + models.IntegerField( + blank=True, + help_text="Average character length of prompts in this dataset", + null=True, + ), + ), + ( + "prompt_count", + models.IntegerField( + blank=True, help_text="Total number of prompts in this dataset", null=True + ), + ), + ( + "use_case", + models.TextField( + blank=True, + help_text="Description of intended use cases for these prompts", + null=True, + ), + ), + ( + "evaluation_criteria", + models.JSONField( + blank=True, + help_text="Criteria or metrics for evaluating prompt effectiveness", + null=True, + ), + ), + ], + options={ + "verbose_name": "Prompt Dataset", + "verbose_name_plural": "Prompt Datasets", + "db_table": "prompt_dataset", + }, + bases=("api.dataset",), + ), + migrations.AddField( + model_name="dataset", + name="dataset_type", + field=models.CharField( + choices=[("DATA", "Data"), ("PROMPT", "Prompt")], default="DATA", max_length=50 + ), + ), + ] diff --git a/api/migrations/0041_alter_promptdataset_domain_and_more.py b/api/migrations/0041_alter_promptdataset_domain_and_more.py new file mode 100755 index 0000000..a6b1c97 --- /dev/null +++ b/api/migrations/0041_alter_promptdataset_domain_and_more.py @@ -0,0 +1,67 @@ +# Generated by Django 5.0.4 on 2026-01-12 14:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0040_promptdataset_dataset_dataset_type"), + ] + + operations = [ + migrations.AlterField( + model_name="promptdataset", + name="domain", + field=models.CharField( + blank=True, + choices=[ + ("HEALTHCARE", "Healthcare"), + ("EDUCATION", "Education"), + ("LEGAL", "Legal"), + ("FINANCE", "Finance"), + ("AGRICULTURE", "Agriculture"), + ("ENVIRONMENT", "Environment"), + ("GOVERNMENT", "Government"), + ("TECHNOLOGY", "Technology"), + ("SCIENCE", "Science"), + ("SOCIAL_SERVICES", "Social Services"), + ("TRANSPORTATION", "Transportation"), + ("ENERGY", "Energy"), + ("GENERAL", "General"), + ("OTHER", "Other"), + ], + help_text="Domain or category (e.g., healthcare, education, legal)", + max_length=200, + null=True, + ), + ), + migrations.AlterField( + model_name="promptdataset", + name="prompt_format", + field=models.CharField( + blank=True, + choices=[ + ("INSTRUCTION", "Instruction"), + ("CHAT", "Chat"), + ("COMPLETION", "Completion"), + ("FEW_SHOT", "Few Shot"), + ("CHAIN_OF_THOUGHT", "Chain Of Thought"), + ("ZERO_SHOT", "Zero Shot"), + ("OTHER", "Other"), + ], + help_text="Format of prompts (e.g., instruction, chat, completion)", + max_length=100, + null=True, + ), + ), + migrations.AlterField( + model_name="promptdataset", + name="target_model_types", + field=models.JSONField( + blank=True, + help_text="List of AI model types these prompts are designed for (e.g., ['GPT', 'LLAMA'])", + null=True, + ), + ), + ] diff --git a/api/migrations/0042_promptresource_and_more.py b/api/migrations/0042_promptresource_and_more.py new file mode 100755 index 0000000..722c058 --- /dev/null +++ b/api/migrations/0042_promptresource_and_more.py @@ -0,0 +1,106 @@ +# Generated by Django 5.0.4 on 2026-01-13 07:31 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0041_alter_promptdataset_domain_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="PromptResource", + fields=[ + ( + "resource", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="prompt_details", + serialize=False, + to="api.resource", + ), + ), + ( + "prompt_format", + models.CharField( + blank=True, + choices=[ + ("INSTRUCTION", "Instruction"), + ("CHAT", "Chat"), + ("COMPLETION", "Completion"), + ("FEW_SHOT", "Few Shot"), + ("CHAIN_OF_THOUGHT", "Chain Of Thought"), + ("ZERO_SHOT", "Zero Shot"), + ("OTHER", "Other"), + ], + help_text="Format of prompts in this file (e.g., instruction, chat, completion)", + max_length=100, + null=True, + ), + ), + ( + "has_system_prompt", + models.BooleanField( + default=False, + help_text="Whether the prompts in this file include system-level instructions", + ), + ), + ( + "has_example_responses", + models.BooleanField( + default=False, + help_text="Whether the prompts in this file include example/expected responses", + ), + ), + ( + "avg_prompt_length", + models.IntegerField( + blank=True, + help_text="Average character length of prompts in this file", + null=True, + ), + ), + ( + "prompt_count", + models.IntegerField( + blank=True, help_text="Total number of prompts in this file", null=True + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ], + options={ + "verbose_name": "Prompt Resource", + "verbose_name_plural": "Prompt Resources", + "db_table": "prompt_resource", + }, + ), + migrations.RemoveField( + model_name="promptdataset", + name="avg_prompt_length", + ), + migrations.RemoveField( + model_name="promptdataset", + name="has_example_responses", + ), + migrations.RemoveField( + model_name="promptdataset", + name="has_system_prompt", + ), + migrations.RemoveField( + model_name="promptdataset", + name="prompt_count", + ), + migrations.RemoveField( + model_name="promptdataset", + name="prompt_format", + ), + migrations.RemoveField( + model_name="promptdataset", + name="use_case", + ), + ] diff --git a/api/migrations/0043_promptdataset_purpose.py b/api/migrations/0043_promptdataset_purpose.py new file mode 100755 index 0000000..f137ae8 --- /dev/null +++ b/api/migrations/0043_promptdataset_purpose.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.4 on 2026-01-14 12:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0042_promptresource_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="promptdataset", + name="purpose", + field=models.CharField( + blank=True, + choices=[ + ("RESEARCH", "Research"), + ("EDUCATION", "Education"), + ("EVALUATION", "Evaluation"), + ("OTHER", "Other"), + ], + help_text="Purpose of the prompt dataset", + max_length=200, + null=True, + ), + ), + ] diff --git a/api/migrations/0044_alter_aimodel_input_schema_alter_aimodel_metadata_and_more.py b/api/migrations/0044_alter_aimodel_input_schema_alter_aimodel_metadata_and_more.py new file mode 100755 index 0000000..00e0e8c --- /dev/null +++ b/api/migrations/0044_alter_aimodel_input_schema_alter_aimodel_metadata_and_more.py @@ -0,0 +1,50 @@ +# Generated by Django 5.0.4 on 2026-02-06 13:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0043_promptdataset_purpose"), + ] + + operations = [ + migrations.AlterField( + model_name="aimodel", + name="input_schema", + field=models.JSONField( + blank=True, + default=dict, + help_text="Expected input format and parameters", + null=True, + ), + ), + migrations.AlterField( + model_name="aimodel", + name="metadata", + field=models.JSONField( + blank=True, + default=dict, + help_text="Additional metadata (training data info, limitations, etc.)", + null=True, + ), + ), + migrations.AlterField( + model_name="aimodel", + name="output_schema", + field=models.JSONField( + blank=True, default=dict, help_text="Expected output format", null=True + ), + ), + migrations.AlterField( + model_name="aimodel", + name="supported_languages", + field=models.JSONField( + blank=True, + default=list, + help_text="List of supported language codes (e.g., ['en', 'es', 'fr'])", + null=True, + ), + ), + ] diff --git a/api/migrations/0045_aimodel_domain.py b/api/migrations/0045_aimodel_domain.py new file mode 100755 index 0000000..c342fb6 --- /dev/null +++ b/api/migrations/0045_aimodel_domain.py @@ -0,0 +1,39 @@ +# Generated by Django 5.0.4 on 2026-02-10 13:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0044_alter_aimodel_input_schema_alter_aimodel_metadata_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="aimodel", + name="domain", + field=models.CharField( + blank=True, + choices=[ + ("HEALTHCARE", "Healthcare"), + ("EDUCATION", "Education"), + ("LEGAL", "Legal"), + ("FINANCE", "Finance"), + ("AGRICULTURE", "Agriculture"), + ("ENVIRONMENT", "Environment"), + ("GOVERNMENT", "Government"), + ("TECHNOLOGY", "Technology"), + ("SCIENCE", "Science"), + ("SOCIAL_SERVICES", "Social Services"), + ("TRANSPORTATION", "Transportation"), + ("ENERGY", "Energy"), + ("GENERAL", "General"), + ("OTHER", "Other"), + ], + help_text="Domain or category (e.g., healthcare, education, legal)", + max_length=200, + null=True, + ), + ), + ] diff --git a/api/migrations/0046_alter_collaborative_slug.py b/api/migrations/0046_alter_collaborative_slug.py new file mode 100755 index 0000000..cb735ce --- /dev/null +++ b/api/migrations/0046_alter_collaborative_slug.py @@ -0,0 +1,30 @@ +# Generated by Django 5.0.4 on 2026-04-30 13:31 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0045_aimodel_domain"), + ] + + operations = [ + migrations.AlterField( + model_name="collaborative", + name="slug", + field=models.SlugField( + blank=True, + max_length=75, + null=True, + unique=True, + validators=[ + django.core.validators.RegexValidator( + message="Slug must be lowercase and contain only alphanumeric characters and hyphens.", + regex="^[a-z0-9]+(?:-[a-z0-9]+)*$", + ) + ], + ), + ), + ] diff --git a/api/migrations/0047_resourcetype_publication_collaborative_publications_and_more.py b/api/migrations/0047_resourcetype_publication_collaborative_publications_and_more.py new file mode 100644 index 0000000..4b9948b --- /dev/null +++ b/api/migrations/0047_resourcetype_publication_collaborative_publications_and_more.py @@ -0,0 +1,226 @@ +# Generated by Django 5.0.4 on 2026-08-26 08:13 + +import api.utils.file_paths +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0046_alter_collaborative_slug"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ResourceType", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + ("name", models.CharField(max_length=75, unique=True)), + ("description", models.CharField(blank=True, max_length=1000, null=True)), + ("slug", models.SlugField(max_length=75, null=True, unique=True)), + ("is_active", models.BooleanField(default=True)), + ], + options={ + "verbose_name": "Resource Type", + "verbose_name_plural": "Resource Types", + "db_table": "resource_type", + "ordering": ["name"], + }, + ), + migrations.CreateModel( + name="Publication", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + ("title", models.CharField(blank=True, max_length=300)), + ("description", models.TextField(blank=True, null=True)), + ("slug", models.SlugField(max_length=255, unique=True)), + ("authors", models.JSONField(blank=True, default=list)), + ("publication_date", models.DateField(blank=True, null=True)), + ( + "license", + models.CharField( + choices=[ + ("GOVERNMENT_OPEN_DATA_LICENSE", "Government Open Data License"), + ("CC_BY_4_0_ATTRIBUTION", "Cc By 4 0 Attribution"), + ( + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "Cc By Sa 4 0 Attribution Share Alike", + ), + ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "Open Data Commons By Attribution", + ), + ("OPEN_DATABASE_LICENSE", "Open Database License"), + ], + default="CC_BY_4_0_ATTRIBUTION", + max_length=50, + ), + ), + ("external_source_link", models.URLField(blank=True, null=True)), + ( + "status", + models.CharField( + choices=[("DRAFT", "Draft"), ("PUBLISHED", "Published")], + default="DRAFT", + max_length=50, + ), + ), + ("download_count", models.IntegerField(default=0)), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "geographies", + models.ManyToManyField( + blank=True, related_name="publications", to="api.geography" + ), + ), + ( + "organization", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="publications", + to="api.organization", + ), + ), + ( + "sectors", + models.ManyToManyField( + blank=True, related_name="publications", to="api.sector" + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="publications", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "resource_type", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="publications", + to="api.resourcetype", + ), + ), + ], + options={ + "verbose_name": "Publication", + "verbose_name_plural": "Publications", + "db_table": "publication", + "ordering": ["-modified"], + }, + ), + migrations.AddField( + model_name="collaborative", + name="publications", + field=models.ManyToManyField(blank=True, to="api.publication"), + ), + migrations.AddField( + model_name="usecase", + name="publications", + field=models.ManyToManyField(blank=True, to="api.publication"), + ), + migrations.CreateModel( + name="PublicationBlock", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + ("position", models.PositiveIntegerField(default=0)), + ( + "block_type", + models.CharField( + choices=[("FILE", "File"), ("YOUTUBE", "Youtube")], max_length=20 + ), + ), + ( + "file", + models.FileField( + blank=True, + max_length=300, + upload_to=api.utils.file_paths._publication_block_directory_path, + ), + ), + ("file_name", models.CharField(blank=True, max_length=300)), + ("file_format", models.CharField(blank=True, max_length=50)), + ("file_size", models.BigIntegerField(blank=True, null=True)), + ("youtube_url", models.URLField(blank=True, null=True)), + ("youtube_video_id", models.CharField(blank=True, max_length=20)), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "publication", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="blocks", + to="api.publication", + ), + ), + ], + options={ + "db_table": "publication_block", + "ordering": ["position"], + "indexes": [ + models.Index( + fields=["publication", "position"], name="publication_publica_bbb551_idx" + ) + ], + }, + ), + migrations.AddConstraint( + model_name="publicationblock", + constraint=models.CheckConstraint( + check=models.Q( + models.Q( + ("block_type", "FILE"), + ("youtube_url__isnull", True), + models.Q(("file", ""), _negated=True), + ), + models.Q( + ("block_type", "YOUTUBE"), ("file", ""), ("youtube_url__isnull", False) + ), + _connector="OR", + ), + name="publicationblock_file_xor_youtube", + ), + ), + migrations.AddIndex( + model_name="publication", + index=models.Index( + fields=["organization", "-modified"], name="publication_organiz_c16e7b_idx" + ), + ), + migrations.AddIndex( + model_name="publication", + index=models.Index(fields=["user", "-modified"], name="publication_user_id_02e48d_idx"), + ), + migrations.AddIndex( + model_name="publication", + index=models.Index(fields=["status"], name="publication_status_2449b8_idx"), + ), + ] diff --git a/authorization/migrations/0001_initial.py b/authorization/migrations/0001_initial.py new file mode 100644 index 0000000..cda857b --- /dev/null +++ b/authorization/migrations/0001_initial.py @@ -0,0 +1,261 @@ +# Generated by Django 5.0.4 on 2025-04-22 14:21 + +import django.contrib.auth.models +import django.contrib.auth.validators +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("api", "0023_alter_usecase_summary"), + ("auth", "0012_alter_user_first_name_max_length"), + ] + + operations = [ + migrations.CreateModel( + name="Role", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=50, unique=True)), + ("description", models.TextField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("can_view", models.BooleanField(default=True)), + ("can_add", models.BooleanField(default=False)), + ("can_change", models.BooleanField(default=False)), + ("can_delete", models.BooleanField(default=False)), + ], + options={ + "db_table": "role", + }, + ), + migrations.CreateModel( + name="User", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("password", models.CharField(max_length=128, verbose_name="password")), + ( + "last_login", + models.DateTimeField( + blank=True, null=True, verbose_name="last login" + ), + ), + ( + "is_superuser", + models.BooleanField( + default=False, + help_text="Designates that this user has all permissions without explicitly assigning them.", + verbose_name="superuser status", + ), + ), + ( + "username", + models.CharField( + error_messages={ + "unique": "A user with that username already exists." + }, + help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.", + max_length=150, + unique=True, + validators=[ + django.contrib.auth.validators.UnicodeUsernameValidator() + ], + verbose_name="username", + ), + ), + ( + "first_name", + models.CharField( + blank=True, max_length=150, verbose_name="first name" + ), + ), + ( + "last_name", + models.CharField( + blank=True, max_length=150, verbose_name="last name" + ), + ), + ( + "email", + models.EmailField( + blank=True, max_length=254, verbose_name="email address" + ), + ), + ( + "is_staff", + models.BooleanField( + default=False, + help_text="Designates whether the user can log into this admin site.", + verbose_name="staff status", + ), + ), + ( + "is_active", + models.BooleanField( + default=True, + help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.", + verbose_name="active", + ), + ), + ( + "date_joined", + models.DateTimeField( + default=django.utils.timezone.now, verbose_name="date joined" + ), + ), + ( + "keycloak_id", + models.CharField( + blank=True, max_length=255, null=True, unique=True + ), + ), + ("bio", models.TextField(blank=True, null=True)), + ( + "profile_picture", + models.ImageField( + blank=True, null=True, upload_to="profile_pictures/" + ), + ), + ( + "groups", + models.ManyToManyField( + blank=True, + help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.", + related_name="user_set", + related_query_name="user", + to="auth.group", + verbose_name="groups", + ), + ), + ( + "user_permissions", + models.ManyToManyField( + blank=True, + help_text="Specific permissions for this user.", + related_name="user_set", + related_query_name="user", + to="auth.permission", + verbose_name="user permissions", + ), + ), + ], + options={ + "db_table": "user", + }, + managers=[ + ("objects", django.contrib.auth.models.UserManager()), + ], + ), + migrations.CreateModel( + name="OrganizationMembership", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "organization", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="api.organization", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "role", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="authorization.role", + ), + ), + ], + options={ + "db_table": "organization_membership", + "unique_together": {("user", "organization")}, + }, + ), + migrations.AddField( + model_name="user", + name="organizations", + field=models.ManyToManyField( + related_name="members", + through="authorization.OrganizationMembership", + to="api.organization", + ), + ), + migrations.CreateModel( + name="DatasetPermission", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "dataset", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.dataset" + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "role", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="authorization.role", + ), + ), + ], + options={ + "db_table": "dataset_permission", + "unique_together": {("user", "dataset")}, + }, + ), + ] diff --git a/authorization/migrations/0002_alter_user_table.py b/authorization/migrations/0002_alter_user_table.py new file mode 100644 index 0000000..29987fa --- /dev/null +++ b/authorization/migrations/0002_alter_user_table.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.4 on 2025-04-28 08:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("authorization", "0001_initial"), + ] + + operations = [ + migrations.AlterModelTable( + name="user", + table="ds_user", + ), + ] diff --git a/authorization/migrations/0003_user_github_profile_user_linkedin_profile_and_more.py b/authorization/migrations/0003_user_github_profile_user_linkedin_profile_and_more.py new file mode 100644 index 0000000..2986fc7 --- /dev/null +++ b/authorization/migrations/0003_user_github_profile_user_linkedin_profile_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 5.0.4 on 2025-05-09 14:11 + +import api.utils.file_paths +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("authorization", "0002_alter_user_table"), + ] + + operations = [ + migrations.AddField( + model_name="user", + name="github_profile", + field=models.URLField(blank=True, null=True), + ), + migrations.AddField( + model_name="user", + name="linkedin_profile", + field=models.URLField(blank=True, null=True), + ), + migrations.AddField( + model_name="user", + name="location", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="user", + name="twitter_profile", + field=models.URLField(blank=True, null=True), + ), + migrations.AlterField( + model_name="user", + name="profile_picture", + field=models.ImageField( + blank=True, + null=True, + upload_to=api.utils.file_paths._user_profile_image_directory_path, + ), + ), + ] diff --git a/authorization/migrations/0004_userconsent.py b/authorization/migrations/0004_userconsent.py new file mode 100644 index 0000000..6e7a152 --- /dev/null +++ b/authorization/migrations/0004_userconsent.py @@ -0,0 +1,50 @@ +# Generated by Django 5.0.4 on 2025-05-27 11:57 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("authorization", "0003_user_github_profile_user_linkedin_profile_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="UserConsent", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("activity_tracking_enabled", models.BooleanField(default=False)), + ("consent_given_at", models.DateTimeField(blank=True, null=True)), + ("consent_updated_at", models.DateTimeField(auto_now=True)), + ( + "consent_ip_address", + models.GenericIPAddressField(blank=True, null=True), + ), + ("consent_user_agent", models.TextField(blank=True, null=True)), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="activity_consent", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "User Consent", + "verbose_name_plural": "User Consents", + "db_table": "user_consent", + }, + ), + ] diff --git a/authorization/migrations/__init__.py b/authorization/migrations/__init__.py new file mode 100644 index 0000000..e69de29 From 59d52a967f0be91b97283723652af5ce837d9b29 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 18:08:29 +0530 Subject: [PATCH 08/20] fix: point prod smoke tests at prod and skip provider-smoke there The Smoke Tests job passed the repo-level HOME_URL_DEV (the dev site) on main too, so every prod deploy's consumer and provider suites tested dev: prod's nginx logged 0 test-browser requests during a prod run's smoke window, dev's logged 8840. Prod rollbacks were being triggered by dev timeouts, and prod's UI was never actually tested. On main, pass the prod URL and set skip_provider, since provider-smoke creates content with the test account. --- .github/workflows/deploy-backend.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 300e584..0a67128 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -259,8 +259,15 @@ jobs: # force_smoke_failure test path. deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }} min_passed: ${{ inputs.force_smoke_failure && 999 || 1 }} + # provider-smoke creates datasets/use cases with the test account, so + # prod deploys run only the read-only api and consumer suites. + skip_provider: ${{ github.ref_name == 'main' }} secrets: - HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + # This job can't see environment-scoped secrets, and the repo-level + # HOME_URL_DEV is the dev site -- passing it on main smoke-tested dev + # while gating a prod deploy. Literal, not a secret, so a missing value + # can't silently fall back to dev through the `&& ||` idiom. + HOME_URL_DEV: ${{ github.ref_name == 'main' && 'https://civicdataspace.in' || secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} From 32d45d32f8aa103e57e6853ef68be8a8a98a6cd8 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 23:30:22 +0530 Subject: [PATCH 09/20] feat: two-stage dev smoke gate, readonly-only prod tests dev: smoke first; functional runs only if smoke passes; either failing rolls back, both passing finalizes. main: only the readonly allowlist runs against prod (provider never), replacing skip_provider. Finalize uses !failure() because functional-tests is skipped on main by design. --- .github/workflows/deploy-backend.yml | 32 ++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 0a67128..fe5d01e 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -259,9 +259,9 @@ jobs: # force_smoke_failure test path. deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }} min_passed: ${{ inputs.force_smoke_failure && 999 || 1 }} - # provider-smoke creates datasets/use cases with the test account, so - # prod deploys run only the read-only api and consumer suites. - skip_provider: ${{ github.ref_name == 'main' }} + # main: only the readonly allowlist ever runs against prod. + # dev: stage 1 of 2 -- functional-tests runs only if this passes. + suite: ${{ github.ref_name == 'main' && 'readonly' || 'smoke' }} secrets: # This job can't see environment-scoped secrets, and the repo-level # HOME_URL_DEV is the dev site -- passing it on main smoke-tested dev @@ -277,10 +277,28 @@ jobs: # and the job fails its preflight. KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + functional-tests: + name: Functional Tests + needs: smoke-tests + # dev only, stage 2: runs only once smoke has passed. Functional tests + # write (datasets, use cases), so they never run against prod. + if: github.ref_name != 'main' + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: functional + api_base_url: ${{ vars.DEV_API_BASE_URL }} + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + rollback-on-smoke-failure: name: Rollback (smoke tests failed) # `deploy` must be in needs: for needs.deploy.result to resolve here. - needs: [deploy, smoke-tests] + needs: [deploy, smoke-tests, functional-tests] # failure()/success() builtins rather than needs.smoke-tests.result -- # both are false on cancellation, which is the behaviour we want; # if: always() would ignore cancellation entirely. @@ -331,8 +349,10 @@ jobs: finalize-deploy: name: Finalize Deploy - needs: [deploy, smoke-tests] - if: success() + needs: [deploy, smoke-tests, functional-tests] + # Not success(): on main functional-tests is skipped by design, and a + # skipped need would skip this job too. Nothing failed + smoke passed. + if: ${{ !failure() && !cancelled() && needs.smoke-tests.result == 'success' }} runs-on: ubuntu-latest environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} timeout-minutes: 10 From 067a53c440670f7ff9e7b8f65a65547108c2f06f Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 23:30:22 +0530 Subject: [PATCH 10/20] feat: full-suite PR gate against dev for PRs into main Runs the full suite (smoke, functional, regression) against dev on every PR into main. deployed_sha is the PR head, so the gate fails unless dev is running exactly the code being merged. --- .github/workflows/pr-gate.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/pr-gate.yml diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml new file mode 100644 index 0000000..0b5b065 --- /dev/null +++ b/.github/workflows/pr-gate.yml @@ -0,0 +1,29 @@ +# Full test suite against dev for every PR into main. Required by branch +# protection on main, so a PR merges (and prod deploys) only once it's green. +# dev already runs the PR's code (it was merged to dev first); deployed_sha +# makes the suite fail if dev is running anything else. +name: PR Gate + +on: + pull_request: + branches: [main] + +concurrency: + group: pr-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + full-suite: + name: Full Suite (dev) + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: full + api_base_url: ${{ vars.DEV_API_BASE_URL }} + deployed_sha: ${{ github.event.pull_request.head.sha }} + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} From 3f78dc8e6be88b1820456749f41b17ce7111a733 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 11:35:12 +0530 Subject: [PATCH 11/20] feat: run provider functional report-only while it is flaky against dev Provider functional tests fail at shifting points against dev (autosave races, slow create flows), so gating on them would roll back dev deploys spuriously. The gating functional stage and the PR gate now skip provider; separate provider jobs run with only_provider and report, without being a dependency of rollback/promote or a required check. --- .github/workflows/deploy-backend.yml | 21 +++++++++++++++++++++ .github/workflows/pr-gate.yml | 16 ++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index fe5d01e..74a10ae 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -287,6 +287,9 @@ jobs: with: suite: functional api_base_url: ${{ vars.DEV_API_BASE_URL }} + # Provider functional is flaky against dev, so it runs report-only in + # provider-functional-report instead of gating the deploy. + skip_provider: true secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} @@ -295,6 +298,24 @@ jobs: TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + provider-functional-report: + name: Provider Functional (report-only) + needs: smoke-tests + # Report-only: rollback and finalize do not list this job in needs, so + # its result never rolls back or blocks a dev deploy. Promote it to a + # gate once the provider suite is stable against dev. + if: github.ref_name != 'main' + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: functional + only_provider: true + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + rollback-on-smoke-failure: name: Rollback (smoke tests failed) # `deploy` must be in needs: for needs.deploy.result to resolve here. diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index 0b5b065..dd8e6e9 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -18,6 +18,9 @@ jobs: uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI with: suite: full + # Provider is flaky against dev: it reports in provider-report below, which + # branch protection does not require. + skip_provider: true api_base_url: ${{ vars.DEV_API_BASE_URL }} deployed_sha: ${{ github.event.pull_request.head.sha }} secrets: @@ -27,3 +30,16 @@ jobs: TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + + provider-report: + name: Provider Full Suite (report-only) + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: full + only_provider: true + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} From ac01ce44a54e7f9e5d8103b7ebd998d7d1a10b16 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 12:25:54 +0530 Subject: [PATCH 12/20] fix: wait for every test before finalizing or rolling back Finalize and rollback did not list the report-only provider job in needs, so a deploy asked for finalize approval while provider tests were still running. Both now wait for it, and condition on the gating jobs' results explicitly instead of failure()/!failure(), which would count the report-only job's result. --- .github/workflows/deploy-backend.yml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 74a10ae..72006f2 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -319,11 +319,14 @@ jobs: rollback-on-smoke-failure: name: Rollback (smoke tests failed) # `deploy` must be in needs: for needs.deploy.result to resolve here. - needs: [deploy, smoke-tests, functional-tests] - # failure()/success() builtins rather than needs.smoke-tests.result -- - # both are false on cancellation, which is the behaviour we want; - # if: always() would ignore cancellation entirely. - if: failure() && needs.deploy.result == 'success' + # provider-functional-report is listed so this waits for every test to + # finish, but its result is deliberately not checked (report-only). + needs: [deploy, smoke-tests, functional-tests, provider-functional-report] + # Explicit gating results, not failure(): failure() would also count the + # report-only provider job. !cancelled() keeps cancellation a no-op. + if: >- + ${{ !cancelled() && needs.deploy.result == 'success' && + (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure') }} runs-on: ubuntu-latest environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} # 50m: must comfortably exceed the Restore previous image step's own @@ -370,10 +373,15 @@ jobs: finalize-deploy: name: Finalize Deploy - needs: [deploy, smoke-tests, functional-tests] - # Not success(): on main functional-tests is skipped by design, and a - # skipped need would skip this job too. Nothing failed + smoke passed. - if: ${{ !failure() && !cancelled() && needs.smoke-tests.result == 'success' }} + # provider-functional-report is listed so this waits for every test to + # finish, but its result is deliberately not checked (report-only). + needs: [deploy, smoke-tests, functional-tests, provider-functional-report] + # Explicit gating results, not success()/!failure(): on main the functional + # jobs are skipped by design, and the provider job must not block. + if: >- + ${{ !cancelled() && needs.deploy.result == 'success' && + needs.smoke-tests.result == 'success' && + (needs.functional-tests.result == 'success' || needs.functional-tests.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} timeout-minutes: 10 From 53d925be5978fca057a710223086ac85c8f0bba9 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 15:18:45 +0530 Subject: [PATCH 13/20] fix: pass KEYCLOAK_CLIENT_SECRET to the report-only provider job org_add_permission needs a Keycloak token to check canAdd. A called workflow only sees secrets its caller passes, and the report-only job passed none, so every org-create test skipped on both workers -- the provider report looked green while never exercising org flows. --- .github/workflows/deploy-backend.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 72006f2..52540c4 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -309,12 +309,16 @@ jobs: with: suite: functional only_provider: true + api_base_url: ${{ vars.DEV_API_BASE_URL }} secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # org_add_permission needs a token to check canAdd; without this every + # org-create test skips on both workers and the report is hollow. + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} rollback-on-smoke-failure: name: Rollback (smoke tests failed) From 788b37ddd6d4f799430a726b8e8d2ce100de0342 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 15:18:45 +0530 Subject: [PATCH 14/20] fix: least-privilege permissions on pr-gate; pass Keycloak secret to provider-report Mirrors main: contents/pull-requests read (the CodeQL finding), and KEYCLOAK_CLIENT_SECRET for provider-report's org-create tests. --- .github/workflows/pr-gate.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index dd8e6e9..332189c 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -8,6 +8,10 @@ on: pull_request: branches: [main] +permissions: + contents: read + pull-requests: read + concurrency: group: pr-gate-${{ github.event.pull_request.number }} cancel-in-progress: true @@ -37,9 +41,13 @@ jobs: with: suite: full only_provider: true + api_base_url: ${{ vars.DEV_API_BASE_URL }} secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # org_add_permission needs a token to check canAdd; without this every + # org-create test skips on both workers and the report is hollow. + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} From eaa5806368b08be7215a83a784612048926e1071 Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 16 Sep 2026 11:52:20 +0530 Subject: [PATCH 15/20] fix: require authentication on the user-info endpoint UserInfoView declared no permission_classes, so the project-wide AllowAny default let anonymous requests reach the handler, where reading .email off AnonymousUser raised AttributeError and returned 500 instead of 401. --- api/views/auth.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/views/auth.py b/api/views/auth.py index 98b1029..ff72a86 100644 --- a/api/views/auth.py +++ b/api/views/auth.py @@ -1,5 +1,5 @@ from rest_framework import status, views -from rest_framework.permissions import AllowAny +from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response from rest_framework_simplejwt.tokens import RefreshToken @@ -83,6 +83,11 @@ class UserInfoView(views.APIView): View for getting the current user's information. """ + # Without this the project-wide AllowAny default lets an anonymous request + # through, and reading .email off AnonymousUser raises a 500 instead of + # returning 401. + permission_classes = [IsAuthenticated] + def get(self, request: Request) -> Response: user = request.user return Response( From cd2c2e11b563b8188086d7074e815cba5a5a267a Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 16 Sep 2026 11:52:20 +0530 Subject: [PATCH 16/20] fix: convert search hits to plain data before caching them cache.set pickles the value. Serialized hits keep AttrList/AttrDict/InnerDoc values for nested fields, and those classes are rebuilt per document type, so pickling raised "it's not the same object as elasticsearch_dsl.document .InnerDoc" and the except clause turned it into a 500 on /api/search/aimodel/. --- api/views/paginated_elastic_view.py | 32 ++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/api/views/paginated_elastic_view.py b/api/views/paginated_elastic_view.py index 7f892bb..c8b77a8 100644 --- a/api/views/paginated_elastic_view.py +++ b/api/views/paginated_elastic_view.py @@ -4,6 +4,7 @@ from django.core.cache import cache from django.http import HttpRequest, HttpResponse from elasticsearch_dsl import Search +from elasticsearch_dsl.utils import AttrDict, AttrList from rest_framework.permissions import AllowAny from rest_framework.response import Response from rest_framework.serializers import Serializer @@ -11,6 +12,25 @@ from api.signals.dataset_signals import SEARCH_CACHE_VERSION_KEY +def as_plain_data(value: Any) -> Any: + """Convert Elasticsearch wrapper objects into plain Python containers. + + Serialized hits keep AttrList/AttrDict/InnerDoc values for nested fields. + Those classes are rebuilt per document type, so pickling one raises + "it's not the same object as elasticsearch_dsl.document.InnerDoc" and the + cache write fails, turning the whole response into a 500. + """ + if isinstance(value, AttrList): + return [as_plain_data(item) for item in value] + if isinstance(value, AttrDict): + return as_plain_data(value.to_dict()) + if isinstance(value, dict): + return {key: as_plain_data(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [as_plain_data(item) for item in value] + return value + + T = TypeVar("T") SearchType = TypeVar("SearchType", bound=Search) SerializerType = TypeVar("SerializerType", bound=Serializer) @@ -173,11 +193,13 @@ def get(self, request: HttpRequest) -> Response: for agg in is_individual_usecase_agg: aggregations["is_individual_usecase"][agg["key"]] = agg["doc_count"] - result: Dict[str, Any] = { - "results": serializer.data, - "total": response.hits.total.value, # type: ignore - "aggregations": aggregations, - } + result: Dict[str, Any] = as_plain_data( + { + "results": serializer.data, + "total": response.hits.total.value, # type: ignore + "aggregations": aggregations, + } + ) # Cache the result cache.set(cache_key, result, timeout=3600) # Cache for 1 hour From e494a29967f46867a2743131615bdd6cbcc7b4c4 Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 16 Sep 2026 11:52:20 +0530 Subject: [PATCH 17/20] test: cover the user-info 401 and the search-cache pickling --- tests/test_search_cache_and_user_info.py | 67 ++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/test_search_cache_and_user_info.py diff --git a/tests/test_search_cache_and_user_info.py b/tests/test_search_cache_and_user_info.py new file mode 100644 index 0000000..e689ebb --- /dev/null +++ b/tests/test_search_cache_and_user_info.py @@ -0,0 +1,67 @@ +"""Regression tests for the two 500s on /api/auth/user/info/ and /api/search/aimodel/.""" + +import pickle + +import pytest +from elasticsearch_dsl import InnerDoc +from elasticsearch_dsl.utils import AttrDict, AttrList +from rest_framework.test import APIRequestFactory + +from api.views.auth import UserInfoView +from api.views.paginated_elastic_view import as_plain_data + + +def nested_doc(**fields: object) -> InnerDoc: + """Build a nested doc the way elasticsearch_dsl does for a nested field. + + The class is rebuilt per document type rather than being the module-level + InnerDoc, which is exactly what pickle refuses to serialize. + """ + cls = type("InnerDoc", (InnerDoc,), {}) + cls.__module__ = "elasticsearch_dsl.document" + doc = cls() + for name, value in fields.items(): + setattr(doc, name, value) + return doc + + +@pytest.mark.django_db +def test_user_info_rejects_anonymous_request() -> None: + """Anonymous callers get 401, not a 500 from reading .email off AnonymousUser. + + The view is called directly: routing it through the test client hides the + bug, because the test settings drop the Keycloak middleware. + """ + request = APIRequestFactory().get("/api/auth/user/info/") + response = UserInfoView.as_view()(request) + assert response.status_code == 401 + + +def test_as_plain_data_makes_search_results_picklable() -> None: + """Nested hits must survive cache.set, which pickles the cached value.""" + result = { + "results": [ + { + "all_providers": AttrList([nested_doc(provider="GPT")]), + "name": AttrDict({"raw": "x"}), + } + ], + "total": 1, + } + + plain = as_plain_data(result) + assert pickle.loads(pickle.dumps(plain)) == plain + assert plain["results"][0]["all_providers"] == [{"provider": "GPT"}] + assert plain["results"][0]["name"] == {"raw": "x"} + assert not _holds_elastic_objects(plain) + + +def _holds_elastic_objects(value: object) -> bool: + """The cache pickles what it is given, so no wrapper may survive anywhere.""" + if isinstance(value, (AttrDict, AttrList, InnerDoc)): + return True + if isinstance(value, dict): + return any(_holds_elastic_objects(item) for item in value.values()) + if isinstance(value, (list, tuple)): + return any(_holds_elastic_objects(item) for item in value) + return False From 3bca7a26dd2a1b56cd3098d7c7ef902ced67aba9 Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 16 Sep 2026 12:13:24 +0530 Subject: [PATCH 18/20] fix(tests): repair the syntax error that blocked test collection options was passed twice and JS true was used instead of True, in five places, so this file never ran - and it took the whole suite down with it. --- .../object_types/charts/test_grouped_bar_chart.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/object_types/charts/test_grouped_bar_chart.py b/tests/object_types/charts/test_grouped_bar_chart.py index 87cdaa7..555bf5c 100644 --- a/tests/object_types/charts/test_grouped_bar_chart.py +++ b/tests/object_types/charts/test_grouped_bar_chart.py @@ -65,8 +65,7 @@ def test_vertical_grouped_bar_chart(self): } chart_details = MockResourceChartDetails( chart_type="BAR", - options={**options, "allow_multi_series": true}, - options=options, + options={**options, "allow_multi_series": True}, ) chart = UnifiedChart(chart_details, self.test_data) result = chart.create_chart() @@ -104,8 +103,7 @@ def test_styling_options(self): } chart_details = MockResourceChartDetails( chart_type="BAR", - options={**options, "allow_multi_series": true}, - options=options, + options={**options, "allow_multi_series": True}, ) chart = UnifiedChart(chart_details, self.test_data) result = chart.create_chart() @@ -136,8 +134,7 @@ def test_value_aggregation(self): } chart_details = MockResourceChartDetails( chart_type="BAR", - options={**options, "allow_multi_series": true}, - options=options, + options={**options, "allow_multi_series": True}, ) chart = UnifiedChart(chart_details, test_data) result = chart.create_chart() @@ -156,8 +153,7 @@ def test_time_based_grouped_bar_chart(self): } chart_details = MockResourceChartDetails( chart_type="BAR", - options={**options, "allow_multi_series": true}, - options=options, + options={**options, "allow_multi_series": True}, ) chart = UnifiedChart(chart_details, self.test_data) result = chart.create_chart() @@ -190,8 +186,7 @@ def test_value_mapping(self): } chart_details = MockResourceChartDetails( chart_type="BAR", - options={**options, "allow_multi_series": true}, - options=options, + options={**options, "allow_multi_series": True}, ) chart = UnifiedChart(chart_details, test_data) result = chart.create_chart() From 4a6ed7eb7bb9d03a5b4b931af0c297ef067a765e Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 16 Sep 2026 12:13:24 +0530 Subject: [PATCH 19/20] ci: run the Django test suite on push, PR and in the regression gate The SDK workflows run a few tests with -p no:django, so views, schema and permissions had no CI coverage at all. Known-stale chart and metadata tests are excluded, tracked in #189. --- .github/workflows/django-tests.yml | 43 ++++++++++++++++++++++++++++++ .github/workflows/pr-gate.yml | 5 ++++ 2 files changed, 48 insertions(+) create mode 100644 .github/workflows/django-tests.yml diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml new file mode 100644 index 0000000..71681e2 --- /dev/null +++ b/.github/workflows/django-tests.yml @@ -0,0 +1,43 @@ +name: Django Tests + +# The SDK workflows run a handful of tests with -p no:django, so nothing ever +# exercised the Django side: views, schema, permissions. This runs that suite. +on: + push: + branches: [dev, main] + pull_request: + branches: [dev, main] + workflow_call: + +jobs: + django-tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + # Not in requirements.txt, which covers the running service only. + pip install pytest pytest-django psycopg2-binary setuptools + + - name: Prepare environment + run: | + cp .env.example .env + mkdir -p logs + + # tests/object_types/charts and the dataset metadata tests were left + # behind by the chart consolidation and fail on main: see #189. + - name: Run tests + run: | + pytest tests/ \ + --ignore=tests/object_types/charts \ + --deselect tests/schema/test_dataset_schema.py::TestAddUpdateDatasetMetadata diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index 332189c..3a804c1 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -17,6 +17,11 @@ concurrency: cancel-in-progress: true jobs: + # Unit tests for this repo's own code, alongside the browser/API suite. + django-tests: + name: Django Tests + uses: ./.github/workflows/django-tests.yml + full-suite: name: Full Suite (dev) uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI From 8a5e1071227055e946a3689aff76cdcce338f887 Mon Sep 17 00:00:00 2001 From: Saqib Date: Wed, 16 Sep 2026 14:22:22 +0530 Subject: [PATCH 20/20] Potential fix for pull request finding 'CodeQL / Workflow does not contain permissions' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/django-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index 71681e2..6bca8c2 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -9,6 +9,9 @@ on: branches: [dev, main] workflow_call: +permissions: + contents: read + jobs: django-tests: runs-on: ubuntu-latest