From 7024d368f171e4482f4cbc5bfb96d00ee7902270 Mon Sep 17 00:00:00 2001 From: Gonzalo Vidal <35148159+Gonza10V@users.noreply.github.com> Date: Wed, 22 Apr 2026 09:53:21 -0600 Subject: [PATCH] Harden config and persistence workflow baseline --- .env.example | 25 +++ .github/workflows/ci.yml | 79 +++++++ .gitignore | 6 + ADR-001.md | 16 +- ARCHITECHTURE.md | 290 +------------------------- ARCHITECTURE.md | 37 ++++ README.md | 211 +++++++------------ docker-compose.yml | 80 ++++--- flapjack_api/.env.example | 16 ++ flapjack_api/.env.prod | 26 ++- flapjack_api/flapjack_api/env.py | 35 ++++ flapjack_api/flapjack_api/health.py | 32 +++ flapjack_api/flapjack_api/settings.py | 91 ++++---- flapjack_api/flapjack_api/tests.py | 32 +++ flapjack_api/flapjack_api/urls.py | 5 +- scripts/backup_postgres.sh | 15 ++ scripts/migrate.sh | 3 + scripts/restore_postgres.sh | 18 ++ 18 files changed, 500 insertions(+), 517 deletions(-) create mode 100644 .env.example create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 ARCHITECTURE.md create mode 100644 flapjack_api/.env.example create mode 100644 flapjack_api/flapjack_api/env.py create mode 100644 flapjack_api/flapjack_api/health.py create mode 100644 flapjack_api/flapjack_api/tests.py create mode 100755 scripts/backup_postgres.sh create mode 100755 scripts/migrate.sh create mode 100755 scripts/restore_postgres.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..50d47c4 --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +# Root-level compose defaults (development only) +POSTGRES_DB=registry +POSTGRES_USER=flapjack +POSTGRES_PASSWORD=change-me +POSTGRES_PORT=5432 +REDIS_PORT=6379 +BACKEND_PORT=8000 +FRONTEND_PORT=3000 + +APP_ENV=development +DJANGO_DEBUG=1 +DJANGO_SECRET_KEY=replace-with-long-random-string +DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1 +DJANGO_CORS_ALLOW_ALL=1 +DJANGO_CORS_ALLOWED_ORIGINS=http://localhost:3000 + +DB_ENGINE=django.db.backends.postgresql +DB_NAME=registry +DB_USER=flapjack +DB_PASSWORD=change-me +DB_HOST=db +DB_PORT=5432 + +REDIS_HOST=redis +REDIS_PORT=6379 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3826e04 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: CI + +on: + push: + pull_request: + +jobs: + backend: + runs-on: ubuntu-latest + defaults: + run: + working-directory: flapjack_api + services: + postgres: + image: postgres:12 + env: + POSTGRES_DB: registry + POSTGRES_USER: flapjack + POSTGRES_PASSWORD: flapjack + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U flapjack -d registry" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 3s + --health-retries 10 + env: + APP_ENV: development + DJANGO_DEBUG: "1" + DJANGO_SECRET_KEY: ci-unsafe-secret + DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1 + DJANGO_CORS_ALLOW_ALL: "1" + DB_ENGINE: django.db.backends.postgresql + DB_NAME: registry + DB_USER: flapjack + DB_PASSWORD: flapjack + DB_HOST: localhost + DB_PORT: "5432" + REDIS_HOST: localhost + REDIS_PORT: "6379" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.7' + - name: Install dependencies + run: pip install -r requirements.txt + - name: Django checks + run: python manage.py check + - name: Run migrations + run: python manage.py migrate --noinput + - name: Run backend tests + run: python manage.py test accounts + + frontend: + runs-on: ubuntu-latest + defaults: + run: + working-directory: flapjack_frontend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '14' + cache: npm + cache-dependency-path: flapjack_frontend/package-lock.json + - name: Install dependencies + run: npm ci + - name: Build frontend + run: npm run build diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..22ea5f4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.env +.env.* +!.env.example +flapjack_api/.env.dev +flapjack_frontend/.env.dev +*.dump diff --git a/ADR-001.md b/ADR-001.md index 12b3b2e..18745c4 100644 --- a/ADR-001.md +++ b/ADR-001.md @@ -2,7 +2,7 @@ ## Status -Proposed +Accepted ## Context @@ -166,3 +166,17 @@ Revisit this ADR if: - the product begins storing large binary artifacts at scale - compliance or security requirements require stronger secret or key management - the team decides to modernize the stack after persistence hardening is complete + + +## Implementation notes (2026-04-21) + +This repository now includes an initial hardening baseline aligned with this ADR: + +- environment-driven Django and database settings with production fail-fast checks +- placeholder-only `.env.example` files +- Docker Compose persistence fixes (`/var/lib/postgresql/data`) and service health checks +- `healthz` endpoint validating DB and Redis reachability +- migration, backup, and restore helper scripts +- CI workflow for backend checks/tests and frontend build + +Further work is still required for managed cloud resource provisioning, backup scheduling ownership, restore drill automation, and object-storage integration for large uploaded artifacts. diff --git a/ARCHITECHTURE.md b/ARCHITECHTURE.md index c4ee673..3135008 100644 --- a/ARCHITECHTURE.md +++ b/ARCHITECHTURE.md @@ -1,289 +1,5 @@ -# Architecture +# Deprecated filename -## Overview +This file has moved to `ARCHITECTURE.md`. -Flapjack is a research data platform for genetic circuit and assay workflows. It combines a browser-based client, a Django/Channels backend, relational persistence, and websocket-based computation/streaming endpoints. - -This document describes: - -- the current observed architecture in the repository -- the intended system boundaries -- operational and security concerns -- the recommended target deployment model - -## System context - -At a high level, Flapjack consists of four runtime layers: - -1. **Frontend client** - - React application served on port 3000 in local development - - calls REST endpoints over HTTP - - connects to websocket endpoints for analysis, plotting, and registry upload flows - -2. **Backend application** - - Django application served as ASGI with Gunicorn + Uvicorn workers - - exposes REST endpoints for the registry and auth flows - - exposes websocket routes through Django Channels - -3. **Primary data store** - - PostgreSQL - - intended to hold the durable system of record for users, metadata, studies, assays, samples, and measurements - -4. **Ephemeral messaging/cache layer** - - Redis - - used for Channels transport and similar short-lived coordination concerns - - must not be treated as the durable source of truth - -## Current observed implementation - -### Frontend - -The frontend is a React application using: - -- React 16 -- `react-app-rewired` -- Ant Design -- Redux -- Plotly - -The local development frontend expects: - -- `REACT_APP_HTTP_API=http://localhost:8000/api/` -- `REACT_APP_WS_API=ws://localhost:8000/ws/` - -### Backend - -The backend is a Django 3 application with: - -- Django REST Framework -- JWT authentication -- Channels -- Redis channel layer -- Gunicorn + Uvicorn for ASGI serving - -Observed URL structure: - -- `/api/auth/register/` -- `/api/auth/log_in/` -- `/api/auth/refresh/` -- `/api/auth/user_info/` -- `/api/auth/log_out/` -- `/api//` - -Observed websocket routes: - -- `/ws/plot` -- `/ws/analysis` -- `/ws/registry` - -### Registry/data API surface - -The registry router currently includes endpoints for: - -- study -- assay -- sample -- dna -- media -- strain -- measurement -- signal -- supplement -- chemical -- vector -- vectorall -- user -- assays_in_study -- vector_in_assay -- strain_in_assay -- media_in_assay -- signal_in_assay - -These resources strongly suggest the core product boundary is a biological experiment and measurement registry with analysis and plotting layered on top. - -## Component boundaries - -### `flapjack_frontend` -Responsible for: - -- user-facing navigation and data interaction -- token-driven authenticated API access -- websocket-driven interactive analysis/plot behavior -- rendering of registry and visualization workflows - -It should not: - -- embed secrets -- own authoritative business rules -- become the source of truth for access control or validation - -### `flapjack_api` -Responsible for: - -- authentication and identity -- authorization and permission enforcement -- persistence rules and data model lifecycle -- REST API behavior -- websocket orchestration -- business logic around registry, analysis, and plotting - -It should not: - -- assume local-only infrastructure -- rely on hardcoded environment-specific settings -- treat Redis as durable storage - -### PostgreSQL -Responsible for: - -- durable relational storage -- transactional integrity -- authoritative system-of-record persistence - -It should not: - -- be configured by hardcoded credentials in repo files -- be used without backups and restore procedures -- depend on ad hoc container-local storage in production - -### Redis -Responsible for: - -- channel layer transport -- short-lived cache/message coordination - -It should not: - -- store authoritative long-term data -- be required to recover historical data after restart - -## Runtime flows - -## 1. Authentication flow - -1. User registers or logs in through `/api/auth/...` -2. Backend issues JWT refresh and access tokens -3. Frontend stores and sends bearer tokens to the backend -4. Refresh token is used to obtain new access tokens -5. Logout blacklists the provided refresh token - -## 2. Registry CRUD flow - -1. Frontend sends authenticated REST requests to `/api/...` -2. Django REST Framework resolves routing and permissions -3. Registry/business logic reads/writes PostgreSQL -4. Response is returned to frontend as JSON - -## 3. Websocket analysis/plot flow - -1. Frontend connects to `/ws/plot`, `/ws/analysis`, or `/ws/registry` -2. Django Channels routes websocket traffic -3. Redis is used as the channel layer backend -4. Backend returns streamed responses or computed results to the client - -## Deployment model - -## Current local model - -Current local development uses a single Docker Compose file with: - -- frontend container -- backend container -- postgres container -- redis container - -This is adequate for local development only. - -## Target production model - -Production should use a split-responsibility model: - -- frontend served as static assets or containerized web app behind TLS -- backend deployed as an ASGI app service -- managed PostgreSQL for durable storage -- managed Redis only if still required for Channels -- object storage for large uploaded files or exports -- secrets injected at runtime from environment or secret manager -- automated backups and restore runbooks - -## Configuration boundaries - -The repository must move toward explicit environment separation: - -- `development` -- `staging` -- `production` - -Recommended config categories: - -- Django secret key -- allowed hosts -- debug mode -- CORS/CSRF origins -- database connection settings -- Redis settings -- file/object storage settings -- JWT/token settings -- logging and observability settings - -No environment-specific secret should be committed to version control. - -## Security posture - -The current repository state indicates the codebase is still operating with development-oriented defaults. - -Immediate security objectives: - -- remove hardcoded secrets and credentials -- disable wildcard hosts/origins outside local development -- define production-safe auth/session/token settings -- add secure deployment defaults -- document incident recovery and restore procedures - -## Observability and operations - -The target operational baseline should include: - -- structured application logs -- clear startup failures when required env vars are missing -- health endpoints for backend, database, and Redis dependencies -- migration strategy documented and scripted -- backup and restore procedures documented and tested -- CI checks for build, lint, and smoke validation - -## Testing expectations - -Current repository structure should evolve toward: - -- backend unit/integration tests for auth and critical registry flows -- frontend tests for critical state and routing behavior -- smoke test that brings up Compose and validates API readiness -- migration test coverage for schema changes that affect persistent data - -## Architecture risks - -Current observed risks include: - -- local-development settings embedded in application code -- hardcoded secrets and database credentials -- production-unsuitable permissive Django settings -- unclear persistence guarantees in current Compose setup -- no documented backup/restore contract -- aging framework/runtime versions that will make maintenance harder over time - -## Open questions - -1. What uploaded artifacts, if any, are currently stored on local container filesystems rather than in PostgreSQL or object storage? -2. Which websocket workloads are latency-sensitive enough to keep on Channels/Redis as-is? -3. What is the expected production deployment target: single VM, container platform, or managed app services? -4. Is there existing legacy data that must be imported before the current database is retired? -5. Which API behaviors are relied on by external clients such as `pyFlapjack` and must remain backward compatible? - -## Architecture principles - -1. PostgreSQL is the durable source of truth. -2. Redis is ephemeral. -3. Code must be runnable locally in Docker. -4. Production config must be explicit and environment-driven. -5. Security and recoverability are mandatory, not optional. -6. Major architecture changes require an ADR. +Please update references to use the correctly spelled file. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..8b77821 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,37 @@ +# Architecture + +## Overview + +Flapjack is a research data platform for genetic circuit and assay workflows. The codebase contains: + +- `flapjack_api`: Django/Channels backend (system of record and business logic) +- `flapjack_frontend`: React frontend +- `docker-compose.yml`: local orchestration for backend, frontend, PostgreSQL, and Redis + +## Runtime boundaries + +- **Primary durable store**: PostgreSQL (registry/auth/domain data) +- **Ephemeral coordination**: Redis (Channels transport) +- **Client**: React (REST + websocket consumer) +- **Backend**: Django REST Framework + Channels (API + websocket orchestration) + +## Security and persistence baseline + +- No hardcoded secrets in committed runtime config. +- Environment variables drive Django secrets, DB config, hosts, and CORS. +- Production should run with managed PostgreSQL + automated encrypted backups + PITR. +- Redis is non-authoritative (losing Redis must not lose operational data). +- Local Docker Compose is development-only and not production hardening by itself. + +## Data placement model + +- **PostgreSQL**: users, auth relations, studies, assays, samples, vectors, strains, media, signals, measurements. +- **Redis**: websocket channel-layer state and short-lived coordination. +- **Object storage (target for scale)**: large uploaded/raw assay files and binary artifacts; DB stores references/metadata. + +## Operational expectations + +- `healthz` endpoint checks DB and Redis reachability. +- Migrations are the schema evolution path. +- Backup and restore must be tested and documented. +- Separate dev/staging/prod env configuration is required. diff --git a/README.md b/README.md index 479a59c..9fe612c 100644 --- a/README.md +++ b/README.md @@ -1,153 +1,82 @@ # Flapjack Fullstack -Flapjack Fullstack is the deployment and development repository for the Flapjack web application: a full-stack system for storing, querying, analyzing, and visualizing genetic circuit and assay data. +Flapjack Fullstack is the full-stack repository for the Flapjack web application. -This repository brings together: - -- `flapjack_api`: Django/Channels backend, REST API, authentication, and websocket endpoints +- `flapjack_api`: Django + DRF + Channels backend - `flapjack_frontend`: React frontend -- `docker-compose.yml`: local orchestration for the app stack - -## Purpose - -This repository exists to make Flapjack runnable as a single system during development and to provide the foundation for a production-ready deployment. - -The application currently supports: - -- user registration, login, token refresh, logout, and user info retrieval -- study, assay, sample, vector, strain, media, signal, measurement, and related registry operations -- websocket-backed analysis, plotting, and registry upload flows -- programmatic access through a documented REST and websocket API - -## Current stack - -### Backend -- Python 3.7 container -- Django 3.0.5 -- Django REST Framework -- Channels + Redis channel layer -- JWT authentication via `djangorestframework-simplejwt` -- Gunicorn + Uvicorn worker for ASGI serving - -### Frontend -- React 16 -- `react-app-rewired` -- Ant Design -- Redux -- Plotly - -### Infrastructure -- Docker Compose -- PostgreSQL 12 -- Redis - -## Repository layout - -```text -. -├── flapjack_api/ -│ ├── flapjack_api/ # Django project config, ASGI/WSGI, routing, settings -│ ├── accounts/ # Authentication endpoints -│ ├── registry/ # Core data registry API -│ ├── analysis/ # Analysis services / websocket handlers -│ ├── plot/ # Plot services / websocket handlers -│ ├── requirements.txt -│ └── Dockerfile -├── flapjack_frontend/ -│ ├── src/ -│ ├── public/ -│ ├── package.json -│ └── Dockerfile -└── docker-compose.yml -Local development -Prerequisites -Docker -Docker Compose -Frontend environment file +- `docker-compose.yml`: local development orchestration + +## Quick architecture + +- **Backend framework**: Django 3.0.5 + Django REST Framework + Channels +- **Frontend framework**: React 16 with `react-app-rewired` +- **Primary data store**: PostgreSQL (durable) +- **Ephemeral dependency**: Redis (channel layer) +- **Auth model**: JWT access/refresh tokens via `djangorestframework-simplejwt` + +See `ARCHITECTURE.md` and `ADR-001.md` for production persistence direction. + +## Local development + +### 1) Prepare environment files + +Copy placeholders and adjust locally: + +```bash +cp .env.example .env +cp flapjack_api/.env.example flapjack_api/.env.dev +cp flapjack_frontend/.env.example flapjack_frontend/.env.dev ``` -Create flapjack_frontend/.env.dev with: +### 2) Start services -REACT_APP_HTTP_API=http://localhost:8000/api/ -REACT_APP_WS_API=ws://localhost:8000/ws/ -Start the stack +```bash docker compose up --build -Run database migrations +``` -Open a shell in the API container: +### 3) Run migrations +```bash docker exec -it flap_api bash +./scripts/migrate.sh +``` -Then run: - -python manage.py migrate -Access the app -Frontend: http://localhost:3000 -Backend API: http://localhost:8000 -Auth endpoints: http://localhost:8000/api/auth/ -Registry endpoints: http://localhost:8000/api/ -Websocket base paths: /ws/plot, /ws/analysis, /ws/registry -Common development tasks -Create new migrations -docker exec -it flap_api bash -python manage.py makemigrations -python manage.py migrate -Rebuild the stack -docker compose down -docker compose up --build -View logs -docker compose logs -f -Current state and important warnings - -This repository is valuable but not yet production-hardened. - -Current code and configuration indicate several issues that must be addressed before using this stack as a trusted system of record: - -secrets and database credentials are currently hardcoded in the repository -Django is configured with permissive development defaults -the current Compose setup should not be assumed to provide production-grade persistence -backup, restore, and disaster recovery workflows are not yet documented in this repository - -Treat the current Docker Compose setup as a local development environment, not as a secure production deployment. - -Production direction - -The intended production target for Flapjack is: - -managed PostgreSQL for durable primary storage -environment-based secrets only -encrypted backups and restore documentation -Redis for ephemeral channel-layer and cache concerns only -object storage for large uploaded data if file volume grows beyond database suitability -separate development, staging, and production configuration - -See ARCHITECTURE.md and ADR-001.md for the recommended target state. - -API and related projects -API docs: https://flapjacksynbio.github.io/flapjack_api -Python client: https://github.com/flapjacksynbio/pyFlapjack -How to work in this repository -For maintainers -keep changes small and reversible -keep Docker Compose runnable -document architecture changes in this repo -avoid introducing new secrets into version control -update docs when runtime behavior changes -For ChatGPT and Codex - -Start with these files before making major changes: - -README.md -ARCHITECTURE.md -PRODUCT.md -AGENT.md -ADR-001.md - -Use them as the working contract for the repository. If code and docs diverge, patch the docs or code explicitly rather than silently working around inconsistencies. - -Next priorities -remove hardcoded secrets and database credentials -fix persistence and deployment assumptions -document backup and restore -add CI and test coverage for critical paths -separate local development settings from production settings +### 4) Access services + +- Frontend: `http://localhost:3000` +- Backend API root: `http://localhost:8000/api/` +- Auth endpoints: `http://localhost:8000/api/auth/` +- Health endpoint: `http://localhost:8000/healthz/` + +## Durability and production guidance + +The local Compose stack is for development only. + +Production baseline: + +- managed PostgreSQL for primary relational persistence +- encrypted automated backups + point-in-time recovery +- TLS in transit +- environment-based secret injection only +- separate dev/staging/prod config +- restore drills and migration workflows + +## Backup and restore scripts + +From inside the backend container (or any host with `pg_dump`/`pg_restore` and env vars set): + +```bash +./scripts/backup_postgres.sh /tmp/flapjack_backup.dump +./scripts/restore_postgres.sh /tmp/flapjack_backup.dump +``` + +Required env vars: + +- `DB_HOST`, `DB_PORT`, `DB_NAME`, `DB_USER`, `DB_PASSWORD` + +## Important security notes + +- Do **not** commit real secrets. +- Do **not** run production with `DJANGO_DEBUG=1`. +- In production, set explicit `DJANGO_ALLOWED_HOSTS` and CORS origins. +- Redis is not an authoritative store. diff --git a/docker-compose.yml b/docker-compose.yml index 96c2b18..61c6899 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,68 +1,94 @@ -version: "3.3" +version: "3.8" services: web: container_name: flapjack_front_together build: - context: ./flapjack_frontend # Adjust this path to your frontend directory + context: ./flapjack_frontend args: app_env: development ports: - - '3000:3000' + - "${FRONTEND_PORT:-3000}:3000" environment: - NODE_ENV=development - env_file: - - ./flapjack_frontend/.env.dev # Adjust path as needed + - REACT_APP_HTTP_API=${REACT_APP_HTTP_API:-http://localhost:8000/api/} + - REACT_APP_WS_API=${REACT_APP_WS_API:-ws://localhost:8000/ws/} volumes: - - './flapjack_frontend:/usr/src/app' - - '/usr/src/app/node_modules' + - ./flapjack_frontend:/usr/src/app + - /usr/src/app/node_modules stdin_open: true depends_on: - - flapjack_api + flapjack_api: + condition: service_healthy flapjack_api: container_name: flap_api restart: always - build: - context: ./flapjack_api # Adjust this path to your backend directory + build: + context: ./flapjack_api volumes: - ./flapjack_api:/var/app/flap - expose: - - "8000" ports: - - "8000:8000" + - "${BACKEND_PORT:-8000}:8000" depends_on: - - db - - redis + db: + condition: service_healthy + redis: + condition: service_healthy environment: - PYTHONPATH=/var/app/flapjack_api + - APP_ENV=${APP_ENV:-development} + - DJANGO_DEBUG=${DJANGO_DEBUG:-1} + - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:-unsafe-dev-secret-key} + - DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS:-localhost,127.0.0.1} + - DJANGO_CORS_ALLOW_ALL=${DJANGO_CORS_ALLOW_ALL:-1} + - DJANGO_CORS_ALLOWED_ORIGINS=${DJANGO_CORS_ALLOWED_ORIGINS:-http://localhost:3000} + - DB_ENGINE=${DB_ENGINE:-django.db.backends.postgresql} + - DB_NAME=${DB_NAME:-registry} + - DB_USER=${DB_USER:-flapjack} + - DB_PASSWORD=${DB_PASSWORD:-change-me} + - DB_HOST=${DB_HOST:-db} + - DB_PORT=${DB_PORT:-5432} + - REDIS_HOST=${REDIS_HOST:-redis} + - REDIS_PORT=${REDIS_PORT:-6379} command: gunicorn flapjack_api.asgi:application -w 4 -k uvicorn.workers.UvicornWorker -b 0.0.0.0:8000 --timeout 3600 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz/')"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s db: container_name: flap_db restart: always image: postgres:12.1 ports: - - "5433:5433" + - "${POSTGRES_PORT:-5432}:5432" environment: - POSTGRES_DB: registry - POSTGRES_USER: guillermo - POSTGRES_PASSWORD: 123456 + POSTGRES_DB: ${POSTGRES_DB:-registry} + POSTGRES_USER: ${POSTGRES_USER:-flapjack} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-change-me} volumes: - - db:/var/pg/data + - db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-flapjack} -d ${POSTGRES_DB:-registry}"] + interval: 10s + timeout: 5s + retries: 5 redis: - image: redis + image: redis:7-alpine volumes: - redis:/data ports: - - 6379 + - "${REDIS_PORT:-6379}:6379" healthcheck: - test: redis-cli ping - interval: 5m - timeout: 30s - retries: 30 + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 10 volumes: redis: - db: \ No newline at end of file + db: diff --git a/flapjack_api/.env.example b/flapjack_api/.env.example new file mode 100644 index 0000000..0490423 --- /dev/null +++ b/flapjack_api/.env.example @@ -0,0 +1,16 @@ +APP_ENV=development +DJANGO_DEBUG=1 +DJANGO_SECRET_KEY=replace-with-long-random-string +DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1 +DJANGO_CORS_ALLOW_ALL=1 +DJANGO_CORS_ALLOWED_ORIGINS=http://localhost:3000 + +DB_ENGINE=django.db.backends.postgresql +DB_NAME=registry +DB_USER=flapjack +DB_PASSWORD=change-me +DB_HOST=db +DB_PORT=5432 + +REDIS_HOST=redis +REDIS_PORT=6379 diff --git a/flapjack_api/.env.prod b/flapjack_api/.env.prod index 3d4d473..193ad4a 100644 --- a/flapjack_api/.env.prod +++ b/flapjack_api/.env.prod @@ -1,9 +1,17 @@ -DEBUG=1 -SECRET_KEY=xb22!w(2y430$dk33y=jz$s@me!l9x!2i1a7mb)b=pp3b_!^2u -DJANGO_ALLOWED_HOSTS=[*] -SQL_ENGINE=django.db.backends.postgresql -SQL_DATABASE=registry -SQL_USER=guillermo -SQL_PASSWORD=123456 -SQL_HOST=db -SQL_PORT=5432 \ No newline at end of file +# Example production environment file. Do not commit real secrets. +APP_ENV=production +DJANGO_DEBUG=0 +DJANGO_SECRET_KEY=replace-me-with-secure-random-key +DJANGO_ALLOWED_HOSTS=api.example.org +DJANGO_CORS_ALLOW_ALL=0 +DJANGO_CORS_ALLOWED_ORIGINS=https://app.example.org + +DB_ENGINE=django.db.backends.postgresql +DB_NAME=registry +DB_USER=registry_user +DB_PASSWORD=replace-me +DB_HOST=managed-postgres-hostname +DB_PORT=5432 + +REDIS_HOST=managed-redis-hostname +REDIS_PORT=6379 diff --git a/flapjack_api/flapjack_api/env.py b/flapjack_api/flapjack_api/env.py new file mode 100644 index 0000000..dfb6a7a --- /dev/null +++ b/flapjack_api/flapjack_api/env.py @@ -0,0 +1,35 @@ +import json +import os + + +def env_str(name, default=None, required=False): + value = os.getenv(name, default) + if required and (value is None or str(value).strip() == ""): + raise RuntimeError(f"Missing required environment variable: {name}") + return value + + +def env_bool(name, default=False): + raw = os.getenv(name) + if raw is None: + return default + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +def env_list(name, default=None): + raw = os.getenv(name) + if raw is None: + return default or [] + + value = raw.strip() + if value == "": + return [] + + if value.startswith("["): + try: + parsed = json.loads(value) + return [str(item).strip() for item in parsed if str(item).strip()] + except json.JSONDecodeError: + pass + + return [item.strip() for item in value.split(",") if item.strip()] diff --git a/flapjack_api/flapjack_api/health.py b/flapjack_api/flapjack_api/health.py new file mode 100644 index 0000000..5c0c75a --- /dev/null +++ b/flapjack_api/flapjack_api/health.py @@ -0,0 +1,32 @@ +import redis +from django.conf import settings +from django.db import connection +from django.http import JsonResponse +from django.views.decorators.http import require_GET + + +@require_GET +def healthz(request): + checks = {} + overall_ok = True + + try: + with connection.cursor() as cursor: + cursor.execute("SELECT 1") + cursor.fetchone() + checks["database"] = "ok" + except Exception as exc: # pragma: no cover + checks["database"] = f"error: {exc}" + overall_ok = False + + redis_host, redis_port = settings.REDIS_HOST, settings.REDIS_PORT + try: + client = redis.Redis(host=redis_host, port=redis_port, socket_connect_timeout=1) + client.ping() + checks["redis"] = "ok" + except Exception as exc: # pragma: no cover + checks["redis"] = f"error: {exc}" + overall_ok = False + + status_code = 200 if overall_ok else 503 + return JsonResponse({"status": "ok" if overall_ok else "error", "checks": checks}, status=status_code) diff --git a/flapjack_api/flapjack_api/settings.py b/flapjack_api/flapjack_api/settings.py index 2f0855a..ce294ce 100755 --- a/flapjack_api/flapjack_api/settings.py +++ b/flapjack_api/flapjack_api/settings.py @@ -1,22 +1,22 @@ import os +from pathlib import Path -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +from .env import env_bool, env_list, env_str +BASE_DIR = Path(__file__).resolve().parent.parent -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ +APP_ENV = env_str("APP_ENV", "development") +IS_PRODUCTION = APP_ENV == "production" -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'xb22!w(2y430$dk33y=jz$s@me!l9x!2i1a7mb)b=pp3b_!^2u' +SECRET_KEY = env_str("DJANGO_SECRET_KEY", required=IS_PRODUCTION, default="unsafe-dev-secret-key") +DEBUG = env_bool("DJANGO_DEBUG", default=not IS_PRODUCTION) -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +if IS_PRODUCTION and DEBUG: + raise RuntimeError("DJANGO_DEBUG cannot be enabled in production") -ALLOWED_HOSTS = ["*"] - - -# Application definition +ALLOWED_HOSTS = env_list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"]) +if IS_PRODUCTION and not ALLOWED_HOSTS: + raise RuntimeError("DJANGO_ALLOWED_HOSTS cannot be empty in production") INSTALLED_APPS = [ 'django.contrib.admin', @@ -49,8 +49,6 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -CORS_ORIGIN_ALLOW_ALL = True - ROOT_URLCONF = 'flapjack_api.urls' TEMPLATES = [ @@ -71,70 +69,61 @@ WSGI_APPLICATION = 'flapjack_api.wsgi.application' +DB_ENGINE = env_str("DB_ENGINE", "django.db.backends.postgresql") +DB_NAME = env_str("DB_NAME", "registry") +DB_USER = env_str("DB_USER", "postgres") +DB_PASSWORD = env_str("DB_PASSWORD", default="", required=IS_PRODUCTION) +DB_HOST = env_str("DB_HOST", "db") +DB_PORT = env_str("DB_PORT", "5432") DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'registry', - 'USER': 'guillermo', - 'PASSWORD': '123456', - 'HOST': 'db', - 'PORT': '5432', + 'ENGINE': DB_ENGINE, + 'NAME': DB_NAME, + 'USER': DB_USER, + 'PASSWORD': DB_PASSWORD, + 'HOST': DB_HOST, + 'PORT': DB_PORT, } } - AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, + {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'}, + {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator'}, + {'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator'}, + {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'}, ] - LANGUAGE_CODE = 'en-us' - TIME_ZONE = 'UTC' - USE_I18N = True - USE_L10N = True - USE_TZ = True - -# Static files STATIC_URL = '/static/' -# rest framework config +CORS_ORIGIN_ALLOW_ALL = env_bool("DJANGO_CORS_ALLOW_ALL", default=not IS_PRODUCTION) +CORS_ALLOWED_ORIGINS = env_list("DJANGO_CORS_ALLOWED_ORIGINS", default=[]) +if IS_PRODUCTION and CORS_ORIGIN_ALLOW_ALL: + raise RuntimeError("DJANGO_CORS_ALLOW_ALL cannot be enabled in production") + REST_FRAMEWORK = { - 'DEFAULT_FILTER_BACKENDS': ( - 'rest_framework_filters.backends.RestFrameworkFilterBackend', - ), + 'DEFAULT_FILTER_BACKENDS': ('rest_framework_filters.backends.RestFrameworkFilterBackend',), 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'PAGE_SIZE': 100, - 'DEFAULT_AUTHENTICATION_CLASSES': [ - 'rest_framework_simplejwt.authentication.JWTAuthentication', - ], - 'DEFAULT_PERMISSION_CLASSES': [ - 'rest_framework.permissions.IsAuthenticated', - ] + 'DEFAULT_AUTHENTICATION_CLASSES': ['rest_framework_simplejwt.authentication.JWTAuthentication'], + 'DEFAULT_PERMISSION_CLASSES': ['rest_framework.permissions.IsAuthenticated'], } +REDIS_HOST = env_str("REDIS_HOST", "redis") +REDIS_PORT = int(env_str("REDIS_PORT", "6379")) + ASGI_APPLICATION = "flapjack_api.routing.application" CHANNEL_LAYERS = { 'default': { 'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': { - "hosts": [('redis', 6379)], + 'hosts': [(REDIS_HOST, REDIS_PORT)], }, }, } diff --git a/flapjack_api/flapjack_api/tests.py b/flapjack_api/flapjack_api/tests.py new file mode 100644 index 0000000..b971179 --- /dev/null +++ b/flapjack_api/flapjack_api/tests.py @@ -0,0 +1,32 @@ +from unittest.mock import MagicMock, patch + +from django.test import RequestFactory, SimpleTestCase + +from flapjack_api.health import healthz + + +class HealthzTests(SimpleTestCase): + def setUp(self): + self.factory = RequestFactory() + + @patch("flapjack_api.health.redis.Redis") + @patch("flapjack_api.health.connection") + def test_healthz_success(self, mock_connection, mock_redis): + mock_connection.cursor.return_value.__enter__.return_value = MagicMock() + mock_redis.return_value = MagicMock() + + response = healthz(self.factory.get("/healthz/")) + + self.assertEqual(response.status_code, 200) + self.assertIn(b'"status": "ok"', response.content) + + @patch("flapjack_api.health.redis.Redis") + @patch("flapjack_api.health.connection") + def test_healthz_failure(self, mock_connection, mock_redis): + mock_connection.cursor.side_effect = Exception("db down") + mock_redis.side_effect = Exception("redis down") + + response = healthz(self.factory.get("/healthz/")) + + self.assertEqual(response.status_code, 503) + self.assertIn(b'"status": "error"', response.content) diff --git a/flapjack_api/flapjack_api/urls.py b/flapjack_api/flapjack_api/urls.py index 14d7a16..6ac9183 100755 --- a/flapjack_api/flapjack_api/urls.py +++ b/flapjack_api/flapjack_api/urls.py @@ -1,8 +1,11 @@ from django.contrib import admin from django.urls import include, path +from .health import healthz + urlpatterns = [ path('', include('registry.urls')), path('api/auth/', include('accounts.urls'), name='accounts'), + path('healthz/', healthz, name='healthz'), path('admin/', admin.site.urls), -] \ No newline at end of file +] diff --git a/scripts/backup_postgres.sh b/scripts/backup_postgres.sh new file mode 100755 index 0000000..3ac66f6 --- /dev/null +++ b/scripts/backup_postgres.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${DB_HOST:?DB_HOST is required}" +: "${DB_PORT:?DB_PORT is required}" +: "${DB_NAME:?DB_NAME is required}" +: "${DB_USER:?DB_USER is required}" +: "${DB_PASSWORD:?DB_PASSWORD is required}" + +TS="$(date -u +%Y%m%dT%H%M%SZ)" +OUT_FILE="${1:-backup_${DB_NAME}_${TS}.dump}" + +export PGPASSWORD="$DB_PASSWORD" +pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -Fc -f "$OUT_FILE" +echo "Backup created: $OUT_FILE" diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 0000000..5555fee --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +python manage.py migrate diff --git a/scripts/restore_postgres.sh b/scripts/restore_postgres.sh new file mode 100755 index 0000000..13a98ef --- /dev/null +++ b/scripts/restore_postgres.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +: "${DB_HOST:?DB_HOST is required}" +: "${DB_PORT:?DB_PORT is required}" +: "${DB_NAME:?DB_NAME is required}" +: "${DB_USER:?DB_USER is required}" +: "${DB_PASSWORD:?DB_PASSWORD is required}" + +BACKUP_FILE="$1" +export PGPASSWORD="$DB_PASSWORD" +pg_restore --clean --if-exists --no-owner -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" "$BACKUP_FILE" +echo "Restore completed from: $BACKUP_FILE"