diff --git a/.gitignore b/.gitignore index 1c07771..a81b2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,9 @@ dist/ # Python __pycache__/ *.pyc + +# Deployment runbooks. Kept out of the repo because they describe one specific +# server — its paths, its host setup, its operational habits — which is detail +# a public repo does not need and an operator reads from their own copy. +DEPLOYMENT.md +DEPLOY-WALKTHROUGH.md diff --git a/DEPLOY-WALKTHROUGH.md b/DEPLOY-WALKTHROUGH.md deleted file mode 100644 index 6932fd5..0000000 --- a/DEPLOY-WALKTHROUGH.md +++ /dev/null @@ -1,921 +0,0 @@ -# Spidder — deployment walkthrough (user: `spidder`) - -A start-to-finish run for a VPS that **already has Docker, Postgres and Redis -installed**. You will not reinstall any of them. What follows creates the -`spidder` user, gives it the access it needs, points the app at the databases you -already have, and puts nginx in front. - -[DEPLOYMENT.md](DEPLOYMENT.md) is the reference version of all of this — read -it when you want the reasoning. This file is the sequence. - -Throughout, replace: - -| Placeholder | With | -| --- | --- | -| `spidder.example.com` | the site hostname | -| `api.spidder.example.com` | the API hostname | -| `your.vps.ip` | the server's address | - ---- - -## Stage 0 — What you already have - -Run these **as your current user** before changing anything. They tell you what -work is genuinely left. - -```bash -# Is Postgres reachable from a container? This is the one that usually fails. -sudo -u postgres psql -c 'SHOW listen_addresses;' - -# Redis: bound where, and does it need a password? -sudo grep -E '^\s*(bind|requirepass|protected-mode)' /etc/redis/redis.conf - -# Does the app's database already exist? -sudo -u postgres psql -c '\l' | grep -i spidder || echo "spidder: NOT created yet" - -# What is the docker bridge address? Every config below refers to it. -ip -4 addr show docker0 | awk '/inet /{print "docker bridge:", $2}' -``` - -Keep the bridge address (usually `172.17.0.1`) — you need it in stages 2 and 3. - -If `listen_addresses` says only `localhost`, **stage 2 is required**. Postgres -being installed is not the same as Postgres being reachable from a container, -and this is the single most common reason a first deploy fails. - ---- - -## Stage 1 — Create the `spidder` user - -As root, or any user with sudo: - -```bash -sudo adduser --disabled-password --gecos "" spidder -``` - -No password: you will reach this account by SSH key only, which is what an -account CI drives should be. - -### Groups - -```bash -sudo usermod -aG sudo spidder # for setup; not needed for deploys -sudo usermod -aG docker spidder # this is the one that matters -``` - -You are **not installing Docker again**. The daemon is already running as root; -the `docker` group is simply the key to its socket at `/var/run/docker.sock`. -Adding `spidder` to that group is the whole of "giving `spidder` Docker access". - -> `docker` group membership is effectively root — a member can mount `/` into a -> container and edit anything. That is normal for a deploy account and it is -> what CI needs, but treat the `spidder` SSH key as a root-equivalent credential. - -### SSH key - -On **your laptop**: - -```bash -ssh-keygen -t ed25519 -C "spidder-deploy" -f ~/.ssh/spidder_deploy -N "" -ssh-copy-id -i ~/.ssh/spidder_deploy.pub spidder@your.vps.ip -``` - -Then verify — this must pass before you go further: - -```bash -ssh -i ~/.ssh/spidder_deploy spidder@your.vps.ip 'id' -``` - -You should see `docker` in the group list. If you don't, the group was added -after the session started; reconnect and check again. - -Finally, confirm Docker works **without sudo** as `spidder`: - -```bash -ssh -i ~/.ssh/spidder_deploy spidder@your.vps.ip 'docker ps' -``` - -A "permission denied on /var/run/docker.sock" here means the group has not -taken effect in that session. Log out fully and back in. - ---- - -## Stage 2 — Let containers reach your Postgres - -Your containers talk to the host database through `host.docker.internal`, which -resolves to the docker bridge. Postgres must be listening there, and must -accept connections from the bridge subnet. - -### Create the role and database - -Skip whichever of these you already did. - -```bash -sudo -u postgres psql -``` - -```sql -CREATE ROLE spidder WITH LOGIN PASSWORD 'PUT_A_LONG_RANDOM_PASSWORD_HERE'; -CREATE DATABASE spidder OWNER spidder; -\q -``` - -Generate that password rather than inventing one: `openssl rand -hex 24`. - -### Listen on the bridge - -```bash -sudo -u postgres psql -c 'SHOW config_file;' # find the exact path -sudo nano /etc/postgresql/16/main/postgresql.conf -``` - -```conf -listen_addresses = 'localhost,172.17.0.1' -``` - -### Allow the bridge subnet - -```bash -sudo nano /etc/postgresql/16/main/pg_hba.conf -``` - -Add this line. It is `172.16.0.0/12`, not the single gateway address, because a -container's own IP is assigned per-network and will not be `172.17.0.1`: - -```conf -host spidder spidder 172.16.0.0/12 scram-sha-256 -``` - -```bash -sudo systemctl restart postgresql -``` - -### Prove it works - -This is the only test that means anything — it runs from inside a container, -exactly like the app will: - -```bash -docker run --rm --add-host=host.docker.internal:host-gateway alpine:3.20 \ - sh -c 'nc -z -w3 host.docker.internal 5432 && echo CONNECTED || echo REFUSED' -``` - -`REFUSED` means `listen_addresses` has not taken effect. Do not continue until -this prints `CONNECTED`. - ---- - -## Stage 3 — Same for Redis - -```bash -sudo nano /etc/redis/redis.conf -``` - -```conf -bind 127.0.0.1 172.17.0.1 -requirepass PUT_ANOTHER_LONG_RANDOM_PASSWORD_HERE -protected-mode yes -appendonly yes -``` - -```bash -sudo systemctl restart redis-server -``` - -> **Do set the password.** A Redis on the bridge with no password is reachable -> by anything that gets a shell in any container — including the sandbox whose -> entire job is running strangers' code. - -Prove it: - -```bash -docker run --rm --add-host=host.docker.internal:host-gateway alpine:3.20 \ - sh -c 'nc -z -w3 host.docker.internal 6379 && echo CONNECTED || echo REFUSED' -``` - ---- - -## Stage 4 — Firewall - -Only SSH and web need to be open. The app ports are published to `127.0.0.1` -and reached by nginx locally; the database ports are reached over the docker -bridge, which `ufw` does not filter. - -```bash -sudo ufw default deny incoming -sudo ufw default allow outgoing -sudo ufw allow OpenSSH -sudo ufw allow 'Nginx Full' -sudo ufw enable -sudo ufw status verbose -``` - -From your **laptop**, these must all hang and time out: - -```bash -nc -zv your.vps.ip 5432 -nc -zv your.vps.ip 6379 -nc -zv your.vps.ip 4001 -``` - -If any connects, stop and fix it before the app is live. - ---- - -## Stage 5 — Get the code, as `spidder` - -```bash -ssh -i ~/.ssh/spidder_deploy spidder@your.vps.ip -``` - -Clone into the `spidder` user's own home directory. No `sudo`, no `chown` — the -user already owns it, which is the whole point of running deploys as `spidder`: - -```bash -cd ~ -git clone https://github.com/TheCodeHeist-Coder/Spidder.git app -cd ~/app -pwd # /home/spidder/app — note this, you need it in stage 11 -``` - -Docker does not care where the compose file lives. The one path-sensitive thing -in it is a bind mount for the Piston provisioning script, and that path is -relative to the compose file, so it travels with the directory. - -Tighten the home directory while you are here. Some distros create it `755`, -which would let any other account on the box read your `.env` — and that file -holds the database password and the JWT secret: - -```bash -chmod 750 /home/spidder -``` - ---- - -## Stage 6 — Write `.env` - -```bash -cp .env.prod.example .env -chmod 600 .env -nano .env -``` - -`chmod 600` matters: this file holds your database password and JWT secret. - -Generate the JWT secret now: - -```bash -openssl rand -hex 32 -``` - -Six values are **required** — compose refuses to start without them: - -```bash -# Note host.docker.internal, NOT localhost. Inside a container, localhost is -# the container itself, and the connection is refused. -DATABASE_URL="postgresql://spidder:YOUR_PG_PASSWORD@host.docker.internal:5432/spidder?schema=public" -REDIS_URL="redis://:YOUR_REDIS_PASSWORD@host.docker.internal:6379" - -JWT_SECRET="paste-the-openssl-output-here" - -# The SITE origin — the origin a browser sends. Not the API's own hostname. -CORS_ORIGINS="https://spidder.example.com" - -# What the BROWSER calls. Compiled into the JS bundle at image build time. -NEXT_PUBLIC_API_URL="https://api.spidder.example.com" -NEXT_PUBLIC_WS_URL="wss://api.spidder.example.com/ws" -``` - -Note the Redis URL shape: `redis://:password@host` — the colon before the -password is not a typo. Redis has no username, so the field is left empty. - -Optional, worth setting on the first deploy so you have an admin account: - -```bash -SUPER_ADMIN_EMAIL="you@example.com" -SUPER_ADMIN_PASSWORD="a-strong-password" -SUPER_ADMIN_USERNAME="admin" -``` - -Remove those three and restart once the account exists, so the credentials stop -sitting in a file on disk. - -### Check it before starting anything - -```bash -docker compose -f docker-compose.prod.yml config >/dev/null && echo "env OK" -``` - -A missing required value names itself here, in a second, instead of halfway -through a rollout. - ---- - -## Stage 7 — First start - -```bash -cd ~/app -docker compose -f docker-compose.prod.yml up -d -``` - -The first run pulls five images, installs Python into the Piston sandbox, then -applies the database schema and seeds the problems. Give it a few minutes. - -Watch it happen: - -```bash -docker compose -f docker-compose.prod.yml logs -f -``` - -Then confirm the shape is right: - -```bash -docker compose -f docker-compose.prod.yml ps -``` - -You are looking for: - -| Service | Expected | -| --- | --- | -| `http-api` | Up (healthy) | -| `ws-server` | Up (healthy) | -| `web` | Up (healthy) | -| `judge-worker` | Up (healthy) | -| `piston` | Up (healthy) | -| `db-init` | **Exited (0)** | -| `piston-init` | **Exited (0)** | - -The two `Exited (0)` are correct, not failures — they are one-shot setup jobs -that run and finish. Any other exit code is a real problem; read its log: - -```bash -docker compose -f docker-compose.prod.yml logs db-init -``` - -### Prove the stack works before adding nginx - -```bash -curl -s localhost:4001/health # {"ok":true} -curl -s localhost:4002/health # {"ok":true} -curl -sI localhost:3001 | head -1 # HTTP/1.1 200 OK -``` - -All three must pass. If they do, the application is fine and anything that -breaks next is nginx or DNS — which is a much smaller haystack. - ---- - -## Stage 8 — DNS - -Point both names at the server, and wait for them to resolve before requesting -certificates. Certbot proves control by answering a challenge on these names. - -``` -A spidder.example.com -> your.vps.ip -A api.spidder.example.com -> your.vps.ip -``` - -```bash -dig +short spidder.example.com -dig +short api.spidder.example.com -``` - -Both must print your server's IP before stage 10. - ---- - -## Stage 9 — nginx, in depth - -nginx is the only way in. The containers publish to `127.0.0.1` only, so -without this nothing is reachable from outside the machine at all. - -Needs sudo — nginx is a host service, not a container. - -### How nginx decides where a request goes - -Two rules, and almost every routing surprise comes from misunderstanding one of -them. - -**First it picks a `server` block, by `Host` header.** Two hostnames point at -this one IP, so nginx reads the `Host:` header the browser sent and matches it -against `server_name`. That is why the site and the API can share port 443 with -no conflict. - -**Then, inside that block, it picks a `location`** — and *not* in file order. -The rules, in priority: - -| Syntax | Meaning | Priority | -| --- | --- | --- | -| `location = /path` | exact match | highest — wins immediately | -| `location ^~ /path` | prefix, stop searching regexes | second | -| `location ~ /re` | regex, first match in file order | third | -| `location /path` | prefix — **longest match wins** | lowest | - -The last row is the one that catches people. `location /ws` and `location /` -are both prefix matches; `/ws` wins for a request to `/ws` because it is -*longer*, not because it appears first. Writing it first is still worth doing — -it makes the intent obvious to the next reader — but the ordering is not what -makes it work. - -### The upgrade map - -Must live at `http` level, not inside a `server`, so it goes in its own file: - -```bash -sudo nano /etc/nginx/conf.d/upgrade.conf -``` - -```nginx -# Maps the request's Upgrade header to the right Connection response header. -# -# Why not just write `proxy_set_header Connection "upgrade"` in the /ws block? -# Because that block also serves the plain HTTP requests that share the same -# connection, and telling nginx every one of them is an upgrade breaks -# keep-alive. This sends "upgrade" only when the client actually asked for one. -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} -``` - -### Shared proxy headers - -Both blocks set the same four headers. Put them in one file and `include` it, -so they cannot drift apart: - -```bash -sudo nano /etc/nginx/conf.d/proxy-common.conf -``` - -```nginx -# Included by every location that proxies to a container. -# -# Without these the app sees every request as coming from 127.0.0.1 (nginx -# itself) on http — so rate limiting by IP would lump all users together, and -# any absolute URL the app generates would come out as http://. -proxy_set_header Host $host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $scheme; -proxy_set_header X-Forwarded-Host $host; -``` - -> `conf.d/*.conf` is included at `http` level, so this file is parsed once and -> the directives are available to every server block. `include` inside a -> `location` then pulls them in where needed. - -### Rate limiting - -Define the zones at `http` level. They are *defined* here and *applied* per -location, which lets one zone protect several endpoints: - -```bash -sudo nano /etc/nginx/conf.d/limits.conf -``` - -```nginx -# 10 MB of shared memory tracks roughly 160,000 addresses — far more than a -# single VPS will ever see at once. -# -# Two zones because the traffic shapes differ. Normal API browsing is bursty -# and harmless; auth endpoints are where credential stuffing lands, and there a -# human never needs more than a few attempts a minute. -limit_req_zone $binary_remote_addr zone=api:10m rate=30r/s; -limit_req_zone $binary_remote_addr zone=auth:10m rate=5r/m; - -# Cap concurrent connections per address. A WebSocket holds one open for the -# length of a battle, so this must be generous enough for several tabs. -limit_conn_zone $binary_remote_addr zone=conn:10m; - -# Return 429 rather than nginx's default 503 — "too many requests" is the -# honest status, and clients back off correctly on it. -limit_req_status 429; -limit_conn_status 429; -``` - -### The site block - -```bash -sudo nano /etc/nginx/sites-available/spidder -``` - -```nginx -server { - listen 80; - listen [::]:80; - server_name spidder.example.com; - - # Hide the nginx version from error pages and the Server header. - server_tokens off; - - # Security headers. These apply to the HTML the site serves. - # - # No Content-Security-Policy here: Next.js injects inline scripts for - # hydration, and a CSP tight enough to be worth having needs nonces wired - # through the app. A half-CSP with 'unsafe-inline' would be theatre. - add_header X-Content-Type-Options "nosniff" always; - add_header X-Frame-Options "SAMEORIGIN" always; - add_header Referrer-Policy "strict-origin-when-cross-origin" always; - - # Uploads are avatars at most. - client_max_body_size 2m; - - # Next's immutable build assets: hashed filenames, safe to cache forever. - # `^~` stops nginx evaluating regex locations for these, which is a small - # win on the highest-volume path on the site. - location ^~ /_next/static/ { - proxy_pass http://127.0.0.1:3001; - include /etc/nginx/conf.d/proxy-common.conf; - - add_header Cache-Control "public, max-age=31536000, immutable"; - access_log off; - } - - # Everything else: the Next.js server. - location / { - proxy_pass http://127.0.0.1:3001; - proxy_http_version 1.1; - include /etc/nginx/conf.d/proxy-common.conf; - - # Next streams server components. Buffering holds the whole response - # until it completes, which delays first paint for no benefit. - proxy_buffering off; - - # If the container is restarting mid-deploy, wait rather than 502. - proxy_connect_timeout 5s; - proxy_read_timeout 60s; - } -} -``` - -### The API block - -This one carries the WebSocket, the auth rate limit, and the deny rule for the -unauthenticated internal endpoint. - -```bash -sudo nano /etc/nginx/sites-available/spidder-api -``` - -```nginx -server { - listen 80; - listen [::]:80; - server_name api.spidder.example.com; - - server_tokens off; - client_max_body_size 2m; - - # Concurrent connections per IP. Generous because each open battle holds a - # WebSocket, and someone may legitimately have two or three tabs. - limit_conn conn 20; - - # --- realtime: ws-server ------------------------------------------------- - location /ws { - proxy_pass http://127.0.0.1:4002; - proxy_http_version 1.1; - - # The two headers that make an upgrade happen. $connection_upgrade - # comes from the map in conf.d/upgrade.conf. - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - include /etc/nginx/conf.d/proxy-common.conf; - - # A battle runs for minutes and the socket is idle between moves. - # nginx's 60s default would cut it mid-match, and the client would - # reconnect into a battle already in progress. - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - - # Realtime frames must not wait for a buffer to fill. - proxy_buffering off; - - # NOT rate limited. A WebSocket is one request that lives for the whole - # battle; limiting it by request rate would do nothing useful, and - # limit_conn above already caps how many a single address can hold. - } - - # --- ws-server health, for uptime monitoring ----------------------------- - # `=` is an exact match, so this cannot accidentally shadow anything. - location = /ws-health { - proxy_pass http://127.0.0.1:4002/health; - include /etc/nginx/conf.d/proxy-common.conf; - access_log off; - } - - # --- BLOCKED: unauthenticated internal endpoint -------------------------- - # ws-server exposes /internal/stats with NO auth — by design, because it is - # meant to be reached only by http-api over the Docker network. See - # apps/ws-server/src/transport/httpApp.ts. - # - # `^~` matters here: it stops nginx searching regex locations, so no regex - # added later can ever take precedence and re-expose this. - location ^~ /internal/ { - return 404; - } - - # --- auth: the tight rate limit ------------------------------------------ - # 5 requests/minute per IP, burst 10. Credential stuffing is the attack - # this blunts; a real person signing in never comes close to the limit. - # - # `nodelay` lets the burst through immediately rather than queuing it — - # a legitimate retry should feel instant, only sustained abuse should stall. - location /auth/ { - limit_req zone=auth burst=10 nodelay; - - proxy_pass http://127.0.0.1:4001; - proxy_http_version 1.1; - include /etc/nginx/conf.d/proxy-common.conf; - proxy_read_timeout 60s; - } - - # --- everything else: http-api ------------------------------------------- - location / { - limit_req zone=api burst=60 nodelay; - - proxy_pass http://127.0.0.1:4001; - proxy_http_version 1.1; - include /etc/nginx/conf.d/proxy-common.conf; - - proxy_connect_timeout 5s; - proxy_read_timeout 60s; - } -} -``` - -> **On CORS.** Do not add `add_header Access-Control-Allow-Origin` here. The -> API already handles CORS itself from `CORS_ORIGINS`, and a second set of -> headers from nginx produces *duplicate* headers, which browsers reject -> outright — the symptom is every cross-origin request failing with a CORS -> error that looks like the app is misconfigured. - -### Enable and test - -```bash -sudo ln -s /etc/nginx/sites-available/spidder /etc/nginx/sites-enabled/ -sudo ln -s /etc/nginx/sites-available/spidder-api /etc/nginx/sites-enabled/ - -# The default block answers for any Host that matches nothing else. Left in -# place it will serve the nginx welcome page to anyone hitting your bare IP. -sudo rm -f /etc/nginx/sites-enabled/default - -sudo nginx -t -``` - -`nginx -t` is not optional. A reload with a broken config leaves the old one -running — but a *restart* would fail and take the site down. - -```bash -sudo systemctl reload nginx -``` - -### Check the routing before adding TLS - -Over plain HTTP first, so a certificate problem cannot be confused with a -routing problem. `--resolve` forces the hostname at your server without -needing DNS to have propagated yet: - -```bash -IP=your.vps.ip - -curl -s --resolve spidder.example.com:80:$IP \ - -o /dev/null -w 'site: %{http_code}\n' http://spidder.example.com/ - -curl -s --resolve api.spidder.example.com:80:$IP \ - http://api.spidder.example.com/health && echo - -curl -s --resolve api.spidder.example.com:80:$IP \ - http://api.spidder.example.com/ws-health && echo - -# MUST be 404. -curl -s --resolve api.spidder.example.com:80:$IP \ - -o /dev/null -w 'internal: %{http_code} (must be 404)\n' \ - http://api.spidder.example.com/internal/stats -``` - -### Useful while debugging - -```bash -# The whole effective config, every include expanded. Answers "is my map -# actually loaded" definitively. -sudo nginx -T | less - -# Confirm the upgrade map is present. -sudo nginx -T | grep -A3 'map $http_upgrade' - -# Which server block answered, and what it did. -sudo tail -f /var/log/nginx/access.log -sudo tail -f /var/log/nginx/error.log -``` - -An `upstream connect failed` in the error log means nginx is fine and the -container is not — check `docker compose ps`. - ---- - -## Stage 10 — HTTPS - -```bash -sudo certbot --nginx \ - -d spidder.example.com \ - -d api.spidder.example.com \ - --agree-tos -m you@example.com --redirect -``` - -Certbot edits both server blocks in place, adding `listen 443 ssl`, the -certificate paths, and an HTTP→HTTPS redirect. Renewal is a systemd timer: - -```bash -sudo certbot renew --dry-run -systemctl list-timers | grep certbot -``` - -### Verify the whole path - -```bash -curl -sI https://spidder.example.com | head -1 # 200 -curl -s https://api.spidder.example.com/health # {"ok":true} -curl -s https://api.spidder.example.com/ws-health # {"ok":true} - -# MUST be 404. A 200 means the deny block is missing or mis-ordered. -curl -s -o /dev/null -w '%{http_code}\n' \ - https://api.spidder.example.com/internal/stats - -# The WebSocket. 101 Switching Protocols means the handshake works. -curl -i -s -N \ - -H "Connection: Upgrade" -H "Upgrade: websocket" \ - -H "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==" \ - -H "Sec-WebSocket-Version: 13" \ - https://api.spidder.example.com/ws | head -1 -``` - -That `101` is the single most important check here. A `200` instead means the -request fell through to http-api — the `/ws` block is missing or ordered after -`location /`. - -Then open the site in a browser, sign in, and start a battle. If the timer -runs and a submission gets judged, the whole path works. - ---- - -## Stage 11 — CI/CD - -Add these under **Settings → Secrets and variables → Actions**: - -| Secret | Value | -| --- | --- | -| `DOCKERHUB_USERNAME` | `codeheist` | -| `DOCKERHUB_TOKEN` | access token from hub.docker.com → Account Settings → Personal access tokens (Read & Write) | -| `VPS_HOST` | `your.vps.ip` | -| `VPS_USER` | `spidder` | -| `VPS_SSH_KEY` | the **whole** contents of `~/.ssh/spidder_deploy` (private key) | -| `VPS_APP_DIR` | `/home/spidder/app` — **absolute**, not `~/app` | -| `NEXT_PUBLIC_API_URL` | `https://api.spidder.example.com` | -| `NEXT_PUBLIC_WS_URL` | `wss://api.spidder.example.com/ws` | - -```bash -cat ~/.ssh/spidder_deploy # copy everything, BEGIN and END lines included -``` - -Note `VPS_USER` is `spidder`, and `NEXT_PUBLIC_*` must match what you put in -`.env` — they are compiled into the browser bundle at image build time, so a -mismatch means the site calls the wrong host and only a rebuild fixes it. - -### First run - -Push to `main`, or use **Actions → Deploy → Run workflow**. It will: - -1. re-run lint, typecheck and tests -2. build five images and push them to Docker Hub -3. SSH in as `spidder`, pull, apply the schema, restart - -On Docker Hub, the free tier allows **one private repository**. Five private -images needs a paid plan — otherwise make the five repos public. They hold only -compiled application code; every credential is injected at runtime from `.env`. - -### Prove the pipeline, not just the app - -Change something visible, push, and confirm the running image actually moved: - -```bash -docker compose -f docker-compose.prod.yml images | grep web -``` - -The tag should be the new commit SHA. - ---- - -## Day-to-day - -As `spidder`, no sudo: - -```bash -cd ~/app -alias dc='docker compose -f docker-compose.prod.yml' - -dc ps # status and health -dc logs -f --tail=100 # everything -dc logs -f judge-worker # one service -dc restart http-api -``` - -Host services need sudo: - -```bash -sudo systemctl status nginx postgresql redis-server -sudo nginx -t && sudo systemctl reload nginx -sudo tail -f /var/log/nginx/error.log -``` - -### Backups - -The database is on the host, so no Docker volume contains it: - -```bash -sudo tee /etc/cron.daily/spidder-backup >/dev/null <<'EOF' -#!/bin/sh -set -eu -mkdir -p /var/backups/spidder -sudo -u postgres pg_dump spidder | gzip \ - > /var/backups/spidder/spidder-$(date +%F).sql.gz -find /var/backups/spidder -name '*.sql.gz' -mtime +14 -delete -EOF -sudo chmod +x /etc/cron.daily/spidder-backup -sudo /etc/cron.daily/spidder-backup && ls -lh /var/backups/spidder -``` - -Run it once by hand as above — a backup script nobody has tested is not a -backup. - -### Rollback - -Re-run **Deploy** on the last good commit from the Actions tab, or pin by hand: - -```bash -cd ~/app -IMAGE_TAG= docker compose -f docker-compose.prod.yml pull -IMAGE_TAG= docker compose -f docker-compose.prod.yml up -d -``` - -This rolls back **code only**. `prisma db push` converges forward, so undoing a -destructive schema change needs the backup above. - ---- - -## When something breaks - -**`docker ps` says permission denied.** The `docker` group has not applied to -this session. Log out completely and back in. - -**Containers restart in a loop.** Read the log — it names the missing thing: - -```bash -docker compose -f docker-compose.prod.yml logs --tail=50 http-api -``` - -**`ECONNREFUSED` to Postgres or Redis.** Almost always `localhost` in `.env` -where it should be `host.docker.internal`, or the host service still listening -on loopback only. Test from inside a container: - -```bash -docker compose -f docker-compose.prod.yml exec http-api \ - node -e "require('net').connect(5432,'host.docker.internal').on('connect',()=>{console.log('ok');process.exit(0)}).on('error',e=>{console.log(e.code);process.exit(1)})" -``` - -**Site loads, but the browser console shows calls to `localhost:4001`.** -`NEXT_PUBLIC_API_URL` was wrong when the image was built. It is compiled into -the bundle — re-run the deploy workflow with the corrected secret. A restart -will not fix it. - -**Battles never start, everything else is fine.** The WebSocket is not -upgrading. Check the `101` from stage 10, then: - -```bash -sudo nginx -T | grep -A3 'map $http_upgrade' -``` - -**Submissions queue but are never judged.** `judge-worker` or `piston` is down, -or Piston has no Python runtime: - -```bash -docker compose -f docker-compose.prod.yml logs judge-worker piston -docker compose -f docker-compose.prod.yml logs piston-init | tail -5 -``` - -**`no space left on device`.** Old images: - -```bash -docker system df -docker image prune -a -f --filter "until=168h" -``` - ---- - -## One gap to know about - -`apps/admin` has **no Dockerfile**, so the separate admin UI is not deployable -yet and nothing above proxies to it. The admin *API* routes live inside -`http-api` and are already served from `api.spidder.example.com`; only the -Next.js dashboard is missing. When it is containerised it needs its own -hostname, its own server block, and ideally an IP allowlist or basic auth at -the nginx layer on top of its login. diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md deleted file mode 100644 index 41bcf41..0000000 --- a/DEPLOYMENT.md +++ /dev/null @@ -1,739 +0,0 @@ -# Deploying Spidder to a VPS - -Postgres, Redis and nginx run **on the host**, managed by systemd. Everything -else — the four app services and the Piston sandbox — runs in Docker, owned by -an unprivileged `deploy` user. Nothing here needs root after setup. - -Two things shape the rest of this guide. Containers cannot reach a host service -by `localhost`, so Postgres and Redis must listen on the Docker bridge and the -firewall must then keep that bridge off the public internet. And the app -containers publish only to `127.0.0.1`, so nginx is the single way in. - -``` - ┌──────────────── VPS ──────────────────┐ - │ │ - browser ──── :443 ──▶ │ nginx (host, systemd) │ - │ │ │ - spidder.example.com ─┼─▶ 127.0.0.1:3001 web │ - │ │ │ - api.spidder.example.com ─┼─▶ 127.0.0.1:4001 http-api │ - /ws │ └─▶ 127.0.0.1:4002 ws-server │ - │ │ │ - │ ┌──────────┴────────────┐ │ - │ ▼ ▼ │ - │ postgres :5432 redis :6379 │ - │ (host, systemd) (host, systemd) │ - │ ▲ ▲ │ - │ └─ host.docker.internal ┘ │ - │ (docker bridge, 172.17.0.1) │ - │ │ - │ judge-worker ──▶ piston (container) │ - │ │ - │ ── docker compose, run as `deploy` ──│ - └───────────────────────────────────────┘ -``` - ---- - -## 1. Create the deploy user - -Everything below runs as a dedicated `deploy` user, never as root. Two reasons -that matter in practice: a mistake in a deploy script cannot touch the rest of -the system, and the SSH key CI holds is scoped to one account you can revoke -without locking yourself out. - -SSH in as root **one last time**: - -```bash -adduser --disabled-password --gecos "" deploy -``` - -`--disabled-password` means the account has no password to guess. You will -reach it by key only, which is what you want for an account CI can drive. - -### Give it sudo, but keep it honest - -```bash -usermod -aG sudo deploy -``` - -The deploy user needs sudo for the *setup* in sections 2–5 — installing -packages, editing Postgres config, managing nginx. It does **not** need sudo -for deploys: once Docker is set up, `docker compose` runs as the user itself. -If you want to lock it down further after setup, remove it from `sudo` and the -CI deploy will still work. - -### Let it use Docker without sudo - -```bash -usermod -aG docker deploy -``` - -This is what makes the CI deploy possible: the workflow SSHes in and runs -`docker compose` directly, with no password prompt to get stuck on. - -> **Be clear-eyed about this.** Membership of the `docker` group is effectively -> root: anyone in it can start a container that mounts `/` and edit anything on -> the host. It is the standard way to run deploys and it is what the CI -> workflow needs, but it means the `deploy` SSH key is a root-equivalent -> credential. Treat it that way — dedicated key, no passphrase reuse, revoke it -> if a laptop goes missing. - -### Set up SSH access - -On **your laptop**, generate a key used for nothing else: - -```bash -ssh-keygen -t ed25519 -C "spidder-deploy" -f ~/.ssh/spidder_deploy -N "" -ssh-copy-id -i ~/.ssh/spidder_deploy.pub deploy@your.vps.ip -``` - -Confirm it works **before** you lock root out: - -```bash -ssh -i ~/.ssh/spidder_deploy deploy@your.vps.ip 'id && sudo -n true && echo "sudo ok"' -``` - -You should see `deploy` in the groups, including `docker`. - -### Close the root door - -Only once the line above succeeds. In `/etc/ssh/sshd_config`: - -```conf -PermitRootLogin no -PasswordAuthentication no -PubkeyAuthentication yes -``` - -```bash -sudo sshd -t # syntax check FIRST — a typo here locks you out -sudo systemctl reload ssh -``` - -Keep your existing session open while you test a *new* one in another -terminal. If the new session fails you still have the old one to fix it with. - ---- - -## 2. Host prerequisites - -Ubuntu 22.04+ or Debian 12+, 2 vCPU / 4 GB RAM minimum. The Next.js build does -not run here — CI builds the images — but Piston plus four Node processes want -headroom. - -As `deploy`: - -```bash -sudo apt update && sudo apt install -y \ - postgresql postgresql-contrib redis-server nginx git ufw \ - certbot python3-certbot-nginx - -# Docker engine + compose plugin -curl -fsSL https://get.docker.com | sudo sh -``` - -You added `deploy` to the `docker` group in section 1, but **group membership -only applies to new sessions**. Log out and back in, then confirm: - -```bash -docker ps # must work with no sudo -``` - -If that still says "permission denied", the session is stale — reconnect. - ---- - -## 3. Postgres - -Create the database and a role that is **not** the superuser: - -```bash -sudo -u postgres psql <<'SQL' -CREATE ROLE spidder WITH LOGIN PASSWORD 'use-a-long-random-password'; -CREATE DATABASE spidder OWNER spidder; -SQL -``` - -Now make it reachable from the containers. Find the bridge address first — -`172.17.0.1` is the common default but not guaranteed: - -```bash -ip -4 addr show docker0 | awk '/inet /{print $2}' # e.g. 172.17.0.1/16 -``` - -Find your config directory (the version number varies): - -```bash -sudo -u postgres psql -c 'SHOW config_file;' -``` - -In `postgresql.conf`: - -```conf -listen_addresses = 'localhost,172.17.0.1' -``` - -In `pg_hba.conf` — the whole private range, because a container's own address -is assigned per-network and will not be the gateway: - -```conf -host spidder spidder 172.16.0.0/12 scram-sha-256 -``` - -```bash -sudo systemctl restart postgresql -``` - -Verify from inside a container, which is the only test that proves the path: - -```bash -docker run --rm --add-host=host.docker.internal:host-gateway alpine:3.20 \ - sh -c 'nc -z -w3 host.docker.internal 5432 && echo CONNECTED || echo REFUSED' -``` - -`REFUSED` means Postgres is still loopback-only — re-check `listen_addresses`. - ---- - -## 4. Redis - -Find the config with `sudo systemctl cat redis-server | grep ExecStart`, then -in `/etc/redis/redis.conf`: - -```conf -bind 127.0.0.1 172.17.0.1 -requirepass use-another-long-random-password -protected-mode yes -appendonly yes -``` - -```bash -sudo systemctl restart redis-server -``` - -> **Set the password.** Leaving Redis open on the bridge means anything that -> gets a shell in any container — including the sandbox that runs submitted -> code — has full read/write access to your job queue. - ---- - -## 5. Firewall - -The bridge is now listening. Close the public door before going further: - -```bash -sudo ufw default deny incoming -sudo ufw default allow outgoing -sudo ufw allow OpenSSH -sudo ufw allow 'Nginx Full' -sudo ufw enable -``` - -Ports 5432 and 6379 are deliberately absent, and so are 3001/4001/4002. The -containers publish those to `127.0.0.1` only and nginx reaches them there; -nothing from outside should. - -Verify from your laptop — all of these should hang and time out: - -```bash -nc -zv your.vps.ip 5432 -nc -zv your.vps.ip 6379 -nc -zv your.vps.ip 4001 -``` - ---- - -## 6. The application - -As `deploy`, in a directory the user owns: - -Clone into the deploy user's own home directory — no `sudo`, no `chown`, since -the user already owns it: - -```bash -cd ~ -git clone https://github.com/TheCodeHeist-Coder/Spidder.git spidder -cd ~/spidder - -cp .env.prod.example .env -chmod 600 .env -$EDITOR .env # fill in every CHANGE_ME -``` - -`chmod 600` matters: that file holds your database password and JWT secret. -Tighten the home directory too — some distros create it `755`, which lets any -other account on the box read it: - -```bash -chmod 750 ~ -``` - -`/opt/spidder` or `/srv/spidder` work equally well if you prefer a conventional -location; they just need `sudo mkdir` and `sudo chown deploy:deploy` first. -Whatever you choose, `VPS_APP_DIR` must match it. - -Generate the secrets rather than inventing them: - -```bash -openssl rand -hex 32 # JWT_SECRET -openssl rand -hex 24 # database / redis passwords -``` - -Then bring it up: - -```bash -docker compose -f docker-compose.prod.yml up -d -docker compose -f docker-compose.prod.yml ps -``` - -`db-init` applies the schema and seeds the problem bank, then exits 0. The app -services wait for it, so the first request never hits an empty database. - ---- - -## 7. nginx and TLS - -The containers publish to `127.0.0.1` only, so nginx is not optional — it is -the only way in. - -Two server blocks: one for the site, one for the API. They are separate -hostnames because the browser bundle is compiled with absolute URLs -(`NEXT_PUBLIC_API_URL`), and keeping the API on its own name means you can move -or scale it later without rebuilding the frontend. - -### DNS first - -Point both names at the VPS before requesting certificates — certbot proves -control by answering an HTTP challenge on them: - -``` -A spidder.example.com -> your.vps.ip -A api.spidder.example.com -> your.vps.ip -``` - -Check it has propagated: `dig +short spidder.example.com`. - -### The site block - -`/etc/nginx/sites-available/spidder`: - -```nginx -server { - listen 80; - listen [::]:80; - server_name spidder.example.com; - - # certbot fills in the TLS config and the redirect below. - location / { - proxy_pass http://127.0.0.1:3001; - proxy_http_version 1.1; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $host; - - # Next.js streams server components; buffering them defeats the point - # and makes the first paint wait for the whole response. - proxy_buffering off; - } - - # Next's immutable build assets. Hashed filenames, so a long cache is safe - # and saves the Node process from serving them at all on repeat visits. - location /_next/static/ { - proxy_pass http://127.0.0.1:3001; - proxy_cache_valid 200 60m; - add_header Cache-Control "public, max-age=31536000, immutable"; - } - - client_max_body_size 2m; -} -``` - -### The API block - -This one carries the WebSocket, and the upgrade headers are the part people -miss — without them battles never start while everything else looks fine. - -`/etc/nginx/sites-available/spidder-api`: - -```nginx -server { - listen 80; - listen [::]:80; - server_name api.spidder.example.com; - - # --- realtime: ws-server ------------------------------------------------- - # Must come before the catch-all `location /`, or the API block below - # swallows it and the upgrade never happens. - location /ws { - proxy_pass http://127.0.0.1:4002; - proxy_http_version 1.1; - - # The three lines that make a WebSocket work. $connection_upgrade is - # defined in the map block below — using a literal "upgrade" here - # breaks ordinary requests that share the connection. - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header Host $host; - - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - # A battle outlives nginx's 60s default, and the socket is idle - # whenever nobody is typing. Without this the connection is cut - # mid-match and the client reconnects into a battle already moving. - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - - # Realtime frames must not be held back waiting for a buffer to fill. - proxy_buffering off; - } - - # --- ws-server's health probe ------------------------------------------- - # Useful for external uptime monitoring. Cheap, and leaks nothing. - location = /ws-health { - proxy_pass http://127.0.0.1:4002/health; - proxy_set_header Host $host; - access_log off; - } - - # --- BLOCKED: ws-server's internal stats --------------------------------- - # /internal/* is server-to-server only and has NO authentication — see - # apps/ws-server/src/transport/httpApp.ts. http-api reaches it over the - # Docker network; it must never be routable from outside. This block is - # belt-and-braces (the proxy_pass above targets 4002 only under /ws), but - # an explicit deny survives someone later adding a broader location. - location /internal/ { - return 404; - } - - # --- everything else: http-api ------------------------------------------- - location / { - proxy_pass http://127.0.0.1:4001; - proxy_http_version 1.1; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_read_timeout 60s; - } - - # Code submissions are the largest thing a client sends, and they are - # text. 2 MB is generous; the app caps source at 100 KB anyway. - client_max_body_size 2m; -} -``` - -### The upgrade map - -`$connection_upgrade` has to be defined at the `http` level, not inside a -server block. Create `/etc/nginx/conf.d/upgrade.conf`: - -```nginx -# Maps the Upgrade request header to the right Connection response header. -# -# A plain `Connection: upgrade` on every request breaks keep-alive for normal -# HTTP; this sends "upgrade" only when the client actually asked for one, and -# "close" otherwise. -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} -``` - -### Enable and test - -```bash -sudo ln -s /etc/nginx/sites-available/spidder /etc/nginx/sites-enabled/ -sudo ln -s /etc/nginx/sites-available/spidder-api /etc/nginx/sites-enabled/ -sudo rm -f /etc/nginx/sites-enabled/default - -sudo nginx -t # ALWAYS before reload -sudo systemctl reload nginx -``` - -### Certificates - -```bash -sudo certbot --nginx \ - -d spidder.example.com \ - -d api.spidder.example.com \ - --agree-tos -m you@example.com --redirect -``` - -`--redirect` adds the HTTP→HTTPS redirect. certbot rewrites both server blocks -in place, adding `listen 443 ssl`, the certificate paths, and a port-80 block -that redirects. Renewal is installed as a systemd timer; check it with: - -```bash -systemctl list-timers | grep certbot -sudo certbot renew --dry-run -``` - -### Verify the whole path - -```bash -curl -I https://spidder.example.com -curl -s https://api.spidder.example.com/health # {"ok":true} -curl -s https://api.spidder.example.com/ws-health # {"ok":true} -curl -s -o /dev/null -w '%{http_code}\n' \ - https://api.spidder.example.com/internal/stats # 404 — must NOT be 200 - -# The WebSocket upgrade. 101 means the handshake worked. -curl -i -N \ - -H "Connection: Upgrade" -H "Upgrade: websocket" \ - -H "Sec-WebSocket-Key: $(openssl rand -base64 16)" \ - -H "Sec-WebSocket-Version: 13" \ - https://api.spidder.example.com/ws 2>&1 | head -1 -``` - -A `200` on `/internal/stats` means the deny block is missing or ordered wrong — -fix it before going live. - -### What about the admin dashboard? - -`apps/admin` has **no `Dockerfile.prod`** and appears in no compose file, so -there is nothing to proxy to yet — that is why no third server block is given -here. The admin API routes live inside `http-api` and are already reachable -through `api.spidder.example.com`; only the separate Next.js UI is missing. - -When it is containerised, it needs its own hostname and a block identical to -the site one, pointing at its port — and it should sit behind something more -than a login form: an IP allowlist (`allow`/`deny`) or basic auth at the nginx -layer, because it is the one surface where a stolen session is worth the most. - -### Point the app at these names - -Two places, and they must agree: - -1. **Repository secrets** — `NEXT_PUBLIC_API_URL=https://api.spidder.example.com` - and `NEXT_PUBLIC_WS_URL=wss://api.spidder.example.com/ws`. Note `wss://`, - not `ws://`: a plain-ws connection from an https page is blocked by the - browser as mixed content. -2. **The server's `.env`** — `CORS_ORIGINS=https://spidder.example.com`. This - is the *site* origin, not the API's; it is the origin the browser sends. - -The `NEXT_PUBLIC_*` pair is compiled into the browser bundle at image build -time, so changing them means re-running the deploy workflow. Changing -`CORS_ORIGINS` only needs a restart. - ---- - -## 8. CI/CD - -### What runs when - -| Workflow | Trigger | What it does | -|---|---|---| -| `ci.yml` | every push, every PR | lint, typecheck, tests, full build | -| `deploy.yml` | push to `main`, or manual | re-verify → build 5 images → push to Docker Hub → roll out | - -Images are built on the runner, never on the VPS. Building a Next.js app on a -small droplet competes with the running site for memory and can OOM halfway, -taking production down to ship a change. A runner builds it, the server pulls -the result. - -Every deploy is tagged with its commit SHA, so a rollback is re-running the -workflow against an older commit rather than reverting and rebuilding. - -### Repository secrets - -`Settings → Secrets and variables → Actions`: - -| Secret | Example | Notes | -|---|---|---| -| `DOCKERHUB_USERNAME` | `codeheist` | your Docker Hub account, lowercase | -| `DOCKERHUB_TOKEN` | `dckr_pat_…` | an **access token**, not your password | -| `VPS_HOST` | `203.0.113.10` | hostname or IP | -| `VPS_USER` | `deploy` | must be in the `docker` group | -| `VPS_SSH_KEY` | `-----BEGIN OPENSSH…` | the **private** key, whole file | -| `VPS_APP_DIR` | `/home/deploy/spidder` | absolute path, not `~` | -| `NEXT_PUBLIC_API_URL` | `https://api.spidder.example.com` | baked into the browser bundle | -| `NEXT_PUBLIC_WS_URL` | `wss://api.spidder.example.com/ws` | note `wss://`, not `ws://` | -| `VPS_SSH_PORT` | `22` | optional | - -### Docker Hub - -Create the access token at **hub.docker.com → Account Settings → Personal -access tokens**, scope **Read & Write**. Use a token rather than your password: -a token can be revoked on its own, and it cannot be used to sign in to the -account itself. - -The workflow pushes five images: - -``` -/spidder-http-api -/spidder-ws-server -/spidder-judge-worker -/spidder-web -/spidder-db-init -``` - -They are created on first push. **The free tier includes one private -repository**, so five private images needs a paid plan — otherwise create them -as public. They hold only compiled application code; every credential is -injected at runtime from the server's `.env`, so nothing secret ships inside -them. The one thing compiled *in* is `NEXT_PUBLIC_*` in the web image, and -those are public URLs visible in any visitor's devtools regardless. - -The compose file already defaults to `codeheist`, so a hand-run -`docker compose` on the VPS resolves the same images the workflow pushes. If -you publish under a different Docker Hub account, set `REGISTRY` in the -server's `.env` to match `DOCKERHUB_USERNAME` — otherwise the server keeps -pulling the original account's images while CI pushes to yours, and the deploy -appears to succeed while changing nothing. - -### The deploy key - -Generate a keypair used by nothing else: - -```bash -ssh-keygen -t ed25519 -C "github-actions" -f ~/.ssh/spidder_deploy -N "" -ssh-copy-id -i ~/.ssh/spidder_deploy.pub deploy@your.vps.ip -cat ~/.ssh/spidder_deploy # paste ALL of this into VPS_SSH_KEY -``` - -### First run - -The workflow logs the server in to Docker Hub before pulling, even when the -images are public. That is on purpose: anonymous pulls are rate-limited per IP -address, and a VPS that shares an address range with other pullers can hit the -ceiling partway through a deploy. The failure is a `429 Too Many Requests` on -`docker compose pull`, which reads like an outage rather than a quota. An -authenticated pull uses the account's own, much higher limit. - -A first deploy failing with `pull access denied` or `repository does not exist` -means the namespace is wrong — check `DOCKERHUB_USERNAME` is your Hub account -(not your GitHub org) and is lowercase. - ---- - -## Operations - -Everything here runs as `deploy`, with no sudo — that is the point of section 1. - -```bash -cd ~/spidder - -# A short alias saves repeating the -f on every command. -alias dc='docker compose -f docker-compose.prod.yml' - -dc ps # what is running, and is it healthy -dc logs -f --tail=100 # everything -dc logs -f http-api # one service -dc restart http-api -dc pull && dc up -d # roll to the latest images by hand -``` - -nginx, Postgres and Redis are host services, so they use systemd and do need -sudo: - -```bash -sudo systemctl status nginx postgresql redis-server -sudo nginx -t && sudo systemctl reload nginx # ALWAYS test before reload -sudo tail -f /var/log/nginx/error.log -``` - -### Backups - -The database is on the host, so it is not in any Docker volume: - -```bash -# /etc/cron.daily/spidder-backup -sudo -u postgres pg_dump spidder | gzip > /var/backups/spidder-$(date +%F).sql.gz -find /var/backups -name 'spidder-*.sql.gz' -mtime +14 -delete -``` - -Restore: - -```bash -gunzip -c /var/backups/spidder-2026-09-13.sql.gz | sudo -u postgres psql spidder -``` - -### Rollback - -The fastest route is the Actions tab: re-run **Deploy** on the last good -commit. To pin by hand on the server instead — `IMAGE_TAG` is the only knob, -since the Docker Hub account is written literally in the compose file: - -```bash -cd ~/spidder -IMAGE_TAG= docker compose -f docker-compose.prod.yml pull -IMAGE_TAG= docker compose -f docker-compose.prod.yml up -d -``` - -Find a SHA from `git log --oneline` or the Actions run list. Note this does -**not** roll the database back: `db push` is forward-converging, so a rollback -across a destructive schema change needs the backup below. - ---- - -## Troubleshooting - -**`ECONNREFUSED` reaching Postgres or Redis.** The host service is not -listening on the bridge. Check `listen_addresses` / `bind`, confirm the bridge -address is what you assumed (`ip -4 addr show docker0`), and check from inside -a container: - -```bash -docker compose -f docker-compose.prod.yml exec http-api \ - node -e "require('net').connect(5432,'host.docker.internal').on('connect',()=>{console.log('ok');process.exit(0)}).on('error',e=>{console.log(e.code);process.exit(1)})" -``` - -**The site loads but nothing works; the browser console shows calls to -`localhost:4001`.** `NEXT_PUBLIC_API_URL` was wrong when the web image was -built. These values are compiled into the client bundle, so fixing the secret -requires a **rebuild**, not a restart — re-run the deploy workflow. - -**Battles never start; everything else is fine.** The WebSocket is not being -upgraded. Check in this order: - -```bash -# 1. Does the handshake reach ws-server at all? 101 = yes. -curl -i -s -N -H "Connection: Upgrade" -H "Upgrade: websocket" \ - -H "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==" -H "Sec-WebSocket-Version: 13" \ - https://api.spidder.example.com/ws | head -1 - -# 2. Is the map defined? Missing $connection_upgrade is the usual cause. -sudo nginx -T | grep -A3 'map $http_upgrade' -``` - -A `200` instead of `101` means the request fell through to `location /` and hit -http-api — the `/ws` block is missing or ordered after the catch-all. A `502` -means it routed correctly but ws-server is down. - -**`403` or `404` from the API on requests the browser makes.** Usually CORS: -`CORS_ORIGINS` on the server must be the **site** origin -(`https://spidder.example.com`), not the API's own hostname. It is the origin -the browser sends, not the one it is talking to. - -**The admin dashboard's live connection count is always zero.** -`WS_SERVER_INTERNAL_URL` must be `http://ws-server:4002` — the code defaults to -`localhost:4002`, which inside a container is the container itself. It is set -in `docker-compose.prod.yml`; confirm it survived any local edit: - -```bash -docker compose -f docker-compose.prod.yml exec http-api printenv WS_SERVER_INTERNAL_URL -``` - -**Submissions queue but never judge.** `judge-worker` or `piston` is down: - -```bash -docker compose -f docker-compose.prod.yml logs judge-worker piston -``` - -`piston-init` must have exited 0 at least once, or no language runtime is -installed and every submission fails to execute. - -**`no space left on device` during a pull.** Old images: - -```bash -docker system df -docker image prune -a -f --filter "until=168h" -``` diff --git a/README.md b/README.md index dc08b3e..cf177a9 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,8 @@ installs the Python 3.12 runtime into the code sandbox — automatically. The ap services wait for all of it, so the first request can never hit an unmigrated database or an empty sandbox. -Developing? Use the hot-reload stack instead: - -```bash -docker compose -f docker-compose.dev.yml up --build -``` +Your source is bind-mounted, so this is already the hot-reload stack: edit a +file on your host and the containers pick it up in about a second. Full instructions, including running without Docker, are in **[SETUP.md](SETUP.md)**. @@ -304,16 +301,19 @@ spidder/ ├── .github/workflows/ │ ├── ci.yml lint · typecheck · test · build │ └── deploy.yml build images → Docker Hub → roll out on the VPS -├── docker-compose.yml full stack in containers (local) -├── docker-compose.dev.yml hot-reload dev stack +├── docker-compose.yml hot-reload dev stack (the default) └── docker-compose.prod.yml VPS deploy — host Postgres/Redis ``` ### Deploying -See [DEPLOYMENT.md](DEPLOYMENT.md). Production runs Postgres and Redis on the -host and only the app services in Docker, so the compose file and the host -setup have to agree — that guide covers both. +`docker-compose.prod.yml` is the deployment stack, and `.github/workflows/deploy.yml` +drives it: build images, push to Docker Hub, roll them out over SSH. Both files +are commented with what they expect. + +Production runs Postgres and Redis on the host and only the app services in +Docker, so the compose file and the host setup have to agree. The detailed +runbook for that is kept out of this repo — it describes one specific server. ### Data model diff --git a/SETUP.md b/SETUP.md index e240ae2..ccc60cf 100644 --- a/SETUP.md +++ b/SETUP.md @@ -40,12 +40,16 @@ docker compose down -v # wipe the database too ### Development, with hot reload -```bash -docker compose -f docker-compose.dev.yml up --build -``` +Nothing more to run — the stack above already does this. Your source is +bind-mounted and the servers restart on change, so edit a file on your host and +the containers pick it up in about a second. + +There are only two compose files, and this is the default one: -Same stack, but your source is bind-mounted and the servers restart on change — -edit a file on your host and the containers pick it up in about a second. +| File | Use | +| --- | --- | +| `docker-compose.yml` | everything above — local development | +| `docker-compose.prod.yml` | deploying to a server — see the comments in that file | ### Ports diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml deleted file mode 100644 index 52b8523..0000000 --- a/docker-compose.dev.yml +++ /dev/null @@ -1,203 +0,0 @@ -############################################################################### -# Spidder — development stack (hot reload). -# -# docker compose -f docker-compose.dev.yml up --build -# -# Same topology as production, but every app runs from its Dockerfile.dev with -# the repo bind-mounted over /app, so edits on the host hot-reload inside the -# container (tsx watch for the services, next dev for the web app). -# -# Open http://localhost:3001 -# -# The `node_modules` anonymous volumes are load-bearing: they shadow the host's -# node_modules so the container keeps its own linux-native install rather than -# the host's (possibly different-arch) one. -############################################################################### - -name: spidder-dev - -x-app-env: &app-env - NODE_ENV: development - DATABASE_URL: postgresql://postgres:postgres@postgres:5432/spidder?schema=public - REDIS_URL: redis://redis:6379 - PISTON_URL: http://piston:2000/api/v2 - JWT_SECRET: ${JWT_SECRET:-dev-secret-change-me-in-production} - CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3001} - -services: - # --- infrastructure ------------------------------------------------------ - postgres: - image: postgres:16-alpine - restart: unless-stopped - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: spidder - # Bound to loopback, and overridable: a Postgres already running on the - # host would otherwise make `compose up` fail with "address already in use". - ports: - - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" - volumes: - - pgdata-dev:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d spidder"] - interval: 5s - timeout: 5s - retries: 10 - - redis: - image: redis:7-alpine - restart: unless-stopped - ports: - - "127.0.0.1:${REDIS_PORT:-6379}:6379" - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 5s - timeout: 3s - retries: 10 - - piston: - image: ghcr.io/engineer-man/piston - restart: unless-stopped - privileged: true - ports: - - "${PISTON_PORT:-2000}:2000" - volumes: - - pistondata-dev:/piston - healthcheck: - # This image ships Node 15 — no global fetch, no curl, no wget. http.get - # is the only probe available to us here. - test: - [ - "CMD", - "node", - "-e", - "require('http').get('http://127.0.0.1:2000/api/v2/runtimes',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))", - ] - interval: 5s - timeout: 5s - retries: 20 - - # --- one-shot initialisers ---------------------------------------------- - piston-init: - image: alpine:3.20 - depends_on: - piston: - condition: service_healthy - environment: - PISTON_URL: http://piston:2000/api/v2 - volumes: - - ./docker/piston-init/provision.sh:/provision.sh:ro - entrypoint: ["/bin/sh", "/provision.sh"] - restart: "no" - - db-init: - build: - context: . - dockerfile: docker/db-init/Dockerfile - depends_on: - postgres: - condition: service_healthy - environment: - DATABASE_URL: postgresql://postgres:postgres@postgres:5432/spidder?schema=public - restart: "no" - - # --- application services (hot reload) ----------------------------------- - http-api: - build: - context: . - dockerfile: apps/http-api/Dockerfile.dev - restart: unless-stopped - depends_on: - postgres: - condition: service_healthy - db-init: - condition: service_completed_successfully - environment: - <<: *app-env - HTTP_API_HOST: 0.0.0.0 - HTTP_API_PORT: 4001 - ports: - - "4001:4001" - volumes: - - .:/app - # Keep the container's linux-native installs and build output. - - /app/node_modules - - /app/apps/http-api/node_modules - - /app/packages/db/node_modules - - /app/packages/db/generated - - ws-server: - build: - context: . - dockerfile: apps/ws-server/Dockerfile.dev - restart: unless-stopped - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - db-init: - condition: service_completed_successfully - environment: - <<: *app-env - WS_SERVER_HOST: 0.0.0.0 - WS_SERVER_PORT: 4002 - ports: - - "4002:4002" - volumes: - - .:/app - - /app/node_modules - - /app/apps/ws-server/node_modules - - /app/packages/db/node_modules - - /app/packages/db/generated - - judge-worker: - build: - context: . - dockerfile: apps/judge-worker/Dockerfile.dev - restart: unless-stopped - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - db-init: - condition: service_completed_successfully - piston-init: - condition: service_completed_successfully - environment: - <<: *app-env - JUDGE_CONCURRENCY: ${JUDGE_CONCURRENCY:-4} - volumes: - - .:/app - - /app/node_modules - - /app/apps/judge-worker/node_modules - - /app/packages/db/node_modules - - /app/packages/db/generated - - web: - build: - context: . - dockerfile: apps/web/Dockerfile.dev - restart: unless-stopped - depends_on: - - http-api - - ws-server - environment: - NODE_ENV: development - PORT: 3001 - # Read at dev-server start; these are the URLs the BROWSER uses. - NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://localhost:4001} - NEXT_PUBLIC_WS_URL: ${NEXT_PUBLIC_WS_URL:-ws://localhost:4002/ws} - ports: - - "3001:3001" - volumes: - - .:/app - - /app/node_modules - - /app/apps/web/node_modules - - /app/apps/web/.next - -volumes: - pgdata-dev: - pistondata-dev: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 8551f14..fcbe5b6 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,120 +1,31 @@ -############################################################################### -# Spidder — production deployment on a VPS. +# Spidder — production deployment. # # docker compose -f docker-compose.prod.yml up -d # -# WHICH COMPOSE FILE IS WHICH -# --------------------------- -# docker-compose.dev.yml infra only, for `pnpm dev` against local sources -# docker-compose.yml the whole stack in containers, one command, for -# trying the app on a laptop -# docker-compose.prod.yml THIS FILE — a real deployment on a real server -# -# DIFFERENT FROM docker-compose.yml IN ONE IMPORTANT WAY -# ------------------------------------------------------ -# Postgres and Redis are NOT containers here. They run on the VPS host, managed -# by systemd, with their own backup and upgrade story. Only the application -# services and the Piston sandbox are containerised. -# -# That means the app containers have to reach back out to the host. On Linux -# that is not automatic: `host.docker.internal` resolves only because each -# service below maps it to `host-gateway`. Without that mapping the containers -# would resolve nothing and every database call would fail at startup. -# -# THE HOST MUST ALLOW IT -# ---------------------- -# Two things are easy to get wrong and both fail the same way — a connection -# that hangs then times out: -# -# 1. Postgres binds to localhost only by default. It must also listen on the -# docker bridge. In postgresql.conf: -# listen_addresses = 'localhost,172.17.0.1' -# and in pg_hba.conf, allow the bridge subnet: -# host spidder spidder 172.16.0.0/12 scram-sha-256 -# -# 2. Redis likewise. In redis.conf: -# bind 127.0.0.1 172.17.0.1 -# requirepass -# Leave `protected-mode yes`. A Redis reachable from the bridge with no -# password is a Redis reachable by anything that gets a shell in any -# container. -# -# See DEPLOYMENT.md for the full host setup, including the firewall rules -# that keep 5432/6379 off the public internet. -# -# IMAGES -# ------ -# Published to Docker Hub under the `codeheist` account, as -# `codeheist/spidder-`. The account is written out literally rather -# than parameterised — it is a fact about this project, not a knob. -# -# IMAGE_TAG is the one part that varies, and defaults to `latest` so a bare -# `docker compose -f docker-compose.prod.yml up -d` works. The deploy workflow -# sets it to the commit SHA, pinning a rollout to one exact build — which is -# what makes a rollback a tag change rather than a revert and rebuild. -# -# Forking? Change `codeheist` here to your own Docker Hub account, and set -# DOCKERHUB_USERNAME to match. Leave it and your server will keep pulling this -# project's images while your CI pushes to yours: a deploy that reports success -# and changes nothing. -# -# SECRETS -# ------- -# Everything below comes from a `.env` file sitting next to this one. There are -# deliberately NO fallback defaults for the credentials: a missing DATABASE_URL -# should stop the deploy, not silently start a stack pointed at a database that -# does not exist. -############################################################################### +# Postgres and Redis run on the VPS itself, not here. This file only runs the +# application containers, and they reach the host through host.docker.internal. +# Setup notes are at the bottom of this file. name: spidder -# Shared by every application service. -# -# `?err` on a value means "fail the deploy if this is unset". It is reserved -# for things that are either secret or environment-specific — exactly the -# values that must never fall back to a development default in production. -x-app-env: &app-env - NODE_ENV: production - DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required — point it at the host Postgres} - REDIS_URL: ${REDIS_URL:?REDIS_URL is required — point it at the host Redis} - PISTON_URL: http://piston:2000/api/v2 - JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required — generate one with `openssl rand -hex 32`} - CORS_ORIGINS: ${CORS_ORIGINS:?CORS_ORIGINS is required — e.g. https://spidder.example.com} - -# Lets a container resolve `host.docker.internal` to the VPS itself, which is -# how it reaches the host-managed Postgres and Redis. Linux needs this spelled -# out; Docker Desktop provides it implicitly, which is why this is a common -# "works on my machine" trap. -x-host-access: &host-access - extra_hosts: - - "host.docker.internal:host-gateway" - -# Keeps one container's runaway logs from filling the VPS disk and taking the -# whole stack down with it. -x-logging: &logging - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - services: - # --- sandbox -------------------------------------------------------------- - # Still a container: Piston needs privileged mode to build its own isolation - # cgroups, and running that directly on the host would be worse, not better. + # --------------------------------------------------------------------------- + # Code sandbox. Runs untrusted user submissions, so it needs privileged mode + # to build its own isolation cgroups. # - # NOT published to the host. Nothing outside this compose network has any - # business reaching an API whose whole job is running untrusted code. + # No published port on purpose: only judge-worker talks to it, over the + # compose network. Nothing outside should reach an API that executes + # arbitrary code. + # --------------------------------------------------------------------------- piston: image: ghcr.io/engineer-man/piston restart: unless-stopped privileged: true volumes: - pistondata:/piston - <<: *logging healthcheck: - # This image ships Node 15 — no global fetch, no curl, no wget. http.get - # is the only probe available to us here. + # The piston image ships Node 15 — no fetch, no curl, no wget. http.get + # is the only probe available. test: [ "CMD", @@ -125,11 +36,19 @@ services: interval: 5s timeout: 5s retries: 20 + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" - # --- one-shot initialisers ------------------------------------------------ - # Installs the Python runtime into Piston. Idempotent; exits 0 when ready. + # --------------------------------------------------------------------------- + # Installs the Python runtime into piston, then exits. Idempotent, so it is + # safe to re-run on every deploy. judge-worker waits for it to finish. + # --------------------------------------------------------------------------- piston-init: image: alpine:3.20 + restart: "no" depends_on: piston: condition: service_healthy @@ -138,24 +57,46 @@ services: volumes: - ./docker/piston-init/provision.sh:/provision.sh:ro entrypoint: ["/bin/sh", "/provision.sh"] - restart: "no" - <<: *logging + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" - # Applies the Prisma schema and seeds problems against the HOST Postgres, - # then exits. Idempotent (`db push` converges, the seed upserts), so it is - # safe to run on every deploy — which is what makes a schema change ship - # automatically with the code that needs it. + # --------------------------------------------------------------------------- + # Applies the Prisma schema to the host Postgres and seeds problems, then + # exits. Every app service waits for it to succeed, so nothing ever starts + # against an unmigrated database. + # + # NOT a database — it is a one-shot client that connects to the Postgres you + # run on the VPS. + # + # WARNING: the seed currently deletes all problems and test cases before + # re-inserting, and `prisma db push` force-converges the schema with no + # migration history. Both are fine against a throwaway dev database and + # dangerous against a real one. See the note at the bottom of this file. + # --------------------------------------------------------------------------- db-init: image: codeheist/spidder-db-init:${IMAGE_TAG:-latest} build: context: . dockerfile: docker/db-init/Dockerfile - environment: - DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} restart: "no" - <<: [*host-access, *logging] + environment: + DATABASE_URL: ${DATABASE_URL:?point this at the host Postgres} + extra_hosts: + - "host.docker.internal:host-gateway" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" - # --- application services ------------------------------------------------- + # --------------------------------------------------------------------------- + # REST API. Published to loopback only — the reverse proxy on the host + # terminates TLS and forwards here. Binding 0.0.0.0 would expose plaintext + # HTTP publicly and bypass the proxy. + # --------------------------------------------------------------------------- http-api: image: codeheist/spidder-http-api:${IMAGE_TAG:-latest} build: @@ -166,30 +107,37 @@ services: db-init: condition: service_completed_successfully environment: - <<: *app-env + NODE_ENV: production + DATABASE_URL: ${DATABASE_URL:?point this at the host Postgres} + REDIS_URL: ${REDIS_URL:?point this at the host Redis} + PISTON_URL: http://piston:2000/api/v2 + JWT_SECRET: ${JWT_SECRET:?generate one with `openssl rand -hex 32`} + CORS_ORIGINS: ${CORS_ORIGINS:?e.g. https://spidder.example.com} HTTP_API_HOST: 0.0.0.0 HTTP_API_PORT: 4001 - # Where http-api reaches ws-server for the admin dashboard's live - # connection counts. Server-to-server over the compose network, never - # through the proxy — that endpoint is unauthenticated by design and - # must not be publicly reachable. - # - # Must be set explicitly: the code defaults to `http://localhost:4002`, - # which inside this container is the container itself. Left unset, the - # admin stats panel fails silently rather than loudly. + # How http-api reaches ws-server for the admin dashboard's live connection + # counts — server to server, never through the proxy. Must be set: the + # code defaults to localhost:4002, which inside this container is itself, + # and the admin stats panel would fail silently. WS_SERVER_INTERNAL_URL: http://ws-server:4002 - # Seeds the first admin account on boot when absent. Optional: leave - # unset and create the admin by hand instead. + # Optional. Creates the first admin account on boot if absent. SUPER_ADMIN_EMAIL: ${SUPER_ADMIN_EMAIL:-} SUPER_ADMIN_PASSWORD: ${SUPER_ADMIN_PASSWORD:-} SUPER_ADMIN_USERNAME: ${SUPER_ADMIN_USERNAME:-} - # Published to LOOPBACK ONLY. The reverse proxy on the host terminates TLS - # and forwards to these; binding 0.0.0.0 here would expose plaintext HTTP - # on the public interface and quietly bypass the proxy. ports: - "127.0.0.1:${HTTP_API_PORT:-4001}:4001" - <<: [*host-access, *logging] + extra_hosts: + - "host.docker.internal:host-gateway" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + # --------------------------------------------------------------------------- + # WebSocket server — live battle state. Loopback only, same proxy reasoning + # as http-api. + # --------------------------------------------------------------------------- ws-server: image: codeheist/spidder-ws-server:${IMAGE_TAG:-latest} build: @@ -200,13 +148,28 @@ services: db-init: condition: service_completed_successfully environment: - <<: *app-env + NODE_ENV: production + DATABASE_URL: ${DATABASE_URL:?point this at the host Postgres} + REDIS_URL: ${REDIS_URL:?point this at the host Redis} + PISTON_URL: http://piston:2000/api/v2 + JWT_SECRET: ${JWT_SECRET:?generate one with `openssl rand -hex 32`} + CORS_ORIGINS: ${CORS_ORIGINS:?e.g. https://spidder.example.com} WS_SERVER_HOST: 0.0.0.0 WS_SERVER_PORT: 4002 ports: - "127.0.0.1:${WS_SERVER_PORT:-4002}:4002" - <<: [*host-access, *logging] + extra_hosts: + - "host.docker.internal:host-gateway" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + # --------------------------------------------------------------------------- + # Judge queue consumer. No published port — it pulls jobs from Redis and + # sends code to piston; nothing connects to it. + # --------------------------------------------------------------------------- judge-worker: image: codeheist/spidder-judge-worker:${IMAGE_TAG:-latest} build: @@ -219,23 +182,36 @@ services: piston-init: condition: service_completed_successfully environment: - <<: *app-env + NODE_ENV: production + DATABASE_URL: ${DATABASE_URL:?point this at the host Postgres} + REDIS_URL: ${REDIS_URL:?point this at the host Redis} + PISTON_URL: http://piston:2000/api/v2 + JWT_SECRET: ${JWT_SECRET:?generate one with `openssl rand -hex 32`} + CORS_ORIGINS: ${CORS_ORIGINS:?e.g. https://spidder.example.com} JUDGE_CONCURRENCY: ${JUDGE_CONCURRENCY:-4} - # No published port: it is a queue consumer, not a server. - <<: [*host-access, *logging] + extra_hosts: + - "host.docker.internal:host-gateway" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + # --------------------------------------------------------------------------- + # Next.js frontend. No host access: it never talks to Postgres or Redis, only + # to http-api and ws-server — and from the browser, not from the container. + # --------------------------------------------------------------------------- web: image: codeheist/spidder-web:${IMAGE_TAG:-latest} build: context: . dockerfile: apps/web/Dockerfile.prod args: - # Inlined into the CLIENT bundle at build time, so these are the URLs a - # BROWSER uses — public https:// origins, never compose service names. - # Changing them requires a rebuild, not a restart; that is why CI passes - # them as build args rather than runtime env. - NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:?NEXT_PUBLIC_API_URL is required — the public API origin} - NEXT_PUBLIC_WS_URL: ${NEXT_PUBLIC_WS_URL:?NEXT_PUBLIC_WS_URL is required — the public WebSocket origin} + # Compiled into the browser bundle at build time, so these are the URLs + # a BROWSER uses — public https:// origins, never compose service names. + # Changing them needs a rebuild, not a restart. + NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:?the public API origin} + NEXT_PUBLIC_WS_URL: ${NEXT_PUBLIC_WS_URL:?the public WebSocket origin} restart: unless-stopped depends_on: http-api: @@ -248,9 +224,80 @@ services: HOSTNAME: 0.0.0.0 ports: - "127.0.0.1:${WEB_PORT:-3001}:3001" - <<: *logging + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" volumes: - # Piston's installed language runtimes. Survives `down`; losing it only means + # Piston's downloaded language runtimes. Losing this volume only means # piston-init downloads Python again on the next boot. pistondata: +# ============================================================================= +# SERVER SETUP +# ============================================================================= +# +# POSTGRES AND REDIS ON THE HOST +# ------------------------------ +# Both run on the VPS, not in this file. The containers reach them through +# `host.docker.internal`, which resolves only because of the `extra_hosts` +# mapping on each service above — Linux needs that spelled out, Docker Desktop +# provides it for free. That difference is a common "works on my machine" trap. +# +# The host services must also listen on the docker bridge, or every connection +# hangs and then times out: +# +# postgresql.conf listen_addresses = 'localhost,172.17.0.1' +# pg_hba.conf host spidder spidder 172.16.0.0/12 scram-sha-256 +# +# redis.conf bind 127.0.0.1 172.17.0.1 +# requirepass +# protected-mode yes +# +# A Redis reachable from the bridge without a password is reachable by anything +# that gets a shell in any container. Keep 5432 and 6379 firewalled off the +# public internet — the bridge is a door for containers, not for the world. +# +# SECRETS +# ------- +# Everything comes from a `.env` file next to this one. Credentials have no +# fallback defaults on purpose: a missing DATABASE_URL should stop the deploy, +# not start a stack pointed at a database that does not exist. +# +# IMAGES +# ------ +# Pulled from Docker Hub as `codeheist/spidder-`. The `build:` blocks +# exist so CI can build and push them; the server only ever pulls. +# +# IMAGE_TAG defaults to `latest` so a bare `up -d` works. The deploy workflow +# sets it to the commit SHA, which is what makes a rollback a tag change rather +# than a revert and rebuild. +# +# Forking? Change `codeheist` above to your own Docker Hub account and set +# DOCKERHUB_USERNAME to match. Leave it, and your server keeps pulling this +# project's images while your CI pushes to yours — a deploy that reports +# success and changes nothing. +# +# KNOWN RISK: db-init +# ------------------- +# db-init is safe to re-run today only because the database is disposable. Two +# things make it dangerous against a database you care about: +# +# 1. prisma/seed.ts calls deleteMany() on problems and test cases before +# re-inserting. That destroys community-submitted and admin-authored +# problems. Once real battles reference a problem, the delete is instead +# REFUSED (Battle.assignedProblemId has no onDelete rule), db-init exits +# non-zero, and every app gated on it refuses to start — a routine deploy +# takes the site down. +# +# 2. `prisma db push` force-converges the schema with no migration history +# and no review step. A column rename becomes a drop and recreate. +# +# Until both are fixed, do not let the deploy workflow run db-init against +# production data. Run it by hand when you intend a schema change: +# +# docker compose -f docker-compose.prod.yml run --rm db-init +# +# And take a backup first: pg_dump on a timer, before real traffic arrives. +# ============================================================================= diff --git a/docker-compose.yml b/docker-compose.yml index 0452c22..9b17925 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,23 +1,34 @@ ############################################################################### -# Spidder — production stack. +# Spidder — development stack (hot reload). # # docker compose up --build # -# Brings up Postgres, Redis, the Piston sandbox (with its Python runtime auto -# installed), applies the DB schema + seed, then starts the API, the realtime -# server, the judge worker, and the web app. +# This is the DEFAULT compose file, so development — the thing done many times +# a day — needs no -f flag. Deployment is the rare, deliberate act, and it is +# the one that spells out its file: +# +# docker-compose.yml THIS FILE — hot-reload dev stack, everything in +# containers including Postgres and Redis +# docker-compose.prod.yml a real deployment on a real server: images pulled +# from Docker Hub, Postgres and Redis on the host +# +# Same topology as production, but every app runs from its Dockerfile.dev with +# the repo bind-mounted over /app, so edits on the host hot-reload inside the +# container (tsx watch for the services, next dev for the web app). # # Open http://localhost:3001 # -# Startup is sequenced with healthchecks: apps wait for db-init and piston-init -# to exit 0, so the first request never hits an unmigrated database or a Piston -# without a Python runtime. +# The `node_modules` anonymous volumes are load-bearing: they shadow the host's +# node_modules so the container keeps its own linux-native install rather than +# the host's (possibly different-arch) one. ############################################################################### -name: spidder +# Distinct from the prod stack's `spidder`, so a local stack and a deployment on +# the same machine never adopt each other's containers or volumes. +name: spidder-dev x-app-env: &app-env - NODE_ENV: production + NODE_ENV: development DATABASE_URL: postgresql://postgres:postgres@postgres:5432/spidder?schema=public REDIS_URL: redis://redis:6379 PISTON_URL: http://piston:2000/api/v2 @@ -38,7 +49,7 @@ services: ports: - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" volumes: - - pgdata:/var/lib/postgresql/data + - pgdata-dev:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d spidder"] interval: 5s @@ -48,18 +59,14 @@ services: redis: image: redis:7-alpine restart: unless-stopped - command: ["redis-server", "--appendonly", "yes"] ports: - "127.0.0.1:${REDIS_PORT:-6379}:6379" - volumes: - - redisdata:/data healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s timeout: 3s retries: 10 - # The code sandbox. Needs privileged mode to build its own isolation cgroups. piston: image: ghcr.io/engineer-man/piston restart: unless-stopped @@ -67,7 +74,7 @@ services: ports: - "${PISTON_PORT:-2000}:2000" volumes: - - pistondata:/piston + - pistondata-dev:/piston healthcheck: # This image ships Node 15 — no global fetch, no curl, no wget. http.get # is the only probe available to us here. @@ -83,7 +90,6 @@ services: retries: 20 # --- one-shot initialisers ---------------------------------------------- - # Installs the Python runtime into Piston. Idempotent; exits 0 when ready. piston-init: image: alpine:3.20 depends_on: @@ -96,7 +102,6 @@ services: entrypoint: ["/bin/sh", "/provision.sh"] restart: "no" - # Applies the Prisma schema and seeds problems. Idempotent; exits 0. db-init: build: context: . @@ -108,11 +113,11 @@ services: DATABASE_URL: postgresql://postgres:postgres@postgres:5432/spidder?schema=public restart: "no" - # --- application services ------------------------------------------------- + # --- application services (hot reload) ----------------------------------- http-api: build: context: . - dockerfile: apps/http-api/Dockerfile.prod + dockerfile: apps/http-api/Dockerfile.dev restart: unless-stopped depends_on: postgres: @@ -123,17 +128,20 @@ services: <<: *app-env HTTP_API_HOST: 0.0.0.0 HTTP_API_PORT: 4001 - # Server-to-server, for the admin dashboard's live connection counts. - # The code defaults to localhost:4002, which in a container is the - # container itself — so it must be named explicitly here. - WS_SERVER_INTERNAL_URL: http://ws-server:4002 ports: - "4001:4001" + volumes: + - .:/app + # Keep the container's linux-native installs and build output. + - /app/node_modules + - /app/apps/http-api/node_modules + - /app/packages/db/node_modules + - /app/packages/db/generated ws-server: build: context: . - dockerfile: apps/ws-server/Dockerfile.prod + dockerfile: apps/ws-server/Dockerfile.dev restart: unless-stopped depends_on: postgres: @@ -148,11 +156,17 @@ services: WS_SERVER_PORT: 4002 ports: - "4002:4002" + volumes: + - .:/app + - /app/node_modules + - /app/apps/ws-server/node_modules + - /app/packages/db/node_modules + - /app/packages/db/generated judge-worker: build: context: . - dockerfile: apps/judge-worker/Dockerfile.prod + dockerfile: apps/judge-worker/Dockerfile.dev restart: unless-stopped depends_on: postgres: @@ -166,30 +180,35 @@ services: environment: <<: *app-env JUDGE_CONCURRENCY: ${JUDGE_CONCURRENCY:-4} + volumes: + - .:/app + - /app/node_modules + - /app/apps/judge-worker/node_modules + - /app/packages/db/node_modules + - /app/packages/db/generated web: build: context: . - dockerfile: apps/web/Dockerfile.prod - args: - # Inlined into the client bundle at build time — these are the URLs the - # BROWSER uses, so they must be host-reachable, not compose-internal. - NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://localhost:4001} - NEXT_PUBLIC_WS_URL: ${NEXT_PUBLIC_WS_URL:-ws://localhost:4002/ws} + dockerfile: apps/web/Dockerfile.dev restart: unless-stopped depends_on: - http-api: - condition: service_healthy - ws-server: - condition: service_healthy + - http-api + - ws-server environment: - NODE_ENV: production + NODE_ENV: development PORT: 3001 - HOSTNAME: 0.0.0.0 + # Read at dev-server start; these are the URLs the BROWSER uses. + NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://localhost:4001} + NEXT_PUBLIC_WS_URL: ${NEXT_PUBLIC_WS_URL:-ws://localhost:4002/ws} ports: - "3001:3001" + volumes: + - .:/app + - /app/node_modules + - /app/apps/web/node_modules + - /app/apps/web/.next volumes: - pgdata: - redisdata: - pistondata: + pgdata-dev: + pistondata-dev: