diff --git a/.claude/skills/configure-k8s-deployment.md b/.claude/skills/configure-k8s-deployment.md index 75a19fe..556c54a 100644 --- a/.claude/skills/configure-k8s-deployment.md +++ b/.claude/skills/configure-k8s-deployment.md @@ -29,7 +29,7 @@ Before asking the user anything, read a small known set of files directly (do no 3. `k8s/base/kustomization.yaml`, `k8s/base/streamlit-deployment.yaml`, `k8s/base/rq-worker-deployment.yaml`, `k8s/base/workspace-pvc.yaml` — confirm the layout still matches the template: - PVC `metadata.name` is `workspaces-pvc`. - Deployments reference `image: openms-streamlit` (the placeholder Kustomize swaps). - - `streamlit-deployment.yaml` has `claimName: workspaces-pvc` and `volume-group: workspaces` (both as a pod label and as the pod-affinity `matchExpressions` value). + - `streamlit-deployment.yaml` has `claimName: workspaces-pvc`. (Co-location of the workspace-using pods is enforced by the shared RWO PVC mount, not by a pod-affinity rule.) 4. `.github/workflows/build-and-test.yml` — confirm which tags CI publishes (the OpenMS template publishes `-full`, `-simple`, `-full`, `-simple`, plus `latest` on `main`-full pushes). If any of those files are missing, renamed, or significantly restructured, stop and ask the user how to proceed. Do not pattern-match the standard answers onto an unknown layout. @@ -155,7 +155,7 @@ spec: storage: # Q6: e.g. 100Gi, 1Ti, 3Ti ``` -Do **not** rename the PVC, the base `kustomization.yaml` resource list, the `claimName` in `streamlit-deployment.yaml`, or the `volume-group` pod-affinity label. Kustomize's `namePrefix` already gives the in-cluster PVC a unique per-fork name; renaming the base creates a 3-file cascade for no benefit. +Do **not** rename the PVC, the base `kustomization.yaml` resource list, or the `claimName` in `streamlit-deployment.yaml`. Kustomize's `namePrefix` already gives the in-cluster PVC a unique per-fork name; renaming the base creates a 3-file cascade for no benefit. Operator caveat (mention in handoff, not your job to verify): in-place expansion of an *already-deployed* PVC requires the StorageClass to have `allowVolumeExpansion: true`. If the operator's `cinder-csi` class does not allow expansion, growing a live PVC requires recreation, not a manifest edit. Resizing on first deploy is unaffected. @@ -164,7 +164,7 @@ Operator caveat (mention in handoff, not your job to verify): in-place expansion After committing the edits, tell the user the next steps belong to a human operator (or CI) and are out of scope for you: 1. Open a PR with the overlay edits and have it reviewed. -2. Merge to `main`. CI (`build-and-test.yml`) rebuilds and pushes the image to GHCR with the tag from Q3. +2. Merge to `main`. CI (`build-and-test.yml`) rebuilds and pushes the image to GHCR with the tag from Q3. The kind integration jobs (`test-nginx`, `test-traefik`) auto-discover slug and Traefik hostnames from the overlay output, so no workflow edits are needed for fork-specific values. 3. Cluster operator runs `kubectl apply -k k8s/overlays/prod/` against the OpenMS cluster. 4. Operator verifies with `kubectl -n openms rollout status deployment/-streamlit` and a browser check on `https://.webapps.openms.de`. @@ -185,4 +185,5 @@ After committing the edits, tell the user the next steps belong to a human opera - [ ] Redis URL written in both Deployment patches (`streamlit` and `rq-worker`) - [ ] Memory-tier component selected - [ ] Storage size in `k8s/base/workspace-pvc.yaml` updated only if the user picked a non-default size; PVC name and `claimName` untouched +- [ ] `.github/workflows/build-and-test.yml` uses dynamic overlay discovery (no `template-app` / `template.webapps.openms.*` literals); patched in if the fork's workflow was on the old hardcoded shape - [ ] Changes committed on a feature branch (no PR opened unless the user asked for one) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 78c4137..7dfeb75 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -158,19 +158,24 @@ jobs: sleep "${i}0" done + - name: Discover overlay identity + run: | + SLUG=$(yq '.commonLabels.app' k8s/overlays/prod/kustomization.yaml) + echo "SLUG=$SLUG" >> "$GITHUB_ENV" + - name: Wait for Redis to be ready run: | - kubectl wait -n openms --for=condition=ready pod -l app=quantms-ddalfq,component=redis --timeout=60s + kubectl wait -n openms --for=condition=ready pod -l app=${SLUG},component=redis --timeout=60s - name: Verify Redis Service is reachable run: | - kubectl run redis-test -n openms --image=redis:7-alpine --rm -i --restart=Never -- redis-cli -h quantms-ddalfq-redis.openms.svc.cluster.local ping + kubectl run redis-test -n openms --image=redis:7-alpine --rm -i --restart=Never -- redis-cli -h ${SLUG}-redis.openms.svc.cluster.local ping - name: Verify all deployments are available run: | - kubectl wait -n openms --for=condition=available deployment -l app=quantms-ddalfq --timeout=180s || true - kubectl get pods -n openms -l app=quantms-ddalfq - kubectl get services -n openms -l app=quantms-ddalfq + kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s || true + kubectl get pods -n openms -l app=${SLUG} + kubectl get services -n openms -l app=${SLUG} - name: Curl both hostnames via nginx ingress run: | @@ -245,29 +250,39 @@ jobs: sleep "${i}0" done + - name: Discover overlay identity + run: | + SLUG=$(yq '.commonLabels.app' k8s/overlays/prod/kustomization.yaml) + TRAEFIK_HOSTS=$(kubectl kustomize k8s/overlays/prod/ \ + | yq 'select(.kind == "IngressRoute") | .spec.routes[0].match' \ + | grep -oP "Host\(\`\K[^\`]+" | tr '\n' ' ') + echo "SLUG=$SLUG" >> "$GITHUB_ENV" + echo "TRAEFIK_HOSTS=$TRAEFIK_HOSTS" >> "$GITHUB_ENV" + - name: Wait for Redis to be ready run: | - kubectl wait -n openms --for=condition=ready pod -l app=quantms-ddalfq,component=redis --timeout=60s + kubectl wait -n openms --for=condition=ready pod -l app=${SLUG},component=redis --timeout=60s - name: Verify all deployments are available run: | - kubectl wait -n openms --for=condition=available deployment -l app=quantms-ddalfq --timeout=180s || true - kubectl get pods -n openms -l app=quantms-ddalfq - kubectl get services -n openms -l app=quantms-ddalfq + kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s || true + kubectl get pods -n openms -l app=${SLUG} + kubectl get services -n openms -l app=${SLUG} - name: Curl both hostnames via Traefik run: | kubectl -n traefik port-forward svc/traefik 8080:80 & PF_PID=$! trap 'kill "$PF_PID" 2>/dev/null || true' EXIT + FIRST_HOST=$(echo ${TRAEFIK_HOSTS} | awk '{print $1}') for i in $(seq 1 30); do sleep 2 - if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8080/_stcore/health -H "Host: opendda.webapps.openms.de"; then + if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8080/_stcore/health -H "Host: ${FIRST_HOST}"; then break fi echo "port-forward / app not ready yet, retry $i" done - for host in opendda.webapps.openms.de opendda.webapps.openms.org; do + for host in ${TRAEFIK_HOSTS}; do curl -fsS --resolve "$host:8080:127.0.0.1" "http://$host:8080/_stcore/health" echo "" echo "$host -> 200 OK" diff --git a/docs/kubernetes-deployment.md b/docs/kubernetes-deployment.md index e30fd9f..d1264c5 100644 --- a/docs/kubernetes-deployment.md +++ b/docs/kubernetes-deployment.md @@ -35,8 +35,8 @@ Every production OpenMS webapp (quantms-web, umetaflow, FLASHApp) deploys via th │ Streamlit Deployment │ │ (N replicas, default 2) │ │ │ - │ [pod affinity: co-locate with │ - │ rq-worker + cleanup-cronjob pods] │ + │ [co-located with rq-worker + │ + │ cleanup pods via shared RWO PVC] │ └────────┬────────────────────────┬───────┘ │ REDIS_URL │ │ │ /workspaces-... @@ -76,9 +76,11 @@ Every production OpenMS webapp (quantms-web, umetaflow, FLASHApp) deploys via th | Traefik IngressRoute | External HTTP entrypoint with sticky sessions | — | — | | nginx Ingress | Alternative HTTP entrypoint used by the CI kind cluster | — | — | -### Pod affinity +### Pod co-location via the RWO PVC -All workspace-using pods (Streamlit, RQ worker, Cleanup) carry a `volume-group: workspaces` label and a `requiredDuringSchedulingIgnoredDuringExecution` pod-affinity rule keyed on `kubernetes.io/hostname`. This forces every workspace-using pod onto the same node, so they can share the `ReadWriteOnce` PVC. +All workspace-using pods (Streamlit, RQ worker, Cleanup) of a given fork mount the same `-workspaces-pvc` (`ReadWriteOnce`, `cinder-csi`). Once the first pod schedules, the volume is attached to that node and the kube-scheduler's `VolumeBinding` plugin pins every subsequent pod that mounts the same PVC to the same node. NodeSelector (`openms.de/memory-tier`) picks which set of nodes the fork is eligible for; the RWO mount picks the specific node within that set. + +There is no pod-affinity rule. Forks are isolated from each other — co-location applies within a fork (because they share a PVC), not across forks (each fork has its own PVC). Co-location is a placement constraint, not a replica cap. The Streamlit deployment can scale to N replicas — they all land on the same node alongside the worker. @@ -130,14 +132,14 @@ Main Streamlit Deployment. Key fields: - Mounts the workspace PVC at `/workspaces-streamlit-template` - Mounts `settings-overrides.json` from the ConfigMap as a `subPath` - Readiness and liveness probes hit `/_stcore/health` -- Pod affinity: `volume-group: workspaces` +- Co-located with the RQ worker (and any cleanup Job) on the node the RWO `workspaces-pvc` is attached to - `seed-demos` initContainer merges image-shipped demos into `.demos/` on the PVC (see [Demo workspaces](#demo-workspaces)) ### `streamlit-service.yaml` ClusterIP Service exposing Streamlit on port 8501. ### `rq-worker-deployment.yaml` -RQ worker Deployment (1 replica). Runs `rq worker openms-workflows --url $REDIS_URL`. Shares the workspace PVC via the same `volume-group: workspaces` affinity rule. +RQ worker Deployment (1 replica). Runs `rq worker openms-workflows --url $REDIS_URL`. Shares the workspace PVC, so it co-locates onto the same node as the Streamlit pods via the RWO mount. ### `cleanup-cronjob.yaml` CronJob that runs `python clean-up-workspaces.py` nightly at 03:00 UTC. Uses `concurrencyPolicy: Forbid`, retains 3 successful and 3 failed jobs. Shares the workspace PVC. diff --git a/k8s/base/cleanup-cronjob.yaml b/k8s/base/cleanup-cronjob.yaml index 05f764d..8648187 100644 --- a/k8s/base/cleanup-cronjob.yaml +++ b/k8s/base/cleanup-cronjob.yaml @@ -15,19 +15,8 @@ spec: metadata: labels: component: cleanup - volume-group: workspaces spec: restartPolicy: OnFailure - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: volume-group - operator: In - values: - - workspaces - topologyKey: kubernetes.io/hostname containers: - name: cleanup image: openms-streamlit diff --git a/k8s/base/rq-worker-deployment.yaml b/k8s/base/rq-worker-deployment.yaml index 091fdc0..021add3 100644 --- a/k8s/base/rq-worker-deployment.yaml +++ b/k8s/base/rq-worker-deployment.yaml @@ -13,18 +13,7 @@ spec: metadata: labels: component: rq-worker - volume-group: workspaces spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: volume-group - operator: In - values: - - workspaces - topologyKey: kubernetes.io/hostname containers: - name: rq-worker image: openms-streamlit diff --git a/k8s/base/streamlit-deployment.yaml b/k8s/base/streamlit-deployment.yaml index 4b936d3..121d438 100644 --- a/k8s/base/streamlit-deployment.yaml +++ b/k8s/base/streamlit-deployment.yaml @@ -13,18 +13,7 @@ spec: metadata: labels: component: streamlit - volume-group: workspaces spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: volume-group - operator: In - values: - - workspaces - topologyKey: kubernetes.io/hostname initContainers: - name: seed-demos image: openms-streamlit