From 544512a2909e2f4854a3b8012ddb17ce3da44643 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Mon, 28 Sep 2026 16:53:00 -0400 Subject: [PATCH 01/31] =?UTF-8?q?=F0=9F=9A=80=20Move=20deploy=20scripts=20?= =?UTF-8?q?to=20kustomize=20format?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 8 - .gitignore | 3 + deploy/README.md | 76 +++++++++ deploy/harbor-minio.yml | 149 ------------------ deploy/harbor-orchestrator-sa.yml | 61 ------- .../cronjob.yaml} | 34 ++-- deploy/intake-poller/intake-poller-ca.yaml | 12 ++ deploy/intake-poller/kustomization.yaml | 8 + deploy/intake-poller/secret.example.yaml | 16 ++ .../deployment.yaml} | 73 ++------- deploy/job-queue/kustomization.yaml | 20 +++ deploy/job-queue/nebius-secret.example.yaml | 19 +++ deploy/job-queue/orchestrator-anyuid.yaml | 13 ++ deploy/job-queue/orchestrator-role.yaml | 22 +++ .../job-queue/orchestrator-rolebinding.yaml | 13 ++ deploy/job-queue/orchestrator-sa.yaml | 7 + deploy/job-queue/pvc.yaml | 14 ++ deploy/job-queue/route.yaml | 18 +++ deploy/job-queue/secret.example.yaml | 12 ++ deploy/job-queue/service.yaml | 19 +++ .../task-anyuid.yaml} | 9 -- deploy/job-queue/task-sa.yaml | 7 + deploy/minio/api-route.yaml | 18 +++ deploy/minio/console-route.yaml | 18 +++ deploy/minio/deployment.yaml | 61 +++++++ deploy/minio/kustomization.yaml | 11 ++ deploy/minio/pvc.yaml | 15 ++ deploy/minio/secret.yaml | 11 ++ deploy/minio/service.yaml | 21 +++ src/coding_agent_bench/job.py | 2 +- tests/openrouter/test_job_spec.py | 2 +- 31 files changed, 466 insertions(+), 306 deletions(-) delete mode 100644 deploy/harbor-minio.yml delete mode 100644 deploy/harbor-orchestrator-sa.yml rename deploy/{intake-cronjob.yml => intake-poller/cronjob.yaml} (83%) create mode 100644 deploy/intake-poller/intake-poller-ca.yaml create mode 100644 deploy/intake-poller/kustomization.yaml create mode 100644 deploy/intake-poller/secret.example.yaml rename deploy/{job-queue-service.yml => job-queue/deployment.yaml} (59%) create mode 100644 deploy/job-queue/kustomization.yaml create mode 100644 deploy/job-queue/nebius-secret.example.yaml create mode 100644 deploy/job-queue/orchestrator-anyuid.yaml create mode 100644 deploy/job-queue/orchestrator-role.yaml create mode 100644 deploy/job-queue/orchestrator-rolebinding.yaml create mode 100644 deploy/job-queue/orchestrator-sa.yaml create mode 100644 deploy/job-queue/pvc.yaml create mode 100644 deploy/job-queue/route.yaml create mode 100644 deploy/job-queue/secret.example.yaml create mode 100644 deploy/job-queue/service.yaml rename deploy/{harbor-task-sa.yml => job-queue/task-anyuid.yaml} (61%) create mode 100644 deploy/job-queue/task-sa.yaml create mode 100644 deploy/minio/api-route.yaml create mode 100644 deploy/minio/console-route.yaml create mode 100644 deploy/minio/deployment.yaml create mode 100644 deploy/minio/kustomization.yaml create mode 100644 deploy/minio/pvc.yaml create mode 100644 deploy/minio/secret.yaml create mode 100644 deploy/minio/service.yaml diff --git a/.env.example b/.env.example index 800e746..c7e609e 100644 --- a/.env.example +++ b/.env.example @@ -32,11 +32,3 @@ NEBIUS_ENABLED=0 # NEBIUS_INSTANCE_NAME_PREFIX=cab-worker # NEBIUS_IDLE_TIMEOUT_SECONDS=600 # HF_TOKEN= - -# Intake Poller (optional) -# GOOGLE_APPLICATION_CREDENTIALS=service-account.json -# GOOGLE_SHEET_ID= -# JOB_QUEUE_URL=http://localhost:8000 -# - SENDER_EMAIL= -# AUTO_APPROVE=false diff --git a/.gitignore b/.gitignore index 55955af..47dad25 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ models.json config.toml data jobs.db +deploy/job-queue/secret.yaml +deploy/job-queue/nebius-secret.yaml +deploy/intake-poller/secret.yaml # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/deploy/README.md b/deploy/README.md index 2767a74..da6ad37 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -222,3 +222,79 @@ configured `SENDER_EMAIL` must be an address permitted by the relay. Each submitted Queue row carries a deterministic idempotency key. If the CronJob is retried after a network timeout, the queue API returns the original job instead of creating a duplicate. + +## GitHub Actions Deployment + +`.github/workflows/deploy.yml` deploys both Kustomize applications with +`oc`: + +- A merged pull request targeting `STAGE` deploys to `STAGE_NAMESPACE`. +- A version tag such as `v0.3.0` deploys to `PROD_NAMESPACE`. + +The workflow expects the Secrets to already exist in the target namespace. It +only verifies them and applies the two Kustomizations. The files below are safe +templates for local reference only; fill them in locally and do not commit them: + +- `deploy/job-queue/secret.example.yaml` +- `deploy/job-queue/nebius-secret.example.yaml` +- `deploy/intake-poller/secret.example.yaml` + +Before the first CI deployment, apply the filled-in templates to each target +namespace: + +```sh +oc project +oc apply -f deploy/job-queue/secret.yaml +oc apply -f deploy/job-queue/nebius-secret.yaml +oc apply -f deploy/intake-poller/secret.yaml +``` + +Repeat these commands for both stage and production. The CI workflow checks for +`job-queue-secret`, `nebius-secret`, `intake-poller-secret`, and +`intake-poller-google-sa` before applying anything. + +### OpenShift setup + +Create one namespace for stage and one for production. In each namespace, create +a deployer ServiceAccount and grant it the permissions needed to create and update +the resources in these Kustomizations. The account must also be allowed to create +the `anyuid` RoleBinding used by the job-queue workload, or an administrator must +apply that binding separately. + +The OpenShift service CA and service-serving certificate operators must be +available. They create `intake-poller-ca` and `job-queue-tls` when the manifests +are applied. The cluster must also be able to pull the image from GHCR. + +Create a token for each environment's deployer ServiceAccount and verify it locally: + +```sh +oc login --server= --token= +oc project +oc auth can-i create deployments +oc auth can-i create rolebindings +``` + +### GitHub setup + +Create GitHub repository variables: + +| Variable | Value | +|----------|-------| +| `STAGE_NAMESPACE` | OpenShift stage namespace | +| `PROD_NAMESPACE` | OpenShift production namespace | + +Create GitHub Environments named `stage` and `production`. Add the OpenShift +connection secrets to both environments, using environment-specific values. +Production can also require an approval reviewer: + +| Secret | Purpose | +|--------|---------| +| `OPENSHIFT_SERVER` | OpenShift API URL | +| `OPENSHIFT_TOKEN` | Token for the namespace deployer ServiceAccount | + +The application secrets listed in the example files are not GitHub secrets. They +are applied directly to OpenShift before deployment. + +The workflow uses the `stage` environment for merged `STAGE` pull requests and +the `production` environment for `v*` tags. It does not run for an unmerged pull +request or for ordinary branch pushes. diff --git a/deploy/harbor-minio.yml b/deploy/harbor-minio.yml deleted file mode 100644 index 6f1f312..0000000 --- a/deploy/harbor-minio.yml +++ /dev/null @@ -1,149 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi - storageClassName: gp3 - volumeMode: Filesystem ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio -type: Opaque -stringData: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio -spec: - replicas: 1 - selector: - matchLabels: - app: harbor-minio - component: minio - template: - metadata: - labels: - app: harbor-minio - component: minio - spec: - containers: - - name: minio - image: quay.io/minio/minio:latest - args: - - server - - /data - - --console-address - - :9001 - envFrom: - - secretRef: - name: harbor-minio - ports: - - containerPort: 9000 - name: api - - containerPort: 9001 - name: console - livenessProbe: - httpGet: - path: /minio/health/live - port: 9000 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /minio/health/ready - port: 9000 - initialDelaySeconds: 10 - periodSeconds: 5 - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: 250m - memory: 512Mi - volumeMounts: - - mountPath: /data - name: data - volumes: - - name: data - persistentVolumeClaim: - claimName: harbor-minio ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio -spec: - ports: - - name: api - port: 9000 - protocol: TCP - targetPort: 9000 - - name: console - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: harbor-minio - component: minio - type: ClusterIP ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio-api -spec: - tls: - termination: edge - insecureEdgeTerminationPolicy: Redirect - port: - targetPort: 9000 - to: - kind: Service - name: harbor-minio - weight: 100 - wildcardPolicy: None ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio-console -spec: - tls: - termination: edge - insecureEdgeTerminationPolicy: Redirect - port: - targetPort: 9001 - to: - kind: Service - name: harbor-minio - weight: 100 - wildcardPolicy: None diff --git a/deploy/harbor-orchestrator-sa.yml b/deploy/harbor-orchestrator-sa.yml deleted file mode 100644 index acf1218..0000000 --- a/deploy/harbor-orchestrator-sa.yml +++ /dev/null @@ -1,61 +0,0 @@ -# -- Orchestrator SA: used by the job pod to manage task pods, builds, etc. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: harbor-orchestrator - labels: - app: harbor ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: harbor-orchestrator - labels: - app: harbor -rules: - - apiGroups: [""] - resources: [pods, pods/exec, pods/log] - verbs: [get, list, watch, create, update, patch, delete] - - apiGroups: ["build.openshift.io"] - resources: [buildconfigs, buildconfigs/instantiatebinary, builds, builds/log] - verbs: [get, list, watch, create, update, patch, delete] - - apiGroups: ["image.openshift.io"] - resources: [imagestreams] - verbs: [get, list, watch, create, update, patch, delete] - - apiGroups: ["image.openshift.io"] - resources: [imagestreamtags] - verbs: [get] - - apiGroups: ["batch"] - resources: [jobs] - verbs: [get, list, watch, create, update, patch, delete] ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: harbor-orchestrator - labels: - app: harbor -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: harbor-orchestrator -subjects: - - kind: ServiceAccount - name: harbor-orchestrator ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: harbor-orchestrator-anyuid - labels: - app: harbor -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:openshift:scc:anyuid -subjects: - - kind: ServiceAccount - name: harbor-orchestrator diff --git a/deploy/intake-cronjob.yml b/deploy/intake-poller/cronjob.yaml similarity index 83% rename from deploy/intake-cronjob.yml rename to deploy/intake-poller/cronjob.yaml index 174934a..1d0e983 100644 --- a/deploy/intake-cronjob.yml +++ b/deploy/intake-poller/cronjob.yaml @@ -1,17 +1,3 @@ ---- -# The service-ca operator injects the service-serving CA into this ConfigMap -# (populating the service-ca.crt key) because of the inject-cabundle annotation. -# Declaring it here keeps the manifest self-contained: the operator only -# populates an existing ConfigMap, it does not create a missing one. -apiVersion: v1 -kind: ConfigMap -metadata: - name: intake-poller-ca - labels: - app: intake-poller - annotations: - service.beta.openshift.io/inject-cabundle: "true" ---- apiVersion: batch/v1 kind: CronJob metadata: @@ -46,9 +32,20 @@ spec: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/google/service-account.json - name: SMTP_HOST - value: smtp.corp.redhat.com + valueFrom: + secretKeyRef: + name: intake-poller-secret + key: SMTP_HOST - name: SMTP_PORT - value: "25" + valueFrom: + secretKeyRef: + name: intake-poller-secret + key: SMTP_PORT + - name: SMTP_STARTTLS + valueFrom: + secretKeyRef: + name: intake-poller-secret + key: SMTP_STARTTLS - name: GOOGLE_SHEET_ID valueFrom: secretKeyRef: @@ -74,6 +71,11 @@ spec: secretKeyRef: name: intake-poller-secret key: AUTO_APPROVE + - name: ALLOW_INSECURE_QUEUE_HTTP + valueFrom: + secretKeyRef: + name: intake-poller-secret + key: ALLOW_INSECURE_QUEUE_HTTP # The queue is reached over its in-cluster service address, which # presents an OpenShift service-serving cert. Trust its CA (injected # into the mounted ConfigMap) so TLS verification succeeds. diff --git a/deploy/intake-poller/intake-poller-ca.yaml b/deploy/intake-poller/intake-poller-ca.yaml new file mode 100644 index 0000000..66b64b2 --- /dev/null +++ b/deploy/intake-poller/intake-poller-ca.yaml @@ -0,0 +1,12 @@ +# The service-ca operator injects the service-serving CA into this ConfigMap +# (populating the service-ca.crt key) because of the inject-cabundle annotation. +# Declaring it here keeps the manifest self-contained: the operator only +# populates an existing ConfigMap, it does not create a missing one +apiVersion: v1 +kind: ConfigMap +metadata: + name: intake-poller-ca + labels: + app: intake-poller + annotations: + service.beta.openshift.io/inject-cabundle: "true" diff --git a/deploy/intake-poller/kustomization.yaml b/deploy/intake-poller/kustomization.yaml new file mode 100644 index 0000000..9a790b1 --- /dev/null +++ b/deploy/intake-poller/kustomization.yaml @@ -0,0 +1,8 @@ +resources: + - ./cronjob.yaml + - ./intake-poller-ca.yaml + +labels: + - pairs: + app: intake-poller + \ No newline at end of file diff --git a/deploy/intake-poller/secret.example.yaml b/deploy/intake-poller/secret.example.yaml new file mode 100644 index 0000000..2a67842 --- /dev/null +++ b/deploy/intake-poller/secret.example.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + name: intake-poller-secret +type: Opaque +stringData: + GOOGLE_SHEET_ID: + JOB_QUEUE_URL: https://job-queue-service + SENDER_EMAIL: ace-model-evals@redhat.com + AUTO_APPROVE: "false" + SMTP_HOST: smtp.corp.redhat.com + SMTP_PORT: "25" + SMTP_STARTTLS: "false" + ALLOW_INSECURE_QUEUE_HTTP: "false" + service-account.json: |- + diff --git a/deploy/job-queue-service.yml b/deploy/job-queue/deployment.yaml similarity index 59% rename from deploy/job-queue-service.yml rename to deploy/job-queue/deployment.yaml index fd233a6..c896d9a 100644 --- a/deploy/job-queue-service.yml +++ b/deploy/job-queue/deployment.yaml @@ -1,19 +1,3 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: job-queue-pvc - labels: - app: job-queue -spec: - resources: - requests: - storage: 1Gi - volumeMode: Filesystem - storageClassName: gp3 - accessModes: - - ReadWriteOnce ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -38,21 +22,20 @@ spec: spec: terminationGracePeriodSeconds: 60 serviceAccountName: harbor-orchestrator - securityContext: - fsGroup: 1001 containers: - image: ghcr.io/redhat-et/coding_agent_bench:v0.2.6 name: job-queue command: ["/bin/sh", "-c"] args: - | - mkdir -p ~/.ssh && \ - ([ -f /app/data/nebius-ssh-key ] || ssh-keygen -t ed25519 -f /app/data/nebius-ssh-key -N "" -q) && \ - chmod 600 /app/data/nebius-ssh-key && \ + if [ ! -f /app/data/nebius-ssh-key ]; then + ssh-keygen -t ed25519 -f /app/data/nebius-ssh-key -N "" -q || exit 1 + fi + test -r /app/data/nebius-ssh-key || exit 1 exec env \ NEBIUS_SSH_PUBLIC_KEY_PATH=/app/data/nebius-ssh-key.pub \ NEBIUS_SSH_PRIVATE_KEY_PATH=/app/data/nebius-ssh-key \ - uv run uvicorn coding_agent_bench.api:app --host 0.0.0.0 --port 8443 \ + uv run --no-sync uvicorn coding_agent_bench.api:app --host 0.0.0.0 --port 8443 \ --ssl-certfile /etc/job-queue/tls/tls.crt \ --ssl-keyfile /etc/job-queue/tls/tls.key resources: @@ -71,11 +54,18 @@ spec: seccompProfile: type: RuntimeDefault env: + - name: HOME + value: /tmp + - name: UV_CACHE_DIR + value: /tmp/uv-cache - name: JOB_STORE_PATH value: /app/data/jobs.db envFrom: - secretRef: name: job-queue-secret + - secretRef: + name: nebius-secret + optional: true ports: - containerPort: 8443 name: https @@ -95,42 +85,3 @@ spec: secretName: job-queue-tls defaultMode: 0440 restartPolicy: Always ---- -kind: Service -apiVersion: v1 -metadata: - name: job-queue-service - labels: - app: job-queue - component: api - annotations: - service.beta.openshift.io/serving-cert-secret-name: job-queue-tls -spec: - selector: - app: job-queue - component: api - type: ClusterIP - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: job-queue-route - labels: - app: job-queue - component: api -spec: - tls: - termination: reencrypt - insecureEdgeTerminationPolicy: Redirect - port: - targetPort: https - to: - kind: Service - name: job-queue-service - weight: 100 - wildcardPolicy: None diff --git a/deploy/job-queue/kustomization.yaml b/deploy/job-queue/kustomization.yaml new file mode 100644 index 0000000..a7c32ce --- /dev/null +++ b/deploy/job-queue/kustomization.yaml @@ -0,0 +1,20 @@ +resources: + # Orchestrator Pod SA + - ./orchestrator-sa.yaml + - ./orchestrator-role.yaml + - ./orchestrator-rolebinding.yaml + - ./orchestrator-anyuid.yaml + # Task Pod SA + - ./task-sa.yaml + - ./task-anyuid.yaml + # Job Queue + - ./pvc.yaml + - ./secret.yaml + - ./nebius-secret.yaml + - ./deployment.yaml + - ./service.yaml + - ./route.yaml + +labels: + - pairs: + app: job-queue diff --git a/deploy/job-queue/nebius-secret.example.yaml b/deploy/job-queue/nebius-secret.example.yaml new file mode 100644 index 0000000..75b6f19 --- /dev/null +++ b/deploy/job-queue/nebius-secret.example.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nebius-secret +type: Opaque +stringData: + NEBIUS_ENABLED: "0" + NEBIUS_SERVICE_ACCOUNT_CREDS: | + + # Use NEBIUS_SERVICE_ACCOUNT_CREDS_PATH instead when the file is mounted in the pod. + NEBIUS_SERVICE_ACCOUNT_CREDS_PATH: "" + NEBIUS_USER: + NEBIUS_PARENT_ID: + NEBIUS_TENANT_ID: + NEBIUS_SERVICE_ACCOUNT_ID: + NEBIUS_SUBNET_ID: + NEBIUS_INSTANCE_NAME_PREFIX: cab-worker + NEBIUS_IDLE_TIMEOUT_SECONDS: "600" + HF_TOKEN: diff --git a/deploy/job-queue/orchestrator-anyuid.yaml b/deploy/job-queue/orchestrator-anyuid.yaml new file mode 100644 index 0000000..6ca8e85 --- /dev/null +++ b/deploy/job-queue/orchestrator-anyuid.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: harbor-orchestrator-anyuid + labels: + app: harbor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:anyuid +subjects: + - kind: ServiceAccount + name: harbor-orchestrator diff --git a/deploy/job-queue/orchestrator-role.yaml b/deploy/job-queue/orchestrator-role.yaml new file mode 100644 index 0000000..7206bfc --- /dev/null +++ b/deploy/job-queue/orchestrator-role.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: harbor-orchestrator + labels: + app: harbor +rules: + - apiGroups: [""] + resources: [pods, pods/exec, pods/log] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: ["build.openshift.io"] + resources: [buildconfigs, buildconfigs/instantiatebinary, builds, builds/log] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: ["image.openshift.io"] + resources: [imagestreams] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: ["image.openshift.io"] + resources: [imagestreamtags] + verbs: [get] + - apiGroups: ["batch"] + resources: [jobs] + verbs: [get, list, watch, create, update, patch, delete] diff --git a/deploy/job-queue/orchestrator-rolebinding.yaml b/deploy/job-queue/orchestrator-rolebinding.yaml new file mode 100644 index 0000000..694f8ba --- /dev/null +++ b/deploy/job-queue/orchestrator-rolebinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: harbor-orchestrator + labels: + app: harbor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: harbor-orchestrator +subjects: + - kind: ServiceAccount + name: harbor-orchestrator diff --git a/deploy/job-queue/orchestrator-sa.yaml b/deploy/job-queue/orchestrator-sa.yaml new file mode 100644 index 0000000..4e8593d --- /dev/null +++ b/deploy/job-queue/orchestrator-sa.yaml @@ -0,0 +1,7 @@ +# -- Orchestrator SA: used by the job pod to manage task pods, builds, etc. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: harbor-orchestrator + labels: + app: harbor diff --git a/deploy/job-queue/pvc.yaml b/deploy/job-queue/pvc.yaml new file mode 100644 index 0000000..e12e4ed --- /dev/null +++ b/deploy/job-queue/pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: job-queue-pvc + labels: + app: job-queue +spec: + resources: + requests: + storage: 1Gi + volumeMode: Filesystem + storageClassName: gp3 + accessModes: + - ReadWriteOnce diff --git a/deploy/job-queue/route.yaml b/deploy/job-queue/route.yaml new file mode 100644 index 0000000..b134c56 --- /dev/null +++ b/deploy/job-queue/route.yaml @@ -0,0 +1,18 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: job-queue-route + labels: + app: job-queue + component: api +spec: + tls: + termination: reencrypt + insecureEdgeTerminationPolicy: Redirect + port: + targetPort: https + to: + kind: Service + name: job-queue-service + weight: 100 + wildcardPolicy: None diff --git a/deploy/job-queue/secret.example.yaml b/deploy/job-queue/secret.example.yaml new file mode 100644 index 0000000..b04d41a --- /dev/null +++ b/deploy/job-queue/secret.example.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: job-queue-secret +type: Opaque +stringData: + # Required by the job-queue API and the intake poller. + API_KEY: + # Add the provider keys needed by submitted jobs. + ANTHROPIC_API_KEY: + OPENAI_API_KEY: + OPENROUTER_API_KEY: diff --git a/deploy/job-queue/service.yaml b/deploy/job-queue/service.yaml new file mode 100644 index 0000000..621b17b --- /dev/null +++ b/deploy/job-queue/service.yaml @@ -0,0 +1,19 @@ +kind: Service +apiVersion: v1 +metadata: + name: job-queue-service + labels: + app: job-queue + component: api + annotations: + service.beta.openshift.io/serving-cert-secret-name: job-queue-tls +spec: + selector: + app: job-queue + component: api + type: ClusterIP + ports: + - name: https + port: 443 + targetPort: 8443 + protocol: TCP diff --git a/deploy/harbor-task-sa.yml b/deploy/job-queue/task-anyuid.yaml similarity index 61% rename from deploy/harbor-task-sa.yml rename to deploy/job-queue/task-anyuid.yaml index 63fc93e..9f8999b 100644 --- a/deploy/harbor-task-sa.yml +++ b/deploy/job-queue/task-anyuid.yaml @@ -1,12 +1,3 @@ -# -- Task SA: used by environment pods, only needs anyuid SCC to run as root. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: harbor-task - labels: - app: harbor ---- - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/deploy/job-queue/task-sa.yaml b/deploy/job-queue/task-sa.yaml new file mode 100644 index 0000000..0497b1f --- /dev/null +++ b/deploy/job-queue/task-sa.yaml @@ -0,0 +1,7 @@ +# -- Task SA: used by environment pods, only needs anyuid SCC to run as root. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: harbor-task + labels: + app: harbor diff --git a/deploy/minio/api-route.yaml b/deploy/minio/api-route.yaml new file mode 100644 index 0000000..dfc974f --- /dev/null +++ b/deploy/minio/api-route.yaml @@ -0,0 +1,18 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio-api +spec: + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + port: + targetPort: 9000 + to: + kind: Service + name: harbor-minio + weight: 100 + wildcardPolicy: None diff --git a/deploy/minio/console-route.yaml b/deploy/minio/console-route.yaml new file mode 100644 index 0000000..c95a8d2 --- /dev/null +++ b/deploy/minio/console-route.yaml @@ -0,0 +1,18 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio-console +spec: + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + port: + targetPort: 9001 + to: + kind: Service + name: harbor-minio + weight: 100 + wildcardPolicy: None diff --git a/deploy/minio/deployment.yaml b/deploy/minio/deployment.yaml new file mode 100644 index 0000000..dd40f24 --- /dev/null +++ b/deploy/minio/deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio +spec: + replicas: 1 + selector: + matchLabels: + app: harbor-minio + component: minio + template: + metadata: + labels: + app: harbor-minio + component: minio + spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + args: + - server + - /data + - --console-address + - :9001 + envFrom: + - secretRef: + name: harbor-minio + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 10 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 250m + memory: 512Mi + volumeMounts: + - mountPath: /data + name: data + volumes: + - name: data + persistentVolumeClaim: + claimName: harbor-minio diff --git a/deploy/minio/kustomization.yaml b/deploy/minio/kustomization.yaml new file mode 100644 index 0000000..b9ead0b --- /dev/null +++ b/deploy/minio/kustomization.yaml @@ -0,0 +1,11 @@ +resources: + - ./pvc.yaml + - ./secret.yaml + - ./deployment.yaml + - ./service.yaml + - ./api-route.yaml + - ./console-route.yaml + +labels: + - pairs: + app: harbor-minio \ No newline at end of file diff --git a/deploy/minio/pvc.yaml b/deploy/minio/pvc.yaml new file mode 100644 index 0000000..7af17c6 --- /dev/null +++ b/deploy/minio/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 400Gi + storageClassName: gp3 + volumeMode: Filesystem diff --git a/deploy/minio/secret.yaml b/deploy/minio/secret.yaml new file mode 100644 index 0000000..122b7f8 --- /dev/null +++ b/deploy/minio/secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio +type: Opaque +stringData: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin diff --git a/deploy/minio/service.yaml b/deploy/minio/service.yaml new file mode 100644 index 0000000..6847bcd --- /dev/null +++ b/deploy/minio/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio +spec: + ports: + - name: api + port: 9000 + protocol: TCP + targetPort: 9000 + - name: console + port: 9001 + protocol: TCP + targetPort: 9001 + selector: + app: harbor-minio + component: minio + type: ClusterIP diff --git a/src/coding_agent_bench/job.py b/src/coding_agent_bench/job.py index 2f193eb..35e9c55 100644 --- a/src/coding_agent_bench/job.py +++ b/src/coding_agent_bench/job.py @@ -109,7 +109,7 @@ def _job_spec( "name": "OPENROUTER_API_KEY", "valueFrom": { "secretKeyRef": { - "name": "openrouter-api-key", + "name": "job-queue-secret", "key": "OPENROUTER_API_KEY", "optional": True, } diff --git a/tests/openrouter/test_job_spec.py b/tests/openrouter/test_job_spec.py index 45a786b..65ff130 100644 --- a/tests/openrouter/test_job_spec.py +++ b/tests/openrouter/test_job_spec.py @@ -11,7 +11,7 @@ def test_job_spec_injects_openrouter_secret_only_when_openrouter(): env = _env_by_name(job._job_spec(["echo", "hi"], openrouter=True)) assert "OPENROUTER_API_KEY" in env ref = env["OPENROUTER_API_KEY"]["valueFrom"]["secretKeyRef"] - assert ref["name"] == "openrouter-api-key" + assert ref["name"] == "job-queue-secret" assert ref["key"] == "OPENROUTER_API_KEY" assert ref["optional"] is True From e5b70f40d686ca8b88c8bce9919e4603fe809b7e Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Mon, 28 Sep 2026 16:53:55 -0400 Subject: [PATCH 02/31] =?UTF-8?q?=F0=9F=91=B7=20Add=20openshift=20deploy?= =?UTF-8?q?=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..fd4d576 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,55 @@ +name: Deploy to OpenShift + +on: + pull_request: + types: [closed] + branches: + - STAGE + push: + tags: + - 'v*' + +permissions: + contents: read + +jobs: + deploy: + if: >- + (github.event_name == 'pull_request' && github.event.pull_request.merged == true) + || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) + runs-on: ubuntu-latest + environment: ${{ github.event_name == 'pull_request' && 'stage' || 'production' }} + env: + NAMESPACE: ${{ github.event_name == 'pull_request' && vars.STAGE_NAMESPACE || vars.PROD_NAMESPACE }} + OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} + OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Log in to OpenShift + uses: redhat-actions/oc-login@v1 + with: + openshift_server_url: ${{ env.OPENSHIFT_SERVER }} + openshift_token: ${{ env.OPENSHIFT_TOKEN }} + namespace: ${{ env.NAMESPACE }} + + - name: Verify application Secrets + shell: bash + run: | + set -euo pipefail + + test -n "$NAMESPACE" + for secret in job-queue-secret nebius-secret intake-poller-secret intake-poller-google-sa; do + oc -n "$NAMESPACE" get secret "$secret" >/dev/null + done + + - name: Apply job queue + run: oc apply -k deploy/job-queue -n "$NAMESPACE" + + - name: Wait for job queue rollout + run: oc rollout status deployment/job-queue -n "$NAMESPACE" --timeout=10m + + - name: Apply intake poller + run: oc apply -k deploy/intake-poller -n "$NAMESPACE" From b4e95b394e6d55312a1397dca4e7058e71faaac2 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Mon, 28 Sep 2026 16:57:08 -0400 Subject: [PATCH 03/31] =?UTF-8?q?=F0=9F=91=B7=20Add=20workflow=20for=20sta?= =?UTF-8?q?ge=20sync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync-stage.yml | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/sync-stage.yml diff --git a/.github/workflows/sync-stage.yml b/.github/workflows/sync-stage.yml new file mode 100644 index 0000000..6e52c5e --- /dev/null +++ b/.github/workflows/sync-stage.yml @@ -0,0 +1,33 @@ +name: Sync stage with main + +on: + pull_request: + branches: + - main + types: + - closed + +permissions: + contents: write + +concurrency: + group: sync-stage + cancel-in-progress: false + +jobs: + sync: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Check out stage + uses: actions/checkout@v4 + with: + ref: stage + fetch-depth: 0 + + - name: Merge main into stage + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git merge origin/main --no-edit + git push origin HEAD:stage From fd4a30da9522efd1f656b8abab561581b4bc29ac Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Mon, 28 Sep 2026 17:21:33 -0400 Subject: [PATCH 04/31] =?UTF-8?q?=F0=9F=92=9A=20Add=20MinIO=20deployment?= =?UTF-8?q?=20to=20deploy=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fd4d576..89786e8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -45,6 +45,12 @@ jobs: oc -n "$NAMESPACE" get secret "$secret" >/dev/null done + - name: Apply MinIO + run: oc apply -k deploy/job-queue -n "$NAMESPACE" + + - name: Wait for MinIO rollout + run: oc rollout status deployment/harbor-minio -n "$NAMESPACE" --timeout=10m + - name: Apply job queue run: oc apply -k deploy/job-queue -n "$NAMESPACE" From abaec82f9a3e8013ab36826141c3c218bc055b4c Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Mon, 28 Sep 2026 17:33:00 -0400 Subject: [PATCH 05/31] =?UTF-8?q?=F0=9F=93=9D=20Update=20documentation=20f?= =?UTF-8?q?or=20new=20deployment=20setup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 2 +- README.md | 149 ++++++++------------ deploy/README.md | 24 +--- deploy/job-queue/nebius-secret.example.yaml | 4 +- 4 files changed, 68 insertions(+), 111 deletions(-) diff --git a/.env.example b/.env.example index c7e609e..bed854e 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,6 @@ NEBIUS_ENABLED=0 # NEBIUS_TENANT_ID= # NEBIUS_SERVICE_ACCOUNT_ID= # NEBIUS_SUBNET_ID= -# NEBIUS_INSTANCE_NAME_PREFIX=cab-worker +# NEBIUS_INSTANCE_NAME_PREFIX=job-queue-worker # NEBIUS_IDLE_TIMEOUT_SECONDS=600 # HF_TOKEN= diff --git a/README.md b/README.md index 5051e40..3103801 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Reproducible benchmarks for coding agents and models using Harbor - [Set up the service](#set-up-the-service) - [Use the service](#use-the-service) - [(Optional) Connect to Nebius](#optional-connect-to-nebius) + - [(Optional) Set Up the Intake Poller](#optional-set-up-the-intake-poller) - [Harbor Command Examples](#harbor-command-examples) - [Claude Code vLLM](#claude-code-vllm) - [Codex vLLM](#codex-vllm) @@ -207,91 +208,39 @@ sequenceDiagram ### Set up the service -1. Log in to your cluster and project: +1. Log in to your cluster: + ```sh oc login --server= --token= - oc project - ``` -2. Create the MinIO service for artifact storage: - ```sh - oc apply -f deploy/harbor-minio.yml - ``` - Note: the default username and password are `(minioadmin, minioadmin)`. - You can update this in the deployment file if needed. -3. Create the orchestrator and task service accounts: - ```sh - oc apply -f deploy/harbor-orchestrator-sa.yml - oc apply -f deploy/harbor-task-sa.yml ``` -4. Create a secret file named `job-queue-secret` with the queue service's - `API_KEY` and any queue or Nebius settings, then apply it: - ```yaml - apiVersion: v1 - kind: Secret - metadata: - name: job-queue-secret - stringData: - API_KEY: - type: Opaque - ``` - If the intake CronJob is deployed, create its separate poller secret: - ```yaml - apiVersion: v1 - kind: Secret - metadata: - name: intake-poller-secret - stringData: - JOB_QUEUE_URL: https:// - GOOGLE_SHEET_ID: - SENDER_EMAIL: ace-model-evals@redhat.com - AUTO_APPROVE: 'false' - type: Opaque - ``` -5. Create the queue service: + +2. Copy and fill in the Secret templates locally. Do not commit the resulting files: + ```sh - oc apply -f deploy/job-queue-service.yml - ``` -6. (Optional) To run jobs against OpenRouter (`server_url: openrouter`), create - an `openrouter-api-key` secret. Job pods mount it automatically (it is - optional, so non-OpenRouter jobs are unaffected): - ```yaml - apiVersion: v1 - kind: Secret - metadata: - name: openrouter-api-key - stringData: - OPENROUTER_API_KEY: - type: Opaque + cp deploy/job-queue/secret.example.yaml deploy/job-queue/secret.yaml + cp deploy/job-queue/nebius-secret.example.yaml deploy/job-queue/nebius-secret.yaml ``` - The queue service itself also needs `OPENROUTER_API_KEY` in its environment - to validate OpenRouter jobs at request time. Add it to `job-queue-secret` - (which the service already loads) or `envFrom` the `openrouter-api-key` - secret in `deploy/job-queue-service.yml`. - - The queue listens on HTTPS inside the cluster. OpenShift's service-serving - certificate operator creates the `job-queue-tls` Secret referenced by the - Deployment, and the Route uses re-encryption so traffic remains encrypted - from the router to the queue pod. Wait for that Secret to appear before - troubleshooting pod startup: + + If you are not using Nebius, you still need to create the secret, but you can leave the default values and they will be ignored. + +3. Deploy the MinIO service to store job artifacts: + ```sh - oc get secret job-queue-tls + oc apply -k deploy/minio -n ``` -Get the route for the deployed service: +4. Deploy the Job Queue service: -```sh -oc get route job-queue-route --output jsonpath='{.spec.host}' -``` - -Set `JOB_QUEUE_URL` in `intake-poller-secret` to this HTTPS route before -applying `deploy/intake-cronjob.yml`. + ```sh + oc apply -k deploy/job-queue -n + ``` -Check that the application is live by visiting the docs: +5. Get the route for the deployed API service: -```sh -export JOB_QUEUE_URL="https://$(oc get route job-queue-route --output jsonpath='{.spec.host}')" -open $JOB_QUEUE_URL/docs -``` + ```sh + export JOB_QUEUE_URL="https://$(oc get route job-queue-route -n --output jsonpath='{.spec.host}')" + open $JOB_QUEUE_URL/docs + ``` ### Use the service @@ -355,30 +304,50 @@ nebius iam auth-public-key generate \ --output ~/.nebius/$SA_ID-credentials.json ``` -Once the service account is created, you can update your job queue secret with the following environment variables needed for Nebius: +Once the service account is created, you can copy and fill in the values in [`deploy/job-queue/nebius-secret.example.yaml`](./deploy/job-queue/nebius-secret.example.yaml): + +```sh +cp deploy/job-queue/nebius-secret.example.yaml deploy/job-queue/nebius-secret.yaml +``` ```yaml -apiVersion: v1 kind: Secret metadata: - name: job-queue-secret + name: nebius-secret +type: Opaque stringData: - API_KEY: - NEBIUS_ENABLED: '1' + NEBIUS_ENABLED: "1" NEBIUS_SERVICE_ACCOUNT_CREDS: | - NEBIUS_PARENT_ID: - NEBIUS_TENANT_ID: - NEBIUS_SERVICE_ACCOUNT_ID: - NEBIUS_SUBNET_ID: + NEBIUS_USER: + NEBIUS_PARENT_ID: + NEBIUS_TENANT_ID: + NEBIUS_SERVICE_ACCOUNT_ID: + NEBIUS_SUBNET_ID: NEBIUS_INSTANCE_NAME_PREFIX: job-queue-worker - NEBIUS_IDLE_TIMEOUT_SECONDS: '600' - HF_TOKEN: -type: Opaque + NEBIUS_IDLE_TIMEOUT_SECONDS: "600" + HF_TOKEN: ``` When creating a job, set `server_url` to `nebius-` to use a managed Nebius instance with the specified GPU resource (e.g. `nebius-h200`, `nebius-b200`). Available resources are defined in `RESOURCE_CONFIG_REGISTRY`. +### (Optional) Set Up the Intake Poller + +The intake poller is an optional CronJob to pull requests from a Google Sheet and submit them to the job queue. +You can read more about this service in the [intake poller docs](./deploy/README.md#intake-poller). + +First, copy the secret in [`deploy/intake-poller/secret.example.yaml`](./deploy/intake-poller/secret.example.yaml) and fill in the values according to the [intake poller docs](./deploy/README.md#intake-poller). Do not commit this file. + +```sh +cp deploy/intake-poller/secret.example.yaml deploy/intake-poller/secret.yaml +``` + +Then create the CronJob: + +```sh +oc apply -k deploy/intake-poller -n +``` + ## Harbor Command Examples **Prerequisites:** @@ -673,7 +642,7 @@ oc project Create ServiceAccounts and RoleBindings to run tasks: ```bash -oc apply -f deploy/harbor-task-sa.yml +oc apply -f deploy/job-queue/harbor-task-sa.yml ``` Then in your `harbor` command, add the flag: @@ -694,14 +663,14 @@ oc project Create ServiceAccounts and RoleBindings to run tasks and orchestrate: ```bash -oc apply -f deploy/harbor-task-sa.yml -oc apply -f deploy/harbor-orchestrator-sa.yml +oc apply -f deploy/job-queue/task-sa.yml +oc apply -f deploy/job-queue/orchestrator-sa.yml ``` Create a MinIO deployment to store your job results: ```bash -oc apply -f deploy/harbor-minio.yml +oc apply -k deploy/minio ``` Using the CLI, start a job with the `--remote` flag enabled and set `--environment openshift`, e.g.: diff --git a/deploy/README.md b/deploy/README.md index da6ad37..33c16ff 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -179,14 +179,19 @@ development. | `JOB_QUEUE_URL` | HTTPS URL for the queue API. Use the cluster's TLS/mTLS endpoint; the poller fails closed instead of using plaintext HTTP. | | `SENDER_EMAIL` | Address notification emails are sent from. Set it to `ace-model-evals@redhat.com`. | | `AUTO_APPROVE` | `"true"` to auto-submit rows with a blank status, otherwise `"false"` | +| `SMTP_HOST` | Host for the SMTP server to send notifications through | +| `SMTP_PORT` | Port for the SMTP server to send notifications through | +| `SMTP_STARTTLS` | Set to `"true"` when the server requires STARTTLS, otherwise `"false"` | +| `ALLOW_INSECURE_QUEUE_HTTP` | Set to `"true"` when the queue is served over HTTP (e.g. locally), otherwise `"false"` | +| `service-account.json` | Content of a GCP service account that has read access to the Google Sheet. | ### Queue TLS -`deploy/job-queue-service.yml` enables OpenShift's service-serving certificate +`deploy/job-queue/deployment.yaml` enables OpenShift's service-serving certificate operator with the `service.beta.openshift.io/serving-cert-secret-name` annotation. The operator creates `job-queue-tls` with `tls.crt`, `tls.key`, and the service CA; the queue mounts that Secret and Uvicorn serves HTTPS on port -8443. The Service exposes it as port 443 and the Route uses `reencrypt` +1. The Service exposes it as port 443 and the Route uses `reencrypt` termination, keeping router-to-pod traffic encrypted as well. Apply the manifest before starting the poller and wait for `job-queue-tls` to be created. @@ -199,21 +204,6 @@ resource token such as `nebius-h200` (or `nebius-b200x8`). The queue service validates that token, provisions the instance, and supplies its endpoint after approval; the requester never needs to know that endpoint. -**`intake-poller-google-sa`** — the Google service-account credential mounted -at `/etc/google/service-account.json` for Sheets access. Notification email is -sent through the internal SMTP relay, so no Gmail mailbox credential or -domain-wide delegation is required. - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: intake-poller-google-sa -type: Opaque -stringData: - service-account.json: -``` - The CronJob sends notifications through `smtp.corp.redhat.com` on port 25. Set `SMTP_HOST` and `SMTP_PORT` on the poller when a different internal relay is required. Set `SMTP_STARTTLS=true` when that relay requires STARTTLS. The diff --git a/deploy/job-queue/nebius-secret.example.yaml b/deploy/job-queue/nebius-secret.example.yaml index 75b6f19..9d22aec 100644 --- a/deploy/job-queue/nebius-secret.example.yaml +++ b/deploy/job-queue/nebius-secret.example.yaml @@ -7,13 +7,11 @@ stringData: NEBIUS_ENABLED: "0" NEBIUS_SERVICE_ACCOUNT_CREDS: | - # Use NEBIUS_SERVICE_ACCOUNT_CREDS_PATH instead when the file is mounted in the pod. - NEBIUS_SERVICE_ACCOUNT_CREDS_PATH: "" NEBIUS_USER: NEBIUS_PARENT_ID: NEBIUS_TENANT_ID: NEBIUS_SERVICE_ACCOUNT_ID: NEBIUS_SUBNET_ID: - NEBIUS_INSTANCE_NAME_PREFIX: cab-worker + NEBIUS_INSTANCE_NAME_PREFIX: job-queue-worker NEBIUS_IDLE_TIMEOUT_SECONDS: "600" HF_TOKEN: From 4322d456961e76aa5069328d0ad4c4c11f843e9a Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Mon, 28 Sep 2026 17:41:47 -0400 Subject: [PATCH 06/31] =?UTF-8?q?=F0=9F=90=9B=20Add=20missing=20key=20sele?= =?UTF-8?q?ction=20for=20google=20SA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/intake-poller/cronjob.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/intake-poller/cronjob.yaml b/deploy/intake-poller/cronjob.yaml index 1d0e983..8728b14 100644 --- a/deploy/intake-poller/cronjob.yaml +++ b/deploy/intake-poller/cronjob.yaml @@ -106,7 +106,10 @@ spec: volumes: - name: google-sa-key secret: - secretName: intake-poller-google-sa + secretName: intake-poller-secret + items: + - key: service-account.json + path: service-account.json - name: service-ca configMap: name: intake-poller-ca From d3bc57c9719643a6ff192148a5573f8584cac79a Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 09:16:22 -0400 Subject: [PATCH 07/31] =?UTF-8?q?=F0=9F=9A=80=20Fix=20image=20tags=20and?= =?UTF-8?q?=20envvars=20using=20overlays?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 6 ++--- .gitignore | 6 ++--- README.md | 22 +++++++++---------- deploy/README.md | 6 ++--- deploy/intake-poller/{ => base}/cronjob.yaml | 0 .../{ => base}/intake-poller-ca.yaml | 0 .../{ => base}/kustomization.yaml | 0 .../{ => base}/secret.example.yaml | 1 + .../overlays/prod/kustomization.yaml | 18 +++++++++++++++ .../overlays/stage/kustomization.yaml | 18 +++++++++++++++ deploy/job-queue/{ => base}/deployment.yaml | 2 +- .../job-queue/{ => base}/kustomization.yaml | 0 .../{ => base}/nebius-secret.example.yaml | 0 .../{ => base}/orchestrator-anyuid.yaml | 0 .../{ => base}/orchestrator-role.yaml | 0 .../{ => base}/orchestrator-rolebinding.yaml | 0 .../job-queue/{ => base}/orchestrator-sa.yaml | 0 deploy/job-queue/{ => base}/pvc.yaml | 0 deploy/job-queue/{ => base}/route.yaml | 0 .../job-queue/{ => base}/secret.example.yaml | 1 + deploy/job-queue/{ => base}/service.yaml | 0 deploy/job-queue/{ => base}/task-anyuid.yaml | 0 deploy/job-queue/{ => base}/task-sa.yaml | 0 .../overlays/prod/kustomization.yaml | 18 +++++++++++++++ .../overlays/stage/kustomization.yaml | 18 +++++++++++++++ deploy/minio/{ => base}/api-route.yaml | 0 deploy/minio/{ => base}/console-route.yaml | 0 deploy/minio/{ => base}/deployment.yaml | 0 deploy/minio/{ => base}/kustomization.yaml | 0 deploy/minio/{ => base}/pvc.yaml | 0 deploy/minio/{ => base}/secret.yaml | 1 + deploy/minio/{ => base}/service.yaml | 0 deploy/minio/overlays/prod/kustomization.yaml | 14 ++++++++++++ .../minio/overlays/stage/kustomization.yaml | 14 ++++++++++++ 34 files changed, 124 insertions(+), 21 deletions(-) rename deploy/intake-poller/{ => base}/cronjob.yaml (100%) rename deploy/intake-poller/{ => base}/intake-poller-ca.yaml (100%) rename deploy/intake-poller/{ => base}/kustomization.yaml (100%) rename deploy/intake-poller/{ => base}/secret.example.yaml (95%) create mode 100644 deploy/intake-poller/overlays/prod/kustomization.yaml create mode 100644 deploy/intake-poller/overlays/stage/kustomization.yaml rename deploy/job-queue/{ => base}/deployment.yaml (97%) rename deploy/job-queue/{ => base}/kustomization.yaml (100%) rename deploy/job-queue/{ => base}/nebius-secret.example.yaml (100%) rename deploy/job-queue/{ => base}/orchestrator-anyuid.yaml (100%) rename deploy/job-queue/{ => base}/orchestrator-role.yaml (100%) rename deploy/job-queue/{ => base}/orchestrator-rolebinding.yaml (100%) rename deploy/job-queue/{ => base}/orchestrator-sa.yaml (100%) rename deploy/job-queue/{ => base}/pvc.yaml (100%) rename deploy/job-queue/{ => base}/route.yaml (100%) rename deploy/job-queue/{ => base}/secret.example.yaml (94%) rename deploy/job-queue/{ => base}/service.yaml (100%) rename deploy/job-queue/{ => base}/task-anyuid.yaml (100%) rename deploy/job-queue/{ => base}/task-sa.yaml (100%) create mode 100644 deploy/job-queue/overlays/prod/kustomization.yaml create mode 100644 deploy/job-queue/overlays/stage/kustomization.yaml rename deploy/minio/{ => base}/api-route.yaml (100%) rename deploy/minio/{ => base}/console-route.yaml (100%) rename deploy/minio/{ => base}/deployment.yaml (100%) rename deploy/minio/{ => base}/kustomization.yaml (100%) rename deploy/minio/{ => base}/pvc.yaml (100%) rename deploy/minio/{ => base}/secret.yaml (91%) rename deploy/minio/{ => base}/service.yaml (100%) create mode 100644 deploy/minio/overlays/prod/kustomization.yaml create mode 100644 deploy/minio/overlays/stage/kustomization.yaml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 89786e8..d515aa6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -46,16 +46,16 @@ jobs: done - name: Apply MinIO - run: oc apply -k deploy/job-queue -n "$NAMESPACE" + run: oc apply -k deploy/minio/overlays/prod -n "$NAMESPACE" - name: Wait for MinIO rollout run: oc rollout status deployment/harbor-minio -n "$NAMESPACE" --timeout=10m - name: Apply job queue - run: oc apply -k deploy/job-queue -n "$NAMESPACE" + run: oc apply -k deploy/job-queue/overlays/prod -n "$NAMESPACE" - name: Wait for job queue rollout run: oc rollout status deployment/job-queue -n "$NAMESPACE" --timeout=10m - name: Apply intake poller - run: oc apply -k deploy/intake-poller -n "$NAMESPACE" + run: oc apply -k deploy/intake-poller/overlays/prod -n "$NAMESPACE" diff --git a/.gitignore b/.gitignore index 47dad25..077bc65 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,9 @@ models.json config.toml data jobs.db -deploy/job-queue/secret.yaml -deploy/job-queue/nebius-secret.yaml -deploy/intake-poller/secret.yaml +deploy/job-queue/base/secret.yaml +deploy/job-queue/base/nebius-secret.yaml +deploy/intake-poller/base/secret.yaml # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index 3103801..0546541 100644 --- a/README.md +++ b/README.md @@ -217,8 +217,8 @@ sequenceDiagram 2. Copy and fill in the Secret templates locally. Do not commit the resulting files: ```sh - cp deploy/job-queue/secret.example.yaml deploy/job-queue/secret.yaml - cp deploy/job-queue/nebius-secret.example.yaml deploy/job-queue/nebius-secret.yaml + cp deploy/job-queue/base/secret.example.yaml deploy/job-queue/base/secret.yaml + cp deploy/job-queue/base/nebius-secret.example.yaml deploy/job-queue/base/nebius-secret.yaml ``` If you are not using Nebius, you still need to create the secret, but you can leave the default values and they will be ignored. @@ -226,13 +226,13 @@ sequenceDiagram 3. Deploy the MinIO service to store job artifacts: ```sh - oc apply -k deploy/minio -n + oc apply -k deploy/minio/overlays/prod -n ``` 4. Deploy the Job Queue service: ```sh - oc apply -k deploy/job-queue -n + oc apply -k deploy/job-queue/overlays/prod -n ``` 5. Get the route for the deployed API service: @@ -304,10 +304,10 @@ nebius iam auth-public-key generate \ --output ~/.nebius/$SA_ID-credentials.json ``` -Once the service account is created, you can copy and fill in the values in [`deploy/job-queue/nebius-secret.example.yaml`](./deploy/job-queue/nebius-secret.example.yaml): +Once the service account is created, you can copy and fill in the values in [`deploy/job-queue/base/nebius-secret.example.yaml`](./deploy/job-queue/base/nebius-secret.example.yaml): ```sh -cp deploy/job-queue/nebius-secret.example.yaml deploy/job-queue/nebius-secret.yaml +cp deploy/job-queue/base/nebius-secret.example.yaml deploy/job-queue/base/nebius-secret.yaml ``` ```yaml @@ -345,7 +345,7 @@ cp deploy/intake-poller/secret.example.yaml deploy/intake-poller/secret.yaml Then create the CronJob: ```sh -oc apply -k deploy/intake-poller -n +oc apply -k deploy/intake-poller/overlays/prod -n ``` ## Harbor Command Examples @@ -642,7 +642,7 @@ oc project Create ServiceAccounts and RoleBindings to run tasks: ```bash -oc apply -f deploy/job-queue/harbor-task-sa.yml +oc apply -f deploy/job-queue/base/harbor-task-sa.yml ``` Then in your `harbor` command, add the flag: @@ -663,14 +663,14 @@ oc project Create ServiceAccounts and RoleBindings to run tasks and orchestrate: ```bash -oc apply -f deploy/job-queue/task-sa.yml -oc apply -f deploy/job-queue/orchestrator-sa.yml +oc apply -f deploy/job-queue/base/task-sa.yml +oc apply -f deploy/job-queue/base/orchestrator-sa.yml ``` Create a MinIO deployment to store your job results: ```bash -oc apply -k deploy/minio +oc apply -k deploy/minio/overlays/prod ``` Using the CLI, start a job with the `--remote` flag enabled and set `--environment openshift`, e.g.: diff --git a/deploy/README.md b/deploy/README.md index 33c16ff..23b9cf5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -234,9 +234,9 @@ namespace: ```sh oc project -oc apply -f deploy/job-queue/secret.yaml -oc apply -f deploy/job-queue/nebius-secret.yaml -oc apply -f deploy/intake-poller/secret.yaml +oc apply -f deploy/job-queue/base/secret.yaml +oc apply -f deploy/job-queue/base/nebius-secret.yaml +oc apply -f deploy/intake-poller/base/secret.yaml ``` Repeat these commands for both stage and production. The CI workflow checks for diff --git a/deploy/intake-poller/cronjob.yaml b/deploy/intake-poller/base/cronjob.yaml similarity index 100% rename from deploy/intake-poller/cronjob.yaml rename to deploy/intake-poller/base/cronjob.yaml diff --git a/deploy/intake-poller/intake-poller-ca.yaml b/deploy/intake-poller/base/intake-poller-ca.yaml similarity index 100% rename from deploy/intake-poller/intake-poller-ca.yaml rename to deploy/intake-poller/base/intake-poller-ca.yaml diff --git a/deploy/intake-poller/kustomization.yaml b/deploy/intake-poller/base/kustomization.yaml similarity index 100% rename from deploy/intake-poller/kustomization.yaml rename to deploy/intake-poller/base/kustomization.yaml diff --git a/deploy/intake-poller/secret.example.yaml b/deploy/intake-poller/base/secret.example.yaml similarity index 95% rename from deploy/intake-poller/secret.example.yaml rename to deploy/intake-poller/base/secret.example.yaml index 2a67842..7711951 100644 --- a/deploy/intake-poller/secret.example.yaml +++ b/deploy/intake-poller/base/secret.example.yaml @@ -12,5 +12,6 @@ stringData: SMTP_PORT: "25" SMTP_STARTTLS: "false" ALLOW_INSECURE_QUEUE_HTTP: "false" + ENVIRONMENT: env service-account.json: |- diff --git a/deploy/intake-poller/overlays/prod/kustomization.yaml b/deploy/intake-poller/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..8aab12c --- /dev/null +++ b/deploy/intake-poller/overlays/prod/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +images: + - name: ghcr.io/redhat-et/coding_agent_bench + newTag: v0.2.6 + +patches: + - target: + kind: Secret + name: intake-poller-secret + patch: | + - op: replace + path: /stringData/ENVIRONMENT + value: prod diff --git a/deploy/intake-poller/overlays/stage/kustomization.yaml b/deploy/intake-poller/overlays/stage/kustomization.yaml new file mode 100644 index 0000000..80f8f2c --- /dev/null +++ b/deploy/intake-poller/overlays/stage/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +images: + - name: ghcr.io/redhat-et/coding_agent_bench + newTag: latest + +patches: + - target: + kind: Secret + name: intake-poller-secret + patch: | + - op: replace + path: /stringData/ENVIRONMENT + value: stage diff --git a/deploy/job-queue/deployment.yaml b/deploy/job-queue/base/deployment.yaml similarity index 97% rename from deploy/job-queue/deployment.yaml rename to deploy/job-queue/base/deployment.yaml index c896d9a..5944172 100644 --- a/deploy/job-queue/deployment.yaml +++ b/deploy/job-queue/base/deployment.yaml @@ -23,7 +23,7 @@ spec: terminationGracePeriodSeconds: 60 serviceAccountName: harbor-orchestrator containers: - - image: ghcr.io/redhat-et/coding_agent_bench:v0.2.6 + - image: ghcr.io/redhat-et/coding_agent_bench:tag name: job-queue command: ["/bin/sh", "-c"] args: diff --git a/deploy/job-queue/kustomization.yaml b/deploy/job-queue/base/kustomization.yaml similarity index 100% rename from deploy/job-queue/kustomization.yaml rename to deploy/job-queue/base/kustomization.yaml diff --git a/deploy/job-queue/nebius-secret.example.yaml b/deploy/job-queue/base/nebius-secret.example.yaml similarity index 100% rename from deploy/job-queue/nebius-secret.example.yaml rename to deploy/job-queue/base/nebius-secret.example.yaml diff --git a/deploy/job-queue/orchestrator-anyuid.yaml b/deploy/job-queue/base/orchestrator-anyuid.yaml similarity index 100% rename from deploy/job-queue/orchestrator-anyuid.yaml rename to deploy/job-queue/base/orchestrator-anyuid.yaml diff --git a/deploy/job-queue/orchestrator-role.yaml b/deploy/job-queue/base/orchestrator-role.yaml similarity index 100% rename from deploy/job-queue/orchestrator-role.yaml rename to deploy/job-queue/base/orchestrator-role.yaml diff --git a/deploy/job-queue/orchestrator-rolebinding.yaml b/deploy/job-queue/base/orchestrator-rolebinding.yaml similarity index 100% rename from deploy/job-queue/orchestrator-rolebinding.yaml rename to deploy/job-queue/base/orchestrator-rolebinding.yaml diff --git a/deploy/job-queue/orchestrator-sa.yaml b/deploy/job-queue/base/orchestrator-sa.yaml similarity index 100% rename from deploy/job-queue/orchestrator-sa.yaml rename to deploy/job-queue/base/orchestrator-sa.yaml diff --git a/deploy/job-queue/pvc.yaml b/deploy/job-queue/base/pvc.yaml similarity index 100% rename from deploy/job-queue/pvc.yaml rename to deploy/job-queue/base/pvc.yaml diff --git a/deploy/job-queue/route.yaml b/deploy/job-queue/base/route.yaml similarity index 100% rename from deploy/job-queue/route.yaml rename to deploy/job-queue/base/route.yaml diff --git a/deploy/job-queue/secret.example.yaml b/deploy/job-queue/base/secret.example.yaml similarity index 94% rename from deploy/job-queue/secret.example.yaml rename to deploy/job-queue/base/secret.example.yaml index b04d41a..533f1fe 100644 --- a/deploy/job-queue/secret.example.yaml +++ b/deploy/job-queue/base/secret.example.yaml @@ -10,3 +10,4 @@ stringData: ANTHROPIC_API_KEY: OPENAI_API_KEY: OPENROUTER_API_KEY: + ENVIRONMENT: env diff --git a/deploy/job-queue/service.yaml b/deploy/job-queue/base/service.yaml similarity index 100% rename from deploy/job-queue/service.yaml rename to deploy/job-queue/base/service.yaml diff --git a/deploy/job-queue/task-anyuid.yaml b/deploy/job-queue/base/task-anyuid.yaml similarity index 100% rename from deploy/job-queue/task-anyuid.yaml rename to deploy/job-queue/base/task-anyuid.yaml diff --git a/deploy/job-queue/task-sa.yaml b/deploy/job-queue/base/task-sa.yaml similarity index 100% rename from deploy/job-queue/task-sa.yaml rename to deploy/job-queue/base/task-sa.yaml diff --git a/deploy/job-queue/overlays/prod/kustomization.yaml b/deploy/job-queue/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..ad1755f --- /dev/null +++ b/deploy/job-queue/overlays/prod/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +images: + - name: ghcr.io/redhat-et/coding_agent_bench + newTag: v0.2.6 + +patches: + - target: + kind: Secret + name: job-queue-secret + patch: | + - op: replace + path: /stringData/ENVIRONMENT + value: prod diff --git a/deploy/job-queue/overlays/stage/kustomization.yaml b/deploy/job-queue/overlays/stage/kustomization.yaml new file mode 100644 index 0000000..888d81a --- /dev/null +++ b/deploy/job-queue/overlays/stage/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +images: + - name: ghcr.io/redhat-et/coding_agent_bench + newTag: latest + +patches: + - target: + kind: Secret + name: job-queue-secret + patch: | + - op: replace + path: /stringData/ENVIRONMENT + value: stage diff --git a/deploy/minio/api-route.yaml b/deploy/minio/base/api-route.yaml similarity index 100% rename from deploy/minio/api-route.yaml rename to deploy/minio/base/api-route.yaml diff --git a/deploy/minio/console-route.yaml b/deploy/minio/base/console-route.yaml similarity index 100% rename from deploy/minio/console-route.yaml rename to deploy/minio/base/console-route.yaml diff --git a/deploy/minio/deployment.yaml b/deploy/minio/base/deployment.yaml similarity index 100% rename from deploy/minio/deployment.yaml rename to deploy/minio/base/deployment.yaml diff --git a/deploy/minio/kustomization.yaml b/deploy/minio/base/kustomization.yaml similarity index 100% rename from deploy/minio/kustomization.yaml rename to deploy/minio/base/kustomization.yaml diff --git a/deploy/minio/pvc.yaml b/deploy/minio/base/pvc.yaml similarity index 100% rename from deploy/minio/pvc.yaml rename to deploy/minio/base/pvc.yaml diff --git a/deploy/minio/secret.yaml b/deploy/minio/base/secret.yaml similarity index 91% rename from deploy/minio/secret.yaml rename to deploy/minio/base/secret.yaml index 122b7f8..880abc3 100644 --- a/deploy/minio/secret.yaml +++ b/deploy/minio/base/secret.yaml @@ -9,3 +9,4 @@ type: Opaque stringData: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin + ENVIRONMENT: env diff --git a/deploy/minio/service.yaml b/deploy/minio/base/service.yaml similarity index 100% rename from deploy/minio/service.yaml rename to deploy/minio/base/service.yaml diff --git a/deploy/minio/overlays/prod/kustomization.yaml b/deploy/minio/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..f008fe9 --- /dev/null +++ b/deploy/minio/overlays/prod/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - target: + kind: Secret + name: harbor-minio + patch: | + - op: replace + path: /stringData/ENVIRONMENT + value: prod diff --git a/deploy/minio/overlays/stage/kustomization.yaml b/deploy/minio/overlays/stage/kustomization.yaml new file mode 100644 index 0000000..ec8493d --- /dev/null +++ b/deploy/minio/overlays/stage/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - target: + kind: Secret + name: harbor-minio + patch: | + - op: replace + path: /stringData/ENVIRONMENT + value: stage From 901248db5bf1b8925aa75963a1f3e7942286c434 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 09:22:38 -0400 Subject: [PATCH 08/31] =?UTF-8?q?=F0=9F=94=A7=20Update=20bumpversion=20con?= =?UTF-8?q?fig=20to=20point=20at=20new=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 354980c..5e90652 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -11,11 +11,11 @@ search = 'version = "{current_version}"' replace = 'version = "{new_version}"' [[tool.bumpversion.files]] -filename = "deploy/job-queue-service.yml" -search = "ghcr.io/redhat-et/coding_agent_bench:v{current_version}" -replace = "ghcr.io/redhat-et/coding_agent_bench:v{new_version}" +filename = "deploy/job-queue/overlays/prod/kustomization.yaml" +search = "v{current_version}" +replace = "v{new_version}" [[tool.bumpversion.files]] -filename = "deploy/intake-cronjob.yml" -search = "ghcr.io/redhat-et/coding_agent_bench:v{current_version}" -replace = "ghcr.io/redhat-et/coding_agent_bench:v{new_version}" +filename = "deploy/intake-poller/overlays/prod/kustomization.yaml" +search = "v{current_version}" +replace = "v{new_version}" From 19c7f6569c617a2a84a87794e1f361a7d24961a7 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 09:27:39 -0400 Subject: [PATCH 09/31] =?UTF-8?q?=F0=9F=91=B7=20Update=20build-push=20CI?= =?UTF-8?q?=20to=20push=20stage=20tag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-push.yml | 1 + deploy/intake-poller/overlays/stage/kustomization.yaml | 2 +- deploy/job-queue/overlays/stage/kustomization.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 38586f6..67b103b 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - stage - 'CICD*' tags: - 'v*' diff --git a/deploy/intake-poller/overlays/stage/kustomization.yaml b/deploy/intake-poller/overlays/stage/kustomization.yaml index 80f8f2c..cbc8a94 100644 --- a/deploy/intake-poller/overlays/stage/kustomization.yaml +++ b/deploy/intake-poller/overlays/stage/kustomization.yaml @@ -6,7 +6,7 @@ resources: images: - name: ghcr.io/redhat-et/coding_agent_bench - newTag: latest + newTag: stage patches: - target: diff --git a/deploy/job-queue/overlays/stage/kustomization.yaml b/deploy/job-queue/overlays/stage/kustomization.yaml index 888d81a..0a4c24c 100644 --- a/deploy/job-queue/overlays/stage/kustomization.yaml +++ b/deploy/job-queue/overlays/stage/kustomization.yaml @@ -6,7 +6,7 @@ resources: images: - name: ghcr.io/redhat-et/coding_agent_bench - newTag: latest + newTag: stage patches: - target: From d9a9cb5930da15bd57cff1e6f4fce44049e52d08 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:15:44 -0400 Subject: [PATCH 10/31] =?UTF-8?q?=F0=9F=90=9B=20Do=20not=20notify=20or=20u?= =?UTF-8?q?pdate=20intake=20in=20stage=20env?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../overlays/stage/kustomization.yaml | 7 ++++ src/coding_agent_bench/intake/poller.py | 24 +++++++---- tests/intake/test_poller.py | 40 +++++++++++++++++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/deploy/intake-poller/overlays/stage/kustomization.yaml b/deploy/intake-poller/overlays/stage/kustomization.yaml index cbc8a94..0ee1d64 100644 --- a/deploy/intake-poller/overlays/stage/kustomization.yaml +++ b/deploy/intake-poller/overlays/stage/kustomization.yaml @@ -9,6 +9,13 @@ images: newTag: stage patches: + - target: + kind: CronJob + name: intake-poller + patch: | + - op: add + path: /spec/suspend + value: true - target: kind: Secret name: intake-poller-secret diff --git a/src/coding_agent_bench/intake/poller.py b/src/coding_agent_bench/intake/poller.py index e39e700..657ab28 100644 --- a/src/coding_agent_bench/intake/poller.py +++ b/src/coding_agent_bench/intake/poller.py @@ -30,6 +30,11 @@ def _auto_approve_enabled() -> bool: return AUTO_APPROVE or os.environ.get("AUTO_APPROVE", "false").lower() == "true" +def _is_stage_environment() -> bool: + """Return whether the poller is running as a non-notifying stage test.""" + return os.environ.get("ENVIRONMENT", "stage").lower() != "prod" + + def _queue_verify() -> str | bool: """Return the CA bundle for verifying the queue's TLS certificate. @@ -110,6 +115,7 @@ def process_rows( ) -> None: """Process approved, in-flight, and pending-notification spreadsheet rows.""" rows = sheets.get_all_rows() + stage = _is_stage_environment() for i, row in enumerate(rows): row_num = i + 1 @@ -120,7 +126,7 @@ def process_rows( if status in TERMINAL_STATUSES: # Completed/failed rows remain eligible for one retry when the # queue state was persisted but the notification was not. - if status in (Status.COMPLETED.value, Status.FAILED.value) and ( + if not stage and status in (Status.COMPLETED.value, Status.FAILED.value) and ( row[Column.NOTIFIED_DONE].strip().upper() != "TRUE" ): _handle_inflight_row( @@ -132,9 +138,9 @@ def process_rows( if status == Status.APPROVED.value or (not status and _auto_approve_enabled()): _handle_new_row( sheets, row, row_num, api_base_url, api_key, - sender_email, + sender_email, notify=not stage, ) - elif status in (Status.QUEUED.value, Status.RUNNING.value): + elif not stage and status in (Status.QUEUED.value, Status.RUNNING.value): _handle_inflight_row( sheets, row, row_num, api_base_url, api_key, sender_email, @@ -150,6 +156,7 @@ def _handle_new_row( api_base_url: str, api_key: str, sender_email: str, + notify: bool = True, ) -> None: """Validate and submit one approved intake row, then notify its submitter.""" agent = row[Column.AGENT].strip() @@ -192,11 +199,12 @@ def _handle_new_row( sheets.update_cell(row_num, Column.JOB_ID, job_id) sheets.update_cell(row_num, Column.STATUS, Status.QUEUED.value) - try: - send_queued_email(email, agent, dataset, model_name, job_id, sender_email) - sheets.update_cell(row_num, Column.NOTIFIED_QUEUED, "TRUE") - except Exception: - logger.exception("Failed to send queued email for row %d", row_num) + if notify: + try: + send_queued_email(email, agent, dataset, model_name, job_id, sender_email) + sheets.update_cell(row_num, Column.NOTIFIED_QUEUED, "TRUE") + except Exception: + logger.exception("Failed to send queued email for row %d", row_num) def _handle_inflight_row( diff --git a/tests/intake/test_poller.py b/tests/intake/test_poller.py index 288aca3..cee1fd2 100644 --- a/tests/intake/test_poller.py +++ b/tests/intake/test_poller.py @@ -107,6 +107,46 @@ def test_empty_status_row_submitted_when_auto_approve(mock_httpx, mock_email): sheets.update_cell.assert_any_call(1, Column.JOB_ID, "uuid-456") +@patch("coding_agent_bench.intake.poller.send_queued_email") +@patch("coding_agent_bench.intake.poller.httpx") +def test_stage_submission_does_not_send_email(mock_httpx, mock_email, monkeypatch): + """Submit stage test rows without sending requester notifications.""" + monkeypatch.setenv("ENVIRONMENT", "stage") + mock_response = MagicMock() + mock_response.json.return_value = {"job_id": "uuid-stage"} + mock_httpx.post.return_value = mock_response + + sheets = MagicMock() + sheets.get_all_rows.return_value = [_make_row(STATUS=Status.APPROVED.value)] + + process_rows(sheets, "http://job-queue-service", "test-key", "bench@example.com") + + mock_email.assert_not_called() + sheets.update_cell.assert_any_call(1, Column.STATUS, Status.QUEUED.value) + sheets.update_cell.assert_any_call(1, Column.JOB_ID, "uuid-stage") + assert all( + call.args != (1, Column.NOTIFIED_QUEUED, "TRUE") + for call in sheets.update_cell.call_args_list + ) + + +@patch("coding_agent_bench.intake.poller.send_completed_email") +@patch("coding_agent_bench.intake.poller.httpx") +def test_stage_does_not_reconcile_inflight_rows(mock_httpx, mock_email, monkeypatch): + """Leave stage jobs non-terminal and avoid completion notifications.""" + monkeypatch.setenv("ENVIRONMENT", "stage") + sheets = MagicMock() + sheets.get_all_rows.return_value = [ + _make_row(STATUS=Status.QUEUED.value, JOB_ID="uuid-stage") + ] + + process_rows(sheets, "http://job-queue-service", "test-key", "bench@example.com") + + mock_httpx.get.assert_not_called() + mock_email.assert_not_called() + sheets.update_cell.assert_not_called() + + @patch("coding_agent_bench.intake.poller.send_queued_email") @patch("coding_agent_bench.intake.poller.httpx") def test_invalid_approved_row_marked_needs_review(mock_httpx, mock_email): From b40f1b74743b59953557c980eabe7f5e374ad152 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:20:29 -0400 Subject: [PATCH 11/31] =?UTF-8?q?=F0=9F=92=9A=20Fix=20issues=20in=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d515aa6..c098e28 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,7 +4,8 @@ on: pull_request: types: [closed] branches: - - STAGE + - stage + - main push: tags: - 'v*' @@ -20,6 +21,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ github.event_name == 'pull_request' && 'stage' || 'production' }} env: + OVERLAY: ${{ github.event_name == 'pull_request' && 'stage' || 'prod' }} NAMESPACE: ${{ github.event_name == 'pull_request' && vars.STAGE_NAMESPACE || vars.PROD_NAMESPACE }} OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} @@ -41,21 +43,21 @@ jobs: set -euo pipefail test -n "$NAMESPACE" - for secret in job-queue-secret nebius-secret intake-poller-secret intake-poller-google-sa; do + for secret in job-queue-secret nebius-secret intake-poller-secret; do oc -n "$NAMESPACE" get secret "$secret" >/dev/null done - name: Apply MinIO - run: oc apply -k deploy/minio/overlays/prod -n "$NAMESPACE" + run: oc apply -k deploy/minio/overlays/$OVERLAY -n "$NAMESPACE" - name: Wait for MinIO rollout run: oc rollout status deployment/harbor-minio -n "$NAMESPACE" --timeout=10m - name: Apply job queue - run: oc apply -k deploy/job-queue/overlays/prod -n "$NAMESPACE" + run: oc apply -k deploy/job-queue/overlays/$OVERLAY -n "$NAMESPACE" - name: Wait for job queue rollout run: oc rollout status deployment/job-queue -n "$NAMESPACE" --timeout=10m - name: Apply intake poller - run: oc apply -k deploy/intake-poller/overlays/prod -n "$NAMESPACE" + run: oc apply -k deploy/intake-poller/overlays/$OVERLAY -n "$NAMESPACE" From 0ec7cf27fa9c842d66948e83e43950e83b1c2d93 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:22:03 -0400 Subject: [PATCH 12/31] =?UTF-8?q?=F0=9F=90=9B=20Use=20secret.example.yaml?= =?UTF-8?q?=20for=20minio=20as=20well?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/minio/base/{secret.yaml => secret.example.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename deploy/minio/base/{secret.yaml => secret.example.yaml} (100%) diff --git a/deploy/minio/base/secret.yaml b/deploy/minio/base/secret.example.yaml similarity index 100% rename from deploy/minio/base/secret.yaml rename to deploy/minio/base/secret.example.yaml From 78e5a947de19947b195549bd1063d464191eaa35 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:33:23 -0400 Subject: [PATCH 13/31] =?UTF-8?q?=F0=9F=90=9B=20Ignore=20minio=20secret?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 077bc65..16aeec6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ models.json config.toml data jobs.db +deploy/minio/base/secret.yaml deploy/job-queue/base/secret.yaml deploy/job-queue/base/nebius-secret.yaml deploy/intake-poller/base/secret.yaml From 014a23a60a16ba01387f6ea436b9bde070ba19bc Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:33:50 -0400 Subject: [PATCH 14/31] =?UTF-8?q?=F0=9F=93=9D=20Update=20docs=20for=20mini?= =?UTF-8?q?o=20secret?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0546541..95c11c1 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,7 @@ sequenceDiagram 2. Copy and fill in the Secret templates locally. Do not commit the resulting files: ```sh + cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml cp deploy/job-queue/base/secret.example.yaml deploy/job-queue/base/secret.yaml cp deploy/job-queue/base/nebius-secret.example.yaml deploy/job-queue/base/nebius-secret.yaml ``` @@ -667,7 +668,13 @@ oc apply -f deploy/job-queue/base/task-sa.yml oc apply -f deploy/job-queue/base/orchestrator-sa.yml ``` -Create a MinIO deployment to store your job results: +Copy the MinIO secret and fill in the values: + +```bash +cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml +``` + +Create the MinIO deployment to store your job results: ```bash oc apply -k deploy/minio/overlays/prod From fb3763269f98bc06f41f90a4a26416205da09355 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:46:29 -0400 Subject: [PATCH 15/31] =?UTF-8?q?=F0=9F=90=9B=20Fix=20deployment=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_deployment_security.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_deployment_security.py b/tests/test_deployment_security.py index 6bc8462..79d6c77 100644 --- a/tests/test_deployment_security.py +++ b/tests/test_deployment_security.py @@ -7,14 +7,22 @@ from coding_agent_bench import api -DEPLOYMENT_PATH = Path(__file__).parents[1] / "deploy" / "job-queue-service.yml" -INTAKE_CRONJOB_PATH = Path(__file__).parents[1] / "deploy" / "intake-cronjob.yml" +DEPLOYMENT_PATHS = ( + Path(__file__).parents[1] / "deploy" / "job-queue" / "base" / "deployment.yaml", + Path(__file__).parents[1] / "deploy" / "job-queue" / "base" / "service.yaml", + Path(__file__).parents[1] / "deploy" / "job-queue" / "base" / "route.yaml", +) +INTAKE_CRONJOB_PATH = ( + Path(__file__).parents[1] / "deploy" / "intake-poller" / "base" / "cronjob.yaml" +) def _deployment_objects() -> dict[str, dict]: """Return queue manifest objects indexed by Kubernetes kind.""" - with DEPLOYMENT_PATH.open() as manifest: - objects = list(yaml.safe_load_all(manifest)) + objects = [] + for path in DEPLOYMENT_PATHS: + with path.open() as manifest: + objects.extend(yaml.safe_load_all(manifest)) return {obj["kind"]: obj for obj in objects} From 3c07befecf9ab3a71cf649ae9b9eb90e7d353bd9 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:46:43 -0400 Subject: [PATCH 16/31] =?UTF-8?q?=F0=9F=93=9D=20Fix=20yml=20->=20yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 95c11c1..2491b65 100644 --- a/README.md +++ b/README.md @@ -643,7 +643,7 @@ oc project Create ServiceAccounts and RoleBindings to run tasks: ```bash -oc apply -f deploy/job-queue/base/harbor-task-sa.yml +oc apply -f deploy/job-queue/base/harbor-task-sa.yaml ``` Then in your `harbor` command, add the flag: @@ -664,8 +664,8 @@ oc project Create ServiceAccounts and RoleBindings to run tasks and orchestrate: ```bash -oc apply -f deploy/job-queue/base/task-sa.yml -oc apply -f deploy/job-queue/base/orchestrator-sa.yml +oc apply -f deploy/job-queue/base/task-sa.yaml +oc apply -f deploy/job-queue/base/orchestrator-sa.yaml ``` Copy the MinIO secret and fill in the values: From e15b05f33dd970dee7c4f3436a7d1601dc2b8dee Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:52:31 -0400 Subject: [PATCH 17/31] =?UTF-8?q?=F0=9F=90=9B=20Run=20stage=20sync=20on=20?= =?UTF-8?q?merged=20branches=20targeting=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sync-stage.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-stage.yml b/.github/workflows/sync-stage.yml index 6e52c5e..47344db 100644 --- a/.github/workflows/sync-stage.yml +++ b/.github/workflows/sync-stage.yml @@ -1,7 +1,7 @@ name: Sync stage with main on: - pull_request: + pull_request_target: branches: - main types: @@ -22,7 +22,8 @@ jobs: - name: Check out stage uses: actions/checkout@v4 with: - ref: stage + repository: ${{ github.repository }} + ref: refs/heads/stage fetch-depth: 0 - name: Merge main into stage From 45b5ca0af9f9a0427cc371e690b5206215d6658a Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 10:55:09 -0400 Subject: [PATCH 18/31] =?UTF-8?q?=F0=9F=90=9B=20Verify=20intake=20poller?= =?UTF-8?q?=20secret=20before=20applying?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c098e28..0e9726b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -46,6 +46,9 @@ jobs: for secret in job-queue-secret nebius-secret intake-poller-secret; do oc -n "$NAMESPACE" get secret "$secret" >/dev/null done + for key in ALLOW_INSECURE_QUEUE_HTTP SMTP_HOST SMTP_PORT SMTP_STARTTLS; do + test -n "$(oc -n "$NAMESPACE" get secret intake-poller-secret -o "jsonpath={.data.$key}")" + done - name: Apply MinIO run: oc apply -k deploy/minio/overlays/$OVERLAY -n "$NAMESPACE" From 09df672c26055e5b1505c0ba5957992e8ee49151 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 11:06:15 -0400 Subject: [PATCH 19/31] =?UTF-8?q?=F0=9F=90=9B=20Remove=20secret=20files=20?= =?UTF-8?q?from=20kustomize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 18 ++++++++++++++---- deploy/job-queue/base/kustomization.yaml | 2 -- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2491b65..d9a1516 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,7 @@ sequenceDiagram oc login --server= --token= ``` -2. Copy and fill in the Secret templates locally. Do not commit the resulting files: +2. Copy the Secret templates locally, fill in their values, and do not commit the resulting files: ```sh cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml @@ -224,6 +224,13 @@ sequenceDiagram If you are not using Nebius, you still need to create the secret, but you can leave the default values and they will be ignored. + Apply the Secrets separately to the target project before deploying the services: + + ```sh + oc apply -f deploy/minio/base/secret.yaml -n + oc apply -f deploy/job-queue/base/secret.yaml -n + ``` + 3. Deploy the MinIO service to store job artifacts: ```sh @@ -305,10 +312,11 @@ nebius iam auth-public-key generate \ --output ~/.nebius/$SA_ID-credentials.json ``` -Once the service account is created, you can copy and fill in the values in [`deploy/job-queue/base/nebius-secret.example.yaml`](./deploy/job-queue/base/nebius-secret.example.yaml): +Once the service account is created, copy [`deploy/job-queue/base/nebius-secret.example.yaml`](./deploy/job-queue/base/nebius-secret.example.yaml), fill in its values, and apply it separately. Do not commit the resulting file: ```sh cp deploy/job-queue/base/nebius-secret.example.yaml deploy/job-queue/base/nebius-secret.yaml +oc apply -f deploy/job-queue/base/nebius-secret.yaml -n ``` ```yaml @@ -337,10 +345,11 @@ When creating a job, set `server_url` to `nebius-` to use a managed Ne The intake poller is an optional CronJob to pull requests from a Google Sheet and submit them to the job queue. You can read more about this service in the [intake poller docs](./deploy/README.md#intake-poller). -First, copy the secret in [`deploy/intake-poller/secret.example.yaml`](./deploy/intake-poller/secret.example.yaml) and fill in the values according to the [intake poller docs](./deploy/README.md#intake-poller). Do not commit this file. +First, copy the secret in [`deploy/intake-poller/secret.example.yaml`](./deploy/intake-poller/secret.example.yaml), fill in the values according to the [intake poller docs](./deploy/README.md#intake-poller), and apply it separately. Do not commit this file. ```sh cp deploy/intake-poller/secret.example.yaml deploy/intake-poller/secret.yaml +oc apply -f deploy/intake-poller/base/secret.yaml -n ``` Then create the CronJob: @@ -668,10 +677,11 @@ oc apply -f deploy/job-queue/base/task-sa.yaml oc apply -f deploy/job-queue/base/orchestrator-sa.yaml ``` -Copy the MinIO secret and fill in the values: +Copy the MinIO secret, fill in the values, and apply it separately. Do not commit the resulting file: ```bash cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml +oc apply -f deploy/minio/base/secret.yaml -n ``` Create the MinIO deployment to store your job results: diff --git a/deploy/job-queue/base/kustomization.yaml b/deploy/job-queue/base/kustomization.yaml index a7c32ce..5e9d787 100644 --- a/deploy/job-queue/base/kustomization.yaml +++ b/deploy/job-queue/base/kustomization.yaml @@ -9,8 +9,6 @@ resources: - ./task-anyuid.yaml # Job Queue - ./pvc.yaml - - ./secret.yaml - - ./nebius-secret.yaml - ./deployment.yaml - ./service.yaml - ./route.yaml From afe0dd0afe740fca235a37bf4d5816158f6e6a91 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 11:08:13 -0400 Subject: [PATCH 20/31] =?UTF-8?q?=F0=9F=90=9B=20Update=20rollout=20strateg?= =?UTF-8?q?y=20for=20minio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/minio/base/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/minio/base/deployment.yaml b/deploy/minio/base/deployment.yaml index dd40f24..0e5aa8b 100644 --- a/deploy/minio/base/deployment.yaml +++ b/deploy/minio/base/deployment.yaml @@ -7,6 +7,8 @@ metadata: name: harbor-minio spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: app: harbor-minio From d4854949453f6f82a26c4bf1a41d1682aac8f66a Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 11:16:29 -0400 Subject: [PATCH 21/31] =?UTF-8?q?=F0=9F=90=9B=20Remove=20secret.yaml=20fro?= =?UTF-8?q?m=20minio=20kustomization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/minio/base/kustomization.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/minio/base/kustomization.yaml b/deploy/minio/base/kustomization.yaml index b9ead0b..68a4872 100644 --- a/deploy/minio/base/kustomization.yaml +++ b/deploy/minio/base/kustomization.yaml @@ -1,6 +1,5 @@ resources: - ./pvc.yaml - - ./secret.yaml - ./deployment.yaml - ./service.yaml - ./api-route.yaml From c87a854318e6e82a6b808a83aeefc0f2de72b73d Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 11:17:23 -0400 Subject: [PATCH 22/31] =?UTF-8?q?=F0=9F=93=9D=20Remove=20reference=20to=20?= =?UTF-8?q?intake-poller-google-sa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 23b9cf5..47f4876 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -240,8 +240,7 @@ oc apply -f deploy/intake-poller/base/secret.yaml ``` Repeat these commands for both stage and production. The CI workflow checks for -`job-queue-secret`, `nebius-secret`, `intake-poller-secret`, and -`intake-poller-google-sa` before applying anything. +`job-queue-secret`, `nebius-secret`, and `intake-poller-secret` before applying anything. ### OpenShift setup From 0228ec713f0ffc6dce1ad8aaf470faf037911b66 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 12:17:24 -0400 Subject: [PATCH 23/31] =?UTF-8?q?=F0=9F=90=9B=20Fix=20environment=20select?= =?UTF-8?q?ion=20when=20merging=20to=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0e9726b..c32dfa3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,10 +19,10 @@ jobs: (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) runs-on: ubuntu-latest - environment: ${{ github.event_name == 'pull_request' && 'stage' || 'production' }} + environment: ${{ (github.event_name == 'push' || github.event.pull_request.base.ref == 'main') && 'production' || 'stage' }} env: - OVERLAY: ${{ github.event_name == 'pull_request' && 'stage' || 'prod' }} - NAMESPACE: ${{ github.event_name == 'pull_request' && vars.STAGE_NAMESPACE || vars.PROD_NAMESPACE }} + OVERLAY: ${{ (github.event_name == 'push' || github.event.pull_request.base.ref == 'main') && 'prod' || 'stage' }} + NAMESPACE: ${{ (github.event_name == 'push' || github.event.pull_request.base.ref == 'main') && vars.PROD_NAMESPACE || vars.STAGE_NAMESPACE }} OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} From 4fd84fb86e59549ffba8dde97c2e25c0accb9463 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 12:22:02 -0400 Subject: [PATCH 24/31] =?UTF-8?q?=F0=9F=90=9B=20Raise=20an=20error=20when?= =?UTF-8?q?=20ENVIRONMENT=20is=20not=20supported?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/coding_agent_bench/intake/poller.py | 7 +++++-- tests/intake/test_poller.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/coding_agent_bench/intake/poller.py b/src/coding_agent_bench/intake/poller.py index 657ab28..1390818 100644 --- a/src/coding_agent_bench/intake/poller.py +++ b/src/coding_agent_bench/intake/poller.py @@ -32,7 +32,10 @@ def _auto_approve_enabled() -> bool: def _is_stage_environment() -> bool: """Return whether the poller is running as a non-notifying stage test.""" - return os.environ.get("ENVIRONMENT", "stage").lower() != "prod" + environment = os.environ.get("ENVIRONMENT", "").lower() + if environment not in {"prod", "stage"}: + raise ValueError("ENVIRONMENT must be set to either 'prod' or 'stage'") + return environment == "stage" def _queue_verify() -> str | bool: @@ -114,8 +117,8 @@ def process_rows( sender_email: str, ) -> None: """Process approved, in-flight, and pending-notification spreadsheet rows.""" - rows = sheets.get_all_rows() stage = _is_stage_environment() + rows = sheets.get_all_rows() for i, row in enumerate(rows): row_num = i + 1 diff --git a/tests/intake/test_poller.py b/tests/intake/test_poller.py index cee1fd2..06b8e9c 100644 --- a/tests/intake/test_poller.py +++ b/tests/intake/test_poller.py @@ -1,7 +1,10 @@ from unittest.mock import MagicMock, patch +import pytest + from coding_agent_bench.intake.config import Column, Status from coding_agent_bench.intake.poller import ( + _is_stage_environment, _queue_verify, _row_idempotency_key, _validate_queue_url, @@ -9,6 +12,22 @@ ) +@pytest.fixture(autouse=True) +def production_environment(monkeypatch): + monkeypatch.setenv("ENVIRONMENT", "prod") + + +def test_environment_must_be_explicitly_supported(monkeypatch): + monkeypatch.delenv("ENVIRONMENT", raising=False) + + with pytest.raises(ValueError, match="ENVIRONMENT"): + _is_stage_environment() + + monkeypatch.setenv("ENVIRONMENT", "development") + with pytest.raises(ValueError, match="ENVIRONMENT"): + _is_stage_environment() + + def _make_row(**overrides) -> list[str]: """Build a representative Queue row with optional column overrides.""" row = [""] * len(Column) From 295669ca81626922999f3d91120d1e3dc50a298f Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 12:31:44 -0400 Subject: [PATCH 25/31] =?UTF-8?q?=F0=9F=9A=9A=20Move=20is=5Fstage=5Fenviro?= =?UTF-8?q?nment=20to=20utils.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/coding_agent_bench/intake/poller.py | 11 ++--------- src/coding_agent_bench/utils.py | 9 +++++++++ tests/conftest.py | 5 +++++ tests/intake/test_poller.py | 19 ------------------- tests/test_utils.py | 13 +++++++++++++ 5 files changed, 29 insertions(+), 28 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_utils.py diff --git a/src/coding_agent_bench/intake/poller.py b/src/coding_agent_bench/intake/poller.py index 1390818..4d9f2ca 100644 --- a/src/coding_agent_bench/intake/poller.py +++ b/src/coding_agent_bench/intake/poller.py @@ -19,6 +19,7 @@ ) from coding_agent_bench.intake.sheets import SheetsClient from coding_agent_bench.intake.validation import validate_row +from coding_agent_bench.utils import is_stage_environment logger = logging.getLogger(__name__) @@ -30,14 +31,6 @@ def _auto_approve_enabled() -> bool: return AUTO_APPROVE or os.environ.get("AUTO_APPROVE", "false").lower() == "true" -def _is_stage_environment() -> bool: - """Return whether the poller is running as a non-notifying stage test.""" - environment = os.environ.get("ENVIRONMENT", "").lower() - if environment not in {"prod", "stage"}: - raise ValueError("ENVIRONMENT must be set to either 'prod' or 'stage'") - return environment == "stage" - - def _queue_verify() -> str | bool: """Return the CA bundle for verifying the queue's TLS certificate. @@ -117,7 +110,7 @@ def process_rows( sender_email: str, ) -> None: """Process approved, in-flight, and pending-notification spreadsheet rows.""" - stage = _is_stage_environment() + stage = is_stage_environment() rows = sheets.get_all_rows() for i, row in enumerate(rows): diff --git a/src/coding_agent_bench/utils.py b/src/coding_agent_bench/utils.py index 37a07c2..e9589b4 100644 --- a/src/coding_agent_bench/utils.py +++ b/src/coding_agent_bench/utils.py @@ -1,3 +1,4 @@ +import os import shlex from pathlib import Path @@ -23,3 +24,11 @@ def validate_remote_skill_sources(skills: list[str] | None) -> None: "Use org/name[@ref] or an HTTP(S) Git URL; local paths are " "only supported for locally orchestrated runs." ) from exc + + +def is_stage_environment() -> bool: + """Return True if the application is running in stage environment.""" + environment = os.environ.get("ENVIRONMENT", "").lower() + if environment not in {"prod", "stage"}: + raise ValueError("ENVIRONMENT must be set to either 'prod' or 'stage'") + return environment == "stage" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..92cbb6f --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +import pytest + +@pytest.fixture(autouse=True) +def production_environment(monkeypatch): + monkeypatch.setenv("ENVIRONMENT", "prod") diff --git a/tests/intake/test_poller.py b/tests/intake/test_poller.py index 06b8e9c..cee1fd2 100644 --- a/tests/intake/test_poller.py +++ b/tests/intake/test_poller.py @@ -1,10 +1,7 @@ from unittest.mock import MagicMock, patch -import pytest - from coding_agent_bench.intake.config import Column, Status from coding_agent_bench.intake.poller import ( - _is_stage_environment, _queue_verify, _row_idempotency_key, _validate_queue_url, @@ -12,22 +9,6 @@ ) -@pytest.fixture(autouse=True) -def production_environment(monkeypatch): - monkeypatch.setenv("ENVIRONMENT", "prod") - - -def test_environment_must_be_explicitly_supported(monkeypatch): - monkeypatch.delenv("ENVIRONMENT", raising=False) - - with pytest.raises(ValueError, match="ENVIRONMENT"): - _is_stage_environment() - - monkeypatch.setenv("ENVIRONMENT", "development") - with pytest.raises(ValueError, match="ENVIRONMENT"): - _is_stage_environment() - - def _make_row(**overrides) -> list[str]: """Build a representative Queue row with optional column overrides.""" row = [""] * len(Column) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..27813bb --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,13 @@ +import pytest + +from coding_agent_bench.utils import is_stage_environment + +def test_environment_must_be_explicitly_supported(monkeypatch): + monkeypatch.delenv("ENVIRONMENT", raising=False) + + with pytest.raises(ValueError, match="ENVIRONMENT"): + is_stage_environment() + + monkeypatch.setenv("ENVIRONMENT", "development") + with pytest.raises(ValueError, match="ENVIRONMENT"): + is_stage_environment() From 6b309659cc232c3a9875b831fef7a7f5c22335bf Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 12:33:43 -0400 Subject: [PATCH 26/31] =?UTF-8?q?=F0=9F=93=9D=20Add=20ENVIRONMENT=20to=20.?= =?UTF-8?q?env.example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/.env.example b/.env.example index bed854e..e8dc0c1 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,7 @@ OPENROUTER_API_KEY= # SMTP_PORT=25 # SMTP_STARTTLS=false # AUTO_APPROVE=false +# ENVIRONMENT=stage # For local-only testing with an HTTP queue, set this explicitly: # ALLOW_INSECURE_QUEUE_HTTP=true From 3bd993c87687b7bb79c7e86da17b2c5b4a498def Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 12:35:47 -0400 Subject: [PATCH 27/31] =?UTF-8?q?=F0=9F=93=9D=20Update=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index d9a1516..723c08e 100644 --- a/README.md +++ b/README.md @@ -219,11 +219,8 @@ sequenceDiagram ```sh cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml cp deploy/job-queue/base/secret.example.yaml deploy/job-queue/base/secret.yaml - cp deploy/job-queue/base/nebius-secret.example.yaml deploy/job-queue/base/nebius-secret.yaml ``` - If you are not using Nebius, you still need to create the secret, but you can leave the default values and they will be ignored. - Apply the Secrets separately to the target project before deploying the services: ```sh @@ -348,7 +345,7 @@ You can read more about this service in the [intake poller docs](./deploy/README First, copy the secret in [`deploy/intake-poller/secret.example.yaml`](./deploy/intake-poller/secret.example.yaml), fill in the values according to the [intake poller docs](./deploy/README.md#intake-poller), and apply it separately. Do not commit this file. ```sh -cp deploy/intake-poller/secret.example.yaml deploy/intake-poller/secret.yaml +cp deploy/intake-poller/base/secret.example.yaml deploy/intake-poller/base/secret.yaml oc apply -f deploy/intake-poller/base/secret.yaml -n ``` From 6a6ae10457bccf16e65525e4b27297be3b3036ad Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 12:40:25 -0400 Subject: [PATCH 28/31] =?UTF-8?q?=F0=9F=92=9A=20Keep=20nebius=20and=20inta?= =?UTF-8?q?ke=20poller=20optional=20in=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c32dfa3..ce57041 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,12 +43,12 @@ jobs: set -euo pipefail test -n "$NAMESPACE" - for secret in job-queue-secret nebius-secret intake-poller-secret; do - oc -n "$NAMESPACE" get secret "$secret" >/dev/null - done - for key in ALLOW_INSECURE_QUEUE_HTTP SMTP_HOST SMTP_PORT SMTP_STARTTLS; do - test -n "$(oc -n "$NAMESPACE" get secret intake-poller-secret -o "jsonpath={.data.$key}")" - done + oc -n "$NAMESPACE" get secret job-queue-secret >/dev/null + if oc -n "$NAMESPACE" get secret intake-poller-secret >/dev/null 2>&1; then + for key in ALLOW_INSECURE_QUEUE_HTTP SMTP_HOST SMTP_PORT SMTP_STARTTLS; do + test -n "$(oc -n "$NAMESPACE" get secret intake-poller-secret -o "jsonpath={.data.$key}")" + done + fi - name: Apply MinIO run: oc apply -k deploy/minio/overlays/$OVERLAY -n "$NAMESPACE" @@ -63,4 +63,7 @@ jobs: run: oc rollout status deployment/job-queue -n "$NAMESPACE" --timeout=10m - name: Apply intake poller - run: oc apply -k deploy/intake-poller/overlays/$OVERLAY -n "$NAMESPACE" + run: | + if oc -n "$NAMESPACE" get secret intake-poller-secret >/dev/null 2>&1; then + oc apply -k deploy/intake-poller/overlays/$OVERLAY -n "$NAMESPACE" + fi From 126026a5b9c99706e0528dd74ae6f752b9f9efb5 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 13:45:34 -0400 Subject: [PATCH 29/31] =?UTF-8?q?=F0=9F=90=9B=20Update=20github=20CI=20dep?= =?UTF-8?q?loy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 4 +++- deploy/README.md | 13 ++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ce57041..fce8860 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,7 +43,9 @@ jobs: set -euo pipefail test -n "$NAMESPACE" - oc -n "$NAMESPACE" get secret job-queue-secret >/dev/null + for secret in harbor-minio job-queue-secret; do + oc -n "$NAMESPACE" get secret "$secret" >/dev/null + done if oc -n "$NAMESPACE" get secret intake-poller-secret >/dev/null 2>&1; then for key in ALLOW_INSECURE_QUEUE_HTTP SMTP_HOST SMTP_PORT SMTP_STARTTLS; do test -n "$(oc -n "$NAMESPACE" get secret intake-poller-secret -o "jsonpath={.data.$key}")" diff --git a/deploy/README.md b/deploy/README.md index 47f4876..076ba39 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -218,29 +218,32 @@ job instead of creating a duplicate. `.github/workflows/deploy.yml` deploys both Kustomize applications with `oc`: -- A merged pull request targeting `STAGE` deploys to `STAGE_NAMESPACE`. +- A merged pull request targeting `stage` deploys to `STAGE_NAMESPACE`. - A version tag such as `v0.3.0` deploys to `PROD_NAMESPACE`. The workflow expects the Secrets to already exist in the target namespace. It only verifies them and applies the two Kustomizations. The files below are safe templates for local reference only; fill them in locally and do not commit them: -- `deploy/job-queue/secret.example.yaml` -- `deploy/job-queue/nebius-secret.example.yaml` -- `deploy/intake-poller/secret.example.yaml` +- `deploy/minio/base/secret.example.yaml` +- `deploy/job-queue/base/secret.example.yaml` +- `deploy/job-queue/base/nebius-secret.example.yaml` +- `deploy/intake-poller/base/secret.example.yaml` Before the first CI deployment, apply the filled-in templates to each target namespace: ```sh oc project +oc apply -f deploy/minio/base/secret.yaml oc apply -f deploy/job-queue/base/secret.yaml oc apply -f deploy/job-queue/base/nebius-secret.yaml oc apply -f deploy/intake-poller/base/secret.yaml ``` Repeat these commands for both stage and production. The CI workflow checks for -`job-queue-secret`, `nebius-secret`, and `intake-poller-secret` before applying anything. +`job-queue-secret` before applying anything. It also checks for `intake-poller-secret` +before applying the intake poller. `nebius-secret` is optional. ### OpenShift setup From cc67e87dac0d4ff91e8c0c8c2448078ce122a26c Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 13:58:39 -0400 Subject: [PATCH 30/31] =?UTF-8?q?=F0=9F=90=9B=20Fix=20ENVIRONMENT=20patchi?= =?UTF-8?q?ng=20in=20deploy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/intake-poller/base/cronjob.yaml | 4 +++- deploy/intake-poller/base/kustomization.yaml | 1 - deploy/intake-poller/base/secret.example.yaml | 1 - deploy/intake-poller/overlays/prod/kustomization.yaml | 6 +++--- deploy/intake-poller/overlays/stage/kustomization.yaml | 6 +++--- deploy/job-queue/base/deployment.yaml | 2 ++ deploy/job-queue/base/secret.example.yaml | 1 - deploy/job-queue/overlays/prod/kustomization.yaml | 6 +++--- deploy/job-queue/overlays/stage/kustomization.yaml | 6 +++--- deploy/minio/base/deployment.yaml | 3 +++ deploy/minio/base/secret.example.yaml | 3 +-- deploy/minio/overlays/prod/kustomization.yaml | 4 ++-- deploy/minio/overlays/stage/kustomization.yaml | 4 ++-- 13 files changed, 25 insertions(+), 22 deletions(-) diff --git a/deploy/intake-poller/base/cronjob.yaml b/deploy/intake-poller/base/cronjob.yaml index 8728b14..96905e3 100644 --- a/deploy/intake-poller/base/cronjob.yaml +++ b/deploy/intake-poller/base/cronjob.yaml @@ -21,7 +21,7 @@ spec: restartPolicy: Never containers: - name: poller - image: ghcr.io/redhat-et/coding_agent_bench:v0.2.6 + image: ghcr.io/redhat-et/coding_agent_bench:tag imagePullPolicy: Always # Run the venv's Python directly. `uv run` re-syncs the project into # /app/.venv and writes a cache under $HOME at startup, both of which @@ -29,6 +29,8 @@ spec: # owned by 1001). The package is already installed in the image. command: ["/app/.venv/bin/python", "-m", "coding_agent_bench.intake.poller"] env: + - name: ENVIRONMENT + value: stage - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/google/service-account.json - name: SMTP_HOST diff --git a/deploy/intake-poller/base/kustomization.yaml b/deploy/intake-poller/base/kustomization.yaml index 9a790b1..562b47c 100644 --- a/deploy/intake-poller/base/kustomization.yaml +++ b/deploy/intake-poller/base/kustomization.yaml @@ -5,4 +5,3 @@ resources: labels: - pairs: app: intake-poller - \ No newline at end of file diff --git a/deploy/intake-poller/base/secret.example.yaml b/deploy/intake-poller/base/secret.example.yaml index 7711951..2a67842 100644 --- a/deploy/intake-poller/base/secret.example.yaml +++ b/deploy/intake-poller/base/secret.example.yaml @@ -12,6 +12,5 @@ stringData: SMTP_PORT: "25" SMTP_STARTTLS: "false" ALLOW_INSECURE_QUEUE_HTTP: "false" - ENVIRONMENT: env service-account.json: |- diff --git a/deploy/intake-poller/overlays/prod/kustomization.yaml b/deploy/intake-poller/overlays/prod/kustomization.yaml index 8aab12c..93d427f 100644 --- a/deploy/intake-poller/overlays/prod/kustomization.yaml +++ b/deploy/intake-poller/overlays/prod/kustomization.yaml @@ -10,9 +10,9 @@ images: patches: - target: - kind: Secret - name: intake-poller-secret + kind: CronJob + name: intake-poller patch: | - op: replace - path: /stringData/ENVIRONMENT + path: /spec/jobTemplate/spec/template/spec/containers/0/env/0/value value: prod diff --git a/deploy/intake-poller/overlays/stage/kustomization.yaml b/deploy/intake-poller/overlays/stage/kustomization.yaml index 0ee1d64..9e9b71a 100644 --- a/deploy/intake-poller/overlays/stage/kustomization.yaml +++ b/deploy/intake-poller/overlays/stage/kustomization.yaml @@ -17,9 +17,9 @@ patches: path: /spec/suspend value: true - target: - kind: Secret - name: intake-poller-secret + kind: CronJob + name: intake-poller patch: | - op: replace - path: /stringData/ENVIRONMENT + path: /spec/jobTemplate/spec/template/spec/containers/0/env/0/value value: stage diff --git a/deploy/job-queue/base/deployment.yaml b/deploy/job-queue/base/deployment.yaml index 5944172..ca420ee 100644 --- a/deploy/job-queue/base/deployment.yaml +++ b/deploy/job-queue/base/deployment.yaml @@ -54,6 +54,8 @@ spec: seccompProfile: type: RuntimeDefault env: + - name: ENVIRONMENT + value: stage - name: HOME value: /tmp - name: UV_CACHE_DIR diff --git a/deploy/job-queue/base/secret.example.yaml b/deploy/job-queue/base/secret.example.yaml index 533f1fe..b04d41a 100644 --- a/deploy/job-queue/base/secret.example.yaml +++ b/deploy/job-queue/base/secret.example.yaml @@ -10,4 +10,3 @@ stringData: ANTHROPIC_API_KEY: OPENAI_API_KEY: OPENROUTER_API_KEY: - ENVIRONMENT: env diff --git a/deploy/job-queue/overlays/prod/kustomization.yaml b/deploy/job-queue/overlays/prod/kustomization.yaml index ad1755f..3340dc0 100644 --- a/deploy/job-queue/overlays/prod/kustomization.yaml +++ b/deploy/job-queue/overlays/prod/kustomization.yaml @@ -10,9 +10,9 @@ images: patches: - target: - kind: Secret - name: job-queue-secret + kind: Deployment + name: job-queue patch: | - op: replace - path: /stringData/ENVIRONMENT + path: /spec/template/spec/containers/0/env/0/value value: prod diff --git a/deploy/job-queue/overlays/stage/kustomization.yaml b/deploy/job-queue/overlays/stage/kustomization.yaml index 0a4c24c..b9a2b4e 100644 --- a/deploy/job-queue/overlays/stage/kustomization.yaml +++ b/deploy/job-queue/overlays/stage/kustomization.yaml @@ -10,9 +10,9 @@ images: patches: - target: - kind: Secret - name: job-queue-secret + kind: Deployment + name: job-queue patch: | - op: replace - path: /stringData/ENVIRONMENT + path: /spec/template/spec/containers/0/env/0/value value: stage diff --git a/deploy/minio/base/deployment.yaml b/deploy/minio/base/deployment.yaml index 0e5aa8b..d598e35 100644 --- a/deploy/minio/base/deployment.yaml +++ b/deploy/minio/base/deployment.yaml @@ -27,6 +27,9 @@ spec: - /data - --console-address - :9001 + env: + - name: ENVIRONMENT + value: stage envFrom: - secretRef: name: harbor-minio diff --git a/deploy/minio/base/secret.example.yaml b/deploy/minio/base/secret.example.yaml index 880abc3..99a1ce2 100644 --- a/deploy/minio/base/secret.example.yaml +++ b/deploy/minio/base/secret.example.yaml @@ -8,5 +8,4 @@ metadata: type: Opaque stringData: MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - ENVIRONMENT: env + MINIO_ROOT_PASSWORD: minioadmin \ No newline at end of file diff --git a/deploy/minio/overlays/prod/kustomization.yaml b/deploy/minio/overlays/prod/kustomization.yaml index f008fe9..25577bc 100644 --- a/deploy/minio/overlays/prod/kustomization.yaml +++ b/deploy/minio/overlays/prod/kustomization.yaml @@ -6,9 +6,9 @@ resources: patches: - target: - kind: Secret + kind: Deployment name: harbor-minio patch: | - op: replace - path: /stringData/ENVIRONMENT + path: /spec/template/spec/containers/0/env/0/value value: prod diff --git a/deploy/minio/overlays/stage/kustomization.yaml b/deploy/minio/overlays/stage/kustomization.yaml index ec8493d..c9c6256 100644 --- a/deploy/minio/overlays/stage/kustomization.yaml +++ b/deploy/minio/overlays/stage/kustomization.yaml @@ -6,9 +6,9 @@ resources: patches: - target: - kind: Secret + kind: Deployment name: harbor-minio patch: | - op: replace - path: /stringData/ENVIRONMENT + path: /spec/template/spec/containers/0/env/0/value value: stage From 8e27a4e8f61f78c25ce4219b7ddc09aa6f248a5a Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Tue, 29 Sep 2026 16:44:15 -0400 Subject: [PATCH 31/31] =?UTF-8?q?=F0=9F=90=9B=20Fix=20stage=20rollouts=20f?= =?UTF-8?q?rom=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 31 +++++++++++-------- .../overlays/stage/kustomization.yaml | 7 +++++ .../overlays/stage/kustomization.yaml | 7 +++++ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fce8860..62fbf5c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,14 +1,11 @@ name: Deploy to OpenShift on: - pull_request: - types: [closed] - branches: - - stage - - main - push: - tags: - - 'v*' + workflow_run: + workflows: + - Build and Push Container Image + types: + - completed permissions: contents: read @@ -16,19 +13,23 @@ permissions: jobs: deploy: if: >- - (github.event_name == 'pull_request' && github.event.pull_request.merged == true) - || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) + github.event.workflow_run.conclusion == 'success' && + (github.event.workflow_run.head_branch == 'main' || + github.event.workflow_run.head_branch == 'stage' || + startsWith(github.event.workflow_run.head_branch, 'v')) runs-on: ubuntu-latest - environment: ${{ (github.event_name == 'push' || github.event.pull_request.base.ref == 'main') && 'production' || 'stage' }} + environment: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && 'production' || 'stage' }} env: - OVERLAY: ${{ (github.event_name == 'push' || github.event.pull_request.base.ref == 'main') && 'prod' || 'stage' }} - NAMESPACE: ${{ (github.event_name == 'push' || github.event.pull_request.base.ref == 'main') && vars.PROD_NAMESPACE || vars.STAGE_NAMESPACE }} + OVERLAY: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && 'prod' || 'stage' }} + NAMESPACE: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && vars.PROD_NAMESPACE || vars.STAGE_NAMESPACE }} OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} steps: - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha }} - name: Log in to OpenShift uses: redhat-actions/oc-login@v1 @@ -61,6 +62,10 @@ jobs: - name: Apply job queue run: oc apply -k deploy/job-queue/overlays/$OVERLAY -n "$NAMESPACE" + - name: Restart stage job queue + if: env.OVERLAY == 'stage' + run: oc rollout restart deployment/job-queue -n "$NAMESPACE" + - name: Wait for job queue rollout run: oc rollout status deployment/job-queue -n "$NAMESPACE" --timeout=10m diff --git a/deploy/intake-poller/overlays/stage/kustomization.yaml b/deploy/intake-poller/overlays/stage/kustomization.yaml index 9e9b71a..29d073b 100644 --- a/deploy/intake-poller/overlays/stage/kustomization.yaml +++ b/deploy/intake-poller/overlays/stage/kustomization.yaml @@ -9,6 +9,13 @@ images: newTag: stage patches: + - target: + kind: CronJob + name: intake-poller + patch: | + - op: add + path: /spec/jobTemplate/spec/template/spec/containers/0/imagePullPolicy + value: Always - target: kind: CronJob name: intake-poller diff --git a/deploy/job-queue/overlays/stage/kustomization.yaml b/deploy/job-queue/overlays/stage/kustomization.yaml index b9a2b4e..6eff519 100644 --- a/deploy/job-queue/overlays/stage/kustomization.yaml +++ b/deploy/job-queue/overlays/stage/kustomization.yaml @@ -9,6 +9,13 @@ images: newTag: stage patches: + - target: + kind: Deployment + name: job-queue + patch: | + - op: add + path: /spec/template/spec/containers/0/imagePullPolicy + value: Always - target: kind: Deployment name: job-queue