👷 Add CICD for deployment to OpenShift

.bumpversion.toml

@@ -11,11 +11,11 @@ search = 'version = "{current_version}"'
 replace = 'version = "{new_version}"'
 
 [[tool.bumpversion.files]]
-filename = "deploy/job-queue-service.yml"
-search = "ghcr.io/redhat-et/coding_agent_bench:v{current_version}"
-replace = "ghcr.io/redhat-et/coding_agent_bench:v{new_version}"
+filename = "deploy/job-queue/overlays/prod/kustomization.yaml"
+search = "v{current_version}"
+replace = "v{new_version}"
 
 [[tool.bumpversion.files]]
-filename = "deploy/intake-cronjob.yml"
-search = "ghcr.io/redhat-et/coding_agent_bench:v{current_version}"
-replace = "ghcr.io/redhat-et/coding_agent_bench:v{new_version}"
+filename = "deploy/intake-poller/overlays/prod/kustomization.yaml"
+search = "v{current_version}"
+replace = "v{new_version}"

.env.example

@@ -16,6 +16,7 @@ OPENROUTER_API_KEY=
 # SMTP_PORT=25
 # SMTP_STARTTLS=false
 # AUTO_APPROVE=false
+# ENVIRONMENT=stage
 # For local-only testing with an HTTP queue, set this explicitly:
 # ALLOW_INSECURE_QUEUE_HTTP=true
 
@@ -29,14 +30,6 @@ NEBIUS_ENABLED=0
 # NEBIUS_TENANT_ID=
 # NEBIUS_SERVICE_ACCOUNT_ID=
 # NEBIUS_SUBNET_ID=
-# NEBIUS_INSTANCE_NAME_PREFIX=cab-worker
+# NEBIUS_INSTANCE_NAME_PREFIX=job-queue-worker
 # NEBIUS_IDLE_TIMEOUT_SECONDS=600
 # HF_TOKEN=
-
-# Intake Poller (optional)
-# GOOGLE_APPLICATION_CREDENTIALS=service-account.json
-# GOOGLE_SHEET_ID=
-# JOB_QUEUE_URL=http://localhost:8000
-#
-  SENDER_EMAIL=
-# AUTO_APPROVE=false

.github/workflows/build-push.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - stage
       - 'CICD*'
     tags:
       - 'v*'

.github/workflows/deploy.yml

@@ -0,0 +1,76 @@
+name: Deploy to OpenShift
+
+on:
+  workflow_run:
+    workflows:
+      - Build and Push Container Image
+    types:
+      - completed
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    if: >-
+      github.event.workflow_run.conclusion == 'success' &&
+      (github.event.workflow_run.head_branch == 'main' ||
+      github.event.workflow_run.head_branch == 'stage' ||
+      startsWith(github.event.workflow_run.head_branch, 'v'))
+    runs-on: ubuntu-latest
+    environment: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && 'production' || 'stage' }}
+    env:
+      OVERLAY: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && 'prod' || 'stage' }}
+      NAMESPACE: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && vars.PROD_NAMESPACE || vars.STAGE_NAMESPACE }}
+      OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }}
+      OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      - name: Log in to OpenShift
+        uses: redhat-actions/oc-login@v1
+        with:
+          openshift_server_url: ${{ env.OPENSHIFT_SERVER }}
+          openshift_token: ${{ env.OPENSHIFT_TOKEN }}
+          namespace: ${{ env.NAMESPACE }}
+
+      - name: Verify application Secrets
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          test -n "$NAMESPACE"
+          for secret in harbor-minio job-queue-secret; do
+            oc -n "$NAMESPACE" get secret "$secret" >/dev/null
+          done
+          if oc -n "$NAMESPACE" get secret intake-poller-secret >/dev/null 2>&1; then
+            for key in ALLOW_INSECURE_QUEUE_HTTP SMTP_HOST SMTP_PORT SMTP_STARTTLS; do
+              test -n "$(oc -n "$NAMESPACE" get secret intake-poller-secret -o "jsonpath={.data.$key}")"
+            done
+          fi
+
+      - name: Apply MinIO
+        run: oc apply -k deploy/minio/overlays/$OVERLAY -n "$NAMESPACE"
+
+      - name: Wait for MinIO rollout
+        run: oc rollout status deployment/harbor-minio -n "$NAMESPACE" --timeout=10m
+
+      - name: Apply job queue
+        run: oc apply -k deploy/job-queue/overlays/$OVERLAY -n "$NAMESPACE"
+
+      - name: Restart stage job queue
+        if: env.OVERLAY == 'stage'
+        run: oc rollout restart deployment/job-queue -n "$NAMESPACE"
+
+      - name: Wait for job queue rollout
+        run: oc rollout status deployment/job-queue -n "$NAMESPACE" --timeout=10m
+
+      - name: Apply intake poller
+        run: |
+          if oc -n "$NAMESPACE" get secret intake-poller-secret >/dev/null 2>&1; then
+            oc apply -k deploy/intake-poller/overlays/$OVERLAY -n "$NAMESPACE"
+          fi

.github/workflows/sync-stage.yml

@@ -0,0 +1,34 @@
+name: Sync stage with main
+
+on:
+  pull_request_target:
+    branches:
+      - main
+    types:
+      - closed
+
+permissions:
+  contents: write
+
+concurrency:
+  group: sync-stage
+  cancel-in-progress: false
+
+jobs:
+  sync:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out stage
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository }}
+          ref: refs/heads/stage
+          fetch-depth: 0
+
+      - name: Merge main into stage
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git merge origin/main --no-edit
+          git push origin HEAD:stage

.gitignore

@@ -3,6 +3,10 @@ models.json
 config.toml
 data
 jobs.db
+deploy/minio/base/secret.yaml
+deploy/job-queue/base/secret.yaml
+deploy/job-queue/base/nebius-secret.yaml
+deploy/intake-poller/base/secret.yaml
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

README.md

@@ -25,6 +25,7 @@ Reproducible benchmarks for coding agents and models using Harbor
   - [Set up the service](#set-up-the-service)
   - [Use the service](#use-the-service)
   - [(Optional) Connect to Nebius](#optional-connect-to-nebius)
+  - [(Optional) Set Up the Intake Poller](#optional-set-up-the-intake-poller)
 - [Harbor Command Examples](#harbor-command-examples)
   - [Claude Code vLLM](#claude-code-vllm)
   - [Codex vLLM](#codex-vllm)
@@ -207,91 +208,44 @@ sequenceDiagram
 
 ### Set up the service
 
-1. Log in to your cluster and project:
+1. Log in to your cluster:
+
     ```sh
     oc login --server=<server> --token=<token>
-    oc project <project>
-    ```
-2. Create the MinIO service for artifact storage:
-    ```sh
-    oc apply -f deploy/harbor-minio.yml
     ```
-    Note: the default username and password are `(minioadmin, minioadmin)`.
-    You can update this in the deployment file if needed.
-3. Create the orchestrator and task service accounts:
+
+2. Copy the Secret templates locally, fill in their values, and do not commit the resulting files:
+
     ```sh
-    oc apply -f deploy/harbor-orchestrator-sa.yml
-    oc apply -f deploy/harbor-task-sa.yml
+    cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml
+    cp deploy/job-queue/base/secret.example.yaml deploy/job-queue/base/secret.yaml
     ```
-4. Create a secret file named `job-queue-secret` with the queue service's
-   `API_KEY` and any queue or Nebius settings, then apply it:
-    ```yaml
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name:  job-queue-secret
-    stringData:
-      API_KEY: <your-api-key>
-    type: Opaque
-    ```
-   If the intake CronJob is deployed, create its separate poller secret:
-   ```yaml
-   apiVersion: v1
-   kind: Secret
-   metadata:
-     name: intake-poller-secret
-   stringData:
-     JOB_QUEUE_URL: https://<queue-route-host>
-     GOOGLE_SHEET_ID: <sheet-id>
-     SENDER_EMAIL: ace-model-evals@redhat.com
-     AUTO_APPROVE: 'false'
-   type: Opaque
-   ```
-5. Create the queue service:
+
+    Apply the Secrets separately to the target project before deploying the services:
+
     ```sh
-    oc apply -f deploy/job-queue-service.yml
+    oc apply -f deploy/minio/base/secret.yaml -n <project>
+    oc apply -f deploy/job-queue/base/secret.yaml -n <project>
     ```
-6. (Optional) To run jobs against OpenRouter (`server_url: openrouter`), create
-   an `openrouter-api-key` secret. Job pods mount it automatically (it is
-   optional, so non-OpenRouter jobs are unaffected):
-    ```yaml
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: openrouter-api-key
-    stringData:
-      OPENROUTER_API_KEY: <your-openrouter-api-key>
-    type: Opaque
-    ```
-    The queue service itself also needs `OPENROUTER_API_KEY` in its environment
-    to validate OpenRouter jobs at request time. Add it to `job-queue-secret`
-    (which the service already loads) or `envFrom` the `openrouter-api-key`
-    secret in `deploy/job-queue-service.yml`.
-
-    The queue listens on HTTPS inside the cluster. OpenShift's service-serving
-    certificate operator creates the `job-queue-tls` Secret referenced by the
-    Deployment, and the Route uses re-encryption so traffic remains encrypted
-    from the router to the queue pod. Wait for that Secret to appear before
-    troubleshooting pod startup:
+
+3. Deploy the MinIO service to store job artifacts:
+
     ```sh
-    oc get secret job-queue-tls
+    oc apply -k deploy/minio/overlays/prod -n <project>
     ```
 
-Get the route for the deployed service:
+4. Deploy the Job Queue service:
 
-```sh
-oc get route job-queue-route --output jsonpath='{.spec.host}'
-```
-
-Set `JOB_QUEUE_URL` in `intake-poller-secret` to this HTTPS route before
-applying `deploy/intake-cronjob.yml`.
+    ```sh
+    oc apply -k deploy/job-queue/overlays/prod -n <project>
+    ```
 
-Check that the application is live by visiting the docs:
+5. Get the route for the deployed API service:
 
-```sh
-export JOB_QUEUE_URL="https://$(oc get route job-queue-route --output jsonpath='{.spec.host}')"
-open $JOB_QUEUE_URL/docs
-```
+    ```sh
+    export JOB_QUEUE_URL="https://$(oc get route job-queue-route -n <project> --output jsonpath='{.spec.host}')"
+    open $JOB_QUEUE_URL/docs
+    ```
 
 ### Use the service
 
@@ -355,30 +309,52 @@ nebius iam auth-public-key generate \
   --output ~/.nebius/$SA_ID-credentials.json
 ```
 
-Once the service account is created, you can update your job queue secret with the following environment variables needed for Nebius:
+Once the service account is created, copy [`deploy/job-queue/base/nebius-secret.example.yaml`](./deploy/job-queue/base/nebius-secret.example.yaml), fill in its values, and apply it separately. Do not commit the resulting file:
+
+```sh
+cp deploy/job-queue/base/nebius-secret.example.yaml deploy/job-queue/base/nebius-secret.yaml
+oc apply -f deploy/job-queue/base/nebius-secret.yaml -n <project>
+```
 
 ```yaml
-apiVersion: v1
 kind: Secret
 metadata:
-  name:  job-queue-secret
+  name: nebius-secret
+type: Opaque
 stringData:
-  API_KEY: <your-api-key>
-  NEBIUS_ENABLED: '1'
+  NEBIUS_ENABLED: "1"
   NEBIUS_SERVICE_ACCOUNT_CREDS: |
     <service-account-file-content>
-  NEBIUS_PARENT_ID: <project-id>
-  NEBIUS_TENANT_ID: <tenant-id>
-  NEBIUS_SERVICE_ACCOUNT_ID: <service-account-id>
-  NEBIUS_SUBNET_ID: <subnet-id>
+  NEBIUS_USER: <nebius-user>
+  NEBIUS_PARENT_ID: <nebius-parent-id>
+  NEBIUS_TENANT_ID: <nebius-tenant-id>
+  NEBIUS_SERVICE_ACCOUNT_ID: <nebius-service-account-id>
+  NEBIUS_SUBNET_ID: <nebius-subnet-id>
   NEBIUS_INSTANCE_NAME_PREFIX: job-queue-worker
-  NEBIUS_IDLE_TIMEOUT_SECONDS: '600'
-  HF_TOKEN: <optional-huggingface-token>
-type: Opaque
+  NEBIUS_IDLE_TIMEOUT_SECONDS: "600"
+  HF_TOKEN: <hugging-face-token>
 ```
 
 When creating a job, set `server_url` to `nebius-<resource>` to use a managed Nebius instance with the specified GPU resource (e.g. `nebius-h200`, `nebius-b200`). Available resources are defined in `RESOURCE_CONFIG_REGISTRY`.
 
+### (Optional) Set Up the Intake Poller
+
+The intake poller is an optional CronJob to pull requests from a Google Sheet and submit them to the job queue.
+You can read more about this service in the [intake poller docs](./deploy/README.md#intake-poller).
+
+First, copy the secret in [`deploy/intake-poller/secret.example.yaml`](./deploy/intake-poller/secret.example.yaml), fill in the values according to the [intake poller docs](./deploy/README.md#intake-poller), and apply it separately. Do not commit this file.
+
+```sh
+cp deploy/intake-poller/base/secret.example.yaml deploy/intake-poller/base/secret.yaml 
+oc apply -f deploy/intake-poller/base/secret.yaml -n <project>
+```
+
+Then create the CronJob:
+
+```sh
+oc apply -k deploy/intake-poller/overlays/prod -n <project>
+```
+
 ## Harbor Command Examples
 
 **Prerequisites:**
@@ -673,7 +649,7 @@ oc project <project>
 Create ServiceAccounts and RoleBindings to run tasks:
 
 ```bash
-oc apply -f deploy/harbor-task-sa.yml
+oc apply -f deploy/job-queue/base/harbor-task-sa.yaml
 ```
 
 Then in your `harbor` command, add the flag:
@@ -694,14 +670,21 @@ oc project <project>
 Create ServiceAccounts and RoleBindings to run tasks and orchestrate:
 
 ```bash
-oc apply -f deploy/harbor-task-sa.yml
-oc apply -f deploy/harbor-orchestrator-sa.yml
+oc apply -f deploy/job-queue/base/task-sa.yaml
+oc apply -f deploy/job-queue/base/orchestrator-sa.yaml
+```
+
+Copy the MinIO secret, fill in the values, and apply it separately. Do not commit the resulting file:
+
+```bash
+cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml
+oc apply -f deploy/minio/base/secret.yaml -n <project>
 ```
 
-Create a MinIO deployment to store your job results:
+Create the MinIO deployment to store your job results:
 
 ```bash
-oc apply -f deploy/harbor-minio.yml
+oc apply -k deploy/minio/overlays/prod
 ```
 
 Using the CLI, start a job with the `--remote` flag enabled and set `--environment openshift`, e.g.: