diff --git a/.bumpversion.toml b/.bumpversion.toml index 354980c..5e90652 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -11,11 +11,11 @@ search = 'version = "{current_version}"' replace = 'version = "{new_version}"' [[tool.bumpversion.files]] -filename = "deploy/job-queue-service.yml" -search = "ghcr.io/redhat-et/coding_agent_bench:v{current_version}" -replace = "ghcr.io/redhat-et/coding_agent_bench:v{new_version}" +filename = "deploy/job-queue/overlays/prod/kustomization.yaml" +search = "v{current_version}" +replace = "v{new_version}" [[tool.bumpversion.files]] -filename = "deploy/intake-cronjob.yml" -search = "ghcr.io/redhat-et/coding_agent_bench:v{current_version}" -replace = "ghcr.io/redhat-et/coding_agent_bench:v{new_version}" +filename = "deploy/intake-poller/overlays/prod/kustomization.yaml" +search = "v{current_version}" +replace = "v{new_version}" diff --git a/.env.example b/.env.example index 800e746..e8dc0c1 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,7 @@ OPENROUTER_API_KEY= # SMTP_PORT=25 # SMTP_STARTTLS=false # AUTO_APPROVE=false +# ENVIRONMENT=stage # For local-only testing with an HTTP queue, set this explicitly: # ALLOW_INSECURE_QUEUE_HTTP=true @@ -29,14 +30,6 @@ NEBIUS_ENABLED=0 # NEBIUS_TENANT_ID= # NEBIUS_SERVICE_ACCOUNT_ID= # NEBIUS_SUBNET_ID= -# NEBIUS_INSTANCE_NAME_PREFIX=cab-worker +# NEBIUS_INSTANCE_NAME_PREFIX=job-queue-worker # NEBIUS_IDLE_TIMEOUT_SECONDS=600 # HF_TOKEN= - -# Intake Poller (optional) -# GOOGLE_APPLICATION_CREDENTIALS=service-account.json -# GOOGLE_SHEET_ID= -# JOB_QUEUE_URL=http://localhost:8000 -# - SENDER_EMAIL= -# AUTO_APPROVE=false diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 38586f6..67b103b 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - stage - 'CICD*' tags: - 'v*' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..62fbf5c --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,76 @@ +name: Deploy to OpenShift + +on: + workflow_run: + workflows: + - Build and Push Container Image + types: + - completed + +permissions: + contents: read + +jobs: + deploy: + if: >- + github.event.workflow_run.conclusion == 'success' && + (github.event.workflow_run.head_branch == 'main' || + github.event.workflow_run.head_branch == 'stage' || + startsWith(github.event.workflow_run.head_branch, 'v')) + runs-on: ubuntu-latest + environment: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && 'production' || 'stage' }} + env: + OVERLAY: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && 'prod' || 'stage' }} + NAMESPACE: ${{ (github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch, 'v')) && vars.PROD_NAMESPACE || vars.STAGE_NAMESPACE }} + OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} + OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha }} + + - name: Log in to OpenShift + uses: redhat-actions/oc-login@v1 + with: + openshift_server_url: ${{ env.OPENSHIFT_SERVER }} + openshift_token: ${{ env.OPENSHIFT_TOKEN }} + namespace: ${{ env.NAMESPACE }} + + - name: Verify application Secrets + shell: bash + run: | + set -euo pipefail + + test -n "$NAMESPACE" + for secret in harbor-minio job-queue-secret; do + oc -n "$NAMESPACE" get secret "$secret" >/dev/null + done + if oc -n "$NAMESPACE" get secret intake-poller-secret >/dev/null 2>&1; then + for key in ALLOW_INSECURE_QUEUE_HTTP SMTP_HOST SMTP_PORT SMTP_STARTTLS; do + test -n "$(oc -n "$NAMESPACE" get secret intake-poller-secret -o "jsonpath={.data.$key}")" + done + fi + + - name: Apply MinIO + run: oc apply -k deploy/minio/overlays/$OVERLAY -n "$NAMESPACE" + + - name: Wait for MinIO rollout + run: oc rollout status deployment/harbor-minio -n "$NAMESPACE" --timeout=10m + + - name: Apply job queue + run: oc apply -k deploy/job-queue/overlays/$OVERLAY -n "$NAMESPACE" + + - name: Restart stage job queue + if: env.OVERLAY == 'stage' + run: oc rollout restart deployment/job-queue -n "$NAMESPACE" + + - name: Wait for job queue rollout + run: oc rollout status deployment/job-queue -n "$NAMESPACE" --timeout=10m + + - name: Apply intake poller + run: | + if oc -n "$NAMESPACE" get secret intake-poller-secret >/dev/null 2>&1; then + oc apply -k deploy/intake-poller/overlays/$OVERLAY -n "$NAMESPACE" + fi diff --git a/.github/workflows/sync-stage.yml b/.github/workflows/sync-stage.yml new file mode 100644 index 0000000..47344db --- /dev/null +++ b/.github/workflows/sync-stage.yml @@ -0,0 +1,34 @@ +name: Sync stage with main + +on: + pull_request_target: + branches: + - main + types: + - closed + +permissions: + contents: write + +concurrency: + group: sync-stage + cancel-in-progress: false + +jobs: + sync: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Check out stage + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: refs/heads/stage + fetch-depth: 0 + + - name: Merge main into stage + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git merge origin/main --no-edit + git push origin HEAD:stage diff --git a/.gitignore b/.gitignore index 55955af..16aeec6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,10 @@ models.json config.toml data jobs.db +deploy/minio/base/secret.yaml +deploy/job-queue/base/secret.yaml +deploy/job-queue/base/nebius-secret.yaml +deploy/intake-poller/base/secret.yaml # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index 5051e40..723c08e 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Reproducible benchmarks for coding agents and models using Harbor - [Set up the service](#set-up-the-service) - [Use the service](#use-the-service) - [(Optional) Connect to Nebius](#optional-connect-to-nebius) + - [(Optional) Set Up the Intake Poller](#optional-set-up-the-intake-poller) - [Harbor Command Examples](#harbor-command-examples) - [Claude Code vLLM](#claude-code-vllm) - [Codex vLLM](#codex-vllm) @@ -207,91 +208,44 @@ sequenceDiagram ### Set up the service -1. Log in to your cluster and project: +1. Log in to your cluster: + ```sh oc login --server= --token= - oc project - ``` -2. Create the MinIO service for artifact storage: - ```sh - oc apply -f deploy/harbor-minio.yml ``` - Note: the default username and password are `(minioadmin, minioadmin)`. - You can update this in the deployment file if needed. -3. Create the orchestrator and task service accounts: + +2. Copy the Secret templates locally, fill in their values, and do not commit the resulting files: + ```sh - oc apply -f deploy/harbor-orchestrator-sa.yml - oc apply -f deploy/harbor-task-sa.yml + cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml + cp deploy/job-queue/base/secret.example.yaml deploy/job-queue/base/secret.yaml ``` -4. Create a secret file named `job-queue-secret` with the queue service's - `API_KEY` and any queue or Nebius settings, then apply it: - ```yaml - apiVersion: v1 - kind: Secret - metadata: - name: job-queue-secret - stringData: - API_KEY: - type: Opaque - ``` - If the intake CronJob is deployed, create its separate poller secret: - ```yaml - apiVersion: v1 - kind: Secret - metadata: - name: intake-poller-secret - stringData: - JOB_QUEUE_URL: https:// - GOOGLE_SHEET_ID: - SENDER_EMAIL: ace-model-evals@redhat.com - AUTO_APPROVE: 'false' - type: Opaque - ``` -5. Create the queue service: + + Apply the Secrets separately to the target project before deploying the services: + ```sh - oc apply -f deploy/job-queue-service.yml + oc apply -f deploy/minio/base/secret.yaml -n + oc apply -f deploy/job-queue/base/secret.yaml -n ``` -6. (Optional) To run jobs against OpenRouter (`server_url: openrouter`), create - an `openrouter-api-key` secret. Job pods mount it automatically (it is - optional, so non-OpenRouter jobs are unaffected): - ```yaml - apiVersion: v1 - kind: Secret - metadata: - name: openrouter-api-key - stringData: - OPENROUTER_API_KEY: - type: Opaque - ``` - The queue service itself also needs `OPENROUTER_API_KEY` in its environment - to validate OpenRouter jobs at request time. Add it to `job-queue-secret` - (which the service already loads) or `envFrom` the `openrouter-api-key` - secret in `deploy/job-queue-service.yml`. - - The queue listens on HTTPS inside the cluster. OpenShift's service-serving - certificate operator creates the `job-queue-tls` Secret referenced by the - Deployment, and the Route uses re-encryption so traffic remains encrypted - from the router to the queue pod. Wait for that Secret to appear before - troubleshooting pod startup: + +3. Deploy the MinIO service to store job artifacts: + ```sh - oc get secret job-queue-tls + oc apply -k deploy/minio/overlays/prod -n ``` -Get the route for the deployed service: +4. Deploy the Job Queue service: -```sh -oc get route job-queue-route --output jsonpath='{.spec.host}' -``` - -Set `JOB_QUEUE_URL` in `intake-poller-secret` to this HTTPS route before -applying `deploy/intake-cronjob.yml`. + ```sh + oc apply -k deploy/job-queue/overlays/prod -n + ``` -Check that the application is live by visiting the docs: +5. Get the route for the deployed API service: -```sh -export JOB_QUEUE_URL="https://$(oc get route job-queue-route --output jsonpath='{.spec.host}')" -open $JOB_QUEUE_URL/docs -``` + ```sh + export JOB_QUEUE_URL="https://$(oc get route job-queue-route -n --output jsonpath='{.spec.host}')" + open $JOB_QUEUE_URL/docs + ``` ### Use the service @@ -355,30 +309,52 @@ nebius iam auth-public-key generate \ --output ~/.nebius/$SA_ID-credentials.json ``` -Once the service account is created, you can update your job queue secret with the following environment variables needed for Nebius: +Once the service account is created, copy [`deploy/job-queue/base/nebius-secret.example.yaml`](./deploy/job-queue/base/nebius-secret.example.yaml), fill in its values, and apply it separately. Do not commit the resulting file: + +```sh +cp deploy/job-queue/base/nebius-secret.example.yaml deploy/job-queue/base/nebius-secret.yaml +oc apply -f deploy/job-queue/base/nebius-secret.yaml -n +``` ```yaml -apiVersion: v1 kind: Secret metadata: - name: job-queue-secret + name: nebius-secret +type: Opaque stringData: - API_KEY: - NEBIUS_ENABLED: '1' + NEBIUS_ENABLED: "1" NEBIUS_SERVICE_ACCOUNT_CREDS: | - NEBIUS_PARENT_ID: - NEBIUS_TENANT_ID: - NEBIUS_SERVICE_ACCOUNT_ID: - NEBIUS_SUBNET_ID: + NEBIUS_USER: + NEBIUS_PARENT_ID: + NEBIUS_TENANT_ID: + NEBIUS_SERVICE_ACCOUNT_ID: + NEBIUS_SUBNET_ID: NEBIUS_INSTANCE_NAME_PREFIX: job-queue-worker - NEBIUS_IDLE_TIMEOUT_SECONDS: '600' - HF_TOKEN: -type: Opaque + NEBIUS_IDLE_TIMEOUT_SECONDS: "600" + HF_TOKEN: ``` When creating a job, set `server_url` to `nebius-` to use a managed Nebius instance with the specified GPU resource (e.g. `nebius-h200`, `nebius-b200`). Available resources are defined in `RESOURCE_CONFIG_REGISTRY`. +### (Optional) Set Up the Intake Poller + +The intake poller is an optional CronJob to pull requests from a Google Sheet and submit them to the job queue. +You can read more about this service in the [intake poller docs](./deploy/README.md#intake-poller). + +First, copy the secret in [`deploy/intake-poller/secret.example.yaml`](./deploy/intake-poller/secret.example.yaml), fill in the values according to the [intake poller docs](./deploy/README.md#intake-poller), and apply it separately. Do not commit this file. + +```sh +cp deploy/intake-poller/base/secret.example.yaml deploy/intake-poller/base/secret.yaml +oc apply -f deploy/intake-poller/base/secret.yaml -n +``` + +Then create the CronJob: + +```sh +oc apply -k deploy/intake-poller/overlays/prod -n +``` + ## Harbor Command Examples **Prerequisites:** @@ -673,7 +649,7 @@ oc project Create ServiceAccounts and RoleBindings to run tasks: ```bash -oc apply -f deploy/harbor-task-sa.yml +oc apply -f deploy/job-queue/base/harbor-task-sa.yaml ``` Then in your `harbor` command, add the flag: @@ -694,14 +670,21 @@ oc project Create ServiceAccounts and RoleBindings to run tasks and orchestrate: ```bash -oc apply -f deploy/harbor-task-sa.yml -oc apply -f deploy/harbor-orchestrator-sa.yml +oc apply -f deploy/job-queue/base/task-sa.yaml +oc apply -f deploy/job-queue/base/orchestrator-sa.yaml +``` + +Copy the MinIO secret, fill in the values, and apply it separately. Do not commit the resulting file: + +```bash +cp deploy/minio/base/secret.example.yaml deploy/minio/base/secret.yaml +oc apply -f deploy/minio/base/secret.yaml -n ``` -Create a MinIO deployment to store your job results: +Create the MinIO deployment to store your job results: ```bash -oc apply -f deploy/harbor-minio.yml +oc apply -k deploy/minio/overlays/prod ``` Using the CLI, start a job with the `--remote` flag enabled and set `--environment openshift`, e.g.: diff --git a/deploy/README.md b/deploy/README.md index 2767a74..076ba39 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -179,14 +179,19 @@ development. | `JOB_QUEUE_URL` | HTTPS URL for the queue API. Use the cluster's TLS/mTLS endpoint; the poller fails closed instead of using plaintext HTTP. | | `SENDER_EMAIL` | Address notification emails are sent from. Set it to `ace-model-evals@redhat.com`. | | `AUTO_APPROVE` | `"true"` to auto-submit rows with a blank status, otherwise `"false"` | +| `SMTP_HOST` | Host for the SMTP server to send notifications through | +| `SMTP_PORT` | Port for the SMTP server to send notifications through | +| `SMTP_STARTTLS` | Set to `"true"` when the server requires STARTTLS, otherwise `"false"` | +| `ALLOW_INSECURE_QUEUE_HTTP` | Set to `"true"` when the queue is served over HTTP (e.g. locally), otherwise `"false"` | +| `service-account.json` | Content of a GCP service account that has read access to the Google Sheet. | ### Queue TLS -`deploy/job-queue-service.yml` enables OpenShift's service-serving certificate +`deploy/job-queue/deployment.yaml` enables OpenShift's service-serving certificate operator with the `service.beta.openshift.io/serving-cert-secret-name` annotation. The operator creates `job-queue-tls` with `tls.crt`, `tls.key`, and the service CA; the queue mounts that Secret and Uvicorn serves HTTPS on port -8443. The Service exposes it as port 443 and the Route uses `reencrypt` +1. The Service exposes it as port 443 and the Route uses `reencrypt` termination, keeping router-to-pod traffic encrypted as well. Apply the manifest before starting the poller and wait for `job-queue-tls` to be created. @@ -199,21 +204,6 @@ resource token such as `nebius-h200` (or `nebius-b200x8`). The queue service validates that token, provisions the instance, and supplies its endpoint after approval; the requester never needs to know that endpoint. -**`intake-poller-google-sa`** — the Google service-account credential mounted -at `/etc/google/service-account.json` for Sheets access. Notification email is -sent through the internal SMTP relay, so no Gmail mailbox credential or -domain-wide delegation is required. - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: intake-poller-google-sa -type: Opaque -stringData: - service-account.json: -``` - The CronJob sends notifications through `smtp.corp.redhat.com` on port 25. Set `SMTP_HOST` and `SMTP_PORT` on the poller when a different internal relay is required. Set `SMTP_STARTTLS=true` when that relay requires STARTTLS. The @@ -222,3 +212,81 @@ configured `SENDER_EMAIL` must be an address permitted by the relay. Each submitted Queue row carries a deterministic idempotency key. If the CronJob is retried after a network timeout, the queue API returns the original job instead of creating a duplicate. + +## GitHub Actions Deployment + +`.github/workflows/deploy.yml` deploys both Kustomize applications with +`oc`: + +- A merged pull request targeting `stage` deploys to `STAGE_NAMESPACE`. +- A version tag such as `v0.3.0` deploys to `PROD_NAMESPACE`. + +The workflow expects the Secrets to already exist in the target namespace. It +only verifies them and applies the two Kustomizations. The files below are safe +templates for local reference only; fill them in locally and do not commit them: + +- `deploy/minio/base/secret.example.yaml` +- `deploy/job-queue/base/secret.example.yaml` +- `deploy/job-queue/base/nebius-secret.example.yaml` +- `deploy/intake-poller/base/secret.example.yaml` + +Before the first CI deployment, apply the filled-in templates to each target +namespace: + +```sh +oc project +oc apply -f deploy/minio/base/secret.yaml +oc apply -f deploy/job-queue/base/secret.yaml +oc apply -f deploy/job-queue/base/nebius-secret.yaml +oc apply -f deploy/intake-poller/base/secret.yaml +``` + +Repeat these commands for both stage and production. The CI workflow checks for +`job-queue-secret` before applying anything. It also checks for `intake-poller-secret` +before applying the intake poller. `nebius-secret` is optional. + +### OpenShift setup + +Create one namespace for stage and one for production. In each namespace, create +a deployer ServiceAccount and grant it the permissions needed to create and update +the resources in these Kustomizations. The account must also be allowed to create +the `anyuid` RoleBinding used by the job-queue workload, or an administrator must +apply that binding separately. + +The OpenShift service CA and service-serving certificate operators must be +available. They create `intake-poller-ca` and `job-queue-tls` when the manifests +are applied. The cluster must also be able to pull the image from GHCR. + +Create a token for each environment's deployer ServiceAccount and verify it locally: + +```sh +oc login --server= --token= +oc project +oc auth can-i create deployments +oc auth can-i create rolebindings +``` + +### GitHub setup + +Create GitHub repository variables: + +| Variable | Value | +|----------|-------| +| `STAGE_NAMESPACE` | OpenShift stage namespace | +| `PROD_NAMESPACE` | OpenShift production namespace | + +Create GitHub Environments named `stage` and `production`. Add the OpenShift +connection secrets to both environments, using environment-specific values. +Production can also require an approval reviewer: + +| Secret | Purpose | +|--------|---------| +| `OPENSHIFT_SERVER` | OpenShift API URL | +| `OPENSHIFT_TOKEN` | Token for the namespace deployer ServiceAccount | + +The application secrets listed in the example files are not GitHub secrets. They +are applied directly to OpenShift before deployment. + +The workflow uses the `stage` environment for merged `STAGE` pull requests and +the `production` environment for `v*` tags. It does not run for an unmerged pull +request or for ordinary branch pushes. diff --git a/deploy/harbor-minio.yml b/deploy/harbor-minio.yml deleted file mode 100644 index 6f1f312..0000000 --- a/deploy/harbor-minio.yml +++ /dev/null @@ -1,149 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi - storageClassName: gp3 - volumeMode: Filesystem ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio -type: Opaque -stringData: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio -spec: - replicas: 1 - selector: - matchLabels: - app: harbor-minio - component: minio - template: - metadata: - labels: - app: harbor-minio - component: minio - spec: - containers: - - name: minio - image: quay.io/minio/minio:latest - args: - - server - - /data - - --console-address - - :9001 - envFrom: - - secretRef: - name: harbor-minio - ports: - - containerPort: 9000 - name: api - - containerPort: 9001 - name: console - livenessProbe: - httpGet: - path: /minio/health/live - port: 9000 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /minio/health/ready - port: 9000 - initialDelaySeconds: 10 - periodSeconds: 5 - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: 250m - memory: 512Mi - volumeMounts: - - mountPath: /data - name: data - volumes: - - name: data - persistentVolumeClaim: - claimName: harbor-minio ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio -spec: - ports: - - name: api - port: 9000 - protocol: TCP - targetPort: 9000 - - name: console - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: harbor-minio - component: minio - type: ClusterIP ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio-api -spec: - tls: - termination: edge - insecureEdgeTerminationPolicy: Redirect - port: - targetPort: 9000 - to: - kind: Service - name: harbor-minio - weight: 100 - wildcardPolicy: None ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - labels: - app: harbor-minio - component: minio - name: harbor-minio-console -spec: - tls: - termination: edge - insecureEdgeTerminationPolicy: Redirect - port: - targetPort: 9001 - to: - kind: Service - name: harbor-minio - weight: 100 - wildcardPolicy: None diff --git a/deploy/harbor-orchestrator-sa.yml b/deploy/harbor-orchestrator-sa.yml deleted file mode 100644 index acf1218..0000000 --- a/deploy/harbor-orchestrator-sa.yml +++ /dev/null @@ -1,61 +0,0 @@ -# -- Orchestrator SA: used by the job pod to manage task pods, builds, etc. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: harbor-orchestrator - labels: - app: harbor ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: harbor-orchestrator - labels: - app: harbor -rules: - - apiGroups: [""] - resources: [pods, pods/exec, pods/log] - verbs: [get, list, watch, create, update, patch, delete] - - apiGroups: ["build.openshift.io"] - resources: [buildconfigs, buildconfigs/instantiatebinary, builds, builds/log] - verbs: [get, list, watch, create, update, patch, delete] - - apiGroups: ["image.openshift.io"] - resources: [imagestreams] - verbs: [get, list, watch, create, update, patch, delete] - - apiGroups: ["image.openshift.io"] - resources: [imagestreamtags] - verbs: [get] - - apiGroups: ["batch"] - resources: [jobs] - verbs: [get, list, watch, create, update, patch, delete] ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: harbor-orchestrator - labels: - app: harbor -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: harbor-orchestrator -subjects: - - kind: ServiceAccount - name: harbor-orchestrator ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: harbor-orchestrator-anyuid - labels: - app: harbor -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:openshift:scc:anyuid -subjects: - - kind: ServiceAccount - name: harbor-orchestrator diff --git a/deploy/intake-cronjob.yml b/deploy/intake-poller/base/cronjob.yaml similarity index 77% rename from deploy/intake-cronjob.yml rename to deploy/intake-poller/base/cronjob.yaml index 174934a..96905e3 100644 --- a/deploy/intake-cronjob.yml +++ b/deploy/intake-poller/base/cronjob.yaml @@ -1,17 +1,3 @@ ---- -# The service-ca operator injects the service-serving CA into this ConfigMap -# (populating the service-ca.crt key) because of the inject-cabundle annotation. -# Declaring it here keeps the manifest self-contained: the operator only -# populates an existing ConfigMap, it does not create a missing one. -apiVersion: v1 -kind: ConfigMap -metadata: - name: intake-poller-ca - labels: - app: intake-poller - annotations: - service.beta.openshift.io/inject-cabundle: "true" ---- apiVersion: batch/v1 kind: CronJob metadata: @@ -35,7 +21,7 @@ spec: restartPolicy: Never containers: - name: poller - image: ghcr.io/redhat-et/coding_agent_bench:v0.2.6 + image: ghcr.io/redhat-et/coding_agent_bench:tag imagePullPolicy: Always # Run the venv's Python directly. `uv run` re-syncs the project into # /app/.venv and writes a cache under $HOME at startup, both of which @@ -43,12 +29,25 @@ spec: # owned by 1001). The package is already installed in the image. command: ["/app/.venv/bin/python", "-m", "coding_agent_bench.intake.poller"] env: + - name: ENVIRONMENT + value: stage - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/google/service-account.json - name: SMTP_HOST - value: smtp.corp.redhat.com + valueFrom: + secretKeyRef: + name: intake-poller-secret + key: SMTP_HOST - name: SMTP_PORT - value: "25" + valueFrom: + secretKeyRef: + name: intake-poller-secret + key: SMTP_PORT + - name: SMTP_STARTTLS + valueFrom: + secretKeyRef: + name: intake-poller-secret + key: SMTP_STARTTLS - name: GOOGLE_SHEET_ID valueFrom: secretKeyRef: @@ -74,6 +73,11 @@ spec: secretKeyRef: name: intake-poller-secret key: AUTO_APPROVE + - name: ALLOW_INSECURE_QUEUE_HTTP + valueFrom: + secretKeyRef: + name: intake-poller-secret + key: ALLOW_INSECURE_QUEUE_HTTP # The queue is reached over its in-cluster service address, which # presents an OpenShift service-serving cert. Trust its CA (injected # into the mounted ConfigMap) so TLS verification succeeds. @@ -104,7 +108,10 @@ spec: volumes: - name: google-sa-key secret: - secretName: intake-poller-google-sa + secretName: intake-poller-secret + items: + - key: service-account.json + path: service-account.json - name: service-ca configMap: name: intake-poller-ca diff --git a/deploy/intake-poller/base/intake-poller-ca.yaml b/deploy/intake-poller/base/intake-poller-ca.yaml new file mode 100644 index 0000000..66b64b2 --- /dev/null +++ b/deploy/intake-poller/base/intake-poller-ca.yaml @@ -0,0 +1,12 @@ +# The service-ca operator injects the service-serving CA into this ConfigMap +# (populating the service-ca.crt key) because of the inject-cabundle annotation. +# Declaring it here keeps the manifest self-contained: the operator only +# populates an existing ConfigMap, it does not create a missing one +apiVersion: v1 +kind: ConfigMap +metadata: + name: intake-poller-ca + labels: + app: intake-poller + annotations: + service.beta.openshift.io/inject-cabundle: "true" diff --git a/deploy/intake-poller/base/kustomization.yaml b/deploy/intake-poller/base/kustomization.yaml new file mode 100644 index 0000000..562b47c --- /dev/null +++ b/deploy/intake-poller/base/kustomization.yaml @@ -0,0 +1,7 @@ +resources: + - ./cronjob.yaml + - ./intake-poller-ca.yaml + +labels: + - pairs: + app: intake-poller diff --git a/deploy/intake-poller/base/secret.example.yaml b/deploy/intake-poller/base/secret.example.yaml new file mode 100644 index 0000000..2a67842 --- /dev/null +++ b/deploy/intake-poller/base/secret.example.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + name: intake-poller-secret +type: Opaque +stringData: + GOOGLE_SHEET_ID: + JOB_QUEUE_URL: https://job-queue-service + SENDER_EMAIL: ace-model-evals@redhat.com + AUTO_APPROVE: "false" + SMTP_HOST: smtp.corp.redhat.com + SMTP_PORT: "25" + SMTP_STARTTLS: "false" + ALLOW_INSECURE_QUEUE_HTTP: "false" + service-account.json: |- + diff --git a/deploy/intake-poller/overlays/prod/kustomization.yaml b/deploy/intake-poller/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..93d427f --- /dev/null +++ b/deploy/intake-poller/overlays/prod/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +images: + - name: ghcr.io/redhat-et/coding_agent_bench + newTag: v0.2.6 + +patches: + - target: + kind: CronJob + name: intake-poller + patch: | + - op: replace + path: /spec/jobTemplate/spec/template/spec/containers/0/env/0/value + value: prod diff --git a/deploy/intake-poller/overlays/stage/kustomization.yaml b/deploy/intake-poller/overlays/stage/kustomization.yaml new file mode 100644 index 0000000..29d073b --- /dev/null +++ b/deploy/intake-poller/overlays/stage/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +images: + - name: ghcr.io/redhat-et/coding_agent_bench + newTag: stage + +patches: + - target: + kind: CronJob + name: intake-poller + patch: | + - op: add + path: /spec/jobTemplate/spec/template/spec/containers/0/imagePullPolicy + value: Always + - target: + kind: CronJob + name: intake-poller + patch: | + - op: add + path: /spec/suspend + value: true + - target: + kind: CronJob + name: intake-poller + patch: | + - op: replace + path: /spec/jobTemplate/spec/template/spec/containers/0/env/0/value + value: stage diff --git a/deploy/job-queue-service.yml b/deploy/job-queue/base/deployment.yaml similarity index 58% rename from deploy/job-queue-service.yml rename to deploy/job-queue/base/deployment.yaml index fd233a6..ca420ee 100644 --- a/deploy/job-queue-service.yml +++ b/deploy/job-queue/base/deployment.yaml @@ -1,19 +1,3 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: job-queue-pvc - labels: - app: job-queue -spec: - resources: - requests: - storage: 1Gi - volumeMode: Filesystem - storageClassName: gp3 - accessModes: - - ReadWriteOnce ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -38,21 +22,20 @@ spec: spec: terminationGracePeriodSeconds: 60 serviceAccountName: harbor-orchestrator - securityContext: - fsGroup: 1001 containers: - - image: ghcr.io/redhat-et/coding_agent_bench:v0.2.6 + - image: ghcr.io/redhat-et/coding_agent_bench:tag name: job-queue command: ["/bin/sh", "-c"] args: - | - mkdir -p ~/.ssh && \ - ([ -f /app/data/nebius-ssh-key ] || ssh-keygen -t ed25519 -f /app/data/nebius-ssh-key -N "" -q) && \ - chmod 600 /app/data/nebius-ssh-key && \ + if [ ! -f /app/data/nebius-ssh-key ]; then + ssh-keygen -t ed25519 -f /app/data/nebius-ssh-key -N "" -q || exit 1 + fi + test -r /app/data/nebius-ssh-key || exit 1 exec env \ NEBIUS_SSH_PUBLIC_KEY_PATH=/app/data/nebius-ssh-key.pub \ NEBIUS_SSH_PRIVATE_KEY_PATH=/app/data/nebius-ssh-key \ - uv run uvicorn coding_agent_bench.api:app --host 0.0.0.0 --port 8443 \ + uv run --no-sync uvicorn coding_agent_bench.api:app --host 0.0.0.0 --port 8443 \ --ssl-certfile /etc/job-queue/tls/tls.crt \ --ssl-keyfile /etc/job-queue/tls/tls.key resources: @@ -71,11 +54,20 @@ spec: seccompProfile: type: RuntimeDefault env: + - name: ENVIRONMENT + value: stage + - name: HOME + value: /tmp + - name: UV_CACHE_DIR + value: /tmp/uv-cache - name: JOB_STORE_PATH value: /app/data/jobs.db envFrom: - secretRef: name: job-queue-secret + - secretRef: + name: nebius-secret + optional: true ports: - containerPort: 8443 name: https @@ -95,42 +87,3 @@ spec: secretName: job-queue-tls defaultMode: 0440 restartPolicy: Always ---- -kind: Service -apiVersion: v1 -metadata: - name: job-queue-service - labels: - app: job-queue - component: api - annotations: - service.beta.openshift.io/serving-cert-secret-name: job-queue-tls -spec: - selector: - app: job-queue - component: api - type: ClusterIP - ports: - - name: https - port: 443 - targetPort: 8443 - protocol: TCP ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: job-queue-route - labels: - app: job-queue - component: api -spec: - tls: - termination: reencrypt - insecureEdgeTerminationPolicy: Redirect - port: - targetPort: https - to: - kind: Service - name: job-queue-service - weight: 100 - wildcardPolicy: None diff --git a/deploy/job-queue/base/kustomization.yaml b/deploy/job-queue/base/kustomization.yaml new file mode 100644 index 0000000..5e9d787 --- /dev/null +++ b/deploy/job-queue/base/kustomization.yaml @@ -0,0 +1,18 @@ +resources: + # Orchestrator Pod SA + - ./orchestrator-sa.yaml + - ./orchestrator-role.yaml + - ./orchestrator-rolebinding.yaml + - ./orchestrator-anyuid.yaml + # Task Pod SA + - ./task-sa.yaml + - ./task-anyuid.yaml + # Job Queue + - ./pvc.yaml + - ./deployment.yaml + - ./service.yaml + - ./route.yaml + +labels: + - pairs: + app: job-queue diff --git a/deploy/job-queue/base/nebius-secret.example.yaml b/deploy/job-queue/base/nebius-secret.example.yaml new file mode 100644 index 0000000..9d22aec --- /dev/null +++ b/deploy/job-queue/base/nebius-secret.example.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nebius-secret +type: Opaque +stringData: + NEBIUS_ENABLED: "0" + NEBIUS_SERVICE_ACCOUNT_CREDS: | + + NEBIUS_USER: + NEBIUS_PARENT_ID: + NEBIUS_TENANT_ID: + NEBIUS_SERVICE_ACCOUNT_ID: + NEBIUS_SUBNET_ID: + NEBIUS_INSTANCE_NAME_PREFIX: job-queue-worker + NEBIUS_IDLE_TIMEOUT_SECONDS: "600" + HF_TOKEN: diff --git a/deploy/job-queue/base/orchestrator-anyuid.yaml b/deploy/job-queue/base/orchestrator-anyuid.yaml new file mode 100644 index 0000000..6ca8e85 --- /dev/null +++ b/deploy/job-queue/base/orchestrator-anyuid.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: harbor-orchestrator-anyuid + labels: + app: harbor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:anyuid +subjects: + - kind: ServiceAccount + name: harbor-orchestrator diff --git a/deploy/job-queue/base/orchestrator-role.yaml b/deploy/job-queue/base/orchestrator-role.yaml new file mode 100644 index 0000000..7206bfc --- /dev/null +++ b/deploy/job-queue/base/orchestrator-role.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: harbor-orchestrator + labels: + app: harbor +rules: + - apiGroups: [""] + resources: [pods, pods/exec, pods/log] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: ["build.openshift.io"] + resources: [buildconfigs, buildconfigs/instantiatebinary, builds, builds/log] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: ["image.openshift.io"] + resources: [imagestreams] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: ["image.openshift.io"] + resources: [imagestreamtags] + verbs: [get] + - apiGroups: ["batch"] + resources: [jobs] + verbs: [get, list, watch, create, update, patch, delete] diff --git a/deploy/job-queue/base/orchestrator-rolebinding.yaml b/deploy/job-queue/base/orchestrator-rolebinding.yaml new file mode 100644 index 0000000..694f8ba --- /dev/null +++ b/deploy/job-queue/base/orchestrator-rolebinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: harbor-orchestrator + labels: + app: harbor +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: harbor-orchestrator +subjects: + - kind: ServiceAccount + name: harbor-orchestrator diff --git a/deploy/job-queue/base/orchestrator-sa.yaml b/deploy/job-queue/base/orchestrator-sa.yaml new file mode 100644 index 0000000..4e8593d --- /dev/null +++ b/deploy/job-queue/base/orchestrator-sa.yaml @@ -0,0 +1,7 @@ +# -- Orchestrator SA: used by the job pod to manage task pods, builds, etc. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: harbor-orchestrator + labels: + app: harbor diff --git a/deploy/job-queue/base/pvc.yaml b/deploy/job-queue/base/pvc.yaml new file mode 100644 index 0000000..e12e4ed --- /dev/null +++ b/deploy/job-queue/base/pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: job-queue-pvc + labels: + app: job-queue +spec: + resources: + requests: + storage: 1Gi + volumeMode: Filesystem + storageClassName: gp3 + accessModes: + - ReadWriteOnce diff --git a/deploy/job-queue/base/route.yaml b/deploy/job-queue/base/route.yaml new file mode 100644 index 0000000..b134c56 --- /dev/null +++ b/deploy/job-queue/base/route.yaml @@ -0,0 +1,18 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: job-queue-route + labels: + app: job-queue + component: api +spec: + tls: + termination: reencrypt + insecureEdgeTerminationPolicy: Redirect + port: + targetPort: https + to: + kind: Service + name: job-queue-service + weight: 100 + wildcardPolicy: None diff --git a/deploy/job-queue/base/secret.example.yaml b/deploy/job-queue/base/secret.example.yaml new file mode 100644 index 0000000..b04d41a --- /dev/null +++ b/deploy/job-queue/base/secret.example.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: job-queue-secret +type: Opaque +stringData: + # Required by the job-queue API and the intake poller. + API_KEY: + # Add the provider keys needed by submitted jobs. + ANTHROPIC_API_KEY: + OPENAI_API_KEY: + OPENROUTER_API_KEY: diff --git a/deploy/job-queue/base/service.yaml b/deploy/job-queue/base/service.yaml new file mode 100644 index 0000000..621b17b --- /dev/null +++ b/deploy/job-queue/base/service.yaml @@ -0,0 +1,19 @@ +kind: Service +apiVersion: v1 +metadata: + name: job-queue-service + labels: + app: job-queue + component: api + annotations: + service.beta.openshift.io/serving-cert-secret-name: job-queue-tls +spec: + selector: + app: job-queue + component: api + type: ClusterIP + ports: + - name: https + port: 443 + targetPort: 8443 + protocol: TCP diff --git a/deploy/harbor-task-sa.yml b/deploy/job-queue/base/task-anyuid.yaml similarity index 61% rename from deploy/harbor-task-sa.yml rename to deploy/job-queue/base/task-anyuid.yaml index 63fc93e..9f8999b 100644 --- a/deploy/harbor-task-sa.yml +++ b/deploy/job-queue/base/task-anyuid.yaml @@ -1,12 +1,3 @@ -# -- Task SA: used by environment pods, only needs anyuid SCC to run as root. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: harbor-task - labels: - app: harbor ---- - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/deploy/job-queue/base/task-sa.yaml b/deploy/job-queue/base/task-sa.yaml new file mode 100644 index 0000000..0497b1f --- /dev/null +++ b/deploy/job-queue/base/task-sa.yaml @@ -0,0 +1,7 @@ +# -- Task SA: used by environment pods, only needs anyuid SCC to run as root. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: harbor-task + labels: + app: harbor diff --git a/deploy/job-queue/overlays/prod/kustomization.yaml b/deploy/job-queue/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..3340dc0 --- /dev/null +++ b/deploy/job-queue/overlays/prod/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +images: + - name: ghcr.io/redhat-et/coding_agent_bench + newTag: v0.2.6 + +patches: + - target: + kind: Deployment + name: job-queue + patch: | + - op: replace + path: /spec/template/spec/containers/0/env/0/value + value: prod diff --git a/deploy/job-queue/overlays/stage/kustomization.yaml b/deploy/job-queue/overlays/stage/kustomization.yaml new file mode 100644 index 0000000..6eff519 --- /dev/null +++ b/deploy/job-queue/overlays/stage/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +images: + - name: ghcr.io/redhat-et/coding_agent_bench + newTag: stage + +patches: + - target: + kind: Deployment + name: job-queue + patch: | + - op: add + path: /spec/template/spec/containers/0/imagePullPolicy + value: Always + - target: + kind: Deployment + name: job-queue + patch: | + - op: replace + path: /spec/template/spec/containers/0/env/0/value + value: stage diff --git a/deploy/minio/base/api-route.yaml b/deploy/minio/base/api-route.yaml new file mode 100644 index 0000000..dfc974f --- /dev/null +++ b/deploy/minio/base/api-route.yaml @@ -0,0 +1,18 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio-api +spec: + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + port: + targetPort: 9000 + to: + kind: Service + name: harbor-minio + weight: 100 + wildcardPolicy: None diff --git a/deploy/minio/base/console-route.yaml b/deploy/minio/base/console-route.yaml new file mode 100644 index 0000000..c95a8d2 --- /dev/null +++ b/deploy/minio/base/console-route.yaml @@ -0,0 +1,18 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio-console +spec: + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + port: + targetPort: 9001 + to: + kind: Service + name: harbor-minio + weight: 100 + wildcardPolicy: None diff --git a/deploy/minio/base/deployment.yaml b/deploy/minio/base/deployment.yaml new file mode 100644 index 0000000..d598e35 --- /dev/null +++ b/deploy/minio/base/deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: harbor-minio + component: minio + template: + metadata: + labels: + app: harbor-minio + component: minio + spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + args: + - server + - /data + - --console-address + - :9001 + env: + - name: ENVIRONMENT + value: stage + envFrom: + - secretRef: + name: harbor-minio + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 10 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 250m + memory: 512Mi + volumeMounts: + - mountPath: /data + name: data + volumes: + - name: data + persistentVolumeClaim: + claimName: harbor-minio diff --git a/deploy/minio/base/kustomization.yaml b/deploy/minio/base/kustomization.yaml new file mode 100644 index 0000000..68a4872 --- /dev/null +++ b/deploy/minio/base/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - ./pvc.yaml + - ./deployment.yaml + - ./service.yaml + - ./api-route.yaml + - ./console-route.yaml + +labels: + - pairs: + app: harbor-minio \ No newline at end of file diff --git a/deploy/minio/base/pvc.yaml b/deploy/minio/base/pvc.yaml new file mode 100644 index 0000000..7af17c6 --- /dev/null +++ b/deploy/minio/base/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 400Gi + storageClassName: gp3 + volumeMode: Filesystem diff --git a/deploy/minio/base/secret.example.yaml b/deploy/minio/base/secret.example.yaml new file mode 100644 index 0000000..99a1ce2 --- /dev/null +++ b/deploy/minio/base/secret.example.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio +type: Opaque +stringData: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin \ No newline at end of file diff --git a/deploy/minio/base/service.yaml b/deploy/minio/base/service.yaml new file mode 100644 index 0000000..6847bcd --- /dev/null +++ b/deploy/minio/base/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor-minio + component: minio + name: harbor-minio +spec: + ports: + - name: api + port: 9000 + protocol: TCP + targetPort: 9000 + - name: console + port: 9001 + protocol: TCP + targetPort: 9001 + selector: + app: harbor-minio + component: minio + type: ClusterIP diff --git a/deploy/minio/overlays/prod/kustomization.yaml b/deploy/minio/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..25577bc --- /dev/null +++ b/deploy/minio/overlays/prod/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - target: + kind: Deployment + name: harbor-minio + patch: | + - op: replace + path: /spec/template/spec/containers/0/env/0/value + value: prod diff --git a/deploy/minio/overlays/stage/kustomization.yaml b/deploy/minio/overlays/stage/kustomization.yaml new file mode 100644 index 0000000..c9c6256 --- /dev/null +++ b/deploy/minio/overlays/stage/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - target: + kind: Deployment + name: harbor-minio + patch: | + - op: replace + path: /spec/template/spec/containers/0/env/0/value + value: stage diff --git a/src/coding_agent_bench/intake/poller.py b/src/coding_agent_bench/intake/poller.py index e39e700..4d9f2ca 100644 --- a/src/coding_agent_bench/intake/poller.py +++ b/src/coding_agent_bench/intake/poller.py @@ -19,6 +19,7 @@ ) from coding_agent_bench.intake.sheets import SheetsClient from coding_agent_bench.intake.validation import validate_row +from coding_agent_bench.utils import is_stage_environment logger = logging.getLogger(__name__) @@ -109,6 +110,7 @@ def process_rows( sender_email: str, ) -> None: """Process approved, in-flight, and pending-notification spreadsheet rows.""" + stage = is_stage_environment() rows = sheets.get_all_rows() for i, row in enumerate(rows): @@ -120,7 +122,7 @@ def process_rows( if status in TERMINAL_STATUSES: # Completed/failed rows remain eligible for one retry when the # queue state was persisted but the notification was not. - if status in (Status.COMPLETED.value, Status.FAILED.value) and ( + if not stage and status in (Status.COMPLETED.value, Status.FAILED.value) and ( row[Column.NOTIFIED_DONE].strip().upper() != "TRUE" ): _handle_inflight_row( @@ -132,9 +134,9 @@ def process_rows( if status == Status.APPROVED.value or (not status and _auto_approve_enabled()): _handle_new_row( sheets, row, row_num, api_base_url, api_key, - sender_email, + sender_email, notify=not stage, ) - elif status in (Status.QUEUED.value, Status.RUNNING.value): + elif not stage and status in (Status.QUEUED.value, Status.RUNNING.value): _handle_inflight_row( sheets, row, row_num, api_base_url, api_key, sender_email, @@ -150,6 +152,7 @@ def _handle_new_row( api_base_url: str, api_key: str, sender_email: str, + notify: bool = True, ) -> None: """Validate and submit one approved intake row, then notify its submitter.""" agent = row[Column.AGENT].strip() @@ -192,11 +195,12 @@ def _handle_new_row( sheets.update_cell(row_num, Column.JOB_ID, job_id) sheets.update_cell(row_num, Column.STATUS, Status.QUEUED.value) - try: - send_queued_email(email, agent, dataset, model_name, job_id, sender_email) - sheets.update_cell(row_num, Column.NOTIFIED_QUEUED, "TRUE") - except Exception: - logger.exception("Failed to send queued email for row %d", row_num) + if notify: + try: + send_queued_email(email, agent, dataset, model_name, job_id, sender_email) + sheets.update_cell(row_num, Column.NOTIFIED_QUEUED, "TRUE") + except Exception: + logger.exception("Failed to send queued email for row %d", row_num) def _handle_inflight_row( diff --git a/src/coding_agent_bench/job.py b/src/coding_agent_bench/job.py index 2f193eb..35e9c55 100644 --- a/src/coding_agent_bench/job.py +++ b/src/coding_agent_bench/job.py @@ -109,7 +109,7 @@ def _job_spec( "name": "OPENROUTER_API_KEY", "valueFrom": { "secretKeyRef": { - "name": "openrouter-api-key", + "name": "job-queue-secret", "key": "OPENROUTER_API_KEY", "optional": True, } diff --git a/src/coding_agent_bench/utils.py b/src/coding_agent_bench/utils.py index 37a07c2..e9589b4 100644 --- a/src/coding_agent_bench/utils.py +++ b/src/coding_agent_bench/utils.py @@ -1,3 +1,4 @@ +import os import shlex from pathlib import Path @@ -23,3 +24,11 @@ def validate_remote_skill_sources(skills: list[str] | None) -> None: "Use org/name[@ref] or an HTTP(S) Git URL; local paths are " "only supported for locally orchestrated runs." ) from exc + + +def is_stage_environment() -> bool: + """Return True if the application is running in stage environment.""" + environment = os.environ.get("ENVIRONMENT", "").lower() + if environment not in {"prod", "stage"}: + raise ValueError("ENVIRONMENT must be set to either 'prod' or 'stage'") + return environment == "stage" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..92cbb6f --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +import pytest + +@pytest.fixture(autouse=True) +def production_environment(monkeypatch): + monkeypatch.setenv("ENVIRONMENT", "prod") diff --git a/tests/intake/test_poller.py b/tests/intake/test_poller.py index 288aca3..cee1fd2 100644 --- a/tests/intake/test_poller.py +++ b/tests/intake/test_poller.py @@ -107,6 +107,46 @@ def test_empty_status_row_submitted_when_auto_approve(mock_httpx, mock_email): sheets.update_cell.assert_any_call(1, Column.JOB_ID, "uuid-456") +@patch("coding_agent_bench.intake.poller.send_queued_email") +@patch("coding_agent_bench.intake.poller.httpx") +def test_stage_submission_does_not_send_email(mock_httpx, mock_email, monkeypatch): + """Submit stage test rows without sending requester notifications.""" + monkeypatch.setenv("ENVIRONMENT", "stage") + mock_response = MagicMock() + mock_response.json.return_value = {"job_id": "uuid-stage"} + mock_httpx.post.return_value = mock_response + + sheets = MagicMock() + sheets.get_all_rows.return_value = [_make_row(STATUS=Status.APPROVED.value)] + + process_rows(sheets, "http://job-queue-service", "test-key", "bench@example.com") + + mock_email.assert_not_called() + sheets.update_cell.assert_any_call(1, Column.STATUS, Status.QUEUED.value) + sheets.update_cell.assert_any_call(1, Column.JOB_ID, "uuid-stage") + assert all( + call.args != (1, Column.NOTIFIED_QUEUED, "TRUE") + for call in sheets.update_cell.call_args_list + ) + + +@patch("coding_agent_bench.intake.poller.send_completed_email") +@patch("coding_agent_bench.intake.poller.httpx") +def test_stage_does_not_reconcile_inflight_rows(mock_httpx, mock_email, monkeypatch): + """Leave stage jobs non-terminal and avoid completion notifications.""" + monkeypatch.setenv("ENVIRONMENT", "stage") + sheets = MagicMock() + sheets.get_all_rows.return_value = [ + _make_row(STATUS=Status.QUEUED.value, JOB_ID="uuid-stage") + ] + + process_rows(sheets, "http://job-queue-service", "test-key", "bench@example.com") + + mock_httpx.get.assert_not_called() + mock_email.assert_not_called() + sheets.update_cell.assert_not_called() + + @patch("coding_agent_bench.intake.poller.send_queued_email") @patch("coding_agent_bench.intake.poller.httpx") def test_invalid_approved_row_marked_needs_review(mock_httpx, mock_email): diff --git a/tests/openrouter/test_job_spec.py b/tests/openrouter/test_job_spec.py index 45a786b..65ff130 100644 --- a/tests/openrouter/test_job_spec.py +++ b/tests/openrouter/test_job_spec.py @@ -11,7 +11,7 @@ def test_job_spec_injects_openrouter_secret_only_when_openrouter(): env = _env_by_name(job._job_spec(["echo", "hi"], openrouter=True)) assert "OPENROUTER_API_KEY" in env ref = env["OPENROUTER_API_KEY"]["valueFrom"]["secretKeyRef"] - assert ref["name"] == "openrouter-api-key" + assert ref["name"] == "job-queue-secret" assert ref["key"] == "OPENROUTER_API_KEY" assert ref["optional"] is True diff --git a/tests/test_deployment_security.py b/tests/test_deployment_security.py index 6bc8462..79d6c77 100644 --- a/tests/test_deployment_security.py +++ b/tests/test_deployment_security.py @@ -7,14 +7,22 @@ from coding_agent_bench import api -DEPLOYMENT_PATH = Path(__file__).parents[1] / "deploy" / "job-queue-service.yml" -INTAKE_CRONJOB_PATH = Path(__file__).parents[1] / "deploy" / "intake-cronjob.yml" +DEPLOYMENT_PATHS = ( + Path(__file__).parents[1] / "deploy" / "job-queue" / "base" / "deployment.yaml", + Path(__file__).parents[1] / "deploy" / "job-queue" / "base" / "service.yaml", + Path(__file__).parents[1] / "deploy" / "job-queue" / "base" / "route.yaml", +) +INTAKE_CRONJOB_PATH = ( + Path(__file__).parents[1] / "deploy" / "intake-poller" / "base" / "cronjob.yaml" +) def _deployment_objects() -> dict[str, dict]: """Return queue manifest objects indexed by Kubernetes kind.""" - with DEPLOYMENT_PATH.open() as manifest: - objects = list(yaml.safe_load_all(manifest)) + objects = [] + for path in DEPLOYMENT_PATHS: + with path.open() as manifest: + objects.extend(yaml.safe_load_all(manifest)) return {obj["kind"]: obj for obj in objects} diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..27813bb --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,13 @@ +import pytest + +from coding_agent_bench.utils import is_stage_environment + +def test_environment_must_be_explicitly_supported(monkeypatch): + monkeypatch.delenv("ENVIRONMENT", raising=False) + + with pytest.raises(ValueError, match="ENVIRONMENT"): + is_stage_environment() + + monkeypatch.setenv("ENVIRONMENT", "development") + with pytest.raises(ValueError, match="ENVIRONMENT"): + is_stage_environment()