STAC-25142 Add GitHub Actions lint + unit-test workflow (agent CI migration phase 1)

.cerberus/cerberus_notify_failure.sh

@@ -23,4 +23,12 @@ PAYLOAD=$(cat << END_OF_PAYLOAD
 END_OF_PAYLOAD
 )
 
-curl --verbose --fail --data "${PAYLOAD?Payload is empty}" "${CERBERUS_LAMBDA_URL?No URL Provided}"
+# Suspend xtrace: with `set -x` the expanded curl command - including the bearer
+# token - would be echoed into the job log.
+set +x
+curl --fail --show-error --silent \
+  --header "Content-Type: application/json" \
+  --header "Authorization: Bearer ${CERBERUS_API_TOKEN?No Cerberus API token provided}" \
+  --data "${PAYLOAD?Payload is empty}" \
+  "${CERBERUS_LAMBDA_URL?No URL Provided}"
+set -x

.github/docker/godeps-cache/Dockerfile

@@ -0,0 +1,63 @@
+# Go dependency cache image for stackstate-agent CI (STAC-25429). Derives FROM the
+# datadog_build CI image and bakes the workspace's external Go module
+# downloads into GOMODCACHE, so consumer jobs get a warm module cache with no
+# per-job `go mod download`/network round-trips. Rebuilt only when the module graph
+# hash changes -- the image tag is content-addressed
+# (see .github/scripts/agent-godeps-cache-metadata.sh).
+#
+# Only the module ZIPS ($GOMODCACHE/cache/download) are kept; the extracted module
+# trees are deleted before the layer closes and Go re-extracts on demand. Measured on
+# a full cache, the extracted trees are ~79% of the bytes but ~97% of the *files*, and
+# image pull cost here is bound by filesystem metadata, not transfer: warm-proxy pulls
+# of the full-cache image spent 0.7m downloading and 4.9m extracting. Shipping ~400k
+# tiny files through overlayfs cost more than it saved (+3.0m pull against ~1.45m of
+# avoided in-build downloading, so ~1.5m/job net loss).
+#
+# No `# syntax=` frontend directive and no BuildKit secret mounts: the base image is
+# pulled by the daemon after `docker login` (creds stay in the daemon, not the
+# build), so the build itself needs no secrets and no Docker Hub frontend pull.
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+# `read -d ''` and `pipefail` are bashisms and RUN defaults to /bin/sh (dash), where
+# the download loop silently iterates zero times and publishes an empty cache.
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Pre-download external modules for every module in the workspace, mirroring what
+# `inv deps` does per module (tasks/libs/common/go.py) at the same parallelism.
+# manifests/ preserves the repo's directory layout so nested go.mod files (and their
+# local `replace ../` targets) resolve. No GOFLAGS override: go.work puts this in
+# workspace mode, where anything other than -mod=readonly is rejected outright. Only
+# the zip cache is kept -- the manifests and extracted trees are removed in the same
+# RUN, so the layer diff only ever contains the final state.
+#
+# The prune needs `chmod -R u+w` first: Go writes the module cache read-only (dirs
+# 0555, files 0444) and `rm -rf` cannot descend into it as a non-root user.
+# cache/vcs holds full git clones for any module fetched outside the proxy; it is not
+# needed to rebuild a module from its zip, so it goes too.
+COPY manifests/ /workspace/
+RUN set -eux; \
+    git config --global --add safe.directory '*'; \
+    export PATH="${PATH}:/usr/local/go/bin"; \
+    cd /workspace; \
+    modules="$(find . -type f -name go.mod | wc -l)"; \
+    echo "workspace modules: ${modules}"; \
+    test "${modules}" -gt 0; \
+    find . -type f -name go.mod -print0 \
+      | xargs -0 -n1 -P8 bash -c \
+          'moddir="$(dirname "$1")"; echo "go mod download :: ${moddir}"; cd "${moddir}" && go mod download' _; \
+    cache="$(go env GOMODCACHE)"; \
+    echo "== populated module cache =="; \
+    du -sh "${cache}"; \
+    echo "files: $(find "${cache}" -type f | wc -l)"; \
+    echo "== zip cache =="; \
+    du -sh "${cache}/cache/download"; \
+    echo "files: $(find "${cache}/cache/download" -type f | wc -l)"; \
+    test "$(du -sm "${cache}/cache/download" | cut -f1)" -gt 100; \
+    find "${cache}" -mindepth 1 -maxdepth 1 ! -name cache -exec chmod -R u+w {} +; \
+    find "${cache}" -mindepth 1 -maxdepth 1 ! -name cache -exec rm -rf {} +; \
+    if [ -d "${cache}/cache/vcs" ]; then chmod -R u+w "${cache}/cache/vcs"; rm -rf "${cache}/cache/vcs"; fi; \
+    echo "== retained in image =="; \
+    du -sh "${cache}"; \
+    echo "files: $(find "${cache}" -type f | wc -l)"; \
+    rm -rf /workspace

.github/scripts/agent-godeps-cache-metadata.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Computes the content-addressed tag for the stackstate-agent Go dependency cache
+# image (STAC-25429) and the two image refs it is referenced by:
+#   * push  ref -> quay.io/stackstate/stackstate-agent-godeps-cache  (authoritative)
+#   * pull  ref -> ${REGISTRY_HOST}/quay/...                         (registry.tooling proxy)
+#
+# The cache lives in its own quay repo rather than the shared sts-ci-images, which also
+# holds the ARC runner images: stackstate-agent is PUBLIC, so whatever credential this
+# workflow carries is reachable from any org member's same-repo branch. A dedicated repo
+# keeps that reach down to a rebuildable cache.
+#
+# Mirrors StackVista/StackGraph .github/scripts/stackgraph-ci-metadata.sh: the tag is
+# a hash of the dependency-defining inputs, so an unchanged module graph reuses the
+# same image across every branch/PR/default build (the tag *is* the cache key), while
+# any change to the graph rotates the tag and a stale cache is never reused.
+
+agent_godeps_compute_metadata() {
+  : "${QUAY_REGISTRY:?QUAY_REGISTRY is required}"     # quay.io
+  : "${REGISTRY_HOST:?REGISTRY_HOST is required}"     # registry.tooling.stackstate.io (quay proxy)
+  : "${BASE_IMAGE_TAG:?BASE_IMAGE_TAG is required}"   # datadog_build tag the cache derives FROM
+  : "${BASE_IMAGE_NAME:?BASE_IMAGE_NAME is required}" # datadog_build_linux_x64 | datadog_build_linux_arm64
+  : "${ARCH:?ARCH is required}"                       # amd64 | arm64
+
+  # Hash inputs: every module manifest (go.work + go.work.sum + modules.yml + all
+  # nested go.mod/go.sum) plus the base image and the two files that define the
+  # cache mechanism itself (Dockerfile + this script), so changing how the cache is
+  # built also rotates the tag.
+  #
+  # The base image NAME is hashed, not just its tag: the amd64 and arm64
+  # datadog_build images share tag 7af9194f, so hashing the tag alone would give
+  # both arches the same cache tag and let one arch's image satisfy the other's
+  # existence check. ARCH is in the tag as well, so the collision is impossible
+  # by construction and the arch is readable off the ref.
+  local godeps_hash
+  godeps_hash="$(
+    {
+      git ls-files -z -- \
+        '*go.mod' '*go.sum' 'go.work' 'go.work.sum' 'modules.yml' \
+        '.github/docker/godeps-cache/Dockerfile' \
+        '.github/scripts/agent-godeps-cache-metadata.sh' \
+        | LC_ALL=C sort -z \
+        | xargs -0 sha256sum
+      printf 'base:%s:%s\n' "${BASE_IMAGE_NAME}" "${BASE_IMAGE_TAG}"
+    } | sha256sum | cut -c1-16
+  )"
+
+  local image_repo="stackstate/stackstate-agent-godeps-cache"
+  local image_tag="godeps-${ARCH}-${godeps_hash}"
+  # shellcheck disable=SC2034  # consumed by the sourcing workflow step
+  ci_image_push="${QUAY_REGISTRY}/${image_repo}:${image_tag}"
+  # shellcheck disable=SC2034  # consumed by the sourcing workflow step
+  ci_image="${REGISTRY_HOST}/quay/${image_repo}:${image_tag}"
+}

.github/workflows/build-binaries.yml

@@ -0,0 +1,259 @@
+name: Binary builds
+
+on:
+  pull_request:
+  push:
+    branches:
+      - stackstate-7.78.2
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  CONDA_ENV: ddpy3
+
+jobs:
+  godeps-cache-amd64:
+    name: Go dependency cache image (amd64)
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    uses: ./.github/workflows/godeps-cache.yml
+    with:
+      arch: amd64
+      build_delay_seconds: 900
+    secrets:
+      REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+      QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+
+  godeps-cache-arm64:
+    name: Go dependency cache image (arm64)
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    uses: ./.github/workflows/godeps-cache.yml
+    with:
+      arch: arm64
+      build_delay_seconds: 0
+    secrets:
+      REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+      QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+
+  build-agent:
+    name: Build agent binary (branded / StackState, ${{ matrix.arch }})
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    needs:
+      - godeps-cache-amd64
+      - godeps-cache-arm64
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: xlarge-public
+            cache_image: ${{ needs.godeps-cache-amd64.outputs.ci_image }}
+          - arch: arm64
+            runner: arm64-xlarge-public
+            cache_image: ${{ needs.godeps-cache-arm64.outputs.ci_image }}
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 120
+    container:
+      image: ${{ matrix.cache_image }} # zizmor: ignore[unpinned-images]
+      credentials:
+        username: ${{ vars.REGISTRY_USER }}
+        password: ${{ secrets.REGISTRY_PASSWORD }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Build branded agent (production, with rtloader)
+        run: |
+          set -eo pipefail
+          export PATH="$PATH:/usr/local/go/bin"
+          . /root/miniforge3/etc/profile.d/conda.sh
+          conda activate "${CONDA_ENV}"
+          # Work volume is owned by the ARC runner uid but the job container runs as
+          # root, so git rejects the repo as "dubious ownership"; mark it safe.
+          git config --global --add safe.directory '*'
+          # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429), keyed
+          # to the module-graph hash, so there is no `go clean -modcache` and no
+          # per-job `inv deps` (go mod download + tidy) reconcile. Materialise
+          # vendor/ for this checkout against the warm cache (offline; no download).
+          go work sync
+          go work vendor
+          # rtloader is the C++/Python bridge the full agent links against; the
+          # cluster-agent doesn't need it, but the production agent does.
+          inv -e rtloader.make
+          inv -e rtloader.install
+          export AGENT_GITHUB_ORG=DataDog
+          export GITHUB_ORG=DataDog
+          export BRANDED=true
+          export AGENT_REPO_NAME=datadog-agent
+          # Rebrand DataDog -> StackState. fix_branding.sh defaults its target dir to
+          # $CI_PROJECT_DIR (a GitLab built-in that is unset here); pass the checkout
+          # root explicitly.
+          ./fix_branding.sh "${GITHUB_WORKSPACE}"
+          # Production (non-race) build -- the shippable binary, distinct from the
+          # race-instrumented build in the unit-test workflow.
+          inv -e agent.build
+          ls -la bin/agent
+
+  build-cluster-agent:
+    name: Build cluster-agent binary (branded / StackState, ${{ matrix.arch }})
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    needs:
+      - godeps-cache-amd64
+      - godeps-cache-arm64
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: xlarge-public
+            cache_image: ${{ needs.godeps-cache-amd64.outputs.ci_image }}
+          - arch: arm64
+            runner: arm64-xlarge-public
+            cache_image: ${{ needs.godeps-cache-arm64.outputs.ci_image }}
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 120
+    container:
+      image: ${{ matrix.cache_image }} # zizmor: ignore[unpinned-images]
+      credentials:
+        username: ${{ vars.REGISTRY_USER }}
+        password: ${{ secrets.REGISTRY_PASSWORD }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Build branded cluster-agent
+        run: |
+          set -eo pipefail
+          export PATH="$PATH:/usr/local/go/bin"
+          . /root/miniforge3/etc/profile.d/conda.sh
+          conda activate "${CONDA_ENV}"
+          # Work volume is owned by the ARC runner uid but the job container runs as
+          # root, so git rejects the repo as "dubious ownership"; mark it safe.
+          git config --global --add safe.directory '*'
+          # GOMODCACHE is pre-warmed by the godeps-cache image (STAC-25429); no
+          # `go clean -modcache` and no per-job `inv deps` reconcile. Materialise
+          # vendor/ for this checkout against the warm cache (offline; no download).
+          go work sync
+          go work vendor
+          export AGENT_GITHUB_ORG=DataDog
+          export GITHUB_ORG=DataDog
+          export BRANDED=true
+          export AGENT_REPO_NAME=datadog-agent
+          # Rebrand DataDog -> StackState; pass the checkout root (CI_PROJECT_DIR is GitLab-only).
+          ./fix_branding.sh "${GITHUB_WORKSPACE}"
+          inv -e cluster-agent.build
+          # version.txt is a build artifact for the downstream packaging/image phases.
+          # deps_deb produced it on GitLab; there is no shared deps job here, so
+          # generate it in-job. inv agent.version is git-based (not module-cache-based)
+          # and cheap. No `-e`: its stdout must be only the version string.
+          inv agent.version -u > version.txt
+          ls -la bin/stackstate-cluster-agent
+
+      - name: Upload cluster-agent build artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: cluster-agent-binary-${{ matrix.arch }}
+          path: |
+            bin/stackstate-cluster-agent/
+            Dockerfiles/cluster-agent/stackstate-cluster.yaml
+            version.txt
+          retention-days: 5
+          if-no-files-found: error
+
+  build-cluster-agent-image:
+    name: Build cluster-agent container image (docker build, no push on PR, ${{ matrix.arch }})
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    needs: build-cluster-agent
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: docker-public
+          - arch: arm64
+            runner: arm64-xlarge-public
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 45
+    env:
+      LOCAL_IMAGE: stackstate-cluster-agent:ci-${{ matrix.arch }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Download cluster-agent binary
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: cluster-agent-binary-${{ matrix.arch }}
+
+      - name: Log in to the registry proxy
+        env:
+          REGISTRY_HOST: ${{ vars.REGISTRY_HOST }}
+          REGISTRY_USER: ${{ vars.REGISTRY_USER }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+        run: |
+          set -eo pipefail
+          # Dockerfiles/cluster-agent pulls its ubuntu builder stage through the
+          # proxy; the BCI stages come from registry.suse.com and need no auth.
+          printf '%s' "${REGISTRY_PASSWORD}" | docker login -u "${REGISTRY_USER}" --password-stdin "${REGISTRY_HOST}"
+
+      - name: Build cluster-agent image
+        run: |
+          set -eo pipefail
+          # actions/download-artifact does not preserve the executable bit (GitLab
+          # artifacts did). The Dockerfile's `chmod +x` targets
+          # opt/stackstate-agent/bin/stackstate-cluster-agent, which is the enclosing
+          # DIRECTORY, not the binary inside it -- so nothing in the build restores
+          # it and the image ships a non-executable agent.
+          chmod +x bin/stackstate-cluster-agent/stackstate-cluster-agent
+          # bin/stackstate-cluster-agent is a DIRECTORY (binary + dist/), and it
+          # must stay one: the Dockerfile COPYs it to
+          # /opt/stackstate-agent/bin/stackstate-cluster-agent/ and entrypoint.sh
+          # puts that directory on PATH so CMD can exec `stackstate-cluster-agent`
+          # by name. Flattening it to a bare binary breaks the image.
+          cp -r bin/stackstate-cluster-agent Dockerfiles/cluster-agent/
+          docker build -t "${LOCAL_IMAGE}" Dockerfiles/cluster-agent
+
+      - name: Smoke test cluster-agent image
+        run: |
+          set -eo pipefail
+          # Bypass entrypoint.sh: it hard-exits without STS_API_KEY, which a
+          # version check has no business needing.
+          docker run --rm \
+            --entrypoint /opt/stackstate-agent/bin/stackstate-cluster-agent/stackstate-cluster-agent \
+            "${LOCAL_IMAGE}" version
+
+  cerberus-notify:
+    name: Report failure to Slack (Cerberus)
+    needs:
+      - godeps-cache-amd64
+      - godeps-cache-arm64
+      - build-agent
+      - build-cluster-agent
+      - build-cluster-agent-image
+    if: >-
+      always()
+      && github.event_name == 'push'
+      && contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/cerberus-notify.yml
+    with:
+      suite: binary-builds
+    secrets:
+      CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }}
+      CERBERUS_API_TOKEN: ${{ secrets.CERBERUS_API_TOKEN }}