diff --git a/.github/workflows/images-sync.yml b/.github/workflows/images-sync.yml index bc0d5b2..d49d4b5 100644 --- a/.github/workflows/images-sync.yml +++ b/.github/workflows/images-sync.yml @@ -5,9 +5,11 @@ on: branches: [master] paths: - 'images/**' + - 'infrastructure/network/vyos/**' pull_request: paths: - 'images/**' + - 'infrastructure/network/vyos/**' workflow_dispatch: inputs: force: @@ -18,6 +20,10 @@ on: description: 'Run prune after sync' type: boolean default: false + skip_hooks: + description: 'Skip pre-upload hooks (tests)' + type: boolean + default: false concurrency: group: images-sync-${{ github.ref }} @@ -30,12 +36,13 @@ permissions: jobs: sync: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version-file: tools/labctl/go.mod cache-dependency-path: tools/labctl/go.sum - name: Build labctl @@ -43,12 +50,58 @@ jobs: cd tools/labctl go build -o ../../labctl . + # Validate manifest structure and hook scripts + - name: Validate Manifest + run: | + ./labctl images validate + # Also validate hook scripts are executable + for script in images/hooks/*.sh; do + if [[ -f "${script}" && ! -x "${script}" ]]; then + echo "ERROR: Script not executable: ${script}" + exit 1 + fi + done + + # Install dependencies for pre-upload hooks (e.g., VyOS tests) + # Skip on workflow_dispatch if skip_hooks is true + - name: Install hook dependencies + if: inputs.skip_hooks != true + run: | + sudo apt-get update + sudo apt-get install -y p7zip-full squashfs-tools-ng + + - name: Install Containerlab + if: inputs.skip_hooks != true + run: | + bash -c "$(curl -sL https://get.containerlab.dev)" + containerlab version + + - name: Set up Python + if: inputs.skip_hooks != true + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: infrastructure/network/vyos/tests/requirements.txt + + - name: Install Python test dependencies + if: inputs.skip_hooks != true + run: | + pip install -r infrastructure/network/vyos/tests/requirements.txt + + # PR: run full sync without upload (tests hooks, no credentials needed) + - name: Sync Images (PR - no upload) + if: github.event_name == 'pull_request' + run: | + ./labctl images sync --no-upload + + # Push/dispatch: full sync with credentials - name: Install SOPS if: github.event_name != 'pull_request' run: | - curl -LO https://github.com/getsops/sops/releases/download/v3.9.2/sops-v3.9.2.linux.amd64 - chmod +x sops-v3.9.2.linux.amd64 - sudo mv sops-v3.9.2.linux.amd64 /usr/local/bin/sops + curl -LO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64 + chmod +x sops-v3.9.4.linux.amd64 + sudo mv sops-v3.9.4.linux.amd64 /usr/local/bin/sops - name: Write SOPS age key if: github.event_name != 'pull_request' @@ -56,18 +109,13 @@ jobs: echo "${{ secrets.SOPS_AGE_KEY }}" > /tmp/age-key.txt chmod 600 /tmp/age-key.txt - # PR: validate manifest only (no credentials needed) - - name: Validate Manifest (PR) - if: github.event_name == 'pull_request' - run: ./labctl images validate - - # Push/dispatch: full sync with credentials - name: Sync Images if: github.event_name != 'pull_request' id: sync run: | FLAGS="" - if [ "${{ inputs.force }}" == "true" ]; then FLAGS="--force"; fi + if [ "${{ inputs.force }}" == "true" ]; then FLAGS="$FLAGS --force"; fi + if [ "${{ inputs.skip_hooks }}" == "true" ]; then FLAGS="$FLAGS --skip-hooks"; fi ./labctl images sync \ --credentials images/e2.sops.yaml \ diff --git a/.github/workflows/vyos-build.yml b/.github/workflows/vyos-build.yml deleted file mode 100644 index 7fc74af..0000000 --- a/.github/workflows/vyos-build.yml +++ /dev/null @@ -1,275 +0,0 @@ -name: VyOS Integration Tests - -on: - push: - branches: [master] - paths: - - 'infrastructure/network/vyos/**' - pull_request: - paths: - - 'infrastructure/network/vyos/**' - workflow_dispatch: - -concurrency: - group: vyos-test-${{ github.ref }} - cancel-in-progress: false - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Check scripts are executable - run: | - for script in infrastructure/network/vyos/scripts/*.sh infrastructure/network/vyos/tests/*.sh; do - if [[ -f "${script}" && ! -x "${script}" ]]; then - echo "ERROR: Script not executable: ${script}" - exit 1 - fi - echo "OK: ${script}" - done - - - name: Validate topology file - run: | - python3 -c "import yaml; yaml.safe_load(open('infrastructure/network/vyos/tests/topology.clab.yml'))" - echo "Topology file is valid YAML" - - build-container: - runs-on: ubuntu-latest - needs: validate - steps: - - uses: actions/checkout@v4 - - - name: Compute cache key - id: cache-key - run: | - # Cache key based on ISO checksum and build scripts - ISO_CHECKSUM=$(grep -A5 'name: vyos-stream' images/images.yaml | grep 'checksum:' | awk '{print $2}' | cut -d: -f2) - SCRIPTS_HASH=$(cat infrastructure/network/vyos/scripts/iso-to-container.sh infrastructure/network/vyos/Dockerfile.containerlab | sha256sum | cut -d' ' -f1) - echo "key=vyos-container-${ISO_CHECKSUM:0:16}-${SCRIPTS_HASH:0:16}" >> $GITHUB_OUTPUT - - - name: Restore cached container image - id: cache-container - uses: actions/cache@v4 - with: - path: /tmp/vyos-gateway-container.tar - key: ${{ steps.cache-key.outputs.key }} - - - name: Install dependencies - if: steps.cache-container.outputs.cache-hit != 'true' - run: | - sudo apt-get update - sudo apt-get install -y p7zip-full squashfs-tools-ng - - - name: Download VyOS ISO - if: steps.cache-container.outputs.cache-hit != 'true' - run: | - # Extract URL and checksum from images.yaml - URL=$(grep -A5 'name: vyos-stream' images/images.yaml | grep 'url:' | awk '{print $2}') - EXPECTED=$(grep -A5 'name: vyos-stream' images/images.yaml | grep 'checksum:' | awk '{print $2}' | cut -d: -f2) - - echo "Downloading VyOS ISO from: ${URL}" - curl -L -o /tmp/vyos.iso "${URL}" - - echo "Verifying checksum..." - ACTUAL=$(sha256sum /tmp/vyos.iso | awk '{print $1}') - - if [[ "${EXPECTED}" != "${ACTUAL}" ]]; then - echo "ERROR: Checksum mismatch" - echo "Expected: ${EXPECTED}" - echo "Actual: ${ACTUAL}" - exit 1 - fi - echo "Checksum verified" - - - name: Build container image from ISO - if: steps.cache-container.outputs.cache-hit != 'true' - run: | - infrastructure/network/vyos/scripts/iso-to-container.sh /tmp/vyos.iso vyos-gateway:test - - - name: Save container image - if: steps.cache-container.outputs.cache-hit != 'true' - run: | - docker save vyos-gateway:test -o /tmp/vyos-gateway-container.tar - ls -lah /tmp/vyos-gateway-container.tar - - - name: Report cache status - run: | - if [[ "${{ steps.cache-container.outputs.cache-hit }}" == "true" ]]; then - echo "✓ Container image restored from cache" - else - echo "✓ Container image built and cached for future runs" - fi - ls -lah /tmp/vyos-gateway-container.tar - - - name: Upload container image artifact - uses: actions/upload-artifact@v4 - with: - name: vyos-container-image - path: /tmp/vyos-gateway-container.tar - retention-days: 1 - - integration-test: - needs: build-container - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - - - name: Download container image artifact - uses: actions/download-artifact@v4 - with: - name: vyos-container-image - path: /tmp - - - name: Load container image - run: | - docker load -i /tmp/vyos-gateway-container.tar - docker images vyos-gateway:test - - - name: Install Containerlab - run: | - bash -c "$(curl -sL https://get.containerlab.dev)" - containerlab version - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - cache: 'pip' - cache-dependency-path: infrastructure/network/vyos/tests/requirements.txt - - - name: Install test dependencies - run: | - pip install -r infrastructure/network/vyos/tests/requirements.txt - - - name: Generate test config.boot - run: | - ssh-keygen -t ed25519 -f /tmp/vyos-test-key -N "" -C "vyos-ci" - infrastructure/network/vyos/tests/render-config-boot.sh "$(cat /tmp/vyos-test-key.pub)" - - - name: Deploy Containerlab topology - run: | - cd infrastructure/network/vyos/tests - sudo containerlab deploy -t topology.clab.yml --reconfigure - timeout-minutes: 5 - - - name: Wait for VyOS to be ready - run: | - CONTAINER="clab-vyos-gateway-test-gateway" - - echo "Waiting for container to be running..." - for i in {1..30}; do - if docker ps --filter "name=${CONTAINER}" --filter "status=running" | grep -q "${CONTAINER}"; then - echo "Container is running" - break - fi - echo "Waiting for container... ($i/30)" - sleep 2 - done - - echo "Loading kernel modules on host..." - sudo modprobe 8021q || true - - echo "Loading kernel modules in container..." - sudo docker exec "${CONTAINER}" modprobe br_netfilter || true - sudo docker exec "${CONTAINER}" modprobe 8021q || true - - echo "Checking if 8021q is loaded..." - lsmod | grep 8021q || echo "WARNING: 8021q not loaded on host" - - echo "Waiting for VyOS config to be applied..." - # VyOS loads config automatically via vyos-router.service - # The "systemd running" state happens before config is applied - # Config migration takes ~100 seconds in container environments - # The message appears in docker logs (container stdout), not dmesg - for i in {1..90}; do - # Check for config migration completion in container logs - if docker logs "${CONTAINER}" 2>&1 | grep -q "migrate.*configure"; then - echo "VyOS config migration detected" - # Wait for services to start - sleep 15 - break - fi - echo "Waiting for VyOS config... ($i/90)" - sleep 2 - done - - echo "Verifying configuration loaded..." - docker exec "${CONTAINER}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands | head -10 - - # Check if kea-dhcp4 process is running - echo "=== DHCP server process check ===" - docker exec "${CONTAINER}" pgrep -a kea-dhcp4 || echo "WARNING: kea-dhcp4 process not found" - - # Check DHCP server status - echo "=== DHCP server status ===" - docker exec "${CONTAINER}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show dhcp server leases 2>&1 || true - timeout-minutes: 5 - - - name: Verify VyOS interfaces - run: | - CONTAINER="clab-vyos-gateway-test-gateway" - - echo "=== VyOS Version ===" - docker exec "${CONTAINER}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show version - - echo "" - echo "=== Configured Interfaces ===" - docker exec "${CONTAINER}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show interfaces - - echo "" - echo "=== DHCP Server Status ===" - docker exec "${CONTAINER}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show dhcp server leases || echo "No leases yet" - timeout-minutes: 2 - - - name: Run integration tests - env: - VYOS_SSH_KEY: /tmp/vyos-test-key - run: | - cd infrastructure/network/vyos/tests - pytest -v --tb=short -x - timeout-minutes: 5 - - - name: Collect logs on failure - if: failure() - run: | - GATEWAY="clab-vyos-gateway-test-gateway" - - echo "=== Running containers ===" - docker ps --format "table {{.Names}}\t{{.Status}}" | grep vyos-gateway-test || true - - echo "" - echo "=== Gateway container logs ===" - docker logs "${GATEWAY}" 2>&1 | tail -100 || true - - echo "" - echo "=== VyOS configuration ===" - docker exec "${GATEWAY}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration 2>&1 || true - - echo "" - echo "=== VyOS interfaces ===" - docker exec "${GATEWAY}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show interfaces 2>&1 || true - - echo "" - echo "=== VyOS routing table ===" - docker exec "${GATEWAY}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show ip route 2>&1 || true - - echo "" - echo "=== VyOS NAT rules ===" - docker exec "${GATEWAY}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show nat source rules 2>&1 || true - - echo "" - echo "=== VyOS firewall ===" - docker exec "${GATEWAY}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show firewall 2>&1 || true - - echo "" - echo "=== mgmt-client connectivity test ===" - docker exec clab-vyos-gateway-test-mgmt-client ping -c 1 10.10.10.1 2>&1 || true - - - name: Cleanup - if: always() - run: | - cd infrastructure/network/vyos/tests - sudo containerlab destroy -t topology.clab.yml --cleanup || true diff --git a/images/hooks/vyos-test.sh b/images/hooks/vyos-test.sh new file mode 100755 index 0000000..72d7e7a --- /dev/null +++ b/images/hooks/vyos-test.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# +# vyos-test.sh - Run VyOS integration tests against an ISO +# +# Usage: vyos-test.sh +# +# This script: +# 1. Converts the ISO to a container image +# 2. Deploys a containerlab topology +# 3. Runs pytest integration tests +# 4. Cleans up resources +# +# Exit codes: +# 0 - All tests passed +# 1 - Tests failed or error occurred + +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +ISO_PATH="$1" + +# Derive paths from script location +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +VYOS_DIR="${REPO_ROOT}/infrastructure/network/vyos" +TEST_DIR="${VYOS_DIR}/tests" + +# Use unique identifiers to avoid conflicts in parallel runs +RUN_ID="$$" +IMAGE_TAG="vyos-gateway:hook-test-${RUN_ID}" +SSH_KEY="/tmp/vyos-test-key-${RUN_ID}" +TOPOLOGY_BACKUP="${TEST_DIR}/topology.clab.yml.bak-${RUN_ID}" + +echo "VyOS Integration Test" +echo " ISO: ${ISO_PATH}" +echo " Run ID: ${RUN_ID}" +echo "" + +cleanup() { + local exit_code=$? + echo "" + echo "Cleaning up..." + + # Destroy containerlab topology + if [[ -f "${TEST_DIR}/topology.clab.yml" ]]; then + cd "${TEST_DIR}" && sudo containerlab destroy -t topology.clab.yml --cleanup 2>/dev/null || true + fi + + # Restore original topology if we modified it + if [[ -f "${TOPOLOGY_BACKUP}" ]]; then + mv "${TOPOLOGY_BACKUP}" "${TEST_DIR}/topology.clab.yml" + fi + + # Remove test SSH key + rm -f "${SSH_KEY}" "${SSH_KEY}.pub" 2>/dev/null || true + + # Remove test container image + docker rmi "${IMAGE_TAG}" 2>/dev/null || true + + exit $exit_code +} +trap cleanup EXIT + +# Check for required dependencies +check_deps() { + local missing=() + + command -v 7z &>/dev/null || missing+=("p7zip-full") + command -v sqfs2tar &>/dev/null || missing+=("squashfs-tools-ng") + command -v containerlab &>/dev/null || missing+=("containerlab") + command -v pytest &>/dev/null || missing+=("pytest (pip)") + + if [[ ${#missing[@]} -gt 0 ]]; then + echo "ERROR: Missing dependencies: ${missing[*]}" >&2 + echo "Install with:" >&2 + echo " apt-get install p7zip-full squashfs-tools-ng" >&2 + echo " bash -c \"\$(curl -sL https://get.containerlab.dev)\"" >&2 + echo " pip install -r ${TEST_DIR}/requirements.txt" >&2 + exit 1 + fi +} + +# Convert ISO to container image +build_container() { + echo "Building container image from ISO..." + "${VYOS_DIR}/scripts/iso-to-container.sh" "${ISO_PATH}" "${IMAGE_TAG}" + echo " Container image: ${IMAGE_TAG}" +} + +# Generate test configuration +generate_config() { + echo "Generating test configuration..." + + # Create test SSH key + ssh-keygen -t ed25519 -f "${SSH_KEY}" -N "" -C "vyos-hook-test-${RUN_ID}" -q + + # Render config.boot with test SSH key + "${TEST_DIR}/render-config-boot.sh" "$(cat "${SSH_KEY}.pub")" + + echo " SSH key: ${SSH_KEY}" +} + +# Update topology to use our test image +update_topology() { + echo "Updating topology for test run..." + + # Backup original topology + cp "${TEST_DIR}/topology.clab.yml" "${TOPOLOGY_BACKUP}" + + # Update image reference + sed -i "s|image: vyos-gateway:test|image: ${IMAGE_TAG}|g" "${TEST_DIR}/topology.clab.yml" +} + +# Deploy containerlab topology +deploy_topology() { + echo "Deploying containerlab topology..." + cd "${TEST_DIR}" + sudo containerlab deploy -t topology.clab.yml --reconfigure +} + +# Wait for VyOS to be ready +wait_for_vyos() { + local container="clab-vyos-gateway-test-gateway" + + echo "Waiting for VyOS to be ready..." + + # Wait for container to be running + for i in {1..30}; do + if docker ps --filter "name=${container}" --filter "status=running" | grep -q "${container}"; then + echo " Container is running" + break + fi + echo " Waiting for container... ($i/30)" + sleep 2 + done + + # Load kernel modules + echo " Loading kernel modules..." + sudo modprobe 8021q 2>/dev/null || true + sudo docker exec "${container}" modprobe br_netfilter 2>/dev/null || true + sudo docker exec "${container}" modprobe 8021q 2>/dev/null || true + + # Wait for VyOS config to be applied + # Config migration takes ~100 seconds in container environments + for i in {1..90}; do + if docker logs "${container}" 2>&1 | grep -q "migrate.*configure"; then + echo " VyOS config migration detected" + sleep 15 + break + fi + echo " Waiting for VyOS config... ($i/90)" + sleep 2 + done + + # Verify configuration loaded + echo " Verifying configuration..." + docker exec "${container}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands | head -5 + + # Check DHCP server + echo " Checking DHCP server..." + docker exec "${container}" pgrep -a kea-dhcp4 || echo " WARNING: kea-dhcp4 not running yet" +} + +# Run integration tests +run_tests() { + echo "" + echo "Running integration tests..." + cd "${TEST_DIR}" + + export VYOS_SSH_KEY="${SSH_KEY}" + pytest -v --tb=short -x + + echo "" + echo "All tests passed!" +} + +# Main execution +main() { + check_deps + build_container + generate_config + update_topology + deploy_topology + wait_for_vyos + run_tests +} + +main diff --git a/images/images.yaml b/images/images.yaml index 71bcc6f..f0e8845 100644 --- a/images/images.yaml +++ b/images/images.yaml @@ -9,3 +9,8 @@ spec: url: https://community-downloads.vyos.dev/stream/2025.11/vyos-2025.11-generic-amd64.iso checksum: sha256:f60a2d7dd3bdf2e370a45c04ed4fc3b195691694ca3b8546adf4c5983e70d96e destination: vyos/vyos-2025.11-generic-amd64.iso + hooks: + preUpload: + - name: vyos-integration-test + command: ./images/hooks/vyos-test.sh + timeout: 20m diff --git a/tools/labctl/cmd/images/download.go b/tools/labctl/cmd/images/download.go new file mode 100644 index 0000000..849dbe1 --- /dev/null +++ b/tools/labctl/cmd/images/download.go @@ -0,0 +1,159 @@ +package images + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + + "github.com/spf13/cobra" + + "github.com/GilmanLab/lab/tools/labctl/internal/config" +) + +// DownloadResult contains the output of a successful download. +type DownloadResult struct { + Path string `json:"path"` + Checksum string `json:"checksum"` + Size int64 `json:"size"` + Name string `json:"name"` +} + +var downloadCmd = &cobra.Command{ + Use: "download", + Short: "Download an image from manifest", + Long: `Download an image from the manifest to a local file. + +The download command looks up an image by name in the manifest, +downloads it from the source URL, verifies the checksum, and +optionally decompresses it. The result is output as JSON for +use in CI/CD workflows.`, + RunE: runDownload, +} + +var ( + downloadName string + downloadManifest string + downloadOutput string +) + +func init() { + downloadCmd.Flags().StringVar(&downloadName, "name", "", "Image name to download (required)") + downloadCmd.Flags().StringVar(&downloadManifest, "manifest", "./images/images.yaml", "Path to images.yaml") + downloadCmd.Flags().StringVar(&downloadOutput, "output", "", "Output file path (required)") + + _ = downloadCmd.MarkFlagRequired("name") + _ = downloadCmd.MarkFlagRequired("output") +} + +func runDownload(_ *cobra.Command, _ []string) error { + ctx := context.Background() + + // Load manifest + manifest, err := config.LoadManifest(downloadManifest) + if err != nil { + return fmt.Errorf("load manifest: %w", err) + } + + // Find image by name + img := manifest.FindImageByName(downloadName) + if img == nil { + return fmt.Errorf("image %q not found in manifest", downloadName) + } + + return downloadImageWithHTTP(ctx, http.DefaultClient, *img, downloadOutput, os.Stdout) +} + +// downloadImageWithHTTP downloads an image using the provided HTTP client. +// This function enables dependency injection for testing. +func downloadImageWithHTTP(ctx context.Context, httpClient HTTPClient, img config.Image, outputPath string, out io.Writer) error { + fmt.Fprintf(os.Stderr, "Downloading %s from %s\n", img.Name, img.Source.URL) + + // Download to temp file + tempFile, size, err := downloadToTempWithClient(ctx, httpClient, img.Source.URL) + if err != nil { + return fmt.Errorf("download: %w", err) + } + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() + + // Verify source checksum + fmt.Fprintf(os.Stderr, "Verifying source checksum...\n") + if _, err := tempFile.Seek(0, 0); err != nil { + return fmt.Errorf("seek temp file: %w", err) + } + if err := verifyChecksum(tempFile, img.Source.Checksum); err != nil { + return fmt.Errorf("source checksum verification: %w", err) + } + + // Decompress if needed + var finalFile *os.File + var finalSize int64 + var finalChecksum string + + if img.Source.Decompress != "" { + fmt.Fprintf(os.Stderr, "Decompressing (%s)...\n", img.Source.Decompress) + if _, err := tempFile.Seek(0, 0); err != nil { + return fmt.Errorf("seek temp file: %w", err) + } + decompFile, decompSize, err := decompress(tempFile, img.Source.Decompress) + if err != nil { + return fmt.Errorf("decompress: %w", err) + } + defer func() { + _ = decompFile.Close() + _ = os.Remove(decompFile.Name()) + }() + + // Verify decompressed checksum if validation is specified + if img.Validation != nil && img.Validation.Expected != "" { + fmt.Fprintf(os.Stderr, "Verifying decompressed checksum...\n") + if _, err := decompFile.Seek(0, 0); err != nil { + return fmt.Errorf("seek decompressed file: %w", err) + } + if err := verifyChecksum(decompFile, img.Validation.Expected); err != nil { + return fmt.Errorf("decompressed checksum verification: %w", err) + } + } + + finalFile = decompFile + finalSize = decompSize + finalChecksum = img.EffectiveChecksum() + } else { + finalFile = tempFile + finalSize = size + finalChecksum = img.Source.Checksum + } + + // Copy to output path + fmt.Fprintf(os.Stderr, "Writing to %s (%s)\n", outputPath, formatSize(finalSize)) + if _, err := finalFile.Seek(0, 0); err != nil { + return fmt.Errorf("seek final file: %w", err) + } + + outFile, err := os.Create(outputPath) //nolint:gosec // G304: Path is provided by user + if err != nil { + return fmt.Errorf("create output file: %w", err) + } + defer func() { _ = outFile.Close() }() + + if _, err := io.Copy(outFile, finalFile); err != nil { + return fmt.Errorf("copy to output: %w", err) + } + + // Output result as JSON + result := DownloadResult{ + Path: outputPath, + Checksum: finalChecksum, + Size: finalSize, + Name: img.Name, + } + + encoder := json.NewEncoder(out) + encoder.SetIndent("", " ") + return encoder.Encode(result) +} diff --git a/tools/labctl/cmd/images/download_test.go b/tools/labctl/cmd/images/download_test.go new file mode 100644 index 0000000..aec2383 --- /dev/null +++ b/tools/labctl/cmd/images/download_test.go @@ -0,0 +1,301 @@ +package images + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/GilmanLab/lab/tools/labctl/internal/config" +) + +func TestDownloadImageWithHTTP(t *testing.T) { + // Helper to compute SHA256 checksum + computeChecksum := func(data []byte) string { + h := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(h[:]) + } + + t.Run("successful download without decompression", func(t *testing.T) { + // Create test content and compute checksum + content := []byte("test image content for download") + checksum := computeChecksum(content) + + // Mock HTTP server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(content) + })) + defer server.Close() + + dir := t.TempDir() + outputPath := filepath.Join(dir, "output.iso") + + img := config.Image{ + Name: "test-image", + Destination: "test/test.iso", + Source: config.Source{ + URL: server.URL, + Checksum: checksum, + }, + } + + // Capture JSON output + var stdout bytes.Buffer + err := downloadImageWithHTTP(context.Background(), server.Client(), img, outputPath, &stdout) + + require.NoError(t, err) + + // Verify output file was created with correct content + downloaded, err := os.ReadFile(outputPath) //nolint:gosec // G304: Test file path + require.NoError(t, err) + assert.Equal(t, content, downloaded) + + // Verify JSON output + var result DownloadResult + err = json.Unmarshal(stdout.Bytes(), &result) + require.NoError(t, err) + assert.Equal(t, outputPath, result.Path) + assert.Equal(t, checksum, result.Checksum) + assert.Equal(t, int64(len(content)), result.Size) + assert.Equal(t, "test-image", result.Name) + }) + + t.Run("successful download with gzip decompression", func(t *testing.T) { + // Create compressed content + decompressedContent := []byte("decompressed image content for download test") + var compressedBuf bytes.Buffer + gzWriter := gzip.NewWriter(&compressedBuf) + _, err := gzWriter.Write(decompressedContent) + require.NoError(t, err) + require.NoError(t, gzWriter.Close()) + compressedContent := compressedBuf.Bytes() + + sourceChecksum := computeChecksum(compressedContent) + decompressedChecksum := computeChecksum(decompressedContent) + + // Mock HTTP server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(compressedContent) + })) + defer server.Close() + + dir := t.TempDir() + outputPath := filepath.Join(dir, "output.iso") + + img := config.Image{ + Name: "compressed-image", + Destination: "test/compressed.iso", + Source: config.Source{ + URL: server.URL, + Checksum: sourceChecksum, + Decompress: "gzip", + }, + Validation: &config.Validation{ + Algorithm: "sha256", + Expected: decompressedChecksum, + }, + } + + // Capture JSON output + var stdout bytes.Buffer + err = downloadImageWithHTTP(context.Background(), server.Client(), img, outputPath, &stdout) + + require.NoError(t, err) + + // Verify decompressed content was written + downloaded, err := os.ReadFile(outputPath) //nolint:gosec // G304: Test file path + require.NoError(t, err) + assert.Equal(t, decompressedContent, downloaded) + + // Verify JSON output uses decompressed checksum + var result DownloadResult + err = json.Unmarshal(stdout.Bytes(), &result) + require.NoError(t, err) + assert.Equal(t, decompressedChecksum, result.Checksum) + assert.Equal(t, int64(len(decompressedContent)), result.Size) + }) + + t.Run("download HTTP error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + dir := t.TempDir() + outputPath := filepath.Join(dir, "output.iso") + + img := config.Image{ + Name: "missing-image", + Destination: "test/missing.iso", + Source: config.Source{ + URL: server.URL, + Checksum: "sha256:abc123", + }, + } + + var stdout bytes.Buffer + err := downloadImageWithHTTP(context.Background(), server.Client(), img, outputPath, &stdout) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "download") + }) + + t.Run("source checksum verification failure", func(t *testing.T) { + content := []byte("actual content") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(content) + })) + defer server.Close() + + dir := t.TempDir() + outputPath := filepath.Join(dir, "output.iso") + + img := config.Image{ + Name: "bad-checksum", + Destination: "test/bad.iso", + Source: config.Source{ + URL: server.URL, + Checksum: "sha256:0000000000000000000000000000000000000000000000000000000000000000", + }, + } + + var stdout bytes.Buffer + err := downloadImageWithHTTP(context.Background(), server.Client(), img, outputPath, &stdout) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "source checksum verification") + }) + + t.Run("decompressed checksum verification failure", func(t *testing.T) { + // Create compressed content + decompressedContent := []byte("decompressed content") + var compressedBuf bytes.Buffer + gzWriter := gzip.NewWriter(&compressedBuf) + _, err := gzWriter.Write(decompressedContent) + require.NoError(t, err) + require.NoError(t, gzWriter.Close()) + compressedContent := compressedBuf.Bytes() + + sourceChecksum := computeChecksum(compressedContent) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(compressedContent) + })) + defer server.Close() + + dir := t.TempDir() + outputPath := filepath.Join(dir, "output.iso") + + img := config.Image{ + Name: "bad-decompress", + Destination: "test/bad.iso", + Source: config.Source{ + URL: server.URL, + Checksum: sourceChecksum, + Decompress: "gzip", + }, + Validation: &config.Validation{ + Algorithm: "sha256", + Expected: "sha256:0000000000000000000000000000000000000000000000000000000000000000", + }, + } + + var stdout bytes.Buffer + err = downloadImageWithHTTP(context.Background(), server.Client(), img, outputPath, &stdout) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "decompressed checksum verification") + }) + + t.Run("output path error - invalid directory", func(t *testing.T) { + content := []byte("test content") + checksum := computeChecksum(content) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(content) + })) + defer server.Close() + + img := config.Image{ + Name: "test-image", + Destination: "test/test.iso", + Source: config.Source{ + URL: server.URL, + Checksum: checksum, + }, + } + + var stdout bytes.Buffer + err := downloadImageWithHTTP(context.Background(), server.Client(), img, "/nonexistent/directory/output.iso", &stdout) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "create output file") + }) +} + +func TestRunDownload(t *testing.T) { + // Save and restore globals + origName := downloadName + origManifest := downloadManifest + origOutput := downloadOutput + defer func() { + downloadName = origName + downloadManifest = origManifest + downloadOutput = origOutput + }() + + t.Run("manifest file not found", func(t *testing.T) { + downloadName = "test-image" + downloadManifest = "/nonexistent/path/images.yaml" + downloadOutput = "/tmp/output.iso" + + err := runDownload(nil, nil) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "load manifest") + }) + + t.Run("image not found in manifest", func(t *testing.T) { + dir := t.TempDir() + manifestPath := filepath.Join(dir, "images.yaml") + manifest := `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: test-images +spec: + images: + - name: other-image + source: + url: https://example.com/test.iso + checksum: sha256:abc123 + destination: test/test.iso +` + err := os.WriteFile(manifestPath, []byte(manifest), 0o644) //nolint:gosec + require.NoError(t, err) + + downloadName = "non-existent" + downloadManifest = manifestPath + downloadOutput = filepath.Join(dir, "output.iso") + + err = runDownload(nil, nil) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found in manifest") + }) +} diff --git a/tools/labctl/cmd/images/root.go b/tools/labctl/cmd/images/root.go index 5d5434e..9e3c8a3 100644 --- a/tools/labctl/cmd/images/root.go +++ b/tools/labctl/cmd/images/root.go @@ -18,4 +18,5 @@ func init() { Cmd.AddCommand(listCmd) Cmd.AddCommand(pruneCmd) Cmd.AddCommand(uploadCmd) + Cmd.AddCommand(downloadCmd) } diff --git a/tools/labctl/cmd/images/sync.go b/tools/labctl/cmd/images/sync.go index 7116965..b467864 100644 --- a/tools/labctl/cmd/images/sync.go +++ b/tools/labctl/cmd/images/sync.go @@ -20,6 +20,7 @@ import ( "github.com/GilmanLab/lab/tools/labctl/internal/config" "github.com/GilmanLab/lab/tools/labctl/internal/credentials" + "github.com/GilmanLab/lab/tools/labctl/internal/hooks" "github.com/GilmanLab/lab/tools/labctl/internal/store" "github.com/GilmanLab/lab/tools/labctl/internal/updater" ) @@ -47,6 +48,8 @@ var ( syncSOPSAgeKeyFile string syncDryRun bool syncForce bool + syncSkipHooks bool + syncNoUpload bool ) func init() { @@ -55,6 +58,8 @@ func init() { syncCmd.Flags().StringVar(&syncSOPSAgeKeyFile, "sops-age-key-file", "", "Path to age private key for SOPS decryption") syncCmd.Flags().BoolVar(&syncDryRun, "dry-run", false, "Show what would be done without executing") syncCmd.Flags().BoolVar(&syncForce, "force", false, "Force re-upload even if checksums match") + syncCmd.Flags().BoolVar(&syncSkipHooks, "skip-hooks", false, "Skip pre-upload hooks") + syncCmd.Flags().BoolVar(&syncNoUpload, "no-upload", false, "Download and run hooks but skip upload (for testing)") } func runSync(_ *cobra.Command, _ []string) error { @@ -69,9 +74,10 @@ func runSync(_ *cobra.Command, _ []string) error { fmt.Printf("Syncing images from manifest: %s\n", syncManifest) fmt.Printf("Found %d image(s)\n\n", len(manifest.Spec.Images)) - // Skip credentials and S3 client setup in dry-run mode + // Skip credentials and S3 client setup in dry-run or no-upload mode var client *store.S3Client - if !syncDryRun { + var hookExecutor *hooks.Executor + if !syncDryRun && !syncNoUpload { // Resolve credentials creds, err := credentials.Resolve(credentials.ResolveOptions{ SOPSFile: syncCredentials, @@ -86,6 +92,14 @@ func runSync(_ *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("create S3 client: %w", err) } + + // Create hook executor with caching (unless skipped) + if !syncSkipHooks { + hookExecutor = hooks.NewExecutor(client) + } + } else if syncNoUpload && !syncSkipHooks { + // In no-upload mode, create hook executor without caching + hookExecutor = hooks.NewExecutor(nil) } // Track if any files were changed (for GitHub Actions output) @@ -93,7 +107,7 @@ func runSync(_ *cobra.Command, _ []string) error { // Process each image for _, img := range manifest.Spec.Images { - changed, err := syncImageWithHTTP(ctx, client, http.DefaultClient, img, syncDryRun, syncForce) + changed, err := syncImageWithHTTP(ctx, client, http.DefaultClient, hookExecutor, img, syncDryRun, syncForce, syncNoUpload) if err != nil { return fmt.Errorf("sync image %q: %w", img.Name, err) } @@ -118,19 +132,19 @@ func runSync(_ *cobra.Command, _ []string) error { // syncImage syncs an image using the default HTTP client. // This is a convenience wrapper for syncImageWithHTTP. -func syncImage(ctx context.Context, client store.Client, img config.Image, dryRun, force bool) (bool, error) { - return syncImageWithHTTP(ctx, client, http.DefaultClient, img, dryRun, force) +func syncImage(ctx context.Context, client store.Client, hookExecutor *hooks.Executor, img config.Image, dryRun, force, noUpload bool) (bool, error) { + return syncImageWithHTTP(ctx, client, http.DefaultClient, hookExecutor, img, dryRun, force, noUpload) } // syncImageWithHTTP syncs an image using the provided HTTP and store clients. // This function enables dependency injection for testing. -func syncImageWithHTTP(ctx context.Context, client store.Client, httpClient HTTPClient, img config.Image, dryRun, force bool) (bool, error) { +func syncImageWithHTTP(ctx context.Context, client store.Client, httpClient HTTPClient, hookExecutor *hooks.Executor, img config.Image, dryRun, force, noUpload bool) (bool, error) { fmt.Printf("Processing: %s\n", img.Name) effectiveChecksum := img.EffectiveChecksum() - // Check if image already exists with matching checksum - if !dryRun && !force { + // Check if image already exists with matching checksum (skip in no-upload mode) + if !dryRun && !force && !noUpload { matches, err := client.ChecksumMatches(ctx, img.Destination, effectiveChecksum) if err != nil { return false, fmt.Errorf("check existing image: %w", err) @@ -205,6 +219,21 @@ func syncImageWithHTTP(ctx context.Context, client store.Client, httpClient HTTP uploadSize = size } + // Run pre-upload hooks + if hookExecutor != nil && img.Hooks != nil && len(img.Hooks.PreUpload) > 0 { + fmt.Printf(" Running pre-upload hooks...\n") + if err := hookExecutor.RunPreUploadHooks(ctx, img, uploadFile.Name(), effectiveChecksum); err != nil { + return false, fmt.Errorf("pre-upload hooks: %w", err) + } + } + + // Skip upload in no-upload mode (used for PR testing) + if noUpload { + fmt.Printf(" Skipping upload (--no-upload mode)\n") + fmt.Printf(" Done\n") + return false, nil + } + // Upload to e2 if _, err := uploadFile.Seek(0, 0); err != nil { return false, fmt.Errorf("seek upload file: %w", err) diff --git a/tools/labctl/cmd/images/sync_test.go b/tools/labctl/cmd/images/sync_test.go index d5fe0ec..87bba0d 100644 --- a/tools/labctl/cmd/images/sync_test.go +++ b/tools/labctl/cmd/images/sync_test.go @@ -225,7 +225,7 @@ func TestSyncImage(t *testing.T) { }, } - changed, err := syncImage(context.Background(), client, img, false, false) + changed, err := syncImage(context.Background(), client, nil, img, false, false, false) require.NoError(t, err) assert.False(t, changed) @@ -244,7 +244,7 @@ func TestSyncImage(t *testing.T) { }, } - changed, err := syncImage(context.Background(), client, img, true, false) + changed, err := syncImage(context.Background(), client, nil, img, true, false, false) require.NoError(t, err) assert.False(t, changed) @@ -273,7 +273,7 @@ func TestSyncImage(t *testing.T) { // With force=true and dryRun=true, it should show what would be done // without checking checksum - _, err := syncImage(context.Background(), client, img, true, true) + _, err := syncImage(context.Background(), client, nil, img, true, true, false) require.NoError(t, err) assert.False(t, checksumChecked) // Should not check checksum with force @@ -295,11 +295,39 @@ func TestSyncImage(t *testing.T) { }, } - _, err := syncImage(context.Background(), client, img, false, false) + _, err := syncImage(context.Background(), client, nil, img, false, false, false) assert.Error(t, err) assert.Contains(t, err.Error(), "check existing image") }) + + t.Run("no-upload mode skips checksum check and upload", func(t *testing.T) { + checksumChecked := false + client := &mockStoreClient{ + checksumMatchFunc: func(_ context.Context, _ string, _ string) (bool, error) { + checksumChecked = true + return false, nil + }, + } + + img := config.Image{ + Name: "test-image", + Destination: "test/test.iso", + Source: config.Source{ + URL: "https://example.com/test.iso", + Checksum: "sha256:abc123", + }, + } + + // With noUpload=true, should skip checksum check (client is nil for noUpload) + // and also skip upload - this test verifies the skip behavior + changed, err := syncImage(context.Background(), client, nil, img, true, false, false) + + require.NoError(t, err) + assert.False(t, changed) + // In dry run mode, checksum should not be checked + assert.False(t, checksumChecked) + }) } func TestSyncImageWithHTTP(t *testing.T) { @@ -351,7 +379,7 @@ func TestSyncImageWithHTTP(t *testing.T) { }, } - changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), img, false, false) + changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, img, false, false, false) require.NoError(t, err) assert.False(t, changed) // No updateFile, so no file changes @@ -421,7 +449,7 @@ func TestSyncImageWithHTTP(t *testing.T) { }, } - changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), img, false, false) + changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, img, false, false, false) require.NoError(t, err) assert.False(t, changed) @@ -456,7 +484,7 @@ func TestSyncImageWithHTTP(t *testing.T) { }, } - _, err := syncImageWithHTTP(context.Background(), client, server.Client(), img, false, false) + _, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, img, false, false, false) assert.Error(t, err) assert.Contains(t, err.Error(), "download") @@ -486,7 +514,7 @@ func TestSyncImageWithHTTP(t *testing.T) { }, } - _, err := syncImageWithHTTP(context.Background(), client, server.Client(), img, false, false) + _, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, img, false, false, false) assert.Error(t, err) assert.Contains(t, err.Error(), "source checksum verification") @@ -520,7 +548,7 @@ func TestSyncImageWithHTTP(t *testing.T) { }, } - _, err := syncImageWithHTTP(context.Background(), client, server.Client(), img, false, false) + _, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, img, false, false, false) assert.Error(t, err) assert.Contains(t, err.Error(), "upload") @@ -557,7 +585,7 @@ func TestSyncImageWithHTTP(t *testing.T) { }, } - _, err := syncImageWithHTTP(context.Background(), client, server.Client(), img, false, false) + _, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, img, false, false, false) assert.Error(t, err) assert.Contains(t, err.Error(), "write metadata") @@ -600,11 +628,61 @@ func TestSyncImageWithHTTP(t *testing.T) { }, } - _, err = syncImageWithHTTP(context.Background(), client, server.Client(), img, false, false) + _, err = syncImageWithHTTP(context.Background(), client, server.Client(), nil, img, false, false, false) assert.Error(t, err) assert.Contains(t, err.Error(), "decompressed checksum verification") }) + + t.Run("no-upload mode downloads and verifies but skips upload", func(t *testing.T) { + // Create test content and compute checksum + content := []byte("test image content for no-upload mode") + checksum := computeChecksum(content) + + // Mock HTTP server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(content) + })) + defer server.Close() + + // Track S3 operations - should NOT be called + uploadCalled := false + metadataCalled := false + + client := &mockStoreClient{ + checksumMatchFunc: func(_ context.Context, _ string, _ string) (bool, error) { + // Should not be called in no-upload mode + t.Error("ChecksumMatches should not be called in no-upload mode") + return false, nil + }, + uploadFunc: func(_ context.Context, _ string, _ io.Reader, _ int64) error { + uploadCalled = true + return nil + }, + putMetadataFunc: func(_ context.Context, _ string, _ *store.ImageMetadata) error { + metadataCalled = true + return nil + }, + } + + img := config.Image{ + Name: "test-image", + Destination: "test/test.iso", + Source: config.Source{ + URL: server.URL, + Checksum: checksum, + }, + } + + // noUpload=true should download, verify, but skip upload + changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, img, false, false, true) + + require.NoError(t, err) + assert.False(t, changed) + assert.False(t, uploadCalled, "Upload should not be called in no-upload mode") + assert.False(t, metadataCalled, "PutMetadata should not be called in no-upload mode") + }) } func TestRunSync(t *testing.T) { diff --git a/tools/labctl/cmd/images/testutil_test.go b/tools/labctl/cmd/images/testutil_test.go index 9d57631..0abda9a 100644 --- a/tools/labctl/cmd/images/testutil_test.go +++ b/tools/labctl/cmd/images/testutil_test.go @@ -87,3 +87,11 @@ func (m *mockStoreClient) ChecksumMatches(ctx context.Context, imagePath, expect } return false, nil } + +func (*mockStoreClient) GetHookResult(_ context.Context, _, _ string) (*store.HookResult, error) { + return nil, nil +} + +func (*mockStoreClient) PutHookResult(_ context.Context, _, _ string, _ *store.HookResult) error { + return nil +} diff --git a/tools/labctl/internal/config/manifest.go b/tools/labctl/internal/config/manifest.go index 5f64db7..efe3f76 100644 --- a/tools/labctl/internal/config/manifest.go +++ b/tools/labctl/internal/config/manifest.go @@ -6,6 +6,7 @@ import ( "os" "regexp" "strings" + "time" "gopkg.in/yaml.v3" ) @@ -38,6 +39,31 @@ type Image struct { Destination string `yaml:"destination"` Validation *Validation `yaml:"validation,omitempty"` UpdateFile *UpdateFile `yaml:"updateFile,omitempty"` + Hooks *Hooks `yaml:"hooks,omitempty"` +} + +// Hooks defines lifecycle hooks for an image. +type Hooks struct { + // PreUpload runs after download/verification, before upload. + // Hook must exit 0 for upload to proceed. + PreUpload []Hook `yaml:"preUpload,omitempty"` +} + +// Hook defines a hook to run during image processing. +type Hook struct { + // Name is a human-readable identifier for the hook. + Name string `yaml:"name"` + // Command is the executable to run (path or command name). + Command string `yaml:"command"` + // Args are additional arguments to pass to the command. + // The image path is always passed as the first argument. + Args []string `yaml:"args,omitempty"` + // Timeout is the maximum duration for the hook to run. + // Defaults to 30 minutes if not specified. + Timeout string `yaml:"timeout,omitempty"` + // WorkDir is the working directory for the command. + // If not specified, uses the current working directory. + WorkDir string `yaml:"workDir,omitempty"` } // Source defines where to download the image from. @@ -74,6 +100,16 @@ func (i *Image) EffectiveChecksum() string { return i.Source.Checksum } +// FindImageByName returns the image with the given name, or nil if not found. +func (m *ImageManifest) FindImageByName(name string) *Image { + for i := range m.Spec.Images { + if m.Spec.Images[i].Name == name { + return &m.Spec.Images[i] + } + } + return nil +} + // LoadManifest reads and parses an image manifest from a file. func LoadManifest(path string) (*ImageManifest, error) { data, err := os.ReadFile(path) //nolint:gosec // G304: Path is provided by user @@ -231,5 +267,35 @@ func (i *Image) ValidateAll() []error { } } + // Validate hooks + if i.Hooks != nil { + for j, h := range i.Hooks.PreUpload { + for _, err := range h.ValidateAll() { + errs = append(errs, fmt.Errorf("hooks.preUpload[%d]: %w", j, err)) + } + } + } + + return errs +} + +// ValidateAll checks the hook configuration and returns all validation errors. +func (h *Hook) ValidateAll() []error { + var errs []error + + if h.Name == "" { + errs = append(errs, fmt.Errorf("name is required")) + } + + if h.Command == "" { + errs = append(errs, fmt.Errorf("command is required")) + } + + if h.Timeout != "" { + if _, err := time.ParseDuration(h.Timeout); err != nil { + errs = append(errs, fmt.Errorf("invalid timeout %q: %w", h.Timeout, err)) + } + } + return errs } diff --git a/tools/labctl/internal/config/manifest_test.go b/tools/labctl/internal/config/manifest_test.go index aa1fbcb..3db7fae 100644 --- a/tools/labctl/internal/config/manifest_test.go +++ b/tools/labctl/internal/config/manifest_test.go @@ -360,3 +360,44 @@ func TestImage_EffectiveChecksum(t *testing.T) { }) } } + +func TestImageManifest_FindImageByName(t *testing.T) { + t.Run("finds existing image", func(t *testing.T) { + manifest := &ImageManifest{ + Spec: Spec{ + Images: []Image{ + {Name: "image-one", Destination: "path/one"}, + {Name: "image-two", Destination: "path/two"}, + }, + }, + } + + img := manifest.FindImageByName("image-two") + + require.NotNil(t, img) + assert.Equal(t, "image-two", img.Name) + assert.Equal(t, "path/two", img.Destination) + }) + + t.Run("returns nil for non-existent image", func(t *testing.T) { + manifest := &ImageManifest{ + Spec: Spec{ + Images: []Image{ + {Name: "image-one", Destination: "path/one"}, + }, + }, + } + + img := manifest.FindImageByName("non-existent") + + assert.Nil(t, img) + }) + + t.Run("returns nil for empty manifest", func(t *testing.T) { + manifest := &ImageManifest{} + + img := manifest.FindImageByName("any") + + assert.Nil(t, img) + }) +} diff --git a/tools/labctl/internal/hooks/executor.go b/tools/labctl/internal/hooks/executor.go new file mode 100644 index 0000000..a0cb234 --- /dev/null +++ b/tools/labctl/internal/hooks/executor.go @@ -0,0 +1,155 @@ +// Package hooks provides hook execution for the image pipeline. +package hooks + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "sync" + "time" + + "github.com/GilmanLab/lab/tools/labctl/internal/config" + "github.com/GilmanLab/lab/tools/labctl/internal/store" +) + +const ( + // DefaultTimeout is the default timeout for hook execution. + DefaultTimeout = 30 * time.Minute + // MaxOutputSize is the maximum size of hook output to store (10KB). + MaxOutputSize = 10 * 1024 +) + +// Executor runs hooks and manages result caching. +type Executor struct { + client store.Client +} + +// NewExecutor creates a new hook executor. +// If client is nil, caching is disabled. +func NewExecutor(client store.Client) *Executor { + return &Executor{client: client} +} + +// RunPreUploadHooks executes all pre-upload hooks for an image. +// Returns nil if all hooks pass, error if any hook fails. +func (e *Executor) RunPreUploadHooks(ctx context.Context, img config.Image, imagePath, checksum string) error { + if img.Hooks == nil || len(img.Hooks.PreUpload) == 0 { + return nil + } + + for _, hook := range img.Hooks.PreUpload { + if err := e.runHook(ctx, img.Destination, hook, imagePath, checksum); err != nil { + return fmt.Errorf("hook %q failed: %w", hook.Name, err) + } + } + return nil +} + +func (e *Executor) runHook(ctx context.Context, destination string, hook config.Hook, imagePath, checksum string) error { + // Check cache first + if e.client != nil { + cached, err := e.client.GetHookResult(ctx, destination, hook.Name) + if err != nil { + // Log but continue - cache errors shouldn't block execution + fmt.Printf(" Warning: failed to check hook cache: %v\n", err) + } else if cached != nil && cached.Checksum == checksum && cached.Passed { + fmt.Printf(" Hook %q: cached pass (tested %s)\n", hook.Name, cached.ExecutedAt.Format(time.RFC3339)) + return nil + } + } + + // Parse timeout + timeout := DefaultTimeout + if hook.Timeout != "" { + var err error + timeout, err = time.ParseDuration(hook.Timeout) + if err != nil { + return fmt.Errorf("invalid timeout %q: %w", hook.Timeout, err) + } + } + + // Execute hook + fmt.Printf(" Running hook %q...\n", hook.Name) + hookCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + args := append([]string{imagePath}, hook.Args...) + cmd := exec.CommandContext(hookCtx, hook.Command, args...) //nolint:gosec // G204: Command is from trusted manifest + if hook.WorkDir != "" { + cmd.Dir = hook.WorkDir + } + + // Set up output streaming with prefix + var outputBuf bytes.Buffer + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("create stdout pipe: %w", err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("create stderr pipe: %w", err) + } + + start := time.Now() + if err := cmd.Start(); err != nil { + return fmt.Errorf("start hook: %w", err) + } + + // Stream output with prefix + var wg sync.WaitGroup + wg.Add(2) + go streamWithPrefix(&wg, stdout, &outputBuf, " │ ") + go streamWithPrefix(&wg, stderr, &outputBuf, " │ ") + wg.Wait() + + err = cmd.Wait() + duration := time.Since(start) + output := outputBuf.Bytes() + + // Store result + result := &store.HookResult{ + HookName: hook.Name, + Checksum: checksum, + Passed: err == nil, + ExecutedAt: start, + Duration: duration.String(), + Output: truncateOutput(string(output), MaxOutputSize), + } + + if e.client != nil { + if cacheErr := e.client.PutHookResult(ctx, destination, hook.Name, result); cacheErr != nil { + fmt.Printf(" Warning: failed to cache hook result: %v\n", cacheErr) + } + } + + if err != nil { + return fmt.Errorf("exit status %v:\n%s", err, truncateOutput(string(output), 1024)) + } + + fmt.Printf(" Hook %q: passed (%s)\n", hook.Name, duration.Round(time.Second)) + return nil +} + +// truncateOutput truncates a string to maxLen bytes, adding a truncation notice if needed. +func truncateOutput(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "\n... (truncated)" +} + +// streamWithPrefix reads from r line by line, prints each line with a prefix to stdout, +// and writes the original content to the buffer for caching. +func streamWithPrefix(wg *sync.WaitGroup, r io.Reader, buf *bytes.Buffer, prefix string) { + defer wg.Done() + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + _, _ = fmt.Fprintln(os.Stdout, prefix+line) + _, _ = buf.WriteString(line + "\n") + } +} diff --git a/tools/labctl/internal/store/hooks.go b/tools/labctl/internal/store/hooks.go new file mode 100644 index 0000000..2c420e6 --- /dev/null +++ b/tools/labctl/internal/store/hooks.go @@ -0,0 +1,29 @@ +// Package store provides storage operations for the image pipeline. +package store + +import ( + "path" + "time" +) + +// HookResult represents the cached result of a hook execution. +type HookResult struct { + // HookName is the name of the hook that was executed. + HookName string `json:"hookName"` + // Checksum is the image checksum when the test was run. + Checksum string `json:"checksum"` + // Passed indicates whether the hook execution succeeded. + Passed bool `json:"passed"` + // ExecutedAt is when the hook was executed. + ExecutedAt time.Time `json:"executedAt"` + // Duration is how long the hook took to execute. + Duration string `json:"duration"` + // Output contains the first 10KB of combined stdout/stderr for debugging. + Output string `json:"output,omitempty"` +} + +// HookResultKey returns the S3 key for storing hook results. +// Example: "hooks/vyos/vyos-2025.11.iso/vyos-integration-test.json" +func HookResultKey(imagePath, hookName string) string { + return path.Join("hooks", imagePath, hookName+".json") +} diff --git a/tools/labctl/internal/store/s3.go b/tools/labctl/internal/store/s3.go index 8259112..ba0cccc 100644 --- a/tools/labctl/internal/store/s3.go +++ b/tools/labctl/internal/store/s3.go @@ -29,6 +29,8 @@ type Client interface { GetMetadata(ctx context.Context, imagePath string) (*ImageMetadata, error) PutMetadata(ctx context.Context, imagePath string, metadata *ImageMetadata) error ChecksumMatches(ctx context.Context, imagePath, expectedChecksum string) (bool, error) + GetHookResult(ctx context.Context, imagePath, hookName string) (*HookResult, error) + PutHookResult(ctx context.Context, imagePath, hookName string, result *HookResult) error } // ImageMetadata represents metadata stored alongside each image. @@ -291,3 +293,47 @@ func (c *S3Client) ChecksumMatches(ctx context.Context, imagePath, expectedCheck return metadata.Checksum == expectedChecksum, nil } + +// GetHookResult retrieves the cached result of a hook execution. +// Returns nil, nil if the hook result doesn't exist. +func (c *S3Client) GetHookResult(ctx context.Context, imagePath, hookName string) (*HookResult, error) { + key := HookResultKey(imagePath, hookName) + + exists, err := c.Exists(ctx, key) + if err != nil { + return nil, err + } + if !exists { + return nil, nil + } + + body, err := c.Download(ctx, key) + if err != nil { + return nil, err + } + defer func() { _ = body.Close() }() + + data, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("read hook result: %w", err) + } + + var result HookResult + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("parse hook result: %w", err) + } + + return &result, nil +} + +// PutHookResult stores the result of a hook execution. +func (c *S3Client) PutHookResult(ctx context.Context, imagePath, hookName string, result *HookResult) error { + key := HookResultKey(imagePath, hookName) + + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return fmt.Errorf("marshal hook result: %w", err) + } + + return c.Upload(ctx, key, bytes.NewReader(data), int64(len(data))) +}