benchmark.yml

name: Memory Tracker Benchmark

on:
  workflow_dispatch:
    inputs:
      commit_range:
        description: 'Commit range to benchmark (e.g., HEAD~10..HEAD)'
        required: true
        default: 'HEAD~5..HEAD'
      binary_id:
        description: 'Binary ID to use for benchmarking'
        required: true
        default: 'default'
      environment_id:
        description: 'Environment ID'
        required: true
        default: 'linux-x86_64'
      server_url:
        description: 'Memory tracker server URL'
        required: false
        default: 'https://memory.python.org'
      cpython_repo:
        description: 'CPython repository URL'
        required: false
        default: 'https://github.com/python/cpython.git'
      configure_flags:
        description: 'Configure flags for CPython build'
        required: false
        default: '-C'
      make_flags:
        description: 'Make flags for CPython build'
        required: false
        default: '-j'
      llvm:
        description: 'LLVM version to use'
        required: false
        default: '19'

jobs:
  benchmark:
    # Keep in sync with the Ubuntu version CPython's CI runs on
    runs-on: ubuntu-26.04
    
    steps:
    - name: Checkout memory tracker
      uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      
    - name: Set up Python
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: '3.11'
        
    - name: Clone CPython repository
      run: |
        git clone ${{ github.event.inputs.cpython_repo }} cpython
        cd cpython
        git fetch --depth=200
        
    - name: Install memory tracker worker
      run: |
        cd worker
        pip install -e .
        
    - name: Install build dependencies
      run: |
        # Install CPython dependencies using their script
        cd cpython
        sudo .github/workflows/posix-deps-apt.sh

        # Install JIT dependencies
        if [ "${{ inputs.binary_id }}" = "jit" ]; then
          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ inputs.llvm }}
        fi

        # Install Memray dependencies
        sudo apt-get install -y \
          python3-dev \
          libdebuginfod-dev \
          libunwind-dev \
          liblz4-dev
          
    - name: Run memory benchmarks
      env:
        MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }}
      run: |
        if [ "${{ inputs.binary_id }}" = "jit" ]; then
          export PATH="$(llvm-config-${{ inputs.llvm }} --bindir):$PATH"
        fi

        # Build command with conditional flags
        CMD="memory-tracker benchmark '${{ github.event.inputs.commit_range }}'"
        CMD="$CMD --repo-path ./cpython"
        CMD="$CMD --binary-id '${{ github.event.inputs.binary_id }}'"
        CMD="$CMD --environment-id '${{ github.event.inputs.environment_id }}'"
        CMD="$CMD --api-base '${{ github.event.inputs.server_url }}'"
        CMD="$CMD --output-dir ./benchmark_results"
        CMD="$CMD --force"
        CMD="$CMD -vv"
        
        # Add configure flags if provided
        if [ -n "${{ github.event.inputs.configure_flags }}" ]; then
          CMD="$CMD --configure-flags='${{ github.event.inputs.configure_flags }}'"
        fi
        
        # Add make flags if provided
        if [ -n "${{ github.event.inputs.make_flags }}" ]; then
          CMD="$CMD --make-flags='${{ github.event.inputs.make_flags }}'"
        fi
        
        echo "Running: $CMD"
        eval $CMD
          
    - name: Upload benchmark results (if failed)
      if: failure()
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: benchmark-logs
        path: |
          *.log
          ./benchmark_results/
        retention-days: 7
        
    - name: Upload benchmark results (on success)
      if: success()
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: benchmark-results
        path: ./benchmark_results/
        retention-days: 30