build.yml

# GitHub Action workflow for testing, building, and releasing the Synapse Python Client.

# - all pushes, releases, and pull requests  are tested against unit tests, and additionally
#     integration tests if account configuration secrets are available.
# - releases are additionally packaged, uploaded as build and release artifacts,
#     and deployed to pypi servers (test.pypi.org for prereleases, and pypi.org for releases)
#     Release tags must conform to our semver versioning, e.g. v1.2.3 in order to be packaged
#     for pypi deployment.
# - all pushes to the `develop` branch of the repository trigger a docker build and push to ghcr.io
#     with the image tag named after the sha of the commit,
#     e.g. `ghcr.io/sage-bionetworks/synapsepythonclient:develop-abc123`
# - all non-prerelease releases trigger a docker build and push to ghcr.io with the image tag named
#     after the release tag, e.g. `ghcr.io/sage-bionetworks/synapsepythonclient:1.2.3`

name: build

on:
  push:
    # we test all pushed branches, but not tags.
    # we only push tags with releases, and we handle releases explicitly
    branches:
      - '**'
    tags-ignore:
      - '**'

  release:
    types:
      - 'published'

# Cancel any previous runs of this workflow that are still in progress.
# This is to avoid running the same tests multiple times concurrently.
concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.13'
      - uses: pre-commit/action@v3.0.1

  # run unit (and integration tests if account secrets available) on our build matrix
  test:
    needs: [pre-commit]
    timeout-minutes: 180

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-22.04, macos-15-intel, windows-2022]

        # if changing the below change the run-integration-tests versions and the check-deploy versions
        # Make sure that we are running the integration tests on the first and last versions of the matrix
        python: ['3.11', '3.12', '3.13', '3.14', '3.15']

    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

      - name: get-dependencies-location
        shell: bash
        run: |
          SITE_PACKAGES_LOCATION=$(python -c "from sysconfig import get_path; print(get_path('purelib'))")
          SITE_BIN_DIR=$(python3 -c "import os; import platform; import sysconfig; pre = sysconfig.get_config_var('prefix'); bindir = os.path.join(pre, 'Scripts' if platform.system() == 'Windows' else 'bin'); print(bindir)")
          echo "site_packages_loc=$SITE_PACKAGES_LOCATION" >> $GITHUB_OUTPUT
          echo "site_bin_dir=$SITE_BIN_DIR" >> $GITHUB_OUTPUT
        id: get-dependencies

      - name: Cache py-dependencies
        id: cache-dependencies
        uses: actions/cache@v4
        env:
          cache-name: cache-py-dependencies
        with:
          path: |
            ${{ steps.get-dependencies.outputs.site_packages_loc }}
            ${{ steps.get-dependencies.outputs.site_bin_dir }}
          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py', 'setup.cfg') }}-v34

      - name: Install py-dependencies
        if: steps.cache-dependencies.outputs.cache-hit != 'true'
        shell: bash
        run: |
          python -m pip install --upgrade pip

          pip install -e ".[boto3,pandas,pysftp,tests,curator]"

          # ensure that numpy c extensions are installed on windows
          # https://stackoverflow.com/a/59346525
          if [ "${{startsWith(runner.os, 'Windows')}}" == "true" ]; then
            pip uninstall -y numpy
            pip uninstall -y setuptools
            pip install setuptools
            pip install numpy
          fi

      - name: run-unit-tests
        shell: bash
        run: |
          pytest -sv --cov-append --cov=. --cov-report xml tests/unit
      - name: Check for Secret availability
        id: secret-check
        if: ${{ contains(fromJSON('["3.11"]'), matrix.python) || contains(fromJSON('["3.15"]'), matrix.python) }}
        # perform secret check & put boolean result as an output
        shell: bash
        run: |
          if [ -z "${{ secrets.encrypted_d17283647768_key }}" ]  || [ -z "${{ secrets.encrypted_d17283647768_iv }}" ]; then
            echo "secrets_available=false" >> $GITHUB_OUTPUT;
          else
            echo "secrets_available=true" >> $GITHUB_OUTPUT;
          fi

          if [ -z "${{ secrets.synapse_personal_access_token }}" ]; then
            echo "synapse_pat_available=false" >> $GITHUB_OUTPUT;
          else
            echo "synapse_pat_available=true" >> $GITHUB_OUTPUT;
          fi

      - name: Download Failed Tests from Previous Attempt
        if: ${{ github.run_attempt > 1 && (contains(fromJSON('["3.11"]'), matrix.python) || contains(fromJSON('["3.15"]'), matrix.python)) && steps.secret-check.outputs.secrets_available == 'true'}}
        uses: actions/download-artifact@v4
        with:
          name: failed-tests-${{ matrix.os }}-${{ matrix.python }}
        continue-on-error: true

      # run integration tests iff the decryption keys for the test configuration are available.
      # they will not be available in pull requests from forks.
      # run integration tests on the oldest and newest supported versions of python.
      # we don't run on the entire matrix to avoid a 3xN set of concurrent tests against
      # the target server where N is the number of supported python versions.
      - name: run-integration-tests
        id: integration_tests
        shell: bash

        # keep versions consistent with the first and last from the strategy matrix
        if: ${{ (contains(fromJSON('["3.11"]'), matrix.python) || contains(fromJSON('["3.15"]'), matrix.python)) && steps.secret-check.outputs.secrets_available == 'true' && github.event_name != 'release'}}
        run: |
          # Set SYNAPSE_PROFILE based on OS and Python version
          if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then
            if [ "${{ matrix.python }}" == "3.11" ]; then
              export SYNAPSE_PROFILE="TestUbuntuMinimumPython"
            elif [ "${{ matrix.python }}" == "3.15" ]; then
              export SYNAPSE_PROFILE="TestUbuntuMaximumPython"
            fi
          elif [ "${{ startsWith(matrix.os, 'windows') }}" == "true" ]; then
            if [ "${{ matrix.python }}" == "3.11" ]; then
              export SYNAPSE_PROFILE="TestWindowsMinimumPython"
            elif [ "${{ matrix.python }}" == "3.15" ]; then
              export SYNAPSE_PROFILE="TestWindowsMaximumPython"
            fi
          elif [ "${{ startsWith(matrix.os, 'macos') }}" == "true" ]; then
            if [ "${{ matrix.python }}" == "3.11" ]; then
              export SYNAPSE_PROFILE="TestMacosMinimumPython"
            elif [ "${{ matrix.python }}" == "3.15" ]; then
              export SYNAPSE_PROFILE="TestMacosMaximumPython"
            fi
          fi

          echo "Using SYNAPSE_PROFILE: $SYNAPSE_PROFILE"

          # decrypt the encrypted test synapse configuration
          openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d
          mv test.synapseConfig ~/.synapseConfig

          if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then
            # on linux only we can build and run a docker container to serve as an SFTP host for our SFTP tests.
            # Docker is not available on GH Action runners on Mac and Windows.

            docker build -t sftp_tests - < tests/integration/synapseclient/core/upload/Dockerfile_sftp
            docker run -d sftp_tests:latest

            # get the internal IP address of the just launched container
            export SFTP_HOST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q))

            printf "[sftp://$SFTP_HOST]\nusername: test\npassword: test\n" >> ~/.synapseConfig

            # add to known_hosts so the ssh connections can be made without any prompting/errors
            mkdir -p ~/.ssh
            ssh-keyscan -H $SFTP_HOST >> ~/.ssh/known_hosts
          fi

            # set env vars used in external bucket tests from secrets
          export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}"
          export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
          export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"

          # Set env vars for OTEL
          export OTEL_EXPORTER_OTLP_ENDPOINT="${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}"
          export OTEL_SERVICE_INSTANCE_ID="${{ vars.OTEL_SERVICE_INSTANCE_ID }}"
          export SYNAPSE_INTEGRATION_TEST_OTEL_ENABLED="${{ vars.SYNAPSE_INTEGRATION_TEST_OTEL_ENABLED }}"
          export OTEL_EXPORTER_OTLP_HEADERS="${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }}"

          # Setup ignore patterns based on Python version
          IGNORE_FLAGS="--ignore=tests/integration/synapseclient/test_command_line_client.py"

          # Check if we should run only failed tests from previous attempt
          if [[ -f failed_tests.txt && -s failed_tests.txt ]]; then
            echo "::notice::Retry attempt ${{ github.run_attempt }} detected - running only previously failed tests"
            cat failed_tests.txt

            # Run only the failed tests
            pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml \
              --html=integration-test-report.html --self-contained-html \
              --junit-xml=test-results.xml \
              -n 4 --dist loadscope \
              $(cat failed_tests.txt | tr '\n' ' ')
          else
            echo "::notice::First attempt or no previous failures - running full integration test suite"

            # use loadscope to avoid issues running tests concurrently that share scoped fixtures
            pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml \
              --html=integration-test-report.html --self-contained-html \
              --junit-xml=test-results.xml \
              tests/integration -n 4 $IGNORE_FLAGS --dist loadscope
          fi

          # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
          pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml --html=cli-test-report.html --self-contained-html tests/integration/synapseclient/test_command_line_client.py

      - name: Extract Failed Tests
        if: always() && steps.integration_tests.outcome == 'failure'
        shell: bash
        run: |
          python -c "
          import xml.etree.ElementTree as ET
          import os
          tree = ET.parse('test-results.xml')
          root = tree.getroot()
          failed = []
          for testcase in root.iter('testcase'):
              if testcase.find('failure') is not None or testcase.find('error') is not None:
                  classname = testcase.get('classname')
                  name = testcase.get('name')
                  file_attr = testcase.get('file')

                  # Use the file attribute if available, otherwise convert classname
                  if file_attr:
                      # file attribute is already in the correct format
                      test_path = f'{file_attr}::{classname.split(\".\")[-1]}::{name}'
                  else:
                      # Convert classname from dot notation to file path
                      # e.g., 'tests.integration.foo.test_bar.TestClass' -> 'tests/integration/foo/test_bar.py::TestClass'
                      parts = classname.split('.')
                      # Find the test file (usually starts with 'test_')
                      module_parts = []
                      class_parts = []
                      found_test_file = False
                      for part in parts:
                          if not found_test_file:
                              module_parts.append(part)
                              if part.startswith('test_'):
                                  found_test_file = True
                          else:
                              class_parts.append(part)

                      file_path = '/'.join(module_parts) + '.py'
                      if class_parts:
                          test_path = f'{file_path}::{\"::\".join(class_parts)}::{name}'
                      else:
                          test_path = f'{file_path}::{name}'

                  failed.append(test_path)

          with open('failed_tests.txt', 'w') as f:
              f.write('\n'.join(failed))
          print(f'Found {len(failed)} failed tests')
          for test in failed:
              print(f'  - {test}')
          print(f'Current attempt: ${{ github.run_attempt }}')
          "

      - name: Upload Failed Tests for Next Attempt
        if: always() && steps.integration_tests.outcome == 'failure' && github.run_attempt < 3
        uses: actions/upload-artifact@v4
        with:
          name: failed-tests-${{ matrix.os }}-${{ matrix.python }}
          path: failed_tests.txt
          retention-days: 2
          overwrite: true

      - name: Fail job if integration tests failed after all retries
        if: always() && steps.integration_tests.outcome == 'failure'
        shell: bash
        run: |
          echo "::error::Integration tests failed after ${{ github.run_attempt }} attempt(s)"
          exit 1

      - name: Upload integration test HTML report
      # make sure report always gets uploaded if the integration tests ran, even if they failed, but skip if the integration tests were skipped
        if: always() && steps.integration_tests.outcome != 'skipped'
        uses: actions/upload-artifact@v4
        with:
          name: integration-test-report-${{ matrix.os }}-${{ matrix.python }}
          path: |
            integration-test-report.html
            cli-test-report.html
          retention-days: 14

      - name: Upload coverage report
        id: upload_coverage_report
        uses: actions/upload-artifact@v4
        if: ${{ contains(fromJSON('["3.13"]'), matrix.python) && contains(fromJSON('["ubuntu-22.04"]'), matrix.os)}}
        with:
          name: coverage-report
          path: coverage.xml

  # run unit tests against older pandas versions, since the test job installs only the newest pandas in the supported range
  test-pandas-versions:
    needs: [pre-commit]
    timeout-minutes: 60

    strategy:
      fail-fast: false
      matrix:
        include:
          # pandas < 2.2.2 is not compatible with numpy 2, so those entries pin numpy 1.26.4 (the last 1.x release)
          # pandas 1.5 is the lower bound in setup.cfg and installs only on Python 3.11
          - python: '3.11'
            pandas: 'pandas==1.5.3'
            numpy: 'numpy==1.26.4'
          # pandas 2.1 is the first version with DataFrame.map and the first to support Python 3.12
          - python: '3.12'
            pandas: 'pandas==2.1.4'
            numpy: 'numpy==1.26.4'
          # pandas 2.2.3 is the first version to support Python 3.13
          - python: '3.13'
            pandas: 'pandas==2.2.3'
            numpy: 'numpy==2.5.4'

    runs-on: ubuntu-22.04

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

      - name: Install py-dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip

          pip install -e ".[boto3,pandas,pysftp,tests,curator]" "${{ matrix.pandas }}" "${{ matrix.numpy }}"

      - name: run-unit-tests
        shell: bash
        run: |
          python -c "import numpy, pandas; print(f'pandas {pandas.__version__}, numpy {numpy.__version__}')"
          pytest -sv tests/unit

  sonarcloud:
    needs: [test]
    if: ${{ always() && !cancelled()}}
    name: SonarCloud
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
      - name: Check coverage-report artifact existence
        id: check_coverage_report
        uses: LIT-Protocol/artifact-exists-action@v0
        with:
          name: 'coverage-report'
      - name: Download coverage report
        uses: actions/download-artifact@v4
        if: steps.check_coverage_report.outputs.exists == 'true'
        with:
          name: coverage-report
      - name: Check coverage.xml file existence
        id: check_coverage_xml
        uses: andstor/file-existence-action@v3
        with:
          files: 'coverage.xml'
        # This is a workaround described in https://community.sonarsource.com/t/sonar-on-github-actions-with-python-coverage-source-issue/36057
      - name: Override Coverage Source Path for Sonar
        if: steps.check_coverage_xml.outputs.files_exists == 'true'
        run: sed -i "s/<source>\/home\/runner\/work\/synapsePythonClient<\/source>/<source>\/github\/workspace<\/source>/g" coverage.xml
      - name: SonarCloud Scan
        uses: SonarSource/sonarqube-scan-action@v6.0.0
        if: ${{ always() }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

  # on a GitHub release, build the pip package and upload it as a GitHub release asset
  package:
    needs: [test, test-pandas-versions, pre-commit]

    runs-on: ubuntu-22.04

    if: github.event_name == 'release'

    outputs:
      sdist-package-name: ${{ steps.build-package.outputs.sdist-package-name }}
      bdist-package-name: ${{ steps.build-package.outputs.bdist-package-name }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: set-release-env
        shell: bash
        run: |
          RELEASE_TAG="${{ github.event.release.tag_name }}"
          if [[ $RELEASE_TAG =~ ^v?([[:digit:]\.]+)(-rc)? ]]; then
            VERSION="${BASH_REMATCH[1]}"
            if [[ "${{ github.event.release.prerelease}}" == "true" ]]; then
              if [[ -z "${BASH_REMATCH[2]}" ]]; then
                echo "A test release tag should end with \"-rc\""
                exit 1
              fi

              # for staging builds we append the build number so we have
              # distinct version numbers between prod and test pypi.
              VERSION="$VERSION.$GITHUB_RUN_NUMBER"
            fi

          else
            echo "Unable to parse deployment version from $RELEASE_TAG"
            exit 1
          fi

          echo "VERSION=$VERSION" >> $GITHUB_ENV

      # ensure that the version file in the package will have the correct version
      # matching the name of the tag
      - name: update-version
        shell: bash
        run: |
          if [[ -n "$VERSION" ]]; then
            sed "s|\"latestVersion\":.*$|\"latestVersion\":\"$VERSION\",|g" synapseclient/synapsePythonClient > temp
            rm synapseclient/synapsePythonClient
            mv temp synapseclient/synapsePythonClient
          fi

      - id: build-package
        shell: bash
        run: |
          python3 -m pip install --upgrade pip
          python3 -m pip install setuptools
          python3 -m pip install wheel
          python3 -m pip install build

          # install synapseclient
          python3 -m pip install .

          # create distribution
          python3 -m build

          SDIST_PACKAGE_NAME="synapseclient-${{env.VERSION}}.tar.gz"
          BDIST_PACKAGE_NAME="synapseclient-${{env.VERSION}}-py3-none-any.whl"
          RELEASE_URL_PREFIX="https://uploads.github.com/repos/${{ github.event.repository.full_name }}/releases/${{ github.event.release.id }}/assets?name="

          echo "sdist-package-name=$SDIST_PACKAGE_NAME" >> $GITHUB_OUTPUT
          echo "bdist-package-name=$BDIST_PACKAGE_NAME" >> $GITHUB_OUTPUT

          echo "sdist-release-url=${RELEASE_URL_PREFIX}${SDIST_PACKAGE_NAME}" >> $GITHUB_OUTPUT
          echo "bdist-release-url=${RELEASE_URL_PREFIX}${BDIST_PACKAGE_NAME}" >> $GITHUB_OUTPUT

      # upload the packages as build artifacts of the GitHub Action

      - name: upload-build-sdist
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.build-package.outputs.sdist-package-name }}
          path: dist/${{ steps.build-package.outputs.sdist-package-name }}

      - name: upload-build-bdist
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.build-package.outputs.bdist-package-name }}
          path: dist/${{ steps.build-package.outputs.bdist-package-name }}

      # upload the packages as artifacts of the GitHub release

      # - name: upload-release-sdist
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.build-package.outputs.sdist-release-url }}
      #     asset_name: ${{ steps.build-package.outputs.sdist-package-name }}
      #     asset_path: dist/${{ steps.build-package.outputs.sdist-package-name }}
      #     asset_content_type: application/gzip

      # - name: upload-release-bdist
      #   uses: actions/upload-release-asset@v1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.build-package.outputs.bdist-release-url }}
      #     asset_name: ${{ steps.build-package.outputs.bdist-package-name }}
      #     asset_path: dist/${{ steps.build-package.outputs.bdist-package-name }}
      #     asset_content_type: application/zip

  # re-download the built package to the appropriate pypi server.
  # we upload prereleases to test.pypi.org and releases to pypi.org.
  deploy:
    needs: package
    runs-on: ubuntu-latest
    environment:
      url: ${{ github.event.release.prerelease == 'true' && 'https://test.pypi.org/p/synapseclient' || 'https://pypi.org/p/synapseclient' }}
      name: pypi
    permissions:
      id-token: write
    steps:
      - name: download-sdist
        uses: actions/download-artifact@v4
        with:
          name: ${{ needs.package.outputs.sdist-package-name }}
          path: dist

      - name: download-bdist
        uses: actions/download-artifact@v4
        with:
          name: ${{ needs.package.outputs.bdist-package-name }}
          path: dist
      - name: deploy-to-test-pypi
        if: 'github.event.release.prerelease'
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/
      - name: deploy-to-prod-pypi
        if: '!github.event.release.prerelease'
        uses: pypa/gh-action-pypi-publish@release/v1

  # on each of our matrix platforms, download the newly uploaded package from pypi and confirm its version
  check-deploy:
    needs: deploy

    strategy:
      matrix:
        os: [ubuntu-24.04, macos-15-intel, windows-2022]

        # python versions should be consistent with the strategy matrix and the runs-integration-tests versions
        python: ['3.11', '3.12', '3.13', '3.14', '3.15']

    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

      - name: check-pypi
        shell: bash
        run: |
          if [[ "${{ github.event.release.prerelease}}" == "false" ]]; then
            PYPI_INDEX_URL="https://pypi.org/simple/"
          else
            PYPI_INDEX_URL="https://test.pypi.org/simple/"
          fi

          RELEASE_TAG="${{ github.event.release.tag_name }}"
          if [[ $RELEASE_TAG =~ ^v?([[:digit:]\.]+)(-rc)? ]]; then
            VERSION="${BASH_REMATCH[1]}"
            if [[ "${{ github.event.release.prerelease}}" == "true" ]]; then
              VERSION="$VERSION.$GITHUB_RUN_NUMBER"
            fi
          else
            echo "Unrecognized release tag"
            exit 1
          fi

          # it can take some time for the packages to become available in pypi after uploading
          for i in 5 10 20 40; do
            if pip3 install --index-url $PYPI_INDEX_URL --extra-index-url https://pypi.org/simple "synapseclient==$VERSION"; then
              ACTUAL_VERSION=$(synapse --version)

              if [ -n "$(echo "$ACTUAL_VERSION" | grep -oF "$VERSION")" ]; then
                echo "Successfully installed version $VERSION"
                exit 0
              else
                echo "Expected version $VERSION, found $ACTUAL_VERSION instead"
                exit 1
              fi
            fi

            sleep $i
          done

          echo "Failed to install expected version $VERSION"
          exit 1

  # containerize the package and upload to the GHCR upon new release (whether pre-release or not)
  # Step 1: Build the Docker image and save as tar for scanning
  ghcr-build-on-release:
    needs: deploy
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    outputs:
      image-tags: ${{ steps.set-tags.outputs.tags }}
      image-name: synapsepythonclient-release
    env:
      TARFILE_NAME: synapsepythonclient-release.tar

    steps:
      - name: Check out the repo
        uses: actions/checkout@v4
      - name: Extract Release Version
        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
        shell: bash
      - name: Set image tags
        id: set-tags
        shell: bash
        run: |
          if [[ "${{ github.event.release.prerelease }}" == "true" ]]; then
            echo "tags=ghcr.io/sage-bionetworks/synapsepythonclient:${{ env.RELEASE_VERSION }}-prerelease" >> $GITHUB_OUTPUT
          else
            echo "tags=ghcr.io/sage-bionetworks/synapsepythonclient:latest,ghcr.io/sage-bionetworks/synapsepythonclient:${{ env.RELEASE_VERSION }}" >> $GITHUB_OUTPUT
          fi
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build Docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: false
          load: true
          provenance: false
          tags: synapsepythonclient-release:local
          file: ./Dockerfile
          platforms: linux/amd64
          cache-from: type=registry,ref=ghcr.io/sage-bionetworks/synapsepythonclient:build-cache
          cache-to: type=inline
      - name: Save Docker image to tar
        run: docker save synapsepythonclient-release:local -o ${{ env.TARFILE_NAME }}
      - name: Upload tar artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.TARFILE_NAME }}
          path: ${{ env.TARFILE_NAME }}
          retention-days: 1

  # Step 2: Scan the built image with Trivy before pushing
  trivy-scan-release:
    needs: [ghcr-build-on-release]
    uses: ./.github/workflows/trivy.yml
    with:
      SOURCE_TYPE: tar
      TARFILE_NAME: synapsepythonclient-release.tar
      IMAGE_NAME: synapsepythonclient-release:local
      EXIT_CODE: 1
    permissions:
      contents: read
      security-events: write
      actions: read

  # Step 3: Push the image to GHCR only if Trivy scan passes
  ghcr-push-on-release:
    needs: [ghcr-build-on-release, trivy-scan-release]
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    env:
      TARFILE_NAME: synapsepythonclient-release.tar

    steps:
      - name: Download scanned tar
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.TARFILE_NAME }}
          path: /tmp
      - name: Load Docker image from tar
        run: docker load -i /tmp/${{ env.TARFILE_NAME }}
      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Tag and push Docker image
        shell: bash
        run: |
          IFS=',' read -ra TAGS <<< "${{ needs.ghcr-build-on-release.outputs.image-tags }}"
          for TAG in "${TAGS[@]}"; do
            docker tag synapsepythonclient-release:local "$TAG"
            docker push "$TAG"
          done

  # containerize the package and upload to the GHCR upon commit in develop
  # Step 1: Build the Docker image and save as tar for scanning
  ghcr-build-on-develop:
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/develop'
    permissions:
      contents: read
      packages: write
    outputs:
      image-tag: ghcr.io/sage-bionetworks/synapsepythonclient:develop-${{ github.sha }}
      image-name: synapsepythonclient-develop
    env:
      TARFILE_NAME: synapsepythonclient-develop.tar

    steps:
      - name: Check out the repo
        uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build Docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          push: false
          load: true
          provenance: false
          tags: synapsepythonclient-develop:local
          file: ./Dockerfile
          platforms: linux/amd64
          cache-from: type=registry,ref=ghcr.io/sage-bionetworks/synapsepythonclient:build-cache
          cache-to: type=inline
      - name: Save Docker image to tar
        run: docker save synapsepythonclient-develop:local -o ${{ env.TARFILE_NAME }}
      - name: Upload tar artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.TARFILE_NAME }}
          path: ${{ env.TARFILE_NAME }}
          retention-days: 1

  # Step 2: Scan the built image with Trivy before pushing
  trivy-scan-develop:
    needs: [ghcr-build-on-develop]
    uses: ./.github/workflows/trivy.yml
    with:
      SOURCE_TYPE: tar
      TARFILE_NAME: synapsepythonclient-develop.tar
      IMAGE_NAME: synapsepythonclient-develop:local
      EXIT_CODE: 1
    permissions:
      contents: read
      security-events: write
      actions: read

  # Step 3: Push the image to GHCR only if Trivy scan passes
  ghcr-push-on-develop:
    needs: [ghcr-build-on-develop, trivy-scan-develop]
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/develop'
    permissions:
      contents: read
      packages: write

    env:
      TARFILE_NAME: synapsepythonclient-develop.tar

    steps:
      - name: Download scanned tar
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.TARFILE_NAME }}
          path: /tmp
      - name: Load Docker image from tar
        run: docker load -i /tmp/${{ env.TARFILE_NAME }}
      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Tag and push Docker image
        run: |
          docker tag synapsepythonclient-develop:local "${{ needs.ghcr-build-on-develop.outputs.image-tag }}"
          docker push "${{ needs.ghcr-build-on-develop.outputs.image-tag }}"