From 59d2c371b733fc6488c6233885a45d32d57631a3 Mon Sep 17 00:00:00 2001 From: Talha Date: Mon, 3 Aug 2026 15:20:27 +0500 Subject: [PATCH 1/8] refactor(release): unify candidate and publication pipeline --- .github/workflows/ci.yml | 42 +-- .github/workflows/containers.yml | 252 ++++++++++-------- .github/workflows/desktop.yml | 12 +- .github/workflows/publish-desktop.yml | 138 ---------- .github/workflows/release-candidate.yml | 205 ++++++++++++++ .github/workflows/release-gate.yml | 40 +++ .github/workflows/release-please.yml | 106 ++++++-- .github/workflows/release-to-pypi.yml | 216 +++++++++------ .github/workflows/release-to-test-pypi.yml | 126 ++++----- .release-please-manifest.json | 3 +- .release-please-manifest.stable.json | 3 +- desktop/THIRD_PARTY_NOTICES.txt | 1 - desktop/VERSION | 1 - desktop/scripts/generate-notices.mjs | 4 +- desktop/src-tauri/src/lib.rs | 29 +- desktop/src/desktopPackagingContracts.test.ts | 5 +- docs/CONTRIBUTING.md | 2 +- docs/desktop.md | 11 +- docs/releasing.md | 148 +++++----- release-please-config.json | 55 ++-- release-please-config.stable.json | 59 ++-- tests/test_packaging.py | 143 +++++----- tests/test_release_contract.py | 55 ++++ utils/check_workflows.sh | 15 ++ utils/prepare_nightly.py | 42 +++ utils/release_contract.py | 87 ++++++ utils/validate_compose.sh | 16 ++ 27 files changed, 1115 insertions(+), 701 deletions(-) delete mode 100644 .github/workflows/publish-desktop.yml create mode 100644 .github/workflows/release-candidate.yml create mode 100644 .github/workflows/release-gate.yml delete mode 100644 desktop/VERSION create mode 100644 tests/test_release_contract.py create mode 100644 utils/check_workflows.sh create mode 100644 utils/prepare_nightly.py create mode 100644 utils/release_contract.py create mode 100644 utils/validate_compose.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17670a5..617c859 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,11 @@ on: required: false default: false type: boolean + package_artifact_name: + description: Preserve the already-smoke-tested distribution under this name. + required: false + default: "" + type: string workflow_dispatch: inputs: checkout_ref: @@ -61,9 +66,14 @@ on: required: false default: true type: boolean + package_artifact_name: + description: Preserve the already-smoke-tested distribution under this name. + required: false + default: "" + type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.checkout_ref || github.ref }} cancel-in-progress: true permissions: @@ -87,6 +97,9 @@ jobs: fetch-depth: 0 ref: ${{ inputs.checkout_ref || github.sha }} + - name: Validate GitHub Actions workflows + run: bash utils/check_workflows.sh + - name: Determine validation scope id: scope shell: bash @@ -238,22 +251,21 @@ jobs: assert find_spec(module) is None, module PY + - name: Preserve the validated distribution + if: >- + needs.scope.outputs.run_suite == 'true' && + matrix.python-version == '3.14' && + inputs.package_artifact_name != '' + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.package_artifact_name }} + path: dist/ + if-no-files-found: error + retention-days: 30 + - name: Validate Compose configuration if: needs.scope.outputs.run_container == 'true' && matrix.python-version == '3.14' - env: - POSTGRES_PASSWORD: ci-postgres-password - VIDXP_ARTIFACT_DOWNLOAD_SECRET: ci-artifact-download-secret-000000 - VIDXP_CONTROL_IMAGE: vidxp-control:ci - VIDXP_PUBLIC_API_HOST: api.example.test - VIDXP_UPLOAD_CLEANUP_TOKEN: ci-upload-cleanup-token-0000000000 - VIDXP_UPLOAD_CORS_ORIGIN_REGEX: '^(https://api\.example\.test)$' - VIDXP_UPLOAD_HANDOFF_PUBLIC_URL: https://api.example.test/upload-handoff - VIDXP_UPLOAD_HANDOFF_SECRET: ci-upload-handoff-secret-000000000 - VIDXP_UPLOAD_PUBLIC_ENDPOINT: https://uploads.example.test/uploads/ - VIDXP_WORKER_IMAGE: vidxp-worker:ci - run: | - docker compose -f compose.yaml config --quiet - docker compose -f compose.coolify.yaml config --quiet + run: bash utils/validate_compose.sh - name: Build container if: needs.scope.outputs.run_container == 'true' && matrix.python-version == '3.14' diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 409d2ea..4bccadb 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -1,50 +1,89 @@ -name: Publish core containers +name: Release containers on: workflow_call: inputs: - channel: - description: Container channel to publish. + operation: + description: Build a candidate or promote its immutable digests. required: true type: string - tag: - description: Existing core tag to build. + checkout_ref: + description: Exact source revision used for a candidate build. + required: false + default: "" + type: string + candidate_id: + description: Stable candidate identifier used for temporary tags. required: true type: string + channel: + description: beta or stable when promoting. + required: false + default: "" + type: string version: - description: Core version without the v prefix. + description: Combined release version without the v prefix. required: true type: string + product_digest: + required: false + default: "" + type: string + control_digest: + required: false + default: "" + type: string + worker_digest: + required: false + default: "" + type: string + outputs: + product_digest: + value: ${{ jobs.stage.outputs.product_digest }} + control_digest: + value: ${{ jobs.stage.outputs.control_digest }} + worker_digest: + value: ${{ jobs.stage.outputs.worker_digest }} permissions: contents: read packages: write jobs: - publish: + stage: + if: inputs.operation == 'stage' runs-on: ubuntu-latest + outputs: + product_digest: ${{ steps.digests.outputs.product }} + control_digest: ${{ steps.digests.outputs.control }} + worker_digest: ${{ steps.digests.outputs.worker }} steps: - uses: actions/checkout@v7 with: - ref: ${{ inputs.tag }} + ref: ${{ inputs.checkout_ref }} + + - name: Verify staging inputs + shell: bash + run: | + [[ "${{ inputs.checkout_ref }}" =~ ^[0-9a-f]{40}$ ]] + [[ "${{ inputs.candidate_id }}" =~ ^[0-9a-f]{40}$ ]] - name: Resolve source revision id: source shell: bash run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - name: Validate Compose deployment configuration + run: bash utils/validate_compose.sh + - uses: docker/setup-buildx-action@v4 - - name: Generate product image metadata + - name: Generate image metadata id: metadata uses: docker/metadata-action@v6 with: images: ghcr.io/${{ github.repository }} - tags: | - type=raw,value=${{ inputs.version }} - type=semver,pattern={{major}}.{{minor}},value=${{ inputs.tag }},enable=${{ inputs.channel == 'stable' }} - type=raw,value=latest,enable=${{ inputs.channel == 'stable' }} - type=raw,value=beta,enable=${{ inputs.channel == 'beta' }} + tags: type=raw,value=candidate-${{ inputs.candidate_id }} labels: | org.opencontainers.image.title=VidXP org.opencontainers.image.description=Local-first video indexing and search @@ -60,7 +99,7 @@ jobs: load: true platforms: linux/amd64 push: false - tags: ${{ steps.metadata.outputs.tags }} + tags: ghcr.io/${{ github.repository }}:candidate-${{ inputs.candidate_id }} labels: ${{ steps.metadata.outputs.labels }} cache-from: type=gha,scope=release-product cache-to: type=gha,mode=max,scope=release-product @@ -73,9 +112,7 @@ jobs: platforms: linux/amd64 push: false target: control - tags: | - ghcr.io/${{ github.repository }}:${{ inputs.version }}-control - ghcr.io/${{ github.repository }}:${{ inputs.channel == 'stable' && 'latest' || 'beta' }}-control + tags: ghcr.io/${{ github.repository }}:candidate-${{ inputs.candidate_id }}-control labels: ${{ steps.metadata.outputs.labels }} cache-from: type=gha,scope=release-control cache-to: type=gha,mode=max,scope=release-control @@ -88,19 +125,19 @@ jobs: platforms: linux/amd64 push: false target: worker - tags: | - ghcr.io/${{ github.repository }}:${{ inputs.version }}-worker - ghcr.io/${{ github.repository }}:${{ inputs.channel == 'stable' && 'latest' || 'beta' }}-worker + tags: ghcr.io/${{ github.repository }}:candidate-${{ inputs.candidate_id }}-worker labels: ${{ steps.metadata.outputs.labels }} cache-from: type=gha,scope=release-worker cache-to: type=gha,mode=max,scope=release-worker - - name: Smoke product image + - name: Smoke candidate images shell: bash run: | - image="ghcr.io/${{ github.repository }}:${{ inputs.version }}" + product="ghcr.io/${{ github.repository }}:candidate-${{ inputs.candidate_id }}" + control="${product}-control" + worker="${product}-worker" docker run --detach --name vidxp-release \ - --publish 127.0.0.1:8501:8501 "$image" + --publish 127.0.0.1:8501:8501 "$product" trap 'docker rm --force vidxp-release >/dev/null 2>&1 || true' EXIT docker exec vidxp-release vidxp --version @@ -114,27 +151,18 @@ jobs: for attempt in {1..30}; do if curl --fail --silent http://127.0.0.1:8501/_stcore/health; then - exit 0 + break fi if [[ "$(docker inspect --format='{{.State.Running}}' vidxp-release)" != "true" ]]; then - break + docker logs vidxp-release + exit 1 fi sleep 2 done - - docker logs vidxp-release - exit 1 - - - name: Smoke server images - shell: bash - run: | - control="ghcr.io/${{ github.repository }}:${{ inputs.version }}-control" - worker="ghcr.io/${{ github.repository }}:${{ inputs.version }}-worker" + curl --fail --silent http://127.0.0.1:8501/_stcore/health docker run --rm "$control" vidxp-api --help docker run --rm "$control" vidxp-mcp --help docker run --rm "$worker" vidxp init --json - docker run --rm "$worker" python -c \ - 'import chromadb, cv2, faster_whisper, huggingface_hub, pooch, psutil, sentence_transformers, torch, transformers; assert psutil.virtual_memory().available > 0' - name: Log in to GitHub Container Registry uses: docker/login-action@v4 @@ -143,89 +171,103 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Protect immutable version tags - id: existing - env: - SOURCE_SHA: ${{ steps.source.outputs.sha }} + - name: Push candidate images and record immutable digests + id: digests shell: bash run: | - check_version_tag() { - local name="$1" - local image="$2" - local local_image="vidxp-release-local:$name" - - docker tag "$image" "$local_image" - if ! docker pull "$image" >/dev/null 2>&1; then - docker tag "$local_image" "$image" - echo "${name}=false" >> "$GITHUB_OUTPUT" - return - fi - - revision="$( - docker inspect \ - --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' \ - "$image" - )" - if [[ "$revision" != "$SOURCE_SHA" ]]; then - echo "::error::$image already belongs to source revision $revision, not $SOURCE_SHA." - exit 1 - fi - echo "${name}=true" >> "$GITHUB_OUTPUT" + digest() { + docker buildx imagetools inspect "$1" --format '{{json .Manifest}}' | + jq -er '.digest' } + base="ghcr.io/${{ github.repository }}:candidate-${{ inputs.candidate_id }}" + docker push "$base" + docker push "${base}-control" + docker push "${base}-worker" + echo "product=$(digest "$base")" >> "$GITHUB_OUTPUT" + echo "control=$(digest "${base}-control")" >> "$GITHUB_OUTPUT" + echo "worker=$(digest "${base}-worker")" >> "$GITHUB_OUTPUT" - check_version_tag \ - product \ - "ghcr.io/${{ github.repository }}:${{ inputs.version }}" - check_version_tag \ - control \ - "ghcr.io/${{ github.repository }}:${{ inputs.version }}-control" - check_version_tag \ - worker \ - "ghcr.io/${{ github.repository }}:${{ inputs.version }}-worker" - - - name: Publish containers + promote: + if: inputs.operation == 'promote' + runs-on: ubuntu-latest + steps: + - name: Verify promotion inputs shell: bash run: | - product="ghcr.io/${{ github.repository }}:${{ inputs.version }}" - control="${product}-control" - worker="${product}-worker" - control_channel="ghcr.io/${{ github.repository }}:${{ inputs.channel == 'stable' && 'latest' || 'beta' }}-control" - worker_channel="ghcr.io/${{ github.repository }}:${{ inputs.channel == 'stable' && 'latest' || 'beta' }}-worker" + [[ "${{ inputs.channel }}" == "beta" || "${{ inputs.channel }}" == "stable" ]] + [[ "${{ inputs.candidate_id }}" =~ ^[0-9a-f]{40}$ ]] + [[ "${{ inputs.product_digest }}" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "${{ inputs.control_digest }}" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "${{ inputs.worker_digest }}" =~ ^sha256:[0-9a-f]{64}$ ]] - if [[ "${{ steps.existing.outputs.product }}" == "true" ]]; then - while read -r image; do - [[ "$image" == "$product" ]] || docker tag "$product" "$image" - done <<< "${{ steps.metadata.outputs.tags }}" - fi - while read -r image; do - if [[ "$image" != "$product" || - "${{ steps.existing.outputs.product }}" != "true" ]]; then - docker push "$image" + - uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Promote the candidate manifests without rebuilding + shell: bash + run: | + repository="ghcr.io/${{ github.repository }}" + candidate="${repository}:candidate-${{ inputs.candidate_id }}" + + promote() { + local source="$1" + local expected="$2" + shift 2 + actual="$( + docker buildx imagetools inspect "$source" \ + --format '{{json .Manifest}}' | jq -er '.digest' + )" + if [[ "$actual" != "$expected" ]]; then + echo "::error::$source resolved to $actual instead of $expected." + exit 1 fi - done <<< "${{ steps.metadata.outputs.tags }}" + for target in "$@"; do + existing="$( + docker buildx imagetools inspect "$target" \ + --format '{{json .Manifest}}' 2>/dev/null | jq -r '.digest // empty' || true + )" + if [[ "$target" == "${repository}:${{ inputs.version }}"* && + -n "$existing" && "$existing" != "$expected" ]]; then + echo "::error::Immutable tag $target already points to $existing." + exit 1 + fi + [[ "$existing" == "$expected" ]] || \ + docker buildx imagetools create \ + --prefer-index=false \ + --tag "$target" \ + "${repository}@${expected}" + done + } - if [[ "${{ steps.existing.outputs.control }}" == "true" ]]; then - docker tag "$control" "$control_channel" + product_tags=("${repository}:${{ inputs.version }}") + control_tags=("${repository}:${{ inputs.version }}-control") + worker_tags=("${repository}:${{ inputs.version }}-worker") + if [[ "${{ inputs.channel }}" == "stable" ]]; then + major_minor="$(cut -d. -f1,2 <<< "${{ inputs.version }}")" + product_tags+=("${repository}:${major_minor}" "${repository}:latest") + control_tags+=("${repository}:latest-control") + worker_tags+=("${repository}:latest-worker") else - docker push "$control" + product_tags+=("${repository}:beta") + control_tags+=("${repository}:beta-control") + worker_tags+=("${repository}:beta-worker") fi - docker push "$control_channel" - if [[ "${{ steps.existing.outputs.worker }}" == "true" ]]; then - docker tag "$worker" "$worker_channel" - else - docker push "$worker" - fi - docker push "$worker_channel" + promote "$candidate" "${{ inputs.product_digest }}" "${product_tags[@]}" + promote "${candidate}-control" "${{ inputs.control_digest }}" "${control_tags[@]}" + promote "${candidate}-worker" "${{ inputs.worker_digest }}" "${worker_tags[@]}" - - name: Verify public container access + - name: Verify public release tags shell: bash run: | docker logout ghcr.io for suffix in "" "-control" "-worker"; do - image="ghcr.io/${{ github.repository }}:${{ inputs.version }}${suffix}" - if ! docker buildx imagetools inspect "$image"; then - echo "::error::Make the GHCR package public, then rerun this workflow for the same tag." - exit 1 - fi + docker buildx imagetools inspect \ + "ghcr.io/${{ github.repository }}:${{ inputs.version }}${suffix}" done diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index a170b69..b5464a3 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -7,6 +7,11 @@ on: description: Commit to build required: true type: string + artifact_retention_days: + description: Number of days to retain built installers. + required: false + default: 14 + type: number workflow_dispatch: inputs: checkout_ref: @@ -14,6 +19,11 @@ on: required: false default: "" type: string + artifact_retention_days: + description: Number of days to retain built installers. + required: false + default: 14 + type: number pull_request: branches: - main @@ -176,4 +186,4 @@ jobs: path: ${{ matrix.target == 'windows' && 'desktop/src-tauri/target/release/bundle/nsis/*-setup.exe' || matrix.target == 'macos' && 'desktop/src-tauri/target/release/bundle/dmg/*.dmg' || 'desktop/src-tauri/target/release/bundle/appimage/*.AppImage' }} if-no-files-found: error compression-level: 0 - retention-days: 14 + retention-days: ${{ inputs.artifact_retention_days || 14 }} diff --git a/.github/workflows/publish-desktop.yml b/.github/workflows/publish-desktop.yml deleted file mode 100644 index 2fd0baa..0000000 --- a/.github/workflows/publish-desktop.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: Publish desktop release - -on: - workflow_dispatch: - inputs: - tag: - description: Existing desktop tag to build and publish. - required: true - type: string - -permissions: - contents: read - -concurrency: - group: desktop-release-${{ inputs.tag }} - cancel-in-progress: false - -jobs: - verify: - runs-on: ubuntu-latest - outputs: - source_sha: ${{ steps.source.outputs.sha }} - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ inputs.tag }} - - - name: Resolve source revision - id: source - shell: bash - run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - - name: Verify desktop tag and version sources - shell: bash - run: | - tag="${{ inputs.tag }}" - if [[ "$tag" != desktop-v* ]]; then - echo "::error::$tag is not a desktop release tag." - exit 1 - fi - if ! git show-ref --verify --quiet "refs/tags/$tag"; then - echo "::error::$tag is not an existing Git tag." - exit 1 - fi - python - "${tag#desktop-v}" <<'PY' - import json - from pathlib import Path - import sys - import tomllib - - expected = sys.argv[1] - desktop = Path("desktop") - runtime_manifest = json.loads( - (desktop / "runtime-manifest.json").read_text() - ) - versions = { - "VERSION": (desktop / "VERSION").read_text().strip(), - "Cargo.toml": tomllib.loads( - (desktop / "src-tauri" / "Cargo.toml").read_text() - )["package"]["version"], - "package.json": json.loads( - (desktop / "package.json").read_text() - )["version"], - "package-lock.json": json.loads( - (desktop / "package-lock.json").read_text() - )["version"], - "package-lock root": json.loads( - (desktop / "package-lock.json").read_text() - )["packages"][""]["version"], - "runtime manifest desktop": runtime_manifest["desktop_version"], - "runtime manifest package": runtime_manifest["package_version"], - "Tauri config": json.loads( - (desktop / "src-tauri" / "tauri.conf.json").read_text() - )["version"], - } - mismatches = { - source: actual - for source, actual in versions.items() - if actual != expected - } - if mismatches: - raise SystemExit( - f"desktop tag expects {expected}; mismatches: {mismatches}" - ) - PY - - build: - needs: verify - uses: ./.github/workflows/desktop.yml - with: - checkout_ref: ${{ inputs.tag }} - - publish: - needs: - - build - - verify - runs-on: ubuntu-latest - permissions: - actions: write - contents: write - steps: - - uses: actions/download-artifact@v8 - with: - pattern: vidxp-desktop-* - path: desktop-dist/ - merge-multiple: true - - - name: Publish complete desktop release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - tag="${{ inputs.tag }}" - version="${tag#desktop-v}" - gh release upload "$tag" desktop-dist/* \ - --repo "$GITHUB_REPOSITORY" \ - --clobber - if [[ "$version" == *-* ]]; then - gh release edit "$tag" \ - --repo "$GITHUB_REPOSITORY" \ - --draft=false \ - --prerelease - else - gh release edit "$tag" \ - --repo "$GITHUB_REPOSITORY" \ - --draft=false \ - --latest - fi - - - name: Synchronize stable source back to main - if: ${{ !contains(inputs.tag, '-b.') }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh workflow run sync-channels.yml \ - --repo "$GITHUB_REPOSITORY" \ - --ref main \ - -f release_sha="${{ needs.verify.outputs.source_sha }}" diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml new file mode 100644 index 0000000..835ea17 --- /dev/null +++ b/.github/workflows/release-candidate.yml @@ -0,0 +1,205 @@ +name: Release candidate + +on: + workflow_dispatch: + inputs: + pull_request: + description: Release Please pull request number. + required: true + type: string + target_branch: + description: main for beta or release for stable. + required: true + type: string + base_sha: + description: Target branch revision included in the candidate. + required: true + type: string + head_sha: + description: Exact Release Please revision to build. + required: true + type: string + +permissions: + contents: read + packages: write + pull-requests: read + statuses: write + +concurrency: + group: release-candidate-${{ inputs.head_sha }} + cancel-in-progress: false + +jobs: + pending: + runs-on: ubuntu-latest + steps: + - name: Mark the exact candidate revision pending + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ inputs.head_sha }} + TARGET_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + shell: bash + run: | + [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] + gh api --method POST "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f state=pending \ + -f context=release/candidate \ + -f description='Building the combined release candidate' \ + -f target_url="$TARGET_URL" + + contract: + needs: pending + runs-on: ubuntu-latest + outputs: + channel: ${{ steps.contract.outputs.channel }} + source_tree: ${{ steps.source.outputs.tree }} + tag: ${{ steps.contract.outputs.tag }} + version: ${{ steps.contract.outputs.version }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + ref: ${{ inputs.head_sha }} + + - name: Verify the candidate source and current pull request + id: source + env: + BASE_SHA: ${{ inputs.base_sha }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ inputs.head_sha }} + PULL_REQUEST: ${{ inputs.pull_request }} + TARGET_BRANCH: ${{ inputs.target_branch }} + shell: bash + run: | + [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$TARGET_BRANCH" == "main" || "$TARGET_BRANCH" == "release" ]] + [[ "$(git rev-parse HEAD)" == "$HEAD_SHA" ]] + git merge-base --is-ancestor "$BASE_SHA" "$HEAD_SHA" + + pr="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PULL_REQUEST")" + [[ "$(jq -r .state <<< "$pr")" == "open" ]] + [[ "$(jq -r .base.ref <<< "$pr")" == "$TARGET_BRANCH" ]] + [[ "$(jq -r .base.sha <<< "$pr")" == "$BASE_SHA" ]] + [[ "$(jq -r .head.sha <<< "$pr")" == "$HEAD_SHA" ]] + [[ "$(jq -r .head.ref <<< "$pr")" == release-please--branches--* ]] + echo "tree=$(git rev-parse 'HEAD^{tree}')" >> "$GITHUB_OUTPUT" + + - name: Validate the combined version contract + id: contract + shell: bash + run: | + channel="${{ inputs.target_branch == 'release' && 'stable' || 'beta' }}" + python utils/release_contract.py --channel "$channel" --github-output \ + >> "$GITHUB_OUTPUT" + + core: + needs: contract + uses: ./.github/workflows/ci.yml + with: + checkout_ref: ${{ inputs.head_sha }} + force_validation: true + package_artifact_name: vidxp-python-dist + provider_platforms: '["ubuntu-24.04","windows-2025","macos-15"]' + python_versions: '["3.11","3.12","3.13","3.14"]' + run_containers: false + + desktop: + needs: contract + uses: ./.github/workflows/desktop.yml + with: + artifact_retention_days: 30 + checkout_ref: ${{ inputs.head_sha }} + + containers: + needs: contract + permissions: + contents: read + packages: write + uses: ./.github/workflows/containers.yml + with: + candidate_id: ${{ inputs.head_sha }} + checkout_ref: ${{ inputs.head_sha }} + operation: stage + version: ${{ needs.contract.outputs.version }} + + finalize: + if: always() + needs: + - pending + - contract + - core + - desktop + - containers + runs-on: ubuntu-latest + steps: + - name: Record the successful candidate + if: >- + needs.pending.result == 'success' && + needs.contract.result == 'success' && + needs.core.result == 'success' && + needs.desktop.result == 'success' && + needs.containers.result == 'success' + env: + CHANNEL: ${{ needs.contract.outputs.channel }} + CONTROL_DIGEST: ${{ needs.containers.outputs.control_digest }} + HEAD_SHA: ${{ inputs.head_sha }} + PRODUCT_DIGEST: ${{ needs.containers.outputs.product_digest }} + SOURCE_TREE: ${{ needs.contract.outputs.source_tree }} + TAG: ${{ needs.contract.outputs.tag }} + VERSION: ${{ needs.contract.outputs.version }} + WORKER_DIGEST: ${{ needs.containers.outputs.worker_digest }} + shell: bash + run: | + jq -n \ + --arg candidate_run_id "$GITHUB_RUN_ID" \ + --arg channel "$CHANNEL" \ + --arg control_digest "$CONTROL_DIGEST" \ + --arg product_digest "$PRODUCT_DIGEST" \ + --arg source_sha "$HEAD_SHA" \ + --arg source_tree "$SOURCE_TREE" \ + --arg tag "$TAG" \ + --arg version "$VERSION" \ + --arg worker_digest "$WORKER_DIGEST" \ + '{candidate_run_id: $candidate_run_id, channel: $channel, + source_sha: $source_sha, source_tree: $source_tree, tag: $tag, + version: $version, containers: {product: $product_digest, + control: $control_digest, worker: $worker_digest}}' \ + > release-candidate.json + + - name: Preserve the candidate manifest + if: >- + needs.pending.result == 'success' && + needs.contract.result == 'success' && + needs.core.result == 'success' && + needs.desktop.result == 'success' && + needs.containers.result == 'success' + uses: actions/upload-artifact@v7 + with: + name: vidxp-release-candidate + path: release-candidate.json + if-no-files-found: error + retention-days: 30 + + - name: Publish the candidate status + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ inputs.head_sha }} + SUCCESS: ${{ needs.pending.result == 'success' && needs.contract.result == 'success' && needs.core.result == 'success' && needs.desktop.result == 'success' && needs.containers.result == 'success' }} + TARGET_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + shell: bash + run: | + if [[ "$SUCCESS" == "true" ]]; then + state=success + description='Combined release candidate is ready' + else + state=failure + description='Combined release candidate failed' + fi + gh api --method POST "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f state="$state" \ + -f context=release/candidate \ + -f description="$description" \ + -f target_url="$TARGET_URL" + [[ "$state" == "success" ]] diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 0000000..7d9c8c3 --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,40 @@ +name: Release gate + +on: + pull_request_target: + branches: + - main + - release + types: + - opened + - synchronize + - reopened + +permissions: + contents: none + statuses: write + +jobs: + classify: + runs-on: ubuntu-latest + steps: + - name: Classify the pull request without checking out its code + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + TARGET_URL: ${{ github.event.pull_request.html_url }} + shell: bash + run: | + if [[ "$HEAD_REF" == release-please--branches--* ]]; then + state=pending + description='Waiting for the combined release candidate' + else + state=success + description='Not a Release Please pull request' + fi + gh api --method POST "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f state="$state" \ + -f context=release/candidate \ + -f description="$description" \ + -f target_url="$TARGET_URL" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 04e3051..2e295ea 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -12,6 +12,7 @@ permissions: contents: write issues: write pull-requests: write + statuses: write concurrency: group: release-please-${{ github.ref_name }} @@ -21,15 +22,7 @@ jobs: release: runs-on: ubuntu-latest steps: - - name: Enable the beta publication workflow - if: github.ref_name == 'main' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: >- - gh api --method PUT - "repos/$GITHUB_REPOSITORY/actions/workflows/release-to-test-pypi.yml/enable" - - - name: Prepare or create component releases + - name: Prepare or create the combined release id: release uses: googleapis/release-please-action@v5 with: @@ -38,30 +31,95 @@ jobs: target-branch: ${{ github.ref_name }} token: ${{ secrets.GITHUB_TOKEN }} - - name: Publish the core release - if: steps.release.outputs.release_created == 'true' + - name: Refresh and validate the open release candidate + if: steps.release.outputs.release_created != 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ steps.release.outputs.tag_name }} + TARGET_BRANCH: ${{ github.ref_name }} shell: bash run: | - if [[ "${{ github.ref_name }}" == "release" ]]; then - workflow="release-to-pypi.yml" - else - workflow="release-to-test-pypi.yml" + pr="$( + gh api "repos/$GITHUB_REPOSITORY/pulls?state=open&base=$TARGET_BRANCH&per_page=100" \ + --jq '[.[] | select(.head.ref | startswith("release-please--branches--"))][0]' + )" + [[ "$pr" != "null" ]] || exit 0 + + number="$(jq -r .number <<< "$pr")" + head="$(jq -r .head.sha <<< "$pr")" + base="$(jq -r .base.sha <<< "$pr")" + update_log="$RUNNER_TEMP/update-branch.log" + if ! gh api --method PUT \ + "repos/$GITHUB_REPOSITORY/pulls/$number/update-branch" \ + -f expected_head_sha="$head" >"$update_log" 2>&1; then + if ! grep -qiE 'already up.to.date|422|head branch was modified' "$update_log"; then + cat "$update_log" + exit 1 + fi fi - gh workflow run "$workflow" \ + + for attempt in {1..45}; do + pr="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$number")" + head="$(jq -r .head.sha <<< "$pr")" + base="$(jq -r .base.sha <<< "$pr")" + comparison="$(gh api "repos/$GITHUB_REPOSITORY/compare/${base}...${head}")" + if [[ "$(jq -r .merge_base_commit.sha <<< "$comparison")" == "$base" ]]; then + break + fi + sleep 2 + done + [[ "$(jq -r .merge_base_commit.sha <<< "$comparison")" == "$base" ]] + + current_status="$( + gh api "repos/$GITHUB_REPOSITORY/commits/$head/status" \ + --jq '[.statuses[] | select(.context == "release/candidate")] | sort_by(.updated_at) | last | .state // ""' + )" + if [[ "$current_status" == "success" ]]; then + echo "Candidate $head is already validated." + exit 0 + fi + + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/workflows/release-candidate.yml" + gh api --method POST "repos/$GITHUB_REPOSITORY/statuses/$head" \ + -f state=pending \ + -f context=release/candidate \ + -f description='Combined release candidate is queued' \ + -f target_url="$run_url" + gh workflow run release-candidate.yml \ --repo "$GITHUB_REPOSITORY" \ - --ref "$TAG" \ - -f tag="$TAG" + --ref "$TARGET_BRANCH" \ + -f pull_request="$number" \ + -f target_branch="$TARGET_BRANCH" \ + -f base_sha="$base" \ + -f head_sha="$head" - - name: Publish the desktop release - if: steps.release.outputs['desktop--release_created'] == 'true' + - name: Dispatch publication of the validated candidate + if: steps.release.outputs.release_created == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ steps.release.outputs['desktop--tag_name'] }} + RELEASE_SHA: ${{ steps.release.outputs.sha }} + TAG: ${{ steps.release.outputs.tag_name }} + TARGET_BRANCH: ${{ github.ref_name }} + shell: bash run: | - gh workflow run publish-desktop.yml \ + pr="$( + gh api -H 'Accept: application/vnd.github+json' \ + "repos/$GITHUB_REPOSITORY/commits/$RELEASE_SHA/pulls" \ + --jq '[.[] | select(.base.ref == env.TARGET_BRANCH and (.head.ref | startswith("release-please--branches--")))][0]' + )" + [[ "$pr" != "null" ]] + head="$(jq -r .head.sha <<< "$pr")" + status="$( + gh api "repos/$GITHUB_REPOSITORY/commits/$head/status" \ + --jq '[.statuses[] | select(.context == "release/candidate")] | sort_by(.updated_at) | last' + )" + [[ "$(jq -r .state <<< "$status")" == "success" ]] + target_url="$(jq -r .target_url <<< "$status")" + [[ "$target_url" =~ /actions/runs/([0-9]+)$ ]] + candidate_run_id="${BASH_REMATCH[1]}" + + gh workflow run release-to-pypi.yml \ --repo "$GITHUB_REPOSITORY" \ --ref "$TAG" \ - -f tag="$TAG" + -f tag="$TAG" \ + -f candidate_run_id="$candidate_run_id" \ + -f candidate_head_sha="$head" diff --git a/.github/workflows/release-to-pypi.yml b/.github/workflows/release-to-pypi.yml index cf54687..28073cc 100644 --- a/.github/workflows/release-to-pypi.yml +++ b/.github/workflows/release-to-pypi.yml @@ -1,99 +1,138 @@ -name: Release (tag → PyPI and GHCR) +name: Publish combined release on: workflow_dispatch: inputs: tag: - description: Existing stable core tag to publish. + description: Existing combined release tag created by Release Please. + required: true + type: string + candidate_run_id: + description: Successful Release candidate workflow run. + required: true + type: string + candidate_head_sha: + description: Exact Release Please head revision that was validated. required: true type: string permissions: + actions: read contents: read - pull-requests: read concurrency: - group: pypi-${{ inputs.tag }} + group: publish-${{ inputs.tag }} cancel-in-progress: false jobs: - validate: - uses: ./.github/workflows/ci.yml - with: - checkout_ref: ${{ inputs.tag }} - force_validation: true - python_versions: '["3.11","3.12","3.13","3.14"]' - provider_platforms: '["ubuntu-24.04","windows-2025","macos-15"]' - run_containers: false - - build: - needs: validate + resolve: runs-on: ubuntu-latest outputs: + channel: ${{ steps.contract.outputs.channel }} + control_digest: ${{ steps.candidate.outputs.control_digest }} + product_digest: ${{ steps.candidate.outputs.product_digest }} source_sha: ${{ steps.source.outputs.sha }} - version: ${{ steps.version.outputs.version }} + version: ${{ steps.contract.outputs.version }} + worker_digest: ${{ steps.candidate.outputs.worker_digest }} steps: - uses: actions/checkout@v7 with: + fetch-depth: 0 ref: ${{ inputs.tag }} - - name: Resolve source revision - id: source - shell: bash - run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - - uses: actions/setup-python@v7 - with: - python-version: "3.14" - cache: pip - - - name: Verify stable tag - id: version + - name: Verify tag and combined version contract + id: contract shell: bash run: | tag="${{ inputs.tag }}" - if [[ "$tag" != v* || "$tag" == *-* ]]; then - echo "::error::$tag is not a stable core release tag." - exit 1 - fi - if ! git show-ref --verify --quiet "refs/tags/$tag"; then - echo "::error::$tag is not an existing Git tag." - exit 1 - fi - version="${tag#v}" - package_version="$( - python -c \ - 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])' - )" - if [[ "$package_version" != "$version" ]]; then - echo "::error::Tag $tag does not match package version $package_version." - exit 1 - fi - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Install build tooling - run: python -m pip install -r utils/build-requirements.txt - - - name: Build distribution - run: bash utils/build_package.sh - - - name: Smoke distribution + git show-ref --verify --quiet "refs/tags/$tag" + if [[ "$tag" == *-* ]]; then channel=beta; else channel=stable; fi + python utils/release_contract.py \ + --channel "$channel" \ + --expected-tag "$tag" \ + --github-output >> "$GITHUB_OUTPUT" + + - name: Download the exact candidate artifacts + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ inputs.candidate_run_id }} shell: bash run: | - python -m venv .release-smoke - .release-smoke/bin/python -m pip install dist/*.whl - .release-smoke/bin/python -m pip check - .release-smoke/bin/vidxp --version + [[ "$RUN_ID" =~ ^[0-9]+$ ]] + run="$(gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion,event,name)" + [[ "$(jq -r .conclusion <<< "$run")" == "success" ]] + [[ "$(jq -r .event <<< "$run")" == "workflow_dispatch" ]] + [[ "$(jq -r .name <<< "$run")" == "Release candidate" ]] + for artifact in \ + vidxp-release-candidate \ + vidxp-python-dist \ + vidxp-desktop-windows-x86_64 \ + vidxp-desktop-macos-aarch64 \ + vidxp-desktop-linux-x86_64; do + gh run download "$RUN_ID" \ + --repo "$GITHUB_REPOSITORY" \ + --name "$artifact" \ + --dir "candidate-artifacts/$artifact" + done + + - name: Match the candidate to the merged release tree + id: candidate + env: + CANDIDATE_HEAD_SHA: ${{ inputs.candidate_head_sha }} + CANDIDATE_RUN_ID: ${{ inputs.candidate_run_id }} + CHANNEL: ${{ steps.contract.outputs.channel }} + TAG: ${{ steps.contract.outputs.tag }} + VERSION: ${{ steps.contract.outputs.version }} + shell: bash + run: | + [[ "$CANDIDATE_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] + candidate="candidate-artifacts/vidxp-release-candidate/release-candidate.json" + jq -e \ + --arg run "$CANDIDATE_RUN_ID" \ + --arg channel "$CHANNEL" \ + --arg sha "$CANDIDATE_HEAD_SHA" \ + --arg tag "$TAG" \ + --arg version "$VERSION" \ + '.candidate_run_id == $run and .channel == $channel and + .source_sha == $sha and .tag == $tag and .version == $version' \ + "$candidate" + + git fetch origin "$CANDIDATE_HEAD_SHA" + candidate_tree="$(git rev-parse "${CANDIDATE_HEAD_SHA}^{tree}")" + release_tree="$(git rev-parse 'HEAD^{tree}')" + recorded_tree="$(jq -r .source_tree "$candidate")" + [[ "$candidate_tree" == "$release_tree" ]] + [[ "$recorded_tree" == "$release_tree" ]] + + echo "product_digest=$(jq -r .containers.product "$candidate")" >> "$GITHUB_OUTPUT" + echo "control_digest=$(jq -r .containers.control "$candidate")" >> "$GITHUB_OUTPUT" + echo "worker_digest=$(jq -r .containers.worker "$candidate")" >> "$GITHUB_OUTPUT" + + - name: Prepare release assets and checksums + id: source + shell: bash + run: | + mkdir release-assets + find candidate-artifacts/vidxp-python-dist \ + candidate-artifacts/vidxp-desktop-* \ + -type f -exec cp '{}' release-assets/ ';' + [[ "$(find release-assets -maxdepth 1 -name '*.whl' | wc -l)" == 1 ]] + [[ "$(find release-assets -maxdepth 1 -name '*.tar.gz' | wc -l)" == 1 ]] + [[ "$(find release-assets -maxdepth 1 -name '*.exe' | wc -l)" == 1 ]] + [[ "$(find release-assets -maxdepth 1 -name '*.dmg' | wc -l)" == 1 ]] + [[ "$(find release-assets -maxdepth 1 -name '*.AppImage' | wc -l)" == 1 ]] + (cd release-assets && sha256sum * > SHA256SUMS) + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@v7 with: - name: pypi-${{ inputs.tag }} - path: dist/ + name: vidxp-release-assets + path: release-assets/ if-no-files-found: error - retention-days: 14 + retention-days: 30 publish-pypi: - needs: build + needs: resolve runs-on: ubuntu-latest environment: pypi permissions: @@ -106,23 +145,25 @@ jobs: - uses: actions/download-artifact@v8 with: - name: pypi-${{ inputs.tag }} - path: dist/ + name: vidxp-release-assets + path: release-assets/ - name: Check PyPI release state id: registry shell: bash run: | + mkdir dist + cp release-assets/*.whl release-assets/*.tar.gz dist/ state="$( python utils/verify_published_distribution.py \ --repository https://pypi.org \ --package vidxp \ - --version "${{ needs.build.outputs.version }}" \ + --version "${{ needs.resolve.outputs.version }}" \ --dist dist )" echo "state=$state" >> "$GITHUB_OUTPUT" - - name: Publish to PyPI + - name: Publish the beta or stable package to PyPI if: steps.registry.outputs.state == 'absent' uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -130,20 +171,24 @@ jobs: publish-containers: needs: - - build + - resolve - publish-pypi permissions: contents: read packages: write uses: ./.github/workflows/containers.yml with: - channel: stable - tag: ${{ inputs.tag }} - version: ${{ needs.build.outputs.version }} + candidate_id: ${{ inputs.candidate_head_sha }} + channel: ${{ needs.resolve.outputs.channel }} + control_digest: ${{ needs.resolve.outputs.control_digest }} + operation: promote + product_digest: ${{ needs.resolve.outputs.product_digest }} + version: ${{ needs.resolve.outputs.version }} + worker_digest: ${{ needs.resolve.outputs.worker_digest }} publish-github: needs: - - build + - resolve - publish-pypi - publish-containers runs-on: ubuntu-latest @@ -153,26 +198,37 @@ jobs: steps: - uses: actions/download-artifact@v8 with: - name: pypi-${{ inputs.tag }} - path: dist/ + name: vidxp-release-assets + path: release-assets/ - - name: Publish complete stable release + - name: Publish the complete combined release env: + CHANNEL: ${{ needs.resolve.outputs.channel }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ inputs.tag }} shell: bash run: | - gh release upload "${{ inputs.tag }}" dist/* \ + gh release upload "$TAG" release-assets/* \ --repo "$GITHUB_REPOSITORY" \ --clobber - gh release edit "${{ inputs.tag }}" \ - --repo "$GITHUB_REPOSITORY" \ - --draft=false + if [[ "$CHANNEL" == "beta" ]]; then + gh release edit "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --draft=false \ + --prerelease + else + gh release edit "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --draft=false \ + --latest + fi - - name: Synchronize stable source back to main + - name: Synchronize a completed stable release back to main + if: needs.resolve.outputs.channel == 'stable' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh workflow run sync-channels.yml \ --repo "$GITHUB_REPOSITORY" \ --ref main \ - -f release_sha="${{ needs.build.outputs.source_sha }}" + -f release_sha="${{ needs.resolve.outputs.source_sha }}" diff --git a/.github/workflows/release-to-test-pypi.yml b/.github/workflows/release-to-test-pypi.yml index c9d0529..a1cf2d7 100644 --- a/.github/workflows/release-to-test-pypi.yml +++ b/.github/workflows/release-to-test-pypi.yml @@ -1,18 +1,25 @@ -name: Prerelease (tag → TestPyPI) +name: Publish nightly package on: + push: + branches: + - main + paths-ignore: + - "**/*.md" + - "docs/**" workflow_dispatch: inputs: - tag: - description: Existing core prerelease tag to publish. - required: true + checkout_ref: + description: Main revision to publish as an ephemeral nightly. + required: false + default: "" type: string permissions: contents: read concurrency: - group: testpypi-${{ inputs.tag }} + group: nightly-testpypi cancel-in-progress: false jobs: @@ -23,54 +30,35 @@ jobs: steps: - uses: actions/checkout@v7 with: - ref: ${{ inputs.tag }} + ref: ${{ inputs.checkout_ref || github.sha }} - uses: actions/setup-python@v7 with: python-version: "3.14" cache: pip - - name: Verify prerelease tag + - name: Assign an ephemeral nightly version id: version shell: bash run: | - tag="${{ inputs.tag }}" - if [[ "$tag" != v*-* ]]; then - echo "::error::$tag is not a core prerelease tag." - exit 1 - fi - if ! git show-ref --verify --quiet "refs/tags/$tag"; then - echo "::error::$tag is not an existing Git tag." - exit 1 - fi - version="${tag#v}" - package_version="$( - python -c \ - 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])' - )" - if [[ "$package_version" != "$version" ]]; then - echo "::error::Tag $tag does not match package version $package_version." - exit 1 - fi + version="$(python utils/prepare_nightly.py --run-number "$GITHUB_RUN_NUMBER")" echo "version=$version" >> "$GITHUB_OUTPUT" - name: Install build tooling run: python -m pip install -r utils/build-requirements.txt - - name: Build distribution - run: bash utils/build_package.sh - - - name: Smoke distribution + - name: Build and smoke the nightly distribution shell: bash run: | - python -m venv .release-smoke - .release-smoke/bin/python -m pip install dist/*.whl - .release-smoke/bin/python -m pip check - .release-smoke/bin/vidxp --version + bash utils/build_package.sh + python -m venv .nightly-smoke + .nightly-smoke/bin/python -m pip install dist/*.whl + .nightly-smoke/bin/python -m pip check + [[ "$(.nightly-smoke/bin/vidxp --version)" == "VidXP ${{ steps.version.outputs.version }}" ]] - uses: actions/upload-artifact@v7 with: - name: testpypi-${{ inputs.tag }} + name: vidxp-nightly-dist path: dist/ if-no-files-found: error retention-days: 14 @@ -85,11 +73,11 @@ jobs: steps: - uses: actions/checkout@v7 with: - ref: ${{ inputs.tag }} + ref: ${{ inputs.checkout_ref || github.sha }} - uses: actions/download-artifact@v8 with: - name: testpypi-${{ inputs.tag }} + name: vidxp-nightly-dist path: dist/ - name: Check TestPyPI release state @@ -105,49 +93,39 @@ jobs: )" echo "state=$state" >> "$GITHUB_OUTPUT" - - name: Publish to TestPyPI + - name: Publish the unique nightly to TestPyPI if: steps.registry.outputs.state == 'absent' uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ packages-dir: dist/ - publish-containers: - needs: - - build - - publish - permissions: - contents: read - packages: write - uses: ./.github/workflows/containers.yml - with: - channel: beta - tag: ${{ inputs.tag }} - version: ${{ needs.build.outputs.version }} - - publish-github: - needs: - - build - - publish - - publish-containers - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/download-artifact@v8 - with: - name: testpypi-${{ inputs.tag }} - path: dist/ - - - name: Publish complete prerelease - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Download and install the exact TestPyPI artifact shell: bash run: | - gh release upload "${{ inputs.tag }}" dist/* \ - --repo "$GITHUB_REPOSITORY" \ - --clobber - gh release edit "${{ inputs.tag }}" \ - --repo "$GITHUB_REPOSITORY" \ - --draft=false \ - --prerelease + for attempt in {1..30}; do + state="$( + python utils/verify_published_distribution.py \ + --repository https://test.pypi.org \ + --package vidxp \ + --version "${{ needs.build.outputs.version }}" \ + --dist dist + )" + [[ "$state" == "identical" ]] && break + sleep 10 + done + [[ "$state" == "identical" ]] + + python -m pip download \ + --index-url https://test.pypi.org/simple \ + --no-deps \ + --no-cache-dir \ + --dest downloaded \ + "vidxp==${{ needs.build.outputs.version }}" + [[ "$(sha256sum dist/*.whl | cut -d' ' -f1)" == \ + "$(sha256sum downloaded/*.whl | cut -d' ' -f1)" ]] + python -m venv .testpypi-smoke + .testpypi-smoke/bin/python -m pip install downloaded/*.whl + .testpypi-smoke/bin/python -m pip check + [[ "$(.testpypi-smoke/bin/vidxp --version)" == \ + "VidXP ${{ needs.build.outputs.version }}" ]] diff --git a/.release-please-manifest.json b/.release-please-manifest.json index f9e5f5f..a44e75a 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,4 +1,3 @@ { - ".": "0.4.0-b", - "desktop": "0.4.0-b" + ".": "0.4.0-b" } diff --git a/.release-please-manifest.stable.json b/.release-please-manifest.stable.json index da42fbd..0ee8c01 100644 --- a/.release-please-manifest.stable.json +++ b/.release-please-manifest.stable.json @@ -1,4 +1,3 @@ { - ".": "0.3.0", - "desktop": "0.3.0" + ".": "0.3.0" } diff --git a/desktop/THIRD_PARTY_NOTICES.txt b/desktop/THIRD_PARTY_NOTICES.txt index d08bb5d..de97bba 100644 --- a/desktop/THIRD_PARTY_NOTICES.txt +++ b/desktop/THIRD_PARTY_NOTICES.txt @@ -8718,7 +8718,6 @@ SOFTWARE. ------------------------------------------------------------------------------- License: MIT License (MIT) Used by: -- vidxp-desktop 0.4.0-b | | - block2 0.6.2 | https://github.com/madsmtm/objc2 | registry+https://github.com/rust-lang/crates.io-index - brotli-decompressor 5.0.3 | https://github.com/dropbox/rust-brotli-decompressor | registry+https://github.com/rust-lang/crates.io-index - dlopen2 0.8.2 | https://github.com/OpenByteDev/dlopen2 | registry+https://github.com/rust-lang/crates.io-index diff --git a/desktop/VERSION b/desktop/VERSION deleted file mode 100644 index 8d1a53c..0000000 --- a/desktop/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.4.0-b diff --git a/desktop/scripts/generate-notices.mjs b/desktop/scripts/generate-notices.mjs index 57d75ab..0063425 100644 --- a/desktop/scripts/generate-notices.mjs +++ b/desktop/scripts/generate-notices.mjs @@ -130,7 +130,9 @@ try { '', uvMitLicense, '', - readFileSync(rustNotices, 'utf8').trim(), + readFileSync(rustNotices, 'utf8') + .replace(/^\- vidxp-desktop [^\n]+\n/m, '') + .trim(), '', frontend.join('\n').trim(), '', diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index b587b20..cede931 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -1010,14 +1010,6 @@ fn base_package_specification(manifest: &RuntimeManifest) -> String { format!("{}=={}", manifest.package_name, manifest.package_version) } -fn package_index(package_version: &str) -> &'static str { - if package_version.split_once('-').is_some() { - "https://test.pypi.org/simple" - } else { - "https://pypi.org/simple" - } -} - fn package_acquisition_arguments(manifest: &RuntimeManifest, python: &Path) -> Vec { vec![ "pip".into(), @@ -1027,7 +1019,7 @@ fn package_acquisition_arguments(manifest: &RuntimeManifest, python: &Path) -> V "--no-config".into(), "--no-deps".into(), "--default-index".into(), - package_index(&manifest.package_version).into(), + manifest.dependency_index.clone(), "--index-strategy".into(), "first-index".into(), base_package_specification(manifest), @@ -3232,10 +3224,9 @@ mod tests { configured_runtime_status, dependency_installation_arguments, desktop_paths_from_roots, display_command, inventory_model_directory, manifest, manifest_digest, normalize_line_endings, normalized_runtime_constraints, package_acquisition_arguments, - package_index, package_specification, read_active_runtime_snapshot, - reconcile_managed_runtime_storage, required_encoder_missing, restore_active_runtime, - selected_capabilities, selected_surfaces, ui_process_action, write_activation_journal, - write_active_runtime, + package_specification, read_active_runtime_snapshot, reconcile_managed_runtime_storage, + required_encoder_missing, restore_active_runtime, selected_capabilities, selected_surfaces, + ui_process_action, write_activation_journal, write_active_runtime, }; use std::{ ffi::OsStr, @@ -3791,7 +3782,7 @@ mod tests { let manifest = manifest().expect("manifest"); let python = Path::new("managed-python"); let constraints = Path::new("runtime-constraints.txt"); - let selected_package_index = package_index(&manifest.package_version); + let selected_package_index = manifest.dependency_index.as_str(); let acquisition = package_acquisition_arguments(&manifest, python); let dependencies = dependency_installation_arguments( &manifest, @@ -3802,15 +3793,7 @@ mod tests { true, ); - let expected_package_index = if manifest.package_version.contains('-') { - "https://test.pypi.org/simple" - } else { - "https://pypi.org/simple" - }; - - assert_eq!(selected_package_index, expected_package_index); - assert_eq!(package_index("0.3.0-b.1"), "https://test.pypi.org/simple"); - assert_eq!(package_index("0.3.0"), "https://pypi.org/simple"); + assert_eq!(selected_package_index, "https://pypi.org/simple"); assert_eq!(manifest.dependency_index, "https://pypi.org/simple"); assert!(acquisition.iter().any(|item| item == "--no-deps")); assert!( diff --git a/desktop/src/desktopPackagingContracts.test.ts b/desktop/src/desktopPackagingContracts.test.ts index 10be9f4..6fffa03 100644 --- a/desktop/src/desktopPackagingContracts.test.ts +++ b/desktop/src/desktopPackagingContracts.test.ts @@ -69,8 +69,11 @@ describe('Desktop packaging and documentation contracts', () => { ]) { expect(workflow).toContain(`- "${path}"`); } - expect(read('.github/workflows/publish-desktop.yml')).toContain( + expect(read('.github/workflows/release-candidate.yml')).toContain( 'uses: ./.github/workflows/desktop.yml', ); + expect(read('.github/workflows/release-to-pypi.yml')).toContain( + 'vidxp-desktop-windows-x86_64', + ); }); }); diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 5ccb89f..8cdb5d4 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -242,7 +242,7 @@ merge describe the public change accurately: Write user-visible `feat`, `fix`, and `perf` subjects for users, not for the implementation history. Internal corrections to an unreleased feature should remain part of that feature rather than appear as fictional public bug fixes. -Release Please prepares the version and component changelogs in a pull request; +Release Please prepares the combined version and changelog in a pull request; do not edit released changelog sections by hand. ## Documentation style diff --git a/docs/desktop.md b/docs/desktop.md index 3a79719..335a1e7 100644 --- a/docs/desktop.md +++ b/docs/desktop.md @@ -118,12 +118,11 @@ Users select dialogue, scene, and actor capabilities independently. Interfaces are selected separately: the browser interface adds the `frontend` extra only when selected. Model preparation can be deferred, and a native folder picker can select a model-cache directory before any model is downloaded. -For the current beta, it first acquires only the exact VidXP package from -TestPyPI with dependency resolution disabled. It then resolves that installed -package's selected extras from production PyPI. This prevents TestPyPI from -becoming a competing source for transitive dependencies. The acquisition index -is derived from the stamped package version: prereleases use TestPyPI and stable -versions use production PyPI. +The managed runtime acquires the exact VidXP package with dependency resolution +disabled, then resolves that package's selected extras. Beta and stable desktop +releases use production PyPI for both steps, so a pinned prerelease and its +normal dependencies come from one authoritative index. TestPyPI is used only +for package-only nightly validation and is never a desktop runtime source. Windows and Linux resolve CPU-only PyTorch wheels using uv's `--torch-backend cpu`; macOS uses native PyPI wheels. The custom PyTorch index is therefore a resolver input and is not embedded as a package URL, avoiding diff --git a/docs/releasing.md b/docs/releasing.md index f037022..05367de 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,73 +1,79 @@ # Release process -VidXP publishes two independently versioned components: - -| Component | Tag | Beta artifacts | Stable artifacts | -|---|---|---|---| -| Core | `v` | TestPyPI and `beta` GHCR images | PyPI and versioned/`latest` GHCR images | -| Desktop | `desktop-v` | Prerelease installers | Stable installers | - -`main` is the default integration branch and the beta channel. `release` is -the protected stable channel. - -## Beta channel - -Feature and fix pull requests target `main`. Release Please maintains a beta -release pull request on that branch from Conventional Commits. - -Merging the beta release pull request: - -1. updates only the changed component versions and changelogs; -2. creates immutable prerelease tags and draft GitHub releases; -3. publishes a changed core to TestPyPI and the `beta` product, control, and - worker images to GHCR; -4. builds changed desktop installers; and -5. exposes each GitHub prerelease only after its complete artifact set exists. - -The desktop runtime manifest pins the selected core package version. A core -release updates that pin, but a new desktop installer is published only when -the desktop component also has a releasable change. - -## Promote beta to stable - -The **Maintain stable promotion PR** workflow keeps one draft pull request from -`main` to `release` whenever the branches differ. Closing it without merging -causes a fresh draft to be created while there are still commits to promote. - -To promote: - -1. freeze merges to `main` for the short promotion window; -2. mark the `main` → `release` draft ready and merge it; -3. review the stable Release Please pull request created on `release`; and -4. merge that pull request to publish stable artifacts. - -A stable core release runs the complete Python and provider matrix, builds and -smokes the distribution, publishes it to PyPI, then builds and smokes all -three GHCR images. Desktop releases build Windows, macOS, and Linux installers. -GitHub releases remain drafts until their complete artifact set is public. - -After all stable releases created from that commit are public, -**Synchronize release channels** fast-forwards `main` to `release`. It refuses -to overwrite new work on `main`. The scheduled run repairs a missed -synchronization after the publication state becomes complete. - -## Publication integrity and retries - -Registry publication is version-locked: - -- an absent Python version is published normally; -- an existing Python version is accepted only when its complete filename and - SHA-256 set exactly matches the distribution built from the immutable tag; -- an existing versioned container tag is accepted only when its source revision - label matches the immutable release tag; and -- any conflict stops publication before moving channel tags or exposing the - GitHub release. - -For an infrastructure failure, rerun the failed jobs from the same workflow -run. If a workflow fix is required, merge the fix to the channel branch and -dispatch the publisher again with the existing tag. Verified matching -artifacts are reused; conflicting artifacts fail instead of being skipped. - -If version selection or changelog content is wrong, do not merge the Release -Please pull request. Correct the commits or configuration and let Release -Please update it. +VidXP ships one version across the Python package, three desktop installers, +and the product, control, and worker container images. + +| Channel | Trigger | Python | Desktop | Containers | GitHub | +|---|---|---|---|---|---| +| Nightly | Non-documentation push to `main` | Unique `.dev…` build on TestPyPI | — | — | Actions artifact only | +| Beta | Merge the Release Please PR into `main` | PyPI prerelease | Windows, macOS, Linux | Versioned and `beta` tags | One prerelease | +| Stable | Merge the Release Please PR into `release` | PyPI release | Windows, macOS, Linux | Versioned and `latest` tags | One latest release | + +`main` is the integration and beta branch. `release` is the stable branch. +Feature and fix pull requests target `main`; the existing promotion workflow +maintains the `main` → `release` draft used to select a stable baseline. + +## Candidate before merge + +Release Please maintains one combined release PR per channel and updates every +version source together. The **Prepare releases** workflow brings that PR up to +date with its target branch and explicitly dispatches **Release candidate**. + +The candidate reuses the normal CI, desktop, and container workflows. It: + +1. validates the exact Release Please head against the current target branch; +2. runs the full Python/provider suite and retains its tested wheel and sdist; +3. builds and tests all three desktop installers and retains them; +4. builds and smokes the three container targets once, pushes temporary + candidate tags, and records their immutable digests; and +5. records a `release/candidate` commit status linked to the Actions run. + +Merging a Release Please PR is allowed only when `release/candidate` succeeds. +The lightweight **Release gate** marks that same context successful for ordinary +PRs, so the rule does not add release builds to normal development changes. + +Configure `release/candidate` as a required status on both channel branches. +Roll this out in order: merge the workflow changes to `main`, enable the rule on +`main`, promote those workflows to `release`, then enable the rule on `release`. +Do not enable the `release` rule before its base branch contains Release gate. + +## Publication after merge + +After a valid release PR is merged, Release Please creates the combined tag and +a draft GitHub release. **Publish combined release** then: + +1. downloads artifacts from the successful candidate run; +2. proves that the merged tag and candidate have the same Git tree and version; +3. publishes the already-tested wheel and sdist to PyPI; +4. promotes the recorded container digests to public version/channel tags + without rebuilding; and +5. uploads the Python and desktop artifacts plus checksums to the same GitHub + release before making it public. + +Beta packages intentionally use real PyPI so the desktop-managed runtime can +resolve its pinned prerelease and normal dependencies from one index. TestPyPI +is reserved for unique nightly package validation. + +Publication is resumable. An existing Python version must have the exact same +filenames and SHA-256 values; immutable container tags must resolve to the +recorded candidate digests. Matching work is reused, while a conflict stops the +run. A failed publication leaves the GitHub release as a draft, and rerunning +the same publisher continues from the same candidate without rebuilding or +rewriting a branch. + +After a stable release is public, **Synchronize release channels** carries the +published version baseline back to `main`. It fast-forwards when possible and +opens a synchronization PR rather than overwriting newer work. + +## Maintainer checklist + +Before merging a Release Please PR: + +- confirm it contains one `v` change across the package and desktop + manifests; +- confirm `release/candidate` points to the latest PR head and succeeded; +- review the generated changelog as product release notes; and +- for stable, confirm the selected `main` baseline was promoted to `release`. + +If version selection or changelog content is wrong, do not merge the release +PR. Correct the commits or configuration and let Release Please update it. diff --git a/release-please-config.json b/release-please-config.json index e74a9f2..ca8369f 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -1,5 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "always-update": true, "bootstrap-sha": "61e8b6f8374f3fa3b8c4012e499b4064d8f5c4d2", "bump-minor-pre-major": false, "bump-patch-for-minor-pre-major": false, @@ -7,77 +8,53 @@ "force-tag-creation": true, "include-v-in-tag": true, "prerelease": true, - "separate-pull-requests": false, "versioning": "prerelease", - "plugins": [ - { - "type": "linked-versions", - "groupName": "VidXP", - "components": [ - "", - "desktop" - ] - } - ], "packages": { ".": { - "exclude-paths": [ - "desktop" - ], - "extra-files": [ - { - "jsonpath": "$.package_version", - "path": "desktop/runtime-manifest.json", - "type": "json" - } - ], - "include-component-in-tag": false, - "package-name": "vidxp", - "prerelease-type": "b", - "release-type": "python" - }, - "desktop": { - "component": "desktop", "extra-files": [ { - "path": "src-tauri/Cargo.toml", + "path": "desktop/src-tauri/Cargo.toml", "type": "generic" }, { - "path": "src-tauri/Cargo.lock", + "path": "desktop/src-tauri/Cargo.lock", "type": "generic" }, { "jsonpath": "$.version", - "path": "package.json", + "path": "desktop/package.json", "type": "json" }, { "jsonpath": "$.version", - "path": "package-lock.json", + "path": "desktop/package-lock.json", "type": "json" }, { "jsonpath": "$.packages[''].version", - "path": "package-lock.json", + "path": "desktop/package-lock.json", "type": "json" }, { "jsonpath": "$.desktop_version", - "path": "runtime-manifest.json", + "path": "desktop/runtime-manifest.json", + "type": "json" + }, + { + "jsonpath": "$.package_version", + "path": "desktop/runtime-manifest.json", "type": "json" }, { "jsonpath": "$.version", - "path": "src-tauri/tauri.conf.json", + "path": "desktop/src-tauri/tauri.conf.json", "type": "json" } ], - "include-component-in-tag": true, - "package-name": "vidxp-desktop", + "include-component-in-tag": false, + "package-name": "vidxp", "prerelease-type": "b", - "release-type": "simple", - "version-file": "VERSION" + "release-type": "python" } } } diff --git a/release-please-config.stable.json b/release-please-config.stable.json index ae6f200..34c8128 100644 --- a/release-please-config.stable.json +++ b/release-please-config.stable.json @@ -1,27 +1,14 @@ { "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "always-update": true, "bootstrap-sha": "61e8b6f8374f3fa3b8c4012e499b4064d8f5c4d2", "bump-minor-pre-major": false, "bump-patch-for-minor-pre-major": false, "draft": true, "force-tag-creation": true, "include-v-in-tag": true, - "separate-pull-requests": false, - "plugins": [ - { - "type": "linked-versions", - "groupName": "VidXP", - "components": [ - "", - "desktop" - ] - } - ], "packages": { ".": { - "exclude-paths": [ - "desktop" - ], "extra-files": [ { "jsonpath": "$['.']", @@ -29,61 +16,47 @@ "type": "json" }, { - "jsonpath": "$.desktop", - "path": ".release-please-manifest.json", - "type": "json" - }, - { - "jsonpath": "$.package_version", - "path": "desktop/runtime-manifest.json", - "type": "json" - } - ], - "include-component-in-tag": false, - "package-name": "vidxp", - "release-type": "python" - }, - "desktop": { - "component": "desktop", - "extra-files": [ - { - "path": "src-tauri/Cargo.toml", + "path": "desktop/src-tauri/Cargo.toml", "type": "generic" }, { - "path": "src-tauri/Cargo.lock", + "path": "desktop/src-tauri/Cargo.lock", "type": "generic" }, { "jsonpath": "$.version", - "path": "package.json", + "path": "desktop/package.json", "type": "json" }, { "jsonpath": "$.version", - "path": "package-lock.json", + "path": "desktop/package-lock.json", "type": "json" }, { "jsonpath": "$.packages[''].version", - "path": "package-lock.json", + "path": "desktop/package-lock.json", "type": "json" }, { "jsonpath": "$.desktop_version", - "path": "runtime-manifest.json", + "path": "desktop/runtime-manifest.json", + "type": "json" + }, + { + "jsonpath": "$.package_version", + "path": "desktop/runtime-manifest.json", "type": "json" }, { "jsonpath": "$.version", - "path": "src-tauri/tauri.conf.json", + "path": "desktop/src-tauri/tauri.conf.json", "type": "json" } ], - "include-component-in-tag": true, - "package-name": "vidxp-desktop", - "release-type": "simple", - "version-file": "VERSION" + "include-component-in-tag": false, + "package-name": "vidxp", + "release-type": "python" } } } diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 868c682..d68139d 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -581,7 +581,15 @@ def test_desktop_manifest_matches_published_package_contract(self): "system", ) - def test_release_please_preserves_desktop_manifests_and_links_versions(self): + def test_combined_release_contract_and_workflow_boundaries(self): + expected_extra_files = { + "desktop/src-tauri/Cargo.toml", + "desktop/src-tauri/Cargo.lock", + "desktop/package.json", + "desktop/package-lock.json", + "desktop/runtime-manifest.json", + "desktop/src-tauri/tauri.conf.json", + } for filename in ( "release-please-config.json", "release-please-config.stable.json", @@ -589,31 +597,32 @@ def test_release_please_preserves_desktop_manifests_and_links_versions(self): config = json.loads( (ROOT / filename).read_text(encoding="utf-8") ) - self.assertNotIn("group-pull-request-title-pattern", config) - linked_versions = [ - plugin - for plugin in config["plugins"] - if plugin["type"] == "linked-versions" - ] - self.assertEqual(len(linked_versions), 1, filename) + self.assertEqual(set(config["packages"]), {"."}, filename) root_package = config["packages"]["."] self.assertFalse(root_package["include-component-in-tag"]) self.assertNotIn("component", root_package) self.assertEqual( - set(linked_versions[0]["components"]), - {"", "desktop"}, + { + entry["path"] + for entry in root_package["extra-files"] + if entry["path"] != ".release-please-manifest.json" + }, + expected_extra_files, filename, ) - - desktop = config["packages"]["desktop"] - self.assertEqual(desktop["version-file"], "VERSION", filename) generic_files = { extra["path"] - for extra in desktop["extra-files"] + for extra in root_package["extra-files"] if extra["type"] == "generic" } - self.assertIn("src-tauri/Cargo.toml", generic_files, filename) - self.assertIn("src-tauri/Cargo.lock", generic_files, filename) + self.assertEqual( + generic_files, + { + "desktop/src-tauri/Cargo.toml", + "desktop/src-tauri/Cargo.lock", + }, + filename, + ) stable = json.loads( (ROOT / "release-please-config.stable.json").read_text( @@ -628,11 +637,16 @@ def test_release_please_preserves_desktop_manifests_and_links_versions(self): } self.assertEqual( beta_manifest_updates, - { - (".release-please-manifest.json", "$['.']"), - (".release-please-manifest.json", "$.desktop"), - }, + {(".release-please-manifest.json", "$['.']")}, ) + for filename in ( + ".release-please-manifest.json", + ".release-please-manifest.stable.json", + ): + manifest_versions = json.loads( + (ROOT / filename).read_text(encoding="utf-8") + ) + self.assertEqual(set(manifest_versions), {"."}, filename) manifest = json.loads( (ROOT / "desktop" / "runtime-manifest.json").read_text( @@ -647,14 +661,18 @@ def test_release_please_preserves_desktop_manifests_and_links_versions(self): ROOT / "desktop" / "src-tauri" / "Cargo.toml" ).read_text(encoding="utf-8") ) - version_file = ( - ROOT / "desktop" / "VERSION" - ).read_text(encoding="utf-8").strip() - self.assertEqual(version_file, manifest["desktop_version"]) - self.assertEqual(version_file, package["version"]) - self.assertEqual(version_file, cargo["package"]["version"]) + project = tomllib.loads( + (ROOT / "pyproject.toml").read_text(encoding="utf-8") + ) + version = project["project"]["version"] + self.assertFalse((ROOT / "desktop" / "VERSION").exists()) + self.assertEqual(version, manifest["desktop_version"]) + self.assertEqual(version, manifest["package_version"]) + self.assertEqual(version, package["version"]) + self.assertEqual(version, cargo["package"]["version"]) + self.assertEqual(manifest["dependency_index"], "https://pypi.org/simple") version_marker = ( - f'version = "{version_file}" # x-release-please-version' + f'version = "{version}" # x-release-please-version' ) for filename in ("Cargo.toml", "Cargo.lock"): self.assertIn( @@ -665,62 +683,41 @@ def test_release_please_preserves_desktop_manifests_and_links_versions(self): filename, ) self.assertNotIn( - f"vidxp=={version_file}", + f"vidxp=={version}", ( ROOT / "desktop" / "src-tauri" / "src" / "lib.rs" ).read_text(encoding="utf-8"), ) - build_command = "bash utils/build_package.sh" - for workflow in ( - ".github/workflows/ci.yml", - ".github/workflows/release-to-test-pypi.yml", - ".github/workflows/release-to-pypi.yml", - ): - self.assertIn( - build_command, - (ROOT / workflow).read_text(encoding="utf-8"), - workflow, - ) + ci = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + nightly = ( + ROOT / ".github/workflows/release-to-test-pypi.yml" + ).read_text(encoding="utf-8") + publisher = ( + ROOT / ".github/workflows/release-to-pypi.yml" + ).read_text(encoding="utf-8") + self.assertIn("bash utils/build_package.sh", ci) + self.assertIn("bash utils/build_package.sh", nightly) + self.assertNotIn("bash utils/build_package.sh", publisher) + self.assertIn("candidate_run_id", publisher) + self.assertIn("source_tree", publisher) + self.assertIn("operation: promote", publisher) release_workflow = ( ROOT / ".github" / "workflows" / "release-please.yml" ).read_text(encoding="utf-8") - self.assertEqual(release_workflow.count('--ref "$TAG"'), 2) - self.assertEqual( - release_workflow.count('--repo "$GITHUB_REPOSITORY"'), - 2, - ) - self.assertNotIn( - '--ref "${{ github.ref_name }}"', - release_workflow, - ) - - publisher_repo_flags = { - "release-to-test-pypi.yml": 2, - "release-to-pypi.yml": 3, - "publish-desktop.yml": 4, - } - for workflow, expected in publisher_repo_flags.items(): - contents = ( - ROOT / ".github" / "workflows" / workflow - ).read_text(encoding="utf-8") - self.assertEqual( - contents.count('--repo "$GITHUB_REPOSITORY"'), - expected, - workflow, - ) - - desktop_publish = ( - ROOT / ".github" / "workflows" / "publish-desktop.yml" - ).read_text(encoding="utf-8") - self.assertIn('"VERSION":', desktop_publish) - self.assertIn('"runtime manifest package":', desktop_publish) - self.assertEqual(desktop_publish.count("--latest"), 1) - core_publish = ( - ROOT / ".github" / "workflows" / "release-to-pypi.yml" + self.assertIn("gh workflow run release-candidate.yml", release_workflow) + self.assertIn("gh workflow run release-to-pypi.yml", release_workflow) + self.assertIn('--ref "$TARGET_BRANCH"', release_workflow) + self.assertIn('--ref "$TAG"', release_workflow) + self.assertFalse( + (ROOT / ".github/workflows/publish-desktop.yml").exists() + ) + gate = ( + ROOT / ".github/workflows/release-gate.yml" ).read_text(encoding="utf-8") - self.assertNotIn("--latest", core_publish) + self.assertIn("pull_request_target:", gate) + self.assertIn("context=release/candidate", gate) for workflow in ("ci.yml", "desktop.yml", "security.yml"): contents = ( diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py new file mode 100644 index 0000000..49c384e --- /dev/null +++ b/tests/test_release_contract.py @@ -0,0 +1,55 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from utils.release_contract import validate +from utils.prepare_nightly import prepare + + +class ReleaseContractTests(unittest.TestCase): + def copy_contract(self, destination: Path) -> None: + root = Path(__file__).resolve().parents[1] + for relative in ( + "pyproject.toml", + "desktop/package.json", + "desktop/package-lock.json", + "desktop/runtime-manifest.json", + "desktop/src-tauri/Cargo.toml", + "desktop/src-tauri/tauri.conf.json", + ): + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((root / relative).read_bytes()) + + def test_beta_contract_matches_every_release_source(self): + self.assertEqual(validate("beta", "v0.4.0-b"), "0.4.0-b") + + def test_rejects_a_divergent_desktop_version(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.copy_contract(root) + package = json.loads((root / "desktop/package.json").read_text()) + package["version"] = "9.9.9" + (root / "desktop/package.json").write_text(json.dumps(package)) + with self.assertRaisesRegex(ValueError, "sources disagree"): + validate("beta", None, root) + + def test_beta_and_stable_channels_are_not_interchangeable(self): + with self.assertRaisesRegex(ValueError, "stable release version"): + validate("stable", None) + + def test_tag_must_match_the_combined_version(self): + with self.assertRaisesRegex(ValueError, "does not match"): + validate("beta", "v0.4.0-b.9") + + def test_nightly_version_is_unique_without_changing_release_sources(self): + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "pyproject.toml" + project.write_text('[project]\nname = "vidxp"\nversion = "0.4.0-b"\n') + self.assertEqual(prepare(project, 12345), "0.4.0.dev12345") + self.assertIn('version = "0.4.0.dev12345"', project.read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/check_workflows.sh b/utils/check_workflows.sh new file mode 100644 index 0000000..6f01aff --- /dev/null +++ b/utils/check_workflows.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +version="1.7.12" +archive="actionlint_${version}_linux_amd64.tar.gz" +checksum="8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" +temporary="$(mktemp -d)" +trap 'rm -rf -- "$temporary"' EXIT + +curl --fail --location --silent --show-error \ + "https://github.com/rhysd/actionlint/releases/download/v${version}/${archive}" \ + --output "$temporary/$archive" +echo "$checksum $temporary/$archive" | sha256sum --check --status +tar -xzf "$temporary/$archive" -C "$temporary" actionlint +"$temporary/actionlint" diff --git a/utils/prepare_nightly.py b/utils/prepare_nightly.py new file mode 100644 index 0000000..b2a5110 --- /dev/null +++ b/utils/prepare_nightly.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Give an ephemeral main-branch build a unique PEP 440 development version.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + + +PROJECT_VERSION = re.compile( + r'(?ms)^(\[project\].*?^version = ")([^"]+)("\s*$)' +) +RELEASE_CORE = re.compile(r"^(\d+\.\d+\.\d+)") + + +def prepare(path: Path, run_number: int) -> str: + contents = path.read_text(encoding="utf-8") + match = PROJECT_VERSION.search(contents) + if match is None: + raise ValueError(f"project version not found in {path}") + core = RELEASE_CORE.match(match.group(2)) + if core is None: + raise ValueError(f"release core not found in {match.group(2)!r}") + version = f"{core.group(1)}.dev{run_number}" + path.write_text( + PROJECT_VERSION.sub(rf"\g<1>{version}\g<3>", contents, count=1), + encoding="utf-8", + ) + return version + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-number", required=True, type=int) + parser.add_argument("--path", type=Path, default=Path("pyproject.toml")) + args = parser.parse_args() + print(prepare(args.path, args.run_number)) + + +if __name__ == "__main__": + main() diff --git a/utils/release_contract.py b/utils/release_contract.py new file mode 100644 index 0000000..068b555 --- /dev/null +++ b/utils/release_contract.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Validate and report the repository's combined release version contract.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import tomllib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +STABLE_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +BETA_VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+-b(?:\.[0-9]+)?$") + + +def _json(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def version_sources(root: Path = ROOT) -> dict[str, str]: + project = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8")) + desktop = root / "desktop" + package = _json(desktop / "package.json") + package_lock = _json(desktop / "package-lock.json") + runtime = _json(desktop / "runtime-manifest.json") + tauri = _json(desktop / "src-tauri" / "tauri.conf.json") + cargo = tomllib.loads( + (desktop / "src-tauri" / "Cargo.toml").read_text(encoding="utf-8") + ) + return { + "pyproject.toml": project["project"]["version"], + "desktop/package.json": package["version"], + "desktop/package-lock.json": package_lock["version"], + "desktop/package-lock.json root": package_lock["packages"][""]["version"], + "desktop/runtime-manifest.json desktop": runtime["desktop_version"], + "desktop/runtime-manifest.json package": runtime["package_version"], + "desktop/src-tauri/Cargo.toml": cargo["package"]["version"], + "desktop/src-tauri/tauri.conf.json": tauri["version"], + } + + +def validate(channel: str, expected_tag: str | None, root: Path = ROOT) -> str: + sources = version_sources(root) + versions = set(sources.values()) + if len(versions) != 1: + detail = ", ".join(f"{source}={version}" for source, version in sources.items()) + raise ValueError(f"release version sources disagree: {detail}") + version = versions.pop() + pattern = STABLE_VERSION if channel == "stable" else BETA_VERSION + if not pattern.fullmatch(version): + raise ValueError(f"{version!r} is not a valid {channel} release version") + tag = f"v{version}" + if expected_tag is not None and tag != expected_tag: + raise ValueError(f"tag {expected_tag!r} does not match repository version {version!r}") + + runtime = _json(root / "desktop" / "runtime-manifest.json") + if runtime["dependency_index"] != "https://pypi.org/simple": + raise ValueError("desktop releases must install VidXP from the public PyPI index") + return version + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--channel", choices=("beta", "stable"), required=True) + parser.add_argument("--expected-tag") + parser.add_argument("--github-output", action="store_true") + args = parser.parse_args() + try: + version = validate(args.channel, args.expected_tag) + except (KeyError, OSError, TypeError, ValueError) as error: + print(f"release contract invalid: {error}", file=sys.stderr) + return 1 + + values = {"channel": args.channel, "tag": f"v{version}", "version": version} + if args.github_output: + for key, value in values.items(): + print(f"{key}={value}") + else: + print(json.dumps(values, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/validate_compose.sh b/utils/validate_compose.sh new file mode 100644 index 0000000..06bee18 --- /dev/null +++ b/utils/validate_compose.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-ci-postgres-password}" +export VIDXP_ARTIFACT_DOWNLOAD_SECRET="${VIDXP_ARTIFACT_DOWNLOAD_SECRET:-ci-artifact-download-secret-000000}" +export VIDXP_CONTROL_IMAGE="${VIDXP_CONTROL_IMAGE:-vidxp-control:ci}" +export VIDXP_PUBLIC_API_HOST="${VIDXP_PUBLIC_API_HOST:-api.example.test}" +export VIDXP_UPLOAD_CLEANUP_TOKEN="${VIDXP_UPLOAD_CLEANUP_TOKEN:-ci-upload-cleanup-token-0000000000}" +export VIDXP_UPLOAD_CORS_ORIGIN_REGEX="${VIDXP_UPLOAD_CORS_ORIGIN_REGEX:-^(https://api\.example\.test)$}" +export VIDXP_UPLOAD_HANDOFF_PUBLIC_URL="${VIDXP_UPLOAD_HANDOFF_PUBLIC_URL:-https://api.example.test/upload-handoff}" +export VIDXP_UPLOAD_HANDOFF_SECRET="${VIDXP_UPLOAD_HANDOFF_SECRET:-ci-upload-handoff-secret-000000000}" +export VIDXP_UPLOAD_PUBLIC_ENDPOINT="${VIDXP_UPLOAD_PUBLIC_ENDPOINT:-https://uploads.example.test/uploads/}" +export VIDXP_WORKER_IMAGE="${VIDXP_WORKER_IMAGE:-vidxp-worker:ci}" + +docker compose -f compose.yaml config --quiet +docker compose -f compose.coolify.yaml config --quiet From 502ee6b5fabf77d9414c896a41ff3ce555219e3f Mon Sep 17 00:00:00 2001 From: Talha Date: Mon, 3 Aug 2026 15:28:19 +0500 Subject: [PATCH 2/8] feat(release): add product-facing release page --- .github/release-intro.md | 30 +++++++ .github/workflows/release-to-pypi.yml | 20 +++++ docs/releasing.md | 9 +- tests/test_packaging.py | 3 + tests/test_release_notes.py | 62 +++++++++++++ utils/render_release_notes.py | 122 ++++++++++++++++++++++++++ 6 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 .github/release-intro.md create mode 100644 tests/test_release_notes.py create mode 100644 utils/render_release_notes.py diff --git a/.github/release-intro.md b/.github/release-intro.md new file mode 100644 index 0000000..e509065 --- /dev/null +++ b/.github/release-intro.md @@ -0,0 +1,30 @@ +## Download VidXP + +{release_notice}VidXP turns video into searchable dialogue, scenes, people, and +inspectable evidence. Choose the desktop app for the guided local setup, or use +the Python package and containers for command-line and server deployments. + +| Platform | Download | +| --- | --- | +| Windows x86-64 | [Installer]({windows_url}) | +| macOS Apple Silicon | [DMG]({macos_url}) | +| Linux x86-64 | [AppImage]({linux_url}) | + +Verify downloaded files against [SHA256SUMS]({checksums_url}). + +### Command line + +```bash +python -m pip install "vidxp[local-worker,mcp]=={version}" +``` + +### Containers + +```bash +docker pull {container_image}:{version} +``` + +Server deployments use the matching `-control` and `-worker` image tags. + +[Installation guide]({installation_url}) · [Server deployment]({deployment_url}) · +[Report an issue]({issues_url}) diff --git a/.github/workflows/release-to-pypi.yml b/.github/workflows/release-to-pypi.yml index 28073cc..5074130 100644 --- a/.github/workflows/release-to-pypi.yml +++ b/.github/workflows/release-to-pypi.yml @@ -196,6 +196,10 @@ jobs: actions: write contents: write steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.tag }} + - uses: actions/download-artifact@v8 with: name: vidxp-release-assets @@ -206,19 +210,35 @@ jobs: CHANNEL: ${{ needs.resolve.outputs.channel }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ inputs.tag }} + VERSION: ${{ needs.resolve.outputs.version }} shell: bash run: | + gh release view "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --json body \ + --jq .body > release-please-notes.md + python utils/render_release_notes.py \ + --assets release-assets \ + --channel "$CHANNEL" \ + --existing release-please-notes.md \ + --output release-notes.md \ + --repository "$GITHUB_REPOSITORY" \ + --tag "$TAG" \ + --template .github/release-intro.md \ + --version "$VERSION" gh release upload "$TAG" release-assets/* \ --repo "$GITHUB_REPOSITORY" \ --clobber if [[ "$CHANNEL" == "beta" ]]; then gh release edit "$TAG" \ --repo "$GITHUB_REPOSITORY" \ + --notes-file release-notes.md \ --draft=false \ --prerelease else gh release edit "$TAG" \ --repo "$GITHUB_REPOSITORY" \ + --notes-file release-notes.md \ --draft=false \ --latest fi diff --git a/docs/releasing.md b/docs/releasing.md index 05367de..1ce79be 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -48,7 +48,14 @@ a draft GitHub release. **Publish combined release** then: 4. promotes the recorded container digests to public version/channel tags without rebuilding; and 5. uploads the Python and desktop artifacts plus checksums to the same GitHub - release before making it public. + release, prepends the product download/install guide to Release Please's + generated changes, and then makes the complete release public. + +The product-facing introduction is maintained once in +`.github/release-intro.md`. The publisher fills it with the exact installer +filenames from the validated candidate and preserves the generated changelog +below it. Re-running publication updates the same marked section instead of +duplicating release notes. Beta packages intentionally use real PyPI so the desktop-managed runtime can resolve its pinned prerelease and normal dependencies from one index. TestPyPI diff --git a/tests/test_packaging.py b/tests/test_packaging.py index d68139d..ecbc283 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -702,6 +702,9 @@ def test_combined_release_contract_and_workflow_boundaries(self): self.assertIn("candidate_run_id", publisher) self.assertIn("source_tree", publisher) self.assertIn("operation: promote", publisher) + self.assertIn("utils/render_release_notes.py", publisher) + self.assertIn("--notes-file release-notes.md", publisher) + self.assertTrue((ROOT / ".github/release-intro.md").is_file()) release_workflow = ( ROOT / ".github" / "workflows" / "release-please.yml" diff --git a/tests/test_release_notes.py b/tests/test_release_notes.py new file mode 100644 index 0000000..af07539 --- /dev/null +++ b/tests/test_release_notes.py @@ -0,0 +1,62 @@ +import tempfile +import unittest +from pathlib import Path + +from utils.render_release_notes import render + + +ROOT = Path(__file__).resolve().parents[1] + + +class ReleaseNotesTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.assets = Path(self.temporary.name) + for name in ( + "VidXP Setup.exe", + "VidXP.dmg", + "VidXP.AppImage", + "SHA256SUMS", + ): + (self.assets / name).touch() + self.template = ( + ROOT / ".github" / "release-intro.md" + ).read_text(encoding="utf-8") + + def render(self, notes: str, channel: str = "stable") -> str: + return render( + template=self.template, + existing_notes=notes, + assets=self.assets, + repository="GrayhatDevelopers/VidXP", + tag="v0.4.0-b", + version="0.4.0-b", + channel=channel, + ) + + def test_release_page_links_exact_assets_before_changelog(self): + result = self.render("## Changelog\n\n* Added evidence boards.") + + self.assertLess(result.index("## Download VidXP"), result.index("## Changelog")) + self.assertIn("VidXP%20Setup.exe", result) + self.assertIn("vidxp[local-worker,mcp]==0.4.0-b", result) + self.assertIn("ghcr.io/grayhatdevelopers/vidxp:0.4.0-b", result) + self.assertNotIn("Beta release", result) + + def test_beta_warning_and_rendering_are_idempotent(self): + first = self.render("* Fixed upload recovery.", channel="beta") + second = self.render(first, channel="beta") + + self.assertEqual(first, second) + self.assertEqual(first.count("Beta release"), 1) + self.assertEqual(first.count("Fixed upload recovery"), 1) + + def test_rejects_an_ambiguous_platform_asset(self): + (self.assets / "another.exe").touch() + with self.assertRaisesRegex(ValueError, "exactly one .*exe"): + self.render("changes") + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/render_release_notes.py b/utils/render_release_notes.py new file mode 100644 index 0000000..7a9a662 --- /dev/null +++ b/utils/render_release_notes.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import argparse +import re +from pathlib import Path +from urllib.parse import quote + +PAGE_START = "" +CHANGELOG_START = "" +PAGE_END = "" +COMPOSED_PAGE = re.compile( + rf"{re.escape(PAGE_START)}.*?{re.escape(CHANGELOG_START)}" + rf"(?P.*?){re.escape(PAGE_END)}", + re.DOTALL, +) + + +def _one_asset(assets: Path, pattern: str) -> Path: + matches = sorted(path for path in assets.glob(pattern) if path.is_file()) + if len(matches) != 1: + raise ValueError( + f"expected exactly one {pattern} release asset, found {len(matches)}" + ) + return matches[0] + + +def _download_url(repository: str, tag: str, asset: Path) -> str: + encoded_tag = quote(tag, safe="") + encoded_name = quote(asset.name, safe="") + return ( + f"https://github.com/{repository}/releases/download/" + f"{encoded_tag}/{encoded_name}" + ) + + +def _original_changelog(notes: str) -> str: + match = COMPOSED_PAGE.fullmatch(notes.strip()) + if match: + return match.group("changelog").strip() + return notes.strip() + + +def render( + *, + template: str, + existing_notes: str, + assets: Path, + repository: str, + tag: str, + version: str, + channel: str, +) -> str: + if channel not in {"beta", "stable"}: + raise ValueError(f"unsupported release channel: {channel}") + + asset_paths = { + "windows": _one_asset(assets, "*.exe"), + "macos": _one_asset(assets, "*.dmg"), + "linux": _one_asset(assets, "*.AppImage"), + "checksums": _one_asset(assets, "SHA256SUMS"), + } + source_root = f"https://github.com/{repository}/blob/{quote(tag, safe='')}" + release_notice = "" + if channel == "beta": + release_notice = ( + "> **Beta release:** This build is ready for testing, but may still " + "contain rough edges. Please report anything unexpected.\n\n" + ) + + intro = template.format( + checksums_url=_download_url( + repository, tag, asset_paths["checksums"] + ), + container_image=f"ghcr.io/{repository.lower()}", + deployment_url=f"{source_root}/docs/deployment/coolify.md", + installation_url=f"{source_root}/INSTALLATION_GUIDE.md", + issues_url=f"https://github.com/{repository}/issues", + linux_url=_download_url(repository, tag, asset_paths["linux"]), + macos_url=_download_url(repository, tag, asset_paths["macos"]), + release_notice=release_notice, + version=version, + windows_url=_download_url(repository, tag, asset_paths["windows"]), + ).strip() + changelog = _original_changelog(existing_notes) + if not changelog: + changelog = "No user-facing changes were listed for this release." + + return ( + f"{PAGE_START}\n{intro}\n\n" + f"## What changed\n\n{CHANGELOG_START}\n" + f"{changelog}\n{PAGE_END}\n" + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Prepend the VidXP download guide to generated release notes." + ) + parser.add_argument("--assets", type=Path, required=True) + parser.add_argument("--channel", choices=("beta", "stable"), required=True) + parser.add_argument("--existing", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repository", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--template", type=Path, required=True) + parser.add_argument("--version", required=True) + args = parser.parse_args() + + rendered = render( + template=args.template.read_text(encoding="utf-8"), + existing_notes=args.existing.read_text(encoding="utf-8"), + assets=args.assets, + repository=args.repository, + tag=args.tag, + version=args.version, + channel=args.channel, + ) + args.output.write_text(rendered, encoding="utf-8") + + +if __name__ == "__main__": + main() From c258f1121436472cce4cc82256208c70c1320130 Mon Sep 17 00:00:00 2001 From: Talha Date: Mon, 3 Aug 2026 15:33:05 +0500 Subject: [PATCH 3/8] fix(ci): satisfy workflow shell checks --- .github/workflows/ci.yml | 10 ++++++---- .github/workflows/containers.yml | 10 ++++++---- .github/workflows/release-please.yml | 2 +- .github/workflows/release-to-pypi.yml | 10 ++++++---- .github/workflows/release-to-test-pypi.yml | 2 +- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 617c859..01b8510 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,9 +134,11 @@ jobs: else needs_python=false fi - echo "run_suite=$run_suite" >> "$GITHUB_OUTPUT" - echo "run_container=$run_container" >> "$GITHUB_OUTPUT" - echo "needs_python=$needs_python" >> "$GITHUB_OUTPUT" + { + echo "run_suite=$run_suite" + echo "run_container=$run_container" + echo "needs_python=$needs_python" + } >> "$GITHUB_OUTPUT" validate: if: needs.scope.outputs.needs_python == 'true' @@ -288,7 +290,7 @@ jobs: docker exec vidxp-ci sh -c \ 'test ! -d "$HOME/.cache/huggingface" && test ! -d "$HOME/.cache/clip"' - for attempt in {1..30}; do + for _ in {1..30}; do if curl --fail --silent http://127.0.0.1:8501/_stcore/health; then exit 0 fi diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 4bccadb..1316857 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -149,7 +149,7 @@ jobs: docker exec vidxp-release sh -c \ 'test ! -d "$HOME/.cache/huggingface" && test ! -d "$HOME/.cache/clip"' - for attempt in {1..30}; do + for _ in {1..30}; do if curl --fail --silent http://127.0.0.1:8501/_stcore/health; then break fi @@ -183,9 +183,11 @@ jobs: docker push "$base" docker push "${base}-control" docker push "${base}-worker" - echo "product=$(digest "$base")" >> "$GITHUB_OUTPUT" - echo "control=$(digest "${base}-control")" >> "$GITHUB_OUTPUT" - echo "worker=$(digest "${base}-worker")" >> "$GITHUB_OUTPUT" + { + echo "product=$(digest "$base")" + echo "control=$(digest "${base}-control")" + echo "worker=$(digest "${base}-worker")" + } >> "$GITHUB_OUTPUT" promote: if: inputs.operation == 'promote' diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 2e295ea..a3dff19 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -57,7 +57,7 @@ jobs: fi fi - for attempt in {1..45}; do + for _ in {1..45}; do pr="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$number")" head="$(jq -r .head.sha <<< "$pr")" base="$(jq -r .base.sha <<< "$pr")" diff --git a/.github/workflows/release-to-pypi.yml b/.github/workflows/release-to-pypi.yml index 5074130..5ea62e7 100644 --- a/.github/workflows/release-to-pypi.yml +++ b/.github/workflows/release-to-pypi.yml @@ -104,9 +104,11 @@ jobs: [[ "$candidate_tree" == "$release_tree" ]] [[ "$recorded_tree" == "$release_tree" ]] - echo "product_digest=$(jq -r .containers.product "$candidate")" >> "$GITHUB_OUTPUT" - echo "control_digest=$(jq -r .containers.control "$candidate")" >> "$GITHUB_OUTPUT" - echo "worker_digest=$(jq -r .containers.worker "$candidate")" >> "$GITHUB_OUTPUT" + { + echo "product_digest=$(jq -r .containers.product "$candidate")" + echo "control_digest=$(jq -r .containers.control "$candidate")" + echo "worker_digest=$(jq -r .containers.worker "$candidate")" + } >> "$GITHUB_OUTPUT" - name: Prepare release assets and checksums id: source @@ -121,7 +123,7 @@ jobs: [[ "$(find release-assets -maxdepth 1 -name '*.exe' | wc -l)" == 1 ]] [[ "$(find release-assets -maxdepth 1 -name '*.dmg' | wc -l)" == 1 ]] [[ "$(find release-assets -maxdepth 1 -name '*.AppImage' | wc -l)" == 1 ]] - (cd release-assets && sha256sum * > SHA256SUMS) + (cd release-assets && sha256sum -- ./* > SHA256SUMS) echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@v7 diff --git a/.github/workflows/release-to-test-pypi.yml b/.github/workflows/release-to-test-pypi.yml index a1cf2d7..2cc019e 100644 --- a/.github/workflows/release-to-test-pypi.yml +++ b/.github/workflows/release-to-test-pypi.yml @@ -103,7 +103,7 @@ jobs: - name: Download and install the exact TestPyPI artifact shell: bash run: | - for attempt in {1..30}; do + for _ in {1..30}; do state="$( python utils/verify_published_distribution.py \ --repository https://test.pypi.org \ From 174f4869f42bd1eedf75b530c286390d6f9875b8 Mon Sep 17 00:00:00 2001 From: Talha Date: Mon, 3 Aug 2026 15:53:41 +0500 Subject: [PATCH 4/8] refactor(ci): centralize validation contracts --- .github/workflows/ci.yml | 142 +----------------- .github/workflows/codeql.yml | 6 +- .github/workflows/containers.yml | 28 +--- .github/workflows/desktop.yml | 34 +---- .github/workflows/security.yml | 13 -- desktop/scripts/verify-windows-gui.ps1 | 14 ++ desktop/src/desktopPackagingContracts.test.ts | 45 ------ docs/CONTRIBUTING.md | 6 + tests/test_ci_scope.py | 40 +++++ tests/test_packaging.py | 71 +-------- utils/check_workflows.sh | 1 + utils/ci_scope.py | 72 +++++++++ utils/smoke_container.sh | 44 ++++++ utils/verify_runtime.py | 116 ++++++++++++++ 14 files changed, 311 insertions(+), 321 deletions(-) create mode 100644 desktop/scripts/verify-windows-gui.ps1 create mode 100644 tests/test_ci_scope.py create mode 100644 utils/ci_scope.py create mode 100644 utils/smoke_container.sh create mode 100644 utils/verify_runtime.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01b8510..fea1a08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,28 +117,9 @@ jobs: exit 0 fi - changed_files="$(git diff --name-only "$BASE_SHA" HEAD)" - if grep -Eq '^(\.github/workflows/ci\.yml$|src/|tests/|utils/|web/upload-page/|LICENSE$|MANIFEST\.in$|pyproject\.toml$|setup\.py$|uv\.lock$)' <<< "$changed_files"; then - run_suite=true - else - run_suite=false - fi - if grep -Eq '^(\.dockerignore$|\.github/workflows/ci\.yml$|Dockerfile$|compose(\.coolify)?\.yaml$|pyproject\.toml$|uv\.lock$|src/vidxp/(requirements/.*\.txt|capabilities/[^/]+/requirements\.txt)$)' <<< "$changed_files"; then - run_container=true - else - run_container=false - fi - - if [[ "$run_suite" == "true" || "$run_container" == "true" ]]; then - needs_python=true - else - needs_python=false - fi - { - echo "run_suite=$run_suite" - echo "run_container=$run_container" - echo "needs_python=$needs_python" - } >> "$GITHUB_OUTPUT" + python utils/ci_scope.py \ + --base "$BASE_SHA" \ + --github-output "$GITHUB_OUTPUT" validate: if: needs.scope.outputs.needs_python == 'true' @@ -221,37 +202,8 @@ jobs: .wheel-smoke/bin/python -m pip install --upgrade pip .wheel-smoke/bin/python -m pip install dist/*.whl .wheel-smoke/bin/python -m pip check - .wheel-smoke/bin/python - <<'PY' - from importlib.util import find_spec - from subprocess import check_output - - import vidxp - from vidxp.capabilities.registry import create_capability_registry - - assert create_capability_registry().names() == ( - "dialogue", "scene", "actor" - ) - help_text = check_output( - [".wheel-smoke/bin/vidxp", "--help"], - text=True, - ) - assert "benchmark" in help_text - benchmark_help = check_output( - [".wheel-smoke/bin/vidxp", "benchmark", "--help"], - text=True, - ) - assert "official benchmark adapters" in benchmark_help - for module in ( - "chromadb", - "cv2", - "faster_whisper", - "sentence_transformers", - "streamlit", - "torch", - "transformers", - ): - assert find_spec(module) is None, module - PY + .wheel-smoke/bin/python utils/verify_runtime.py minimal \ + --vidxp-executable .wheel-smoke/bin/vidxp - name: Preserve the validated distribution if: >- @@ -275,33 +227,7 @@ jobs: - name: Smoke container if: needs.scope.outputs.run_container == 'true' && matrix.python-version == '3.14' - shell: bash - run: | - docker run --detach --name vidxp-ci \ - --publish 127.0.0.1:8501:8501 vidxp:ci - trap 'docker rm --force vidxp-ci >/dev/null 2>&1 || true' EXIT - - docker exec vidxp-ci vidxp --version - docker exec vidxp-ci vidxp init --json - docker exec vidxp-ci python -c \ - 'import chromadb, cv2, faster_whisper, huggingface_hub, pooch, psutil, sentence_transformers, torch, transformers; assert psutil.virtual_memory().available > 0' - docker exec vidxp-ci python -c \ - 'import importlib.metadata as m; assert not any(d.metadata["Name"].lower().startswith("nvidia-") for d in m.distributions())' - docker exec vidxp-ci sh -c \ - 'test ! -d "$HOME/.cache/huggingface" && test ! -d "$HOME/.cache/clip"' - - for _ in {1..30}; do - if curl --fail --silent http://127.0.0.1:8501/_stcore/health; then - exit 0 - fi - if [[ "$(docker inspect --format='{{.State.Running}}' vidxp-ci)" != "true" ]]; then - break - fi - sleep 2 - done - - docker logs vidxp-ci - exit 1 + run: bash utils/smoke_container.sh vidxp:ci provider-platform-smoke: if: needs.scope.outputs.run_suite == 'true' @@ -339,58 +265,4 @@ jobs: VIDXP_ALLOW_MODEL_DOWNLOADS: "false" - name: Verify platform and CPU runtime - shell: bash - run: | - uv run --no-sync python - <<'PY' - from importlib import metadata - import platform - import chromadb - import cv2 - import faster_whisper - import huggingface_hub - import pooch - import psutil - import sentence_transformers - import torch - import transformers - - from vidxp.runtime import resolve_backends - - assert hasattr(cv2, "FaceDetectorYN") - assert hasattr(cv2, "FaceRecognizerSF") - assert hasattr(faster_whisper, "WhisperModel") - assert hasattr(faster_whisper, "BatchedInferencePipeline") - assert hasattr( - sentence_transformers.SentenceTransformer, - "encode_query", - ) - assert hasattr( - sentence_transformers.SentenceTransformer, - "encode_document", - ) - assert hasattr(transformers, "AutoModel") - assert hasattr(transformers, "AutoProcessor") - assert hasattr(chromadb, "PersistentClient") - assert hasattr(huggingface_hub, "snapshot_download") - assert hasattr(pooch, "retrieve") - assert psutil.virtual_memory().available > 0 - assert torch.version.cuda is None, torch.version.cuda - installed = { - distribution.metadata["Name"].lower().replace("_", "-") - for distribution in metadata.distributions() - if distribution.metadata.get("Name") - } - leaks = sorted( - name - for name in installed - if name in {"cuda-toolkit", "cuda-bindings", "triton"} - or name.startswith("nvidia-") - or name.startswith("pytorch-triton") - ) - assert not leaks, leaks - - profile = resolve_backends("cpu") - assert profile.torch_device == "cpu" - if platform.system() == "Darwin": - assert platform.machine() == "arm64" - PY + run: uv run --no-sync python utils/verify_runtime.py cpu diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8eb1ef8..6bdee2f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -6,15 +6,13 @@ on: - main paths: - ".github/workflows/codeql.yml" - - "src/**/*.py" - - "utils/**/*.py" + - "**/*.py" push: branches: - main paths: - ".github/workflows/codeql.yml" - - "src/**/*.py" - - "utils/**/*.py" + - "**/*.py" schedule: - cron: "17 4 * * 6" diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 1316857..d857091 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -136,33 +136,7 @@ jobs: product="ghcr.io/${{ github.repository }}:candidate-${{ inputs.candidate_id }}" control="${product}-control" worker="${product}-worker" - docker run --detach --name vidxp-release \ - --publish 127.0.0.1:8501:8501 "$product" - trap 'docker rm --force vidxp-release >/dev/null 2>&1 || true' EXIT - - docker exec vidxp-release vidxp --version - docker exec vidxp-release vidxp init --json - docker exec vidxp-release python -c \ - 'import chromadb, cv2, faster_whisper, huggingface_hub, pooch, psutil, sentence_transformers, torch, transformers; assert psutil.virtual_memory().available > 0' - docker exec vidxp-release python -c \ - 'import importlib.metadata as m; assert not any(d.metadata["Name"].lower().startswith("nvidia-") for d in m.distributions())' - docker exec vidxp-release sh -c \ - 'test ! -d "$HOME/.cache/huggingface" && test ! -d "$HOME/.cache/clip"' - - for _ in {1..30}; do - if curl --fail --silent http://127.0.0.1:8501/_stcore/health; then - break - fi - if [[ "$(docker inspect --format='{{.State.Running}}' vidxp-release)" != "true" ]]; then - docker logs vidxp-release - exit 1 - fi - sleep 2 - done - curl --fail --silent http://127.0.0.1:8501/_stcore/health - docker run --rm "$control" vidxp-api --help - docker run --rm "$control" vidxp-mcp --help - docker run --rm "$worker" vidxp init --json + bash utils/smoke_container.sh "$product" "$control" "$worker" - name: Log in to GitHub Container Registry uses: docker/login-action@v4 diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index b5464a3..4cb26d1 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -30,18 +30,13 @@ on: - release paths: - "desktop/**" - - "src/vidxp/requirements/**" - - "src/vidxp/capabilities/*/requirements.txt" - - "src/vidxp/local_probe.py" - - "src/vidxp/frontend.py" - - "src/vidxp/cli.py" - - "src/vidxp/cli_commands/probe.py" - - "tests/test_local_probe.py" - - "tests/test_cli.py" - - "tests/test_frontend.py" - - "tests/test_frontend_app.py" - - "tests/test_packaging.py" + - "src/**" + - "tests/**" + - "utils/**" + - "LICENSE" + - "MANIFEST.in" - "pyproject.toml" + - "setup.py" - "uv.lock" - ".github/workflows/desktop.yml" @@ -162,22 +157,7 @@ jobs: - name: Verify the Windows app uses the GUI subsystem if: runner.os == 'Windows' - shell: pwsh - run: | - $path = "desktop/src-tauri/target/release/vidxp-desktop.exe" - $stream = [System.IO.File]::OpenRead($path) - try { - $reader = [System.IO.BinaryReader]::new($stream) - $stream.Position = 0x3c - $peOffset = $reader.ReadInt32() - $stream.Position = $peOffset + 4 + 20 + 68 - $subsystem = $reader.ReadUInt16() - if ($subsystem -ne 2) { - throw "Expected Windows GUI subsystem 2, found $subsystem." - } - } finally { - $stream.Dispose() - } + run: powershell -ExecutionPolicy Bypass -File desktop/scripts/verify-windows-gui.ps1 - name: Preserve the desktop installer uses: actions/upload-artifact@v7 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index b3a2765..bf519e0 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -5,19 +5,6 @@ on: branches: - main - release - paths: - - ".github/dependabot.yml" - - ".github/workflows/**" - - "pyproject.toml" - - "uv.lock" - - "src/**/*.txt" - - "utils/build-requirements.txt" - - "desktop/package.json" - - "desktop/package-lock.json" - - "desktop/runtime-constraints.txt" - - "desktop/sidecars.json" - - "desktop/src-tauri/Cargo.toml" - - "desktop/src-tauri/Cargo.lock" concurrency: group: security-${{ github.event.pull_request.number || github.ref }} diff --git a/desktop/scripts/verify-windows-gui.ps1 b/desktop/scripts/verify-windows-gui.ps1 new file mode 100644 index 0000000..524724e --- /dev/null +++ b/desktop/scripts/verify-windows-gui.ps1 @@ -0,0 +1,14 @@ +$path = "desktop/src-tauri/target/release/vidxp-desktop.exe" +$stream = [System.IO.File]::OpenRead($path) +try { + $reader = [System.IO.BinaryReader]::new($stream) + $stream.Position = 0x3c + $peOffset = $reader.ReadInt32() + $stream.Position = $peOffset + 4 + 20 + 68 + $subsystem = $reader.ReadUInt16() + if ($subsystem -ne 2) { + throw "Expected Windows GUI subsystem 2, found $subsystem." + } +} finally { + $stream.Dispose() +} diff --git a/desktop/src/desktopPackagingContracts.test.ts b/desktop/src/desktopPackagingContracts.test.ts index 6fffa03..cdbb9b7 100644 --- a/desktop/src/desktopPackagingContracts.test.ts +++ b/desktop/src/desktopPackagingContracts.test.ts @@ -31,49 +31,4 @@ describe('Desktop packaging and documentation contracts', () => { expect(windows.app.windows[0]).toMatchObject({ decorations: false, shadow: false }); }); - it('exposes deterministic write and check commands for the model catalog', () => { - const packageJson = JSON.parse(read('desktop/package.json')); - expect(packageJson.scripts['model-catalog:write']).toBe( - 'uv run --frozen python scripts/model-catalog.py --write', - ); - expect(packageJson.scripts['model-catalog:check']).toBe( - 'uv run --frozen python scripts/model-catalog.py --check', - ); - }); - - it('keeps public contributor commands and target ownership guidance accurate', () => { - const contributing = read('docs/CONTRIBUTING.md'); - expect(contributing).toContain( - 'cargo install cargo-about --version 0.9.1 --locked --features cli', - ); - expect(contributing).toContain('npm --prefix desktop run notices:write'); - expect(contributing).toContain('npm --prefix desktop run model-catalog:write'); - - const installation = read('INSTALLATION_GUIDE.md'); - expect(installation).toContain('Use an existing installation'); - expect(installation).toContain('the installation stays externally owned'); - expect(installation).toContain('Prepare / verify models'); - expect(installation).toContain('Open VidXP'); - }); - - it('keeps pull-request desktop validation on every advertised platform and probe trigger', () => { - const workflow = read('.github/workflows/desktop.yml'); - expect(workflow).toContain('target: [windows, macos, linux]'); - for (const path of [ - 'src/vidxp/local_probe.py', - 'src/vidxp/frontend.py', - 'src/vidxp/cli.py', - 'src/vidxp/cli_commands/probe.py', - 'tests/test_local_probe.py', - 'tests/test_packaging.py', - ]) { - expect(workflow).toContain(`- "${path}"`); - } - expect(read('.github/workflows/release-candidate.yml')).toContain( - 'uses: ./.github/workflows/desktop.yml', - ); - expect(read('.github/workflows/release-to-pypi.yml')).toContain( - 'vidxp-desktop-windows-x86_64', - ); - }); }); diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 8cdb5d4..9a0c23f 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -149,6 +149,12 @@ npm --prefix desktop run check docker compose config --quiet ``` +Pull-request CI treats Markdown and the documentation directory as +documentation-only. Tests and Desktop-only changes run the code suite without +building a container; product, tooling, workflow, and unknown new paths default +to the code suite plus container validation. Desktop and CodeQL triggers use +directory or language globs rather than enumerating individual source files. + Desktop validation requires the pinned uv sidecar before Rust tests: ```bash diff --git a/tests/test_ci_scope.py b/tests/test_ci_scope.py new file mode 100644 index 0000000..cb86c17 --- /dev/null +++ b/tests/test_ci_scope.py @@ -0,0 +1,40 @@ +import unittest + +from utils.ci_scope import Scope, classify + + +class CiScopeTests(unittest.TestCase): + def test_documentation_only_changes_skip_code_validation(self): + self.assertEqual( + classify(["README.md", "docs/releasing.md"]), + Scope(run_suite=False, run_container=False), + ) + + def test_tests_and_desktop_changes_skip_container_builds(self): + self.assertEqual( + classify(["tests/test_new_feature.py", "desktop/src/App.tsx"]), + Scope(run_suite=True, run_container=False), + ) + + def test_product_and_workflow_changes_validate_containers(self): + for path in ( + "src/vidxp/new_feature.py", + "web/upload-page/src/app.js", + "utils/build_package.sh", + ".github/workflows/future.yml", + ): + with self.subTest(path=path): + self.assertEqual( + classify([path]), + Scope(run_suite=True, run_container=True), + ) + + def test_unknown_new_roots_default_to_full_validation(self): + self.assertEqual( + classify(["future-product/component.rs"]), + Scope(run_suite=True, run_container=True), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_packaging.py b/tests/test_packaging.py index ecbc283..49ad79d 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -581,7 +581,7 @@ def test_desktop_manifest_matches_published_package_contract(self): "system", ) - def test_combined_release_contract_and_workflow_boundaries(self): + def test_combined_release_version_contract(self): expected_extra_files = { "desktop/src-tauri/Cargo.toml", "desktop/src-tauri/Cargo.lock", @@ -689,75 +689,6 @@ def test_combined_release_contract_and_workflow_boundaries(self): ).read_text(encoding="utf-8"), ) - ci = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") - nightly = ( - ROOT / ".github/workflows/release-to-test-pypi.yml" - ).read_text(encoding="utf-8") - publisher = ( - ROOT / ".github/workflows/release-to-pypi.yml" - ).read_text(encoding="utf-8") - self.assertIn("bash utils/build_package.sh", ci) - self.assertIn("bash utils/build_package.sh", nightly) - self.assertNotIn("bash utils/build_package.sh", publisher) - self.assertIn("candidate_run_id", publisher) - self.assertIn("source_tree", publisher) - self.assertIn("operation: promote", publisher) - self.assertIn("utils/render_release_notes.py", publisher) - self.assertIn("--notes-file release-notes.md", publisher) - self.assertTrue((ROOT / ".github/release-intro.md").is_file()) - - release_workflow = ( - ROOT / ".github" / "workflows" / "release-please.yml" - ).read_text(encoding="utf-8") - self.assertIn("gh workflow run release-candidate.yml", release_workflow) - self.assertIn("gh workflow run release-to-pypi.yml", release_workflow) - self.assertIn('--ref "$TARGET_BRANCH"', release_workflow) - self.assertIn('--ref "$TAG"', release_workflow) - self.assertFalse( - (ROOT / ".github/workflows/publish-desktop.yml").exists() - ) - gate = ( - ROOT / ".github/workflows/release-gate.yml" - ).read_text(encoding="utf-8") - self.assertIn("pull_request_target:", gate) - self.assertIn("context=release/candidate", gate) - - for workflow in ("ci.yml", "desktop.yml", "security.yml"): - contents = ( - ROOT / ".github" / "workflows" / workflow - ).read_text(encoding="utf-8") - self.assertIn(" - release", contents, workflow) - self.assertIn("github.base_ref != 'release'", contents, workflow) - self.assertIn("github.head_ref != 'main'", contents, workflow) - desktop_ci = ( - ROOT / ".github" / "workflows" / "desktop.yml" - ).read_text(encoding="utf-8") - self.assertIn("target: [windows, macos, linux]", desktop_ci) - self.assertNotIn( - "!startsWith(github.head_ref, 'release-please--branches--')", - desktop_ci, - ) - - promotion = ( - ROOT / ".github" / "workflows" / "promotion-pr.yml" - ).read_text(encoding="utf-8") - self.assertIn('"$status" == "diverged"', promotion) - synchronization = ( - ROOT / ".github" / "workflows" / "sync-channels.yml" - ).read_text(encoding="utf-8") - self.assertIn( - 'git push origin "$publication_sha:refs/heads/main"', - synchronization, - ) - self.assertNotIn( - 'git push origin "origin/release:refs/heads/main"', - synchronization, - ) - self.assertIn( - '--title "chore(release): synchronize stable baseline"', - synchronization, - ) - def test_windows_release_binary_uses_the_gui_subsystem(self): main = ( ROOT / "desktop" / "src-tauri" / "src" / "main.rs" diff --git a/utils/check_workflows.sh b/utils/check_workflows.sh index 6f01aff..66cc4f6 100644 --- a/utils/check_workflows.sh +++ b/utils/check_workflows.sh @@ -13,3 +13,4 @@ curl --fail --location --silent --show-error \ echo "$checksum $temporary/$archive" | sha256sum --check --status tar -xzf "$temporary/$archive" -C "$temporary" actionlint "$temporary/actionlint" +shellcheck utils/*.sh desktop/scripts/*.sh diff --git a/utils/ci_scope.py b/utils/ci_scope.py new file mode 100644 index 0000000..834dd27 --- /dev/null +++ b/utils/ci_scope.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import argparse +import subprocess +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Scope: + run_suite: bool + run_container: bool + + +def _normalize(path: str) -> str: + normalized = path.strip().replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + return normalized + + +def _is_documentation(path: str) -> bool: + return path.startswith("docs/") or path.endswith(".md") + + +def _is_container_neutral(path: str) -> bool: + return path.startswith(("desktop/", "skills/", "tests/")) + + +def classify(changed_files: list[str] | tuple[str, ...]) -> Scope: + code_paths = [ + path + for value in changed_files + if (path := _normalize(value)) and not _is_documentation(path) + ] + return Scope( + run_suite=bool(code_paths), + run_container=any( + not _is_container_neutral(path) for path in code_paths + ), + ) + + +def changed_files(base: str, head: str) -> list[str]: + completed = subprocess.run( + ["git", "diff", "--name-only", base, head], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.splitlines() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Select stable CI scopes from changed repository paths." + ) + parser.add_argument("--base", required=True) + parser.add_argument("--head", default="HEAD") + parser.add_argument("--github-output", type=Path, required=True) + args = parser.parse_args() + + scope = classify(changed_files(args.base, args.head)) + needs_python = scope.run_suite or scope.run_container + with args.github_output.open("a", encoding="utf-8") as output: + output.write(f"run_suite={str(scope.run_suite).lower()}\n") + output.write(f"run_container={str(scope.run_container).lower()}\n") + output.write(f"needs_python={str(needs_python).lower()}\n") + + +if __name__ == "__main__": + main() diff --git a/utils/smoke_container.sh b/utils/smoke_container.sh new file mode 100644 index 0000000..f944137 --- /dev/null +++ b/utils/smoke_container.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 1 && "$#" -ne 3 ]]; then + echo "usage: $0 PRODUCT_IMAGE [CONTROL_IMAGE WORKER_IMAGE]" >&2 + exit 2 +fi + +product="$1" +control="${2:-}" +worker="${3:-}" +container="vidxp-smoke-${GITHUB_RUN_ID:-local}" +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +cleanup() { + docker rm --force "$container" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker run --detach --name "$container" \ + --publish 127.0.0.1:8501:8501 "$product" +docker exec "$container" vidxp --version +docker exec "$container" vidxp init --json +docker exec -i "$container" python - cpu < "$root/utils/verify_runtime.py" +docker exec "$container" sh -c \ + 'test ! -d "$HOME/.cache/huggingface" && test ! -d "$HOME/.cache/clip"' + +for _ in {1..30}; do + if curl --fail --silent http://127.0.0.1:8501/_stcore/health; then + break + fi + if [[ "$(docker inspect --format='{{.State.Running}}' "$container")" != "true" ]]; then + docker logs "$container" + exit 1 + fi + sleep 2 +done +curl --fail --silent http://127.0.0.1:8501/_stcore/health + +if [[ -n "$control" ]]; then + docker run --rm "$control" vidxp-api --help + docker run --rm "$control" vidxp-mcp --help + docker run --rm "$worker" vidxp init --json +fi diff --git a/utils/verify_runtime.py b/utils/verify_runtime.py new file mode 100644 index 0000000..ec20bd0 --- /dev/null +++ b/utils/verify_runtime.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import argparse +import platform +import subprocess +from importlib import metadata +from importlib.util import find_spec + + +def require(condition: bool, message: str) -> None: + if not condition: + raise RuntimeError(message) + + +def verify_minimal(executable: str) -> None: + from vidxp.capabilities.registry import create_capability_registry + + require( + create_capability_registry().names() == ("dialogue", "scene", "actor"), + "minimal wheel does not expose the expected capability registry", + ) + help_text = subprocess.check_output([executable, "--help"], text=True) + require("benchmark" in help_text, "minimal CLI does not expose benchmarks") + benchmark_help = subprocess.check_output( + [executable, "benchmark", "--help"], text=True + ) + require( + "official benchmark adapters" in benchmark_help, + "minimal CLI benchmark help is incomplete", + ) + unexpected = [ + module + for module in ( + "chromadb", + "cv2", + "faster_whisper", + "sentence_transformers", + "streamlit", + "torch", + "transformers", + ) + if find_spec(module) is not None + ] + require(not unexpected, f"minimal wheel installed optional modules: {unexpected}") + + +def verify_cpu() -> None: + import chromadb + import cv2 + import faster_whisper + import huggingface_hub + import pooch + import psutil + import sentence_transformers + import torch + import transformers + + from vidxp.runtime import resolve_backends + + required_attributes = ( + (cv2, "FaceDetectorYN"), + (cv2, "FaceRecognizerSF"), + (faster_whisper, "WhisperModel"), + (faster_whisper, "BatchedInferencePipeline"), + (sentence_transformers.SentenceTransformer, "encode_query"), + (sentence_transformers.SentenceTransformer, "encode_document"), + (transformers, "AutoModel"), + (transformers, "AutoProcessor"), + (chromadb, "PersistentClient"), + (huggingface_hub, "snapshot_download"), + (pooch, "retrieve"), + ) + missing = [ + f"{owner.__name__}.{attribute}" + for owner, attribute in required_attributes + if not hasattr(owner, attribute) + ] + require(not missing, f"CPU runtime APIs are missing: {missing}") + require(psutil.virtual_memory().available > 0, "memory probe returned no capacity") + require(torch.version.cuda is None, f"CPU torch exposes CUDA {torch.version.cuda}") + + installed = { + distribution.metadata["Name"].lower().replace("_", "-") + for distribution in metadata.distributions() + if distribution.metadata.get("Name") + } + leaks = sorted( + name + for name in installed + if name in {"cuda-toolkit", "cuda-bindings", "triton"} + or name.startswith(("nvidia-", "pytorch-triton")) + ) + require(not leaks, f"CPU runtime contains GPU packages: {leaks}") + + profile = resolve_backends("cpu") + require(profile.torch_device == "cpu", "CPU profile selected another device") + if platform.system() == "Darwin": + require(platform.machine() == "arm64", "macOS runtime is not Apple Silicon") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Verify installed VidXP runtimes.") + parser.add_argument("profile", choices=("minimal", "cpu")) + parser.add_argument("--vidxp-executable") + args = parser.parse_args() + + if args.profile == "minimal": + if not args.vidxp_executable: + parser.error("minimal verification requires --vidxp-executable") + verify_minimal(args.vidxp_executable) + else: + verify_cpu() + + +if __name__ == "__main__": + main() From fdd66b730237e1cfd687ab00c3d8b78d51275299 Mon Sep 17 00:00:00 2001 From: Talha Date: Mon, 3 Aug 2026 15:58:47 +0500 Subject: [PATCH 5/8] fix(ci): enforce current Python lockfile --- .github/workflows/ci.yml | 5 ++++- uv.lock | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fea1a08..2a65027 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,12 +149,15 @@ jobs: - name: Install repository tooling run: | python -m pip install --upgrade pip + python -m pip install "uv==0.12.0" python -m pip install -r utils/build-requirements.txt + - name: Verify Python lockfile + run: uv lock --check + - name: Install test surface if: needs.scope.outputs.run_suite == 'true' run: | - python -m pip install "uv==0.12.0" uv sync \ --frozen \ --extra all \ diff --git a/uv.lock b/uv.lock index 4c2701f..d7bee06 100644 --- a/uv.lock +++ b/uv.lock @@ -4126,7 +4126,7 @@ wheels = [ [[package]] name = "vidxp" -version = "0.2.1b1" +version = "0.4.0b0" source = { editable = "." } dependencies = [ { name = "dbos" }, From 99da21daaca24b385d2f8fb0c460b50f72378cde Mon Sep 17 00:00:00 2001 From: Talha Date: Mon, 3 Aug 2026 16:01:42 +0500 Subject: [PATCH 6/8] fix(release): synchronize Python lock version --- release-please-config.json | 4 ++++ release-please-config.stable.json | 4 ++++ tests/test_packaging.py | 7 +++++++ uv.lock | 2 +- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/release-please-config.json b/release-please-config.json index ca8369f..141c650 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -12,6 +12,10 @@ "packages": { ".": { "extra-files": [ + { + "path": "uv.lock", + "type": "generic" + }, { "path": "desktop/src-tauri/Cargo.toml", "type": "generic" diff --git a/release-please-config.stable.json b/release-please-config.stable.json index 34c8128..c919200 100644 --- a/release-please-config.stable.json +++ b/release-please-config.stable.json @@ -15,6 +15,10 @@ "path": ".release-please-manifest.json", "type": "json" }, + { + "path": "uv.lock", + "type": "generic" + }, { "path": "desktop/src-tauri/Cargo.toml", "type": "generic" diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 49ad79d..7e35d8b 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -583,6 +583,7 @@ def test_desktop_manifest_matches_published_package_contract(self): def test_combined_release_version_contract(self): expected_extra_files = { + "uv.lock", "desktop/src-tauri/Cargo.toml", "desktop/src-tauri/Cargo.lock", "desktop/package.json", @@ -618,6 +619,7 @@ def test_combined_release_version_contract(self): self.assertEqual( generic_files, { + "uv.lock", "desktop/src-tauri/Cargo.toml", "desktop/src-tauri/Cargo.lock", }, @@ -682,6 +684,11 @@ def test_combined_release_version_contract(self): ).read_text(encoding="utf-8"), filename, ) + self.assertIn( + version_marker, + (ROOT / "uv.lock").read_text(encoding="utf-8"), + "uv.lock", + ) self.assertNotIn( f"vidxp=={version}", ( diff --git a/uv.lock b/uv.lock index d7bee06..cd0a125 100644 --- a/uv.lock +++ b/uv.lock @@ -4126,7 +4126,7 @@ wheels = [ [[package]] name = "vidxp" -version = "0.4.0b0" +version = "0.4.0-b" # x-release-please-version source = { editable = "." } dependencies = [ { name = "dbos" }, From 133a5edffd1098214dc3f86f047a092f5347cb0f Mon Sep 17 00:00:00 2001 From: Talha Date: Mon, 3 Aug 2026 16:29:49 +0500 Subject: [PATCH 7/8] fix(release): make Cargo lock updates regeneration-safe --- desktop/src-tauri/Cargo.lock | 2 +- release-please-config.json | 3 ++- release-please-config.stable.json | 3 ++- tests/test_packaging.py | 41 ++++++++++++++++++++++++------- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index c88fea6..66c6e71 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -4425,7 +4425,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vidxp-desktop" -version = "0.4.0-b" # x-release-please-version +version = "0.4.0-b" dependencies = [ "atomic-write-file", "hex", diff --git a/release-please-config.json b/release-please-config.json index 141c650..0eba41b 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -21,8 +21,9 @@ "type": "generic" }, { + "jsonpath": "$.package[?(@.name.value==\"vidxp-desktop\")].version", "path": "desktop/src-tauri/Cargo.lock", - "type": "generic" + "type": "toml" }, { "jsonpath": "$.version", diff --git a/release-please-config.stable.json b/release-please-config.stable.json index c919200..3c95045 100644 --- a/release-please-config.stable.json +++ b/release-please-config.stable.json @@ -24,8 +24,9 @@ "type": "generic" }, { + "jsonpath": "$.package[?(@.name.value==\"vidxp-desktop\")].version", "path": "desktop/src-tauri/Cargo.lock", - "type": "generic" + "type": "toml" }, { "jsonpath": "$.version", diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 7e35d8b..80f0b0d 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -621,7 +621,20 @@ def test_combined_release_version_contract(self): { "uv.lock", "desktop/src-tauri/Cargo.toml", - "desktop/src-tauri/Cargo.lock", + }, + filename, + ) + toml_files = { + extra["path"]: extra["jsonpath"] + for extra in root_package["extra-files"] + if extra["type"] == "toml" + } + self.assertEqual( + toml_files, + { + "desktop/src-tauri/Cargo.lock": ( + '$.package[?(@.name.value=="vidxp-desktop")].version' + ) }, filename, ) @@ -676,14 +689,24 @@ def test_combined_release_version_contract(self): version_marker = ( f'version = "{version}" # x-release-please-version' ) - for filename in ("Cargo.toml", "Cargo.lock"): - self.assertIn( - version_marker, - ( - ROOT / "desktop" / "src-tauri" / filename - ).read_text(encoding="utf-8"), - filename, - ) + self.assertIn( + version_marker, + ( + ROOT / "desktop" / "src-tauri" / "Cargo.toml" + ).read_text(encoding="utf-8"), + "Cargo.toml", + ) + cargo_lock = tomllib.loads( + ( + ROOT / "desktop" / "src-tauri" / "Cargo.lock" + ).read_text(encoding="utf-8") + ) + desktop_lock = next( + package + for package in cargo_lock["package"] + if package["name"] == "vidxp-desktop" + ) + self.assertEqual(desktop_lock["version"], version) self.assertIn( version_marker, (ROOT / "uv.lock").read_text(encoding="utf-8"), From d4782c0b002bb935978f8945e7e19c172399c476 Mon Sep 17 00:00:00 2001 From: Talha Date: Mon, 3 Aug 2026 16:29:49 +0500 Subject: [PATCH 8/8] fix(ci): add stable pull request validation gate --- .github/workflows/ci.yml | 58 ++++++++++++++++++------ .github/workflows/desktop.yml | 31 +++---------- docs/releasing.md | 19 ++++++-- tests/test_ci_scope.py | 79 +++++++++++++++++++++++++++++---- utils/ci_scope.py | 83 +++++++++++++++++++++++++++++++++-- 5 files changed, 216 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a65027..4d3a7b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,14 +82,11 @@ permissions: jobs: scope: - if: >- - github.event_name != 'pull_request' || - github.base_ref != 'release' || - github.head_ref != 'main' runs-on: ubuntu-latest outputs: needs_python: ${{ steps.scope.outputs.needs_python }} run_container: ${{ steps.scope.outputs.run_container }} + run_desktop: ${{ steps.scope.outputs.run_desktop }} run_suite: ${{ steps.scope.outputs.run_suite }} steps: - uses: actions/checkout@v7 @@ -105,21 +102,18 @@ jobs: shell: bash env: BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_REF: ${{ github.base_ref }} FORCE_VALIDATION: ${{ inputs.force_validation || false }} + HEAD_REF: ${{ github.head_ref }} RUN_CONTAINERS: ${{ inputs.run_containers || false }} run: | - if [[ "$FORCE_VALIDATION" == "true" ]]; then - { - echo "run_suite=true" - echo "run_container=$RUN_CONTAINERS" - echo "needs_python=true" - } >> "$GITHUB_OUTPUT" - exit 0 - fi - python utils/ci_scope.py \ --base "$BASE_SHA" \ - --github-output "$GITHUB_OUTPUT" + --base-ref "$BASE_REF" \ + --force-validation "$FORCE_VALIDATION" \ + --github-output "$GITHUB_OUTPUT" \ + --head-ref "$HEAD_REF" \ + --run-containers "$RUN_CONTAINERS" validate: if: needs.scope.outputs.needs_python == 'true' @@ -269,3 +263,39 @@ jobs: - name: Verify platform and CPU runtime run: uv run --no-sync python utils/verify_runtime.py cpu + + desktop: + if: >- + github.event_name == 'pull_request' && + needs.scope.outputs.run_desktop == 'true' + needs: scope + uses: ./.github/workflows/desktop.yml + with: + checkout_ref: ${{ github.sha }} + + required: + if: always() && github.event_name == 'pull_request' + needs: + - scope + - validate + - provider-platform-smoke + - desktop + name: validation/required + runs-on: ubuntu-latest + steps: + - name: Require every applicable validation job + env: + DESKTOP_RESULT: ${{ needs.desktop.result }} + PROVIDER_RESULT: ${{ needs.provider-platform-smoke.result }} + SCOPE_RESULT: ${{ needs.scope.result }} + VALIDATE_RESULT: ${{ needs.validate.result }} + shell: bash + run: | + [[ "$SCOPE_RESULT" == "success" ]] + for result in \ + "$DESKTOP_RESULT" \ + "$PROVIDER_RESULT" \ + "$VALIDATE_RESULT" + do + [[ "$result" == "success" || "$result" == "skipped" ]] + done diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 4cb26d1..554c343 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -24,22 +24,6 @@ on: required: false default: 14 type: number - pull_request: - branches: - - main - - release - paths: - - "desktop/**" - - "src/**" - - "tests/**" - - "utils/**" - - "LICENSE" - - "MANIFEST.in" - - "pyproject.toml" - - "setup.py" - - "uv.lock" - - ".github/workflows/desktop.yml" - permissions: contents: read @@ -49,10 +33,6 @@ concurrency: jobs: build: - if: >- - github.event_name != 'pull_request' || - github.base_ref != 'release' || - github.head_ref != 'main' name: ${{ matrix.target == 'windows' && 'Windows x86-64 NSIS' || matrix.target == 'macos' && 'macOS Apple Silicon DMG' || 'Linux x86-64 AppImage' }} runs-on: ${{ matrix.target == 'windows' && 'windows-2025' || matrix.target == 'macos' && 'macos-15' || 'ubuntu-24.04' }} strategy: @@ -89,21 +69,22 @@ jobs: version: "0.12.0" enable-cache: false - - uses: dtolnay/rust-toolchain@1.97.1 + - id: rust-toolchain + uses: dtolnay/rust-toolchain@1.97.1 - name: Install locked license tooling run: cargo install cargo-about --version 0.9.1 --locked --features cli - name: Restore Rust build cache - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: | ~/.cargo/registry ~/.cargo/git desktop/src-tauri/target - key: desktop-rust-${{ runner.os }}-${{ runner.arch }}-1.97.1-${{ hashFiles('desktop/src-tauri/Cargo.lock') }} + key: desktop-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('desktop/src-tauri/Cargo.lock') }} restore-keys: | - desktop-rust-${{ runner.os }}-${{ runner.arch }}-1.97.1- + desktop-rust-${{ runner.os }}-${{ runner.arch }}-${{ steps.rust-toolchain.outputs.cachekey }}- - name: Install locked desktop tooling run: npm ci @@ -138,7 +119,7 @@ jobs: - name: Restore the pinned uv sidecar id: uv-sidecar-cache - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: desktop/src-tauri/binaries key: desktop-uv-sidecar-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('desktop/sidecars.json') }} diff --git a/docs/releasing.md b/docs/releasing.md index 1ce79be..cddcddd 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -32,10 +32,21 @@ Merging a Release Please PR is allowed only when `release/candidate` succeeds. The lightweight **Release gate** marks that same context successful for ordinary PRs, so the rule does not add release builds to normal development changes. -Configure `release/candidate` as a required status on both channel branches. -Roll this out in order: merge the workflow changes to `main`, enable the rule on -`main`, promote those workflows to `release`, then enable the rule on `release`. -Do not enable the `release` rule before its base branch contains Release gate. +Require these GitHub Actions statuses on both channel branches: + +- `validation/required` aggregates the applicable Python, provider, container, + and Desktop checks. It reports without heavy builds for documentation-only + changes and Release Please PRs. +- `dependency-review` rejects newly introduced high-severity dependencies. +- `release/candidate` stays pending on Release Please PRs until the complete + retained candidate succeeds; ordinary PRs receive an immediate success. + +Require pull-request branches to be up to date before merging so the checked +tree cannot differ from the eventual merge. Roll the rules out in order: merge +the workflow changes to `main`, observe all three statuses on a new or refreshed +PR, enable them on `main`, promote the workflow baseline to `release`, and then +enable the same requirements there. Do not enable the `release` rules before +its base branch contains the validation and release-gate workflows. ## Publication after merge diff --git a/tests/test_ci_scope.py b/tests/test_ci_scope.py index cb86c17..54aa0ee 100644 --- a/tests/test_ci_scope.py +++ b/tests/test_ci_scope.py @@ -1,19 +1,23 @@ import unittest -from utils.ci_scope import Scope, classify +from utils.ci_scope import Scope, classify, select_scope class CiScopeTests(unittest.TestCase): def test_documentation_only_changes_skip_code_validation(self): self.assertEqual( classify(["README.md", "docs/releasing.md"]), - Scope(run_suite=False, run_container=False), + Scope( + run_suite=False, + run_container=False, + run_desktop=False, + ), ) def test_tests_and_desktop_changes_skip_container_builds(self): self.assertEqual( classify(["tests/test_new_feature.py", "desktop/src/App.tsx"]), - Scope(run_suite=True, run_container=False), + Scope(run_suite=True, run_container=False, run_desktop=True), ) def test_product_and_workflow_changes_validate_containers(self): @@ -24,15 +28,74 @@ def test_product_and_workflow_changes_validate_containers(self): ".github/workflows/future.yml", ): with self.subTest(path=path): - self.assertEqual( - classify([path]), - Scope(run_suite=True, run_container=True), - ) + scope = classify([path]) + self.assertTrue(scope.run_suite) + self.assertTrue(scope.run_container) + + def test_desktop_uses_stable_product_and_packaging_boundaries(self): + for path in ( + "desktop/src/App.tsx", + "src/vidxp/settings.py", + "tests/test_packaging.py", + "utils/build_package.sh", + "pyproject.toml", + "uv.lock", + ".github/workflows/desktop.yml", + ): + with self.subTest(path=path): + self.assertTrue(classify([path]).run_desktop) + + for path in ( + "README.md", + "docs/releasing.md", + "skills/vidxp-ingest-video/SKILL.md", + "web/upload-page/src/app.js", + ): + with self.subTest(path=path): + self.assertFalse(classify([path]).run_desktop) def test_unknown_new_roots_default_to_full_validation(self): self.assertEqual( classify(["future-product/component.rs"]), - Scope(run_suite=True, run_container=True), + Scope(run_suite=True, run_container=True, run_desktop=True), + ) + + def test_release_candidates_defer_to_the_candidate_build(self): + self.assertEqual( + select_scope( + ["pyproject.toml", "uv.lock"], + base_ref="main", + head_ref="release-please--branches--main", + ), + Scope( + run_suite=False, + run_container=False, + run_desktop=False, + ), + ) + + def test_main_to_release_sync_does_not_rebuild_validated_main(self): + self.assertEqual( + select_scope( + ["src/vidxp/app.py"], + base_ref="release", + head_ref="main", + ), + Scope( + run_suite=False, + run_container=False, + run_desktop=False, + ), + ) + + def test_forced_candidate_validation_ignores_pr_scope(self): + self.assertEqual( + select_scope( + [], + force_validation=True, + run_containers=True, + ), + Scope(run_suite=True, run_container=True, run_desktop=False), ) diff --git a/utils/ci_scope.py b/utils/ci_scope.py index 834dd27..68d5582 100644 --- a/utils/ci_scope.py +++ b/utils/ci_scope.py @@ -10,6 +10,7 @@ class Scope: run_suite: bool run_container: bool + run_desktop: bool def _normalize(path: str) -> str: @@ -27,6 +28,29 @@ def _is_container_neutral(path: str) -> bool: return path.startswith(("desktop/", "skills/", "tests/")) +def _affects_desktop(path: str) -> bool: + return path.startswith(("desktop/", "src/", "tests/", "utils/")) or path in { + ".github/workflows/ci.yml", + ".github/workflows/desktop.yml", + "LICENSE", + "MANIFEST.in", + "pyproject.toml", + "setup.py", + "uv.lock", + } + + +def _is_unknown_product_path(path: str) -> bool: + return not path.startswith( + (".github/", "desktop/", "skills/", "src/", "tests/", "utils/", "web/") + ) and path not in { + ".dockerignore", + "compose.coolify.yaml", + "compose.yaml", + "Dockerfile", + } + + def classify(changed_files: list[str] | tuple[str, ...]) -> Scope: code_paths = [ path @@ -35,12 +59,39 @@ def classify(changed_files: list[str] | tuple[str, ...]) -> Scope: ] return Scope( run_suite=bool(code_paths), - run_container=any( - not _is_container_neutral(path) for path in code_paths + run_container=any(not _is_container_neutral(path) for path in code_paths), + run_desktop=any( + _affects_desktop(path) or _is_unknown_product_path(path) + for path in code_paths ), ) +def select_scope( + changed_files: list[str] | tuple[str, ...], + *, + base_ref: str = "", + head_ref: str = "", + force_validation: bool = False, + run_containers: bool = False, +) -> Scope: + if force_validation: + return Scope( + run_suite=True, + run_container=run_containers, + run_desktop=False, + ) + if head_ref.startswith("release-please--branches--") or ( + base_ref == "release" and head_ref == "main" + ): + return Scope( + run_suite=False, + run_container=False, + run_desktop=False, + ) + return classify(changed_files) + + def changed_files(base: str, head: str) -> list[str]: completed = subprocess.run( ["git", "diff", "--name-only", base, head], @@ -56,15 +107,41 @@ def main() -> None: description="Select stable CI scopes from changed repository paths." ) parser.add_argument("--base", required=True) + parser.add_argument("--base-ref", default="") + parser.add_argument( + "--force-validation", + choices=("true", "false"), + default="false", + ) parser.add_argument("--head", default="HEAD") + parser.add_argument("--head-ref", default="") parser.add_argument("--github-output", type=Path, required=True) + parser.add_argument( + "--run-containers", + choices=("true", "false"), + default="false", + ) args = parser.parse_args() - scope = classify(changed_files(args.base, args.head)) + paths = ( + [] + if args.force_validation == "true" + or args.head_ref.startswith("release-please--branches--") + or (args.base_ref == "release" and args.head_ref == "main") + else changed_files(args.base, args.head) + ) + scope = select_scope( + paths, + base_ref=args.base_ref, + head_ref=args.head_ref, + force_validation=args.force_validation == "true", + run_containers=args.run_containers == "true", + ) needs_python = scope.run_suite or scope.run_container with args.github_output.open("a", encoding="utf-8") as output: output.write(f"run_suite={str(scope.run_suite).lower()}\n") output.write(f"run_container={str(scope.run_container).lower()}\n") + output.write(f"run_desktop={str(scope.run_desktop).lower()}\n") output.write(f"needs_python={str(needs_python).lower()}\n")