diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index e40adb4c54b0..b5ec95d13592 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -8,6 +8,10 @@ resources: type: git name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/canary + - repository: azure-sdk-build-tools + type: git + name: internal/azure-sdk-build-tools + ref: refs/tags/azure-sdk-build-tools_20260702.2 parameters: - name: stages diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 5357beb17f49..7f58eaa8880f 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -13,6 +13,24 @@ parameters: PackageSourceOverride: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" stages: + # Binary signing stages, emitted per artifact that opted in with `signBinaries: true`. + # + # INVARIANT: this compile-time gate must remain a superset of every gate that references a + # Sign_* stage, namely the release gate below and the Integration stage's dependsOn. If they + # drift so a Sign_* stage is referenced but not emitted, the pipeline fails to compile on an + # unresolvable dependsOn. "internal and not a PR build" is the loosest of the three, so it is + # repeated verbatim in all of them, and the narrower "which run reasons actually sign" logic + # lives in the stage's runtime condition instead of being duplicated here. + # + # Signing never runs on PR builds: the public project cannot reach internal/azure-sdk-build-tools, + # and ESRP must not be handed unreviewed code. + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - template: /eng/pipelines/templates/stages/sign-binaries.yml + parameters: + Artifact: ${{ artifact }} + DependsOn: ${{ parameters.DependsOn }} # Release stages are compiled for: # * internal manual runs (existing behavior), and # * internal post-merge CI on 'main' (auto-release). For auto-release, only the packages changed by a @@ -46,389 +64,40 @@ stages: EnableUvAuth: true - ${{ each artifact in parameters.Artifacts }}: - - stage: 'Release_${{artifact.safename}}' - displayName: 'Release: ${{artifact.name}}' - dependsOn: - - ${{ parameters.DependsOn }} - - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - - AutoReleasePrepare - variables: - - template: /eng/pipelines/templates/variables/image.yml - - template: /eng/common/pipelines/templates/variables/api-review-break-glass.yml - # Auto-release CI: only release when the shared prepare stage flagged this artifact as changed. - # Manual runs: every declared artifact remains eligible (existing behavior). - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - condition: and(succeeded(), eq(dependencies.AutoReleasePrepare.outputs['ResolveAutoReleasePackages.resolve.ReleaseArtifact_${{ artifact.safename }}'], 'true'), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + - template: /eng/pipelines/templates/stages/release-artifact.yml + parameters: + Artifact: ${{ artifact }} + # Packages that ship compiled binaries release from the artifact rebuilt by their + # Sign_ stage. Everything else keeps releasing from the build output, so a + # service can mix both kinds of packages freely. + ${{ if eq(artifact.signBinaries, true) }}: + ArtifactName: packages_${{ artifact.safeName }}_signed + DependsOn: Sign_${{ artifact.safeName }} ${{ else }}: - condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) - jobs: - - job: TagRepository - displayName: "Create release tag" - condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - template: /eng/common/pipelines/templates/steps/retain-run.yml - - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: "azure-template" - ServiceDirectory: "template" - TestPipeline: ${{ parameters.TestPipeline }} - - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: true - - - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml - parameters: - PackageName: ${{artifact.name}} - ServiceDirectory: ${{parameters.ServiceDirectory}} - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} - - - script: | - python -m pip install "./eng/tools/azure-sdk-tools" - displayName: Install tool dependencies - - - task: PythonScript@0 - displayName: Verify Dependency Presence - condition: and(succeeded(), ne(variables['Skip.VerifyDependencies'], 'true')) - inputs: - scriptPath: 'scripts/devops_tasks/verify_dependencies_present.py' - arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' - - - task: PythonScript@0 - displayName: Verify CI enabled - condition: succeeded() - inputs: - scriptPath: 'scripts/devops_tasks/verify_ci_enabled.py' - arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' - - - pwsh: | - Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - PackageRepository: PyPI - ReleaseSha: $(Build.SourceVersion) - RepoId: Azure/azure-sdk-for-python - WorkingDirectory: $(System.DefaultWorkingDirectory) - AuthToken: '' - - - ${{if ne(artifact.skipPublishPackage, 'true')}}: - - deployment: PublishPackage - displayName: "Publish to ${{ parameters.PublicFeed }}" - condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - # Auto-release runs after merge on main and must not wait on the package-publish approval gate. - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - environment: none - ${{ else }}: - environment: ${{ parameters.PublicPublishEnvironment }} - dependsOn: TagRepository - - templateContext: - type: releaseJob - isProduction: true - inputs: - - input: pipelineArtifact - artifactName: release_artifact - targetPath: $(Pipeline.Workspace)/release_artifact - - input: pipelineArtifact - artifactName: packages_extended - targetPath: $(Pipeline.Workspace)/packages_extended - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - strategy: - runOnce: - deploy: - steps: - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - EnableTwineAuth: false - EnablePipAuth: true - EnableUvAuth: false - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - - - script: | - python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt - displayName: Install Release Dependencies - - - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: - - pwsh: | - $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - New-Item -ItemType Directory -Force -Path $esrpDirectory - - Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}" ` - | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` - | Copy-Item -Destination $esrpDirectory - - Get-ChildItem $esrpDirectory - displayName: Isolate files for ESRP Publish - - - template: /eng/pipelines/templates/steps/esrp-publish.yml - parameters: - targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{artifact.name}}" - - - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: - - task: TwineAuthenticate@0 - displayName: 'Authenticate to feed: ${{parameters.PublicFeed}}' - inputs: - artifactFeeds: ${{parameters.PublicFeed}} - - - script: | - set -e - twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.whl - echo "Uploaded whl to devops feed" - twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tar.gz - echo "Uploaded sdist to devops feed" - displayName: 'Publish package to feed: ${{parameters.PublicFeed}}' - - - task: TwineAuthenticate@0 - displayName: 'Authenticate to feed: ${{parameters.DevFeedName}}' - inputs: - artifactFeeds: ${{parameters.DevFeedName}} - - - script: | - set -e - twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.whl - echo "Uploaded whl to devops feed" - twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tar.gz - echo "Uploaded sdist to devops feed" - displayName: 'Publish package to feed: ${{parameters.DevFeedName}}' - - - job: MarkPackageReleaseCompletion - displayName: "Mark package release completion" - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - EnableTwineAuth: false - EnablePipAuth: true - EnableUvAuth: false - - - template: /eng/common/pipelines/templates/steps/mark-release-completion.yml - parameters: - ConfigFileDir: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo' - PackageArtifactName: ${{artifact.name}} - - # Management-plane-only workaround: skip marking packages that do not generate an APIView revision. - - pwsh: | - $packageInfoPath = '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ artifact.name }}.json' - $packageInfo = Get-Content -Raw -Path $packageInfoPath | ConvertFrom-Json - $isManagementPackage = $packageInfo.SdkType -eq 'mgmt' - Write-Host "##vso[task.setvariable variable=IsManagementPackage]$isManagementPackage" - displayName: Check package SDK type - - - task: AzureCLI@2 - displayName: Mark Package Released - condition: >- - and( - succeeded(), - ne(variables['IsManagementPackage'], 'true'), - not( - and( - eq(variables['Skip.MarkPackageReleased'], 'true'), - eq(variables['IsRequesterAuthorizedToSkipApiReview'], 'true') - ) - ) - ) - inputs: - azureSubscription: "ADO to ARH Service Connection" - scriptType: pscore - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/scripts/Mark-PackageReleased.ps1 - arguments: > - -PackageInfoFiles @('$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ artifact.name }}.json') - -RepoOwner 'Azure' - -AzSdkExePath '$(AZSDK)' - workingDirectory: $(Pipeline.Workspace) - - - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - - job: PublishGitHubIODocs - displayName: Publish Docs to GitHubIO Blob Storage - condition: >- - and( - succeeded(), - ne(variables['Skip.PublishDocs'], 'true'), - ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') - ) - dependsOn: PublishPackage - - pool: - name: azsdk-pool - image: windows-2022 - os: windows - - steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - - - download: current - artifact: ${{parameters.DocArtifact}} - timeoutInMinutes: 5 - - - pwsh: | - if (Test-Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}") { - Get-ChildItem -Recurse "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" - } - else { - New-Item -ItemType Directory -Force -Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" - } - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - - template: /eng/common/pipelines/templates/steps/publish-blobs.yml - parameters: - FolderForUpload: '$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}' - TargetLanguage: 'python' - ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}' - - - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - job: PublishDocs - displayName: Docs.MS Release - condition: >- - and( - succeeded(), - ne(variables['Skip.PublishDocs'], 'true'), - ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') - ) - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/**/*.md - - .github/CODEOWNERS - - - download: current - - # py2docfx requires Python >= 3.12.x, match docs pipeline version specification - - task: UsePythonVersion@0 - displayName: 'Use Python 3.12.x' - inputs: - versionSpec: '3.12.x' - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml - parameters: - PackageInfoLocations: - - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - Language: 'python' - SparseCheckoutPaths: - - docs-ref-services/ - - metadata/ - - - job: UpdatePackageVersion - displayName: "Update Package Version" - condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - dependsOn: PublishPackage - - pool: - image: ubuntu-24.04 - name: azsdk-pool - os: linux - - steps: - - checkout: self - - task: UsePythonVersion@0 - - - template: /eng/pipelines/templates/steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - script: | - python -m pip install "./eng/tools/azure-sdk-tools" - displayName: Install versioning tool dependencies - - - pwsh: | - sdk_increment_version --package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }} - if (Test-Path component-detection-pip-report.json) { - Write-Host "Deleting component-detection-pip-report.json" - rm component-detection-pip-report.json - } - displayName: Increment package version - - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - RepoName: azure-sdk-for-python - PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) - CommitMsg: "Increment package version after release of ${{ artifact.name }}" - PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - AuthToken: '' - - - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - - template: /eng/pipelines/templates/jobs/smoke.tests.yml - parameters: - Daily: false - ArtifactName: ${{ parameters.ArtifactName }} - Artifact: ${{ artifact }} - DevFeedName: ${{ parameters.DevFeedName }} + ArtifactName: ${{ parameters.ArtifactName }} + DependsOn: ${{ parameters.DependsOn }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} + DocArtifact: ${{ parameters.DocArtifact }} + DevFeedName: ${{ parameters.DevFeedName }} + PublicFeed: ${{ parameters.PublicFeed }} + PublicPublishEnvironment: ${{ parameters.PublicPublishEnvironment }} + TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} + TargetDocRepoName: ${{ parameters.TargetDocRepoName }} - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - stage: Integration - dependsOn: ${{parameters.DependsOn}} + # Signing stages are dependencies purely for ordering, so their signed artifacts exist + # before this stage downloads them. The condition still keys off the build stage only, + # exactly as before, so a signing failure degrades to skipping that one package's alpha + # publish rather than taking down the dev feed publish for the whole service. + # The gate below must stay identical to the signing emission gate at the top of this file. + dependsOn: + - ${{ parameters.DependsOn }} + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - Sign_${{ artifact.safeName }} condition: succeededOrFailed('${{parameters.DependsOn}}') jobs: - job: PublishPackages @@ -443,6 +112,16 @@ stages: artifact: ${{parameters.ArtifactName}} timeoutInMinutes: 5 + # Signed artifacts are downloaded best effort. If a Sign_* stage failed or was + # skipped the artifact is absent, and only that package's alpha publish is skipped. + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{ if eq(artifact.signBinaries, true) }}: + - download: current + artifact: packages_${{ artifact.safeName }}_signed + timeoutInMinutes: 5 + continueOnError: true + - task: UsePythonVersion@0 - template: ../steps/auth-dev-feed.yml @@ -456,33 +135,16 @@ stages: - ${{ each artifact in parameters.Artifacts }}: - ${{if ne(artifact.skipPublishDevFeed, 'true')}}: - - - pwsh: | - # If BuildTargetingString is set, check whether this artifact matches any of the - # (possibly comma-separated) glob patterns before attempting to publish. - # This handles scoped builds where only a subset of packages are built. - $targetingString = $env:BUILDTARGETINGSTRING - if ($targetingString) { - $globs = $targetingString -split "," - $isTargeted = $globs | Where-Object { "${{artifact.name}}" -like $_.Trim() } - if (-not $isTargeted) { - Write-Host "Package '${{artifact.name}}' does not match BuildTargetingString '$targetingString'. Skipping integration publish." - exit 0 - } - } - - $fileCount = (Get-ChildItem $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} | ? {$_.Name -match "-[0-9]*.[0-9]*.[0-9]*a[0-9]*" } | Measure-Object).Count - - if ($fileCount -eq 0) { - Write-Host "No alpha packages for ${{artifact.name}} to publish." - exit 0 - } - - twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.whl - echo "Uploaded whl to devops feed $(DevFeedName)" - twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.tar.gz - echo "Uploaded sdist to devops feed $(DevFeedName)" - displayName: 'Publish ${{artifact.name}} alpha package' + - template: /eng/pipelines/templates/steps/publish-alpha-package.yml + parameters: + Artifact: ${{ artifact }} + # Packages that ship compiled binaries publish the wheels rebuilt by + # their Sign_ stage; everything else publishes straight from + # the build output, so a service can mix both kinds of packages freely. + ${{ if and(eq(artifact.signBinaries, true), ne(variables['Build.Reason'], 'PullRequest')) }}: + PackagePath: $(Pipeline.Workspace)/packages_${{ artifact.safeName }}_signed/${{ artifact.name }} + ${{ else }}: + PackagePath: $(Pipeline.Workspace)/${{ parameters.ArtifactName }}/${{ artifact.name }} - job: PublishDocsToNightlyBranch dependsOn: PublishPackages diff --git a/eng/pipelines/templates/stages/release-artifact.yml b/eng/pipelines/templates/stages/release-artifact.yml new file mode 100644 index 000000000000..01f5991af01b --- /dev/null +++ b/eng/pipelines/templates/stages/release-artifact.yml @@ -0,0 +1,428 @@ +# Emits the `Release_` stage for a single artifact. +# +# Extracted verbatim from archetype-python-release.yml so the release jobs keep exactly one +# definition. Artifacts that ship compiled binaries release from a different pipeline artifact +# (`packages__signed`) and depend on a different upstream stage (`Sign_`) +# than everything else. Parameterizing just those two values here avoids forking the release +# job definitions into signed and unsigned copies that would inevitably drift. +# +# Callers resolve ArtifactName and DependsOn per artifact; every job below is otherwise +# unchanged from the shared release workflow. + +parameters: + - name: Artifact + type: object + # Pipeline artifact this package releases from. `packages_extended` for ordinary packages, + # `packages__signed` for packages that opted in with `signBinaries: true`. + - name: ArtifactName + type: string + # Upstream stage. The build stage for ordinary packages, `Sign_` when signing. + - name: DependsOn + type: string + - name: ServiceDirectory + type: string + default: 'not-specified' + - name: TestPipeline + type: boolean + default: false + - name: DocArtifact + type: string + default: 'documentation' + - name: DevFeedName + type: string + default: 'public/azure-sdk-for-python' + - name: PublicFeed + type: string + default: PyPi + - name: PublicPublishEnvironment + type: string + default: package-publish + - name: TargetDocRepoOwner + type: string + default: '' + - name: TargetDocRepoName + type: string + default: '' + +stages: + # Property lookups on a declared object parameter are case sensitive, unlike the loop variable + # this stage was extracted from, so this must match the `safeName` spelling used in ci.yml. + - stage: 'Release_${{parameters.Artifact.safeName}}' + displayName: 'Release: ${{parameters.Artifact.name}}' + dependsOn: + - ${{ parameters.DependsOn }} + - ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + - AutoReleasePrepare + variables: + - template: /eng/pipelines/templates/variables/image.yml + - template: /eng/common/pipelines/templates/variables/api-review-break-glass.yml + # Auto-release CI: only release when the shared prepare stage flagged this artifact as changed. + # Manual runs: every declared artifact remains eligible (existing behavior). + ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + condition: and(succeeded(), eq(dependencies.AutoReleasePrepare.outputs['ResolveAutoReleasePackages.resolve.ReleaseArtifact_${{ parameters.Artifact.safeName }}'], 'true'), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + ${{ else }}: + condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + jobs: + - job: TagRepository + displayName: "Create release tag" + condition: and(succeeded(), ne(variables['Skip.TagRepository'], 'true')) + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - template: /eng/common/pipelines/templates/steps/retain-run.yml + + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} + + - template: /eng/common/pipelines/templates/steps/verify-changelog.yml + parameters: + PackageName: ${{parameters.Artifact.name}} + ServiceName: ${{parameters.ServiceDirectory}} + ForRelease: true + + - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml + parameters: + PackageName: ${{parameters.Artifact.name}} + ServiceDirectory: ${{parameters.ServiceDirectory}} + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} + + - script: | + python -m pip install "./eng/tools/azure-sdk-tools" + displayName: Install tool dependencies + + - task: PythonScript@0 + displayName: Verify Dependency Presence + condition: and(succeeded(), ne(variables['Skip.VerifyDependencies'], 'true')) + inputs: + scriptPath: 'scripts/devops_tasks/verify_dependencies_present.py' + arguments: '--package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }}' + + - task: PythonScript@0 + displayName: Verify CI enabled + condition: succeeded() + inputs: + scriptPath: 'scripts/devops_tasks/verify_ci_enabled.py' + arguments: '--package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }}' + + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}} + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}} + PackageRepository: PyPI + ReleaseSha: $(Build.SourceVersion) + RepoId: Azure/azure-sdk-for-python + WorkingDirectory: $(System.DefaultWorkingDirectory) + AuthToken: '' + + - ${{if ne(parameters.Artifact.skipPublishPackage, 'true')}}: + - deployment: PublishPackage + displayName: "Publish to ${{ parameters.PublicFeed }}" + condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) + # Auto-release runs after merge on main and must not wait on the package-publish approval gate. + ${{ if and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + environment: none + ${{ else }}: + environment: ${{ parameters.PublicPublishEnvironment }} + dependsOn: TagRepository + + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + artifactName: release_artifact + targetPath: $(Pipeline.Workspace)/release_artifact + - input: pipelineArtifact + artifactName: packages_extended + targetPath: $(Pipeline.Workspace)/packages_extended + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + strategy: + runOnce: + deploy: + steps: + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + EnableTwineAuth: false + EnablePipAuth: true + EnableUvAuth: false + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.10' + + - script: | + python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt + displayName: Install Release Dependencies + + - ${{ if eq(parameters.PublicFeed, 'PyPi') }}: + - pwsh: | + $esrpDirectory = "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" + New-Item -ItemType Directory -Force -Path $esrpDirectory + + Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" ` + | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } ` + | Copy-Item -Destination $esrpDirectory + + Get-ChildItem $esrpDirectory + displayName: Isolate files for ESRP Publish + + - template: /eng/pipelines/templates/steps/esrp-publish.yml + parameters: + targetFolder: "$(Pipeline.Workspace)/esrp-release/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}" + + - ${{ if ne(parameters.PublicFeed, 'PyPi') }}: + - task: TwineAuthenticate@0 + displayName: 'Authenticate to feed: ${{parameters.PublicFeed}}' + inputs: + artifactFeeds: ${{parameters.PublicFeed}} + + - script: | + set -e + twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.whl + echo "Uploaded whl to devops feed" + twine upload --repository ${{parameters.PublicFeed}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.tar.gz + echo "Uploaded sdist to devops feed" + displayName: 'Publish package to feed: ${{parameters.PublicFeed}}' + + - task: TwineAuthenticate@0 + displayName: 'Authenticate to feed: ${{parameters.DevFeedName}}' + inputs: + artifactFeeds: ${{parameters.DevFeedName}} + + - script: | + set -e + twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.whl + echo "Uploaded whl to devops feed" + twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}/*.tar.gz + echo "Uploaded sdist to devops feed" + displayName: 'Publish package to feed: ${{parameters.DevFeedName}}' + + - job: MarkPackageReleaseCompletion + displayName: "Mark package release completion" + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + EnableTwineAuth: false + EnablePipAuth: true + EnableUvAuth: false + + - template: /eng/common/pipelines/templates/steps/mark-release-completion.yml + parameters: + ConfigFileDir: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo' + PackageArtifactName: ${{parameters.Artifact.name}} + + # Management-plane-only workaround: skip marking packages that do not generate an APIView revision. + - pwsh: | + $packageInfoPath = '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ parameters.Artifact.name }}.json' + $packageInfo = Get-Content -Raw -Path $packageInfoPath | ConvertFrom-Json + $isManagementPackage = $packageInfo.SdkType -eq 'mgmt' + Write-Host "##vso[task.setvariable variable=IsManagementPackage]$isManagementPackage" + displayName: Check package SDK type + + - task: AzureCLI@2 + displayName: Mark Package Released + condition: >- + and( + succeeded(), + ne(variables['IsManagementPackage'], 'true'), + not( + and( + eq(variables['Skip.MarkPackageReleased'], 'true'), + eq(variables['IsRequesterAuthorizedToSkipApiReview'], 'true') + ) + ) + ) + inputs: + azureSubscription: "ADO to ARH Service Connection" + scriptType: pscore + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/scripts/Mark-PackageReleased.ps1 + arguments: > + -PackageInfoFiles @('$(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{ parameters.Artifact.name }}.json') + -RepoOwner 'Azure' + -AzSdkExePath '$(AZSDK)' + workingDirectory: $(Pipeline.Workspace) + + - ${{if ne(parameters.Artifact.skipPublishDocGithubIo, 'true')}}: + - job: PublishGitHubIODocs + displayName: Publish Docs to GitHubIO Blob Storage + condition: >- + and( + succeeded(), + ne(variables['Skip.PublishDocs'], 'true'), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') + ) + dependsOn: PublishPackage + + pool: + name: azsdk-pool + image: windows-2022 + os: windows + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - download: current + artifact: ${{parameters.DocArtifact}} + timeoutInMinutes: 5 + + - pwsh: | + if (Test-Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}") { + Get-ChildItem -Recurse "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}" + } + else { + New-Item -ItemType Directory -Force -Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}" + } + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + + - template: /eng/common/pipelines/templates/steps/publish-blobs.yml + parameters: + FolderForUpload: '$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{parameters.Artifact.name}}' + TargetLanguage: 'python' + ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{parameters.Artifact.name}}' + + - ${{if ne(parameters.Artifact.skipPublishDocMs, 'true')}}: + - job: PublishDocs + displayName: Docs.MS Release + condition: >- + and( + succeeded(), + ne(variables['Skip.PublishDocs'], 'true'), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr') + ) + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - sdk/**/*.md + - .github/CODEOWNERS + + - download: current + + # py2docfx requires Python >= 3.12.x, match docs pipeline version specification + - task: UsePythonVersion@0 + displayName: 'Use Python 3.12.x' + inputs: + versionSpec: '3.12.x' + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{parameters.Artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'python' + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ + + - job: UpdatePackageVersion + displayName: "Update Package Version" + condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + dependsOn: PublishPackage + + pool: + image: ubuntu-24.04 + name: azsdk-pool + os: linux + + steps: + - checkout: self + - task: UsePythonVersion@0 + + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - script: | + python -m pip install "./eng/tools/azure-sdk-tools" + displayName: Install versioning tool dependencies + + - pwsh: | + sdk_increment_version --package-name ${{ parameters.Artifact.name }} --service ${{ parameters.ServiceDirectory }} + if (Test-Path component-detection-pip-report.json) { + Write-Host "Deleting component-detection-pip-report.json" + rm component-detection-pip-report.json + } + displayName: Increment package version + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: azure-sdk-for-python + PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) + CommitMsg: "Increment package version after release of ${{ parameters.Artifact.name }}" + PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + AuthToken: '' + + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: + - template: /eng/pipelines/templates/jobs/smoke.tests.yml + parameters: + Daily: false + ArtifactName: ${{ parameters.ArtifactName }} + Artifact: ${{ parameters.Artifact }} + DevFeedName: ${{ parameters.DevFeedName }} diff --git a/eng/pipelines/templates/stages/sign-binaries.yml b/eng/pipelines/templates/stages/sign-binaries.yml new file mode 100644 index 000000000000..498c467aa0fd --- /dev/null +++ b/eng/pipelines/templates/stages/sign-binaries.yml @@ -0,0 +1,240 @@ +# Emits a single `Sign_` stage for one artifact that ships compiled binaries. +# +# Packages opt in from their service ci.yml by setting `signBinaries: true` on the artifact. +# This stage is emitted by archetype-python-release.yml, which owns the gating that decides +# when signing runs (release + scheduled builds, never PR builds). +# +# The mac and windows wheels built by the Build stage contain unsigned binaries. This stage +# unpacks only this artifact's wheels, sends the binaries through ESRP, rebuilds the wheels, +# and republishes them as `packages__signed`. +# +# The published artifact deliberately mirrors the layout of `packages_extended`: +# +# packages__signed/ +# PackageInfo/.json +# /*.whl, *.tar.gz, apistub tokens +# +# so that every downstream release job works against it unchanged, just by swapping the +# artifact name. `packages_extended` itself is left untouched and still contains the +# unsigned wheels for this package alongside every other package in the service. + +parameters: + - name: Artifact + type: object + - name: DependsOn + type: string + default: Build + +stages: + - stage: Sign_${{ parameters.Artifact.safeName }} + displayName: 'Sign: ${{ parameters.Artifact.name }}' + dependsOn: ${{ parameters.DependsOn }} + # Intentionally does not check Skip.Release or SetDevVersion. Signing must run whenever + # *either* consumer might need the signed wheels: the Release_ stage or the + # Integration dev feed publish. Skip logic belongs on those consumers, which already carry + # it. Adding it here would propagate through Integration's dependency and silently break + # the nightly and manual dev version alpha publishes. + # + # Run reasons are filtered here rather than at compile time so the emission gate in + # archetype-python-release.yml can stay short enough to repeat verbatim wherever a + # Sign_* stage is referenced. Release builds are manual runs and auto-release CI on main; + # scheduled builds sign the alpha packages that the Integration stage publishes. + # + # The 'Schedule' term is load bearing and coupled to daily-dev-build-variable.yml, which + # sets SetDevVersion=true only when Build.Reason is exactly 'Schedule'. Alpha wheels exist + # only when that is true, so Integration needs signed wheels under exactly the same reasons. + # If that check ever widens (for example to ScheduleForced), widen this list to match or + # scheduled signing silently stops and the alpha publish falls back to skipping the package. + # Reasons absent from this list (BatchedCI, ScheduleForced, IndividualCI off main) produce + # neither alpha wheels nor a release stage, so skipping signing for them is correct. + condition: >- + and( + succeeded(), + ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr'), + or( + in(variables['Build.Reason'], 'Manual', '', 'Schedule'), + and( + eq(variables['Build.Reason'], 'IndividualCI'), + eq(variables['Build.SourceBranch'], 'refs/heads/main') + ) + ) + ) + + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + + jobs: + - job: Sign_macOS + displayName: Sign macOS Wheels + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + # The default sparse checkout always includes /eng, which is all these jobs need. + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - task: DownloadPipelineArtifact@2 + displayName: Download unsigned mac wheels + inputs: + artifactName: packages_mac + # Only this artifact's wheels. Other packages in the service are untouched. + itemPattern: '${{ parameters.Artifact.name }}/**' + targetPath: $(Build.ArtifactStagingDirectory)/packages_mac + + - pwsh: | + python eng/scripts/wheel_signing/extract_sign_inputs.py ` + --platform mac ` + --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_mac/${{ parameters.Artifact.name }}" ` + --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` + --sign-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" + displayName: Extract mac wheel binaries + + - template: pipelines/steps/azd-cli-mac-signing.yml@azure-sdk-build-tools + parameters: + MacPath: "$(Build.ArtifactStagingDirectory)" + MacPattern: "mac-sign-input.zip" + Notarize: false + + - pwsh: | + python eng/scripts/wheel_signing/repackage_signed_wheels.py ` + --platform mac ` + --work-dir "$(Build.ArtifactStagingDirectory)/mac-sign-work" ` + --signed-input-zip "$(Build.ArtifactStagingDirectory)/mac-sign-input.zip" ` + --output-wheels-dir "$(Build.ArtifactStagingDirectory)/mac-wheels-signed" + displayName: Repackage mac wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/mac-wheels-signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_mac_signed' + + - job: Sign_Windows + displayName: Sign Windows Wheels + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + # The default sparse checkout always includes /eng, which is all these jobs need. + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - task: DownloadPipelineArtifact@2 + displayName: Download unsigned windows wheels + inputs: + artifactName: packages_windows + itemPattern: '${{ parameters.Artifact.name }}/**' + targetPath: $(Build.ArtifactStagingDirectory)/packages_windows + + - pwsh: | + python eng/scripts/wheel_signing/extract_sign_inputs.py ` + --platform windows ` + --wheels-dir "$(Build.ArtifactStagingDirectory)/packages_windows/${{ parameters.Artifact.name }}" ` + --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` + --sign-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" + displayName: Extract windows wheel binaries + + - template: pipelines/steps/azd-cli-win-signing.yml@azure-sdk-build-tools + parameters: + WinPath: "$(Build.ArtifactStagingDirectory)/win-sign-input" + WinPattern: '**/*.pyd' + + - pwsh: | + python eng/scripts/wheel_signing/repackage_signed_wheels.py ` + --platform windows ` + --work-dir "$(Build.ArtifactStagingDirectory)/win-sign-work" ` + --signed-input-dir "$(Build.ArtifactStagingDirectory)/win-sign-input" ` + --output-wheels-dir "$(Build.ArtifactStagingDirectory)/win-wheels-signed" + displayName: Repackage windows wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/win-wheels-signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_win_signed' + + - job: Assemble + displayName: Assemble signed artifact + dependsOn: + - Sign_macOS + - Sign_Windows + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + - checkout: none + + # packages_extended is the merged output of the three platform builds plus the + # apistub tokens and PackageInfo. Take this package's slice of it as the base, + # then overwrite the mac and windows wheels with their signed replacements. + - task: DownloadPipelineArtifact@2 + displayName: Download package files + inputs: + artifactName: packages_extended + itemPattern: | + ${{ parameters.Artifact.name }}/** + PackageInfo/${{ parameters.Artifact.name }}.json + targetPath: $(Build.ArtifactStagingDirectory)/signed + + - task: DownloadPipelineArtifact@2 + displayName: Download signed mac wheels + inputs: + artifactName: packages_${{ parameters.Artifact.safeName }}_mac_signed + targetPath: $(Build.ArtifactStagingDirectory)/mac_signed + + - task: DownloadPipelineArtifact@2 + displayName: Download signed windows wheels + inputs: + artifactName: packages_${{ parameters.Artifact.safeName }}_win_signed + targetPath: $(Build.ArtifactStagingDirectory)/win_signed + + - pwsh: | + $ErrorActionPreference = 'Stop' + + $packageDir = "$(Build.ArtifactStagingDirectory)/signed/${{ parameters.Artifact.name }}" + $packageInfo = "$(Build.ArtifactStagingDirectory)/signed/PackageInfo/${{ parameters.Artifact.name }}.json" + + if (-not (Test-Path $packageDir)) { + throw "Expected package folder '$packageDir' in packages_extended." + } + if (-not (Test-Path $packageInfo)) { + throw "Expected package info file '$packageInfo' in packages_extended." + } + + $signedWheels = @( + Get-ChildItem "$(Build.ArtifactStagingDirectory)/mac_signed" -Recurse -Filter "*.whl" + Get-ChildItem "$(Build.ArtifactStagingDirectory)/win_signed" -Recurse -Filter "*.whl" + ) + + if (-not $signedWheels) { + throw "No signed wheels were produced for ${{ parameters.Artifact.name }}." + } + + foreach ($wheel in $signedWheels) { + $target = Join-Path $packageDir $wheel.Name + if (-not (Test-Path $target)) { + throw "Signed wheel '$($wheel.Name)' has no unsigned counterpart in packages_extended. The signed artifact would not match the build output." + } + Write-Host "Replacing $($wheel.Name) with its signed build." + Copy-Item -Path $wheel.FullName -Destination $target -Force + } + + Write-Host "`nFinal contents of the signed artifact:" + Get-ChildItem -Recurse "$(Build.ArtifactStagingDirectory)/signed" | Select-Object -ExpandProperty FullName + displayName: Overlay signed wheels + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/signed' + ArtifactName: 'packages_${{ parameters.Artifact.safeName }}_signed' diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 5f026f19ded9..a4bf41dba5d1 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -91,8 +91,8 @@ steps: BuildTargetingString: ${{ parameters.BuildTargetingString }} PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - # todo, walk the artifacts and ensure that one which includes an extension package is present - # if not, we only need to build on linux. if so, we need to build on all platforms + # Decides whether the mac/windows builds produce anything, based on whether any targeted package + # declares compiled binaries. If none do, only the linux build runs. - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml parameters: PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo diff --git a/eng/pipelines/templates/steps/publish-alpha-package.yml b/eng/pipelines/templates/steps/publish-alpha-package.yml new file mode 100644 index 000000000000..dc246652c884 --- /dev/null +++ b/eng/pipelines/templates/steps/publish-alpha-package.yml @@ -0,0 +1,49 @@ +# Publishes one package's alpha (daily dev build) wheel and sdist to the dev feed. +# +# Extracted from the Integration stage in archetype-python-release.yml so the package path can +# vary per artifact. Packages built with `signBinaries: true` publish from their signed +# artifact; everything else publishes from `packages_extended`. + +parameters: + - name: Artifact + type: object + # Folder holding this package's built files, already resolved by the caller. + - name: PackagePath + type: string + +steps: + - pwsh: | + # The signed artifact is absent when a Sign_* stage failed or was skipped. Skip this one + # package rather than failing the dev feed publish for every other package in the service. + # Logged as a build issue, not just Write-Warning, so a signing failure degrading to + # "this package was not published" is visible in the run summary instead of buried in logs. + if (-not (Test-Path "${{ parameters.PackagePath }}")) { + Write-Host "##vso[task.logissue type=warning]No build output at '${{ parameters.PackagePath }}'. Skipping alpha publish for ${{ parameters.Artifact.name }}. If this package sets signBinaries, check whether its Sign stage failed or was skipped." + exit 0 + } + + # If BuildTargetingString is set, check whether this artifact matches any of the + # (possibly comma-separated) glob patterns before attempting to publish. + # This handles scoped builds where only a subset of packages are built. + $targetingString = $env:BUILDTARGETINGSTRING + if ($targetingString) { + $globs = $targetingString -split "," + $isTargeted = $globs | Where-Object { "${{ parameters.Artifact.name }}" -like $_.Trim() } + if (-not $isTargeted) { + Write-Host "Package '${{ parameters.Artifact.name }}' does not match BuildTargetingString '$targetingString'. Skipping integration publish." + exit 0 + } + } + + $fileCount = (Get-ChildItem ${{ parameters.PackagePath }} | ? {$_.Name -match "-[0-9]*.[0-9]*.[0-9]*a[0-9]*" } | Measure-Object).Count + + if ($fileCount -eq 0) { + Write-Host "No alpha packages for ${{ parameters.Artifact.name }} to publish." + exit 0 + } + + twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) ${{ parameters.PackagePath }}/*-*a*.whl + echo "Uploaded whl to devops feed $(DevFeedName)" + twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) ${{ parameters.PackagePath }}/*-*a*.tar.gz + echo "Uploaded sdist to devops feed $(DevFeedName)" + displayName: 'Publish ${{ parameters.Artifact.name }} alpha package' diff --git a/eng/pipelines/templates/steps/resolve-build-platforms.yml b/eng/pipelines/templates/steps/resolve-build-platforms.yml index 43db2e725352..2de46faf2585 100644 --- a/eng/pipelines/templates/steps/resolve-build-platforms.yml +++ b/eng/pipelines/templates/steps/resolve-build-platforms.yml @@ -4,15 +4,32 @@ parameters: default: '' steps: - # when we merge pipeline v3, this check will change to examining the targeting string $(TargetingString) - # as the generate-pr-diff call + resolution will be present in resolve-package-targeting.yml. - # until then, we simply check to see if we're targeting storage service directory + # Packages that ship compiled binaries opt in with `signBinaries: true` on their artifact entry in + # the service ci.yml. Save-Package-Properties copies that artifact entry verbatim into each package + # info file as `ArtifactDetails`, so the opt-in is readable here without threading the artifact list + # through every caller. + # + # resolve-package-targeting.yml runs immediately before this and deletes the package info files that + # this run is not targeting, so the folder is already narrowed to the packages being built; on 'auto' + # pull request builds save-package-properties.yml narrows it to the PR diff first. This deliberately + # keeps reading the folder rather than the $(TargetingString) those steps also set, because that + # variable carries only package names and the decision below needs an artifact property. - pwsh: | - $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json" ` - | ForEach-Object { $_.Name.Replace(".json", "") } + $binaryPackages = @() - if ($packageProperties -contains "azure-storage-extensions") { - Write-Host "Targeting storage, enabling extension build." + foreach ($packageInfoPath in (Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json")) { + $packageInfo = Get-Content -Raw -Path $packageInfoPath.FullName | ConvertFrom-Json + + if ($packageInfo.ArtifactDetails.signBinaries -eq $true) { + $binaryPackages += $packageInfo.Name + } + } + + if ($binaryPackages) { + Write-Host "Targeting package(s) with compiled binaries ($($binaryPackages -join ', ')), enabling extension build." Write-Host "##vso[task.setvariable variable=ENABLE_EXTENSION_BUILD]true" } + else { + Write-Host "No targeted package declares compiled binaries, building on linux only." + } displayName: Check extension package presence diff --git a/eng/scripts/wheel_signing/extract_sign_inputs.py b/eng/scripts/wheel_signing/extract_sign_inputs.py new file mode 100644 index 000000000000..045dc4ea1c5c --- /dev/null +++ b/eng/scripts/wheel_signing/extract_sign_inputs.py @@ -0,0 +1,178 @@ +import argparse +import json +import shutil +import zipfile +from pathlib import Path +from typing import Dict, List + + +SIGNABLE_SUFFIXES = {".so", ".dylib", ".dll", ".pyd"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Extract wheel files, collect signable binaries, and generate " + "signing payloads plus a manifest for wheel repackaging." + ) + ) + parser.add_argument("--platform", choices=["mac", "windows"], required=True) + parser.add_argument("--wheels-dir", required=True, help="Directory containing input wheel files.") + parser.add_argument("--work-dir", required=True, help="Working directory for unpacked wheels and manifest.") + parser.add_argument( + "--sign-input-zip", + default=None, + help="Output zip containing binaries to sign (required for --platform mac).", + ) + parser.add_argument( + "--sign-input-dir", + default=None, + help="Output folder containing binaries to sign (required for --platform windows).", + ) + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> None: + if args.platform == "mac": + if not args.sign_input_zip: + raise ValueError("--sign-input-zip is required for --platform mac.") + if args.sign_input_dir: + raise ValueError("--sign-input-dir is not valid for --platform mac.") + else: + if not args.sign_input_dir: + raise ValueError("--sign-input-dir is required for --platform windows.") + if args.sign_input_zip: + raise ValueError("--sign-input-zip is not valid for --platform windows.") + + +def reset_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def collect_wheels(wheels_dir: Path) -> List[Path]: + if not wheels_dir.is_dir(): + raise FileNotFoundError(f"Wheel directory not found: {wheels_dir}") + return sorted(wheels_dir.rglob("*.whl")) + + +def collect_signable_files(unpacked_wheel_dir: Path) -> List[Path]: + files = [] + for path in sorted(unpacked_wheel_dir.rglob("*")): + if path.is_file() and path.suffix.lower() in SIGNABLE_SUFFIXES: + files.append(path) + return files + + +def write_manifest(manifest_path: Path, manifest_data: Dict) -> None: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + with manifest_path.open("w", encoding="utf-8") as handle: + json.dump(manifest_data, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def create_zip_from_dir(source_dir: Path, output_zip: Path) -> None: + output_zip.parent.mkdir(parents=True, exist_ok=True) + if output_zip.exists(): + output_zip.unlink() + + with zipfile.ZipFile(output_zip, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for file_path in sorted(source_dir.rglob("*")): + if file_path.is_file(): + archive.write(file_path, file_path.relative_to(source_dir).as_posix()) + + +def main() -> None: + args = parse_args() + validate_args(args) + + wheels_dir = Path(args.wheels_dir).resolve() + work_dir = Path(args.work_dir).resolve() + unpack_root = work_dir / "unpacked" + manifest_path = work_dir / "signing-manifest.json" + + if args.platform == "windows": + payload_dir = Path(args.sign_input_dir).resolve() + else: + payload_dir = work_dir / "mac-sign-input" + + reset_dir(work_dir) + reset_dir(unpack_root) + reset_dir(payload_dir) + + wheels = collect_wheels(wheels_dir) + print(f"Platform: {args.platform}") + print(f"Input wheels dir: {wheels_dir}") + print(f"Work dir: {work_dir}") + + manifest: Dict[str, object] = { + "platform": args.platform, + "wheels": [], + "entries": [], + } + + payload_index = 0 + + for wheel_path in wheels: + unpack_dir_name = wheel_path.name[:-4] + unpacked_wheel_dir = unpack_root / unpack_dir_name + unpacked_wheel_dir.mkdir(parents=True, exist_ok=True) + print(f"[EXTRACT] wheel={wheel_path.name} unpack_dir={unpacked_wheel_dir}") + + with zipfile.ZipFile(wheel_path, "r") as archive: + archive.extractall(unpacked_wheel_dir) + + manifest["wheels"].append( + { + "wheel_filename": wheel_path.name, + "unpack_dir": unpack_dir_name, + } + ) + + signable_files = collect_signable_files(unpacked_wheel_dir) + print(f"[EXTRACT] wheel={wheel_path.name} signable_count={len(signable_files)}") + for signable_file in signable_files: + relative_path = signable_file.relative_to(unpacked_wheel_dir).as_posix() + payload_subdir = f"{payload_index:05d}" + payload_index += 1 + + payload_path = f"{payload_subdir}/{signable_file.name}" + payload_file = payload_dir / payload_subdir / signable_file.name + payload_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(signable_file, payload_file) + print( + "[MAP_EXTRACT] " + f"payload={payload_path} " + f"source_wheel={wheel_path.name} " + f"source_relative_path={relative_path} " + f"source_file={signable_file} " + f"payload_file={payload_file}" + ) + + manifest["entries"].append( + { + "wheel_filename": wheel_path.name, + "unpack_dir": unpack_dir_name, + "relative_path": relative_path, + "payload_path": payload_path, + } + ) + + write_manifest(manifest_path, manifest) + + if args.platform == "mac": + sign_zip = Path(args.sign_input_zip).resolve() + create_zip_from_dir(payload_dir, sign_zip) + + print(f"Wheels processed: {len(wheels)}") + print(f"Signable binaries collected: {len(manifest['entries'])}") + print(f"Manifest: {manifest_path}") + if args.platform == "mac": + print(f"Sign payload zip: {Path(args.sign_input_zip).resolve()}") + else: + print(f"Sign payload dir: {payload_dir}") + + +if __name__ == "__main__": + main() diff --git a/eng/scripts/wheel_signing/repackage_signed_wheels.py b/eng/scripts/wheel_signing/repackage_signed_wheels.py new file mode 100644 index 000000000000..0fa25172ca3d --- /dev/null +++ b/eng/scripts/wheel_signing/repackage_signed_wheels.py @@ -0,0 +1,199 @@ +import argparse +import base64 +import csv +import hashlib +import json +import shutil +import tempfile +import zipfile +from pathlib import Path +from typing import Dict, List + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Reinsert signed binaries into unpacked wheel trees and rebuild wheel files " + "with original filenames." + ) + ) + parser.add_argument("--platform", choices=["mac", "windows"], required=True) + parser.add_argument("--work-dir", required=True, help="Working directory created by extract_sign_inputs.py.") + parser.add_argument( + "--signed-input-zip", + default=None, + help="Signed binary zip payload (required for --platform mac).", + ) + parser.add_argument( + "--signed-input-dir", + default=None, + help="Signed binary folder payload (required for --platform windows).", + ) + parser.add_argument("--output-wheels-dir", required=True, help="Directory where rebuilt wheel files are written.") + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> None: + if args.platform == "mac": + if not args.signed_input_zip: + raise ValueError("--signed-input-zip is required for --platform mac.") + if args.signed_input_dir: + raise ValueError("--signed-input-dir is not valid for --platform mac.") + else: + if not args.signed_input_dir: + raise ValueError("--signed-input-dir is required for --platform windows.") + if args.signed_input_zip: + raise ValueError("--signed-input-zip is not valid for --platform windows.") + + +def load_manifest(manifest_path: Path) -> Dict: + if not manifest_path.is_file(): + raise FileNotFoundError(f"Manifest not found: {manifest_path}") + with manifest_path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def reset_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def get_digest_and_size(file_path: Path) -> List[str]: + content = file_path.read_bytes() + digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).decode("ascii").rstrip("=") + return [f"sha256={digest}", str(len(content))] + + +def find_record_path(unpacked_wheel_dir: Path) -> Path: + candidates = sorted(unpacked_wheel_dir.glob("*.dist-info/RECORD")) + if len(candidates) != 1: + raise RuntimeError( + f"Expected exactly one RECORD file under {unpacked_wheel_dir}, found {len(candidates)}." + ) + return candidates[0] + + +def rewrite_record(unpacked_wheel_dir: Path) -> None: + record_path = find_record_path(unpacked_wheel_dir) + record_rel = record_path.relative_to(unpacked_wheel_dir).as_posix() + + rows: List[List[str]] = [] + for file_path in sorted(unpacked_wheel_dir.rglob("*")): + if not file_path.is_file(): + continue + rel = file_path.relative_to(unpacked_wheel_dir).as_posix() + if rel == record_rel: + continue + digest, size = get_digest_and_size(file_path) + rows.append([rel, digest, size]) + + rows.append([record_rel, "", ""]) + + with record_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle, lineterminator="\n") + writer.writerows(rows) + + +def build_wheel(unpacked_wheel_dir: Path, output_wheel_path: Path) -> None: + output_wheel_path.parent.mkdir(parents=True, exist_ok=True) + if output_wheel_path.exists(): + output_wheel_path.unlink() + + rewrite_record(unpacked_wheel_dir) + + with zipfile.ZipFile(output_wheel_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for file_path in sorted(unpacked_wheel_dir.rglob("*")): + if file_path.is_file(): + archive.write(file_path, file_path.relative_to(unpacked_wheel_dir).as_posix()) + + +def main() -> None: + args = parse_args() + validate_args(args) + + work_dir = Path(args.work_dir).resolve() + unpack_root = work_dir / "unpacked" + manifest_path = work_dir / "signing-manifest.json" + output_wheels_dir = Path(args.output_wheels_dir).resolve() + + manifest = load_manifest(manifest_path) + if manifest.get("platform") != args.platform: + raise RuntimeError( + f"Manifest platform '{manifest.get('platform')}' does not match argument platform '{args.platform}'." + ) + + if not unpack_root.is_dir(): + raise FileNotFoundError(f"Unpacked wheel directory not found: {unpack_root}") + + print(f"Platform: {args.platform}") + print(f"Work dir: {work_dir}") + print(f"Manifest: {manifest_path}") + + with tempfile.TemporaryDirectory(prefix="signed-binaries-") as tmpdir: + if args.platform == "mac": + signed_payload_dir = Path(tmpdir) / "signed-payload" + signed_payload_dir.mkdir(parents=True, exist_ok=True) + signed_zip = Path(args.signed_input_zip).resolve() + if not signed_zip.is_file(): + raise FileNotFoundError(f"Signed payload zip not found: {signed_zip}") + with zipfile.ZipFile(signed_zip, "r") as archive: + archive.extractall(signed_payload_dir) + print(f"Signed payload zip: {signed_zip}") + else: + signed_payload_dir = Path(args.signed_input_dir).resolve() + if not signed_payload_dir.is_dir(): + raise FileNotFoundError(f"Signed payload directory not found: {signed_payload_dir}") + print(f"Signed payload dir: {signed_payload_dir}") + + for entry in manifest.get("entries", []): + wheel_filename = entry["wheel_filename"] + unpack_dir = entry["unpack_dir"] + relative_path = entry["relative_path"] + payload_path = entry["payload_path"] + + source_signed_binary = signed_payload_dir / payload_path + target_binary = unpack_root / unpack_dir / relative_path + + if not source_signed_binary.is_file(): + raise FileNotFoundError(f"Signed binary missing: {source_signed_binary}") + if not target_binary.is_file(): + raise FileNotFoundError(f"Target binary missing in unpacked wheel: {target_binary}") + + shutil.copy2(source_signed_binary, target_binary) + print( + "[MAP_REPACKAGE] " + f"payload={payload_path} " + f"target_wheel={wheel_filename} " + f"target_relative_path={relative_path} " + f"signed_source={source_signed_binary} " + f"target_file={target_binary}" + ) + + reset_dir(output_wheels_dir) + + rebuilt_count = 0 + for wheel_info in manifest.get("wheels", []): + wheel_filename = wheel_info["wheel_filename"] + unpack_dir = wheel_info["unpack_dir"] + + unpacked_wheel_dir = unpack_root / unpack_dir + if not unpacked_wheel_dir.is_dir(): + raise FileNotFoundError(f"Unpacked wheel directory missing: {unpacked_wheel_dir}") + + output_wheel_path = output_wheels_dir / wheel_filename + print( + "[REBUILD] " + f"wheel={wheel_filename} " + f"unpacked_dir={unpacked_wheel_dir} " + f"output_wheel={output_wheel_path}" + ) + build_wheel(unpacked_wheel_dir, output_wheel_path) + rebuilt_count += 1 + + print(f"Wheels rebuilt: {rebuilt_count}") + print(f"Output wheels dir: {output_wheels_dir}") + + +if __name__ == "__main__": + main() diff --git a/sdk/storage/azure-storage-extensions/CHANGELOG.md b/sdk/storage/azure-storage-extensions/CHANGELOG.md index 8f529305a53b..ceaea28525d4 100644 --- a/sdk/storage/azure-storage-extensions/CHANGELOG.md +++ b/sdk/storage/azure-storage-extensions/CHANGELOG.md @@ -7,4 +7,5 @@ ## 0.1.0 (2026-07-28) ### Features Added + - Initial release. diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index a05078529031..da3d9c40a91c 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -55,6 +55,9 @@ extends: safeName: azurestoragequeue - name: azure-storage-extensions safeName: azurestorageextensions + # Ships compiled binaries, so its wheels are routed through a dedicated + # Sign_azurestorageextensions stage before release and dev feed publishing. + signBinaries: true triggeringPaths: - /sdk/storage/azure-storage-blob - /sdk/storage/azure-storage-file-datalake diff --git a/sdk/template/azure-template-two/CHANGELOG.md b/sdk/template/azure-template-two/CHANGELOG.md new file mode 100644 index 000000000000..3c34c1341e89 --- /dev/null +++ b/sdk/template/azure-template-two/CHANGELOG.md @@ -0,0 +1,9 @@ +# Release History + +## 1.0.0b1 (Unreleased) + +### Features Added + +- Initial release. Adds `azure.template.two.istrue.is_true`, a minimal native + function used to exercise binary wheel builds and signing in the release + pipeline. diff --git a/sdk/template/azure-template-two/LICENSE b/sdk/template/azure-template-two/LICENSE new file mode 100644 index 000000000000..63447fd8bbbf --- /dev/null +++ b/sdk/template/azure-template-two/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) Microsoft Corporation. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/sdk/template/azure-template-two/MANIFEST.in b/sdk/template/azure-template-two/MANIFEST.in new file mode 100644 index 000000000000..cf0b5dd1bf0b --- /dev/null +++ b/sdk/template/azure-template-two/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include azure *.h *.c *.pyi py.typed +include *.md +include LICENSE +recursive-include tests *.py diff --git a/sdk/template/azure-template-two/README.md b/sdk/template/azure-template-two/README.md new file mode 100644 index 000000000000..9bfb7fd0b374 --- /dev/null +++ b/sdk/template/azure-template-two/README.md @@ -0,0 +1,47 @@ +# Binary template package for the Azure SDK for Python + +**This package is a test fixture for the Azure SDK engineering system and is not intended for direct use.** + +`azure-template-two` is the binary counterpart to `azure-template`. It ships a minimal C extension so that the build and release pipelines have a package that produces both platform specific wheels and an sdist. Use it to exercise `cibuildwheel`, binary signing, and publishing without depending on a real service library. + +The extension exposes a single function that takes no arguments and always returns `True`: + +```python +from azure.template.two import istrue + +istrue.is_true() # True +``` + +## Installation + +```bash +pip install azure-template-two +``` + +### Prerequisites +* Python 3.10 or later is required to use this package. For more details, please read our page on [Azure SDK for Python version support policy](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/python_version_support_policy.md). + +## Building locally + +Wheels are produced with [cibuildwheel](https://cibuildwheel.pypa.io/), configured in `pyproject.toml` to build CPython 3.10 for x86_64 Linux, x86_64 Windows, and arm64 macOS. Because the extension compiles against the CPython limited API, each wheel is tagged `abi3` and remains importable on later interpreters. + +To build and test for the current platform only: + +```bash +pip install cibuildwheel +cibuildwheel --only cp310-macosx_arm64 sdk/template/azure-template-two +``` + +A plain in-place build is also enough for local development: + +```bash +pip install -e sdk/template/azure-template-two +pytest sdk/template/azure-template-two/tests +``` + +## Contributing +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/sdk/template/azure-template-two/azure/template/two/__init__.py b/sdk/template/azure-template-two/azure/template/two/__init__.py new file mode 100644 index 000000000000..e67f0c1442ad --- /dev/null +++ b/sdk/template/azure-template-two/azure/template/two/__init__.py @@ -0,0 +1,9 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from . import istrue + +__all__ = ["istrue"] diff --git a/sdk/template/azure-template-two/azure/template/two/_version.py b/sdk/template/azure-template-two/azure/template/two/_version.py new file mode 100644 index 000000000000..e5b9bca3a368 --- /dev/null +++ b/sdk/template/azure-template-two/azure/template/two/_version.py @@ -0,0 +1,7 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +VERSION = "1.0.0b1" diff --git a/sdk/template/azure-template-two/azure/template/two/istrue.pyi b/sdk/template/azure-template-two/azure/template/two/istrue.pyi new file mode 100644 index 000000000000..d801ac38da39 --- /dev/null +++ b/sdk/template/azure-template-two/azure/template/two/istrue.pyi @@ -0,0 +1,13 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +# pylint: skip-file + +def is_true() -> bool: + """Return True. + + :returns: True, always. + """ + ... diff --git a/sdk/template/azure-template-two/azure/template/two/py.typed b/sdk/template/azure-template-two/azure/template/two/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/template/azure-template-two/azure/template/two/src/istruemodule.c b/sdk/template/azure-template-two/azure/template/two/src/istruemodule.c new file mode 100644 index 000000000000..73dc3dedc819 --- /dev/null +++ b/sdk/template/azure-template-two/azure/template/two/src/istruemodule.c @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#define PY_SSIZE_T_CLEAN +#include + +// This package exists to give the release pipeline a package that ships both a +// platform specific binary wheel and an sdist. The implementation is therefore +// deliberately trivial: one function that takes no arguments and returns True. +static PyObject* is_true(PyObject* self, PyObject* Py_UNUSED(ignored)) { + Py_RETURN_TRUE; +} + +static PyMethodDef istrue_methods[] = { + { "is_true", (PyCFunction)is_true, METH_NOARGS, "Return True." }, + { NULL, NULL, 0, NULL } +}; + +static PyModuleDef istrue_module = { + PyModuleDef_HEAD_INIT, + "istrue", // Module name to use with Python import statements + "Minimal native extension used to exercise binary packaging and signing.", + 0, + istrue_methods +}; + +PyMODINIT_FUNC +PyInit_istrue() { + return PyModule_Create(&istrue_module); +} diff --git a/sdk/template/azure-template-two/dev_requirements.txt b/sdk/template/azure-template-two/dev_requirements.txt new file mode 100644 index 000000000000..190d2e5520b2 --- /dev/null +++ b/sdk/template/azure-template-two/dev_requirements.txt @@ -0,0 +1,2 @@ +-e ../../../eng/tools/azure-sdk-tools +pytest diff --git a/sdk/template/azure-template-two/pyproject.toml b/sdk/template/azure-template-two/pyproject.toml new file mode 100644 index 000000000000..6adde4576eb3 --- /dev/null +++ b/sdk/template/azure-template-two/pyproject.toml @@ -0,0 +1,92 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------- + +[build-system] +requires = ["setuptools>=77.0.3", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-template-two" +authors = [ + { name = "Microsoft Corporation", email = "azuresdkengsysadmins@microsoft.com" }, +] +description = "Microsoft Azure Template Package (Binary) Client Library for Python" +license = "MIT" +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +requires-python = ">=3.10" +dependencies = [] +keywords = ["azure", "azure sdk"] +dynamic = ["version", "readme"] + +[project.urls] +"Bug Reports" = "https://github.com/Azure/azure-sdk-for-python/issues" +repository = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/template/azure-template-two" + +[tool.setuptools.dynamic] +version = { attr = "azure.template.two._version.VERSION" } +readme = { file = ["README.md"], content-type = "text/markdown" } + +# `azure` and `azure.template` are namespace directories owned by other +# distributions, so only the leaf package is shipped from here. The C sources +# live in a `src` directory rather than one named after the extension so that +# they cannot shadow the compiled module when importing from the source tree. +[tool.setuptools.packages.find] +exclude = [ + "tests*", + "samples*", + "doc*", + "azure", + "azure.template", + "azure.template.two.src", +] + +[tool.azure-sdk-build] +# This package is a C extension with no public Python surface beyond a single +# function, so the Python-oriented checks are not meaningful here. This mirrors +# the settings used by the other binary package in the repo, +# sdk/storage/azure-storage-extensions. +mypy = false +pyright = false +type_check_samples = false +verifytypes = false +pylint = false +sphinx = false +verifysdist = false +verifywhl = false +sdist = false +breaking = false +suppressed_skip_warnings = ["*"] + +[tool.azure-sdk-conda] +in_bundle = false + +# One CPython 3.10 build per platform. Because the extension is compiled +# against the limited API the resulting wheels are tagged abi3 and remain +# importable on later interpreters, so this is the smallest matrix that still +# covers all three operating systems. +[tool.cibuildwheel] +build = ["cp310-*"] +skip = ["*musllinux*"] +test-requires = "pytest" +test-command = "pytest {project}/tests" + +[tool.cibuildwheel.linux] +archs = "x86_64" + +[tool.cibuildwheel.macos] +archs = "arm64" + +[tool.cibuildwheel.windows] +archs = "AMD64" diff --git a/sdk/template/azure-template-two/sdk_packaging.toml b/sdk/template/azure-template-two/sdk_packaging.toml new file mode 100644 index 000000000000..901bc8ccbfa6 --- /dev/null +++ b/sdk/template/azure-template-two/sdk_packaging.toml @@ -0,0 +1,2 @@ +[packaging] +auto_update = false diff --git a/sdk/template/azure-template-two/setup.py b/sdk/template/azure-template-two/setup.py new file mode 100644 index 000000000000..a6a78778c9c6 --- /dev/null +++ b/sdk/template/azure-template-two/setup.py @@ -0,0 +1,45 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# This file remains for two pieces of config that setuptools cannot yet +# express declaratively in pyproject.toml without opting into experimental +# config: +# 1. The limited-API C extension(s) — [tool.setuptools.ext-modules] is +# still experimental. +# 2. The abi3 wheel tag (py_limited_api) — the equivalent +# [tool.distutils.bdist_wheel] table is also still experimental. + +import re +from pathlib import Path + +from setuptools import setup, Extension + +PACKAGE_NAME = "azure-template-two" + +# `name` and `version` are declared in pyproject.toml (version dynamically via +# [tool.setuptools.dynamic]). They are repeated in the setup() call below only +# because doc-warden's README verification parses setup.py and requires both +# kwargs to be present. The version is read from _version.py rather than +# imported so it works under doc-warden's AST exec as well as a normal build; +# both run with the working directory set to the package root. +version_text = Path("azure", "template", "two", "_version.py").read_text() +version = re.search(r'VERSION = "(.*?)"', version_text).group(1) + + +setup( + name=PACKAGE_NAME, + version=version, + ext_package="azure.template.two", + ext_modules=[ + Extension( + "istrue", + ["azure/template/two/src/istruemodule.c"], + define_macros=[("Py_LIMITED_API", "0x030A0000")], + py_limited_api=True, + ), + ], + options={"bdist_wheel": {"py_limited_api": "cp310"}}, +) diff --git a/sdk/template/azure-template-two/tests/test_istrue.py b/sdk/template/azure-template-two/tests/test_istrue.py new file mode 100644 index 000000000000..dca18e7486e0 --- /dev/null +++ b/sdk/template/azure-template-two/tests/test_istrue.py @@ -0,0 +1,24 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from azure.template.two import istrue + + +def test_is_true_returns_true(): + assert istrue.is_true() is True + + +def test_is_true_takes_no_arguments(): + import pytest + + with pytest.raises(TypeError): + istrue.is_true(1) + + +def test_extension_is_native(): + # The wheel is only worth signing if this really is a compiled module + # rather than a pure Python fallback. + assert istrue.__file__.endswith((".so", ".pyd", ".dylib")) diff --git a/sdk/template/ci.yml b/sdk/template/ci.yml index 0683cad79cb6..404932379483 100644 --- a/sdk/template/ci.yml +++ b/sdk/template/ci.yml @@ -61,3 +61,15 @@ extends: Artifacts: - name: azure-template safeName: azuretemplate + # Ships compiled binaries, so its wheels are routed through a dedicated + # Sign_azuretemplatetwo stage before release and dev feed publishing. Paired + # with the pure Python azure-template above, this service covers both kinds + # of package in one release. + - name: azure-template-two + safeName: azuretemplatetwo + signBinaries: true + # Pure C-based package, not generating docs at this moment. Doc publishing + # resolves a single wheel per package, which does not hold for a package + # that ships one wheel per platform. + skipPublishDocGithubIo: true + skipPublishDocMs: true