From ff7572617bfe491801230bf441bf00b181e51746 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 9 Oct 2026 09:56:01 -0700 Subject: [PATCH 1/2] Update common Docker engineering infrastructure from mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:90213836518e8ef7c33153d00abeaafbd7e8d6eb276576920c2cb31b4c7dbd8e Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9aa5e3ce-8de5-43db-b6e8-1d2e3a104d4c --- eng/docker-tools/CHANGELOG.md | 65 ++++++++++++++++ eng/docker-tools/DEV-GUIDE.md | 18 ++++- .../templates/jobs/build-images.yml | 5 +- .../templates/jobs/post-build.yml | 8 +- eng/docker-tools/templates/jobs/publish.yml | 75 +++++++------------ .../templates/stages/build-and-test.yml | 9 +++ .../stages/dotnet/build-and-test.yml | 2 + .../stages/dotnet/build-test-publish-repo.yml | 2 + .../templates/steps/annotate-eol-digests.yml | 43 ----------- .../templates/steps/generate-appsettings.yml | 18 ++++- .../templates/steps/init-common.yml | 6 ++ .../templates/steps/init-imagebuilder.yml | 5 ++ .../templates/steps/publish-artifact.yml | 6 +- .../templates/variables/common.yml | 2 - .../templates/variables/docker-images.yml | 2 +- 15 files changed, 159 insertions(+), 107 deletions(-) delete mode 100644 eng/docker-tools/templates/steps/annotate-eol-digests.yml diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md index 0936f6feb6..adcc236c34 100644 --- a/eng/docker-tools/CHANGELOG.md +++ b/eng/docker-tools/CHANGELOG.md @@ -4,6 +4,71 @@ All breaking changes and new features in `eng/docker-tools` will be documented i --- +## 2026-10-05: Lifecycle metadata commands combined + +- Pull request: [#2252](https://github.com/dotnet/docker-tools/pull/2252) +- Issues: [#2153](https://github.com/dotnet/docker-tools/issues/2153), [#2066](https://github.com/dotnet/docker-tools/issues/2066) + +ImageBuilder now uses one command, `attachLifecycleMetadata`, to add lifecycle metadata to images. +The shared publish and cleanup templates use the new command. Repositories that only use these +templates do not need to make any changes. + +Repositories that call ImageBuilder commands directly must use these replacements: + +| Old command | New command | +| --- | --- | +| `generateEolAnnotationDataForPublish` and `annotateEolDigests` | `attachLifecycleMetadata unsupported` | +| `generateEolAnnotationDataForAllImages` and `annotateEolDigests` | `attachLifecycleMetadata all` | +| `annotateEolDigests` with a data file | `attachLifecycleMetadata file` | +| `waitForMarAnnotationIngestion` | Add `--wait-for-ingestion` to an `attachLifecycleMetadata` command | + +The publish job no longer creates lifecycle metadata data files before it attaches the metadata. +Update any automation that reads those intermediate files. + +Lifecycle metadata can now be marked as internal. Internal metadata is never published. + +--- + +## 2026-10-05: Image-level syndication + +- Issue: [#2240](https://github.com/dotnet/docker-tools/issues/2240) + +Syndication has moved from tag-level to image-level. All platform tags, shared +tags, signatures, and referrers are replicated 1:1 during publishing. + +`Build` and `Post_Build` stages no longer create syndicated tags or manifest +lists. Image info records the destination as `syndicatedRepo` instead of +recording separate `syndicatedDigests`, because syndicated images have the same +digests as the primary images. Custom destination tag names and selective +platform syndication are no longer supported. + +--- + +## 2026-08-10: Pre-ImageBuilder build customization + +Build pipeline templates now accept `customPreImageBuilderBuildSteps`. These steps run after +ImageBuilder is available but before repository content is copied into the Linux ImageBuilder +image. Repositories can use the hook to stage files into Docker build contexts, such as shared +`eng/common` content required by Dockerfiles. + +--- + +## 2026-07-28: Publish stage artifacts consolidated + +The Publish stage now uploads `$(Build.ArtifactStagingDirectory)` once as +`publish-attempt-$(System.JobAttempt)`. This replaces these separate artifacts: + +- `image-info-final-$(System.JobAttempt)` +- `eol-annotation-data-$(System.JobAttempt)` +- `annotation-digests--$(System.JobAttempt)` +- `source-build-id` + +Consumers of those artifact names must download the consolidated artifact instead. Files retain +their staging-directory paths, including `imageInfo/`, `eol-annotation-data/`, +`annotation-digests/`, and `sourceBuildId/source-build-id.txt`. + +--- + ## 2026-06-11: Configurable per-registry referrer-lookup rate limit - Issue: [#2141](https://github.com/dotnet/docker-tools/issues/2141) diff --git a/eng/docker-tools/DEV-GUIDE.md b/eng/docker-tools/DEV-GUIDE.md index 9d4eaee7c4..20c8a02832 100644 --- a/eng/docker-tools/DEV-GUIDE.md +++ b/eng/docker-tools/DEV-GUIDE.md @@ -302,7 +302,7 @@ The publish stage does more than just push images. Here's the sequence: 5. **Wait for Doc Ingestion** — Ensures README changes are live 6. **Merge & Publish Image Info** — Updates the versions repo with new image metadata 7. **Ingest Kusto Image Info** — Sends telemetry to Kusto for analytics -8. **Generate & Apply EOL Annotations** — Marks images with end-of-life dates +8. **Attach Lifecycle Metadata** - `attachLifecycleMetadata unsupported` marks unsupported images with end-of-life dates 9. **Post Publish Notification** — Creates GitHub issues/notifications about the publish ### Dry-Run Mode @@ -389,6 +389,22 @@ To force a rebuild regardless of cache state, set the `noCache` parameter to `tr ## Common Customization Patterns +### Pattern: Staging Files into Docker Build Contexts + +Use `customPreImageBuilderBuildSteps` to modify repository content immediately before the +repository is copied into the Linux ImageBuilder image. For example, a repository can stage +shared `eng/common` files next to Dockerfiles whose build contexts cannot access the repository +root: + +```yaml +customPreImageBuilderBuildSteps: +- powershell: ./eng/Stage-EngCommon.ps1 + displayName: Stage eng/common in Docker Build Contexts +``` + +The steps run once per Linux build job, after ImageBuilder is available and before the +`Dockerfile.WithRepo` image is built. + ### Pattern: Adding Build Arguments Pass Dockerfile `ARG` values via ImageBuilder: diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml index 7327b6d697..33b866f6da 100644 --- a/eng/docker-tools/templates/jobs/build-images.yml +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -10,6 +10,8 @@ parameters: # Custom steps that run after ImageBuilder is set up but before the build starts. # Use for build-specific initialization (e.g., setting variables, additional setup). customBuildInitSteps: [] + # Custom steps that modify repository content before it is copied into the ImageBuilder image. + customPreImageBuilderBuildSteps: [] publishConfig: null versionsRepoRef: "" noCache: false @@ -41,6 +43,7 @@ jobs: versionsRepoRef: ${{ parameters.versionsRepoRef }} cleanupDocker: true customInitSteps: ${{ parameters.customInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} - ${{ parameters.customBuildInitSteps }} - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self parameters: @@ -140,4 +143,4 @@ jobs: displayName: Publish SBOM internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - condition: ne(variables['BuildImages.builtImages'], '') + condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) diff --git a/eng/docker-tools/templates/jobs/post-build.yml b/eng/docker-tools/templates/jobs/post-build.yml index 32b9c7999c..8a07759bd1 100644 --- a/eng/docker-tools/templates/jobs/post-build.yml +++ b/eng/docker-tools/templates/jobs/post-build.yml @@ -11,7 +11,6 @@ jobs: variables: imageInfosSubDir: "/image-infos" imageInfosHostDir: "$(Build.ArtifactStagingDirectory)$(imageInfosSubDir)" - imageInfosContainerDir: "$(artifactsPath)$(imageInfosSubDir)" imageInfosOutputSubDir: "/output" sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom" steps: @@ -73,11 +72,10 @@ jobs: exit 0 } - New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force $(runImageBuilderCmd) mergeImageInfo ` --manifest $(manifest) ` - $(imageInfosContainerDir) ` - $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json ` + image-infos ` + image-infos/output/image-info.json ` $(manifestVariables) name: MergeImageInfoFiles displayName: Merge Image Info Files @@ -89,7 +87,7 @@ jobs: condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true'), ne(variables['Build.Reason'], 'PullRequest')) args: >- createManifestList - '$(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json' + 'image-infos/output/image-info.json' --repo-prefix '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}' --os-type '*' --architecture '*' diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml index 5839be2d76..05fb29205e 100644 --- a/eng/docker-tools/templates/jobs/publish.yml +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -37,8 +37,6 @@ jobs: value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ] - name: imageInfoHostDir value: $(Build.ArtifactStagingDirectory)/imageInfo - - name: imageInfoContainerDir - value: $(artifactsPath)/imageInfo - name: sourceBuildIdOutputDir value: $(Build.ArtifactStagingDirectory)/sourceBuildId - name: commitOverrideArg @@ -46,6 +44,8 @@ jobs: value: --commit-override $(Build.SourceVersion) ${{ else }}: value: '' + - name: eolAnnotationsWaitArg + value: $[ iif(eq(variables['waitForIngestionEnabled'], 'true'), '--wait-for-ingestion', '') ] - ${{ parameters.customPublishVariables }} steps: @@ -99,7 +99,7 @@ jobs: - script: > $(runImageBuilderCmd) trimUnchangedPlatforms - '$(imageInfoContainerDir)/image-info.json' + 'imageInfo/image-info.json' displayName: Trim Unchanged Images - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self @@ -113,23 +113,15 @@ jobs: --os-type '*' --architecture '*' --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}' - --image-info '$(imageInfoContainerDir)/image-info.json' + --image-info 'imageInfo/image-info.json' $(dryRunArg) $(imageBuilder.pathArgs) $(imageBuilder.commonCmdArgs) - - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self - parameters: - path: $(imageInfoHostDir) - artifactName: image-info-final-$(System.JobAttempt) - displayName: Publish Image Info File Artifact - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - - template: /eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} - imageInfoPath: '$(imageinfoContainerDir)/image-info.json' + imageInfoPath: 'imageInfo/image-info.json' minQueueTime: $(imageQueueTime) dryRunArg: $(dryRunArg) condition: succeeded() @@ -139,9 +131,6 @@ jobs: dryRunArg: $(dryRunArg) condition: and(succeeded(), eq(variables['publishReadme'], 'true')) - - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data - displayName: Create EOL Annotation Data Directory - - script: |- cd $(versionsRepoRoot) git pull origin $(gitHubVersionsRepoInfo.branch) @@ -155,13 +144,13 @@ jobs: - script: > $(runImageBuilderCmd) mergeImageInfo - $(imageInfoContainerDir) - $(imageInfoContainerDir)/full-image-info-new.json + imageInfo + imageInfo/full-image-info-new.json $(manifestVariables) $(dryRunArg) --manifest $(manifest) --publish - --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json + --initial-image-info-path imageInfo/full-image-info-orig.json $(commitOverrideArg) condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) displayName: Merge Image Info @@ -178,7 +167,7 @@ jobs: condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true')) args: >- ingestKustoImageInfo - '$(imageInfoContainerDir)/image-info.json' + 'imageInfo/image-info.json' '$(kusto.cluster)' '$(kusto.database)' '$(kusto.imageTable)' @@ -190,36 +179,26 @@ jobs: - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self parameters: - displayName: Generate EOL Annotation Data + displayName: Attach Lifecycle Metadata + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) internalProjectName: internal condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) args: >- - generateEolAnnotationDataForPublish + attachLifecycleMetadata unsupported '${{ parameters.publishConfig.PublishRegistry.server }}' '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}' - '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' - '$(imageInfoContainerDir)/full-image-info-orig.json' - '$(imageInfoContainerDir)/full-image-info-new.json' - $(generateEolAnnotationDataExtraOptions) + 'imageInfo/full-image-info-orig.json' + 'imageInfo/full-image-info-new.json' + $(eolAnnotationsWaitArg) $(dryRunArg) - - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self - parameters: - path: $(Build.ArtifactStagingDirectory)/eol-annotation-data - artifactName: eol-annotation-data-$(System.JobAttempt) - displayName: Publish EOL Annotation Data Artifact - internalProjectName: internal - publicProjectName: public - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - - - template: /eng/docker-tools/templates/steps/annotate-eol-digests.yml@self - parameters: - acr: ${{ parameters.publishConfig.PublishRegistry }} - dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json - - script: > $(runImageBuilderCmd) publishImageInfo - '$(imageInfoContainerDir)/full-image-info-new.json' + 'imageInfo/full-image-info-new.json' '$(gitHubVersionsRepoInfo.userName)' '$(gitHubVersionsRepoInfo.email)' $(gitHubVersionsRepoInfo.authArgs) @@ -248,7 +227,7 @@ jobs: $(runImageBuilderCmd) postPublishNotification '$(publishNotificationRepoName)' '$(branchName)' - '$(imageInfoContainerDir)/image-info.json' + 'imageInfo/image-info.json' $(Build.BuildId) '$(System.AccessToken)' '$(azdoOrgName)' @@ -263,9 +242,7 @@ jobs: --task "🟪 Wait for MCR Doc Ingestion" --task "🟪 Publish Image Info" --task "🟪 Ingest Kusto Image Info" - --task "🟪 Generate EOL Annotation Data" - --task "🟪 Annotate EOL Images (${{ parameters.publishConfig.PublishRegistry.server }})" - --task "🟪 Wait for Annotation Ingestion (${{ parameters.publishConfig.PublishRegistry.server }})" + --task "🟪 Attach Lifecycle Metadata" $(dryRunArg) $(imageBuilder.commonCmdArgs) displayName: Post Publish Notification @@ -285,8 +262,10 @@ jobs: - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self parameters: - path: $(sourceBuildIdOutputDir) - artifactName: source-build-id - displayName: Publish Source Build ID Artifact + path: $(Build.ArtifactStagingDirectory) + artifactName: publish-attempt-$(System.JobAttempt) + displayName: Publish Artifacts internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + # Always upload, even if the pipeline fails or is canceled. + condition: always() diff --git a/eng/docker-tools/templates/stages/build-and-test.yml b/eng/docker-tools/templates/stages/build-and-test.yml index a03804aa89..6d602fe401 100644 --- a/eng/docker-tools/templates/stages/build-and-test.yml +++ b/eng/docker-tools/templates/stages/build-and-test.yml @@ -13,6 +13,8 @@ parameters: # Custom steps that run after ImageBuilder is set up but before the build starts. # Use for build-specific initialization (e.g., setting variables, additional setup). customBuildInitSteps: [] + # Custom steps that modify repository content before it is copied into the ImageBuilder image. + customPreImageBuilderBuildSteps: [] customTestInitSteps: [] sourceBuildPipelineRunId: "" # When true, the Post-Build stage runs even if the Build stage failed (succeededOrFailed). @@ -113,6 +115,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -128,6 +131,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -143,6 +147,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -158,6 +163,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -173,6 +179,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -188,6 +195,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -203,6 +211,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} diff --git a/eng/docker-tools/templates/stages/dotnet/build-and-test.yml b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml index 5b7e23379b..81eee2c9e7 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-and-test.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml @@ -29,6 +29,7 @@ parameters: linuxArmBuildJobTimeout: 60 windowsAmdBuildJobTimeout: 60 customBuildInitSteps: [] + customPreImageBuilderBuildSteps: [] # Test parameters testMatrixType: platformVersionedOs @@ -58,6 +59,7 @@ stages: testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} customInitSteps: ${{ parameters.customInitSteps }} customTestInitSteps: ${{ parameters.customTestInitSteps }} windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} diff --git a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml index 2c924ef0d7..9102d994cb 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml @@ -21,6 +21,7 @@ parameters: linuxArmBuildJobTimeout: 60 windowsAmdBuildJobTimeout: 60 customBuildInitSteps: [] + customPreImageBuilderBuildSteps: [] # Test parameters testMatrixType: platformVersionedOs @@ -60,6 +61,7 @@ stages: linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} customInitSteps: ${{ parameters.customInitSteps }} # Test sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/steps/annotate-eol-digests.yml b/eng/docker-tools/templates/steps/annotate-eol-digests.yml deleted file mode 100644 index 8e2f7571be..0000000000 --- a/eng/docker-tools/templates/steps/annotate-eol-digests.yml +++ /dev/null @@ -1,43 +0,0 @@ -parameters: -- name: acr - type: object -# Path to EOL annotation data JSON file generated by 'generateEolAnnotationData*' command -- name: dataFile - type: string - -steps: - - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests - displayName: Create Annotation Digests Directory - - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Annotate EOL Images (${{ parameters.acr.server }}) - internalProjectName: internal - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - args: >- - annotateEolDigests - "${{ parameters.dataFile }}" - "${{ parameters.acr.server }}" - "${{ parameters.acr.repoPrefix }}" - $(artifactsPath)/annotation-digests/annotation-digests.txt - $(dryRunArg) - - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self - parameters: - path: $(Build.ArtifactStagingDirectory)/annotation-digests - artifactName: annotation-digests-${{ parameters.acr.server }}-$(System.JobAttempt) - displayName: Publish Annotation Digests List (${{ parameters.acr.server }}) - internalProjectName: internal - publicProjectName: public - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self - parameters: - displayName: Wait for Annotation Ingestion (${{ parameters.acr.server }}) - serviceConnections: - - name: mar - id: $(marStatus.serviceConnection.id) - tenantId: $(marStatus.serviceConnection.tenantId) - clientId: $(marStatus.serviceConnection.clientId) - internalProjectName: internal - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'), eq(variables['waitForIngestionEnabled'], 'true')) - args: >- - waitForMarAnnotationIngestion - $(artifactsPath)/annotation-digests/annotation-digests.txt diff --git a/eng/docker-tools/templates/steps/generate-appsettings.yml b/eng/docker-tools/templates/steps/generate-appsettings.yml index b1243e7038..f1b5bf89f2 100644 --- a/eng/docker-tools/templates/steps/generate-appsettings.yml +++ b/eng/docker-tools/templates/steps/generate-appsettings.yml @@ -20,7 +20,17 @@ steps: - powershell: |- # Escape backslashes for JSON compatibility (Windows paths like D:\a\_work become D:\\a\\_work) $artifactStagingDirectory = "${{ parameters.artifactStagingDirectory }}" -replace '\\', '\\' - $appsettingsJsonContent = @" + + if ("$(Build.Reason)" -eq "PullRequest") { + $appsettingsJsonContent = @" + { + "BuildConfiguration": { + "ArtifactStagingDirectory": "$artifactStagingDirectory" + } + } + "@ + } else { + $appsettingsJsonContent = @" { "PublishConfiguration": ${{ convertToJson(parameters.publishConfig) }}, "BuildConfiguration": { @@ -28,7 +38,9 @@ steps: } } "@ + } + Set-Content -Path "appsettings.json" -Value $appsettingsJsonContent Get-Content -Path "appsettings.json" - displayName: Output publish configuration - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), ${{ parameters.condition }}) + displayName: Output ImageBuilder configuration + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/init-common.yml b/eng/docker-tools/templates/steps/init-common.yml index eda572618c..25d22a7b1e 100644 --- a/eng/docker-tools/templates/steps/init-common.yml +++ b/eng/docker-tools/templates/steps/init-common.yml @@ -34,6 +34,11 @@ parameters: type: stepList default: [] +# Custom steps that modify repository content before it is copied into the ImageBuilder image. +- name: customPreImageBuilderBuildSteps + type: stepList + default: [] + # Registry and authentication configuration for publishing images. # Contains server URLs, repo prefixes, subscriptions, and resource groups. # When null, build/publish steps that require registry access will be skipped. @@ -245,3 +250,4 @@ steps: publishConfig: ${{ parameters.publishConfig }} condition: ${{ parameters.condition }} customInitSteps: ${{ parameters.customInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} diff --git a/eng/docker-tools/templates/steps/init-imagebuilder.yml b/eng/docker-tools/templates/steps/init-imagebuilder.yml index b85c62b6c4..1e4f57a1f8 100644 --- a/eng/docker-tools/templates/steps/init-imagebuilder.yml +++ b/eng/docker-tools/templates/steps/init-imagebuilder.yml @@ -21,6 +21,10 @@ parameters: type: stepList default: [] +- name: customPreImageBuilderBuildSteps + type: stepList + default: [] + steps: # Custom ImageBuilder setup (e.g., bootstrap from source) - ${{ if gt(length(parameters.customInitSteps), 0) }}: @@ -67,6 +71,7 @@ steps: # The withrepo image layers the checked-out repository into the ImageBuilder # container at /repo, so ImageBuilder can access manifests and Dockerfiles - ${{ if eq(parameters.dockerClientOS, 'linux') }}: + - ${{ parameters.customPreImageBuilderBuildSteps }} - script: >- docker build -t $(imageNames.imageBuilder.withrepo) diff --git a/eng/docker-tools/templates/steps/publish-artifact.yml b/eng/docker-tools/templates/steps/publish-artifact.yml index 72ce47005a..6095a1dd6b 100644 --- a/eng/docker-tools/templates/steps/publish-artifact.yml +++ b/eng/docker-tools/templates/steps/publish-artifact.yml @@ -11,7 +11,7 @@ parameters: type: string - name: condition type: string - default: 'true' + default: succeeded() steps: - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: @@ -20,9 +20,9 @@ steps: path: ${{ parameters.path }} artifact: ${{ parameters.artifactName }} displayName: ${{ parameters.displayName }} - condition: and(succeeded(), ${{ parameters.condition }}) + condition: ${{ parameters.condition }} - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - publish: ${{ parameters.path }} artifact: ${{ parameters.artifactName }} displayName: ${{ parameters.displayName }} - condition: and(succeeded(), ${{ parameters.condition }}) + condition: ${{ parameters.condition }} diff --git a/eng/docker-tools/templates/variables/common.yml b/eng/docker-tools/templates/variables/common.yml index 680dc12930..b6ce7442e5 100644 --- a/eng/docker-tools/templates/variables/common.yml +++ b/eng/docker-tools/templates/variables/common.yml @@ -16,8 +16,6 @@ variables: value: "" - name: imageBuilderDockerRunExtraOptions value: "" -- name: generateEolAnnotationDataExtraOptions - value: "" - name: productVersionComponents value: 2 - name: imageInfoVariant diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index 46549a7668..5b3599bc7c 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:b2ee4d5f90beae94be5bdde2840d8e1cc52a8b67db50513b49e1c7f640f38045 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:90213836518e8ef7c33153d00abeaafbd7e8d6eb276576920c2cb31b4c7dbd8e imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner From 2fb1ecf5c4f05640695512fb2ec3cfc1bc96d120 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 9 Oct 2026 09:56:01 -0700 Subject: [PATCH 2/2] Update Aspire Dashboard syndication Move legacy syndication to the image-level schema required by the latest ImageBuilder and update its manifest tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9aa5e3ce-8de5-43db-b6e8-1d2e3a104d4c --- manifest.json | 53 +++---------- .../AspireDashboardManifestTests.cs | 77 ++----------------- .../Microsoft.DotNet.Docker.Tests/Manifest.cs | 1 + 3 files changed, 16 insertions(+), 115 deletions(-) diff --git a/manifest.json b/manifest.json index deecf50acc..bb968de2be 100644 --- a/manifest.json +++ b/manifest.json @@ -14216,26 +14216,11 @@ "images": [ { "productVersion": "$(aspire-dashboard|product-version)", + "syndication": "$(syndicatedAspireDashboardRepo)", "sharedTags": { - "$(aspire-dashboard|fixed-tag)": { - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)" - } - }, - "$(aspire-dashboard|minor-tag)": { - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)" - } - }, - "$(aspire-dashboard|major-tag)": { - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)", - "destinationTags": [ - "$(aspire-dashboard|major-tag)", - "latest" - ] - } - }, + "$(aspire-dashboard|fixed-tag)": {}, + "$(aspire-dashboard|minor-tag)": {}, + "$(aspire-dashboard|major-tag)": {}, "latest": {} }, "platforms": [ @@ -14249,22 +14234,13 @@ "osVersion": "azurelinux3.0-distroless", "tags": { "$(aspire-dashboard|fixed-tag)-amd64": { - "docType": "Undocumented", - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)" - } + "docType": "Undocumented" }, "$(aspire-dashboard|minor-tag)-amd64": { - "docType": "Undocumented", - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)" - } + "docType": "Undocumented" }, "$(aspire-dashboard|major-tag)-amd64": { - "docType": "Undocumented", - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)" - } + "docType": "Undocumented" } } }, @@ -14279,22 +14255,13 @@ "osVersion": "azurelinux3.0-distroless", "tags": { "$(aspire-dashboard|fixed-tag)-arm64v8": { - "docType": "Undocumented", - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)" - } + "docType": "Undocumented" }, "$(aspire-dashboard|minor-tag)-arm64v8": { - "docType": "Undocumented", - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)" - } + "docType": "Undocumented" }, "$(aspire-dashboard|major-tag)-arm64v8": { - "docType": "Undocumented", - "syndication": { - "repo": "$(syndicatedAspireDashboardRepo)" - } + "docType": "Undocumented" } }, "variant": "v8" diff --git a/tests/Microsoft.DotNet.Docker.Tests/AspireDashboardManifestTests.cs b/tests/Microsoft.DotNet.Docker.Tests/AspireDashboardManifestTests.cs index a7b571bfd2..aea4a37ebf 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/AspireDashboardManifestTests.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/AspireDashboardManifestTests.cs @@ -2,9 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Collections.Generic; using System.Linq; -using System.Text.RegularExpressions; using Newtonsoft.Json.Linq; using Xunit; @@ -42,11 +40,13 @@ public void LegacySyndication_IsLimitedToAspire13() { int majorVersion = int.Parse(Config.GetVariableValue("aspire-dashboard|major-tag")); JObject aspireDashboardRepo = GetAspireDashboardRepo(); - List<(string Tag, string Repo)> syndications = GetSyndications(aspireDashboardRepo).ToList(); + JArray images = Assert.IsType(aspireDashboardRepo["images"]); + JObject image = Assert.IsType(Assert.Single(images)); + string? syndicatedRepo = image.Value("syndication"); if (majorVersion != 13) { - Assert.Empty(syndications); + Assert.Null(syndicatedRepo); return; } @@ -58,78 +58,11 @@ public void LegacySyndication_IsLimitedToAspire13() Assert.Equal( expectedLegacyRepo, Config.Manifest.Value["variables"]?["syndicatedAspireDashboardRepo"]?.Value()); - Assert.All(syndications, syndication => Assert.Equal(SyndicatedRepoVariable, syndication.Repo)); - - string fixedTag = Config.GetVariableValue("aspire-dashboard|fixed-tag"); - string minorTag = Config.GetVariableValue("aspire-dashboard|minor-tag"); - string majorTag = Config.GetVariableValue("aspire-dashboard|major-tag"); - JObject sharedTags = (JObject)aspireDashboardRepo["images"]!.Single()["sharedTags"]!; - Assert.Equal( - [majorTag, "latest"], - sharedTags["$(aspire-dashboard|major-tag)"]!["syndication"]!["destinationTags"]! - .Values() - .Select(destinationTag => ResolveTag(destinationTag!))); - Assert.Null(sharedTags["latest"]!["syndication"]); - - HashSet expectedTags = - [ - fixedTag, - minorTag, - majorTag, - "latest", - $"{fixedTag}-amd64", - $"{minorTag}-amd64", - $"{majorTag}-amd64", - $"{fixedTag}-arm64v8", - $"{minorTag}-arm64v8", - $"{majorTag}-arm64v8" - ]; - - Assert.Equal(expectedTags.Count, syndications.Count); - Assert.True( - expectedTags.SetEquals(syndications.Select(syndication => syndication.Tag)), - $"Expected syndication for: {string.Join(", ", expectedTags.Order())}"); + Assert.Equal(SyndicatedRepoVariable, syndicatedRepo); } private static JObject GetAspireDashboardRepo() => Config.Manifest.Value["repos"]! .Children() .Single(repo => repo.Value("id") == AspireDashboardId); - - private static IEnumerable<(string Tag, string Repo)> GetSyndications(JObject repo) - { - foreach (JObject image in repo["images"]!.Children()) - { - IEnumerable tagGroups = - [ - (JObject)image["sharedTags"]!, - .. image["platforms"]! - .Children() - .Select(platform => (JObject)platform["tags"]!) - ]; - - foreach (JProperty tag in tagGroups.SelectMany(tagGroup => tagGroup.Properties())) - { - JToken? syndication = tag.Value["syndication"]; - string? syndicatedRepo = syndication?["repo"]?.Value(); - if (syndicatedRepo is not null) - { - IEnumerable destinationTags = syndication!["destinationTags"] is JArray configuredDestinationTags - ? configuredDestinationTags.Values().Select(destinationTag => destinationTag!) - : [tag.Name]; - - foreach (string destinationTag in destinationTags) - { - yield return (ResolveTag(destinationTag), syndicatedRepo); - } - } - } - } - } - - private static string ResolveTag(string tag) => - Regex.Replace( - tag, - @"\$\((?[\w:\-.|]+)\)", - match => Config.GetVariableValue(match.Groups["variable"].Value)); } diff --git a/tests/Microsoft.DotNet.Docker.Tests/Manifest.cs b/tests/Microsoft.DotNet.Docker.Tests/Manifest.cs index e946cb17f5..4b46fbdc24 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/Manifest.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/Manifest.cs @@ -24,6 +24,7 @@ public class Image { public Dictionary SharedTags { get; set; } = new Dictionary(); public string ProductVersion { get; set; } = string.Empty; + public string Syndication { get; set; } = string.Empty; public List Platforms { get; set; } = new List(); }