From e8d6522034571fbe3f9b9833e3a01b2b02051d95 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 13 Oct 2026 20:54:58 -0700 Subject: [PATCH 1/6] build(release): cross-compile the release tools in presubmit Presubmit only built the release tools for each CI host's own platform, so 8 of the 11 platforms in dev/platforms were only built when the release workflow ran. A new target, //dev/release_artifacts:cross_compiled_artifacts, builds the tools for the platforms that the host can cross-compile for, with a split transition on --platforms. BazelCI's default builds of //... on Linux x86_64, macOS arm64, and Windows x86_64 thus cover all 11 platforms without new jobs. BazelCI has no Windows Arm machines, so Windows x86_64 hosts also get a Rust toolchain for Arm Windows. Arm Windows needs Visual Studio's optional ARM64 build tools. A config setting detects them through toolchain resolution, so Windows machines without them skip that platform instead of failing. BazelCI's Windows jobs set --//dev/release_artifacts:require_msvc_arm64 to make missing tools an error there. On Linux x86_64, developers' machines included, builds of //... now also cross-compile the 5 musl variants. The release workflow still builds aarch64-pc-windows-msvc and x86_64-apple-darwin natively. Work towards #4216 --- .bazelci/presubmit.yml | 7 ++ MODULE.bazel | 19 ++- dev/platforms/BUILD.bazel | 9 +- dev/release_artifacts/BUILD.bazel | 97 ++++++++++++++ .../cc_toolchain_available_config_setting.bzl | 118 ++++++++++++++++++ .../cross_compiled_files.bzl | 46 +++++++ 6 files changed, 289 insertions(+), 7 deletions(-) create mode 100644 dev/release_artifacts/cc_toolchain_available_config_setting.bzl create mode 100644 dev/release_artifacts/cross_compiled_files.bzl diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index dc13b38e14..8fe871f418 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -298,8 +298,15 @@ tasks: name: "Default: Windows, Bazel {bazel}" platform: windows bazel: ${{ bazel }} + build_flags: + <<: *reusable_config_build_flags + # Fail if the image lacks Visual Studio's ARM64 build tools; see + # dev/release_artifacts/BUILD.bazel. Tests set it too, so that they reuse + # the build's analysis cache. + ? "--//dev/release_artifacts:require_msvc_arm64" test_flags: ? "--test_tag_filters=-integration-test,-fix-windows" + ? "--//dev/release_artifacts:require_msvc_arm64" rbe_min: <<: *minimum_supported_version <<: *reusable_config diff --git a/MODULE.bazel b/MODULE.bazel index 532c0c1832..989aa89965 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -230,10 +230,11 @@ register_toolchains( dev_dependency = True, ) -# A Rust toolchain to cross-compile the release artifacts for 32-bit Windows -# from Windows x86_64, with MSVC's x86 tools. Unlike the Linux ones, it isn't -# registered here: no default toolchain targets 32-bit Windows, and rules_rust -# registers it. +# Rust toolchains to cross-compile the release artifacts for 32-bit and Arm +# Windows from Windows x86_64, with MSVC's cross tools; the Arm one needs +# Visual Studio's optional ARM64 build tools. Unlike the Linux ones, they +# aren't registered here: no default toolchain targets these platforms from +# Windows x86_64, and rules_rust registers them. rust.repository_set( # As above, its x86_64 toolchain's name must sort after the default one's. # On Windows, rules_rust names toolchains after a hash of the full name: @@ -250,6 +251,16 @@ rust.repository_set( versions = ["1.97.1"], ) +# Another target for the same set; the first tag sets the shared attributes. +rust.repository_set( + name = "rust_release_windows", + target_compatible_with = [ + "@platforms//cpu:aarch64", + "@platforms//os:windows", + ], + target_triple = "aarch64-pc-windows-msvc", +) + # Use the Rust-based build tools when developing rules_python itself. register_toolchains( "//dev/dev_only_toolchains:exe_zip_maker_toolchain", diff --git a/dev/platforms/BUILD.bazel b/dev/platforms/BUILD.bazel index 90657fb1a3..0e57c210d6 100644 --- a/dev/platforms/BUILD.bazel +++ b/dev/platforms/BUILD.bazel @@ -5,9 +5,12 @@ # aarch64 macOS file. s390x isn't covered yet: Rust ships no musl std for it, # and a static glibc build needs a C cross toolchain. # -# Linux x86_64 hosts cross-compile for the musl ones, and Windows x86_64 hosts -# for i686-pc-windows-msvc, with the Rust toolchains in MODULE.bazel. The -# others are built on hosts that match them. +# Linux x86_64 hosts can cross-compile for the musl ones, Windows x86_64 hosts +# for the other Windows ones, and macOS arm64 hosts for x86_64-apple-darwin, +# with the Rust toolchains in MODULE.bazel and rules_rust's defaults. +# //dev/release_artifacts:cross_compiled_artifacts cross-compiles them all; the +# release workflow builds aarch64-pc-windows-msvc and x86_64-apple-darwin on +# hosts that match them instead. package(default_visibility = ["//:__subpackages__"]) diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index 4c7f904b8c..58c665025d 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel @@ -1,5 +1,11 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load( + ":cc_toolchain_available_config_setting.bzl", + "cc_toolchain_available_config_setting", +) +load(":cross_compiled_files.bzl", "cross_compiled_files") load(":release_files.bzl", "release_files") package(default_visibility = ["//:__subpackages__"]) @@ -14,8 +20,99 @@ release_files( tags = [] if BZLMOD_ENABLED else ["manual"], ) +# artifacts_for_release for the other platforms in dev/platforms that the host +# can cross-compile for, so that builds of //... on each host cover them. The +# select() matches the target platform, which is the host's unless --platforms +# is set. +# +# To reproduce a failure, build artifacts_for_release with +# --platforms=//dev/platforms:; see build.sh. If one breaks presubmit +# without a related change, tag this target "manual" until it's fixed. +cross_compiled_files( + name = "cross_compiled_artifacts", + src = ":artifacts_for_release", + platforms = select({ + ":is_linux_x86_64": [ + "//dev/platforms:aarch64-unknown-linux-musl", + "//dev/platforms:armv7-unknown-linux-musleabihf", + "//dev/platforms:i686-unknown-linux-musl", + "//dev/platforms:powerpc64le-unknown-linux-musl", + "//dev/platforms:riscv64gc-unknown-linux-musl", + ], + ":is_macos_aarch64": ["//dev/platforms:x86_64-apple-darwin"], + ":is_windows_x86_64": ["//dev/platforms:i686-pc-windows-msvc"], + "//conditions:default": [], + }) + select({ + ":is_windows_x86_64_with_msvc_arm64": [ + "//dev/platforms:aarch64-pc-windows-msvc", + ], + "//conditions:default": [], + }), + # See artifacts_for_release. + tags = [] if BZLMOD_ENABLED else ["manual"], +) + +# Matches on Windows x86_64 if Visual Studio's optional ARM64 build tools +# (Microsoft.VisualStudio.Component.VC.Tools.ARM64) are installed, so that the +# C++ toolchain for Windows arm64 works. Without them, aarch64-pc-windows-msvc +# is skipped, or the build fails if require_msvc_arm64 is set. +cc_toolchain_available_config_setting( + name = "is_windows_x86_64_with_msvc_arm64", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + ], + platform = "//dev/platforms:aarch64-pc-windows-msvc", + required_flag = ":require_msvc_arm64", +) + +# BazelCI's Windows jobs set this, since their images have the ARM64 build +# tools, so that losing them fails the build instead of going unnoticed. +bool_flag( + name = "require_msvc_arm64", + build_setting_default = False, +) + +config_setting( + name = "is_linux_x86_64", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], +) + +config_setting( + name = "is_macos_aarch64", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + ], +) + +config_setting( + name = "is_windows_x86_64", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + ], +) + bzl_library( name = "release_files", srcs = ["release_files.bzl"], deps = ["//python/private:common_labels"], ) + +bzl_library( + name = "cc_toolchain_available_config_setting", + srcs = ["cc_toolchain_available_config_setting.bzl"], + deps = [ + "@bazel_skylib//rules:common_settings", + "@rules_cc//cc:find_cc_toolchain_bzl", + ], +) + +bzl_library( + name = "cross_compiled_files", + srcs = ["cross_compiled_files.bzl"], +) diff --git a/dev/release_artifacts/cc_toolchain_available_config_setting.bzl b/dev/release_artifacts/cc_toolchain_available_config_setting.bzl new file mode 100644 index 0000000000..ce09e4695b --- /dev/null +++ b/dev/release_artifacts/cc_toolchain_available_config_setting.bzl @@ -0,0 +1,118 @@ +"""Config setting for whether a C++ toolchain can build for a platform.""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load( + "@rules_cc//cc:find_cc_toolchain.bzl", + "find_cc_toolchain", + "use_cc_toolchain", +) + +_ProbeInfo = provider( + doc = "Whether a usable C++ toolchain was found.", + fields = {"available": "bool: whether a usable C++ toolchain was found."}, +) + +def _probe_impl(ctx): + cc_toolchain = find_cc_toolchain(ctx, mandatory = False) + + # rules_cc registers its MSVC toolchains even if Visual Studio lacks the + # tools for their CPU, and then runs an error script instead of them. + available = cc_toolchain != None and ( + "vc_installation_error" not in cc_toolchain.compiler_executable + ) + return [_ProbeInfo(available = available)] + +_probe = rule( + implementation = _probe_impl, + toolchains = use_cc_toolchain(mandatory = False), +) + +def _platform_transition_impl(_settings, attr): + return {"//command_line_option:platforms": str(attr.platform)} + +_platform_transition = transition( + implementation = _platform_transition_impl, + inputs = [], + outputs = ["//command_line_option:platforms"], +) + +def _cc_toolchain_available_impl(ctx): + for constraint in ctx.attr.constraint_values: + if not ctx.target_platform_has_constraint( + constraint[platform_common.ConstraintValueInfo], + ): + return [config_common.FeatureFlagInfo(value = "no")] + + # With a transition, ctx.attr.probe is a list. + available = ctx.attr.probe[0][_ProbeInfo].available + required = ( + ctx.attr.required_flag != None and + ctx.attr.required_flag[BuildSettingInfo].value + ) + if required and not available: + fail("{} is set, but no usable C++ toolchain can build for {}.".format( + ctx.attr.required_flag.label, + ctx.attr.platform.label, + )) + return [config_common.FeatureFlagInfo(value = "yes" if available else "no")] + +_cc_toolchain_available = rule( + implementation = _cc_toolchain_available_impl, + attrs = { + "constraint_values": attr.label_list( + providers = [platform_common.ConstraintValueInfo], + ), + "platform": attr.label( + mandatory = True, + providers = [platform_common.PlatformInfo], + ), + "probe": attr.label( + cfg = _platform_transition, + mandatory = True, + providers = [_ProbeInfo], + ), + "required_flag": attr.label(providers = [BuildSettingInfo]), + }, +) + +def cc_toolchain_available_config_setting( + name, + platform, + constraint_values = [], + required_flag = None, + visibility = None): + """Matches if a usable C++ toolchain can build for `platform`. + + Toolchain resolution for `platform` finds the toolchain, so the result + reflects the registered toolchains and execution platforms. A toolchain + that runs rules_cc's error script instead of a compiler isn't usable. + + Args: + name: The name of the `config_setting`. + platform: The platform to look for a C++ toolchain for. + constraint_values: Constraint values that the target platform must + also have, as in `config_setting`. + required_flag: The label of a `bool_flag`. When it's true and the + target platform has `constraint_values`, a missing toolchain fails + the build instead of not matching. + visibility: The visibility of the `config_setting`. + """ + _probe( + name = name + "_probe", + tags = ["manual"], + visibility = ["//visibility:private"], + ) + _cc_toolchain_available( + name = name + "_flag", + constraint_values = constraint_values, + platform = platform, + probe = ":" + name + "_probe", + required_flag = required_flag, + tags = ["manual"], + visibility = ["//visibility:private"], + ) + native.config_setting( + name = name, + flag_values = {":" + name + "_flag": "yes"}, + visibility = visibility, + ) diff --git a/dev/release_artifacts/cross_compiled_files.bzl b/dev/release_artifacts/cross_compiled_files.bzl new file mode 100644 index 0000000000..e05ba7bcb8 --- /dev/null +++ b/dev/release_artifacts/cross_compiled_files.bzl @@ -0,0 +1,46 @@ +"""Rule to build a target for several platforms in one build.""" + +def _platforms_transition_impl(_settings, attr): + # A split transition: one configuration per platform, keyed by its label. + return { + str(platform): {"//command_line_option:platforms": str(platform)} + for platform in attr.platforms + } + +_platforms_transition = transition( + implementation = _platforms_transition_impl, + inputs = [], + outputs = ["//command_line_option:platforms"], +) + +def _cross_compiled_files_impl(ctx): + # Without platforms, the transition leaves src in this target's + # configuration, so skip it. + if not ctx.attr.platforms: + return [DefaultInfo()] + + # With a transition, ctx.attr.src is a list: one target per platform. + files = [src[DefaultInfo].files for src in ctx.attr.src] + return [DefaultInfo(files = depset(transitive = files))] + +cross_compiled_files = rule( + implementation = _cross_compiled_files_impl, + doc = """ +Builds `src` for each platform in `platforms`, in one build. + +`--platforms` selects one target platform for the whole build; this builds +`src` for every platform in `platforms` instead. With no platforms, it builds +nothing. +""", + attrs = { + "platforms": attr.label_list( + doc = "The platforms to build `src` for.", + providers = [platform_common.PlatformInfo], + ), + "src": attr.label( + cfg = _platforms_transition, + doc = "The target to build.", + mandatory = True, + ), + }, +) From dd4556bffcabcc7398fbbfd2877c69e4efba81e1 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 13 Oct 2026 23:29:28 -0700 Subject: [PATCH 2/6] build(release): set the ARM64 strict flag through --config=ci Name the strict MSVC ARM64 flag in the shared .bazelrc's ci config, which BazelCI's default Windows jobs now pass, instead of in presubmit.yml. Also recognize all of rules_cc's error scripts in the toolchain probe, and apply wording fixes from code review. --- .bazelci/presubmit.yml | 8 +++--- .bazelrc | 9 ++++-- RELEASING.md | 3 ++ dev/platforms/BUILD.bazel | 5 ++-- dev/release_artifacts/BUILD.bazel | 20 ++++++++----- .../cc_toolchain_available_config_setting.bzl | 28 +++++++++++-------- .../cross_compiled_files.bzl | 5 ++-- 7 files changed, 48 insertions(+), 30 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 8fe871f418..23560b270c 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -300,13 +300,13 @@ tasks: bazel: ${{ bazel }} build_flags: <<: *reusable_config_build_flags - # Fail if the image lacks Visual Studio's ARM64 build tools; see - # dev/release_artifacts/BUILD.bazel. Tests set it too, so that they reuse + # Among other things, makes missing Visual Studio ARM64 build tools an + # error; see .bazelrc. test_flags repeat it so that `bazel test` reuses # the build's analysis cache. - ? "--//dev/release_artifacts:require_msvc_arm64" + ? "--config=ci" test_flags: ? "--test_tag_filters=-integration-test,-fix-windows" - ? "--//dev/release_artifacts:require_msvc_arm64" + ? "--config=ci" rbe_min: <<: *minimum_supported_version <<: *reusable_config diff --git a/.bazelrc b/.bazelrc index 12d1b33865..c1fb9043b2 100644 --- a/.bazelrc +++ b/.bazelrc @@ -48,13 +48,18 @@ common --disk_cache=~/.cache/bazel/bazel-disk-cache common --repository_cache=~/.cache/bazel/bazel-repo-cache # Hardlinks save disk space and copy time when unpacking repositories common --experimental_repository_cache_hardlinks=true -# To work around bug on bazel 7 -common:ci --experimental_repository_cache_hardlinks=false # Drop `experimental_` prefix once Bazel 7 is no longer supported common --experimental_downloader_config=downloader_config.cfg common --http_timeout_scaling=10.0 common --experimental_repository_downloader_retries=10 +# Only BazelCI's default Windows jobs pass --config=ci (.bazelci/presubmit.yml). +# To work around bug on bazel 7 +common:ci --experimental_repository_cache_hardlinks=false +# Make missing Visual Studio ARM64 build tools an error on Windows x86_64; see +# dev/release_artifacts/BUILD.bazel. +common:ci --//dev/release_artifacts:require_msvc_arm64=true + # Additional config to use for readthedocs builds. # See .readthedocs.yml for additional flags that can only be determined from # the runtime environment. diff --git a/RELEASING.md b/RELEASING.md index cbd9bc1b0a..2bf5e59fd0 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -73,6 +73,9 @@ The release workflow also builds `//dev/release_artifacts:artifacts_for_release` for each platform to verify it builds. For now, the files aren't attached to the release, so a failed "Build tools" job doesn't block it. +Presubmit builds it for every platform too: natively on each CI host, and +through `//dev/release_artifacts:cross_compiled_artifacts` for the others. + ### Manually publishing to PyPI If PyPI publishing failed or was skipped during the main release, the PyPI diff --git a/dev/platforms/BUILD.bazel b/dev/platforms/BUILD.bazel index 0e57c210d6..17f19cc357 100644 --- a/dev/platforms/BUILD.bazel +++ b/dev/platforms/BUILD.bazel @@ -6,8 +6,9 @@ # and a static glibc build needs a C cross toolchain. # # Linux x86_64 hosts can cross-compile for the musl ones, Windows x86_64 hosts -# for the other Windows ones, and macOS arm64 hosts for x86_64-apple-darwin, -# with the Rust toolchains in MODULE.bazel and rules_rust's defaults. +# for the other Windows ones (Arm needs Visual Studio's ARM64 build tools), and +# macOS arm64 hosts for x86_64-apple-darwin, with the Rust toolchains in +# MODULE.bazel and rules_rust's defaults. # //dev/release_artifacts:cross_compiled_artifacts cross-compiles them all; the # release workflow builds aarch64-pc-windows-msvc and x86_64-apple-darwin on # hosts that match them instead. diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index 58c665025d..bd226a1ee9 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel @@ -22,12 +22,15 @@ release_files( # artifacts_for_release for the other platforms in dev/platforms that the host # can cross-compile for, so that builds of //... on each host cover them. The -# select() matches the target platform, which is the host's unless --platforms -# is set. +# select()s match the target platform, which is the host's unless --platforms +# is set. Arm Windows needs its own select(): Windows x86_64 hosts can match +# both is_windows_x86_64 and is_windows_x86_64_with_msvc_arm64, and in a single +# select() that ambiguous match would be an error. # # To reproduce a failure, build artifacts_for_release with -# --platforms=//dev/platforms:; see build.sh. If one breaks presubmit -# without a related change, tag this target "manual" until it's fixed. +# --platforms=//dev/platforms:; see build.sh. If a platform's build +# breaks presubmit without a related change, tag this target "manual" until +# it's fixed. cross_compiled_files( name = "cross_compiled_artifacts", src = ":artifacts_for_release", @@ -55,7 +58,8 @@ cross_compiled_files( # Matches on Windows x86_64 if Visual Studio's optional ARM64 build tools # (Microsoft.VisualStudio.Component.VC.Tools.ARM64) are installed, so that the # C++ toolchain for Windows arm64 works. Without them, aarch64-pc-windows-msvc -# is skipped, or the build fails if require_msvc_arm64 is set. +# is skipped, or the build fails if require_msvc_arm64 is set. After installing +# them, run `bazel fetch --configure --force` so that rules_cc detects them. cc_toolchain_available_config_setting( name = "is_windows_x86_64_with_msvc_arm64", constraint_values = [ @@ -66,8 +70,10 @@ cc_toolchain_available_config_setting( required_flag = ":require_msvc_arm64", ) -# BazelCI's Windows jobs set this, since their images have the ARM64 build -# tools, so that losing them fails the build instead of going unnoticed. +# BazelCI's default Windows jobs (the `windows` task in .bazelci/presubmit.yml) +# set this through --config=ci (see .bazelrc), since their images have the +# ARM64 build tools, so that losing them fails the build instead of going +# unnoticed. bool_flag( name = "require_msvc_arm64", build_setting_default = False, diff --git a/dev/release_artifacts/cc_toolchain_available_config_setting.bzl b/dev/release_artifacts/cc_toolchain_available_config_setting.bzl index ce09e4695b..44122da195 100644 --- a/dev/release_artifacts/cc_toolchain_available_config_setting.bzl +++ b/dev/release_artifacts/cc_toolchain_available_config_setting.bzl @@ -8,17 +8,18 @@ load( ) _ProbeInfo = provider( - doc = "Whether a usable C++ toolchain was found.", + doc = "The result of looking for a C++ toolchain.", fields = {"available": "bool: whether a usable C++ toolchain was found."}, ) def _probe_impl(ctx): cc_toolchain = find_cc_toolchain(ctx, mandatory = False) - # rules_cc registers its MSVC toolchains even if Visual Studio lacks the - # tools for their CPU, and then runs an error script instead of them. + # rules_cc registers its Windows toolchains even if their tools are + # missing, and then runs an error script instead of them, e.g. + # vc_installation_error_arm64.bat or clang_installation_error.bat. available = cc_toolchain != None and ( - "vc_installation_error" not in cc_toolchain.compiler_executable + "installation_error" not in cc_toolchain.compiler_executable ) return [_ProbeInfo(available = available)] @@ -50,10 +51,10 @@ def _cc_toolchain_available_impl(ctx): ctx.attr.required_flag[BuildSettingInfo].value ) if required and not available: - fail("{} is set, but no usable C++ toolchain can build for {}.".format( - ctx.attr.required_flag.label, - ctx.attr.platform.label, - )) + fail(( + "{} is set, but no usable C++ toolchain can build for {}. " + + "Install one, or unset the flag." + ).format(ctx.attr.required_flag.label, ctx.attr.platform.label)) return [config_common.FeatureFlagInfo(value = "yes" if available else "no")] _cc_toolchain_available = rule( @@ -78,14 +79,15 @@ _cc_toolchain_available = rule( def cc_toolchain_available_config_setting( name, platform, - constraint_values = [], + constraint_values = None, required_flag = None, visibility = None): """Matches if a usable C++ toolchain can build for `platform`. Toolchain resolution for `platform` finds the toolchain, so the result reflects the registered toolchains and execution platforms. A toolchain - that runs rules_cc's error script instead of a compiler isn't usable. + that runs one of rules_cc's error scripts instead of a compiler isn't + usable. Args: name: The name of the `config_setting`. @@ -93,10 +95,12 @@ def cc_toolchain_available_config_setting( constraint_values: Constraint values that the target platform must also have, as in `config_setting`. required_flag: The label of a `bool_flag`. When it's true and the - target platform has `constraint_values`, a missing toolchain fails - the build instead of not matching. + target platform has `constraint_values`, a missing or unusable + toolchain fails the build instead of not matching. visibility: The visibility of the `config_setting`. """ + constraint_values = constraint_values or [] + _probe( name = name + "_probe", tags = ["manual"], diff --git a/dev/release_artifacts/cross_compiled_files.bzl b/dev/release_artifacts/cross_compiled_files.bzl index e05ba7bcb8..a2688439f7 100644 --- a/dev/release_artifacts/cross_compiled_files.bzl +++ b/dev/release_artifacts/cross_compiled_files.bzl @@ -28,9 +28,8 @@ cross_compiled_files = rule( doc = """ Builds `src` for each platform in `platforms`, in one build. -`--platforms` selects one target platform for the whole build; this builds -`src` for every platform in `platforms` instead. With no platforms, it builds -nothing. +`--platforms` selects only one target platform per build. With no platforms, +this builds nothing. """, attrs = { "platforms": attr.label_list( From ccda8a66ceaa9c85b88756caba945cb4a0eee1fc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 13 Oct 2026 23:57:17 -0700 Subject: [PATCH 3/6] build(release): move cross_compiled_files into release_files.bzl Keep the release build rules in one file: the cross_compiled_files rule and its transition move unchanged into release_files.bzl, and cross_compiled_files.bzl and its bzl_library are removed. --- dev/release_artifacts/BUILD.bazel | 8 +--- .../cross_compiled_files.bzl | 45 ------------------ dev/release_artifacts/release_files.bzl | 46 ++++++++++++++++++- 3 files changed, 46 insertions(+), 53 deletions(-) delete mode 100644 dev/release_artifacts/cross_compiled_files.bzl diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index bd226a1ee9..0785760c27 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel @@ -5,8 +5,7 @@ load( ":cc_toolchain_available_config_setting.bzl", "cc_toolchain_available_config_setting", ) -load(":cross_compiled_files.bzl", "cross_compiled_files") -load(":release_files.bzl", "release_files") +load(":release_files.bzl", "cross_compiled_files", "release_files") package(default_visibility = ["//:__subpackages__"]) @@ -117,8 +116,3 @@ bzl_library( "@rules_cc//cc:find_cc_toolchain_bzl", ], ) - -bzl_library( - name = "cross_compiled_files", - srcs = ["cross_compiled_files.bzl"], -) diff --git a/dev/release_artifacts/cross_compiled_files.bzl b/dev/release_artifacts/cross_compiled_files.bzl deleted file mode 100644 index a2688439f7..0000000000 --- a/dev/release_artifacts/cross_compiled_files.bzl +++ /dev/null @@ -1,45 +0,0 @@ -"""Rule to build a target for several platforms in one build.""" - -def _platforms_transition_impl(_settings, attr): - # A split transition: one configuration per platform, keyed by its label. - return { - str(platform): {"//command_line_option:platforms": str(platform)} - for platform in attr.platforms - } - -_platforms_transition = transition( - implementation = _platforms_transition_impl, - inputs = [], - outputs = ["//command_line_option:platforms"], -) - -def _cross_compiled_files_impl(ctx): - # Without platforms, the transition leaves src in this target's - # configuration, so skip it. - if not ctx.attr.platforms: - return [DefaultInfo()] - - # With a transition, ctx.attr.src is a list: one target per platform. - files = [src[DefaultInfo].files for src in ctx.attr.src] - return [DefaultInfo(files = depset(transitive = files))] - -cross_compiled_files = rule( - implementation = _cross_compiled_files_impl, - doc = """ -Builds `src` for each platform in `platforms`, in one build. - -`--platforms` selects only one target platform per build. With no platforms, -this builds nothing. -""", - attrs = { - "platforms": attr.label_list( - doc = "The platforms to build `src` for.", - providers = [platform_common.PlatformInfo], - ), - "src": attr.label( - cfg = _platforms_transition, - doc = "The target to build.", - mandatory = True, - ), - }, -) diff --git a/dev/release_artifacts/release_files.bzl b/dev/release_artifacts/release_files.bzl index b5bde5f65f..ae6563c6c4 100644 --- a/dev/release_artifacts/release_files.bzl +++ b/dev/release_artifacts/release_files.bzl @@ -1,4 +1,4 @@ -"""Macro to build the files for a release and list where they are.""" +"""Macro and rule to build the files for a release and list where they are.""" load("//python/private:common_labels.bzl", "labels") # buildifier: disable=bzl-visibility @@ -102,3 +102,47 @@ def release_files(name, srcs, **kwargs): }), **kwargs ) + +def _platforms_transition_impl(_settings, attr): + # A split transition: one configuration per platform, keyed by its label. + return { + str(platform): {"//command_line_option:platforms": str(platform)} + for platform in attr.platforms + } + +_platforms_transition = transition( + implementation = _platforms_transition_impl, + inputs = [], + outputs = ["//command_line_option:platforms"], +) + +def _cross_compiled_files_impl(ctx): + # Without platforms, the transition leaves src in this target's + # configuration, so skip it. + if not ctx.attr.platforms: + return [DefaultInfo()] + + # With a transition, ctx.attr.src is a list: one target per platform. + files = [src[DefaultInfo].files for src in ctx.attr.src] + return [DefaultInfo(files = depset(transitive = files))] + +cross_compiled_files = rule( + implementation = _cross_compiled_files_impl, + doc = """ +Builds `src` for each platform in `platforms`, in one build. + +`--platforms` selects only one target platform per build. With no platforms, +this builds nothing. +""", + attrs = { + "platforms": attr.label_list( + doc = "The platforms to build `src` for.", + providers = [platform_common.PlatformInfo], + ), + "src": attr.label( + cfg = _platforms_transition, + doc = "The target to build.", + mandatory = True, + ), + }, +) From 56c566cd6c0e4c05f38c1e152db89fd4c6d6cf2b Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 14 Oct 2026 00:12:06 -0700 Subject: [PATCH 4/6] tests: build the cross-compiled release files in test runs BazelCI's test step and --config=fast-tests build only tests and their dependencies, so add a build_test for //dev/release_artifacts:cross_compiled_artifacts. --- tests/dev/BUILD.bazel | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/dev/BUILD.bazel diff --git a/tests/dev/BUILD.bazel b/tests/dev/BUILD.bazel new file mode 100644 index 0000000000..5ed87cd1df --- /dev/null +++ b/tests/dev/BUILD.bazel @@ -0,0 +1,12 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility + +# Builds the cross-compiled release files in test runs too, e.g. with +# --build_tests_only. +build_test( + name = "cross_compiled_artifacts_build_test", + # Under WORKSPACE, rules_rust is a stub, so the target can't build; see + # //dev/release_artifacts:artifacts_for_release. + tags = [] if BZLMOD_ENABLED else ["manual"], + targets = ["//dev/release_artifacts:cross_compiled_artifacts"], +) From 020b5fa42b9fb3e32d8a71c9ed359dcc3c55f896 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 14 Oct 2026 16:45:18 -0700 Subject: [PATCH 5/6] ci: pass --config=ci in the default and RBE jobs Only the Windows jobs passed --config=ci, so the CI settings in .bazelrc applied to them alone. The shared build and test flag sets now include it, so every default and RBE bzlmod job uses it in both steps, which keeps `bazel test` reusing the build's analysis cache. A new reusable_config_test_flags anchor mirrors the build one. The Windows and rolling jobs merge it; the RBE jobs, which set their own flags, list --config=ci explicitly. --- .bazelci/presubmit.yml | 21 ++++++++++++--------- .bazelrc | 3 ++- dev/release_artifacts/BUILD.bazel | 7 +++---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 23560b270c..0c8938656f 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -40,11 +40,16 @@ buildifier: ? "--keep_going" ? "--build_tag_filters=-integration-test" ? "--verbose_failures" + # Among other things, makes missing Visual Studio ARM64 build tools an + # error on Windows; see .bazelrc. test_flags repeat it so that `bazel test` + # reuses the build's analysis cache. + ? "--config=ci" test_targets: - "--" - "..." - test_flags: + test_flags: &reusable_config_test_flags ? "--test_tag_filters=-integration-test" + ? "--config=ci" .common_workspace_flags_min_bazel: &common_workspace_flags_min_bazel build_flags: ? "--experimental_repository_cache_hardlinks=false" @@ -245,8 +250,8 @@ tasks: - "--" - "//tests/..." test_flags: + <<: *reusable_config_test_flags ? "--keep_going" - ? "--test_tag_filters=-integration-test" ubuntu_workspace: <<: *reusable_config <<: *common_workspace_flags @@ -298,15 +303,10 @@ tasks: name: "Default: Windows, Bazel {bazel}" platform: windows bazel: ${{ bazel }} - build_flags: - <<: *reusable_config_build_flags - # Among other things, makes missing Visual Studio ARM64 build tools an - # error; see .bazelrc. test_flags repeat it so that `bazel test` reuses - # the build's analysis cache. - ? "--config=ci" test_flags: + <<: *reusable_config_test_flags + # Bazel uses the last --test_tag_filters, so this replaces the merged one. ? "--test_tag_filters=-integration-test,-fix-windows" - ? "--config=ci" rbe_min: <<: *minimum_supported_version <<: *reusable_config @@ -320,6 +320,7 @@ tasks: # build kite cc toolchain. ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" ? "--build_tag_filters=-docs" + ? "--config=ci" test_flags: ? "--test_tag_filters=-integration-test,-acceptance-test,-docs" # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, @@ -327,6 +328,7 @@ tasks: # on Bazel 5.4 and earlier. To workaround this, manually specify the # build kite cc toolchain. ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" + ? "--config=ci" rbe: <<: *reusable_config name: "RBE: Ubuntu" @@ -337,6 +339,7 @@ tasks: test_flags: ? "--test_tag_filters=-integration-test,-acceptance-test" ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" + ? "--config=ci" integration_test_build_file_generation_ubuntu_minimum_supported_workspace: <<: *minimum_supported_version diff --git a/.bazelrc b/.bazelrc index c1fb9043b2..24aba939a6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -53,7 +53,8 @@ common --experimental_downloader_config=downloader_config.cfg common --http_timeout_scaling=10.0 common --experimental_repository_downloader_retries=10 -# Only BazelCI's default Windows jobs pass --config=ci (.bazelci/presubmit.yml). +# BazelCI's default and RBE bzlmod jobs pass --config=ci; see +# .bazelci/presubmit.yml. # To work around bug on bazel 7 common:ci --experimental_repository_cache_hardlinks=false # Make missing Visual Studio ARM64 build tools an error on Windows x86_64; see diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index 0785760c27..d81b5294b1 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel @@ -69,10 +69,9 @@ cc_toolchain_available_config_setting( required_flag = ":require_msvc_arm64", ) -# BazelCI's default Windows jobs (the `windows` task in .bazelci/presubmit.yml) -# set this through --config=ci (see .bazelrc), since their images have the -# ARM64 build tools, so that losing them fails the build instead of going -# unnoticed. +# BazelCI sets this through --config=ci (see .bazelrc). Its Windows images have +# the ARM64 build tools, so losing them fails the build instead of going +# unnoticed. It has no effect unless the target platform is Windows x86_64. bool_flag( name = "require_msvc_arm64", build_setting_default = False, From 70984afec5b9b722b59bae5c64e63e0182ee2f2a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 14 Oct 2026 21:13:18 -0700 Subject: [PATCH 6/6] ci: merge the shared flags into the RBE jobs The RBE jobs listed --config=ci by hand. Merging the shared flag sets instead keeps them in sync with the other jobs; rbe_min thereby also gains --keep_going and --verbose_failures. --- .bazelci/presubmit.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 0c8938656f..219a1e5b7b 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -35,6 +35,9 @@ buildifier: # As a regression test for #225, check that wheel targets still build when # their package path is qualified with the repo name. - "@rules_python//examples/wheel/..." + # Tasks can merge these into their own build_flags or test_flags with `<<:`. + # To change a merged flag, such as a tag filter, list it again: Bazel uses the + # last value. build_flags: &reusable_config_build_flags ? "--experimental_repository_cache_hardlinks=false" ? "--keep_going" @@ -305,7 +308,6 @@ tasks: bazel: ${{ bazel }} test_flags: <<: *reusable_config_test_flags - # Bazel uses the last --test_tag_filters, so this replaces the merged one. ? "--test_tag_filters=-integration-test,-fix-windows" rbe_min: <<: *minimum_supported_version @@ -313,22 +315,21 @@ tasks: name: "RBE: Ubuntu, minimum Bazel" platform: rbe_ubuntu2404 build_flags: - ? "--experimental_repository_cache_hardlinks=false" + <<: *reusable_config_build_flags # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, # which prevents cc toolchain autodetection from working correctly # on Bazel 5.4 and earlier. To workaround this, manually specify the # build kite cc toolchain. ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" ? "--build_tag_filters=-docs" - ? "--config=ci" test_flags: + <<: *reusable_config_test_flags ? "--test_tag_filters=-integration-test,-acceptance-test,-docs" # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, # which prevents cc toolchain autodetection from working correctly # on Bazel 5.4 and earlier. To workaround this, manually specify the # build kite cc toolchain. ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" - ? "--config=ci" rbe: <<: *reusable_config name: "RBE: Ubuntu" @@ -337,9 +338,9 @@ tasks: # See https://github.com/bazelbuild/rules_python/issues/2499 bazel: 8.x test_flags: + <<: *reusable_config_test_flags ? "--test_tag_filters=-integration-test,-acceptance-test" ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" - ? "--config=ci" integration_test_build_file_generation_ubuntu_minimum_supported_workspace: <<: *minimum_supported_version