diff --git a/.tekton/python-builder-pull-request.yaml b/.tekton/python-builder-pull-request.yaml index b628605..0c69e9a 100644 --- a/.tekton/python-builder-pull-request.yaml +++ b/.tekton/python-builder-pull-request.yaml @@ -27,11 +27,19 @@ spec: value: 5d - name: dockerfile value: Containerfile + - name: build-image-index + value: "true" + - name: build-platforms + value: + - linux/x86_64 + - linux/arm64 + - linux/s390x + - linux/ppc64le pipelineSpec: description: | This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. - _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + _Uses `buildah-remote` (multi-platform-controller) to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ params: - description: Source Repository URL @@ -72,10 +80,15 @@ spec: description: Build a source image. name: build-source-image type: string - - default: "false" + - default: "true" description: Add built image into an OCI image index name: build-image-index type: string + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array - default: docker description: The format for the resulting image's mediaType. Valid values are oci or docker. name: buildah-format @@ -193,7 +206,12 @@ spec: workspace: git-auth - name: netrc workspace: netrc - - name: build-container + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images params: - name: IMAGE value: $(params.output-image) @@ -235,14 +253,16 @@ spec: - $(params.build-args-file) - name: PREFETCH_ARTIFACT value: $(tasks.prefetch-dependencies.results.PREFETCH_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" runAfter: - prefetch-dependencies taskRef: params: - name: name - value: buildah-oci-ta + value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.14.1@sha256:c02e7ce2ba4ee2fece1320bdef977d517557f4b909731143d35be30c9788977d + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.14.1@sha256:2ca404d74b6b81af71034f4a50490a2c4c5591d34c88ddb14fbb495659f80ff3 - name: kind value: task resolver: bundles @@ -254,11 +274,11 @@ spec: value: $(params.build-image-index) - name: IMAGES value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - $(tasks.build-images.results.IMAGE_REF[*]) - name: BUILDAH_FORMAT value: $(params.buildah-format) runAfter: - - build-container + - build-images taskRef: params: - name: name @@ -526,7 +546,7 @@ spec: - name: netrc optional: true taskRunSpecs: - - pipelineTaskName: build-container + - pipelineTaskName: build-images timeout: 3h stepSpecs: - name: build diff --git a/.tekton/python-builder-push.yaml b/.tekton/python-builder-push.yaml index 49cec9f..9779283 100644 --- a/.tekton/python-builder-push.yaml +++ b/.tekton/python-builder-push.yaml @@ -24,11 +24,19 @@ spec: value: quay.io/redhat-user-workloads/lightwell-python-tenant/python-builder/python-builder:{{revision}} - name: dockerfile value: Containerfile + - name: build-image-index + value: "true" + - name: build-platforms + value: + - linux/x86_64 + - linux/arm64 + - linux/s390x + - linux/ppc64le pipelineSpec: description: | This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. - _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + _Uses `buildah-remote` (multi-platform-controller) to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ params: - description: Source Repository URL @@ -69,10 +77,15 @@ spec: description: Build a source image. name: build-source-image type: string - - default: "false" + - default: "true" description: Add built image into an OCI image index name: build-image-index type: string + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array - default: docker description: The format for the resulting image's mediaType. Valid values are oci or docker. name: buildah-format @@ -190,7 +203,12 @@ spec: workspace: git-auth - name: netrc workspace: netrc - - name: build-container + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images params: - name: IMAGE value: $(params.output-image) @@ -232,14 +250,16 @@ spec: - $(params.build-args-file) - name: PREFETCH_ARTIFACT value: $(tasks.prefetch-dependencies.results.PREFETCH_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" runAfter: - prefetch-dependencies taskRef: params: - name: name - value: buildah-oci-ta + value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.14.1@sha256:c02e7ce2ba4ee2fece1320bdef977d517557f4b909731143d35be30c9788977d + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.14.1@sha256:2ca404d74b6b81af71034f4a50490a2c4c5591d34c88ddb14fbb495659f80ff3 - name: kind value: task resolver: bundles @@ -251,11 +271,11 @@ spec: value: $(params.build-image-index) - name: IMAGES value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - $(tasks.build-images.results.IMAGE_REF[*]) - name: BUILDAH_FORMAT value: $(params.buildah-format) runAfter: - - build-container + - build-images taskRef: params: - name: name @@ -523,7 +543,7 @@ spec: - name: netrc optional: true taskRunSpecs: - - pipelineTaskName: build-container + - pipelineTaskName: build-images timeout: 3h stepSpecs: - name: build diff --git a/Containerfile b/Containerfile index c9e854a..080774a 100644 --- a/Containerfile +++ b/Containerfile @@ -1,6 +1,11 @@ # default to latest supported policy, x86_64 ARG BASEIMAGE=registry.access.redhat.com/ubi8/ubi@sha256:c322aa68f2d630b36b9faab64094357f62720be2f56f7099376ca2d6c4fe389b ARG POLICY=manylinux_2_28 +# buildah --platform sets TARGETARCH (amd64/arm64/s390x/ppc64le). AUDITWHEEL_ARCH +# / PLAT are baked from TARGETARCH via runtime_base_packages_ stages below +# (OCI amd64→x86_64, arm64→aarch64). PLATFORM remains for local overrides / +# build_utils.sh and manylinux-entrypoint still remap from uname as a safety net. +ARG TARGETARCH=amd64 ARG PLATFORM=x86_64 ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-14/root ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst @@ -21,13 +26,18 @@ ARG OPENSSL_SHARED_PREFIX=/opt/_internal/openssl-3.6 FROM $BASEIMAGE AS runtime_base_packages ARG POLICY +ARG TARGETARCH ARG PLATFORM ARG DEVTOOLSET_ROOTPATH ARG LD_LIBRARY_PATH_ARG ARG PREPEND_PATH LABEL maintainer="Red Hat" -ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} +# Temporary defaults for package-install RUNs; entrypoint/build_utils remap from +# uname on non-x86. Arch-specific stages below override the baked image ENV. +ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=x86_64 AUDITWHEEL_PLAT=${POLICY}_x86_64 +# TARGETARCH is consumed by buildah multi-platform builds; keep it visible for debugging. +ENV TARGETARCH=${TARGETARCH} ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} @@ -47,6 +57,26 @@ ENTRYPOINT ["manylinux-entrypoint"] COPY build_scripts/install-runtime-packages.sh /opt/_internal/build_scripts/ RUN manylinux-entrypoint /opt/_internal/build_scripts/install-runtime-packages.sh +# Bake AUDITWHEEL_ARCH/PLAT for this TARGETARCH (buildah sets TARGETARCH per platform). +FROM runtime_base_packages AS runtime_base_packages_amd64 +ARG POLICY +ENV AUDITWHEEL_ARCH=x86_64 AUDITWHEEL_PLAT=${POLICY}_x86_64 + +FROM runtime_base_packages AS runtime_base_packages_arm64 +ARG POLICY +ENV AUDITWHEEL_ARCH=aarch64 AUDITWHEEL_PLAT=${POLICY}_aarch64 + +FROM runtime_base_packages AS runtime_base_packages_s390x +ARG POLICY +ENV AUDITWHEEL_ARCH=s390x AUDITWHEEL_PLAT=${POLICY}_s390x + +FROM runtime_base_packages AS runtime_base_packages_ppc64le +ARG POLICY +ENV AUDITWHEEL_ARCH=ppc64le AUDITWHEEL_PLAT=${POLICY}_ppc64le + +FROM runtime_base_packages_${TARGETARCH} AS runtime_base_arch + + # prepare cross-compilation support FROM --platform=linux/${MANYLINUX_BUILDARCH} ghcr.io/mayeut/static-clang:${MANYLINUX_CLANG_VERSION} AS static_clang_bin @@ -61,7 +91,7 @@ FROM scratch AS static_clang COPY --from=static_clang_prepare /tmp/cross-compiler / -FROM runtime_base_packages AS runtime_base +FROM runtime_base_arch AS runtime_base COPY build_scripts/install-autoconf.sh /opt/_internal/build_scripts/ RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \ export AUTOCONF_ROOT=autoconf-2.72 && \ @@ -176,11 +206,11 @@ RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \ FROM build_base AS build_rust COPY build_scripts/build-rust.sh /opt/_internal/build_scripts/ # The unversioned .../rustup/dist path always serves the newest rustup, so the -# version is pinned in the URL; otherwise RUSTUP_HASH breaks on every release. +# version is pinned in the URL. Per-arch rustup-init sha256 digests live in +# build-rust.sh (matched to this RUSTUP_VERSION). RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \ export RUST_VERSION=1.98.1 && \ export RUSTUP_VERSION=1.29.1 && \ - export RUSTUP_HASH=dda7234360b7f578ca8b0ddcb80145646fa61a67c1720a5abc7051b35c9fcb71 && \ export RUSTUP_DOWNLOAD_URL=https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION} && \ /tmp/cross-compiler/entrypoint /opt/_internal/build_scripts/build-rust.sh diff --git a/IMPLEMENTATION_SUMMARY.md b/IMPLEMENTATION_SUMMARY.md index 28b66d9..d0cb9c2 100644 --- a/IMPLEMENTATION_SUMMARY.md +++ b/IMPLEMENTATION_SUMMARY.md @@ -294,9 +294,9 @@ git checkout -- Containerfile build_scripts/build_utils.sh ## Known Limitations ### Rust Multi-Architecture -- Current implementation has x86_64 hash only -- For aarch64/i686 builds, need architecture-specific rustup-init hashes -- Script handles architecture detection, just needs proper hashes +- `build-rust.sh` maps `AUDITWHEEL_ARCH` to the rustup-init triple and sha256 + for x86_64, aarch64, s390x, ppc64le, i686, and armv7l +- Digests must be refreshed when bumping `RUSTUP_VERSION` in the Containerfile ### Build Time Increase - Phase 2 adds ~5-10 minutes (mainly Rust installation) diff --git a/MIGRATION_STATUS.md b/MIGRATION_STATUS.md index f1e23d1..08c4753 100644 --- a/MIGRATION_STATUS.md +++ b/MIGRATION_STATUS.md @@ -33,7 +33,7 @@ This document tracks the implementation of the migration from AlmaLinux 8 to UBI 1. **Rust 1.84.0 + Cargo** ⚠️ CRITICAL - Script: `build_scripts/build-rust.sh` - Method: rustup installation - - Hash: `4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10` (x86_64) + - Hashes: per-arch rustup-init digests in `build-rust.sh` (x86_64, aarch64, s390x, ppc64le, …) - Wrappers created for `rustc`, `cargo`, `rustup` in `/usr/local/bin` 2. **libjpeg-turbo 3.0.1** @@ -240,9 +240,9 @@ podman run --rm plumbing-builder:ubi8-phase4 \ ## Known Issues / Notes ### Rust Architecture Support -- Current hash is for x86_64 only -- For multi-arch builds, need architecture-specific hashes for rustup-init -- Alternative: Download rustup-init without hash verification (not recommended) +- `build-rust.sh` selects the rustup-init triple and sha256 per `AUDITWHEEL_ARCH` + (x86_64, aarch64, s390x, ppc64le, i686, armv7l) +- Digests are pinned for the `RUSTUP_VERSION` archive URL set in the Containerfile ### OpenBLAS Tuning - Uses `DYNAMIC_ARCH=1` for runtime CPU detection diff --git a/VERSION_UPDATES.md b/VERSION_UPDATES.md index 43029a7..f0c3b31 100644 --- a/VERSION_UPDATES.md +++ b/VERSION_UPDATES.md @@ -41,6 +41,7 @@ # Rust 1.94.0 (rustup-init for x86_64) RUST_VERSION=1.94.0 RUSTUP_HASH=4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10 +# Multi-arch: per-arch rustup-init digests now live in build_scripts/build-rust.sh # libjpeg-turbo 3.1.3 LIBJPEG_TURBO_VERSION=3.1.3 @@ -146,6 +147,7 @@ LIBTIFF_HASH=b92017489bdc1db3a4c97191aa4b75366673cb746de0dce5d7a749d5954681ba ```bash # Rust (rustup-init) https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init +# Multi-arch triples/hashes: see build_scripts/build-rust.sh (archive//…) # libjpeg-turbo https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.1.3/libjpeg-turbo-3.1.3.tar.gz diff --git a/build_scripts/build-arrow.sh b/build_scripts/build-arrow.sh index ad14839..b04d592 100755 --- a/build_scripts/build-arrow.sh +++ b/build_scripts/build-arrow.sh @@ -24,6 +24,13 @@ check_sha256sum "${ARROW_ROOT}.tar.gz" "${ARROW_HASH}" tar xfz "${ARROW_ROOT}.tar.gz" pushd "${ARROW_ROOT}/cpp" +# Arrow 25.0.1 / 26.0.0 pin xsimd 14.2.0, which fails Clang VSX on ppc64le +# (cannot use 'long' with '__vector bool'). 14.3.0 has the VSX Clang fix. +sed -i \ + -e 's/^ARROW_XSIMD_BUILD_VERSION=.*/ARROW_XSIMD_BUILD_VERSION=14.3.0/' \ + -e 's/^ARROW_XSIMD_BUILD_SHA256_CHECKSUM=.*/ARROW_XSIMD_BUILD_SHA256_CHECKSUM=b3d50e7a73fbf4642ceef30131c93414901d69eee41c2a5302db650b03e2c792/' \ + thirdparty/versions.txt + cmake_extra_args=() target_arch="${AUDITWHEEL_ARCH:?AUDITWHEEL_ARCH must be set}" if [[ "$target_arch" == "x86_64" ]]; then diff --git a/build_scripts/build-rust.sh b/build_scripts/build-rust.sh index b5bd8dc..835e48e 100755 --- a/build_scripts/build-rust.sh +++ b/build_scripts/build-rust.sh @@ -13,25 +13,44 @@ source "${MY_DIR}/build_utils.sh" # Install a more recent Rust check_var "${RUST_VERSION}" -check_var "${RUSTUP_HASH}" check_var "${RUSTUP_DOWNLOAD_URL}" PREFIX=/opt/_internal/rust-${RUST_VERSION} -# Download and verify rustup-init +# Download and verify rustup-init. Hashes are for rustup-init from +# https://static.rust-lang.org/rustup/archive///rustup-init +# (pinned via RUSTUP_DOWNLOAD_URL in the Containerfile; currently 1.29.1). RUSTUP_INIT="rustup-init" -if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then - RUSTUP_ARCH="x86_64-unknown-linux-gnu" -elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ]; then - RUSTUP_ARCH="aarch64-unknown-linux-gnu" -elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then - RUSTUP_ARCH="i686-unknown-linux-gnu" -elif [ "${AUDITWHEEL_ARCH}" == "armv7l" ]; then - RUSTUP_ARCH="armv7-unknown-linux-gnueabihf" -else - echo "Unsupported architecture: ${AUDITWHEEL_ARCH}" - exit 1 -fi +case "${AUDITWHEEL_ARCH}" in + x86_64) + RUSTUP_ARCH="x86_64-unknown-linux-gnu" + RUSTUP_HASH="dda7234360b7f578ca8b0ddcb80145646fa61a67c1720a5abc7051b35c9fcb71" + ;; + aarch64) + RUSTUP_ARCH="aarch64-unknown-linux-gnu" + RUSTUP_HASH="15f6e4ce9f583b929c996c91562bad6d4454f3281de858b02cdfdef615fac433" + ;; + s390x) + RUSTUP_ARCH="s390x-unknown-linux-gnu" + RUSTUP_HASH="312ac9390467fd7870f8aff6557c7cb3bff0a04bb11b263173fdfd81586da641" + ;; + ppc64le) + RUSTUP_ARCH="powerpc64le-unknown-linux-gnu" + RUSTUP_HASH="86ebc5b46b20013fca57dff4f83aa0c325ee90914626ce1c80ae76e126e48afb" + ;; + i686) + RUSTUP_ARCH="i686-unknown-linux-gnu" + RUSTUP_HASH="5b993042a9f0d3577592f08cb76f8b850951487c2556d90f6e9846abbb083a88" + ;; + armv7l) + RUSTUP_ARCH="armv7-unknown-linux-gnueabihf" + RUSTUP_HASH="6f34abb0d553273ce08306ea3adb758d0171f21090cc5ad5426f474ded5504d1" + ;; + *) + echo "Unsupported architecture: ${AUDITWHEEL_ARCH}" + exit 1 + ;; +esac fetch_source "${RUSTUP_INIT}" "${RUSTUP_DOWNLOAD_URL}/${RUSTUP_ARCH}" check_sha256sum "${RUSTUP_INIT}" "${RUSTUP_HASH}" diff --git a/build_scripts/build_utils.sh b/build_scripts/build_utils.sh index 711ade7..9355615 100755 --- a/build_scripts/build_utils.sh +++ b/build_scripts/build_utils.sh @@ -14,6 +14,23 @@ export BASE_POLICY=manylinux PACKAGE_MANAGER=dnf OS_ID_LIKE=rhel +# Containerfile defaults PLATFORM/AUDITWHEEL_ARCH to x86_64. Native multi-arch +# builds (buildah --platform / MPC VMs) run on the target CPU, so correct the +# arch from uname when the default does not match the host. Explicit non-x86_64 +# PLATFORM values are left alone (including linux32 cross cases). +if [ "${AUDITWHEEL_ARCH:-}" = "x86_64" ]; then + _manylinux_host_arch="$(uname -m)" + case "${_manylinux_host_arch}" in + aarch64|s390x|ppc64le) + export AUDITWHEEL_ARCH="${_manylinux_host_arch}" + if [ -n "${AUDITWHEEL_POLICY:-}" ]; then + export AUDITWHEEL_PLAT="${AUDITWHEEL_POLICY}_${AUDITWHEEL_ARCH}" + fi + ;; + esac + unset _manylinux_host_arch +fi + function check_var { if [ -z "$1" ]; then echo "required variable not defined" diff --git a/manylinux-entrypoint b/manylinux-entrypoint index e1289ac..6f4684b 100755 --- a/manylinux-entrypoint +++ b/manylinux-entrypoint @@ -2,6 +2,21 @@ set -eu +# Match build_utils.sh: Containerfile defaults AUDITWHEEL_ARCH to x86_64, but +# multi-arch image builds run natively on the target CPU. +if [ "${AUDITWHEEL_ARCH:-}" = "x86_64" ]; then + _manylinux_host_arch="$(uname -m)" + case "${_manylinux_host_arch}" in + aarch64|s390x|ppc64le) + export AUDITWHEEL_ARCH="${_manylinux_host_arch}" + if [ -n "${AUDITWHEEL_POLICY:-}" ]; then + export AUDITWHEEL_PLAT="${AUDITWHEEL_POLICY}_${AUDITWHEEL_ARCH}" + fi + ;; + esac + unset _manylinux_host_arch +fi + if [ "${AUDITWHEEL_ARCH}" == "i686" ] && [ "$(uname -m)" == "x86_64" ]; then linux32 "$@" elif [ "${AUDITWHEEL_ARCH}" == "armv7l" ] && [ "$(uname -m)" != "armv7l" ]; then