From d33b9234a55585efba4a8898ed7617f34923b1c1 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:53:25 -0400 Subject: [PATCH 01/14] Add a persistent ThinLTO cache for release builds The ThinLTO link of `clickhouse` and `clickhouse-keeper` dominates the release build: ~82 min of a ~117 min `amd_release` build on a 16-core, 62 GB builder. lld can cache the optimized native object of every module (`--thinlto-cache-dir`), keyed by the module's bitcode hash, its import set and the LTO configuration, and reuse it on the next link when nothing in that key changed. Only the parallel backend phase is cached; the single-threaded thin-link and the final native link still run in full. Measured on the same builder class with the production `-g` configuration (sha e3f85bc69bf, same commit built twice): 114.5 min cold, 65.5 min warm. Forty-six commits of churn on top of a warm cache cost about 3 min, since a header change only invalidates modules whose emitted IR changed. The cache lives on an attached volume at `/mnt/clickhouse_thinlto`. praktika gains `--volume-if-exists=HOST:CONTAINER` in a job's docker settings, which mounts the host path only on runners that have it rather than creating an empty directory everywhere as `--volume` does, so the build job enables the cache by the directory's presence and builders without the volume run unchanged. lld prunes inside the linker after each link's backend has run, removing the least recently used entries until the cache fits the size target, so the policy is size-only (`prune_after=0s`, lld's own default is 7 days) at 90% of the volume. The volume must be mounted `relatime` so that hits refresh access times. One `amd_release` `-g` population is ~15 GB. The build job also prints the output of its `sccache --show-stats` and cache stats commands, which `Shell.check` otherwise discards. Co-Authored-By: Claude Mythos 5.1 --- CMakeLists.txt | 16 ++++++++++++++++ ci/defs/job_configs.py | 4 ++++ ci/jobs/build_clickhouse.py | 19 +++++++++++++++++++ ci/praktika/runner.py | 13 +++++++++++++ 4 files changed, 52 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 68b194ad629c3..54d294a529e99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -493,6 +493,22 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--lto-whole-program-visibility") endif() + # Persistent ThinLTO cache. lld stores the optimized native object of every + # module keyed by the module hash, its import set and the LTO flags, and + # reuses it on the next link when nothing in that key changed. Only the + # backend codegen of the link is cached; the thin-link and the final native + # link still run in full. Entries are content-addressed, so concurrent + # writers cannot corrupt each other. + set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + if (THINLTO_CACHE_DIR) + if (NOT LINKER_NAME MATCHES "lld") + message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") + endif () + message (STATUS "Using ThinLTO cache in ${THINLTO_CACHE_DIR} with policy '${THINLTO_CACHE_POLICY}'") + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-cache-dir=${THINLTO_CACHE_DIR} -Wl,--thinlto-cache-policy=${THINLTO_CACHE_POLICY}") + endif () + if (ENABLE_CFI) # CFI checks: virtual dispatch type safety and bad downcasts via static_cast. # cfi-icall is omitted because C function pointers in contrib libraries (compiled without diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index eaec79a77d782..9811e6f39f60c 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,6 +29,10 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" + # Attached volume holding the ThinLTO cache of release builds, see + # `use_thinlto_cache` in build_clickhouse.py. Mounted only on builders that + # have it, so the build job can key on the directory's presence. + "+--volume-if-exists=/mnt/clickhouse_thinlto:/thinlto-cache" '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' ) BINARY_DOCKER_COMMAND_SANITIZER = ( diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 23eec7551006d..02ed4a0636cd9 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -105,6 +105,7 @@ def parse_args(): def run_shell(name, command, **kwargs): print(f"\n>>>> {name}\n") + kwargs.setdefault("verbose", True) Shell.check(command, **kwargs) print(f"\n<<<< {name}\n") @@ -214,6 +215,19 @@ def main(): ): cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1" + # ThinLTO cache: the link reuses the optimized native objects of modules + # that did not change since the previous link in this directory. It is an + # attached volume that BINARY_DOCKER_COMMAND mounts only on builders that + # have it, so a builder whose disk is wiped between jobs has no directory + # here and does not pay for writing a cache it will never read. + thinlto_cache_dir = "/thinlto-cache" + use_thinlto_cache = "ENABLE_THINLTO=1" in cmake_cmd and os.path.isdir( + thinlto_cache_dir + ) + if use_thinlto_cache: + print(f"ThinLTO cache is enabled in {thinlto_cache_dir}") + cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir}" + # PGO/BOLT profile integration for release builds. The sccache-warmup builds # are deliberately excluded: they exist only to populate the shared compiler # cache, and PGO/BOLT belong to the real release builds. @@ -463,6 +477,11 @@ def do_checkout(): results.append(retry_cmake) run_shell("sccache stats", "sccache --show-stats") + if use_thinlto_cache: + run_shell( + "thinlto cache stats", + f"du -sh {thinlto_cache_dir} && find {thinlto_cache_dir} -type f | wc -l && df -h {thinlto_cache_dir}", + ) if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") clang_tidy_cache_log = "./ci/tmp/clang-tidy-cache.log" diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 6d105b87468ac..5185f4b81640b 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -480,6 +480,19 @@ def _run( if not job.timeout_shell_cleanup: job.timeout_shell_cleanup = f"docker rm -f {container_name}" workdir = f"--workdir={current_dir}" + # `--volume-if-exists=HOST:CONTAINER` mounts HOST only on runners + # that have it (e.g. an attached cache volume), instead of creating + # an empty directory everywhere like `--volume` does. + optional_volumes = [ + s for s in settings if s.startswith("--volume-if-exists=") + ] + for setting in optional_volumes: + settings.remove(setting) + volume = setting.removeprefix("--volume-if-exists=") + if Path(volume.split(":")[0]).exists(): + settings.append(f"--volume={volume}") + else: + print(f"NOTE: optional volume [{volume}] is not present on this runner") for setting in settings: if setting.startswith("--volume"): volume = setting.removeprefix("--volume=").split(":")[0] From d92a33d2bf38430826e9ffef802b8c1463da6826 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:53:25 -0400 Subject: [PATCH 02/14] Link clickhouse and clickhouse-keeper concurrently in ThinLTO builds Each ThinLTO link spends most of its time in a single-threaded thin-link phase, so two links overlap almost entirely. Measured on a 16-core, 62 GB builder with zram: 114.5 -> 83 min cold and 65.5 -> 48.5 min warm (with the ThinLTO cache), with a combined memory peak of ~48 GB. `limit_jobs.cmake` caps ThinLTO links from a budget of ~100 GB per `-g` link and applied that cap even to an explicit `PARALLEL_LINK_JOBS`, so a 62 GB builder could never link both binaries at once. The cap now applies only to the auto-calculated value, and the build job sets two link jobs for every ThinLTO flavour. Co-Authored-By: Claude Mythos 5.1 --- ci/jobs/build_clickhouse.py | 9 +++++++++ cmake/limit_jobs.cmake | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 02ed4a0636cd9..981b2a64e5c72 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -228,6 +228,15 @@ def main(): print(f"ThinLTO cache is enabled in {thinlto_cache_dir}") cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir}" + # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link + # spends most of its time in a single-threaded thin-link phase, so the two + # overlap almost entirely: measured on a 16-core, 62 GB builder with zram, + # 114.5 -> 83 min cold and 65.5 -> 48.5 min warm. The memory-derived + # default in limit_jobs.cmake budgets ~100 GB per `-g` link and would + # allow only one; the real peak of both links together was ~48 GB. + if "ENABLE_THINLTO=1" in cmake_cmd: + cmake_cmd += " -DPARALLEL_LINK_JOBS=2" + # PGO/BOLT profile integration for release builds. The sccache-warmup builds # are deliberately excluded: they exist only to populate the shared compiler # cache, and PGO/BOLT belong to the real release builds. diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 8be747c6f27f0..366e3f71a8ba4 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -28,6 +28,10 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) endif() endif() +# Remember whether the link-job limit was set explicitly: the ThinLTO memory +# cap below only applies to the auto-calculated value. +set (PARALLEL_LINK_JOBS_EXPLICIT ${PARALLEL_LINK_JOBS}) + if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) math(EXPR PARALLEL_LINK_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY}) @@ -54,7 +58,7 @@ endif () # links than physically fit in memory - otherwise the linker is OOM-killed. # With debug info disabled (-g0, DISABLE_ALL_DEBUG_SYMBOLS) a link needs only # ~50 GB, so two still fit and we keep them parallel. -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1) +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1 AND NOT PARALLEL_LINK_JOBS_EXPLICIT) if (DISABLE_ALL_DEBUG_SYMBOLS) set (MAX_THINLTO_LINK_MEMORY 51200) # megabytes per concurrent ThinLTO link, measured ~49 GB with -g0 else () From 02e720836ff43fdc35367cc2ec7b359ac64b948e Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:13:24 -0400 Subject: [PATCH 03/14] Reduce runner swappiness, we are wasting available ram --- .github/actions/runner_setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/runner_setup/action.yml b/.github/actions/runner_setup/action.yml index e978d8f57a741..a9f9d0d341f41 100644 --- a/.github/actions/runner_setup/action.yml +++ b/.github/actions/runner_setup/action.yml @@ -25,7 +25,7 @@ runs: ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0 sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0 - sudo sysctl vm.swappiness=200 + sudo sysctl vm.swappiness=80 - name: Install awscli shell: bash run: | From 5b2a13913502d7603024a26debbed405d25a3e38 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:32:45 -0400 Subject: [PATCH 04/14] enable lto cache volume on builders Cache does not belong in meta label since builder instances are also used for some tests --- .github/workflows/fast_builds.yml | 4 ++-- .github/workflows/master.yml | 8 ++++---- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/pull_request_community.yml | 4 ++-- ci/defs/defs.py | 4 ++-- ci/defs/job_configs.py | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index eaabcc3a63286..7b6b245777e36 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -800,7 +800,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -844,7 +844,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 62f6d64627657..b88decb436b4e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -859,7 +859,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -903,7 +903,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -947,7 +947,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -991,7 +991,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c638bc99d1e54..5f2c4ba6cde87 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -847,7 +847,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -889,7 +889,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index e3c6626cbac83..6cdd7707d4d1b 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/ci/defs/defs.py b/ci/defs/defs.py index cd3681eece7e4..68d93cbed1b60 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "volume-clickhouse_thinlto-200GB"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "volume-clickhouse_thinlto-200GB"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 9811e6f39f60c..63707a3c7a692 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -462,7 +462,7 @@ class JobConfigs: Job.ParamSet( parameter=BuildTypes.AMD_CFI, provides=[ArtifactNames.CH_AMD_CFI], - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_ARM, timeout=4 * 3600, ), ) @@ -505,12 +505,12 @@ class JobConfigs: ).parametrize( Job.ParamSet( parameter=BuildTypes.AMD_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_ARM, timeout=3 * 3600, ), Job.ParamSet( parameter=BuildTypes.ARM_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.ARM_LARGE, + runs_on=RunnerLabels.BUILDER_ARM, ), ) ] From d151e0f7417c4dda63bc4e508a8737f6d91e8a30 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:47:53 -0400 Subject: [PATCH 05/14] Cap the ThinLTO cache at 180 GB as well as 90% of its filesystem --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 54d294a529e99..971a108900ed6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -500,7 +500,7 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) # link still run in full. Entries are content-addressed, so concurrent # writers cannot corrupt each other. set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") - set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%:cache_size_bytes=180g" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") if (THINLTO_CACHE_DIR) if (NOT LINKER_NAME MATCHES "lld") message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") From 1e896cbf4dbf70663d433f8f577b220ebe932418 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:53:59 -0400 Subject: [PATCH 06/14] fix volume labels --- .github/workflows/fast_builds.yml | 4 ++-- .github/workflows/master.yml | 8 ++++---- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/pull_request_community.yml | 4 ++-- ci/defs/defs.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 7b6b245777e36..61899d96e066b 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -800,7 +800,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -844,7 +844,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b88decb436b4e..7ed2d154c9a94 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -859,7 +859,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -903,7 +903,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -947,7 +947,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -991,7 +991,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5f2c4ba6cde87..51a52c4b607ce 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -847,7 +847,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -889,7 +889,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 6cdd7707d4d1b..a41b454610754 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/ci/defs/defs.py b/ci/defs/defs.py index 68d93cbed1b60..139f9d543cd42 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "volume-clickhouse_thinlto-200GB"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "volume-clickhouse_thinlto-200GB"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-volume-clickhouse_thinlto-200GB"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-volume-clickhouse_thinlto-200GB"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", From 62318a6b5dbb3217c737214c8a4999dbdf489892 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:13:02 -0400 Subject: [PATCH 07/14] Make the ThinLTO cache stats work on a volume with `lost+found` On the builder volume the root-owned `lost+found` directory is unreadable for the build user, so `du` failed and the `&&` chain stopped before the entry count and `df`. Exclude it from `du`, count only lld's own `llvmcache-*` entries, and run the three commands independently. https://github.com/Altinity/ClickHouse/actions/runs/40545556921/job/122623854938 Co-Authored-By: Claude Opus 5.5 --- ci/jobs/build_clickhouse.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 981b2a64e5c72..8a67a904e96f1 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -489,7 +489,11 @@ def do_checkout(): if use_thinlto_cache: run_shell( "thinlto cache stats", - f"du -sh {thinlto_cache_dir} && find {thinlto_cache_dir} -type f | wc -l && df -h {thinlto_cache_dir}", + # The volume's root-owned `lost+found` is unreadable for the + # build user, so count only lld's own entries. + f"du -sh --exclude=lost+found {thinlto_cache_dir}; " + f"find {thinlto_cache_dir} -maxdepth 1 -name 'llvmcache-*' | wc -l; " + f"df -h {thinlto_cache_dir}", ) if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") From 0d56da2db82589aa5dac5929c71407bb811e12ea Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:10:48 -0400 Subject: [PATCH 08/14] Split the ThinLTO cache volumes by target architecture amd and arm cache entries never share keys, so a build only benefits from a volume where a build for the same target ran recently. With one volume name, the scaler hands each build whichever detached volume it finds first, and a master push runs two amd and two arm ThinLTO builds at once. Give each target its own meta label, `altinity-thinlto-cache-amd` and `altinity-thinlto-cache-arm`, which the runner controller expands into the volume (`clickhouse_thinlto_amd`, `clickhouse_thinlto_arm`) and its location, so neither size nor location is fixed in the repository. The build job mounts whichever of the per-target volumes is present, or the single `/mnt/clickhouse_thinlto` directory that static builders keep on their own disk for both targets. The 180 GB cap in the cache policy stays, since those builders hold both targets' entries. `arm_release` now runs on `BUILDER_ARM`, and `amd_cfi` and the amd cache warmup build on `BUILDER_AMD`. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/fast_builds.yml | 4 ++-- .github/workflows/master.yml | 8 ++++---- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/pull_request_community.yml | 4 ++-- ci/defs/defs.py | 4 ++-- ci/defs/job_configs.py | 17 +++++++++++------ 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 61899d96e066b..ff8a7c80dc874 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -800,7 +800,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -844,7 +844,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7ed2d154c9a94..b65ab01074036 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -859,7 +859,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -903,7 +903,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -947,7 +947,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -991,7 +991,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 51a52c4b607ce..e8244433825c2 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -847,7 +847,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -889,7 +889,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index a41b454610754..0844cfb532a7a 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-volume-clickhouse_thinlto-200GB] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/ci/defs/defs.py b/ci/defs/defs.py index 139f9d543cd42..12d32fd34d46f 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-volume-clickhouse_thinlto-200GB"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-volume-clickhouse_thinlto-200GB"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-amd"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-arm"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 63707a3c7a692..bcb528e8d5259 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,9 +29,14 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" - # Attached volume holding the ThinLTO cache of release builds, see - # `use_thinlto_cache` in build_clickhouse.py. Mounted only on builders that - # have it, so the build job can key on the directory's presence. + # ThinLTO cache of release builds, see `use_thinlto_cache` in + # build_clickhouse.py. Mounted only on builders that have it, so the build + # job can key on the directory's presence. Cloud builders get one attached + # volume per target architecture (see `RunnerLabels.BUILDER_AMD`); static + # builders keep a single directory on their own disk for both targets. A + # builder has exactly one of the three. + "+--volume-if-exists=/mnt/clickhouse_thinlto_amd:/thinlto-cache" + "+--volume-if-exists=/mnt/clickhouse_thinlto_arm:/thinlto-cache" "+--volume-if-exists=/mnt/clickhouse_thinlto:/thinlto-cache" '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' ) @@ -447,7 +452,7 @@ class JobConfigs: ArtifactNames.RPM_ARM_RELEASE, ArtifactNames.TGZ_ARM_RELEASE, ], - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.BUILDER_ARM, ), ) release_build_jobs_with_examples = [ @@ -462,7 +467,7 @@ class JobConfigs: Job.ParamSet( parameter=BuildTypes.AMD_CFI, provides=[ArtifactNames.CH_AMD_CFI], - runs_on=RunnerLabels.BUILDER_ARM, + runs_on=RunnerLabels.BUILDER_AMD, timeout=4 * 3600, ), ) @@ -505,7 +510,7 @@ class JobConfigs: ).parametrize( Job.ParamSet( parameter=BuildTypes.AMD_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.BUILDER_ARM, + runs_on=RunnerLabels.BUILDER_AMD, timeout=3 * 3600, ), Job.ParamSet( From fe70d03bdc2d007c3f0035a5a2c40a38fae7aa0c Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:42:53 -0400 Subject: [PATCH 09/14] Move the ThinLTO cache from builder volumes to Hetzner Object Storage Attached volumes tied the cache to whichever builder a job landed on, and the runner scaler never reused a volume for builders made from a snapshot image. The cache now travels as one archive per branch, build type and debug-info flavour in the Hetzner bucket from the submodule cache: `altinity-builds-cache/ci_ch_cache/thinlto//-.tar.zst`. Every ThinLTO build that links restores the archive before `ninja`. PR builds use their base branch's archive. Pushes and internal PRs then upload it again, so `-g` archives come from `MasterCI` and `-g0` ones from PR builds; community PRs read anonymously and write nothing, like the shared compiler cache. The cache-warmup builds only compile, so they skip it. The upload keeps exactly the entries the build used: restored entries get an access time of zero, so under `relatime` the next read sets it, and the entries still at zero are dropped before compressing with `zstd --long`. That keeps an archive at one build's working set, about 15 GB raw for `-g`, and lld's own pruning is off. A restore that fails leaves an empty cache rather than a partly extracted one, and an archive over 20 GB (`-g`) or 5 GB (`-g0`) is not uploaded. This removes the praktika `--volume-if-exists` setting, the cache mounts and the per-target builder labels, and puts `amd_cfi` and the cache-warmup builds back on `ARM_LARGE`. Related: https://github.com/Altinity/ClickHouse/pull/2740 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/fast_builds.yml | 4 +- .github/workflows/master.yml | 8 +- .github/workflows/pull_request.yml | 4 +- .github/workflows/pull_request_community.yml | 4 +- CMakeLists.txt | 4 +- ci/defs/defs.py | 4 +- ci/defs/job_configs.py | 17 +- ci/jobs/build_clickhouse.py | 172 ++++++++++++++++--- ci/praktika/runner.py | 13 -- 9 files changed, 171 insertions(+), 59 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index c5538b8a7e8fb..26a61911e0462 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -802,7 +802,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -846,7 +846,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "Fast Builds" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 3672c1c649003..90d8b9e15d717 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -861,7 +861,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_llvm_coverage_per_test)' --workflow "MasterCI" --ci --timestamp build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -905,7 +905,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "MasterCI" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" @@ -949,7 +949,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_release)' --workflow "MasterCI" --ci --timestamp build_amd_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (amd_release_pr_cache_warmup)" @@ -993,7 +993,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release_pr_cache_warmup)' --workflow "MasterCI" --ci --timestamp build_arm_release_pr_cache_warmup: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlX3ByX2NhY2hlX3dhcm11cCk=') }} name: "Build (arm_release_pr_cache_warmup)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2949239dfbcdf..e16505d822493 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -849,7 +849,7 @@ jobs: run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3." build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -891,7 +891,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_release)' --workflow "PR" --ci --timestamp build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 0844cfb532a7a..e3c6626cbac83 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -817,7 +817,7 @@ jobs: retention-days: 14 build_amd_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-amd] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" @@ -909,7 +909,7 @@ jobs: retention-days: 14 build_arm_release: - runs-on: [self-hosted, altinity-on-demand, altinity-builder, altinity-thinlto-cache-arm] + runs-on: [self-hosted, altinity-on-demand, altinity-builder] needs: [config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} name: "Build (arm_release)" diff --git a/CMakeLists.txt b/CMakeLists.txt index 971a108900ed6..ffcd02c5d0696 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -500,7 +500,9 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) # link still run in full. Entries are content-addressed, so concurrent # writers cannot corrupt each other. set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") - set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=90%:cache_size_bytes=180g" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + # lld prunes after each link by default. That is off here, since CI keeps + # exactly the entries a build used; set a policy for a long-lived cache dir. + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=0%:cache_size_bytes=0:cache_size_files=0" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") if (THINLTO_CACHE_DIR) if (NOT LINKER_NAME MATCHES "lld") message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") diff --git a/ci/defs/defs.py b/ci/defs/defs.py index 12d32fd34d46f..cd3681eece7e4 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -18,8 +18,8 @@ class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-amd"] - BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder", "altinity-thinlto-cache-arm"] + BUILDER_AMD = ["self-hosted", "altinity-on-demand", "altinity-builder"] + BUILDER_ARM = ["self-hosted", "altinity-on-demand", "altinity-builder"] FUNC_TESTER_AMD = ["self-hosted", "altinity-on-demand", "altinity-func-tester"] FUNC_TESTER_ARM = [ "self-hosted", diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index bcb528e8d5259..b79745981beb2 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,16 +29,9 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" - # ThinLTO cache of release builds, see `use_thinlto_cache` in - # build_clickhouse.py. Mounted only on builders that have it, so the build - # job can key on the directory's presence. Cloud builders get one attached - # volume per target architecture (see `RunnerLabels.BUILDER_AMD`); static - # builders keep a single directory on their own disk for both targets. A - # builder has exactly one of the three. - "+--volume-if-exists=/mnt/clickhouse_thinlto_amd:/thinlto-cache" - "+--volume-if-exists=/mnt/clickhouse_thinlto_arm:/thinlto-cache" - "+--volume-if-exists=/mnt/clickhouse_thinlto:/thinlto-cache" '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' + # Writes to the ThinLTO cache in Hetzner Object Storage; jobs without them read anonymously + '+--env=HETZNER_S3_ACCESS_KEY="$HETZNER_S3_ACCESS_KEY"+--env=HETZNER_S3_SECRET_KEY="$HETZNER_S3_SECRET_KEY"' ) BINARY_DOCKER_COMMAND_SANITIZER = ( BINARY_DOCKER_COMMAND + "+--security-opt seccomp=unconfined" @@ -467,7 +460,7 @@ class JobConfigs: Job.ParamSet( parameter=BuildTypes.AMD_CFI, provides=[ArtifactNames.CH_AMD_CFI], - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.ARM_LARGE, timeout=4 * 3600, ), ) @@ -510,12 +503,12 @@ class JobConfigs: ).parametrize( Job.ParamSet( parameter=BuildTypes.AMD_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.ARM_LARGE, timeout=3 * 3600, ), Job.ParamSet( parameter=BuildTypes.ARM_RELEASE_PR_CACHE_WARMUP, - runs_on=RunnerLabels.BUILDER_ARM, + runs_on=RunnerLabels.ARM_LARGE, ), ) ] diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 8a67a904e96f1..f8207263d0b04 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -6,8 +6,10 @@ from ci.jobs.scripts.clickhouse_version import CHVersion from ci.praktika.info import Info from ci.praktika.result import Result +from ci.praktika.s3 import S3 from ci.praktika.settings import Settings from ci.praktika.utils import MetaClasses, Shell, Utils +from ci.settings.altinity_overrides import CI_CACHE_BUCKET current_directory = Utils.cwd() build_dir = f"{current_directory}/ci/tmp/build" @@ -15,6 +17,18 @@ repo_path_normalized = "/ClickHouse" build_dir_normalized = f"{repo_path_normalized}/ci/tmp/build" +# ThinLTO cache: lld stores the optimized native object of every module it +# links, and the next link of an unchanged module reuses it instead of running +# the backend again. The cache travels between builders as one archive per +# branch, build type and debug-info flavour. +THINLTO_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache/thinlto" +thinlto_cache_dir = f"{temp_dir}/thinlto_cache" +thinlto_cache_dir_normalized = f"{repo_path_normalized}/ci/tmp/thinlto_cache" +thinlto_cache_archive = f"{temp_dir}/thinlto_cache.tar.zst" +# An archive holds the entries one build used, about 15 GB for a `-g` build of +# `clickhouse` and `clickhouse-keeper`. Much more means pruning went wrong. +THINLTO_CACHE_MAX_BYTES = {"g": 20 << 30, "g0": 5 << 30} + BUILD_TYPE_TO_CMAKE = { BuildTypes.AMD_DEBUG: f" cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=Debug -DENABLE_THINLTO=0 -DSANITIZE= -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE} -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=1 -DENABLE_LEXER_TEST=1 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DENABLE_BUZZHOUSE=1", BuildTypes.AMD_RELEASE: f" cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=None -DENABLE_THINLTO=1 -DSANITIZE= -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE} -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=0 -DENABLE_LEXER_TEST=0 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1", @@ -103,6 +117,111 @@ def parse_args(): return parser.parse_args() +def thinlto_cache_flavour(cmake_cmd): + return "g0" if "-DDISABLE_ALL_DEBUG_SYMBOLS=1" in cmake_cmd else "g" + + +def thinlto_cache_s3_path(info, build_type, cmake_cmd): + """PR builds share the cache of their base branch, other builds use their own branch.""" + branch = info.base_branch if info.pr_number > 0 else info.git_branch + if not branch: + return None + return f"{THINLTO_CACHE_S3_PATH}/{branch}/{build_type}-{thinlto_cache_flavour(cmake_cmd)}.tar.zst" + + +def restore_thinlto_cache(s3_path): + shutil.rmtree(thinlto_cache_dir, ignore_errors=True) + os.makedirs(thinlto_cache_dir) + if not S3.head_object(s3_path): + print(f"ThinLTO cache miss: {s3_path}") + return True + try: + S3.copy_file_from_s3(s3_path=s3_path, local_path=thinlto_cache_archive) + Shell.check( + f"set -o pipefail; zstd -d --long=27 -c {thinlto_cache_archive} | tar -xf - -C {thinlto_cache_dir}", + verbose=True, + strict=True, + ) + except BaseException: + # A partly extracted entry would be linked as is: start cold instead. + shutil.rmtree(thinlto_cache_dir, ignore_errors=True) + os.makedirs(thinlto_cache_dir) + raise + finally: + if os.path.exists(thinlto_cache_archive): + os.remove(thinlto_cache_archive) + # With an access time of zero, the kernel records the next read of an + # entry even under `relatime`, which is how the upload step tells the + # entries this build used from the ones it did not. + count = 0 + size = 0 + with os.scandir(thinlto_cache_dir) as entries: + for entry in entries: + stat = entry.stat(follow_symlinks=False) + os.utime(entry.path, (0, stat.st_mtime)) + count += 1 + size += stat.st_size + print(f"ThinLTO cache restored: {count} entries, {size / (1 << 30):.1f} GiB") + return True + + +def thinlto_cache_tracks_reads(): + probe = f"{thinlto_cache_dir}/.atime-probe" + with open(probe, "w") as f: + f.write("probe") + os.utime(probe, (0, os.stat(probe).st_mtime)) + with open(probe) as f: + f.read() + tracked = os.stat(probe).st_atime > 0 + os.remove(probe) + return tracked + + +def upload_thinlto_cache(s3_path, flavour): + """Keep the entries this build read or wrote, then upload them.""" + assert thinlto_cache_tracks_reads(), ( + f"{thinlto_cache_dir} does not record file reads (mounted with noatime?), " + "so the entries this build used are unknown" + ) + kept = 0 + removed = 0 + size = 0 + with os.scandir(thinlto_cache_dir) as entries: + for entry in entries: + stat = entry.stat(follow_symlinks=False) + # Anything else is a temporary file of an interrupted link. + used = entry.name.startswith("llvmcache-") and entry.is_file( + follow_symlinks=False + ) and stat.st_atime > 0 + if used: + kept += 1 + size += stat.st_size + else: + os.remove(entry.path) + removed += 1 + print( + f"ThinLTO cache: kept {kept} entries ({size / (1 << 30):.1f} GiB), " + f"removed {removed} unused" + ) + assert size <= THINLTO_CACHE_MAX_BYTES[flavour], ( + f"ThinLTO cache of {size} bytes exceeds the {THINLTO_CACHE_MAX_BYTES[flavour]} " + f"bytes expected for a `-{flavour}` build, not uploading it" + ) + Shell.check( + f"set -o pipefail; tar -cf - -C {thinlto_cache_dir} . | zstd -T0 -3 --long=27 -c > {thinlto_cache_archive}", + verbose=True, + strict=True, + ) + print( + f"ThinLTO cache archive: {os.path.getsize(thinlto_cache_archive) / (1 << 30):.1f} GiB" + ) + S3.copy_file_to_s3( + s3_path=s3_path, local_path=thinlto_cache_archive, with_rename=True + ) + os.remove(thinlto_cache_archive) + return True + + def run_shell(name, command, **kwargs): print(f"\n>>>> {name}\n") kwargs.setdefault("verbose", True) @@ -215,18 +334,16 @@ def main(): ): cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1" - # ThinLTO cache: the link reuses the optimized native objects of modules - # that did not change since the previous link in this directory. It is an - # attached volume that BINARY_DOCKER_COMMAND mounts only on builders that - # have it, so a builder whose disk is wiped between jobs has no directory - # here and does not pay for writing a cache it will never read. - thinlto_cache_dir = "/thinlto-cache" - use_thinlto_cache = "ENABLE_THINLTO=1" in cmake_cmd and os.path.isdir( - thinlto_cache_dir - ) - if use_thinlto_cache: - print(f"ThinLTO cache is enabled in {thinlto_cache_dir}") - cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir}" + # ThinLTO cache for every build that links with ThinLTO. The cache-warmup + # builds only compile. Everyone reads the archive (community PRs anonymously); + # pushes and internal PRs write it, like the shared compiler cache. + thinlto_cache_s3 = None + if "ENABLE_THINLTO=1" in cmake_cmd and not cache_warmup: + thinlto_cache_s3 = thinlto_cache_s3_path(info, build_type, cmake_cmd) + thinlto_cache_writable = not info.is_local_run and not info.is_community_pr + if thinlto_cache_s3: + print(f"ThinLTO cache: {thinlto_cache_s3}, writable: {thinlto_cache_writable}") + cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir_normalized}" # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link # spends most of its time in a single-threaded thin-link phase, so the two @@ -445,6 +562,15 @@ def do_checkout(): else: build_command = f"command time -v ninja {targets}" + if thinlto_cache_s3: + results.append( + Result.from_commands_run( + name="Restore ThinLTO cache", + command=restore_thinlto_cache, + command_args=[thinlto_cache_s3], + ) + ) + build_result_index = len(results) results.append( Result.from_commands_run( @@ -485,16 +611,20 @@ def do_checkout(): else: results.append(retry_cmake) - run_shell("sccache stats", "sccache --show-stats") - if use_thinlto_cache: - run_shell( - "thinlto cache stats", - # The volume's root-owned `lost+found` is unreadable for the - # build user, so count only lld's own entries. - f"du -sh --exclude=lost+found {thinlto_cache_dir}; " - f"find {thinlto_cache_dir} -maxdepth 1 -name 'llvmcache-*' | wc -l; " - f"df -h {thinlto_cache_dir}", + if ( + thinlto_cache_s3 + and thinlto_cache_writable + and results[build_result_index].is_ok() + ): + results.append( + Result.from_commands_run( + name="Upload ThinLTO cache", + command=upload_thinlto_cache, + command_args=[thinlto_cache_s3, thinlto_cache_flavour(cmake_cmd)], + ) ) + + run_shell("sccache stats", "sccache --show-stats") if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") clang_tidy_cache_log = "./ci/tmp/clang-tidy-cache.log" diff --git a/ci/praktika/runner.py b/ci/praktika/runner.py index 3d6013ce2d9f1..a09b09c3471e3 100644 --- a/ci/praktika/runner.py +++ b/ci/praktika/runner.py @@ -481,19 +481,6 @@ def _run( if not job.timeout_shell_cleanup: job.timeout_shell_cleanup = f"docker rm -f {container_name}" workdir = f"--workdir={current_dir}" - # `--volume-if-exists=HOST:CONTAINER` mounts HOST only on runners - # that have it (e.g. an attached cache volume), instead of creating - # an empty directory everywhere like `--volume` does. - optional_volumes = [ - s for s in settings if s.startswith("--volume-if-exists=") - ] - for setting in optional_volumes: - settings.remove(setting) - volume = setting.removeprefix("--volume-if-exists=") - if Path(volume.split(":")[0]).exists(): - settings.append(f"--volume={volume}") - else: - print(f"NOTE: optional volume [{volume}] is not present on this runner") for setting in settings: if setting.startswith("--volume"): volume = setting.removeprefix("--volume=").split(":")[0] From d55de5fde5f8f442fe07b220e4ed96fefacea3a3 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:36:34 -0400 Subject: [PATCH 10/14] Map the Hetzner cache bucket to its public endpoint `S3.copy_file_to_s3` uploaded the ThinLTO cache archive, then raised `KeyError: 'altinity-builds-cache'` while building the link it returns, which failed the release builds after a successful upload. https://github.com/Altinity/ClickHouse/actions/runs/41109373051/job/124345996580 Co-Authored-By: Claude Opus 5.5 --- ci/settings/altinity_overrides.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 101c6f492a047..aac84ca4835dd 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -29,6 +29,8 @@ class RunnerLabels: S3_BUCKET_TO_HTTP_ENDPOINT = { S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT, S3_REPORT_BUCKET_NAME: S3_REPORT_BUCKET_HTTP_ENDPOINT, + # S3.copy_file_to_s3 returns a public link to every upload + "altinity-builds-cache": "altinity-builds-cache.hel1.your-objectstorage.com", } ENABLE_ARTIFACTS_REPORT = True From 0d136eafa537f8101929d362e6bdc65712562378 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:49:48 -0500 Subject: [PATCH 11/14] Stop building `clickhouse-examples` in the ARM release builds Nothing in Altinity CI uses it: its only consumer, `Parser memory check`, is disabled and compares against a master binary on the upstream bucket. Building it gave `arm_release` three ThinLTO links for two link slots. `clickhouse-examples` and `clickhouse-keeper` held both slots for 17 minutes while the static libraries `clickhouse` needs waited behind them in the same pool, so the main binary linked alone afterwards for another 26 minutes. https://github.com/Altinity/ClickHouse/actions/runs/41151170196/job/125318650979 Related: https://github.com/Altinity/ClickHouse/pull/2716 Co-Authored-By: Claude Opus 5.5 --- ci/defs/job_configs.py | 5 ++--- ci/workflows/fast_builds.py | 5 +++-- ci/workflows/master.py | 5 +++-- ci/workflows/pull_request.py | 5 +++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index b79745981beb2..ebecfb315cb49 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -492,10 +492,9 @@ class JobConfigs: # master data directly comparable to a PR build (official master builds # keep debug symbols, PR builds strip them). The hook uploads only for the # arm variant (see _PROFILED_BUILDS in build_profile_hook.py). + # NOTE (strtgbb): no --build-examples for the arm warmup, PR builds no longer build clickhouse-examples sccache_warmup_build_jobs = [ - job.set_command(f"{job.command} --build-examples") - if f"({BuildTypes.ARM_RELEASE_PR_CACHE_WARMUP})" in job.name - else job + job for job in common_build_job_config.set_post_hooks( post_hooks=[ # "python3 ./ci/jobs/scripts/job_hooks/build_profile_hook.py", diff --git a/ci/workflows/fast_builds.py b/ci/workflows/fast_builds.py index 0e10c801cfd7f..d55b75d675485 100644 --- a/ci/workflows/fast_builds.py +++ b/ci/workflows/fast_builds.py @@ -19,7 +19,8 @@ builds_for_release_branch = [ job.unset_provides("unittest") - for job in JobConfigs.build_jobs + JobConfigs.release_build_jobs_with_examples + # NOTE (strtgbb): no clickhouse-examples, nothing uses it and it delays the ThinLTO link of clickhouse + for job in JobConfigs.build_jobs + JobConfigs.release_build_jobs if "coverage" not in job.name ] @@ -58,7 +59,7 @@ artifacts=[ *clickhouse_binaries_with_tags, *ArtifactConfigs.clickhouse_binaries_gh, - ArtifactConfigs.clickhouse_examples, + # ArtifactConfigs.clickhouse_examples, *ArtifactConfigs.clickhouse_debians, *ArtifactConfigs.clickhouse_rpms, *ArtifactConfigs.clickhouse_tgzs, diff --git a/ci/workflows/master.py b/ci/workflows/master.py index fbe4d824b1494..bede045233dd6 100644 --- a/ci/workflows/master.py +++ b/ci/workflows/master.py @@ -44,7 +44,8 @@ *JobConfigs.build_jobs, # *JobConfigs.build_llvm_coverage_job, JobConfigs.coverage_build_jobs[1], - *JobConfigs.release_build_jobs_with_examples, + # NOTE (strtgbb): no clickhouse-examples, nothing uses it and it delays the ThinLTO link of clickhouse + *JobConfigs.release_build_jobs, *JobConfigs.sccache_warmup_build_jobs, # *[ # NOTE (strtgbb): we don't run special build jobs @@ -104,7 +105,7 @@ *AltinityArtifactConfigs.signed_hashes, ArtifactConfigs.fuzzers, ArtifactConfigs.fuzzers_corpus, - ArtifactConfigs.clickhouse_examples, + # ArtifactConfigs.clickhouse_examples, # *ArtifactConfigs.llvm_profdata_file, # ArtifactConfigs.llvm_coverage_info_file, ], diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index 67d9239cc5b21..9180e760ae8df 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -94,7 +94,8 @@ ], *[ job.set_run_after(REGULAR_BUILD_NAMES) - for job in JobConfigs.release_build_jobs_with_examples + # NOTE (strtgbb): no clickhouse-examples, nothing uses it and it delays the ThinLTO link of clickhouse + for job in JobConfigs.release_build_jobs ], # *[ # NOTE (strtgbb): we don't run special build jobs # job.set_run_after(CORE_BLOCKING_JOB_NAMES) @@ -255,7 +256,7 @@ ArtifactConfigs.wasm_parser, ArtifactConfigs.fuzzers, ArtifactConfigs.fuzzers_corpus, - ArtifactConfigs.clickhouse_examples, + # ArtifactConfigs.clickhouse_examples, # *ArtifactConfigs.llvm_profdata_file, # ArtifactConfigs.llvm_coverage_info_file, ArtifactConfigs.toolchain_pgo_bolt_amd, From 1fda95195560b65c9fae58638a0d70cd25bfb4c1 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:24:45 -0500 Subject: [PATCH 12/14] get the artifact bucket from the repository variable Uploads and the workflow report read S3_ARTIFACT_BUCKET. Generation still runs when the variable is unset, and the checked-in workflows emit vars.S3_ARTIFACT_BUCKET. Co-authored-by: Cursor --- .../create_workflow_report.py | 142 +++++++++++------- .github/workflows/fast_builds.yml | 3 +- .github/workflows/grype_scan.yml | 2 +- .github/workflows/init_praktika.yml | 1 + .github/workflows/master.yml | 3 +- .github/workflows/pull_request.yml | 3 +- .github/workflows/pull_request_community.yml | 3 +- .../workflows/regression-reusable-suite.yml | 3 +- ci/defs/job_configs.py | 4 +- ci/praktika/validator.py | 23 +-- ci/praktika/yaml_additional_templates.py | 2 +- ci/praktika/yaml_generator.py | 3 + ci/settings/altinity_overrides.py | 10 +- tests/ci/env_helper.py | 7 +- .../03536_ch_as_client_and_local.sh | 2 +- 15 files changed, 129 insertions(+), 82 deletions(-) diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index 705c9ad6ff101..4cb3e09d8caba 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -20,16 +20,35 @@ from clickhouse_driver import Client from clickhouse_driver.errors import ServerException import boto3 -from botocore.exceptions import NoCredentialsError +from botocore.exceptions import ClientError, NoCredentialsError import yaml DATABASE_HOST_VAR = "CHECKS_DATABASE_HOST" DATABASE_USER_VAR = "CLICKHOUSE_TEST_STAT_LOGIN" DATABASE_PASSWORD_VAR = "CLICKHOUSE_TEST_STAT_PASSWORD" -S3_BUCKET = "altinity-build-artifacts" +S3_BUCKET = os.getenv("S3_ARTIFACT_BUCKET") GITHUB_REPO = "Altinity/ClickHouse" -GITHUB_GIT_URL = f"https://github.com/{GITHUB_REPO}.git" + + +def _repo_and_run_id(actions_run_url: str) -> tuple[str, str]: + parts = urllib.parse.urlparse(actions_run_url).path.strip("/").split("/") + if ( + len(parts) >= 5 + and parts[2] == "actions" + and parts[3] == "runs" + and parts[0] + and parts[1] + and parts[4] + ): + return f"{parts[0]}/{parts[1]}", parts[4] + return GITHUB_REPO, actions_run_url.rstrip("/").split("/")[-1] + + +def _github_git_url(repo: str) -> str: + return f"https://github.com/{repo}.git" + + GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN") # Release tags are fetched here rather than refs/tags/ so local tags are never touched. RELEASE_TAG_REF_PREFIX = "refs/workflow-report/tags/" @@ -73,7 +92,9 @@ def _results_link_has_ref(target_url: str | None, ref: str) -> bool: return f"REF={ref}&" in target_url or f"/REFs/{ref}/" in target_url -def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame: +def get_commit_statuses( + sha: str, pr_number: int, ref: str, repo: str = GITHUB_REPO +) -> pd.DataFrame: """ Fetch commit statuses for a given SHA and return as a pandas DataFrame. Handles pagination to get all statuses. @@ -95,7 +116,7 @@ def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame: "Accept": "application/vnd.github.v3+json", } - url = f"https://api.github.com/repos/{GITHUB_REPO}/commits/{sha}/statuses" + url = f"https://api.github.com/repos/{repo}/commits/{sha}/statuses" all_data = [] @@ -150,7 +171,7 @@ def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame: ).reset_index(drop=True) -def get_pr_info_from_number(pr_number: str) -> dict: +def get_pr_info_from_number(pr_number: str, repo: str = GITHUB_REPO) -> dict: """ Fetch pull request information for a given PR number. @@ -165,7 +186,7 @@ def get_pr_info_from_number(pr_number: str) -> dict: "Accept": "application/vnd.github.v3+json", } - url = f"https://api.github.com/repos/{GITHUB_REPO}/pulls/{pr_number}" + url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" response = requests.get(url, headers=headers) if response.status_code != 200: @@ -176,7 +197,7 @@ def get_pr_info_from_number(pr_number: str) -> dict: return response.json() -def get_run_details(run_id: str) -> dict: +def get_run_details(run_id: str, repo: str = GITHUB_REPO) -> dict: """ Fetch run details for a given run URL. """ @@ -185,7 +206,7 @@ def get_run_details(run_id: str) -> dict: "Accept": "application/vnd.github.v3+json", } - url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}" + url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}" response = requests.get(url, headers=headers) if response.status_code != 200: @@ -407,9 +428,11 @@ def _git_is_shallow(cwd: str | None) -> bool: return p.stdout.strip() == "true" -def _git_fetch(options: list[str], refspecs: list[str], cwd: str | None) -> None: +def _git_fetch( + options: list[str], refspecs: list[str], cwd: str | None, repo: str = GITHUB_REPO +) -> None: p = subprocess.run( - ["git", "fetch", *options, GITHUB_GIT_URL, *refspecs], + ["git", "fetch", *options, _github_git_url(repo), *refspecs], cwd=cwd, capture_output=True, text=True, @@ -466,7 +489,9 @@ def _filter_releases_to_version_line( ] -def _fetch_release_tags(releases: list[dict], cwd: str) -> None: +def _fetch_release_tags( + releases: list[dict], cwd: str, repo: str = GITHUB_REPO +) -> None: """Fetch release tags into RELEASE_TAG_REF_PREFIX; skip tags missing remotely.""" for rel in releases: tag_name = rel.get("tag_name") @@ -477,6 +502,7 @@ def _fetch_release_tags(releases: list[dict], cwd: str) -> None: ["--no-tags"], [f"+refs/tags/{tag_name}:{RELEASE_TAG_REF_PREFIX}{tag_name}"], cwd, + repo, ) except Exception as e: # NOTE (strtgbb): one deleted remote tag must not abort the section, but @@ -484,17 +510,17 @@ def _fetch_release_tags(releases: list[dict], cwd: str) -> None: print(f"WARNING: skipping release tag {tag_name}: {e}") -def _fetch_commit(commit_sha: str, cwd: str) -> None: - """Fetch the workflow commit from GITHUB_GIT_URL. +def _fetch_commit(commit_sha: str, cwd: str, repo: str = GITHUB_REPO) -> None: + """Fetch the workflow commit from the repository under test. Local checkouts might be a different branch and might not have the run SHA yet. --filter/--unshallow would convert a full clone into a partial one, so those options are used only when the checkout is already shallow. """ if _git_is_shallow(cwd): - _git_fetch(["--filter=tree:0", "--unshallow"], [commit_sha], cwd) + _git_fetch(["--filter=tree:0", "--unshallow"], [commit_sha], cwd, repo) else: - _git_fetch(["--no-tags"], [commit_sha], cwd) + _git_fetch(["--no-tags"], [commit_sha], cwd, repo) def get_prs_in_release_dataframe( @@ -514,7 +540,7 @@ def get_prs_in_release_dataframe( Returns (dataframe, baseline commit time as UTC YYYY-MM-DDTHH:MM:SSZ or None). """ - _fetch_commit(commit_sha, cwd) + _fetch_commit(commit_sha, cwd, repo) resolved = _git_rev_parse(commit_sha, cwd) if not resolved: raise Exception(f"Cannot resolve commit: {commit_sha!r}") @@ -523,7 +549,7 @@ def get_prs_in_release_dataframe( releases = _filter_releases_to_version_line( _get_github_releases(repo), branch_ref, commit_sha, cwd ) - _fetch_release_tags(releases, cwd) + _fetch_release_tags(releases, cwd, repo) _, baseline_sha, _ = _find_release_baseline( branch_ref, releases, cwd, commit_sha=commit_sha @@ -564,7 +590,9 @@ def _parse_release_ref(ref_name: str) -> tuple[str, str, str | None] | None: return None -def _prs_in_release_search_url(branch_name: str, baseline_time: str | None) -> str | None: +def _prs_in_release_search_url( + branch_name: str, baseline_time: str | None, repo: str = GITHUB_REPO +) -> str | None: if not baseline_time: return None # Tag-triggered runs expose the tag as head_branch; search needs the PR base @@ -575,7 +603,7 @@ def _prs_in_release_search_url(branch_name: str, baseline_time: str | None) -> s branch_name = f"{flavour}-{minor_version}" query = f"is:pr base:{branch_name} merged:>{baseline_time}" return ( - f"https://github.com/{GITHUB_REPO}/pulls?" + f"https://github.com/{repo}/pulls?" + urllib.parse.urlencode({"q": query}) ) @@ -890,15 +918,20 @@ def get_new_fails_this_pr( return new_fails_df -def _workflow_config_s3_url(pr_number: int, commit_sha: str, ref_name: str) -> str: +def _s3_get_json(key: str): + s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL")) + response = s3_client.get_object(Bucket=S3_BUCKET, Key=key) + return json.loads(response["Body"].read().decode("utf-8")) + + +def _workflow_config_s3_key(pr_number: int, commit_sha: str, ref_name: str) -> str: if pr_number == 0: return ( - f"https://{S3_BUCKET}.s3.amazonaws.com/REFs/{ref_name}/{commit_sha}/" + f"REFs/{ref_name}/{commit_sha}/" "config_workflow/workflow_config_masterci.json" ) return ( - f"https://{S3_BUCKET}.s3.amazonaws.com/PRs/{pr_number}/{commit_sha}/" - "config_workflow/workflow_config_pr.json" + f"PRs/{pr_number}/{commit_sha}/config_workflow/workflow_config_pr.json" ) @@ -908,18 +941,14 @@ def get_workflow_config(pr_number: int, commit_sha: str, ref_name: str) -> dict: On failure, warn and return a minimal empty config (same shape recreation uses). """ - url = _workflow_config_s3_url(pr_number, commit_sha, ref_name) + key = _workflow_config_s3_key(pr_number, commit_sha, ref_name) try: - response = requests.get(url, timeout=30) - if response.status_code != 200: - print( - f"WARNING:Failed to fetch workflow config from {url}: " - f"{response.status_code}" - ) - return {"cache_jobs": {}} - return response.json() + return _s3_get_json(key) except Exception as e: - print(f"WARNING:Failed to fetch workflow config from {url}: {e}") + print( + f"WARNING:Failed to fetch workflow config from s3://{S3_BUCKET}/{key}: {e}", + file=sys.stderr, + ) return {"cache_jobs": {}} @@ -1130,18 +1159,16 @@ def fetch_workflow_result_json( ref_param = f"PR={pr_number}" workflow_name = "PR" - status_file = f"result_{workflow_name.lower()}.json" - s3_path = ( - f"https://{S3_BUCKET}.s3.amazonaws.com/" - f"{ref_param.replace('=', 's/')}/{commit_sha}/{status_file}" + key = ( + f"{ref_param.replace('=', 's/')}/{commit_sha}/" + f"result_{workflow_name.lower()}.json" ) try: - response = requests.get(s3_path, timeout=30) - if response.status_code != 200: - return None - return response.json() + return _s3_get_json(key) + except ClientError: + return None except Exception as e: - print(f"WARNING:Failed to fetch workflow result from {s3_path}: {e}") + print(f"WARNING:Failed to fetch workflow result from s3://{S3_BUCKET}/{key}: {e}") return None @@ -1182,7 +1209,9 @@ def add_labels_to_checks_fails( return df[cols] -def format_results_as_html_table(results, *, branch_name: str = "") -> str: +def format_results_as_html_table( + results, *, branch_name: str = "", repo: str = GITHUB_REPO +) -> str: if not isinstance(results, pd.DataFrame): return results @@ -1213,7 +1242,7 @@ def format_col_name(col_name: str) -> str: f'{s}' ), "PR Number": lambda n: url_to_html_link( - f"https://github.com/{GITHUB_REPO}/pull/{n}" + f"https://github.com/{repo}/pull/{n}" ), "PR Labels": lambda labels: format_pr_labels_with_verification( labels, branch_name=branch_name @@ -1361,12 +1390,14 @@ def create_workflow_report( print(f"{DATABASE_PASSWORD_VAR} is not set") if not GITHUB_TOKEN: print("GITHUB_TOKEN is not set") - if not all([host, user, password, GITHUB_TOKEN]): + if not S3_BUCKET: + print("S3_ARTIFACT_BUCKET is not set") + if not all([host, user, password, GITHUB_TOKEN, S3_BUCKET]): raise Exception("Required environment variables are not set") - run_id = actions_run_url.split("/")[-1] + github_repo, run_id = _repo_and_run_id(actions_run_url) - run_details = get_run_details(run_id) + run_details = get_run_details(run_id, github_repo) branch_name = run_details.get("head_branch", "unknown branch") if pr_number is None or commit_sha is None: if pr_number is None: @@ -1391,7 +1422,9 @@ def create_workflow_report( results_dfs = { "prs_in_release": [], - "job_statuses": get_commit_statuses(commit_sha, pr_number, branch_name), + "job_statuses": get_commit_statuses( + commit_sha, pr_number, branch_name, github_repo + ), "checks_fails": add_labels_to_checks_fails( get_checks_fails(db_client, commit_sha, branch_name), workflow_result, @@ -1408,11 +1441,11 @@ def create_workflow_report( if pr_number == 0 and not mark_preview: try: prs_df, baseline_time = get_prs_in_release_dataframe( - branch_name, cwd=os.getcwd(), commit_sha=commit_sha + branch_name, cwd=os.getcwd(), repo=github_repo, commit_sha=commit_sha ) results_dfs["prs_in_release"] = prs_df prs_in_release_search_url = _prs_in_release_search_url( - branch_name, baseline_time + branch_name, baseline_time, github_repo ) except Exception as e: print(f"Error in get_prs_in_release_dataframe: {e}") @@ -1442,8 +1475,8 @@ def create_workflow_report( pr_info_html = f"Release ({branch_name})" else: try: - pr_info = get_pr_info_from_number(pr_number) - pr_info_html = f""" + pr_info = get_pr_info_from_number(pr_number, github_repo) + pr_info_html = f""" #{pr_info.get("number")} ({pr_info.get("base", {}).get('ref')} <- {pr_info.get("head", {}).get('ref')}) {pr_info.get("title")} """ results_dfs["pr_new_fails"] = get_new_fails_this_pr( @@ -1478,7 +1511,7 @@ def create_workflow_report( # Define the context for rendering context = { "title": "ClickHouse® CI Workflow Run Report", - "github_repo": GITHUB_REPO, + "github_repo": github_repo, "s3_bucket": S3_BUCKET, "pr_info_html": pr_info_html, "pr_number": pr_number, @@ -1513,6 +1546,7 @@ def create_workflow_report( else format_results_as_html_table( results_dfs["prs_in_release"], branch_name=branch_name, + repo=github_repo, ) ), "ci_jobs_status_html": format_results_as_html_table( diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 26a61911e0462..e99daf68a6190 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -47,6 +47,7 @@ env: ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} @@ -83,7 +84,7 @@ jobs: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY - name: Prepare env script diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index f57125e695e8a..249d66394138e 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -85,7 +85,7 @@ jobs: if: always() id: upload_results env: - S3_BUCKET: "altinity-build-artifacts" + S3_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} PR_NUMBER: ${{ env.PR_NUMBER || github.event.pull_request.number || 0 }} DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} diff --git a/.github/workflows/init_praktika.yml b/.github/workflows/init_praktika.yml index 42aaaeedde00f..7b083d8f7529d 100644 --- a/.github/workflows/init_praktika.yml +++ b/.github/workflows/init_praktika.yml @@ -12,6 +12,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} jobs: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 90d8b9e15d717..d083682d537c6 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -62,6 +62,7 @@ env: ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} @@ -98,7 +99,7 @@ jobs: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY - name: Prepare env script diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7924f9837029b..412c04d7b8718 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -37,6 +37,7 @@ env: ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} @@ -74,7 +75,7 @@ jobs: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY - name: Prepare env script diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index bf1a63d7d667f..e711dba7b23cc 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -20,6 +20,7 @@ env: DISABLE_CI_MERGE_COMMIT: ${{ vars.DISABLE_CI_MERGE_COMMIT || '0' }} DISABLE_CI_CACHE: ${{ github.event.inputs.no_cache || '0' }} CHECKOUT_REF: ${{ vars.DISABLE_CI_MERGE_COMMIT == '1' && github.event.pull_request.head.sha || '' }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} @@ -57,7 +58,7 @@ jobs: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY - name: Prepare env script diff --git a/.github/workflows/regression-reusable-suite.yml b/.github/workflows/regression-reusable-suite.yml index 51d064ea0f011..6cf3e3ffbdd26 100644 --- a/.github/workflows/regression-reusable-suite.yml +++ b/.github/workflows/regression-reusable-suite.yml @@ -93,6 +93,7 @@ jobs: build_sha: ${{ inputs.build_sha }} pr_number: ${{ github.event.number }} artifacts: builds + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} regression_args: ${{ inputs.regression_args }} # Args args: --test-to-end @@ -130,7 +131,7 @@ jobs: - name: 📦 Get deb url env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ + S3_BASE_URL: https://${{ vars.S3_ARTIFACT_BUCKET }}.s3.amazonaws.com/ PR_NUMBER: ${{ github.event.pull_request.number || 0 }} run: | REPORTS_PATH=${{ runner.temp }}/reports_dir diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index eaec79a77d782..e14a5294a36dd 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,7 +29,7 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" - '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' + '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"+--env=S3_ARTIFACT_BUCKET="$S3_ARTIFACT_BUCKET"' ) BINARY_DOCKER_COMMAND_SANITIZER = ( BINARY_DOCKER_COMMAND + "+--security-opt seccomp=unconfined" @@ -254,7 +254,7 @@ class JobConfigs: runs_on=RunnerLabels.AMD_LARGE, command="python3 ./ci/jobs/fast_test.py", # --network=host required for ec2 metadata http endpoint to work - run_in_docker="altinityinfra/fasttest+--network=host+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/var/log:/var/log+--volume=.:/ClickHouse+--env=AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID+--env=AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY", + run_in_docker="altinityinfra/fasttest+--network=host+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/var/log:/var/log+--volume=.:/ClickHouse+--env=AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID+--env=AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY+--env=S3_ARTIFACT_BUCKET=$S3_ARTIFACT_BUCKET", digest_config=fast_test_digest_config, result_name_for_cidb="Tests", needs_submodules=True, diff --git a/ci/praktika/validator.py b/ci/praktika/validator.py index 6ade4d53033a9..ebc7df543c794 100644 --- a/ci/praktika/validator.py +++ b/ci/praktika/validator.py @@ -143,10 +143,11 @@ def is_valid_cron_field(field: str) -> bool: f"Must be Artifact.Config type, not {type(artifact)}: [{artifact}]", workflow.name, ) - if artifact.is_s3_artifact(): - assert ( - Settings.S3_ARTIFACT_PATH - ), "Provide S3_ARTIFACT_PATH setting in any .py file in ./ci/settings/* to be able to use s3 for artifacts" + # NOTE (strtgbb): S3_ARTIFACT_BUCKET is set on the job, not when workflows are generated + # if artifact.is_s3_artifact(): + # assert ( + # Settings.S3_ARTIFACT_PATH + # ), "Provide S3_ARTIFACT_PATH setting in any .py file in ./ci/settings/* to be able to use s3 for artifacts" for job in workflow.jobs: if job.requires and workflow.artifacts: @@ -164,9 +165,10 @@ def is_valid_cron_field(field: str) -> bool: Settings.CI_CONFIG_RUNS_ON ), f"Runner label to run workflow config job must be provided via CACHE_CONFIG_RUNS_ON setting if enable_cache=True, workflow [{workflow.name}]" - assert ( - Settings.CACHE_S3_PATH - ), f"CACHE_S3_PATH Setting must be defined if enable_cache=True, workflow [{workflow.name}]" + # NOTE (strtgbb): S3_ARTIFACT_BUCKET is set on the job, not when workflows are generated + # assert ( + # Settings.CACHE_S3_PATH + # ), f"CACHE_S3_PATH Setting must be defined if enable_cache=True, workflow [{workflow.name}]" if workflow.dockers: if Settings.ENABLE_MULTIPLATFORM_DOCKER_IN_ONE_JOB == False: @@ -200,9 +202,10 @@ def is_valid_cron_field(field: str) -> bool: ) if workflow.enable_report: - assert ( - Settings.S3_REPORT_BUCKET - ), f"S3_REPORT_BUCKET Setting must be defined if enable_html=True, workflow [{workflow.name}]" + # NOTE (strtgbb): S3_ARTIFACT_BUCKET is set on the job, not when workflows are generated + # assert ( + # Settings.S3_REPORT_BUCKET + # ), f"S3_REPORT_BUCKET Setting must be defined if enable_html=True, workflow [{workflow.name}]" assert ( Settings.S3_BUCKET_TO_HTTP_ENDPOINT ), f"S3_BUCKET_TO_HTTP_ENDPOINT Setting must be defined if enable_html=True, workflow [{workflow.name}]" diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 2869279da9962..ac8102a5dce20 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -33,7 +33,7 @@ class AltinityWorkflowTemplates: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY """ # Additional jobs diff --git a/ci/praktika/yaml_generator.py b/ci/praktika/yaml_generator.py index d5ec52d57d276..6e51ac60fc6f0 100644 --- a/ci/praktika/yaml_generator.py +++ b/ci/praktika/yaml_generator.py @@ -729,6 +729,9 @@ def _all_needs(job_name: str) -> set: format_kwargs[ "ENV_SECRETS" ] += AltinityWorkflowTemplates.ADDITIONAL_GLOBAL_ENV + format_kwargs[ + "ENV_SECRETS" + ] += " S3_ARTIFACT_BUCKET: ${{{{ vars.S3_ARTIFACT_BUCKET }}}}\n" template_1 = base_template.strip().format( NAME=self.workflow_config.name, diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 101c6f492a047..687bf8e69a63d 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -1,9 +1,9 @@ import os -S3_BUCKET_NAME = "altinity-build-artifacts" -S3_REPORT_BUCKET_NAME = "altinity-build-artifacts" -S3_BUCKET_HTTP_ENDPOINT = "altinity-build-artifacts.s3.amazonaws.com" -S3_REPORT_BUCKET_HTTP_ENDPOINT = "altinity-build-artifacts.s3.amazonaws.com" +S3_BUCKET_NAME = os.environ.get("S3_ARTIFACT_BUCKET") or "" +S3_REPORT_BUCKET_NAME = S3_BUCKET_NAME +S3_BUCKET_HTTP_ENDPOINT = f"{S3_BUCKET_NAME}.s3.amazonaws.com" +S3_REPORT_BUCKET_HTTP_ENDPOINT = S3_BUCKET_HTTP_ENDPOINT class RunnerLabels: @@ -23,7 +23,7 @@ class RunnerLabels: DOCKER_BUILD_AND_MERGE_RUNS_ON = RunnerLabels.STYLE_CHECK_AMD DOCKER_BUILD_ARM_RUNS_ON = RunnerLabels.STYLE_CHECK_ARM -CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" +CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" if S3_BUCKET_NAME else "" HTML_S3_PATH = S3_REPORT_BUCKET_NAME S3_REPORT_BUCKET = S3_REPORT_BUCKET_NAME S3_BUCKET_TO_HTTP_ENDPOINT = { diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index ac22e90a92880..b2df6ab3964c0 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -31,10 +31,11 @@ RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) S3_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") -S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-build-artifacts") -S3_BUILDS_BUCKET_PUBLIC = "altinity-build-artifacts" +S3_ARTIFACT_BUCKET = os.getenv("S3_ARTIFACT_BUCKET") +S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET") or S3_ARTIFACT_BUCKET +S3_BUILDS_BUCKET_PUBLIC = S3_ARTIFACT_BUCKET S3_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") -S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") +S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET") or S3_ARTIFACT_BUCKET S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) diff --git a/tests/queries/0_stateless/03536_ch_as_client_and_local.sh b/tests/queries/0_stateless/03536_ch_as_client_and_local.sh index 323843d0b0c72..ec4d5ce92bc16 100755 --- a/tests/queries/0_stateless/03536_ch_as_client_and_local.sh +++ b/tests/queries/0_stateless/03536_ch_as_client_and_local.sh @@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CLICKHOUSE_BINARY_CH=${CLICKHOUSE_BINARY/clickhouse/ch} +CLICKHOUSE_BINARY_CH=${CLICKHOUSE_BINARY/%clickhouse/ch} # Invocation with unknown tool name prints help: ${CLICKHOUSE_BINARY} test 2>&1 | grep -F 'Use one of the following commands' From da3051ac1c58e71ec26e7f056e0e1d88be5d4059 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:12:43 -0500 Subject: [PATCH 13/14] parameterize the submodule cache bucket --- .github/workflows/fast_builds.yml | 1 + .github/workflows/master.yml | 1 + .github/workflows/pull_request.yml | 1 + .github/workflows/pull_request_community.yml | 1 + ci/praktika/yaml_generator.py | 3 +++ ci/settings/altinity_overrides.py | 10 +++++----- 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index e99daf68a6190..660d5edeaec98 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -48,6 +48,7 @@ env: HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} + HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }} diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d083682d537c6..59451790d0301 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -63,6 +63,7 @@ env: HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} + HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 412c04d7b8718..7e6d53cf12872 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -38,6 +38,7 @@ env: HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} + HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }} diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index e711dba7b23cc..f4e2bc78d6c1c 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -21,6 +21,7 @@ env: DISABLE_CI_CACHE: ${{ github.event.inputs.no_cache || '0' }} CHECKOUT_REF: ${{ vars.DISABLE_CI_MERGE_COMMIT == '1' && github.event.pull_request.head.sha || '' }} S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} + HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }} diff --git a/ci/praktika/yaml_generator.py b/ci/praktika/yaml_generator.py index 6e51ac60fc6f0..bb0f5a61a85b9 100644 --- a/ci/praktika/yaml_generator.py +++ b/ci/praktika/yaml_generator.py @@ -732,6 +732,9 @@ def _all_needs(job_name: str) -> set: format_kwargs[ "ENV_SECRETS" ] += " S3_ARTIFACT_BUCKET: ${{{{ vars.S3_ARTIFACT_BUCKET }}}}\n" + format_kwargs[ + "ENV_SECRETS" + ] += " HETZNER_CACHE_BUCKET: ${{{{ vars.HETZNER_CACHE_BUCKET }}}}\n" template_1 = base_template.strip().format( NAME=self.workflow_config.name, diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 687bf8e69a63d..9558a7a4ca809 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -64,10 +64,11 @@ class RunnerLabels: ENABLE_SUBMODULE_CLONE_AUTH = True # Most runners are in Hetzner (hel1): cache there to avoid AWS egress. -CI_CACHE_BUCKET = "altinity-builds-cache" -SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" -S3_EXTERNAL_BUCKETS = { - CI_CACHE_BUCKET: { +CI_CACHE_BUCKET = os.environ.get("HETZNER_CACHE_BUCKET") or "" +SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" if CI_CACHE_BUCKET else "" +S3_EXTERNAL_BUCKETS = {} +if CI_CACHE_BUCKET: + S3_EXTERNAL_BUCKETS[CI_CACHE_BUCKET] = { "endpoint_url": "https://hel1.your-objectstorage.com", "region": "hel1", "access_key_env": "HETZNER_S3_ACCESS_KEY", @@ -76,7 +77,6 @@ class RunnerLabels: # netcup runners get ~2.5 MB/s per connection from hel1 "download_concurrency": 32, } -} DISABLED_WORKFLOWS = [ "backport_branches.py", From fd6d813e1a2f1695b06c47c8e6b94d385e4d24c3 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:10:10 -0500 Subject: [PATCH 14/14] unify the two bucket maps --- ci/praktika/info.py | 8 ++++---- ci/praktika/s3.py | 17 +++++++++++++---- ci/praktika/settings.py | 7 +++---- ci/praktika/validator.py | 12 ++++++------ ci/settings/altinity_overrides.py | 11 ++++------- ci/settings/settings.py | 6 +++--- 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/ci/praktika/info.py b/ci/praktika/info.py index c77dc98a08ab3..e38853a6e38ee 100644 --- a/ci/praktika/info.py +++ b/ci/praktika/info.py @@ -203,9 +203,9 @@ def get_specific_report_url( assert branch ref_param = f"REF={branch}" path = Settings.S3_REPORT_BUCKET - for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items(): + for bucket, config in (Settings.S3_EXTERNAL_BUCKETS or {}).items(): if bucket in path: - path = path.replace(bucket, endpoint) + path = path.replace(bucket, config["http_endpoint"]) break workflow_name = workflow_name or self.env.WORKFLOW_NAME res = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?{ref_param}&sha={sha}&name_0={urllib.parse.quote(workflow_name, safe='')}" @@ -223,9 +223,9 @@ def get_specific_report_url_static(pr_number, branch, sha, job_name, workflow_na assert branch ref_param = f"REF={branch}" path = Settings.S3_REPORT_BUCKET - for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items(): + for bucket, config in (Settings.S3_EXTERNAL_BUCKETS or {}).items(): if bucket in path: - path = path.replace(bucket, endpoint) + path = path.replace(bucket, config["http_endpoint"]) break res = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?{ref_param}&sha={sha}&name_0={urllib.parse.quote(workflow_name, safe='')}" if job_name: diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index 863718e8b915d..871c0e6837585 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -65,7 +65,17 @@ class S3: @classmethod def _external_bucket_config(cls, bucket): - return (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + config = (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + if not config or not config.get("endpoint_url"): + return None + return config + + @classmethod + def _http_endpoint(cls, bucket): + config = (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) or {} + endpoint = config.get("http_endpoint") + assert endpoint, f"S3 bucket [{bucket}] has no http_endpoint in S3_EXTERNAL_BUCKETS" + return endpoint @classmethod def _reads_anonymously(cls, bucket): @@ -280,8 +290,7 @@ def _upload(): print(f"WARNING: Failed to record upload usage for {local_path}: {e}") bucket = s3_path.split("/")[0] - endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket] - assert endpoint + endpoint = cls._http_endpoint(bucket) return quote(f"https://{s3_full_path}".replace(bucket, endpoint), safe=":/?&=") @classmethod @@ -643,7 +652,7 @@ def _upload_file_to_s3( def _dump_urls(cls, s3_path): # TODO: add support for path with '*' bucket, name = s3_path.split("/")[0], s3_path.split("/")[-1] - endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket] + endpoint = cls._http_endpoint(bucket) with open(Settings.ARTIFACT_URLS_FILE, "w", encoding="utf-8") as f: json.dump( diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index 3385cd6b57368..cd26c7dac0a9c 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -128,9 +128,9 @@ class _Settings: # Optional: upstream report bucket to merge issue catalogs from (e.g. "clickhouse-test-reports") S3_UPSTREAM_REPORT_BUCKET: str = "" HTML_PAGE_FILE: str = "./ci/praktika/json.html" - S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None - # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read, download_concurrency}} - # The keys are read from the named env vars; with public_read, jobs without them read unsigned. + # {bucket: {http_endpoint, endpoint_url?, region?, access_key_env?, secret_key_env?, public_read?, download_concurrency?}} + # http_endpoint is the public host spliced into links. endpoint_url marks a non-AWS bucket. + # Keys are read from the named env vars; with public_read, jobs without them read unsigned. # download_concurrency (optional): parallel ranged GETs, for links with a low rate per connection. S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"]) @@ -182,7 +182,6 @@ class _Settings: "CLOUD_INFRASTRUCTURE_CONFIG_PATH", "EVENT_FEED_S3_PATH", "AWS_REGION", - "S3_BUCKET_TO_HTTP_ENDPOINT", "S3_EXTERNAL_BUCKETS", "SUBMODULE_CACHE_S3_PATH", "TEXT_CONTENT_EXTENSIONS", diff --git a/ci/praktika/validator.py b/ci/praktika/validator.py index ebc7df543c794..382e76dd8bfcf 100644 --- a/ci/praktika/validator.py +++ b/ci/praktika/validator.py @@ -206,13 +206,13 @@ def is_valid_cron_field(field: str) -> bool: # assert ( # Settings.S3_REPORT_BUCKET # ), f"S3_REPORT_BUCKET Setting must be defined if enable_html=True, workflow [{workflow.name}]" + report_bucket = Settings.S3_REPORT_BUCKET.split("/")[0] + http_endpoint = ( + (Settings.S3_EXTERNAL_BUCKETS or {}).get(report_bucket) or {} + ).get("http_endpoint") assert ( - Settings.S3_BUCKET_TO_HTTP_ENDPOINT - ), f"S3_BUCKET_TO_HTTP_ENDPOINT Setting must be defined if enable_html=True, workflow [{workflow.name}]" - assert ( - Settings.S3_REPORT_BUCKET.split("/")[0] - in Settings.S3_BUCKET_TO_HTTP_ENDPOINT - ), f"S3_BUCKET_TO_HTTP_ENDPOINT Setting must include bucket name [{Settings.S3_REPORT_BUCKET}] from S3_REPORT_BUCKET, workflow [{workflow.name}]" + http_endpoint + ), f"S3_EXTERNAL_BUCKETS Setting must include http_endpoint for bucket [{report_bucket}] from S3_REPORT_BUCKET, workflow [{workflow.name}]" if workflow.enable_cache: for artifact in workflow.artifacts or []: diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index f841432035951..0cde40c212e75 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -26,9 +26,9 @@ class RunnerLabels: CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" if S3_BUCKET_NAME else "" HTML_S3_PATH = S3_REPORT_BUCKET_NAME S3_REPORT_BUCKET = S3_REPORT_BUCKET_NAME -S3_BUCKET_TO_HTTP_ENDPOINT = { - S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT, - S3_REPORT_BUCKET_NAME: S3_REPORT_BUCKET_HTTP_ENDPOINT, +S3_EXTERNAL_BUCKETS = { + S3_BUCKET_NAME: {"http_endpoint": S3_BUCKET_HTTP_ENDPOINT}, + S3_REPORT_BUCKET_NAME: {"http_endpoint": S3_REPORT_BUCKET_HTTP_ENDPOINT}, } ENABLE_ARTIFACTS_REPORT = True @@ -66,14 +66,11 @@ class RunnerLabels: # Most runners are in Hetzner (hel1): cache there to avoid AWS egress. CI_CACHE_BUCKET = os.environ.get("HETZNER_CACHE_BUCKET") or "" SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" if CI_CACHE_BUCKET else "" -S3_EXTERNAL_BUCKETS = {} if CI_CACHE_BUCKET: # S3.copy_file_to_s3 returns a public link to every upload - S3_BUCKET_TO_HTTP_ENDPOINT[CI_CACHE_BUCKET] = ( - f"{CI_CACHE_BUCKET}.hel1.your-objectstorage.com" - ) S3_EXTERNAL_BUCKETS[CI_CACHE_BUCKET] = { "endpoint_url": "https://hel1.your-objectstorage.com", + "http_endpoint": f"{CI_CACHE_BUCKET}.hel1.your-objectstorage.com", "region": "hel1", "access_key_env": "HETZNER_S3_ACCESS_KEY", "secret_key_env": "HETZNER_S3_SECRET_KEY", diff --git a/ci/settings/settings.py b/ci/settings/settings.py index 04a1c2d2b1f0f..1ce7dcbb64c37 100644 --- a/ci/settings/settings.py +++ b/ci/settings/settings.py @@ -25,9 +25,9 @@ class RunnerLabels: CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" ENABLE_SUBMODULE_CACHE = True S3_REPORT_BUCKET = S3_REPORT_BUCKET_NAME -S3_BUCKET_TO_HTTP_ENDPOINT = { - S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT, - S3_REPORT_BUCKET_NAME: S3_REPORT_BUCKET_HTTP_ENDPOINT, +S3_EXTERNAL_BUCKETS = { + S3_BUCKET_NAME: {"http_endpoint": S3_BUCKET_HTTP_ENDPOINT}, + S3_REPORT_BUCKET_NAME: {"http_endpoint": S3_REPORT_BUCKET_HTTP_ENDPOINT}, } COMPRESS_THRESHOLD_MB = 32 TEXT_CONTENT_EXTENSIONS = [