diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index 705c9ad6ff101..4cb3e09d8caba 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -20,16 +20,35 @@ from clickhouse_driver import Client from clickhouse_driver.errors import ServerException import boto3 -from botocore.exceptions import NoCredentialsError +from botocore.exceptions import ClientError, NoCredentialsError import yaml DATABASE_HOST_VAR = "CHECKS_DATABASE_HOST" DATABASE_USER_VAR = "CLICKHOUSE_TEST_STAT_LOGIN" DATABASE_PASSWORD_VAR = "CLICKHOUSE_TEST_STAT_PASSWORD" -S3_BUCKET = "altinity-build-artifacts" +S3_BUCKET = os.getenv("S3_ARTIFACT_BUCKET") GITHUB_REPO = "Altinity/ClickHouse" -GITHUB_GIT_URL = f"https://github.com/{GITHUB_REPO}.git" + + +def _repo_and_run_id(actions_run_url: str) -> tuple[str, str]: + parts = urllib.parse.urlparse(actions_run_url).path.strip("/").split("/") + if ( + len(parts) >= 5 + and parts[2] == "actions" + and parts[3] == "runs" + and parts[0] + and parts[1] + and parts[4] + ): + return f"{parts[0]}/{parts[1]}", parts[4] + return GITHUB_REPO, actions_run_url.rstrip("/").split("/")[-1] + + +def _github_git_url(repo: str) -> str: + return f"https://github.com/{repo}.git" + + GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN") # Release tags are fetched here rather than refs/tags/ so local tags are never touched. RELEASE_TAG_REF_PREFIX = "refs/workflow-report/tags/" @@ -73,7 +92,9 @@ def _results_link_has_ref(target_url: str | None, ref: str) -> bool: return f"REF={ref}&" in target_url or f"/REFs/{ref}/" in target_url -def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame: +def get_commit_statuses( + sha: str, pr_number: int, ref: str, repo: str = GITHUB_REPO +) -> pd.DataFrame: """ Fetch commit statuses for a given SHA and return as a pandas DataFrame. Handles pagination to get all statuses. @@ -95,7 +116,7 @@ def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame: "Accept": "application/vnd.github.v3+json", } - url = f"https://api.github.com/repos/{GITHUB_REPO}/commits/{sha}/statuses" + url = f"https://api.github.com/repos/{repo}/commits/{sha}/statuses" all_data = [] @@ -150,7 +171,7 @@ def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame: ).reset_index(drop=True) -def get_pr_info_from_number(pr_number: str) -> dict: +def get_pr_info_from_number(pr_number: str, repo: str = GITHUB_REPO) -> dict: """ Fetch pull request information for a given PR number. @@ -165,7 +186,7 @@ def get_pr_info_from_number(pr_number: str) -> dict: "Accept": "application/vnd.github.v3+json", } - url = f"https://api.github.com/repos/{GITHUB_REPO}/pulls/{pr_number}" + url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" response = requests.get(url, headers=headers) if response.status_code != 200: @@ -176,7 +197,7 @@ def get_pr_info_from_number(pr_number: str) -> dict: return response.json() -def get_run_details(run_id: str) -> dict: +def get_run_details(run_id: str, repo: str = GITHUB_REPO) -> dict: """ Fetch run details for a given run URL. """ @@ -185,7 +206,7 @@ def get_run_details(run_id: str) -> dict: "Accept": "application/vnd.github.v3+json", } - url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}" + url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}" response = requests.get(url, headers=headers) if response.status_code != 200: @@ -407,9 +428,11 @@ def _git_is_shallow(cwd: str | None) -> bool: return p.stdout.strip() == "true" -def _git_fetch(options: list[str], refspecs: list[str], cwd: str | None) -> None: +def _git_fetch( + options: list[str], refspecs: list[str], cwd: str | None, repo: str = GITHUB_REPO +) -> None: p = subprocess.run( - ["git", "fetch", *options, GITHUB_GIT_URL, *refspecs], + ["git", "fetch", *options, _github_git_url(repo), *refspecs], cwd=cwd, capture_output=True, text=True, @@ -466,7 +489,9 @@ def _filter_releases_to_version_line( ] -def _fetch_release_tags(releases: list[dict], cwd: str) -> None: +def _fetch_release_tags( + releases: list[dict], cwd: str, repo: str = GITHUB_REPO +) -> None: """Fetch release tags into RELEASE_TAG_REF_PREFIX; skip tags missing remotely.""" for rel in releases: tag_name = rel.get("tag_name") @@ -477,6 +502,7 @@ def _fetch_release_tags(releases: list[dict], cwd: str) -> None: ["--no-tags"], [f"+refs/tags/{tag_name}:{RELEASE_TAG_REF_PREFIX}{tag_name}"], cwd, + repo, ) except Exception as e: # NOTE (strtgbb): one deleted remote tag must not abort the section, but @@ -484,17 +510,17 @@ def _fetch_release_tags(releases: list[dict], cwd: str) -> None: print(f"WARNING: skipping release tag {tag_name}: {e}") -def _fetch_commit(commit_sha: str, cwd: str) -> None: - """Fetch the workflow commit from GITHUB_GIT_URL. +def _fetch_commit(commit_sha: str, cwd: str, repo: str = GITHUB_REPO) -> None: + """Fetch the workflow commit from the repository under test. Local checkouts might be a different branch and might not have the run SHA yet. --filter/--unshallow would convert a full clone into a partial one, so those options are used only when the checkout is already shallow. """ if _git_is_shallow(cwd): - _git_fetch(["--filter=tree:0", "--unshallow"], [commit_sha], cwd) + _git_fetch(["--filter=tree:0", "--unshallow"], [commit_sha], cwd, repo) else: - _git_fetch(["--no-tags"], [commit_sha], cwd) + _git_fetch(["--no-tags"], [commit_sha], cwd, repo) def get_prs_in_release_dataframe( @@ -514,7 +540,7 @@ def get_prs_in_release_dataframe( Returns (dataframe, baseline commit time as UTC YYYY-MM-DDTHH:MM:SSZ or None). """ - _fetch_commit(commit_sha, cwd) + _fetch_commit(commit_sha, cwd, repo) resolved = _git_rev_parse(commit_sha, cwd) if not resolved: raise Exception(f"Cannot resolve commit: {commit_sha!r}") @@ -523,7 +549,7 @@ def get_prs_in_release_dataframe( releases = _filter_releases_to_version_line( _get_github_releases(repo), branch_ref, commit_sha, cwd ) - _fetch_release_tags(releases, cwd) + _fetch_release_tags(releases, cwd, repo) _, baseline_sha, _ = _find_release_baseline( branch_ref, releases, cwd, commit_sha=commit_sha @@ -564,7 +590,9 @@ def _parse_release_ref(ref_name: str) -> tuple[str, str, str | None] | None: return None -def _prs_in_release_search_url(branch_name: str, baseline_time: str | None) -> str | None: +def _prs_in_release_search_url( + branch_name: str, baseline_time: str | None, repo: str = GITHUB_REPO +) -> str | None: if not baseline_time: return None # Tag-triggered runs expose the tag as head_branch; search needs the PR base @@ -575,7 +603,7 @@ def _prs_in_release_search_url(branch_name: str, baseline_time: str | None) -> s branch_name = f"{flavour}-{minor_version}" query = f"is:pr base:{branch_name} merged:>{baseline_time}" return ( - f"https://github.com/{GITHUB_REPO}/pulls?" + f"https://github.com/{repo}/pulls?" + urllib.parse.urlencode({"q": query}) ) @@ -890,15 +918,20 @@ def get_new_fails_this_pr( return new_fails_df -def _workflow_config_s3_url(pr_number: int, commit_sha: str, ref_name: str) -> str: +def _s3_get_json(key: str): + s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL")) + response = s3_client.get_object(Bucket=S3_BUCKET, Key=key) + return json.loads(response["Body"].read().decode("utf-8")) + + +def _workflow_config_s3_key(pr_number: int, commit_sha: str, ref_name: str) -> str: if pr_number == 0: return ( - f"https://{S3_BUCKET}.s3.amazonaws.com/REFs/{ref_name}/{commit_sha}/" + f"REFs/{ref_name}/{commit_sha}/" "config_workflow/workflow_config_masterci.json" ) return ( - f"https://{S3_BUCKET}.s3.amazonaws.com/PRs/{pr_number}/{commit_sha}/" - "config_workflow/workflow_config_pr.json" + f"PRs/{pr_number}/{commit_sha}/config_workflow/workflow_config_pr.json" ) @@ -908,18 +941,14 @@ def get_workflow_config(pr_number: int, commit_sha: str, ref_name: str) -> dict: On failure, warn and return a minimal empty config (same shape recreation uses). """ - url = _workflow_config_s3_url(pr_number, commit_sha, ref_name) + key = _workflow_config_s3_key(pr_number, commit_sha, ref_name) try: - response = requests.get(url, timeout=30) - if response.status_code != 200: - print( - f"WARNING:Failed to fetch workflow config from {url}: " - f"{response.status_code}" - ) - return {"cache_jobs": {}} - return response.json() + return _s3_get_json(key) except Exception as e: - print(f"WARNING:Failed to fetch workflow config from {url}: {e}") + print( + f"WARNING:Failed to fetch workflow config from s3://{S3_BUCKET}/{key}: {e}", + file=sys.stderr, + ) return {"cache_jobs": {}} @@ -1130,18 +1159,16 @@ def fetch_workflow_result_json( ref_param = f"PR={pr_number}" workflow_name = "PR" - status_file = f"result_{workflow_name.lower()}.json" - s3_path = ( - f"https://{S3_BUCKET}.s3.amazonaws.com/" - f"{ref_param.replace('=', 's/')}/{commit_sha}/{status_file}" + key = ( + f"{ref_param.replace('=', 's/')}/{commit_sha}/" + f"result_{workflow_name.lower()}.json" ) try: - response = requests.get(s3_path, timeout=30) - if response.status_code != 200: - return None - return response.json() + return _s3_get_json(key) + except ClientError: + return None except Exception as e: - print(f"WARNING:Failed to fetch workflow result from {s3_path}: {e}") + print(f"WARNING:Failed to fetch workflow result from s3://{S3_BUCKET}/{key}: {e}") return None @@ -1182,7 +1209,9 @@ def add_labels_to_checks_fails( return df[cols] -def format_results_as_html_table(results, *, branch_name: str = "") -> str: +def format_results_as_html_table( + results, *, branch_name: str = "", repo: str = GITHUB_REPO +) -> str: if not isinstance(results, pd.DataFrame): return results @@ -1213,7 +1242,7 @@ def format_col_name(col_name: str) -> str: f'{s}' ), "PR Number": lambda n: url_to_html_link( - f"https://github.com/{GITHUB_REPO}/pull/{n}" + f"https://github.com/{repo}/pull/{n}" ), "PR Labels": lambda labels: format_pr_labels_with_verification( labels, branch_name=branch_name @@ -1361,12 +1390,14 @@ def create_workflow_report( print(f"{DATABASE_PASSWORD_VAR} is not set") if not GITHUB_TOKEN: print("GITHUB_TOKEN is not set") - if not all([host, user, password, GITHUB_TOKEN]): + if not S3_BUCKET: + print("S3_ARTIFACT_BUCKET is not set") + if not all([host, user, password, GITHUB_TOKEN, S3_BUCKET]): raise Exception("Required environment variables are not set") - run_id = actions_run_url.split("/")[-1] + github_repo, run_id = _repo_and_run_id(actions_run_url) - run_details = get_run_details(run_id) + run_details = get_run_details(run_id, github_repo) branch_name = run_details.get("head_branch", "unknown branch") if pr_number is None or commit_sha is None: if pr_number is None: @@ -1391,7 +1422,9 @@ def create_workflow_report( results_dfs = { "prs_in_release": [], - "job_statuses": get_commit_statuses(commit_sha, pr_number, branch_name), + "job_statuses": get_commit_statuses( + commit_sha, pr_number, branch_name, github_repo + ), "checks_fails": add_labels_to_checks_fails( get_checks_fails(db_client, commit_sha, branch_name), workflow_result, @@ -1408,11 +1441,11 @@ def create_workflow_report( if pr_number == 0 and not mark_preview: try: prs_df, baseline_time = get_prs_in_release_dataframe( - branch_name, cwd=os.getcwd(), commit_sha=commit_sha + branch_name, cwd=os.getcwd(), repo=github_repo, commit_sha=commit_sha ) results_dfs["prs_in_release"] = prs_df prs_in_release_search_url = _prs_in_release_search_url( - branch_name, baseline_time + branch_name, baseline_time, github_repo ) except Exception as e: print(f"Error in get_prs_in_release_dataframe: {e}") @@ -1442,8 +1475,8 @@ def create_workflow_report( pr_info_html = f"Release ({branch_name})" else: try: - pr_info = get_pr_info_from_number(pr_number) - pr_info_html = f""" + pr_info = get_pr_info_from_number(pr_number, github_repo) + pr_info_html = f""" #{pr_info.get("number")} ({pr_info.get("base", {}).get('ref')} <- {pr_info.get("head", {}).get('ref')}) {pr_info.get("title")} """ results_dfs["pr_new_fails"] = get_new_fails_this_pr( @@ -1478,7 +1511,7 @@ def create_workflow_report( # Define the context for rendering context = { "title": "ClickHouse® CI Workflow Run Report", - "github_repo": GITHUB_REPO, + "github_repo": github_repo, "s3_bucket": S3_BUCKET, "pr_info_html": pr_info_html, "pr_number": pr_number, @@ -1513,6 +1546,7 @@ def create_workflow_report( else format_results_as_html_table( results_dfs["prs_in_release"], branch_name=branch_name, + repo=github_repo, ) ), "ci_jobs_status_html": format_results_as_html_table( diff --git a/.github/actions/runner_setup/action.yml b/.github/actions/runner_setup/action.yml index e978d8f57a741..a9f9d0d341f41 100644 --- a/.github/actions/runner_setup/action.yml +++ b/.github/actions/runner_setup/action.yml @@ -25,7 +25,7 @@ runs: ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0 sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0 - sudo sysctl vm.swappiness=200 + sudo sysctl vm.swappiness=80 - name: Install awscli shell: bash run: | diff --git a/.github/workflows/fast_builds.yml b/.github/workflows/fast_builds.yml index 26a61911e0462..660d5edeaec98 100644 --- a/.github/workflows/fast_builds.yml +++ b/.github/workflows/fast_builds.yml @@ -47,6 +47,8 @@ env: ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} + HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }} @@ -83,7 +85,7 @@ jobs: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY - name: Prepare env script diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index f57125e695e8a..249d66394138e 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -85,7 +85,7 @@ jobs: if: always() id: upload_results env: - S3_BUCKET: "altinity-build-artifacts" + S3_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} PR_NUMBER: ${{ env.PR_NUMBER || github.event.pull_request.number || 0 }} DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} diff --git a/.github/workflows/init_praktika.yml b/.github/workflows/init_praktika.yml index 42aaaeedde00f..7b083d8f7529d 100644 --- a/.github/workflows/init_praktika.yml +++ b/.github/workflows/init_praktika.yml @@ -12,6 +12,7 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} jobs: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 90d8b9e15d717..59451790d0301 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -62,6 +62,8 @@ env: ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} + HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }} @@ -98,7 +100,7 @@ jobs: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY - name: Prepare env script diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7924f9837029b..7e6d53cf12872 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -37,6 +37,8 @@ env: ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }} HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} + HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }} @@ -74,7 +76,7 @@ jobs: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY - name: Prepare env script diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index bf1a63d7d667f..f4e2bc78d6c1c 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -20,6 +20,8 @@ env: DISABLE_CI_MERGE_COMMIT: ${{ vars.DISABLE_CI_MERGE_COMMIT || '0' }} DISABLE_CI_CACHE: ${{ github.event.inputs.no_cache || '0' }} CHECKOUT_REF: ${{ vars.DISABLE_CI_MERGE_COMMIT == '1' && github.event.pull_request.head.sha || '' }} + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} + HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }} @@ -57,7 +59,7 @@ jobs: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY - name: Prepare env script diff --git a/.github/workflows/regression-reusable-suite.yml b/.github/workflows/regression-reusable-suite.yml index 51d064ea0f011..6cf3e3ffbdd26 100644 --- a/.github/workflows/regression-reusable-suite.yml +++ b/.github/workflows/regression-reusable-suite.yml @@ -93,6 +93,7 @@ jobs: build_sha: ${{ inputs.build_sha }} pr_number: ${{ github.event.number }} artifacts: builds + S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }} regression_args: ${{ inputs.regression_args }} # Args args: --test-to-end @@ -130,7 +131,7 @@ jobs: - name: 📦 Get deb url env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ + S3_BASE_URL: https://${{ vars.S3_ARTIFACT_BUCKET }}.s3.amazonaws.com/ PR_NUMBER: ${{ github.event.pull_request.number || 0 }} run: | REPORTS_PATH=${{ runner.temp }}/reports_dir diff --git a/CMakeLists.txt b/CMakeLists.txt index 68b194ad629c3..ffcd02c5d0696 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -493,6 +493,24 @@ if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--lto-whole-program-visibility") endif() + # Persistent ThinLTO cache. lld stores the optimized native object of every + # module keyed by the module hash, its import set and the LTO flags, and + # reuses it on the next link when nothing in that key changed. Only the + # backend codegen of the link is cached; the thin-link and the final native + # link still run in full. Entries are content-addressed, so concurrent + # writers cannot corrupt each other. + set (THINLTO_CACHE_DIR "" CACHE PATH "Directory for the ThinLTO cache (lld --thinlto-cache-dir); empty disables the cache") + # lld prunes after each link by default. That is off here, since CI keeps + # exactly the entries a build used; set a policy for a long-lived cache dir. + set (THINLTO_CACHE_POLICY "prune_after=0s:cache_size=0%:cache_size_bytes=0:cache_size_files=0" CACHE STRING "Pruning policy for the ThinLTO cache (lld --thinlto-cache-policy)") + if (THINLTO_CACHE_DIR) + if (NOT LINKER_NAME MATCHES "lld") + message (FATAL_ERROR "THINLTO_CACHE_DIR requires lld, but LINKER_NAME is '${LINKER_NAME}'") + endif () + message (STATUS "Using ThinLTO cache in ${THINLTO_CACHE_DIR} with policy '${THINLTO_CACHE_POLICY}'") + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-cache-dir=${THINLTO_CACHE_DIR} -Wl,--thinlto-cache-policy=${THINLTO_CACHE_POLICY}") + endif () + if (ENABLE_CFI) # CFI checks: virtual dispatch type safety and bad downcasts via static_cast. # cfi-icall is omitted because C function pointers in contrib libraries (compiled without diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index eaec79a77d782..51c6c33ddab70 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -29,7 +29,9 @@ f"+--memory={Utils.physical_memory() * 95 // 100}" f"+--memory-reservation={Utils.physical_memory() * 9 // 10}" f"+--volume=.:/ClickHouse" - '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"' + '+--env=AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"+--env=AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"+--env=S3_ARTIFACT_BUCKET="$S3_ARTIFACT_BUCKET"' + # Writes to the ThinLTO cache in Hetzner Object Storage; jobs without them read anonymously + '+--env=HETZNER_S3_ACCESS_KEY="$HETZNER_S3_ACCESS_KEY"+--env=HETZNER_S3_SECRET_KEY="$HETZNER_S3_SECRET_KEY"+--env=HETZNER_CACHE_BUCKET="$HETZNER_CACHE_BUCKET"' ) BINARY_DOCKER_COMMAND_SANITIZER = ( BINARY_DOCKER_COMMAND + "+--security-opt seccomp=unconfined" @@ -254,7 +256,7 @@ class JobConfigs: runs_on=RunnerLabels.AMD_LARGE, command="python3 ./ci/jobs/fast_test.py", # --network=host required for ec2 metadata http endpoint to work - run_in_docker="altinityinfra/fasttest+--network=host+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/var/log:/var/log+--volume=.:/ClickHouse+--env=AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID+--env=AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY", + run_in_docker="altinityinfra/fasttest+--network=host+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/var/log:/var/log+--volume=.:/ClickHouse+--env=AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID+--env=AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY+--env=S3_ARTIFACT_BUCKET=$S3_ARTIFACT_BUCKET", digest_config=fast_test_digest_config, result_name_for_cidb="Tests", needs_submodules=True, @@ -443,7 +445,7 @@ class JobConfigs: ArtifactNames.RPM_ARM_RELEASE, ArtifactNames.TGZ_ARM_RELEASE, ], - runs_on=RunnerLabels.BUILDER_AMD, + runs_on=RunnerLabels.BUILDER_ARM, ), ) release_build_jobs_with_examples = [ @@ -490,10 +492,9 @@ class JobConfigs: # master data directly comparable to a PR build (official master builds # keep debug symbols, PR builds strip them). The hook uploads only for the # arm variant (see _PROFILED_BUILDS in build_profile_hook.py). + # NOTE (strtgbb): no --build-examples for the arm warmup, PR builds no longer build clickhouse-examples sccache_warmup_build_jobs = [ - job.set_command(f"{job.command} --build-examples") - if f"({BuildTypes.ARM_RELEASE_PR_CACHE_WARMUP})" in job.name - else job + job for job in common_build_job_config.set_post_hooks( post_hooks=[ # "python3 ./ci/jobs/scripts/job_hooks/build_profile_hook.py", diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 23eec7551006d..f8207263d0b04 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -6,8 +6,10 @@ from ci.jobs.scripts.clickhouse_version import CHVersion from ci.praktika.info import Info from ci.praktika.result import Result +from ci.praktika.s3 import S3 from ci.praktika.settings import Settings from ci.praktika.utils import MetaClasses, Shell, Utils +from ci.settings.altinity_overrides import CI_CACHE_BUCKET current_directory = Utils.cwd() build_dir = f"{current_directory}/ci/tmp/build" @@ -15,6 +17,18 @@ repo_path_normalized = "/ClickHouse" build_dir_normalized = f"{repo_path_normalized}/ci/tmp/build" +# ThinLTO cache: lld stores the optimized native object of every module it +# links, and the next link of an unchanged module reuses it instead of running +# the backend again. The cache travels between builders as one archive per +# branch, build type and debug-info flavour. +THINLTO_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache/thinlto" +thinlto_cache_dir = f"{temp_dir}/thinlto_cache" +thinlto_cache_dir_normalized = f"{repo_path_normalized}/ci/tmp/thinlto_cache" +thinlto_cache_archive = f"{temp_dir}/thinlto_cache.tar.zst" +# An archive holds the entries one build used, about 15 GB for a `-g` build of +# `clickhouse` and `clickhouse-keeper`. Much more means pruning went wrong. +THINLTO_CACHE_MAX_BYTES = {"g": 20 << 30, "g0": 5 << 30} + BUILD_TYPE_TO_CMAKE = { BuildTypes.AMD_DEBUG: f" cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=Debug -DENABLE_THINLTO=0 -DSANITIZE= -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE} -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=1 -DENABLE_LEXER_TEST=1 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DENABLE_BUZZHOUSE=1", BuildTypes.AMD_RELEASE: f" cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=None -DENABLE_THINLTO=1 -DSANITIZE= -DENABLE_CHECK_HEAVY_BUILDS=1 -DBUILD_STRIPPED_BINARY=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DCMAKE_C_COMPILER={ToolSet.COMPILER_C} -DCMAKE_CXX_COMPILER={ToolSet.COMPILER_CPP} -DCOMPILER_CACHE={ToolSet.COMPILER_CACHE} -DCMAKE_TOOLCHAIN_FILE={repo_path_normalized}/cmake/linux/toolchain-x86_64.cmake -DENABLE_BUILD_PROFILING=1 -DENABLE_TESTS=0 -DENABLE_LEXER_TEST=0 -DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1", @@ -103,8 +117,114 @@ def parse_args(): return parser.parse_args() +def thinlto_cache_flavour(cmake_cmd): + return "g0" if "-DDISABLE_ALL_DEBUG_SYMBOLS=1" in cmake_cmd else "g" + + +def thinlto_cache_s3_path(info, build_type, cmake_cmd): + """PR builds share the cache of their base branch, other builds use their own branch.""" + branch = info.base_branch if info.pr_number > 0 else info.git_branch + if not branch: + return None + return f"{THINLTO_CACHE_S3_PATH}/{branch}/{build_type}-{thinlto_cache_flavour(cmake_cmd)}.tar.zst" + + +def restore_thinlto_cache(s3_path): + shutil.rmtree(thinlto_cache_dir, ignore_errors=True) + os.makedirs(thinlto_cache_dir) + if not S3.head_object(s3_path): + print(f"ThinLTO cache miss: {s3_path}") + return True + try: + S3.copy_file_from_s3(s3_path=s3_path, local_path=thinlto_cache_archive) + Shell.check( + f"set -o pipefail; zstd -d --long=27 -c {thinlto_cache_archive} | tar -xf - -C {thinlto_cache_dir}", + verbose=True, + strict=True, + ) + except BaseException: + # A partly extracted entry would be linked as is: start cold instead. + shutil.rmtree(thinlto_cache_dir, ignore_errors=True) + os.makedirs(thinlto_cache_dir) + raise + finally: + if os.path.exists(thinlto_cache_archive): + os.remove(thinlto_cache_archive) + # With an access time of zero, the kernel records the next read of an + # entry even under `relatime`, which is how the upload step tells the + # entries this build used from the ones it did not. + count = 0 + size = 0 + with os.scandir(thinlto_cache_dir) as entries: + for entry in entries: + stat = entry.stat(follow_symlinks=False) + os.utime(entry.path, (0, stat.st_mtime)) + count += 1 + size += stat.st_size + print(f"ThinLTO cache restored: {count} entries, {size / (1 << 30):.1f} GiB") + return True + + +def thinlto_cache_tracks_reads(): + probe = f"{thinlto_cache_dir}/.atime-probe" + with open(probe, "w") as f: + f.write("probe") + os.utime(probe, (0, os.stat(probe).st_mtime)) + with open(probe) as f: + f.read() + tracked = os.stat(probe).st_atime > 0 + os.remove(probe) + return tracked + + +def upload_thinlto_cache(s3_path, flavour): + """Keep the entries this build read or wrote, then upload them.""" + assert thinlto_cache_tracks_reads(), ( + f"{thinlto_cache_dir} does not record file reads (mounted with noatime?), " + "so the entries this build used are unknown" + ) + kept = 0 + removed = 0 + size = 0 + with os.scandir(thinlto_cache_dir) as entries: + for entry in entries: + stat = entry.stat(follow_symlinks=False) + # Anything else is a temporary file of an interrupted link. + used = entry.name.startswith("llvmcache-") and entry.is_file( + follow_symlinks=False + ) and stat.st_atime > 0 + if used: + kept += 1 + size += stat.st_size + else: + os.remove(entry.path) + removed += 1 + print( + f"ThinLTO cache: kept {kept} entries ({size / (1 << 30):.1f} GiB), " + f"removed {removed} unused" + ) + assert size <= THINLTO_CACHE_MAX_BYTES[flavour], ( + f"ThinLTO cache of {size} bytes exceeds the {THINLTO_CACHE_MAX_BYTES[flavour]} " + f"bytes expected for a `-{flavour}` build, not uploading it" + ) + Shell.check( + f"set -o pipefail; tar -cf - -C {thinlto_cache_dir} . | zstd -T0 -3 --long=27 -c > {thinlto_cache_archive}", + verbose=True, + strict=True, + ) + print( + f"ThinLTO cache archive: {os.path.getsize(thinlto_cache_archive) / (1 << 30):.1f} GiB" + ) + S3.copy_file_to_s3( + s3_path=s3_path, local_path=thinlto_cache_archive, with_rename=True + ) + os.remove(thinlto_cache_archive) + return True + + def run_shell(name, command, **kwargs): print(f"\n>>>> {name}\n") + kwargs.setdefault("verbose", True) Shell.check(command, **kwargs) print(f"\n<<<< {name}\n") @@ -214,6 +334,26 @@ def main(): ): cmake_cmd += " -DDISABLE_ALL_DEBUG_SYMBOLS=1" + # ThinLTO cache for every build that links with ThinLTO. The cache-warmup + # builds only compile. Everyone reads the archive (community PRs anonymously); + # pushes and internal PRs write it, like the shared compiler cache. + thinlto_cache_s3 = None + if "ENABLE_THINLTO=1" in cmake_cmd and not cache_warmup: + thinlto_cache_s3 = thinlto_cache_s3_path(info, build_type, cmake_cmd) + thinlto_cache_writable = not info.is_local_run and not info.is_community_pr + if thinlto_cache_s3: + print(f"ThinLTO cache: {thinlto_cache_s3}, writable: {thinlto_cache_writable}") + cmake_cmd += f" -DTHINLTO_CACHE_DIR={thinlto_cache_dir_normalized}" + + # Link `clickhouse` and `clickhouse-keeper` concurrently. Each ThinLTO link + # spends most of its time in a single-threaded thin-link phase, so the two + # overlap almost entirely: measured on a 16-core, 62 GB builder with zram, + # 114.5 -> 83 min cold and 65.5 -> 48.5 min warm. The memory-derived + # default in limit_jobs.cmake budgets ~100 GB per `-g` link and would + # allow only one; the real peak of both links together was ~48 GB. + if "ENABLE_THINLTO=1" in cmake_cmd: + cmake_cmd += " -DPARALLEL_LINK_JOBS=2" + # PGO/BOLT profile integration for release builds. The sccache-warmup builds # are deliberately excluded: they exist only to populate the shared compiler # cache, and PGO/BOLT belong to the real release builds. @@ -422,6 +562,15 @@ def do_checkout(): else: build_command = f"command time -v ninja {targets}" + if thinlto_cache_s3: + results.append( + Result.from_commands_run( + name="Restore ThinLTO cache", + command=restore_thinlto_cache, + command_args=[thinlto_cache_s3], + ) + ) + build_result_index = len(results) results.append( Result.from_commands_run( @@ -462,6 +611,19 @@ def do_checkout(): else: results.append(retry_cmake) + if ( + thinlto_cache_s3 + and thinlto_cache_writable + and results[build_result_index].is_ok() + ): + results.append( + Result.from_commands_run( + name="Upload ThinLTO cache", + command=upload_thinlto_cache, + command_args=[thinlto_cache_s3, thinlto_cache_flavour(cmake_cmd)], + ) + ) + run_shell("sccache stats", "sccache --show-stats") if build_type in (BuildTypes.AMD_TIDY, BuildTypes.ARM_TIDY): run_shell("clang-tidy-cache stats", "clang-tidy-cache.py --show-stats") diff --git a/ci/praktika/info.py b/ci/praktika/info.py index c77dc98a08ab3..e38853a6e38ee 100644 --- a/ci/praktika/info.py +++ b/ci/praktika/info.py @@ -203,9 +203,9 @@ def get_specific_report_url( assert branch ref_param = f"REF={branch}" path = Settings.S3_REPORT_BUCKET - for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items(): + for bucket, config in (Settings.S3_EXTERNAL_BUCKETS or {}).items(): if bucket in path: - path = path.replace(bucket, endpoint) + path = path.replace(bucket, config["http_endpoint"]) break workflow_name = workflow_name or self.env.WORKFLOW_NAME res = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?{ref_param}&sha={sha}&name_0={urllib.parse.quote(workflow_name, safe='')}" @@ -223,9 +223,9 @@ def get_specific_report_url_static(pr_number, branch, sha, job_name, workflow_na assert branch ref_param = f"REF={branch}" path = Settings.S3_REPORT_BUCKET - for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items(): + for bucket, config in (Settings.S3_EXTERNAL_BUCKETS or {}).items(): if bucket in path: - path = path.replace(bucket, endpoint) + path = path.replace(bucket, config["http_endpoint"]) break res = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?{ref_param}&sha={sha}&name_0={urllib.parse.quote(workflow_name, safe='')}" if job_name: diff --git a/ci/praktika/s3.py b/ci/praktika/s3.py index 863718e8b915d..871c0e6837585 100644 --- a/ci/praktika/s3.py +++ b/ci/praktika/s3.py @@ -65,7 +65,17 @@ class S3: @classmethod def _external_bucket_config(cls, bucket): - return (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + config = (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) + if not config or not config.get("endpoint_url"): + return None + return config + + @classmethod + def _http_endpoint(cls, bucket): + config = (Settings.S3_EXTERNAL_BUCKETS or {}).get(bucket) or {} + endpoint = config.get("http_endpoint") + assert endpoint, f"S3 bucket [{bucket}] has no http_endpoint in S3_EXTERNAL_BUCKETS" + return endpoint @classmethod def _reads_anonymously(cls, bucket): @@ -280,8 +290,7 @@ def _upload(): print(f"WARNING: Failed to record upload usage for {local_path}: {e}") bucket = s3_path.split("/")[0] - endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket] - assert endpoint + endpoint = cls._http_endpoint(bucket) return quote(f"https://{s3_full_path}".replace(bucket, endpoint), safe=":/?&=") @classmethod @@ -643,7 +652,7 @@ def _upload_file_to_s3( def _dump_urls(cls, s3_path): # TODO: add support for path with '*' bucket, name = s3_path.split("/")[0], s3_path.split("/")[-1] - endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket] + endpoint = cls._http_endpoint(bucket) with open(Settings.ARTIFACT_URLS_FILE, "w", encoding="utf-8") as f: json.dump( diff --git a/ci/praktika/settings.py b/ci/praktika/settings.py index 3385cd6b57368..cd26c7dac0a9c 100644 --- a/ci/praktika/settings.py +++ b/ci/praktika/settings.py @@ -128,9 +128,9 @@ class _Settings: # Optional: upstream report bucket to merge issue catalogs from (e.g. "clickhouse-test-reports") S3_UPSTREAM_REPORT_BUCKET: str = "" HTML_PAGE_FILE: str = "./ci/praktika/json.html" - S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None - # Non-AWS buckets: {bucket: {endpoint_url, region, access_key_env, secret_key_env, public_read, download_concurrency}} - # The keys are read from the named env vars; with public_read, jobs without them read unsigned. + # {bucket: {http_endpoint, endpoint_url?, region?, access_key_env?, secret_key_env?, public_read?, download_concurrency?}} + # http_endpoint is the public host spliced into links. endpoint_url marks a non-AWS bucket. + # Keys are read from the named env vars; with public_read, jobs without them read unsigned. # download_concurrency (optional): parallel ranged GETs, for links with a low rate per connection. S3_EXTERNAL_BUCKETS: Optional[Dict[str, Dict[str, Any]]] = None TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"]) @@ -182,7 +182,6 @@ class _Settings: "CLOUD_INFRASTRUCTURE_CONFIG_PATH", "EVENT_FEED_S3_PATH", "AWS_REGION", - "S3_BUCKET_TO_HTTP_ENDPOINT", "S3_EXTERNAL_BUCKETS", "SUBMODULE_CACHE_S3_PATH", "TEXT_CONTENT_EXTENSIONS", diff --git a/ci/praktika/validator.py b/ci/praktika/validator.py index 6ade4d53033a9..382e76dd8bfcf 100644 --- a/ci/praktika/validator.py +++ b/ci/praktika/validator.py @@ -143,10 +143,11 @@ def is_valid_cron_field(field: str) -> bool: f"Must be Artifact.Config type, not {type(artifact)}: [{artifact}]", workflow.name, ) - if artifact.is_s3_artifact(): - assert ( - Settings.S3_ARTIFACT_PATH - ), "Provide S3_ARTIFACT_PATH setting in any .py file in ./ci/settings/* to be able to use s3 for artifacts" + # NOTE (strtgbb): S3_ARTIFACT_BUCKET is set on the job, not when workflows are generated + # if artifact.is_s3_artifact(): + # assert ( + # Settings.S3_ARTIFACT_PATH + # ), "Provide S3_ARTIFACT_PATH setting in any .py file in ./ci/settings/* to be able to use s3 for artifacts" for job in workflow.jobs: if job.requires and workflow.artifacts: @@ -164,9 +165,10 @@ def is_valid_cron_field(field: str) -> bool: Settings.CI_CONFIG_RUNS_ON ), f"Runner label to run workflow config job must be provided via CACHE_CONFIG_RUNS_ON setting if enable_cache=True, workflow [{workflow.name}]" - assert ( - Settings.CACHE_S3_PATH - ), f"CACHE_S3_PATH Setting must be defined if enable_cache=True, workflow [{workflow.name}]" + # NOTE (strtgbb): S3_ARTIFACT_BUCKET is set on the job, not when workflows are generated + # assert ( + # Settings.CACHE_S3_PATH + # ), f"CACHE_S3_PATH Setting must be defined if enable_cache=True, workflow [{workflow.name}]" if workflow.dockers: if Settings.ENABLE_MULTIPLATFORM_DOCKER_IN_ONE_JOB == False: @@ -200,16 +202,17 @@ def is_valid_cron_field(field: str) -> bool: ) if workflow.enable_report: + # NOTE (strtgbb): S3_ARTIFACT_BUCKET is set on the job, not when workflows are generated + # assert ( + # Settings.S3_REPORT_BUCKET + # ), f"S3_REPORT_BUCKET Setting must be defined if enable_html=True, workflow [{workflow.name}]" + report_bucket = Settings.S3_REPORT_BUCKET.split("/")[0] + http_endpoint = ( + (Settings.S3_EXTERNAL_BUCKETS or {}).get(report_bucket) or {} + ).get("http_endpoint") assert ( - Settings.S3_REPORT_BUCKET - ), f"S3_REPORT_BUCKET Setting must be defined if enable_html=True, workflow [{workflow.name}]" - assert ( - Settings.S3_BUCKET_TO_HTTP_ENDPOINT - ), f"S3_BUCKET_TO_HTTP_ENDPOINT Setting must be defined if enable_html=True, workflow [{workflow.name}]" - assert ( - Settings.S3_REPORT_BUCKET.split("/")[0] - in Settings.S3_BUCKET_TO_HTTP_ENDPOINT - ), f"S3_BUCKET_TO_HTTP_ENDPOINT Setting must include bucket name [{Settings.S3_REPORT_BUCKET}] from S3_REPORT_BUCKET, workflow [{workflow.name}]" + http_endpoint + ), f"S3_EXTERNAL_BUCKETS Setting must include http_endpoint for bucket [{report_bucket}] from S3_REPORT_BUCKET, workflow [{workflow.name}]" if workflow.enable_cache: for artifact in workflow.artifacts or []: diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 2869279da9962..ac8102a5dce20 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -33,7 +33,7 @@ class AltinityWorkflowTemplates: else PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA" fi - REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html + REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY """ # Additional jobs diff --git a/ci/praktika/yaml_generator.py b/ci/praktika/yaml_generator.py index d5ec52d57d276..bb0f5a61a85b9 100644 --- a/ci/praktika/yaml_generator.py +++ b/ci/praktika/yaml_generator.py @@ -729,6 +729,12 @@ def _all_needs(job_name: str) -> set: format_kwargs[ "ENV_SECRETS" ] += AltinityWorkflowTemplates.ADDITIONAL_GLOBAL_ENV + format_kwargs[ + "ENV_SECRETS" + ] += " S3_ARTIFACT_BUCKET: ${{{{ vars.S3_ARTIFACT_BUCKET }}}}\n" + format_kwargs[ + "ENV_SECRETS" + ] += " HETZNER_CACHE_BUCKET: ${{{{ vars.HETZNER_CACHE_BUCKET }}}}\n" template_1 = base_template.strip().format( NAME=self.workflow_config.name, diff --git a/ci/settings/altinity_overrides.py b/ci/settings/altinity_overrides.py index 101c6f492a047..0cde40c212e75 100644 --- a/ci/settings/altinity_overrides.py +++ b/ci/settings/altinity_overrides.py @@ -1,9 +1,9 @@ import os -S3_BUCKET_NAME = "altinity-build-artifacts" -S3_REPORT_BUCKET_NAME = "altinity-build-artifacts" -S3_BUCKET_HTTP_ENDPOINT = "altinity-build-artifacts.s3.amazonaws.com" -S3_REPORT_BUCKET_HTTP_ENDPOINT = "altinity-build-artifacts.s3.amazonaws.com" +S3_BUCKET_NAME = os.environ.get("S3_ARTIFACT_BUCKET") or "" +S3_REPORT_BUCKET_NAME = S3_BUCKET_NAME +S3_BUCKET_HTTP_ENDPOINT = f"{S3_BUCKET_NAME}.s3.amazonaws.com" +S3_REPORT_BUCKET_HTTP_ENDPOINT = S3_BUCKET_HTTP_ENDPOINT class RunnerLabels: @@ -23,12 +23,12 @@ class RunnerLabels: DOCKER_BUILD_AND_MERGE_RUNS_ON = RunnerLabels.STYLE_CHECK_AMD DOCKER_BUILD_ARM_RUNS_ON = RunnerLabels.STYLE_CHECK_ARM -CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" +CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" if S3_BUCKET_NAME else "" HTML_S3_PATH = S3_REPORT_BUCKET_NAME S3_REPORT_BUCKET = S3_REPORT_BUCKET_NAME -S3_BUCKET_TO_HTTP_ENDPOINT = { - S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT, - S3_REPORT_BUCKET_NAME: S3_REPORT_BUCKET_HTTP_ENDPOINT, +S3_EXTERNAL_BUCKETS = { + S3_BUCKET_NAME: {"http_endpoint": S3_BUCKET_HTTP_ENDPOINT}, + S3_REPORT_BUCKET_NAME: {"http_endpoint": S3_REPORT_BUCKET_HTTP_ENDPOINT}, } ENABLE_ARTIFACTS_REPORT = True @@ -64,11 +64,13 @@ class RunnerLabels: ENABLE_SUBMODULE_CLONE_AUTH = True # Most runners are in Hetzner (hel1): cache there to avoid AWS egress. -CI_CACHE_BUCKET = "altinity-builds-cache" -SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" -S3_EXTERNAL_BUCKETS = { - CI_CACHE_BUCKET: { +CI_CACHE_BUCKET = os.environ.get("HETZNER_CACHE_BUCKET") or "" +SUBMODULE_CACHE_S3_PATH = f"{CI_CACHE_BUCKET}/ci_ch_cache" if CI_CACHE_BUCKET else "" +if CI_CACHE_BUCKET: + # S3.copy_file_to_s3 returns a public link to every upload + S3_EXTERNAL_BUCKETS[CI_CACHE_BUCKET] = { "endpoint_url": "https://hel1.your-objectstorage.com", + "http_endpoint": f"{CI_CACHE_BUCKET}.hel1.your-objectstorage.com", "region": "hel1", "access_key_env": "HETZNER_S3_ACCESS_KEY", "secret_key_env": "HETZNER_S3_SECRET_KEY", @@ -76,7 +78,6 @@ class RunnerLabels: # netcup runners get ~2.5 MB/s per connection from hel1 "download_concurrency": 32, } -} DISABLED_WORKFLOWS = [ "backport_branches.py", diff --git a/ci/settings/settings.py b/ci/settings/settings.py index 04a1c2d2b1f0f..1ce7dcbb64c37 100644 --- a/ci/settings/settings.py +++ b/ci/settings/settings.py @@ -25,9 +25,9 @@ class RunnerLabels: CACHE_S3_PATH = f"{S3_BUCKET_NAME}/ci_ch_cache" ENABLE_SUBMODULE_CACHE = True S3_REPORT_BUCKET = S3_REPORT_BUCKET_NAME -S3_BUCKET_TO_HTTP_ENDPOINT = { - S3_BUCKET_NAME: S3_BUCKET_HTTP_ENDPOINT, - S3_REPORT_BUCKET_NAME: S3_REPORT_BUCKET_HTTP_ENDPOINT, +S3_EXTERNAL_BUCKETS = { + S3_BUCKET_NAME: {"http_endpoint": S3_BUCKET_HTTP_ENDPOINT}, + S3_REPORT_BUCKET_NAME: {"http_endpoint": S3_REPORT_BUCKET_HTTP_ENDPOINT}, } COMPRESS_THRESHOLD_MB = 32 TEXT_CONTENT_EXTENSIONS = [ diff --git a/ci/workflows/fast_builds.py b/ci/workflows/fast_builds.py index 0e10c801cfd7f..d55b75d675485 100644 --- a/ci/workflows/fast_builds.py +++ b/ci/workflows/fast_builds.py @@ -19,7 +19,8 @@ builds_for_release_branch = [ job.unset_provides("unittest") - for job in JobConfigs.build_jobs + JobConfigs.release_build_jobs_with_examples + # NOTE (strtgbb): no clickhouse-examples, nothing uses it and it delays the ThinLTO link of clickhouse + for job in JobConfigs.build_jobs + JobConfigs.release_build_jobs if "coverage" not in job.name ] @@ -58,7 +59,7 @@ artifacts=[ *clickhouse_binaries_with_tags, *ArtifactConfigs.clickhouse_binaries_gh, - ArtifactConfigs.clickhouse_examples, + # ArtifactConfigs.clickhouse_examples, *ArtifactConfigs.clickhouse_debians, *ArtifactConfigs.clickhouse_rpms, *ArtifactConfigs.clickhouse_tgzs, diff --git a/ci/workflows/master.py b/ci/workflows/master.py index fbe4d824b1494..bede045233dd6 100644 --- a/ci/workflows/master.py +++ b/ci/workflows/master.py @@ -44,7 +44,8 @@ *JobConfigs.build_jobs, # *JobConfigs.build_llvm_coverage_job, JobConfigs.coverage_build_jobs[1], - *JobConfigs.release_build_jobs_with_examples, + # NOTE (strtgbb): no clickhouse-examples, nothing uses it and it delays the ThinLTO link of clickhouse + *JobConfigs.release_build_jobs, *JobConfigs.sccache_warmup_build_jobs, # *[ # NOTE (strtgbb): we don't run special build jobs @@ -104,7 +105,7 @@ *AltinityArtifactConfigs.signed_hashes, ArtifactConfigs.fuzzers, ArtifactConfigs.fuzzers_corpus, - ArtifactConfigs.clickhouse_examples, + # ArtifactConfigs.clickhouse_examples, # *ArtifactConfigs.llvm_profdata_file, # ArtifactConfigs.llvm_coverage_info_file, ], diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index 2b2fc2035788b..f7fdfdd22eb27 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -94,7 +94,8 @@ ], *[ job.set_run_after(REGULAR_BUILD_NAMES) - for job in JobConfigs.release_build_jobs_with_examples + # NOTE (strtgbb): no clickhouse-examples, nothing uses it and it delays the ThinLTO link of clickhouse + for job in JobConfigs.release_build_jobs ], # *[ # NOTE (strtgbb): we don't run special build jobs # job.set_run_after(CORE_BLOCKING_JOB_NAMES) @@ -259,7 +260,7 @@ ArtifactConfigs.wasm_parser, ArtifactConfigs.fuzzers, ArtifactConfigs.fuzzers_corpus, - ArtifactConfigs.clickhouse_examples, + # ArtifactConfigs.clickhouse_examples, # *ArtifactConfigs.llvm_profdata_file, # ArtifactConfigs.llvm_coverage_info_file, ArtifactConfigs.toolchain_pgo_bolt_amd, diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 8be747c6f27f0..366e3f71a8ba4 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -28,6 +28,10 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) endif() endif() +# Remember whether the link-job limit was set explicitly: the ThinLTO memory +# cap below only applies to the auto-calculated value. +set (PARALLEL_LINK_JOBS_EXPLICIT ${PARALLEL_LINK_JOBS}) + if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) math(EXPR PARALLEL_LINK_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY}) @@ -54,7 +58,7 @@ endif () # links than physically fit in memory - otherwise the linker is OOM-killed. # With debug info disabled (-g0, DISABLE_ALL_DEBUG_SYMBOLS) a link needs only # ~50 GB, so two still fit and we keep them parallel. -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1) +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 1 AND NOT PARALLEL_LINK_JOBS_EXPLICIT) if (DISABLE_ALL_DEBUG_SYMBOLS) set (MAX_THINLTO_LINK_MEMORY 51200) # megabytes per concurrent ThinLTO link, measured ~49 GB with -g0 else () diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index ac22e90a92880..b2df6ab3964c0 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -31,10 +31,11 @@ RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) S3_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") -S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-build-artifacts") -S3_BUILDS_BUCKET_PUBLIC = "altinity-build-artifacts" +S3_ARTIFACT_BUCKET = os.getenv("S3_ARTIFACT_BUCKET") +S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET") or S3_ARTIFACT_BUCKET +S3_BUILDS_BUCKET_PUBLIC = S3_ARTIFACT_BUCKET S3_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") -S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") +S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET") or S3_ARTIFACT_BUCKET S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) diff --git a/tests/queries/0_stateless/03536_ch_as_client_and_local.sh b/tests/queries/0_stateless/03536_ch_as_client_and_local.sh index 323843d0b0c72..ec4d5ce92bc16 100755 --- a/tests/queries/0_stateless/03536_ch_as_client_and_local.sh +++ b/tests/queries/0_stateless/03536_ch_as_client_and_local.sh @@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CLICKHOUSE_BINARY_CH=${CLICKHOUSE_BINARY/clickhouse/ch} +CLICKHOUSE_BINARY_CH=${CLICKHOUSE_BINARY/%clickhouse/ch} # Invocation with unknown tool name prints help: ${CLICKHOUSE_BINARY} test 2>&1 | grep -F 'Use one of the following commands'