Antalya 26.8 - ThinLTO link cache

.github/actions/create_workflow_report/create_workflow_report.py

@@ -20,16 +20,35 @@
 from clickhouse_driver import Client
 from clickhouse_driver.errors import ServerException
 import boto3
-from botocore.exceptions import NoCredentialsError
+from botocore.exceptions import ClientError, NoCredentialsError
 import yaml
 
 
 DATABASE_HOST_VAR = "CHECKS_DATABASE_HOST"
 DATABASE_USER_VAR = "CLICKHOUSE_TEST_STAT_LOGIN"
 DATABASE_PASSWORD_VAR = "CLICKHOUSE_TEST_STAT_PASSWORD"
-S3_BUCKET = "altinity-build-artifacts"
+S3_BUCKET = os.getenv("S3_ARTIFACT_BUCKET")
 GITHUB_REPO = "Altinity/ClickHouse"
-GITHUB_GIT_URL = f"https://github.com/{GITHUB_REPO}.git"
+
+
+def _repo_and_run_id(actions_run_url: str) -> tuple[str, str]:
+    parts = urllib.parse.urlparse(actions_run_url).path.strip("/").split("/")
+    if (
+        len(parts) >= 5
+        and parts[2] == "actions"
+        and parts[3] == "runs"
+        and parts[0]
+        and parts[1]
+        and parts[4]
+    ):
+        return f"{parts[0]}/{parts[1]}", parts[4]
+    return GITHUB_REPO, actions_run_url.rstrip("/").split("/")[-1]
+
+
+def _github_git_url(repo: str) -> str:
+    return f"https://github.com/{repo}.git"
+
+
 GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN")
 # Release tags are fetched here rather than refs/tags/ so local tags are never touched.
 RELEASE_TAG_REF_PREFIX = "refs/workflow-report/tags/"
@@ -73,7 +92,9 @@ def _results_link_has_ref(target_url: str | None, ref: str) -> bool:
     return f"REF={ref}&" in target_url or f"/REFs/{ref}/" in target_url
 
 
-def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame:
+def get_commit_statuses(
+    sha: str, pr_number: int, ref: str, repo: str = GITHUB_REPO
+) -> pd.DataFrame:
     """
     Fetch commit statuses for a given SHA and return as a pandas DataFrame.
     Handles pagination to get all statuses.
@@ -95,7 +116,7 @@ def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame:
         "Accept": "application/vnd.github.v3+json",
     }
 
-    url = f"https://api.github.com/repos/{GITHUB_REPO}/commits/{sha}/statuses"
+    url = f"https://api.github.com/repos/{repo}/commits/{sha}/statuses"
 
     all_data = []
 
@@ -150,7 +171,7 @@ def get_commit_statuses(sha: str, pr_number: int, ref: str) -> pd.DataFrame:
     ).reset_index(drop=True)
 
 
-def get_pr_info_from_number(pr_number: str) -> dict:
+def get_pr_info_from_number(pr_number: str, repo: str = GITHUB_REPO) -> dict:
     """
     Fetch pull request information for a given PR number.
 
@@ -165,7 +186,7 @@ def get_pr_info_from_number(pr_number: str) -> dict:
         "Accept": "application/vnd.github.v3+json",
     }
 
-    url = f"https://api.github.com/repos/{GITHUB_REPO}/pulls/{pr_number}"
+    url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}"
     response = requests.get(url, headers=headers)
 
     if response.status_code != 200:
@@ -176,7 +197,7 @@ def get_pr_info_from_number(pr_number: str) -> dict:
     return response.json()
 
 
-def get_run_details(run_id: str) -> dict:
+def get_run_details(run_id: str, repo: str = GITHUB_REPO) -> dict:
     """
     Fetch run details for a given run URL.
     """
@@ -185,7 +206,7 @@ def get_run_details(run_id: str) -> dict:
         "Accept": "application/vnd.github.v3+json",
     }
 
-    url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}"
+    url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}"
     response = requests.get(url, headers=headers)
 
     if response.status_code != 200:
@@ -407,9 +428,11 @@ def _git_is_shallow(cwd: str | None) -> bool:
     return p.stdout.strip() == "true"
 
 
-def _git_fetch(options: list[str], refspecs: list[str], cwd: str | None) -> None:
+def _git_fetch(
+    options: list[str], refspecs: list[str], cwd: str | None, repo: str = GITHUB_REPO
+) -> None:
     p = subprocess.run(
-        ["git", "fetch", *options, GITHUB_GIT_URL, *refspecs],
+        ["git", "fetch", *options, _github_git_url(repo), *refspecs],
         cwd=cwd,
         capture_output=True,
         text=True,
@@ -466,7 +489,9 @@ def _filter_releases_to_version_line(
     ]
 
 
-def _fetch_release_tags(releases: list[dict], cwd: str) -> None:
+def _fetch_release_tags(
+    releases: list[dict], cwd: str, repo: str = GITHUB_REPO
+) -> None:
     """Fetch release tags into RELEASE_TAG_REF_PREFIX; skip tags missing remotely."""
     for rel in releases:
         tag_name = rel.get("tag_name")
@@ -477,24 +502,25 @@ def _fetch_release_tags(releases: list[dict], cwd: str) -> None:
                 ["--no-tags"],
                 [f"+refs/tags/{tag_name}:{RELEASE_TAG_REF_PREFIX}{tag_name}"],
                 cwd,
+                repo,
             )
         except Exception as e:
             # NOTE (strtgbb): one deleted remote tag must not abort the section, but
             # if it is the newest release the baseline moves back to an older one
             print(f"WARNING: skipping release tag {tag_name}: {e}")
 
 
-def _fetch_commit(commit_sha: str, cwd: str) -> None:
-    """Fetch the workflow commit from GITHUB_GIT_URL.
+def _fetch_commit(commit_sha: str, cwd: str, repo: str = GITHUB_REPO) -> None:
+    """Fetch the workflow commit from the repository under test.
 
     Local checkouts might be a different branch and might not have the run SHA yet.
     --filter/--unshallow would convert a full clone into a partial one, so those
     options are used only when the checkout is already shallow.
     """
     if _git_is_shallow(cwd):
-        _git_fetch(["--filter=tree:0", "--unshallow"], [commit_sha], cwd)
+        _git_fetch(["--filter=tree:0", "--unshallow"], [commit_sha], cwd, repo)
     else:
-        _git_fetch(["--no-tags"], [commit_sha], cwd)
+        _git_fetch(["--no-tags"], [commit_sha], cwd, repo)
 
 
 def get_prs_in_release_dataframe(
@@ -514,7 +540,7 @@ def get_prs_in_release_dataframe(
 
     Returns (dataframe, baseline commit time as UTC YYYY-MM-DDTHH:MM:SSZ or None).
     """
-    _fetch_commit(commit_sha, cwd)
+    _fetch_commit(commit_sha, cwd, repo)
     resolved = _git_rev_parse(commit_sha, cwd)
     if not resolved:
         raise Exception(f"Cannot resolve commit: {commit_sha!r}")
@@ -523,7 +549,7 @@ def get_prs_in_release_dataframe(
     releases = _filter_releases_to_version_line(
         _get_github_releases(repo), branch_ref, commit_sha, cwd
     )
-    _fetch_release_tags(releases, cwd)
+    _fetch_release_tags(releases, cwd, repo)
 
     _, baseline_sha, _ = _find_release_baseline(
         branch_ref, releases, cwd, commit_sha=commit_sha
@@ -564,7 +590,9 @@ def _parse_release_ref(ref_name: str) -> tuple[str, str, str | None] | None:
     return None
 
 
-def _prs_in_release_search_url(branch_name: str, baseline_time: str | None) -> str | None:
+def _prs_in_release_search_url(
+    branch_name: str, baseline_time: str | None, repo: str = GITHUB_REPO
+) -> str | None:
     if not baseline_time:
         return None
     # Tag-triggered runs expose the tag as head_branch; search needs the PR base
@@ -575,7 +603,7 @@ def _prs_in_release_search_url(branch_name: str, baseline_time: str | None) -> s
         branch_name = f"{flavour}-{minor_version}"
     query = f"is:pr base:{branch_name} merged:>{baseline_time}"
     return (
-        f"https://github.com/{GITHUB_REPO}/pulls?"
+        f"https://github.com/{repo}/pulls?"
         + urllib.parse.urlencode({"q": query})
     )
 
@@ -890,15 +918,20 @@ def get_new_fails_this_pr(
     return new_fails_df
 
 
-def _workflow_config_s3_url(pr_number: int, commit_sha: str, ref_name: str) -> str:
+def _s3_get_json(key: str):
+    s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL"))
+    response = s3_client.get_object(Bucket=S3_BUCKET, Key=key)
+    return json.loads(response["Body"].read().decode("utf-8"))
+
+
+def _workflow_config_s3_key(pr_number: int, commit_sha: str, ref_name: str) -> str:
     if pr_number == 0:
         return (
-            f"https://{S3_BUCKET}.s3.amazonaws.com/REFs/{ref_name}/{commit_sha}/"
+            f"REFs/{ref_name}/{commit_sha}/"
             "config_workflow/workflow_config_masterci.json"
         )
     return (
-        f"https://{S3_BUCKET}.s3.amazonaws.com/PRs/{pr_number}/{commit_sha}/"
-        "config_workflow/workflow_config_pr.json"
+        f"PRs/{pr_number}/{commit_sha}/config_workflow/workflow_config_pr.json"
     )
 
 
@@ -908,18 +941,14 @@ def get_workflow_config(pr_number: int, commit_sha: str, ref_name: str) -> dict:
 
     On failure, warn and return a minimal empty config (same shape recreation uses).
     """
-    url = _workflow_config_s3_url(pr_number, commit_sha, ref_name)
+    key = _workflow_config_s3_key(pr_number, commit_sha, ref_name)
     try:
-        response = requests.get(url, timeout=30)
-        if response.status_code != 200:
-            print(
-                f"WARNING:Failed to fetch workflow config from {url}: "
-                f"{response.status_code}"
-            )
-            return {"cache_jobs": {}}
-        return response.json()
+        return _s3_get_json(key)
     except Exception as e:
-        print(f"WARNING:Failed to fetch workflow config from {url}: {e}")
+        print(
+            f"WARNING:Failed to fetch workflow config from s3://{S3_BUCKET}/{key}: {e}",
+            file=sys.stderr,
+        )
         return {"cache_jobs": {}}
 
 
@@ -1130,18 +1159,16 @@ def fetch_workflow_result_json(
         ref_param = f"PR={pr_number}"
         workflow_name = "PR"
 
-    status_file = f"result_{workflow_name.lower()}.json"
-    s3_path = (
-        f"https://{S3_BUCKET}.s3.amazonaws.com/"
-        f"{ref_param.replace('=', 's/')}/{commit_sha}/{status_file}"
+    key = (
+        f"{ref_param.replace('=', 's/')}/{commit_sha}/"
+        f"result_{workflow_name.lower()}.json"
     )
     try:
-        response = requests.get(s3_path, timeout=30)
-        if response.status_code != 200:
-            return None
-        return response.json()
+        return _s3_get_json(key)
+    except ClientError:
+        return None
     except Exception as e:
-        print(f"WARNING:Failed to fetch workflow result from {s3_path}: {e}")
+        print(f"WARNING:Failed to fetch workflow result from s3://{S3_BUCKET}/{key}: {e}")
         return None
 
 
@@ -1182,7 +1209,9 @@ def add_labels_to_checks_fails(
     return df[cols]
 
 
-def format_results_as_html_table(results, *, branch_name: str = "") -> str:
+def format_results_as_html_table(
+    results, *, branch_name: str = "", repo: str = GITHUB_REPO
+) -> str:
     if not isinstance(results, pd.DataFrame):
         return results
 
@@ -1213,7 +1242,7 @@ def format_col_name(col_name: str) -> str:
             f'<span data-sort="{CVE_SEVERITY_ORDER.get(str(s).lower(), 6)}">{s}</span>'
         ),
         "PR Number": lambda n: url_to_html_link(
-            f"https://github.com/{GITHUB_REPO}/pull/{n}"
+            f"https://github.com/{repo}/pull/{n}"
         ),
         "PR Labels": lambda labels: format_pr_labels_with_verification(
             labels, branch_name=branch_name
@@ -1361,12 +1390,14 @@ def create_workflow_report(
         print(f"{DATABASE_PASSWORD_VAR} is not set")
     if not GITHUB_TOKEN:
         print("GITHUB_TOKEN is not set")
-    if not all([host, user, password, GITHUB_TOKEN]):
+    if not S3_BUCKET:
+        print("S3_ARTIFACT_BUCKET is not set")
+    if not all([host, user, password, GITHUB_TOKEN, S3_BUCKET]):
         raise Exception("Required environment variables are not set")
 
-    run_id = actions_run_url.split("/")[-1]
+    github_repo, run_id = _repo_and_run_id(actions_run_url)
 
-    run_details = get_run_details(run_id)
+    run_details = get_run_details(run_id, github_repo)
     branch_name = run_details.get("head_branch", "unknown branch")
     if pr_number is None or commit_sha is None:
         if pr_number is None:
@@ -1391,7 +1422,9 @@ def create_workflow_report(
 
     results_dfs = {
         "prs_in_release": [],
-        "job_statuses": get_commit_statuses(commit_sha, pr_number, branch_name),
+        "job_statuses": get_commit_statuses(
+            commit_sha, pr_number, branch_name, github_repo
+        ),
         "checks_fails": add_labels_to_checks_fails(
             get_checks_fails(db_client, commit_sha, branch_name),
             workflow_result,
@@ -1408,11 +1441,11 @@ def create_workflow_report(
     if pr_number == 0 and not mark_preview:
         try:
             prs_df, baseline_time = get_prs_in_release_dataframe(
-                branch_name, cwd=os.getcwd(), commit_sha=commit_sha
+                branch_name, cwd=os.getcwd(), repo=github_repo, commit_sha=commit_sha
             )
             results_dfs["prs_in_release"] = prs_df
             prs_in_release_search_url = _prs_in_release_search_url(
-                branch_name, baseline_time
+                branch_name, baseline_time, github_repo
             )
         except Exception as e:
             print(f"Error in get_prs_in_release_dataframe: {e}")
@@ -1442,8 +1475,8 @@ def create_workflow_report(
         pr_info_html = f"Release ({branch_name})"
     else:
         try:
-            pr_info = get_pr_info_from_number(pr_number)
-            pr_info_html = f"""<a href="https://github.com/{GITHUB_REPO}/pull/{pr_info["number"]}">
+            pr_info = get_pr_info_from_number(pr_number, github_repo)
+            pr_info_html = f"""<a href="https://github.com/{github_repo}/pull/{pr_info["number"]}">
                     #{pr_info.get("number")} ({pr_info.get("base", {}).get('ref')} <- {pr_info.get("head", {}).get('ref')})  {pr_info.get("title")}
                     </a>"""
             results_dfs["pr_new_fails"] = get_new_fails_this_pr(
@@ -1478,7 +1511,7 @@ def create_workflow_report(
     # Define the context for rendering
     context = {
         "title": "ClickHouse® CI Workflow Run Report",
-        "github_repo": GITHUB_REPO,
+        "github_repo": github_repo,
         "s3_bucket": S3_BUCKET,
         "pr_info_html": pr_info_html,
         "pr_number": pr_number,
@@ -1513,6 +1546,7 @@ def create_workflow_report(
             else format_results_as_html_table(
                 results_dfs["prs_in_release"],
                 branch_name=branch_name,
+                repo=github_repo,
             )
         ),
         "ci_jobs_status_html": format_results_as_html_table(

.github/actions/runner_setup/action.yml

@@ -25,7 +25,7 @@ runs:
           ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB
           .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0
           sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0
-          sudo sysctl vm.swappiness=200
+          sudo sysctl vm.swappiness=80
     - name: Install awscli
       shell: bash
       run: |

.github/workflows/fast_builds.yml

@@ -47,6 +47,8 @@ env:
   ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
   HETZNER_S3_ACCESS_KEY: ${{ secrets.HETZNER_S3_ACCESS_KEY }}
   HETZNER_S3_SECRET_KEY: ${{ secrets.HETZNER_S3_SECRET_KEY }}
+  S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }}
+  HETZNER_CACHE_BUCKET: ${{ vars.HETZNER_CACHE_BUCKET }}
 
 
 
@@ -83,7 +85,7 @@ jobs:
           else
             PREFIX="PRs/$PR_NUMBER/$COMMIT_SHA"
           fi
-          REPORT_LINK=https://s3.amazonaws.com/altinity-build-artifacts/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html
+          REPORT_LINK=https://s3.amazonaws.com/${{ vars.S3_ARTIFACT_BUCKET }}/$PREFIX/$GITHUB_RUN_ID/ci_run_report.html
           echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY
 
       - name: Prepare env script

.github/workflows/grype_scan.yml

@@ -85,7 +85,7 @@ jobs:
           if: always()
           id: upload_results
           env:
-            S3_BUCKET: "altinity-build-artifacts"
+            S3_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }}
             COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
             PR_NUMBER: ${{ env.PR_NUMBER || github.event.pull_request.number || 0 }}
             DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}

.github/workflows/init_praktika.yml

@@ -12,6 +12,7 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  S3_ARTIFACT_BUCKET: ${{ vars.S3_ARTIFACT_BUCKET }}
 
 jobs: