From 51b9e323823eaf3ce445355c32c004f0186f9edb Mon Sep 17 00:00:00 2001 From: "swift-dns-helper[bot]" <295214583+swift-dns-helper[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:53:19 +0000 Subject: [PATCH] Sync files from swift-dns/shared@c43863a5057e44c8434d2334b4b80947a6e8d9ea --- .github/lsan-suppressions.txt | 41 +++ .github/scripts/check-relevance.sh | 291 ++++++---------- .github/scripts/commit-signed.sh | 326 ++++++++++++++++++ .github/scripts/floor-baseline.sh | 2 +- .github/scripts/floor-thresholds.sh | 2 +- .github/scripts/latest-non-bot-commit.sh | 37 ++ .github/scripts/log-threshold-changes.sh | 161 +++++++++ .github/scripts/wait-for-swift-exit.sh | 24 ++ .github/workflows/benchmarks.yml | 188 +++++----- .../workflows/export-benchmark-results.yml | 43 ++- .github/workflows/integration-tests.yml | 131 +++++-- .github/workflows/lint.yml | 31 +- .github/workflows/nightly-tests.yml | 134 +++++-- .github/workflows/unit-tests.yml | 112 +++++- .../workflows/update-benchmark-thresholds.yml | 91 +++-- scripts/benchmark.sh | 13 + scripts/format.sh | 47 +++ 17 files changed, 1278 insertions(+), 396 deletions(-) create mode 100644 .github/lsan-suppressions.txt create mode 100644 .github/scripts/commit-signed.sh create mode 100644 .github/scripts/latest-non-bot-commit.sh create mode 100644 .github/scripts/log-threshold-changes.sh create mode 100644 .github/scripts/wait-for-swift-exit.sh create mode 100644 scripts/benchmark.sh create mode 100644 scripts/format.sh diff --git a/.github/lsan-suppressions.txt b/.github/lsan-suppressions.txt new file mode 100644 index 0000000..764e30d --- /dev/null +++ b/.github/lsan-suppressions.txt @@ -0,0 +1,41 @@ +# LeakSanitizer suppressions for macOS. +# +# Every entry is Darwin runtime state that is per-thread or per-image and is +# never freed by design: libdispatch worker threads outlive exit(), so their +# thread-local state is still reachable-but-unfreed when LSan runs at exit. +# None of these are reachable from library code. +# +# A `leak:` pattern is matched as a substring against every frame of the +# allocation stack. Run with `print_suppressions=1` to see which entries +# actually fire; entries that stop firing should be deleted. + +# ObjC autorelease pool pages. +leak:AutoreleasePoolPage +leak:objc_object::rootAutorelease2 + +# ObjC per-thread class-initialization bookkeeping. +leak:_setThisThreadIsInitializingClass +leak:_fetchInitializingClassList +leak:fetch_cache +leak:id2data + +# CoreFoundation thread-specific data table. +leak:__CFTSDGetTable +leak:_CFGetTSDCreateIfNeeded + +# CFRunLoop mach port bookkeeping. +leak:__CFRunLoopServiceMachPort + +# Foundation platform initialization, at image load. +leak:_NSInitializePlatform + +# dyld thread-local variable instantiation. +leak:instantiateVariable +leak:_tlv_get_addr + +# Swift runtime per-thread task context. +leak:SwiftTLSContext + +# libdispatch source handler continuations. +leak:_dispatch_continuation_alloc_from_heap +leak:_dispatch_source_set_handler diff --git a/.github/scripts/check-relevance.sh b/.github/scripts/check-relevance.sh index bc51452..a87e82a 100755 --- a/.github/scripts/check-relevance.sh +++ b/.github/scripts/check-relevance.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -Eeuo pipefail shopt -s failglob @@ -8,205 +8,122 @@ log() { printf -- "** %s\n" "$*" >&2; } error() { printf -- "** ERROR: %s\n" "$*" >&2; } fatal() { error "$@"; exit 1; } -readonly target_kind="${TARGET_KIND:?a short description of the checked targets shown in logs, e.g. 'BenchmarkPlugin' or 'test'}" -readonly target_filter="${TARGET_FILTER:?a jq filter that picks the checked targets out of .targets[], e.g. select(.type == \"test\")}" -readonly package_path="${PACKAGE_PATH:?the path of the SwiftPM package, e.g. 'Benchmarks' or '.'}" -readonly github_context_json="${GITHUB_OBJECT:?the toJson(github) context}" - -# Optional: newline-separated extra repo-relative paths that force a run when changed. -extra_force_run_paths=() -if [[ -n "${FORCE_RUN_PATHS:-}" ]]; then - mapfile -t extra_force_run_paths <<< "${FORCE_RUN_PATHS}" -fi -readonly extra_force_run_paths - -if [[ "${package_path}" == "." ]]; then - readonly package_prefix="" -else - readonly package_prefix="${package_path}/" -fi - -# Require a pull request event -event_name="$(jq -r '.event_name' <<< "${github_context_json}")" -readonly event_name - -if [[ "${event_name}" != pull_request* ]]; then - log "Not a Pull request event: '${event_name}'; Won't check for relevance." - printf 'true\n' - exit 0 -fi - -base_sha="$(jq -r '.event.pull_request.base.sha' <<< "${github_context_json}")" -head_sha="$(jq -r '.event.pull_request.head.sha' <<< "${github_context_json}")" -readonly base_sha head_sha -[[ "${base_sha}" =~ ^[0-9a-f]{40}$ && "${head_sha}" =~ ^[0-9a-f]{40}$ ]] \ - || fatal "could not read base/head sha from github context. -base_sha: '${base_sha}' -head_sha: '${head_sha}'" - -mapfile -d '' -t changed_files < <( - git diff -z --name-only "${base_sha}...${head_sha}" -) -readonly changed_files - -# - Check force-run paths -readonly force_run_paths=( - "${package_prefix}Package.swift" - "${package_prefix}Package.resolved" - ".github/scripts/check-relevance.sh" - "${extra_force_run_paths[@]}" -) - -forces_run() { - local changed_file="${1:?forces_run requires a changed file path}" - local force_path - - for force_path in "${force_run_paths[@]}"; do - if [[ "${changed_file}" == "${force_path}" || "${changed_file}" == "${force_path}"/* ]]; then - return 0 - fi - done - return 1 -} +readonly repository="${GITHUB_REPOSITORY:?the 'owner/repo' slug, e.g. 'swift-dns/swift-dns'}" +readonly workflow_ref="${GITHUB_WORKFLOW_REF:?the workflow ref, e.g. 'swift-dns/swift-dns/.github/workflows/unit-tests.yml@refs/heads/main'}" +readonly head_sha="${HEAD_SHA:?the sha of the commit this workflow is running for}" +readonly run_id="${GITHUB_RUN_ID:?the id of the current workflow run}" +readonly run_attempt="${GITHUB_RUN_ATTEMPT:?the attempt number of the current workflow run}" +readonly runner_name="${RUNNER_NAME:-}" +readonly github_token="${GITHUB_TOKEN:?a token with 'contents: read' and 'actions: read' permissions}" -for changed_file in "${changed_files[@]}"; do - [[ -n "${changed_file}" ]] || continue +# Both the benchmark and the threshold-update workflows commit with this subject prefix. +readonly benchmark_update_subject_prefix="Update of benchmark thresholds" - if forces_run "${changed_file}"; then - log "Force-run path changed: '${changed_file}'; will run." - printf 'true\n' - exit 0 - fi -done +readonly workflow_path="${workflow_ref%%@*}" +readonly workflow_file="${workflow_path##*/}" -# - Set up to check SwiftPM target graph for modified dependencies -repo_root="$(git rev-parse --show-toplevel)" -readonly repo_root - -package_dump_json="$(swift package --package-path "${package_path}" dump-package)" -readonly package_dump_json - -mapfile -d '' -t local_target_names < <( - jq --raw-output0 '.targets[].name' <<< "${package_dump_json}" -) -readonly local_target_names -[[ "${#local_target_names[@]}" -gt 0 ]] \ - || fatal "swift package dump-package found no targets in package path '${package_path}'" - -is_local_target() { - local candidate_name="${1:?is_local_target requires a target name}" - local target_name - - for target_name in "${local_target_names[@]}"; do - if [[ "${target_name}" == "${candidate_name}" ]]; then - return 0 - fi - done - return 1 -} - -get_target_dependencies() { - local target_name="${1:?get_target_dependencies requires a target name}" - - jq --raw-output0 --arg t "${target_name}" ' - .targets[] - | select(.name == $t) - | (.dependencies[]? | (.byName[0]? // .target[0]?) // empty), - (.pluginUsages[]?.plugin[0] // empty) - ' <<< "${package_dump_json}" -} +run_and_exit() { + local reason="${1:?run_and_exit requires a reason}" -# - Get all targets selected by the caller-provided filter -# SwiftPM dumps each plugin usage as {"plugin": [name, package]}, so .plugin[0] is the plugin name. -readonly target_filter_program='.targets[] | '"${target_filter}"' | .name' -mapfile -d '' -t checked_targets < <(jq --raw-output0 "${target_filter_program}" <<< "${package_dump_json}") - -if [[ "${#checked_targets[@]}" -eq 0 ]]; then - log "No ${target_kind} targets found among ${#local_target_names[@]} target(s) in '${package_path}': $(IFS=' '; printf '%s' "${local_target_names[*]}"); will return true just to be safe." + log "${reason}; will run." printf 'true\n' exit 0 -fi - -# - Find all local dependencies of the checked targets -declare -A seen=() -declare -a targets_to_visit=() -for target_name in "${checked_targets[@]}"; do - seen["${target_name}"]=1 - targets_to_visit+=("${target_name}") -done - -for (( i = 0; i < ${#targets_to_visit[@]}; i++ )); do - target_name="${targets_to_visit[i]}" +} - while IFS= read -r -d '' dependency; do - [[ -n "${dependency}" ]] || continue +github_api() { + local endpoint="${1:?github_api requires an api endpoint}" - if [[ -z "${seen[${dependency}]:-}" ]] && is_local_target "${dependency}"; then - seen["${dependency}"]=1 - targets_to_visit+=("${dependency}") - fi - done < <(get_target_dependencies "${target_name}") -done + curl --silent --show-error --fail --location \ + --header "Accept: application/vnd.github+json" \ + --header "Authorization: Bearer ${github_token}" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/${endpoint}" \ + || return 1 + return 0 +} -# - Find relevant directories to targets_to_visit -declare -A relevant_directories=() -for target_name in "${targets_to_visit[@]}"; do - target_subpath="$( - jq -r \ - --arg t "${target_name}" \ - '.targets[] | select(.name == $t) | .path // ""' <<< "${package_dump_json}" - )" - target_type="$( - jq -r \ - --arg t "${target_name}" \ - '.targets[] | select(.name == $t) | .type' <<< "${package_dump_json}" - )" +is_benchmark_update_commit() { + local commit_json="${1:?is_benchmark_update_commit requires a commit json}" + local description="${2:?is_benchmark_update_commit requires a description of the commit}" + local author subject - # Skip non-local binary targets - if [[ "${target_type}" == "binary" && -z "${target_subpath}" ]]; then - continue - fi + author="$(jq -r '.commit.author.name // ""' <<< "${commit_json}")" + subject="$(jq -r '(.commit.message // "") | split("\n")[0]' <<< "${commit_json}")" + log "${description} commit is authored by '${author}' with subject '${subject}'." - if [[ "${target_subpath}" == "." ]]; then - target_dir="${package_path}" - elif [[ -n "${target_subpath}" ]]; then - target_dir="${package_prefix}${target_subpath}" - elif [[ "${target_type}" == "plugin" ]]; then - target_dir="${package_prefix}Plugins/${target_name}" - elif [[ "${target_type}" == "test" ]]; then - target_dir="${package_prefix}Tests/${target_name}" - else - target_dir="${package_prefix}Sources/${target_name}" + if [[ "${author}" == *"[bot]" && "${subject}" == "${benchmark_update_subject_prefix}"* ]]; then + return 0 fi - [[ -d "${target_dir}" ]] || fatal "source directory not found for target '${target_name}': ${target_dir}" - absolute_target_dir="$(realpath "${target_dir}")" - relative_target_dir="${absolute_target_dir#"${repo_root}"/}" - relevant_directories["${relative_target_dir}"]=1 -done - -log "Relevant directories (${#relevant_directories[@]}), derived from ${target_kind} targets in '${package_path}':" -for relevant_dir in "${!relevant_directories[@]}"; do log " ${relevant_dir}/"; done + return 1 +} -# - See if any of the directories have had any changes -declare -a matched_files=() -for changed_file in "${changed_files[@]}"; do - [[ -n "${changed_file}" ]] || continue +head_commit_json="$(github_api "repos/${repository}/commits/${head_sha}")" \ + || fatal "could not fetch commit '${head_sha}' of '${repository}'" +readonly head_commit_json + +is_benchmark_update_commit "${head_commit_json}" "Head ${head_sha:0:7}" \ + || run_and_exit "Head commit ${head_sha:0:7} is not a benchmark thresholds update" + +parent_sha="$(jq -r '.parents[0].sha // ""' <<< "${head_commit_json}")" +readonly parent_sha +[[ "${parent_sha}" =~ ^[0-9a-f]{40}$ ]] \ + || run_and_exit "Head commit ${head_sha:0:7} has no parent commit to compare against" + +parent_commit_json="$(github_api "repos/${repository}/commits/${parent_sha}")" \ + || fatal "could not fetch commit '${parent_sha}' of '${repository}'" +readonly parent_commit_json + +is_benchmark_update_commit "${parent_commit_json}" "Parent ${parent_sha:0:7}" \ + || run_and_exit "Parent commit ${parent_sha:0:7} is not a benchmark thresholds update" + +[[ -n "${runner_name}" ]] \ + || run_and_exit "RUNNER_NAME is not set, so the current job cannot be identified" + +current_run_jobs_json="$( + github_api "repos/${repository}/actions/runs/${run_id}/attempts/${run_attempt}/jobs?per_page=100" +)" || run_and_exit "Could not fetch the jobs of the current run ${run_id}" +readonly current_run_jobs_json + +# A GitHub runner only ever hosts one running job at a time, so this identifies the current job, +# matrix values included, without having to reconstruct its display name by hand. +job_name="$( + jq -r --arg runner_name "${runner_name}" ' + [.jobs[] | select(.runner_name == $runner_name and .status == "in_progress") | .name] + | if length == 1 then .[0] else "" end + ' <<< "${current_run_jobs_json}" +)" +readonly job_name +[[ -n "${job_name}" ]] \ + || run_and_exit "Could not identify the current job among the jobs of run ${run_id} using runner '${runner_name}'" + +parent_runs_json="$( + github_api "repos/${repository}/actions/workflows/${workflow_file}/runs?head_sha=${parent_sha}&per_page=100" +)" || run_and_exit "Could not fetch the '${workflow_file}' runs of parent commit ${parent_sha:0:7}" +readonly parent_runs_json + +mapfile -t parent_run_ids < <(jq -r '.workflow_runs[].id' <<< "${parent_runs_json}") +readonly parent_run_ids +[[ "${#parent_run_ids[@]}" -gt 0 ]] \ + || run_and_exit "No '${workflow_file}' run found for parent commit ${parent_sha:0:7}" + +# A run is cancelled as a whole when a newer commit supersedes it, even though the jobs that had +# already finished did succeed, so this looks at the job instead of at the run that contains it. +for parent_run_id in "${parent_run_ids[@]}"; do + parent_run_jobs_json="$( + github_api "repos/${repository}/actions/runs/${parent_run_id}/jobs?per_page=100" + )" || continue + + succeeded="$( + jq --arg job_name "${job_name}" \ + '[.jobs[] | select(.name == $job_name and .conclusion == "success")] | length' \ + <<< "${parent_run_jobs_json}" + )" - for relevant_dir in "${!relevant_directories[@]}"; do - if [[ "${changed_file}" == "${relevant_dir}" || "${changed_file}" == "${relevant_dir}"/* ]]; then - matched_files+=("${changed_file}") - continue 2 - fi - done + if [[ "${succeeded}" -gt 0 ]]; then + log "Both ${head_sha:0:7} and its parent ${parent_sha:0:7} are benchmark thresholds updates, and '${job_name}' succeeded on the parent in run ${parent_run_id}; skipping." + printf 'false\n' + exit 0 + fi done -if [[ "${#matched_files[@]}" -eq 0 ]]; then - log "No relevant changes among ${#changed_files[@]} changed file(s) in ${base_sha:0:7}...${head_sha:0:7}; skipping." - printf 'false\n' - exit 0 -fi - -log "Relevant changes detected (${#matched_files[@]} of ${#changed_files[@]} changed file(s) in ${base_sha:0:7}...${head_sha:0:7}):" -for changed_file in "${matched_files[@]}"; do log " ${changed_file}"; done -printf 'true\n' -exit 0 +run_and_exit "No successful '${job_name}' job found for parent commit ${parent_sha:0:7} in ${#parent_run_ids[@]} '${workflow_file}' run(s)" diff --git a/.github/scripts/commit-signed.sh b/.github/scripts/commit-signed.sh new file mode 100644 index 0000000..8518ba8 --- /dev/null +++ b/.github/scripts/commit-signed.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +shopt -s failglob +IFS=$'\n\t' + +log() { printf -- "** %s\n" "$*" >&2; } +error() { printf -- "** ERROR: %s\n" "$*" >&2; } +fatal() { error "$@"; exit 1; } + +readonly token="${GH_TOKEN:?GH_TOKEN must be a token allowed to write contents to REPOSITORY}" +readonly repository="${REPOSITORY:?REPOSITORY must be the owner/name of the repository to commit to, e.g. 'swift-dns/swift-dns'}" +readonly branch="${BRANCH:?BRANCH must be the branch to create the signed commit on, e.g. 'thr-update/main'}" +readonly base_sha="${BASE_SHA:?BASE_SHA must be the 40-char commit SHA the branch is force-reset to before committing}" +readonly commit_message="${COMMIT_MESSAGE:?COMMIT_MESSAGE must be the commit message; its first line becomes the headline}" +readonly work_dir="${WORK_DIR:?WORK_DIR must point at the checked-out repository holding the changes to commit}" +readonly output_file="${OUTPUT_FILE:?OUTPUT_FILE must be the file path to write 'has-changes' and 'commit-sha' to}" +readonly pathspec="${PATHSPEC-}" +readonly api_url="${GITHUB_API_URL:-https://api.github.com}" +readonly graphql_url="${GITHUB_GRAPHQL_URL:-https://api.github.com/graphql}" + +if [[ ! "${repository}" =~ ^[^/]+/[^/]+$ ]]; then + fatal "REPOSITORY is not in 'owner/name' form: '${repository}'" +fi +if [[ ! "${base_sha}" =~ ^[0-9a-f]{40}$ ]]; then + fatal "BASE_SHA is not a 40-char commit SHA: '${base_sha}'" +fi +if [[ -z "${commit_message//[[:space:]]/}" ]]; then + fatal "COMMIT_MESSAGE is blank; the GraphQL commit headline cannot be empty" +fi +[[ -d "${work_dir}" ]] || fatal "WORK_DIR directory does not exist: '${work_dir}'" + +readonly staging_branch="${branch}-staging" + +workspace="$(mktemp -d)" || fatal "Failed to create a temporary workspace directory" +readonly workspace +staging_branch_touched=0 +trap cleanup EXIT + +readonly changes_file="${workspace}/changes.jsonl" +readonly additions_file="${workspace}/additions.json" +readonly deletions_file="${workspace}/deletions.json" +readonly payload_file="${workspace}/payload.json" +readonly response_file="${workspace}/response.json" +readonly branch_head_file="${workspace}/branch-head.json" + +# Performs a GitHub API request, writing the body to a file and printing the HTTP status. +github_api() { + local method="${1:?github_api requires an HTTP method}" + local url="${2:?github_api requires a URL}" + local request_body_file="${3?github_api requires a request body file path, empty for none}" + local response_body_file="${4:?github_api requires a response body file path}" + + local -a curl_args=( + --silent + --show-error + --request "${method}" + --header "Authorization: Bearer ${token}" + --header "Accept: application/vnd.github+json" + --header "X-GitHub-Api-Version: 2022-11-28" + --output "${response_body_file}" + --write-out '%{http_code}' + ) + + if [[ -n "${request_body_file}" ]]; then + if [[ ! -f "${request_body_file}" ]]; then + fatal "github_api request body file does not exist: '${request_body_file}'" + fi + curl_args+=( + --header "Content-Type: application/json" + --data-binary "@${request_body_file}" + ) + fi + + : > "${response_body_file}" + curl "${curl_args[@]}" "${url}" || error "Request failed: ${method} ${url}" + return 0 +} + +api_failure_details() { + local status="${1:?api_failure_details requires an HTTP status}" + local response_body_file="${2:?api_failure_details requires a response body file path}" + + printf -- 'HTTP %s\n%s' "${status}" "$(cat "${response_body_file}")" + return 0 +} + +git_in_work_dir() { + git -C "${work_dir}" "$@" + return "$?" +} + +# Collects the changed paths, splitting them into GraphQL 'additions' and 'deletions'. +# Returns 1 when the working tree holds no changes within PATHSPEC. +collect_file_changes() { + local -a status_args=(status --porcelain=v1 -z --untracked-files=all) + if [[ -n "${pathspec}" ]]; then + status_args+=(-- "${pathspec}") + fi + + local -a changed_paths=() + local entry index_status worktree_status changed_path original_path + while IFS= read -r -d '' entry; do + index_status="${entry:0:1}" + worktree_status="${entry:1:1}" + changed_path="${entry:3}" + + if [[ "${index_status}" == "U" || "${worktree_status}" == "U" ]]; then + fatal "Unmerged path in '${work_dir}': '${changed_path}'" + fi + + changed_paths+=("${changed_path}") + + if [[ "${index_status}" == "R" || "${index_status}" == "C" ]]; then + if ! IFS= read -r -d '' original_path; then + fatal "Missing original path for rename/copy entry: '${entry}'" + fi + changed_paths+=("${original_path}") + fi + done < <(git_in_work_dir "${status_args[@]}") + + if [[ "${#changed_paths[@]}" -eq 0 ]]; then + return 1 + fi + + local file_path contents + for changed_path in "${changed_paths[@]}"; do + file_path="${work_dir}/${changed_path}" + if [[ -e "${file_path}" || -L "${file_path}" ]]; then + if ! contents="$(base64 < "${file_path}" | tr -d '\n')"; then + fatal "Failed to base64-encode '${file_path}'" + fi + jq --null-input --arg path "${changed_path}" --arg contents "${contents}" \ + '{path: $path, contents: $contents}' + else + jq --null-input --arg path "${changed_path}" '{path: $path}' + fi + done > "${changes_file}" + + jq --slurp 'map(select(has("contents"))) | unique_by(.path)' \ + "${changes_file}" > "${additions_file}" + jq --slurp 'map(select(has("contents") | not)) | unique_by(.path)' \ + "${changes_file}" > "${deletions_file}" + + local addition_count deletion_count + addition_count="$(jq length "${additions_file}")" + deletion_count="$(jq length "${deletions_file}")" + log "Collected ${addition_count} addition(s) and ${deletion_count} deletion(s)." + return 0 +} + +# Prints the tree SHA the commit would produce, so an unchanged branch is left alone. +desired_tree_sha() { + local -a add_args=(add --all --) + if [[ -n "${pathspec}" ]]; then + add_args+=("${pathspec}") + fi + + if ! git_in_work_dir "${add_args[@]}"; then + fatal "Failed to stage the changes in '${work_dir}'" + fi + if ! git_in_work_dir write-tree; then + fatal "Failed to write the staged tree in '${work_dir}'" + fi + return 0 +} + +# Fetches the remote branch head into 'branch_head_file'; returns 1 when the branch is absent. +fetch_remote_branch_head() { + local url="${api_url}/repos/${repository}/branches/${branch}" + local status + status="$(github_api GET "${url}" "" "${branch_head_file}")" + + if [[ "${status}" == "404" ]]; then + return 1 + fi + if [[ "${status}" != "200" ]]; then + fatal "Failed to read branch '${branch}' of '${repository}':" \ + "$(api_failure_details "${status}" "${branch_head_file}")" + fi + return 0 +} + +point_branch_at_commit() { + local target_branch="${1:?point_branch_at_commit requires a branch name}" + local target_sha="${2:?point_branch_at_commit requires a 40-char commit SHA}" + local create_url="${api_url}/repos/${repository}/git/refs" + local update_url="${api_url}/repos/${repository}/git/refs/heads/${target_branch}" + local status + + jq --null-input --arg ref "refs/heads/${target_branch}" --arg sha "${target_sha}" \ + '{ref: $ref, sha: $sha}' > "${payload_file}" + status="$(github_api POST "${create_url}" "${payload_file}" "${response_file}")" + + if [[ "${status}" == "201" ]]; then + log "Created branch '${target_branch}' at ${target_sha:0:7}." + return 0 + fi + if [[ "${status}" != "422" ]]; then + fatal "Failed to create branch '${target_branch}' of '${repository}':" \ + "$(api_failure_details "${status}" "${response_file}")" + fi + + jq --null-input --arg sha "${target_sha}" '{sha: $sha, force: true}' > "${payload_file}" + status="$(github_api PATCH "${update_url}" "${payload_file}" "${response_file}")" + if [[ "${status}" != "200" ]]; then + fatal "Failed to force-update '${target_branch}' of '${repository}' to ${target_sha}:" \ + "$(api_failure_details "${status}" "${response_file}")" + fi + + log "Force-updated branch '${target_branch}' to ${target_sha:0:7}." + return 0 +} + +# Deletes the branch without failing the run, so cleanup never masks the real error. +delete_branch() { + local target_branch="${1:?delete_branch requires a branch name}" + local url="${api_url}/repos/${repository}/git/refs/heads/${target_branch}" + local status + status="$(github_api DELETE "${url}" "" "${response_file}")" + + if [[ "${status}" != "204" && "${status}" != "404" && "${status}" != "422" ]]; then + error "Failed to delete branch '${target_branch}' of '${repository}':" \ + "$(api_failure_details "${status}" "${response_file}")" + fi + return 0 +} + +cleanup() { + if [[ "${staging_branch_touched}" == "1" ]]; then + delete_branch "${staging_branch}" + fi + + rm -rf "${workspace}" + return 0 +} + +# Creates the commit through the GraphQL API so GitHub signs it, and prints its OID. +create_signed_commit() { + local target_branch="${1:?create_signed_commit requires a branch name}" + local headline body status commit_oid + headline="${commit_message%%$'\n'*}" + body="${commit_message#"${headline}"}" + body="${body#$'\n'}" + body="${body#$'\n'}" + + jq --null-input \ + --arg repository "${repository}" \ + --arg branch "${target_branch}" \ + --arg headline "${headline}" \ + --arg body "${body}" \ + --arg expected_head_oid "${base_sha}" \ + --slurpfile additions "${additions_file}" \ + --slurpfile deletions "${deletions_file}" \ + '{ + query: "mutation CommitOnBranch($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { oid url } } }", + variables: { + input: { + branch: {repositoryNameWithOwner: $repository, branchName: $branch}, + message: {headline: $headline, body: $body}, + expectedHeadOid: $expected_head_oid, + fileChanges: {additions: $additions[0], deletions: $deletions[0]} + } + } + }' > "${payload_file}" + + status="$(github_api POST "${graphql_url}" "${payload_file}" "${response_file}")" + if [[ "${status}" != "200" ]]; then + fatal "GraphQL createCommitOnBranch request failed:" \ + "$(api_failure_details "${status}" "${response_file}")" + fi + if jq --exit-status 'has("errors")' "${response_file}" > /dev/null; then + fatal "GraphQL createCommitOnBranch returned errors:" \ + "$(jq --compact-output '.errors' "${response_file}")" + fi + + commit_oid="$(jq --raw-output '.data.createCommitOnBranch.commit.oid' "${response_file}")" + if [[ ! "${commit_oid}" =~ ^[0-9a-f]{40}$ ]]; then + fatal "GraphQL createCommitOnBranch returned an unexpected OID: '${commit_oid}'" + fi + + printf -- '%s' "${commit_oid}" + return 0 +} + +if ! git_in_work_dir rev-parse --git-dir > /dev/null 2>&1; then + fatal "WORK_DIR is not a git repository: '${work_dir}'" +fi + +if ! collect_file_changes; then + log "No changes in '${work_dir}' under pathspec '${pathspec:-.}'; nothing to commit." + printf -- 'has-changes=false\n' >> "${output_file}" + exit 0 +fi + +wanted_tree="$(desired_tree_sha)" +readonly wanted_tree + +if fetch_remote_branch_head; then + branch_head_sha="$(jq --raw-output '.commit.sha' "${branch_head_file}")" + branch_tree_sha="$(jq --raw-output '.commit.commit.tree.sha' "${branch_head_file}")" + + if [[ "${wanted_tree}" == "${branch_tree_sha}" ]]; then + log "Branch '${branch}' already holds tree ${wanted_tree:0:7}; no new commit needed." + { + printf -- 'has-changes=true\n' + printf -- 'commit-sha=%s\n' "${branch_head_sha}" + } >> "${output_file}" + exit 0 + fi +fi + +staging_branch_touched=1 +point_branch_at_commit "${staging_branch}" "${base_sha}" + +commit_sha="$(create_signed_commit "${staging_branch}")" +readonly commit_sha + +point_branch_at_commit "${branch}" "${commit_sha}" + +{ + printf -- 'has-changes=true\n' + printf -- 'commit-sha=%s\n' "${commit_sha}" +} >> "${output_file}" + +log "✅ Created signed commit ${commit_sha:0:7} on '${repository}@${branch}'." diff --git a/.github/scripts/floor-baseline.sh b/.github/scripts/floor-baseline.sh index 4ca7b32..d997e77 100755 --- a/.github/scripts/floor-baseline.sh +++ b/.github/scripts/floor-baseline.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -Eeuo pipefail shopt -s nullglob diff --git a/.github/scripts/floor-thresholds.sh b/.github/scripts/floor-thresholds.sh index c3fdcd0..a01aaff 100755 --- a/.github/scripts/floor-thresholds.sh +++ b/.github/scripts/floor-thresholds.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -Eeuo pipefail shopt -s nullglob diff --git a/.github/scripts/latest-non-bot-commit.sh b/.github/scripts/latest-non-bot-commit.sh new file mode 100644 index 0000000..3f5a3b1 --- /dev/null +++ b/.github/scripts/latest-non-bot-commit.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +shopt -s failglob +IFS=$'\n\t' + +log() { printf -- "** %s\n" "$*" >&2; } +error() { printf -- "** ERROR: %s\n" "$*" >&2; } +fatal() { error "$@"; exit 1; } + +readonly start_ref="${START_REF:?START_REF must be the commit or ref whose history is walked}" +readonly max_depth="${MAX_DEPTH:-100}" +readonly bot_author_suffix="[bot]" + +if ! git rev-parse --verify --quiet "${start_ref}^{commit}" > /dev/null; then + fatal "Failed to resolve '${start_ref}' to a commit" +fi + +# Prints ' ' of the newest commit reachable from 'start_ref' that is not +# authored by a GitHub App or Actions bot, whose author names all end in '[bot]'. +find_latest_non_bot_commit() { + local -a log_args=(log -z --max-count="${max_depth}" --format='%h%x1f%an%x1f%s' "${start_ref}") + + local entry short_sha author_name subject + while IFS= read -r -d '' entry; do + IFS=$'\x1f' read -r short_sha author_name subject <<< "${entry}" + if [[ "${author_name}" == *"${bot_author_suffix}" ]]; then + continue + fi + printf -- '%s %s' "${short_sha}" "${subject}" + return 0 + done < <(git "${log_args[@]}") + + fatal "No commit authored by a non-bot found within ${max_depth} commits of '${start_ref}'" +} + +find_latest_non_bot_commit diff --git a/.github/scripts/log-threshold-changes.sh b/.github/scripts/log-threshold-changes.sh new file mode 100644 index 0000000..2b32d43 --- /dev/null +++ b/.github/scripts/log-threshold-changes.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +shopt -s failglob +IFS=$'\n\t' + +log() { printf -- "** %s\n" "$*" >&2; } +error() { printf -- "** ERROR: %s\n" "$*" >&2; } +fatal() { error "$@"; exit 1; } + +readonly thresholds_path="${THRESHOLDS_PATH:?THRESHOLDS_PATH must point at the benchmark Thresholds directory}" +readonly base_branch="${BASE_BRANCH:-main}" +readonly summary_file="${SUMMARY_FILE:-${GITHUB_STEP_SUMMARY:-}}" +readonly max_summary_bytes="${MAX_SUMMARY_BYTES:-524288}" + +if [[ ! -d "${thresholds_path}" ]]; then + fatal "Thresholds directory not found: '${thresholds_path}'" +fi +if ! git rev-parse --git-dir > /dev/null 2>&1; then + fatal "Not a git repository, so the threshold changes cannot be diffed: '${PWD}'" +fi + +workspace="$(mktemp -d)" || fatal "Failed to create a temporary workspace directory" +readonly workspace +trap 'rm -rf "${workspace}"' EXIT + +readonly patch_file="${workspace}/thresholds.patch" + +# Prints the commit the thresholds are compared against. A stale local 'origin/' is still a +# valid base to recover from, so a failed fetch only downgrades the base instead of failing the run. +resolve_base_commit() { + local remote_ref="origin/${base_branch}" + + if ! git fetch --no-tags --quiet origin "${base_branch}"; then + log "Could not fetch '${base_branch}' from origin; falling back to the local '${remote_ref}'." + fi + + local resolved_commit + if ! resolved_commit="$(git rev-parse --verify --quiet "${remote_ref}^{commit}")"; then + fatal "Failed to resolve '${remote_ref}'; there is no base to compare thresholds against." + fi + + printf -- '%s' "${resolved_commit}" + return 0 +} + +# Writes the threshold diff against 'base_commit' to 'destination'. Threshold files the benchmark +# run created are still untracked, so they are marked intent-to-add to make 'git diff' include them. +write_threshold_patch() { + local base_commit="${1:?write_threshold_patch requires a base commit SHA}" + local destination="${2:?write_threshold_patch requires a destination patch file path}" + + if ! git add --intent-to-add -- "${thresholds_path}"; then + fatal "Failed to mark the untracked threshold files under '${thresholds_path}' for diffing" + fi + + local -a diff_args=(diff --no-color "${base_commit}" -- "${thresholds_path}") + local diff_status=0 + git "${diff_args[@]}" > "${destination}" || diff_status="$?" + + if ! git reset --quiet -- "${thresholds_path}"; then + log "Failed to unmark the intent-to-add threshold files; the index still holds them." + fi + + if [[ "${diff_status}" -ne 0 ]]; then + fatal "Failed to diff '${thresholds_path}' against ${base_commit}; git exited ${diff_status}." + fi + + return 0 +} + +count_patched_files() { + local patch_source="${1:?count_patched_files requires a patch file path}" + + grep -c -- '^diff --git ' "${patch_source}" || true + return 0 +} + +# Job log copies carry per-line timestamps, so the summary copy below is the one to recover from. +print_patch_to_job_log() { + local base_commit="${1:?print_patch_to_job_log requires a base commit SHA}" + local patch_source="${2:?print_patch_to_job_log requires a patch file path}" + + printf -- '===== BEGIN THRESHOLD PATCH vs %s (%s) =====\n' "${base_branch}" "${base_commit}" + cat -- "${patch_source}" + printf -- '===== END THRESHOLD PATCH =====\n' + return 0 +} + +append_patch_to_summary() { + local base_commit="${1:?append_patch_to_summary requires a base commit SHA}" + local patch_source="${2:?append_patch_to_summary requires a patch file path}" + local patched_files="${3:?append_patch_to_summary requires a patched file count}" + + if [[ -z "${summary_file}" ]]; then + log "Neither SUMMARY_FILE nor GITHUB_STEP_SUMMARY is set; skipping the summary section." + return 0 + fi + + local heading="## Benchmark threshold changes vs '${base_branch}' (${base_commit:0:7})" + printf -- '\n%s\n' "${heading}" >> "${summary_file}" + + if [[ "${patched_files}" -eq 0 ]]; then + printf -- '\n%s\n' "No threshold file changes." >> "${summary_file}" + return 0 + fi + + local patch_bytes + patch_bytes="$(wc -c < "${patch_source}" | tr -d '[:space:]')" + + local recovery_notice + recovery_notice="${patched_files} file(s) changed." + recovery_notice+=" Recover them by saving the diff below as 'thresholds.patch', then:" + + { + printf -- '\n%s\n' "${recovery_notice}" + printf -- '\n%s\n' '```sh' + printf -- '%s\n' "git checkout ${base_commit}" + printf -- '%s\n' "git apply thresholds.patch" + printf -- '%s\n' '```' + printf -- '\n%s\n' '
' + printf -- '%s\n' ' Click to expand threshold changes ' + printf -- '\n%s\n' '```diff' + } >> "${summary_file}" + + if [[ "${patch_bytes}" -gt "${max_summary_bytes}" ]]; then + head -c "${max_summary_bytes}" "${patch_source}" >> "${summary_file}" + local truncation_notice + truncation_notice="... truncated at ${max_summary_bytes} bytes of ${patch_bytes};" + truncation_notice+=" the full patch is in the job log." + printf -- '\n%s\n' "${truncation_notice}" >> "${summary_file}" + else + cat -- "${patch_source}" >> "${summary_file}" + fi + + { + printf -- '%s\n' '```' + printf -- '\n%s\n' '
' + } >> "${summary_file}" + + return 0 +} + +base_commit_sha="$(resolve_base_commit)" +readonly base_commit_sha + +write_threshold_patch "${base_commit_sha}" "${patch_file}" + +patched_file_count="$(count_patched_files "${patch_file}")" +readonly patched_file_count + +append_patch_to_summary "${base_commit_sha}" "${patch_file}" "${patched_file_count}" + +if [[ "${patched_file_count}" -eq 0 ]]; then + log "✅ No threshold changes compared to '${base_branch}' (${base_commit_sha:0:7})." + exit 0 +fi + +print_patch_to_job_log "${base_commit_sha}" "${patch_file}" + +log "✅ Logged ${patched_file_count} changed threshold file(s) vs ${base_commit_sha:0:7}." diff --git a/.github/scripts/wait-for-swift-exit.sh b/.github/scripts/wait-for-swift-exit.sh new file mode 100644 index 0000000..7091c30 --- /dev/null +++ b/.github/scripts/wait-for-swift-exit.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +IFS=$'\n\t' + +log() { printf -- "** %s\n" "$*" >&2; } +error() { printf -- "** ERROR: %s\n" "$*" >&2; } +fatal() { error "$@"; exit 1; } + +readonly max_wait_seconds="${MAX_WAIT_SECONDS:-30}" + +waited_seconds=0 +while swift_processes="$(pgrep -l swift)"; do + if [[ "${waited_seconds}" -ge "${max_wait_seconds}" ]]; then + fatal "Swift processes were still running after ${max_wait_seconds}s:" "${swift_processes}" + fi + if [[ "$((waited_seconds % 10))" -eq 0 ]]; then + log "Waiting for swift processes to exit:" "${swift_processes}" + fi + sleep 1 + waited_seconds=$((waited_seconds + 1)) +done + +log "✅ No swift processes running after waiting ${waited_seconds}s." diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 9039bdb..ee6b50b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -8,6 +8,13 @@ on: push: { branches: [main] } schedule: - cron: "0 6 * * 6" + workflow_dispatch: + inputs: + swift_version: + description: "Swiftly Swift toolchain selector string to run benchmarks with. Empty uses the machine default. Example: '6.4.x-snapshot'. Must be pre-installed on the machine." + required: false + type: string + default: "" defaults: run: @@ -19,82 +26,16 @@ env: THRESHOLDS_PATH: "Benchmarks/Thresholds/" jobs: - check-benchmark-relevance: - runs-on: ubuntu-latest - - container: swift:6.3-noble - - timeout-minutes: 10 - - permissions: - pull-requests: write # For the skip comment - contents: read - - outputs: - should-run: ${{ steps.decide.outputs.should-run }} - - steps: - - name: Generate swift-dns-helper token - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ vars.SWIFT_DNS_HELPER_CLIENT_ID }} - private-key: ${{ secrets.SWIFT_DNS_HELPER_PRIVATE_KEY }} - - - name: Install jq - run: apt-get update -y && apt-get install -y jq - - - name: Check out code - uses: actions/checkout@v7 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Configure git - run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - - name: Decide whether benchmarks must run - id: decide - env: - GITHUB_OBJECT: ${{ toJson(github) }} - run: | - HAS_BENCHMARKS="$([[ -d "Benchmarks" ]] && printf 'true' || printf 'false')" - BENCHMARKS_CHANGED="$(.github/scripts/check-benchmark-relevance.sh)" - SHOULD_RUN="$([[ "${HAS_BENCHMARKS}" == "true" && "${BENCHMARKS_CHANGED}" == "true" ]] && printf 'true' || printf 'false')" - - printf '%s\n' "has-benchmarks=${HAS_BENCHMARKS}" >> "${GITHUB_OUTPUT}" - printf '%s\n' "benchmarks-changed=${BENCHMARKS_CHANGED}" >> "${GITHUB_OUTPUT}" - printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" - - - name: Construct skip comment - if: steps.decide.outputs.should-run == 'false' && startsWith(github.event_name, 'pull_request') - run: | - printf '%s\n' "## [Benchmark](${RUN_LINK}) Report" >> skip-comment.md - if [[ '${{ steps.decide.outputs.has-benchmarks }}' == 'false' ]]; then - printf '%s\n' '**⏭️ No benchmarks present — benchmarks skipped ⏭️**' >> skip-comment.md - else - printf '%s\n' '**⏭️ No benchmark-relevant changes — benchmarks skipped ⏭️**' >> skip-comment.md - fi - - - name: Comment in PR - if: steps.decide.outputs.should-run == 'false' && startsWith(github.event_name, 'pull_request') - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 - with: - github-token: ${{ steps.app-token.outputs.token }} - pr-number: ${{ github.event.pull_request.number }} - file-path: skip-comment.md - comment-tag: benchmark-ci-comment - benchmark-vs-thresholds: runs-on: [self-hosted, linux, x64] - needs: check-benchmark-relevance - if: ${{ !cancelled() && needs.check-benchmark-relevance.outputs.should-run != 'false' }} - permissions: pull-requests: write # For the comment action contents: write # For committing threshold updates + env: + TOOLCHAIN_MODIFIER: ${{ inputs.swift_version && format('+{0}', inputs.swift_version) || '' }} + steps: - name: Generate swift-dns-helper token id: app-token @@ -104,7 +45,7 @@ jobs: private-key: ${{ secrets.SWIFT_DNS_HELPER_PRIVATE_KEY }} - name: Enable full performance - run: /usr/bin/ordo-performance + run: set-perf-mode performance - name: Find commit info id: commit-info @@ -124,7 +65,7 @@ jobs: printf '%s\n' "pr_number=${PR_NUMBER}" >> "${GITHUB_OUTPUT}" SHORT_NAME="$(printf '%s' "${COMMIT_SHA}" | cut -c 1-7)" FILES_LINK="https://github.com/${{ github.repository }}/pull/${PR_NUMBER}/commits/${COMMIT_SHA}" - elif [[ "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "schedule" ]]; then + elif [[ "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then printf '%s\n' "ref=${COMMIT_SHA}" >> "${GITHUB_OUTPUT}" printf '%s\n' "sha=${COMMIT_SHA}" >> "${GITHUB_OUTPUT}" printf '%s\n' "title=Current-Branch" >> "${GITHUB_OUTPUT}" @@ -163,22 +104,35 @@ jobs: - name: Configure git run: | git config --global --add safe.directory "${GITHUB_WORKSPACE}" - git config --global user.name "${{ steps.app-token.outputs.app-slug }}[bot]" - git config --global user.email "${{ vars.SWIFT_DNS_HELPER_BOT_ID }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com" git remote set-url origin "https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/${{ github.repository }}.git" + - name: Resolve Swift toolchain for cache key + id: swift-toolchain + run: | + SWIFT_VERSION_OUTPUT="$(swiftly run swift ${TOOLCHAIN_MODIFIER} --version)" + TOOLCHAIN_ID="$(printf '%s\n' "${SWIFT_VERSION_OUTPUT}" | sed -n 's/.*Swift version \([^ ]*\).*/\1/p' | head -n 1)" + if [[ -z "${TOOLCHAIN_ID}" ]]; then + printf '%s\n' "Failed to resolve Swift toolchain version from 'swift --version' output:" >&2 + printf '%s\n' "${SWIFT_VERSION_OUTPUT}" >&2 + exit 1 + fi + printf '%s\n' "version=${TOOLCHAIN_ID}" >> "${GITHUB_OUTPUT}" + - name: Restore .build if: ${{ runner.debug != '1' }} id: restore-cache uses: actions/cache/restore@v6 with: path: Benchmarks/.build - key: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.commit-info.outputs.sha }}" - restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-" + key: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-${{ steps.commit-info.outputs.sha }}" + restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-" - name: Run benchmarks for '${{ steps.commit-info.outputs.title }}' run: | - swift package -c release \ + run-with-priority \ + swiftly run swift ${TOOLCHAIN_MODIFIER} \ + package \ + -c release \ --package-path Benchmarks \ --allow-writing-to-package-directory \ benchmark baseline update \ @@ -186,7 +140,9 @@ jobs: - name: Read benchmark result run: | - swift package -c release \ + swiftly run swift ${TOOLCHAIN_MODIFIER} \ + package \ + -c release \ --package-path Benchmarks \ --allow-writing-to-package-directory \ benchmark baseline read \ @@ -210,7 +166,9 @@ jobs: # Disable 'set -e' to prevent the script from exiting on non-zero exit codes set +e - swift package -c release \ + swiftly run swift ${TOOLCHAIN_MODIFIER} \ + package \ + -c release \ --package-path Benchmarks \ --allow-writing-to-package-directory \ benchmark thresholds check \ @@ -220,20 +178,38 @@ jobs: --no-progress \ --format markdown \ >> comparison.md - printf '%s\n' "exit-status=$?" >> "${GITHUB_OUTPUT}" + EXIT_STATUS=$? set -e + printf '%s\n' "exit-status=${EXIT_STATUS}" >> "${GITHUB_OUTPUT}" + + # SwiftPM reduces a command plugin's result to success/failure, so the benchmark plugin's + # own exit codes never survive; the outcome has to come from what the plugin printed. + if [[ "${EXIT_STATUS}" -eq 0 ]]; then + OUTCOME='within-thresholds' + elif grep -qF 'Deviations worse than threshold' comparison.md; then + OUTCOME='regression' + elif grep -qF 'Deviations better than threshold' comparison.md; then + OUTCOME='improvement' + else + OUTCOME='failed' + fi + + printf '%s\n' "outcome=${OUTCOME}" >> "${GITHUB_OUTPUT}" + - name: Cache .build if: steps.restore-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v6 with: path: Benchmarks/.build - key: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.commit-info.outputs.sha }}" + key: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-${{ steps.commit-info.outputs.sha }}" - name: Update thresholds based on the failed benchmark if: steps.comparison.outputs.exit-status != '0' run: | - swift package -c release \ + swiftly run swift ${TOOLCHAIN_MODIFIER} \ + package \ + -c release \ --package-path Benchmarks \ --allow-writing-to-package-directory \ benchmark thresholds update \ @@ -259,11 +235,22 @@ jobs: printf '%s\n' "changes-count=0" >> "${GITHUB_OUTPUT}" fi + - name: Log threshold changes against main as backup + if: steps.comparison.outputs.exit-status != '0' + continue-on-error: true + env: + BASE_BRANCH: main + run: .github/scripts/log-threshold-changes.sh + - name: Commit threshold changes if: steps.check-modifications.outputs.has-modifications == 'true' id: commit-update env: BOT_NAME: "${{ steps.app-token.outputs.app-slug }}[bot]" + GH_TOKEN: ${{ steps.app-token.outputs.token }} + REPOSITORY: ${{ github.repository }} + WORK_DIR: "." + PATHSPEC: ${{ env.THRESHOLDS_PATH }} run: | SOURCE_BRANCH="${{ github.head_ref || github.ref_name }}" @@ -284,15 +271,23 @@ jobs: printf '%s\n' "branch=${TARGET_BRANCH}" >> "${GITHUB_OUTPUT}" + BASE_SHA="$(git rev-parse "origin/${SOURCE_BRANCH}")" git checkout -B "${TARGET_BRANCH}" "origin/${SOURCE_BRANCH}" - git add "${{ env.THRESHOLDS_PATH }}" - git commit -m "Update of benchmark thresholds with the deviated baseline - - Committed to ${TARGET_BRANCH} by ${RUN_LINK}" - git push -f origin "${TARGET_BRANCH}" || printf '%s\n' "Failed to push threshold changes to '${TARGET_BRANCH}'; continuing without failing the job." - - printf '%s\n' "commit-sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + BENCHMARKED_COMMIT="$(START_REF="${BASE_SHA}" .github/scripts/latest-non-bot-commit.sh)" + COMMIT_MESSAGE="$( + printf -- '%s\n\n%s\n%s\n' \ + "Update of benchmark thresholds with the deviated baseline" \ + "Benchmarked commit: ${BENCHMARKED_COMMIT}" \ + "Committed to ${TARGET_BRANCH} by ${RUN_LINK}" + )" + BRANCH="${TARGET_BRANCH}" + OUTPUT_FILE="${GITHUB_OUTPUT}" + export BRANCH BASE_SHA COMMIT_MESSAGE OUTPUT_FILE + + if ! .github/scripts/commit-signed.sh; then + printf '%s\n' "Failed to commit thresholds to '${TARGET_BRANCH}'; continuing anyway." + fi - name: Construct result comment if the benchmark-process succeeded run: | @@ -300,17 +295,14 @@ jobs: EXIT_CODE='${{ steps.comparison.outputs.exit-status }}' - case "${EXIT_CODE}" in - 0) + case '${{ steps.comparison.outputs.outcome }}' in + within-thresholds) printf '%s\n' '**✅ Benchmark results are within the specified thresholds ✅**' >> comment.md ;; - 1) - printf '%s\n' '**❌ Benchmark results are outside the specified thresholds 📊**' >> comment.md - ;; - 2) + regression) printf '%s\n' '**❌ Benchmark results are worse than the specified thresholds 📉**' >> comment.md ;; - 4) + improvement) printf '%s\n' '**❌ Benchmark results are better than the specified thresholds 📈**' >> comment.md ;; *) @@ -318,8 +310,8 @@ jobs: ;; esac - if [[ '${{ steps.check-modifications.outputs.has-modifications }}' == 'true' ]]; then - COMMIT_SHA='${{ steps.commit-update.outputs.commit-sha }}' + COMMIT_SHA='${{ steps.commit-update.outputs.commit-sha }}' + if [[ -n "${COMMIT_SHA}" ]]; then BRANCH_NAME='${{ steps.commit-update.outputs.branch }}' SHORT_SHA="$(printf '%s' "${COMMIT_SHA}" | cut -c 1-7)" COMMIT_LINK="https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}" @@ -376,7 +368,7 @@ jobs: - name: Enable powersave mode if: always() - run: /usr/bin/ordo-performance powersave + run: set-perf-mode powersave - name: Exit with correct status run: exit ${{ steps.comparison.outputs.exit-status }} diff --git a/.github/workflows/export-benchmark-results.yml b/.github/workflows/export-benchmark-results.yml index 9e32a3e..c2e4cb7 100644 --- a/.github/workflows/export-benchmark-results.yml +++ b/.github/workflows/export-benchmark-results.yml @@ -18,6 +18,11 @@ on: description: "Regex filter for benchmark names to skip. If not provided, no benchmarks will be skipped." required: false type: string + swift_version: + description: "Swiftly Swift toolchain selector string to run benchmarks with. Empty uses the machine default. Example: '6.4.x-snapshot'. Must be pre-installed on the machine." + required: false + type: string + default: "" defaults: run: @@ -27,15 +32,14 @@ jobs: export-benchmark-results: runs-on: [self-hosted, linux, x64] - container: swift:6.3-noble - timeout-minutes: 1440 + env: + TOOLCHAIN_MODIFIER: ${{ inputs.swift_version && format('+{0}', inputs.swift_version) || '' }} + steps: - - name: Install dependencies - run: | - apt-get update - apt-get install -y zstd + - name: Enable full performance + run: set-perf-mode performance - name: Check out code uses: actions/checkout@v7 @@ -46,14 +50,26 @@ jobs: - name: Configure git run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Resolve Swift toolchain for cache key + id: swift-toolchain + run: | + SWIFT_VERSION_OUTPUT="$(swiftly run swift ${TOOLCHAIN_MODIFIER} --version)" + TOOLCHAIN_ID="$(printf '%s\n' "${SWIFT_VERSION_OUTPUT}" | sed -n 's/.*Swift version \([^ ]*\).*/\1/p' | head -n 1)" + if [[ -z "${TOOLCHAIN_ID}" ]]; then + printf '%s\n' "Failed to resolve Swift toolchain version from 'swift --version' output:" >&2 + printf '%s\n' "${SWIFT_VERSION_OUTPUT}" >&2 + exit 1 + fi + printf '%s\n' "version=${TOOLCHAIN_ID}" >> "${GITHUB_OUTPUT}" + - name: Restore .build if: ${{ runner.debug != '1' }} id: restore-cache uses: actions/cache/restore@v6 with: path: Benchmarks/.build - key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.sha }}" - restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-" + key: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-${{ github.sha }}" + restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-" - name: Run benchmarks env: @@ -62,7 +78,10 @@ jobs: SKIP: ${{ inputs.skip }} run: | COMMAND=( - swift package -c release + run-with-priority + swiftly run swift ${TOOLCHAIN_MODIFIER} + package + -c release --package-path Benchmarks --allow-writing-to-directory="${PWD}" benchmark run @@ -88,7 +107,7 @@ jobs: uses: actions/cache/save@v6 with: path: Benchmarks/.build - key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.sha }}" + key: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-${{ github.sha }}" - name: Calculate archive file name id: calculate-archive-file-name @@ -120,3 +139,7 @@ jobs: with: name: "${{ steps.calculate-archive-file-name.outputs.file_name }}" path: "*.histogram.txt" + + - name: Enable powersave mode + if: always() + run: set-perf-mode powersave diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 59df59f..4268b49 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -31,6 +31,10 @@ jobs: timeout-minutes: 15 + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + steps: - name: Check out ${{ github.event.repository.name }} uses: actions/checkout@v7 @@ -40,10 +44,19 @@ jobs: - name: Install dependencies run: | apt-get update - apt-get install -y zstd curl tcpdump + apt-get install -y zstd curl tcpdump jq + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" - name: Restore .build - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-cache" uses: actions/cache/restore@v6 with: @@ -52,21 +65,24 @@ jobs: restore-keys: "${{ github.event.repository.name }}-integration-tests-${{ matrix.config.mode }}-build-${{ matrix.swift-image.name }}-${{ runner.os }}-" - name: Build ${{ github.event.repository.name }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | swift build \ --build-tests \ + --disable-xctest \ --explicit-target-dependency-import-check error \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} - name: Cache .build - if: steps.restore-cache.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: .build key: "${{ github.event.repository.name }}-integration-tests-${{ matrix.config.mode }}-build-${{ matrix.swift-image.name }}-${{ runner.os }}-${{ hashFiles('./Package.resolved') }}" - name: Run tests + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: tests-run run: | tcpdump -w dns.pcap 'port 53 and (host 8.8.4.4 or host 240.1.2.3)' & @@ -77,6 +93,7 @@ jobs: swift test \ --filter "IntegrationTests." \ --skip-build \ + --disable-xctest \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} printf '%s\n' "exit-status=$?" >> "${GITHUB_OUTPUT}" @@ -85,6 +102,7 @@ jobs: kill "${TCPDUMP_PID}" - name: Calculate capture file name + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: calculate-capture-file-name run: | RESULT="${{ steps.tests-run.outputs.exit-status == 0 && 'success' || 'failure' }}" @@ -98,12 +116,14 @@ jobs: mv dns.pcap "${FILE_NAME}.pcap" - name: Archive packet capture + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} uses: actions/upload-artifact@v7 with: name: "${{ steps.calculate-capture-file-name.outputs.file_name }}" path: "${{ steps.calculate-capture-file-name.outputs.file_name }}.pcap" - name: Exit with correct status + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: exit ${{ steps.tests-run.outputs.exit-status }} cache-swiftly-toolchain-for-macos: @@ -119,14 +139,35 @@ jobs: timeout-minutes: 20 + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + env: SWIFTLY_HOME_DIR: "/Users/runner/.swiftly" SWIFTLY_BIN_DIR: "/Users/runner/.swiftly/bin" SWIFTLY_TOOLCHAINS_DIR: "/Users/runner/Library/Developer/Toolchains" steps: + - name: Check out ${{ github.event.repository.name }} + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Update bash via Homebrew + run: brew install bash + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" + - name: Restore swiftly home - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-home" uses: actions/cache/restore@v6 with: @@ -134,7 +175,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-home-integration-tests-${{ matrix.swift.version }}-${{ runner.os }}" - name: Restore swiftly bin - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-bin" uses: actions/cache/restore@v6 with: @@ -142,7 +183,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-bin-integration-tests-${{ matrix.swift.version }}-${{ runner.os }}" - name: Restore swiftly toolchains - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-toolchains" uses: actions/cache/restore@v6 with: @@ -150,6 +191,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-toolchain-integration-tests-${{ matrix.swift.version }}-${{ runner.os }}" - name: Update swift via Swiftly + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # Installation commands from https://www.swift.org/install/macos curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg @@ -165,21 +207,21 @@ jobs: swiftly use --assume-yes ${{ matrix.swift.version }} - name: Cache swiftly home - if: steps.restore-home.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-home.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_HOME_DIR }}" key: "${{ github.event.repository.name }}-swiftly-home-integration-tests-${{ matrix.swift.version }}-${{ runner.os }}" - name: Cache swiftly bin - if: steps.restore-bin.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-bin.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_BIN_DIR }}" key: "${{ github.event.repository.name }}-swiftly-bin-integration-tests-${{ matrix.swift.version }}-${{ runner.os }}" - name: Cache swiftly toolchains - if: steps.restore-toolchains.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-toolchains.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_TOOLCHAINS_DIR }}" @@ -208,26 +250,45 @@ jobs: needs: cache-swiftly-toolchain-for-macos + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + env: SWIFTLY_HOME_DIR: "/Users/runner/.swiftly" SWIFTLY_BIN_DIR: "/Users/runner/.swiftly/bin" SWIFTLY_TOOLCHAINS_DIR: "/Users/runner/Library/Developer/Toolchains" steps: + - name: Check out ${{ github.event.repository.name }} + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Update bash via Homebrew + run: brew install bash + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" + - name: Setup appropriate Xcode version - uses: maxim-lobanov/setup-xcode@v1 + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} + uses: maxim-lobanov/setup-xcode@ed7a3b1fda3918c0306d1b724322adc0b8cc0a90 # v1.7.0 with: xcode-version: ${{ matrix.xcode.version }} - name: Install tcpdump for packet capture + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: brew install tcpdump - - name: Check out ${{ github.event.repository.name }} - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Restore swiftly home + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-home" uses: actions/cache/restore@v6 with: @@ -235,6 +296,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-home-integration-tests-${{ matrix.swift.version }}-${{ runner.os }}" - name: Restore swiftly bin + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-bin" uses: actions/cache/restore@v6 with: @@ -242,6 +304,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-bin-integration-tests-${{ matrix.swift.version }}-${{ runner.os }}" - name: Restore swiftly toolchains + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-toolchains" uses: actions/cache/restore@v6 with: @@ -249,6 +312,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-toolchain-integration-tests-${{ matrix.swift.version }}-${{ runner.os }}" - name: Update swift via Swiftly + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # Workaround some kind of GitHub caching issue where it struggles to restore or cache the swiftly toolchain BLOCK_SIZE="$(du -s '${{ env.SWIFTLY_TOOLCHAINS_DIR }}' | awk '{print $1}')" @@ -273,7 +337,7 @@ jobs: swiftly use --assume-yes ${{ matrix.swift.version }} - name: Restore .build - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-cache" uses: actions/cache/restore@v6 with: @@ -282,22 +346,25 @@ jobs: restore-keys: "${{ github.event.repository.name }}-integration-tests-${{ matrix.config.mode }}-build-${{ matrix.swift.version }}-${{ matrix.macos.version }}-" - name: Build ${{ github.event.repository.name }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | swift build \ --build-tests \ + --disable-xctest \ --enable-code-coverage \ --explicit-target-dependency-import-check error \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} - name: Cache .build - if: steps.restore-cache.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: .build key: "${{ github.event.repository.name }}-integration-tests-${{ matrix.config.mode }}-build-${{ matrix.swift.version }}-${{ matrix.macos.version }}-${{ hashFiles('./Package.resolved') }}" - name: Run tests while capturing packets + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: tests-run run: | sudo tcpdump -w dns.pcap 'port 53 and (host 8.8.4.4 or host 240.1.2.3)' & @@ -308,6 +375,7 @@ jobs: swift test \ --filter "IntegrationTests." \ --skip-build \ + --disable-xctest \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} printf '%s\n' "exit-status=$?" >> "${GITHUB_OUTPUT}" @@ -316,6 +384,7 @@ jobs: kill "${TCPDUMP_PID}" - name: Calculate capture file name + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: calculate-capture-file-name run: | RESULT="${{ steps.tests-run.outputs.exit-status == 0 && 'success' || 'failure' }}" @@ -329,12 +398,14 @@ jobs: mv dns.pcap "${FILE_NAME}.pcap" - name: Archive packet capture + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} uses: actions/upload-artifact@v7 with: name: "${{ steps.calculate-capture-file-name.outputs.file_name }}" path: "${{ steps.calculate-capture-file-name.outputs.file_name }}.pcap" - name: Exit with correct status + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: exit ${{ steps.tests-run.outputs.exit-status }} android-integration-tests: @@ -353,21 +424,37 @@ jobs: timeout-minutes: 30 - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y zstd + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + steps: - name: Check out code uses: actions/checkout@v7 with: persist-credentials: false + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" + + - name: Install dependencies + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} + run: | + sudo apt-get update + sudo apt-get install -y zstd + - name: Replace symlinks with the actual files + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: find . -type l -exec sh -c 'LINKED="$(readlink -f "{}")"; rm "{}"; cp -a "${LINKED}" "{}"' \; - name: Run unit tests + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} uses: skiptools/swift-android-action@2044b79660f201ccef8cbce62877e4ec5409f9c8 # v2.9.5 with: # Ubuntu runners low on space causes the emulator to fail to install diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index aa43252..e819c0e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,6 +10,9 @@ defaults: run: shell: bash --noprofile --norc -Eeuo pipefail {0} +env: + RUN_LINK: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + jobs: lint: runs-on: ubuntu-latest @@ -41,9 +44,6 @@ jobs: - name: Configure git run: | git config --global --add safe.directory "${GITHUB_WORKSPACE}" - git config --global user.name "${{ steps.app-token.outputs.app-slug }}[bot]" - git config --global user.email "${{ vars.SWIFT_DNS_HELPER_BOT_ID }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com" - git remote set-url origin "https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/${{ github.repository }}.git" - name: Check config file run: | @@ -55,7 +55,7 @@ jobs: - name: Lint id: lint run: | - apt -q update && apt -yq install curl + apt -q update && apt -yq install curl jq curl -s https://raw.githubusercontent.com/swiftlang/github-workflows/refs/heads/main/.github/workflows/scripts/check-swift-format.sh | bash printf '%s\n' "exit-status=$?" >> "${GITHUB_OUTPUT}" @@ -75,17 +75,24 @@ jobs: - name: Commit formatting changes to a new branch if: steps.check-modifications.outputs.has-modifications == 'true' id: commit-update + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + REPOSITORY: ${{ github.repository }} + WORK_DIR: "." run: | CURRENT_BRANCH="${{ github.head_ref || github.ref_name }}" NEW_BRANCH="format-update/${{ github.run_id }}/${CURRENT_BRANCH}" printf '%s\n' "new-branch=${NEW_BRANCH}" >> "${GITHUB_OUTPUT}" - git checkout -b "${NEW_BRANCH}" - git add . - git commit -m "Formatting update - - Committed to ${NEW_BRANCH} by ${RUN_LINK}" - git push origin "${NEW_BRANCH}" - - printf '%s\n' "commit-sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + BASE_SHA="$(git rev-parse HEAD)" + COMMIT_MESSAGE="$( + printf -- '%s\n\n%s\n' \ + "Formatting update" \ + "Committed to ${NEW_BRANCH} by ${RUN_LINK}" + )" + BRANCH="${NEW_BRANCH}" + OUTPUT_FILE="${GITHUB_OUTPUT}" + export BRANCH BASE_SHA COMMIT_MESSAGE OUTPUT_FILE + + .github/scripts/commit-signed.sh diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index 2225373..bb2b8cd 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -29,6 +29,13 @@ jobs: timeout-minutes: 15 + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + + env: + ASAN_OPTIONS: "detect_leaks=1" + steps: - name: Check out ${{ github.event.repository.name }} uses: actions/checkout@v7 @@ -38,10 +45,19 @@ jobs: - name: Install dependencies run: | apt-get update - apt-get install -y zstd curl + apt-get install -y zstd curl jq + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" - name: Restore .build - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-cache" uses: actions/cache/restore@v6 with: @@ -50,29 +66,35 @@ jobs: restore-keys: "${{ github.event.repository.name }}-nightly-unit-tests-${{ matrix.config.mode }}-build-${{ matrix.swift-image.name }}-${{ runner.os }}-" - name: Build ${{ github.event.repository.name }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # keeping code coverage flag to check for regressions swift build \ --build-tests \ + --disable-xctest \ --enable-code-coverage \ + --sanitize=address \ --explicit-target-dependency-import-check error \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} - name: Cache .build - if: steps.restore-cache.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: .build key: "${{ github.event.repository.name }}-nightly-unit-tests-${{ matrix.config.mode }}-build-${{ matrix.swift-image.name }}-${{ runner.os }}-${{ hashFiles('./Package.resolved') }}" - name: Run tests + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # keeping code coverage flag to check for regressions swift test \ --skip "IntegrationTests." \ --skip-build \ + --disable-xctest \ --enable-code-coverage \ + --sanitize=address \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} @@ -87,17 +109,39 @@ jobs: timeout-minutes: 20 + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + env: SWIFTLY_HOME_DIR: "/Users/runner/.swiftly" SWIFTLY_BIN_DIR: "/Users/runner/.swiftly/bin" SWIFTLY_TOOLCHAINS_DIR: "/Users/runner/Library/Developer/Toolchains" steps: + - name: Check out ${{ github.event.repository.name }} + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Update bash via Homebrew + run: brew install bash + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" + - name: Set today's date + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: printf '%s\n' "TODAY=$(date "+%Y-%m-%d")" >> "${GITHUB_ENV}" - name: Restore swiftly home - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-home" uses: actions/cache/restore@v6 with: @@ -105,7 +149,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-home-${{ matrix.swift.version }}-${{ env.TODAY }}-${{ runner.os }}" - name: Restore swiftly bin - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-bin" uses: actions/cache/restore@v6 with: @@ -113,7 +157,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-bin-${{ matrix.swift.version }}-${{ env.TODAY }}-${{ runner.os }}" - name: Restore swiftly toolchains - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-toolchains" uses: actions/cache/restore@v6 with: @@ -121,6 +165,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-toolchain-${{ matrix.swift.version }}-${{ env.TODAY }}-${{ runner.os }}" - name: Update swift via Swiftly + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # Installation commands from https://www.swift.org/install/macos curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg @@ -136,21 +181,21 @@ jobs: swiftly use --assume-yes ${{ matrix.swift.version }} - name: Cache swiftly home - if: steps.restore-home.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-home.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_HOME_DIR }}" key: "${{ github.event.repository.name }}-swiftly-home-${{ matrix.swift.version }}-${{ env.TODAY }}-${{ runner.os }}" - name: Cache swiftly bin - if: steps.restore-bin.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-bin.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_BIN_DIR }}" key: "${{ github.event.repository.name }}-swiftly-bin-${{ matrix.swift.version }}-${{ env.TODAY }}-${{ runner.os }}" - name: Cache swiftly toolchains - if: steps.restore-toolchains.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-toolchains.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_TOOLCHAINS_DIR }}" @@ -177,26 +222,48 @@ jobs: timeout-minutes: 20 + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + env: + ASAN_OPTIONS: "detect_leaks=1" + LSAN_OPTIONS: "suppressions=.github/lsan-suppressions.txt:print_suppressions=1" + UBSAN_OPTIONS: "halt_on_error=1:print_stacktrace=1" SWIFTLY_HOME_DIR: "/Users/runner/.swiftly" SWIFTLY_BIN_DIR: "/Users/runner/.swiftly/bin" SWIFTLY_TOOLCHAINS_DIR: "/Users/runner/Library/Developer/Toolchains" steps: - - name: Setup appropriate Xcode version - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: ${{ matrix.xcode.version }} - - name: Check out ${{ github.event.repository.name }} uses: actions/checkout@v7 with: persist-credentials: false + - name: Update bash via Homebrew + run: brew install bash + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" + + - name: Setup appropriate Xcode version + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} + uses: maxim-lobanov/setup-xcode@ed7a3b1fda3918c0306d1b724322adc0b8cc0a90 # v1.7.0 + with: + xcode-version: ${{ matrix.xcode.version }} + - name: Set today's date + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: printf '%s\n' "TODAY=$(date "+%Y-%m-%d")" >> "${GITHUB_ENV}" - name: Restore swiftly home + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-home" uses: actions/cache/restore@v6 with: @@ -204,6 +271,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-home-${{ matrix.swift.version }}-${{ env.TODAY }}-${{ runner.os }}" - name: Restore swiftly bin + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-bin" uses: actions/cache/restore@v6 with: @@ -211,6 +279,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-bin-${{ matrix.swift.version }}-${{ env.TODAY }}-${{ runner.os }}" - name: Restore swiftly toolchains + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-toolchains" uses: actions/cache/restore@v6 with: @@ -218,6 +287,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-toolchain-${{ matrix.swift.version }}-${{ env.TODAY }}-${{ runner.os }}" - name: Update swift via Swiftly + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # Workaround some kind of GitHub caching issue where it struggles to restore or cache the swiftly toolchain BLOCK_SIZE="$(du -s '${{ env.SWIFTLY_TOOLCHAINS_DIR }}' | awk '{print $1}')" @@ -242,7 +312,7 @@ jobs: swiftly use --assume-yes ${{ matrix.swift.version }} - name: Restore .build - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-cache" uses: actions/cache/restore@v6 with: @@ -251,28 +321,34 @@ jobs: restore-keys: "${{ github.event.repository.name }}-unit-tests-${{ matrix.config.mode }}-build-${{ matrix.swift.version }}-${{ matrix.macos.version }}-" - name: Build ${{ github.event.repository.name }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | swift build \ --build-tests \ + --disable-xctest \ --enable-code-coverage \ + --sanitize=address --sanitize=undefined \ --explicit-target-dependency-import-check error \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} - name: Cache .build - if: steps.restore-cache.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: .build key: "${{ github.event.repository.name }}-unit-tests-${{ matrix.config.mode }}-build-${{ matrix.swift.version }}-${{ matrix.macos.version }}-${{ hashFiles('./Package.resolved') }}" - name: Run tests + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # keeping code coverage flag to check for regressions swift test \ --skip "IntegrationTests." \ --skip-build \ + --disable-xctest \ --enable-code-coverage \ + --sanitize=address --sanitize=undefined \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} @@ -290,21 +366,37 @@ jobs: timeout-minutes: 30 - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y zstd + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + steps: - name: Check out code uses: actions/checkout@v7 with: persist-credentials: false + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" + + - name: Install dependencies + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} + run: | + sudo apt-get update + sudo apt-get install -y zstd + - name: Replace symlinks with the actual files + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: find . -type l -exec sh -c 'LINKED="$(readlink -f "{}")"; rm "{}"; cp -a "${LINKED}" "{}"' \; - name: Run unit tests + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} uses: skiptools/swift-android-action@2044b79660f201ccef8cbce62877e4ec5409f9c8 # v2.9.5 with: # Ubuntu runners low on space causes the emulator to fail to install diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9db093e..66f7972 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -29,6 +29,13 @@ jobs: timeout-minutes: 15 + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + + env: + ASAN_OPTIONS: "detect_leaks=1" + steps: - name: Check out ${{ github.event.repository.name }} uses: actions/checkout@v7 @@ -38,10 +45,19 @@ jobs: - name: Install dependencies run: | apt-get update - apt-get install -y zstd curl + apt-get install -y zstd curl jq + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" - name: Restore .build - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-cache" uses: actions/cache/restore@v6 with: @@ -50,32 +66,39 @@ jobs: restore-keys: "${{ github.event.repository.name }}-unit-tests-${{ matrix.config.mode }}-build-${{ matrix.swift-image.name }}-${{ runner.os }}-" - name: Build ${{ github.event.repository.name }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | swift build \ --build-tests \ + --disable-xctest \ --enable-code-coverage \ + --sanitize=address \ --explicit-target-dependency-import-check error \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} - name: Cache .build - if: steps.restore-cache.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: .build key: "${{ github.event.repository.name }}-unit-tests-${{ matrix.config.mode }}-build-${{ matrix.swift-image.name }}-${{ runner.os }}-${{ hashFiles('./Package.resolved') }}" - name: Run tests + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | swift test \ --skip "IntegrationTests." \ --skip-build \ + --disable-xctest \ --enable-code-coverage \ + --sanitize=address \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} - name: Submit code coverage - uses: vapor/swift-codecov-action@v0.3 + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} + uses: vapor/swift-codecov-action@2f478f2fd22e06ca363967d0fb6c6593a8548cee # v0.3.5 with: build_parameters: "-c ${{ matrix.config.mode }}" codecov_token: "${{ secrets.CODECOV_TOKEN }}" @@ -91,14 +114,35 @@ jobs: timeout-minutes: 20 + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + env: SWIFTLY_HOME_DIR: "/Users/runner/.swiftly" SWIFTLY_BIN_DIR: "/Users/runner/.swiftly/bin" SWIFTLY_TOOLCHAINS_DIR: "/Users/runner/Library/Developer/Toolchains" steps: + - name: Check out ${{ github.event.repository.name }} + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Update bash via Homebrew + run: brew install bash + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" + - name: Restore swiftly home - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-home" uses: actions/cache/restore@v6 with: @@ -106,7 +150,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-home-${{ matrix.swift.version }}-${{ runner.os }}" - name: Restore swiftly bin - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-bin" uses: actions/cache/restore@v6 with: @@ -114,7 +158,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-bin-${{ matrix.swift.version }}-${{ runner.os }}" - name: Restore swiftly toolchains - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-toolchains" uses: actions/cache/restore@v6 with: @@ -122,6 +166,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-toolchain-${{ matrix.swift.version }}-${{ runner.os }}" - name: Update swift via Swiftly + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # Installation commands from https://www.swift.org/install/macos curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg @@ -137,21 +182,21 @@ jobs: swiftly use --assume-yes ${{ matrix.swift.version }} - name: Cache swiftly home - if: steps.restore-home.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-home.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_HOME_DIR }}" key: "${{ github.event.repository.name }}-swiftly-home-${{ matrix.swift.version }}-${{ runner.os }}" - name: Cache swiftly bin - if: steps.restore-bin.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-bin.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_BIN_DIR }}" key: "${{ github.event.repository.name }}-swiftly-bin-${{ matrix.swift.version }}-${{ runner.os }}" - name: Cache swiftly toolchains - if: steps.restore-toolchains.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-toolchains.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: "${{ env.SWIFTLY_TOOLCHAINS_DIR }}" @@ -178,23 +223,44 @@ jobs: timeout-minutes: 20 + permissions: + actions: read # For reading the previous commit's workflow runs + contents: read + env: + ASAN_OPTIONS: "detect_leaks=1" + LSAN_OPTIONS: "suppressions=.github/lsan-suppressions.txt:print_suppressions=1" + UBSAN_OPTIONS: "halt_on_error=1:print_stacktrace=1" SWIFTLY_HOME_DIR: "/Users/runner/.swiftly" SWIFTLY_BIN_DIR: "/Users/runner/.swiftly/bin" SWIFTLY_TOOLCHAINS_DIR: "/Users/runner/Library/Developer/Toolchains" steps: - - name: Setup appropriate Xcode version - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: ${{ matrix.xcode.version }} - - name: Check out ${{ github.event.repository.name }} uses: actions/checkout@v7 with: persist-credentials: false + - name: Update bash via Homebrew + run: brew install bash + + - name: Check relevance + id: check-relevance + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + SHOULD_RUN="$(.github/scripts/check-relevance.sh)" + printf '%s\n' "should-run=${SHOULD_RUN}" >> "${GITHUB_OUTPUT}" + + - name: Setup appropriate Xcode version + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} + uses: maxim-lobanov/setup-xcode@ed7a3b1fda3918c0306d1b724322adc0b8cc0a90 # v1.7.0 + with: + xcode-version: ${{ matrix.xcode.version }} + - name: Restore swiftly home + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-home" uses: actions/cache/restore@v6 with: @@ -202,6 +268,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-home-${{ matrix.swift.version }}-${{ runner.os }}" - name: Restore swiftly bin + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-bin" uses: actions/cache/restore@v6 with: @@ -209,6 +276,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-bin-${{ matrix.swift.version }}-${{ runner.os }}" - name: Restore swiftly toolchains + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} id: "restore-toolchains" uses: actions/cache/restore@v6 with: @@ -216,6 +284,7 @@ jobs: key: "${{ github.event.repository.name }}-swiftly-toolchain-${{ matrix.swift.version }}-${{ runner.os }}" - name: Update swift via Swiftly + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | # Workaround some kind of GitHub caching issue where it struggles to restore or cache the swiftly toolchain BLOCK_SIZE="$(du -s '${{ env.SWIFTLY_TOOLCHAINS_DIR }}' | awk '{print $1}')" @@ -240,7 +309,7 @@ jobs: swiftly use --assume-yes ${{ matrix.swift.version }} - name: Restore .build - if: ${{ runner.debug != '1' }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' && runner.debug != '1' }} id: "restore-cache" uses: actions/cache/restore@v6 with: @@ -249,32 +318,39 @@ jobs: restore-keys: "${{ github.event.repository.name }}-unit-tests-${{ matrix.config.mode }}-build-${{ matrix.swift.version }}-${{ matrix.macos.version }}-" - name: Build ${{ github.event.repository.name }} + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | swift build \ --build-tests \ + --disable-xctest \ --enable-code-coverage \ + --sanitize=address --sanitize=undefined \ --explicit-target-dependency-import-check error \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} - name: Cache .build - if: steps.restore-cache.outputs.cache-hit != 'true' + if: ${{ steps.check-relevance.outputs.should-run == 'true' && steps.restore-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: path: .build key: "${{ github.event.repository.name }}-unit-tests-${{ matrix.config.mode }}-build-${{ matrix.swift.version }}-${{ matrix.macos.version }}-${{ hashFiles('./Package.resolved') }}" - name: Run tests + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} run: | swift test \ --skip "IntegrationTests." \ --skip-build \ + --disable-xctest \ --enable-code-coverage \ + --sanitize=address --sanitize=undefined \ -Xswiftc -require-explicit-sendable \ -c ${{ matrix.config.mode }} - name: Submit code coverage - uses: vapor/swift-codecov-action@v0.3 + if: ${{ steps.check-relevance.outputs.should-run == 'true' }} + uses: vapor/swift-codecov-action@2f478f2fd22e06ca363967d0fb6c6593a8548cee # v0.3.5 with: build_parameters: "-c ${{ matrix.config.mode }}" codecov_token: "${{ secrets.CODECOV_TOKEN }}" diff --git a/.github/workflows/update-benchmark-thresholds.yml b/.github/workflows/update-benchmark-thresholds.yml index aa08244..6351f70 100644 --- a/.github/workflows/update-benchmark-thresholds.yml +++ b/.github/workflows/update-benchmark-thresholds.yml @@ -27,6 +27,11 @@ on: description: "Regex filter for benchmark names to skip. If not provided, no benchmarks will be skipped." required: false type: string + swift_version: + description: "Swiftly Swift toolchain selector string to run benchmarks with. Empty uses the machine default. Example: '6.4.x-snapshot'. Must be pre-installed on the machine." + required: false + type: string + default: "" defaults: run: @@ -45,6 +50,9 @@ jobs: pull-requests: write # For the comment action contents: write # For the committing the changes + env: + TOOLCHAIN_MODIFIER: ${{ inputs.swift_version && format('+{0}', inputs.swift_version) || '' }} + steps: - name: Generate swift-dns-helper token id: app-token-initial @@ -54,7 +62,7 @@ jobs: private-key: ${{ secrets.SWIFT_DNS_HELPER_PRIVATE_KEY }} - name: Enable full performance - run: /usr/bin/ordo-performance + run: set-perf-mode performance - name: Construct in-progress comment if: ${{ inputs.pr_number != '' }} @@ -80,18 +88,28 @@ jobs: - name: Configure git run: | git config --global --add safe.directory "${GITHUB_WORKSPACE}" - git config --global user.name "${{ steps.app-token-initial.outputs.app-slug }}[bot]" - git config --global user.email "${{ vars.SWIFT_DNS_HELPER_BOT_ID }}+${{ steps.app-token-initial.outputs.app-slug }}[bot]@users.noreply.github.com" git remote set-url origin "https://x-access-token:${{ steps.app-token-initial.outputs.token }}@github.com/${{ github.repository }}.git" + - name: Resolve Swift toolchain for cache key + id: swift-toolchain + run: | + SWIFT_VERSION_OUTPUT="$(swiftly run swift ${TOOLCHAIN_MODIFIER} --version)" + TOOLCHAIN_ID="$(printf '%s\n' "${SWIFT_VERSION_OUTPUT}" | sed -n 's/.*Swift version \([^ ]*\).*/\1/p' | head -n 1)" + if [[ -z "${TOOLCHAIN_ID}" ]]; then + printf '%s\n' "Failed to resolve Swift toolchain version from 'swift --version' output:" >&2 + printf '%s\n' "${SWIFT_VERSION_OUTPUT}" >&2 + exit 1 + fi + printf '%s\n' "version=${TOOLCHAIN_ID}" >> "${GITHUB_OUTPUT}" + - name: Restore .build if: ${{ runner.debug != '1' }} id: restore-cache uses: actions/cache/restore@v6 with: path: Benchmarks/.build - key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.sha }}" - restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-" + key: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-${{ github.sha }}" + restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-" - name: Update benchmark thresholds env: @@ -101,7 +119,10 @@ jobs: RUN_COUNT: ${{ inputs.run_count }} run: | COMMAND=( - swift package -c release + run-with-priority + swiftly run swift ${TOOLCHAIN_MODIFIER} + package + -c release --package-path Benchmarks --allow-writing-to-package-directory benchmark thresholds update @@ -129,7 +150,11 @@ jobs: uses: actions/cache/save@v6 with: path: Benchmarks/.build - key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.sha }}" + key: "swiftpm-benchmark-build-${{ runner.os }}-${{ steps.swift-toolchain.outputs.version }}-${{ github.sha }}" + + - name: Wait for swift processes to exit + if: always() + run: .github/scripts/wait-for-swift-exit.sh - name: Floor static thresholds and baselines if: always() @@ -149,6 +174,12 @@ jobs: printf '%s\n' "changes-count=0" >> "${GITHUB_OUTPUT}" fi + - name: Log threshold changes against main as backup + if: always() + env: + BASE_BRANCH: main + run: .github/scripts/log-threshold-changes.sh + - name: Regenerate swift-dns-helper token if: always() id: app-token @@ -160,29 +191,36 @@ jobs: - name: Commit threshold changes to another branch if: always() && (steps.check-modifications.outputs.has-modifications == 'true') id: commit-update + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + REPOSITORY: ${{ github.repository }} + WORK_DIR: "." + PATHSPEC: ${{ env.THRESHOLDS_PATH }} run: | - # Re-point the remote at the fresh token - git remote set-url origin "https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/${{ github.repository }}.git" - CURRENT_BRANCH="${{ github.head_ref || github.ref_name }}" NEW_BRANCH="auto-thr-update/${CURRENT_BRANCH}" printf '%s\n' "new-branch=${NEW_BRANCH}" >> "${GITHUB_OUTPUT}" + BASE_SHA="$(git rev-parse "origin/${CURRENT_BRANCH}")" git checkout -B "${NEW_BRANCH}" "origin/${CURRENT_BRANCH}" - git add "${{ env.THRESHOLDS_PATH }}" - git commit -m "Update of benchmark thresholds with the new baselines - - Run count: '${{ inputs.run_count }}' - Target: '${{ inputs.target }}' - Filter: '${{ inputs.filter }}' - Skip: '${{ inputs.skip }}' - - Committed to ${NEW_BRANCH} by ${RUN_LINK}" - - git push -f origin "${NEW_BRANCH}" - printf '%s\n' "commit-sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + BENCHMARKED_COMMIT="$(START_REF="${BASE_SHA}" .github/scripts/latest-non-bot-commit.sh)" + COMMIT_MESSAGE="$( + printf -- '%s\n\n%s\n%s\n%s\n%s\n\n%s\n%s\n' \ + "Update of benchmark thresholds with the new baselines" \ + "Max run count: '${{ inputs.run_count }}'" \ + "Target: '${{ inputs.target }}'" \ + "Filter: '${{ inputs.filter }}'" \ + "Skip: '${{ inputs.skip }}'" \ + "Benchmarked commit: ${BENCHMARKED_COMMIT}" \ + "Committed to ${NEW_BRANCH} by ${RUN_LINK}" + )" + BRANCH="${NEW_BRANCH}" + OUTPUT_FILE="${GITHUB_OUTPUT}" + export BRANCH BASE_SHA COMMIT_MESSAGE OUTPUT_FILE + + .github/scripts/commit-signed.sh - name: Construct result comment if: always() @@ -190,9 +228,10 @@ jobs: printf '%s\n' "## [Benchmark Thresholds Update](${RUN_LINK}) Report" >> comment.md printf '%s\n' 'Thresholds update completed with ${{ steps.check-modifications.outputs.changes-count }} files changed.' >> comment.md - if [[ '${{ steps.check-modifications.outputs.has-modifications }}' == 'true' ]]; then - SHORT_SHA="$(printf '%s' '${{ steps.commit-update.outputs.commit-sha }}' | cut -c 1-7)" - COMMIT_LINK="https://github.com/${{ github.repository }}/commit/${{ steps.commit-update.outputs.commit-sha }}" + COMMIT_SHA='${{ steps.commit-update.outputs.commit-sha }}' + if [[ -n "${COMMIT_SHA}" ]]; then + SHORT_SHA="$(printf '%s' "${COMMIT_SHA}" | cut -c 1-7)" + COMMIT_LINK="https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}" BRANCH_NAME="${{ steps.commit-update.outputs.new-branch }}" BRANCH_LINK="https://github.com/${{ github.repository }}/tree/${BRANCH_NAME}" printf '\n' >> comment.md @@ -219,4 +258,4 @@ jobs: - name: Enable powersave mode if: always() - run: /usr/bin/ordo-performance powersave + run: set-perf-mode powersave diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh new file mode 100644 index 0000000..2627830 --- /dev/null +++ b/scripts/benchmark.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -Eeuo pipefail + +# This script is in `./scripts` directory so `./scripts/..` would be the same as `./`. +script_dir="$(dirname "${BASH_SOURCE[0]}")" +base_dir="${script_dir}/.." + +swift package -c release \ + --package-path "${base_dir}/Benchmarks" \ + benchmark run \ + --path "${base_dir}/Benchmarks/Thresholds" \ + "$@" diff --git a/scripts/format.sh b/scripts/format.sh new file mode 100644 index 0000000..c8de86c --- /dev/null +++ b/scripts/format.sh @@ -0,0 +1,47 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the Swift.org open source project +## +## Copyright (c) 2024 Apple Inc. and the Swift project authors +## Licensed under Apache License v2.0 with Runtime Library Exception +## +## See https://swift.org/LICENSE.txt for license information +## See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +## +##===----------------------------------------------------------------------===## + +# source: https://raw.githubusercontent.com/swiftlang/github-workflows/refs/heads/main/.github/workflows/scripts/check-swift-format.sh + +set -Eeuo pipefail + +log() { printf -- "** %s\n" "$*" >&2; } +error() { printf -- "** ERROR: %s\n" "$*" >&2; } +fatal() { error "$@"; exit 1; } + + +if [[ -f .swiftformatignore ]]; then + log "Found swiftformatignore file..." + + log "Running swift format format..." + tr '\n' '\0' < .swiftformatignore| xargs -0 -I% printf '":(exclude)%" '| xargs git ls-files -z '*.swift' | xargs -0 swift format format --parallel --in-place + + log "Running swift format lint..." + + tr '\n' '\0' < .swiftformatignore | xargs -0 -I% printf '":(exclude)%" '| xargs git ls-files -z '*.swift' | xargs -0 swift format lint --strict --parallel +else + log "Running swift format format..." + git ls-files -z '*.swift' | xargs -0 swift format format --parallel --in-place + + log "Running swift format lint..." + + git ls-files -z '*.swift' | xargs -0 swift format lint --strict --parallel +fi + + + +log "Checking for modified files..." + +GIT_PAGER='' git diff --exit-code '*.swift' + +log "✅ Found no formatting issues."