From e398e7294e5b70391f8a19e36b0a0c607a127e84 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 3 Nov 2026 19:58:13 +0000 Subject: [PATCH 1/8] ci: configure systems tests using cloud build for google-cloud-bigquery-storage --- ci/cloudbuild/system-test.yaml | 116 +++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 ci/cloudbuild/system-test.yaml diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml new file mode 100644 index 000000000000..8a03af203889 --- /dev/null +++ b/ci/cloudbuild/system-test.yaml @@ -0,0 +1,116 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - name: 'us-central1-docker.pkg.dev/client-library-releases/release-images-prod/python-multi@sha256:d529954c373542171f9fc411e0102cebe542b511e320a965edf8820285f4f122' + id: 'run-system-tests-shard' + env: + # Populate standard GCP project environment variables expected by system test suites. + - 'GOOGLE_CLOUD_PROJECT=${PROJECT_ID}' + - 'GCLOUD_PROJECT=${PROJECT_ID}' + - 'PROJECT_ID=${PROJECT_ID}' + # Pass sharding and diff parameters consumed by ci/get_package_shards.py. + - 'MAX_SHARDS=${_MAX_SHARDS}' + - 'SHARD_INDEX=${_SHARD_INDEX}' + - 'BUILD_TYPE=${_BUILD_TYPE}' + - 'TARGET_BRANCH=${_BASE_BRANCH}' + - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' + - 'ENABLED_PACKAGES=${_ENABLED_PACKAGES}' + - 'PACKAGE_WEIGHTS=${_PACKAGE_WEIGHTS}' + # Specify the nox session consumed by ci/run_conditional_tests.sh. + - 'NOX_SESSION=${_NOX_SESSION}' + script: | + #!/usr/bin/env bash + set -eo pipefail + + # Fetch target branch history so git diff in ci/get_package_shards.py can detect modified packages. + git fetch origin "${TARGET_BRANCH}" --depth=50 || true + + # Import ci/get_package_shards.py without modifying the script on disk. + # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. + PACKAGE_LIST=$(python3 -c ' + import io, json, os, sys + from contextlib import redirect_stdout + + sys.path.insert(0, "ci") + import get_package_shards as gps + + enabled = { + p.strip().rstrip("/") + "/" + for p in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() + if p.strip() + } + if enabled: + orig_get_packages = gps.get_packages + gps.get_packages = lambda: [p for p in orig_get_packages() if p in enabled] + + buf = io.StringIO() + with redirect_stdout(buf): + if hasattr(gps, "main"): + gps.main() + else: + import runpy + runpy.run_path("ci/get_package_shards.py", run_name="__main__") + + lines = [l for l in buf.getvalue().splitlines() if l.strip()] + data = json.loads(lines[-1]) if lines else [] + shards = data.get("include", data) if isinstance(data, dict) else data + + idx = int(os.environ.get("SHARD_INDEX", "0")) + print(shards[idx].get("packages", "") if idx < len(shards) else "") + ') + + # Report immediate green (< 10s) if no enabled packages in this PR map to this shard index. + if [[ -z "${PACKAGE_LIST}" ]]; then + echo "No enabled packages assigned to shard ${SHARD_INDEX}. Exiting 0." + exit 0 + fi + + # Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS is unset. + # Set a placeholder value for noxfile.py, then unset it at pytest startup so google.auth.default() + # uses native Cloud Build VM metadata server credentials (ADC) automatically. + if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then + mkdir -p /tmp/cb_pytest_shim + cat << 'EOF' > /tmp/cb_pytest_shim/cb_adc_shim.py + import os + if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC": + os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + EOF + export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" + export PYTHONPATH="/tmp/cb_pytest_shim:${PYTHONPATH:-}" + export PYTEST_ADDOPTS="-p cb_adc_shim ${PYTEST_ADDOPTS:-}" + fi + + # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. + export PACKAGE_LIST + echo "=== System Shard ${SHARD_INDEX}/${MAX_SHARDS} running ${NOX_SESSION} for: ${PACKAGE_LIST} ===" + ./ci/run_conditional_tests.sh + +# Hard 30-minute (1800s) CI timeout ceiling. +timeout: 1800s + +substitutions: + _MAX_SHARDS: '1' + _SHARD_INDEX: '0' + _BUILD_TYPE: 'presubmit' + _BASE_BRANCH: 'main' + _TEST_ALL_PACKAGES: 'false' + _ENABLED_PACKAGES: 'packages/google-cloud-bigquery-storage' + _NOX_SESSION: 'system-3.12' + _PACKAGE_WEIGHTS: >- + google-cloud-bigquery-storage:5 + +options: + machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY From 097dddd9b039ba4a2a438b90ca943688c76abd76 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 18:17:17 +0000 Subject: [PATCH 2/8] fix build --- ci/cloudbuild/system-test.yaml | 39 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 8a03af203889..1e9aef67168c 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -28,20 +28,25 @@ steps: - 'TEST_ALL_PACKAGES=${_TEST_ALL_PACKAGES}' - 'ENABLED_PACKAGES=${_ENABLED_PACKAGES}' - 'PACKAGE_WEIGHTS=${_PACKAGE_WEIGHTS}' - # Specify the nox session consumed by ci/run_conditional_tests.sh. + # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. - 'NOX_SESSION=${_NOX_SESSION}' + - 'TEST_TYPE=${_NOX_SESSION}' script: | #!/usr/bin/env bash set -eo pipefail - # Fetch target branch history so git diff in ci/get_package_shards.py can detect modified packages. - git fetch origin "${TARGET_BRANCH}" --depth=50 || true + # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. + export PY_VERSION="${NOX_SESSION#system-}" + + # Deepen both HEAD and target branch history so git diff in ci/get_package_shards.py + # can find the merge base in Cloud Build's shallow checkout. + git fetch --no-tags origin "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" --depth=200 || true + git fetch --no-tags origin HEAD --depth=200 || true # Import ci/get_package_shards.py without modifying the script on disk. # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. PACKAGE_LIST=$(python3 -c ' - import io, json, os, sys - from contextlib import redirect_stdout + import os, sys sys.path.insert(0, "ci") import get_package_shards as gps @@ -53,19 +58,17 @@ steps: } if enabled: orig_get_packages = gps.get_packages - gps.get_packages = lambda: [p for p in orig_get_packages() if p in enabled] - - buf = io.StringIO() - with redirect_stdout(buf): - if hasattr(gps, "main"): - gps.main() - else: - import runpy - runpy.run_path("ci/get_package_shards.py", run_name="__main__") + def filtered_get_packages(*args, **kwargs): + pkgs = {} + for name, paths in orig_get_packages(*args, **kwargs).items(): + kept = [p for p in paths if p in enabled] + if kept: + pkgs[name] = kept + return pkgs + gps.get_packages = filtered_get_packages - lines = [l for l in buf.getvalue().splitlines() if l.strip()] - data = json.loads(lines[-1]) if lines else [] - shards = data.get("include", data) if isinstance(data, dict) else data + packages = gps.get_packages_to_test() + shards = gps.group_packages(packages) idx = int(os.environ.get("SHARD_INDEX", "0")) print(shards[idx].get("packages", "") if idx < len(shards) else "") @@ -88,7 +91,7 @@ steps: os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) EOF export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" - export PYTHONPATH="/tmp/cb_pytest_shim:${PYTHONPATH:-}" + export PYTHONPATH="/tmp/cb_pytest_shim${PYTHONPATH:+:${PYTHONPATH}}" export PYTEST_ADDOPTS="-p cb_adc_shim ${PYTEST_ADDOPTS:-}" fi From 5f311ca6f3c0854d64f144a16cc430e58093e0c8 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 18:51:10 +0000 Subject: [PATCH 3/8] fix build --- ci/cloudbuild/system-test.yaml | 70 ++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 1e9aef67168c..6c8bbec80a26 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -31,6 +31,9 @@ steps: # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. - 'NOX_SESSION=${_NOX_SESSION}' - 'TEST_TYPE=${_NOX_SESSION}' + # Configure parallel package execution and unbuffered Python output for real-time logging. + - 'PARALLEL_WORKERS=${_PARALLEL_WORKERS}' + - 'PYTHONUNBUFFERED=1' script: | #!/usr/bin/env bash set -eo pipefail @@ -38,10 +41,15 @@ steps: # Derive PY_VERSION (e.g. "3.12" from "system-3.12") required by ci/run_single_test.sh. export PY_VERSION="${NOX_SESSION#system-}" - # Deepen both HEAD and target branch history so git diff in ci/get_package_shards.py - # can find the merge base in Cloud Build's shallow checkout. - git fetch --no-tags origin "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" --depth=200 || true - git fetch --no-tags origin HEAD --depth=200 || true + # Deepen both the checked-out PR commit and target branch history so git diff in + # ci/get_package_shards.py finds the merge base in Cloud Build's shallow checkout, + # and populate local refs/heads/${TARGET_BRANCH} so ci/run_conditional_tests.sh's + # local insteadOf fetch succeeds silently. + git fetch --no-tags --quiet origin \ + "$(git rev-parse HEAD)" \ + "+refs/heads/${TARGET_BRANCH}:refs/remotes/origin/${TARGET_BRANCH}" \ + "+refs/heads/${TARGET_BRANCH}:refs/heads/${TARGET_BRANCH}" \ + --depth=200 || true # Import ci/get_package_shards.py without modifying the script on disk. # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. @@ -80,20 +88,57 @@ steps: exit 0 fi - # Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS is unset. - # Set a placeholder value for noxfile.py, then unset it at pytest startup so google.auth.default() - # uses native Cloud Build VM metadata server credentials (ADC) automatically. - if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then - mkdir -p /tmp/cb_pytest_shim - cat << 'EOF' > /tmp/cb_pytest_shim/cb_adc_shim.py + # Create runtime shims in /tmp/cloudbuild_shims so tracked repo files remain unmodified: + # - cloudbuild_pytest_plugin.py: Unsets the placeholder GOOGLE_APPLICATION_CREDENTIALS + # value ("CLOUD_BUILD_ADC" satisfies noxfile.py's env check) so google.auth.default() + # uses Cloud Build VM ADC, and flushes per-test results so `py.test --quiet` logs live. + # - nox wrapper: Streams [package]-prefixed output to fd 3 in real time across parallel + # workers, and only writes to stdout on failure so run_conditional_tests.sh does not + # print passing logs twice. + mkdir -p /tmp/cloudbuild_shims + cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py import os + if os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") == "CLOUD_BUILD_ADC": os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) + + + def pytest_runtest_logreport(report): + if report.when == "call" or ( + report.when in ("setup", "teardown") and report.failed + ): + print( + f"{report.outcome.upper()} {report.nodeid} ({report.duration:.1f}s)", + flush=True, + ) + EOF + + cat << 'EOF' > /tmp/cloudbuild_shims/nox + #!/usr/bin/env bash + PKG=$(basename "$PWD") + REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) + TMP_LOG=$(mktemp) + echo "[${PKG}] Starting nox $*" >&3 + "$REAL_NOX" "$@" 2>&1 | awk -v p="[${PKG}] " -v f="${TMP_LOG}" '{ + print $0 > f; fflush(f); + print p $0 > "/dev/fd/3"; fflush("/dev/fd/3"); + }' + RET=${PIPESTATUS[0]} + if [[ ${RET} -ne 0 ]]; then + cat "${TMP_LOG}" + fi + rm -f "${TMP_LOG}" + exit ${RET} EOF + chmod +x /tmp/cloudbuild_shims/nox + + if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" - export PYTHONPATH="/tmp/cb_pytest_shim${PYTHONPATH:+:${PYTHONPATH}}" - export PYTEST_ADDOPTS="-p cb_adc_shim ${PYTEST_ADDOPTS:-}" fi + export PATH="/tmp/cloudbuild_shims:${PATH}" + export PYTHONPATH="/tmp/cloudbuild_shims${PYTHONPATH:+:${PYTHONPATH}}" + export PYTEST_ADDOPTS="-p cloudbuild_pytest_plugin ${PYTEST_ADDOPTS:-}" + exec 3>&1 # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. export PACKAGE_LIST @@ -106,6 +151,7 @@ timeout: 1800s substitutions: _MAX_SHARDS: '1' _SHARD_INDEX: '0' + _PARALLEL_WORKERS: '4' _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' From adc86ba38aa0b9fec48da7ca3ea82f0e75111403 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 19:28:42 +0000 Subject: [PATCH 4/8] fix build --- ci/cloudbuild/system-test.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 6c8bbec80a26..4b16eeb812e2 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -91,10 +91,9 @@ steps: # Create runtime shims in /tmp/cloudbuild_shims so tracked repo files remain unmodified: # - cloudbuild_pytest_plugin.py: Unsets the placeholder GOOGLE_APPLICATION_CREDENTIALS # value ("CLOUD_BUILD_ADC" satisfies noxfile.py's env check) so google.auth.default() - # uses Cloud Build VM ADC, and flushes per-test results so `py.test --quiet` logs live. + # uses Cloud Build VM ADC, and enables verbose mode so `py.test --quiet` logs each test line. # - nox wrapper: Streams [package]-prefixed output to fd 3 in real time across parallel - # workers, and only writes to stdout on failure so run_conditional_tests.sh does not - # print passing logs twice. + # workers, and on failure outputs only the FAILURES summary to stdout for run_conditional_tests.sh. mkdir -p /tmp/cloudbuild_shims cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py import os @@ -103,14 +102,9 @@ steps: os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None) - def pytest_runtest_logreport(report): - if report.when == "call" or ( - report.when in ("setup", "teardown") and report.failed - ): - print( - f"{report.outcome.upper()} {report.nodeid} ({report.duration:.1f}s)", - flush=True, - ) + def pytest_configure(config): + config.option.quiet = 0 + config.option.verbose = max(config.option.verbose, 1) EOF cat << 'EOF' > /tmp/cloudbuild_shims/nox @@ -125,7 +119,7 @@ steps: }' RET=${PIPESTATUS[0]} if [[ ${RET} -ne 0 ]]; then - cat "${TMP_LOG}" + sed -n '/=== FAILURES ===/,$p' "${TMP_LOG}" fi rm -f "${TMP_LOG}" exit ${RET} From 1900e38a103bc2c5e77fb153d62af7b65a482342 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 19:33:03 +0000 Subject: [PATCH 5/8] fix build --- ci/cloudbuild/system-test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 4b16eeb812e2..716f98f695b6 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -59,6 +59,11 @@ steps: sys.path.insert(0, "ci") import get_package_shards as gps + # Only run system tests for packages directly modified in the PR (unless + # TEST_ALL_PACKAGES=true on postsubmit), ignoring .github/, ci/, or core package changes. + gps.CI_INFRASTRUCTURE_DIRS = set() + gps.CORE_PACKAGES = set() + enabled = { p.strip().rstrip("/") + "/" for p in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() From 9d07f27aae34e29e52f34f6d45f7b47d42d5e276 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 19:49:50 +0000 Subject: [PATCH 6/8] fix build --- ci/cloudbuild/system-test.yaml | 143 ++++++++++++++++++++++----------- 1 file changed, 98 insertions(+), 45 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 716f98f695b6..41baa97a54e8 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -31,8 +31,10 @@ steps: # Specify the nox session consumed by ci/run_conditional_tests.sh and ci/run_single_test.sh. - 'NOX_SESSION=${_NOX_SESSION}' - 'TEST_TYPE=${_NOX_SESSION}' - # Configure parallel package execution and unbuffered Python output for real-time logging. + # Configure parallel package execution, time budget/threshold, and unbuffered Python output. - 'PARALLEL_WORKERS=${_PARALLEL_WORKERS}' + - 'TIME_BUDGET_MINUTES=${_TIME_BUDGET_MINUTES}' + - 'HARD_TIMEOUT_MINUTES=${_HARD_TIMEOUT_MINUTES}' - 'PYTHONUNBUFFERED=1' script: | #!/usr/bin/env bash @@ -54,37 +56,46 @@ steps: # Import ci/get_package_shards.py without modifying the script on disk. # Filter get_packages() in memory to _ENABLED_PACKAGES so unmigrated packages are ignored. PACKAGE_LIST=$(python3 -c ' - import os, sys + import os + import sys sys.path.insert(0, "ci") - import get_package_shards as gps - - # Only run system tests for packages directly modified in the PR (unless - # TEST_ALL_PACKAGES=true on postsubmit), ignoring .github/, ci/, or core package changes. - gps.CI_INFRASTRUCTURE_DIRS = set() - gps.CORE_PACKAGES = set() - - enabled = { - p.strip().rstrip("/") + "/" - for p in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() - if p.strip() + import get_package_shards + + # Only run system tests for packages directly modified in the pull request + # (unless TEST_ALL_PACKAGES=true on postsubmit), ignoring changes to .github/, + # ci/, or core dependency packages. + get_package_shards.CI_INFRASTRUCTURE_DIRS = set() + get_package_shards.CORE_PACKAGES = set() + + # Normalize enabled package paths to include a trailing slash so they match + # the directory format ("packages//") returned by get_package_shards.get_packages(). + enabled_packages = { + package_path.strip().rstrip("/") + "/" + for package_path in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() + if package_path.strip() } - if enabled: - orig_get_packages = gps.get_packages + if enabled_packages: + original_get_packages = get_package_shards.get_packages + def filtered_get_packages(*args, **kwargs): - pkgs = {} - for name, paths in orig_get_packages(*args, **kwargs).items(): - kept = [p for p in paths if p in enabled] - if kept: - pkgs[name] = kept - return pkgs - gps.get_packages = filtered_get_packages - - packages = gps.get_packages_to_test() - shards = gps.group_packages(packages) - - idx = int(os.environ.get("SHARD_INDEX", "0")) - print(shards[idx].get("packages", "") if idx < len(shards) else "") + filtered_packages = {} + for package_name, package_paths in original_get_packages(*args, **kwargs).items(): + kept_paths = [ + path for path in package_paths if path in enabled_packages + ] + if kept_paths: + filtered_packages[package_name] = kept_paths + return filtered_packages + + get_package_shards.get_packages = filtered_get_packages + + # Compute the packages that need testing and distribute them across balanced shards. + packages_to_test = get_package_shards.get_packages_to_test() + shards = get_package_shards.group_packages(packages_to_test) + + shard_index = int(os.environ.get("SHARD_INDEX", "0")) + print(shards[shard_index].get("packages", "") if shard_index < len(shards) else "") ') # Report immediate green (< 10s) if no enabled packages in this PR map to this shard index. @@ -93,13 +104,18 @@ steps: exit 0 fi - # Create runtime shims in /tmp/cloudbuild_shims so tracked repo files remain unmodified: - # - cloudbuild_pytest_plugin.py: Unsets the placeholder GOOGLE_APPLICATION_CREDENTIALS - # value ("CLOUD_BUILD_ADC" satisfies noxfile.py's env check) so google.auth.default() - # uses Cloud Build VM ADC, and enables verbose mode so `py.test --quiet` logs each test line. - # - nox wrapper: Streams [package]-prefixed output to fd 3 in real time across parallel - # workers, and on failure outputs only the FAILURES summary to stdout for run_conditional_tests.sh. + # Create runtime shims in /tmp/cloudbuild_shims so tracked repository files remain unmodified. mkdir -p /tmp/cloudbuild_shims + + # 1. Pytest plugin (cloudbuild_pytest_plugin.py): + # - Package noxfile.py files skip the system session if GOOGLE_APPLICATION_CREDENTIALS + # is unset. We export a placeholder value ("CLOUD_BUILD_ADC") to satisfy noxfile.py, + # then remove it here at pytest startup so google.auth.default() automatically uses + # the Cloud Build worker VM's Application Default Credentials (ADC). + # - Package noxfile.py files invoke `py.test --quiet`, which only outputs dots (".") + # without newlines and causes Cloud Build logs to appear frozen for several minutes. + # Overriding `quiet = 0` and `verbose = 1` makes pytest print one newline-terminated + # status line as each individual test finishes. cat << 'EOF' > /tmp/cloudbuild_shims/cloudbuild_pytest_plugin.py import os @@ -112,31 +128,66 @@ steps: config.option.verbose = max(config.option.verbose, 1) EOF + # 2. Nox wrapper (/tmp/cloudbuild_shims/nox): + # - When PARALLEL_WORKERS > 1, `ci/run_conditional_tests.sh` redirects each worker's + # stdout/stderr (file descriptors 1 and 2) to a temporary log file, which otherwise + # produces no console output until the entire package finishes. + # - Before invoking `ci/run_conditional_tests.sh`, we duplicate the container's live + # stdout to file descriptor 3 (`exec 3>&1`). This wrapper streams every output line + # prefixed with `[]` directly to file descriptor 3 in real time while + # also saving the raw output to a temporary file (`TEMP_LOG_FILE`). + # - At completion, it calculates how much of the per-package time budget was used + # (e.g., 18 minutes out of a 20-minute budget = 90%) and logs the summary. + # - If the nox session fails, it outputs the `=== FAILURES ===` section to stdout so + # `ci/run_conditional_tests.sh` prints a clean failure summary at the end of the run. cat << 'EOF' > /tmp/cloudbuild_shims/nox #!/usr/bin/env bash - PKG=$(basename "$PWD") + PACKAGE_NAME=$(basename "$PWD") REAL_NOX=$(which -a nox | grep -v '/tmp/cloudbuild_shims' | head -n 1) - TMP_LOG=$(mktemp) - echo "[${PKG}] Starting nox $*" >&3 - "$REAL_NOX" "$@" 2>&1 | awk -v p="[${PKG}] " -v f="${TMP_LOG}" '{ - print $0 > f; fflush(f); - print p $0 > "/dev/fd/3"; fflush("/dev/fd/3"); + TEMP_LOG_FILE=$(mktemp) + START_TIME=$(date +%s) + echo "[${PACKAGE_NAME}] Starting nox $*" >&3 + "$REAL_NOX" "$@" 2>&1 | awk -v package_prefix="[${PACKAGE_NAME}] " -v log_file="${TEMP_LOG_FILE}" '{ + print $0 > log_file; fflush(log_file); + print package_prefix $0 > "/dev/fd/3"; fflush("/dev/fd/3"); }' - RET=${PIPESTATUS[0]} - if [[ ${RET} -ne 0 ]]; then - sed -n '/=== FAILURES ===/,$p' "${TMP_LOG}" + EXIT_CODE=${PIPESTATUS[0]} + + # Calculate elapsed time and percentage of the target time budget spent on this package. + ELAPSED_SECONDS=$(( $(date +%s) - START_TIME )) + BUDGET_MINUTES="${TIME_BUDGET_MINUTES:-20}" + HARD_LIMIT_MINUTES="${HARD_TIMEOUT_MINUTES:-30}" + BUDGET_SECONDS=$(( BUDGET_MINUTES * 60 )) + BUDGET_PERCENTAGE=$(( (ELAPSED_SECONDS * 100 + BUDGET_SECONDS / 2) / BUDGET_SECONDS )) + BUDGET_MESSAGE=$(printf "Time budget usage: %d%% (%dm %02ds / %dm budget, %dm hard threshold)" \ + "${BUDGET_PERCENTAGE}" "$(( ELAPSED_SECONDS / 60 ))" "$(( ELAPSED_SECONDS % 60 ))" \ + "${BUDGET_MINUTES}" "${HARD_LIMIT_MINUTES}") + echo "[${PACKAGE_NAME}] ${BUDGET_MESSAGE}" >&3 + echo "${BUDGET_MESSAGE}" + + # Emit failure details to stdout (captured by run_conditional_tests.sh) only when tests fail + # so passing runs do not print their logs a second time. + if [[ ${EXIT_CODE} -ne 0 ]]; then + sed -n '/=== FAILURES ===/,$p' "${TEMP_LOG_FILE}" fi - rm -f "${TMP_LOG}" - exit ${RET} + rm -f "${TEMP_LOG_FILE}" + exit ${EXIT_CODE} EOF chmod +x /tmp/cloudbuild_shims/nox + # Set placeholder credentials for noxfile.py if not already set. if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then export GOOGLE_APPLICATION_CREDENTIALS="CLOUD_BUILD_ADC" fi + + # Prepend /tmp/cloudbuild_shims to PATH and PYTHONPATH so our nox wrapper and pytest + # plugin are picked up automatically by ci/run_single_test.sh and pytest. export PATH="/tmp/cloudbuild_shims:${PATH}" export PYTHONPATH="/tmp/cloudbuild_shims${PYTHONPATH:+:${PYTHONPATH}}" export PYTEST_ADDOPTS="-p cloudbuild_pytest_plugin ${PYTEST_ADDOPTS:-}" + + # Open file descriptor 3 pointing to the container's live stdout so child workers + # spawned by xargs in ci/run_conditional_tests.sh can stream real-time logs. exec 3>&1 # Export PACKAGE_LIST and delegate execution to the existing monorepo test runner script. @@ -151,6 +202,8 @@ substitutions: _MAX_SHARDS: '1' _SHARD_INDEX: '0' _PARALLEL_WORKERS: '4' + _TIME_BUDGET_MINUTES: '20' + _HARD_TIMEOUT_MINUTES: '30' _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' From 7abcd782b9b6f72d8819fd18c0176811c575dcb0 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 20:09:22 +0000 Subject: [PATCH 7/8] fix build --- ci/cloudbuild/system-test.yaml | 36 +++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 41baa97a54e8..7ef6439c8b2d 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -165,6 +165,27 @@ steps: echo "[${PACKAGE_NAME}] ${BUDGET_MESSAGE}" >&3 echo "${BUDGET_MESSAGE}" + # Compare actual runtime in minutes against the configured weight in PACKAGE_WEIGHTS + # (defaulting to 1). Only propose an adjustment when the difference is at least 3 minutes + # (>= 15% of the 20-minute budget) to avoid noise from normal test runtime variance. + PROPOSED_WEIGHT=$(( (ELAPSED_SECONDS + 30) / 60 )) + if [[ ${PROPOSED_WEIGHT} -lt 1 ]]; then + PROPOSED_WEIGHT=1 + fi + CONFIGURED_WEIGHT=$(echo "${PACKAGE_WEIGHTS:-}" | tr ' ' '\n' | awk -F: -v package_name="${PACKAGE_NAME}" ' + $1 == package_name { gsub(/[[:space:]]/, "", $2); if ($2 != "") print $2 } + ' | tail -n 1) + CONFIGURED_WEIGHT="${CONFIGURED_WEIGHT:-1}" + WEIGHT_DIFFERENCE=$(( PROPOSED_WEIGHT - CONFIGURED_WEIGHT )) + if [[ ${WEIGHT_DIFFERENCE} -lt 0 ]]; then + WEIGHT_DIFFERENCE=$(( -WEIGHT_DIFFERENCE )) + fi + if [[ ${WEIGHT_DIFFERENCE} -ge 3 ]]; then + WEIGHT_MESSAGE="Weight adjustment needed for ${PACKAGE_NAME}: configured=${CONFIGURED_WEIGHT}, proposed=${PROPOSED_WEIGHT} (${PACKAGE_NAME}:${PROPOSED_WEIGHT})" + echo "[${PACKAGE_NAME}] ${WEIGHT_MESSAGE}" >&3 + echo "${WEIGHT_MESSAGE}" + fi + # Emit failure details to stdout (captured by run_conditional_tests.sh) only when tests fail # so passing runs do not print their logs a second time. if [[ ${EXIT_CODE} -ne 0 ]]; then @@ -207,10 +228,19 @@ substitutions: _BUILD_TYPE: 'presubmit' _BASE_BRANCH: 'main' _TEST_ALL_PACKAGES: 'false' - _ENABLED_PACKAGES: 'packages/google-cloud-bigquery-storage' + _ENABLED_PACKAGES: >- + packages/bigquery-magics + packages/google-cloud-bigquery-storage + packages/google-cloud-compute + packages/google-cloud-datastore + packages/google-cloud-dns + packages/google-cloud-error-reporting + packages/google-cloud-logging + packages/google-resumable-media + packages/pandas-gbq + packages/sqlalchemy-bigquery _NOX_SESSION: 'system-3.12' - _PACKAGE_WEIGHTS: >- - google-cloud-bigquery-storage:5 + _PACKAGE_WEIGHTS: '' options: machineType: 'E2_HIGHCPU_8' From 329d1ff64bb2fb24b701904d812bac3af8888296 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 5 Nov 2026 20:24:38 +0000 Subject: [PATCH 8/8] run tests on every change to ci --- ci/cloudbuild/system-test.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/cloudbuild/system-test.yaml b/ci/cloudbuild/system-test.yaml index 7ef6439c8b2d..0e5e924ee379 100644 --- a/ci/cloudbuild/system-test.yaml +++ b/ci/cloudbuild/system-test.yaml @@ -62,14 +62,12 @@ steps: sys.path.insert(0, "ci") import get_package_shards - # Only run system tests for packages directly modified in the pull request - # (unless TEST_ALL_PACKAGES=true on postsubmit), ignoring changes to .github/, - # ci/, or core dependency packages. - get_package_shards.CI_INFRASTRUCTURE_DIRS = set() - get_package_shards.CORE_PACKAGES = set() - # Normalize enabled package paths to include a trailing slash so they match # the directory format ("packages//") returned by get_package_shards.get_packages(). + # When CI infrastructure (ci/, .github/) or core dependencies change, + # get_package_shards.get_packages_to_test() automatically selects all packages + # returned by get_packages(). Passing handwritten_only=False to original_get_packages + # ensures every package explicitly listed in _ENABLED_PACKAGES is included. enabled_packages = { package_path.strip().rstrip("/") + "/" for package_path in os.environ.get("ENABLED_PACKAGES", "").replace(",", " ").split() @@ -80,7 +78,9 @@ steps: def filtered_get_packages(*args, **kwargs): filtered_packages = {} - for package_name, package_paths in original_get_packages(*args, **kwargs).items(): + for package_name, package_paths in original_get_packages( + handwritten_only=False + ).items(): kept_paths = [ path for path in package_paths if path in enabled_packages ]