From bd6d1a05654e8413b78a2a6572715959086c7e31 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 12:19:41 +0000 Subject: [PATCH 1/6] impl(bq_driver): Adding performance pipeline for linux --- .../integration-production-bq-driver-dm.sh | 36 ++ ci/cloudbuild/builds/lib/benchmark_results.py | 417 ++++++++++++++++++ .../builds/linux-bq-driver-benchmark.sh | 358 +++++++++++++++ .../linux-bq-driver-benchmark-ci.yaml | 29 ++ .../internal/odbc_sql_execute_utils.cc | 7 + 5 files changed, 847 insertions(+) create mode 100644 ci/cloudbuild/builds/lib/benchmark_results.py create mode 100755 ci/cloudbuild/builds/linux-bq-driver-benchmark.sh create mode 100644 ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index b58c5ddfec..bd702f1094 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -73,6 +73,7 @@ io::run cmake -B "$BUILD_DIR" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ -DODBC_DEMO_TESTING=ON \ @@ -81,6 +82,41 @@ io::run cmake -B "$BUILD_DIR" \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out +# --------------------------------------------------------------------------- +# Publish Google driver .so for performance benchmarks +# --------------------------------------------------------------------------- + +if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then + DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + + if [[ ! -f "$DRIVER_SO" ]]; then + echo "ERROR: Google ODBC driver .so was not found:" + echo " $DRIVER_SO" + exit 1 + fi + + echo "Google driver found:" + ls -lh "$DRIVER_SO" + + # Sanitize branch name for use in GCS path. + SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' + ) + + PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + + echo "Uploading Google driver artifact..." + echo "Branch: ${SANITIZED_BRANCH}" + + gcloud storage cp \ + "$DRIVER_SO" \ + "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + + echo "Google driver benchmark artifact uploaded:" + echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +fi + # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" mapfile -t ctest_args < <(ctest::common_args) diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py new file mode 100644 index 0000000000..a26391a8f7 --- /dev/null +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -0,0 +1,417 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import re +import statistics +from pathlib import Path + +TIME_RE = re.compile(r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)") + +FAILED_RE = re.compile( + r"\[\s*FAILED\s*\]\s+(.+?)(?:\s+\([\d.]+\s*(?:ns|us|ms|s)\))?\s*$" +) + + +def clean_test_name(name): + """Normalize GTest test names for comparison.""" + name = name.strip().rstrip(",") + + # Ignore GTest summary lines such as: + # + # 19 tests, listed below: + # + if re.match(r"^\d+\s+tests?,\s+listed below:$", name): + return None + + # Remove GTest parameter description. + # + # Example: + # + # Benchmark/all_bq_types_2, where GetParam() = + # ("all_bq_types_2", "SELECT * FROM ...") + # + # becomes: + # + # Benchmark/all_bq_types_2 + # + name = re.sub( + r",\s*where\s+GetParam\(\)\s*=.*$", + "", + name, + ) + + # Remove everything before the first '.'. + # + # Example: + # + # Instantiation/TestSuite.TestCase/0 + # + # becomes: + # + # TestCase/0 + if "." in name: + name = name.split(".", 1)[1] + + # Remove legacy HTAPI suffixes. + name = re.sub( + r"/(?:With|Without)HTAPI$", + "", + name, + ) + + return name + + +def parse_time_to_ms(value, unit): + """Convert a GTest duration to milliseconds.""" + if unit == "s": + return value * 1000.0 + + if unit == "us": + return value / 1000.0 + + if unit == "ns": + return value / 1_000_000.0 + + return value + + +def format_ms(value): + """Format milliseconds for the benchmark table.""" + if value is None: + return "N/A" + + return f"{value:.0f}ms" + + +def parse_gtest_output(path): + """ + Parse repeated GTest benchmark output. + + A benchmark is run multiple times. + + Rules: + 1. Test passes in every iteration: + -> use median execution time. + + 2. Test fails in ANY iteration: + -> return None, displayed as N/A. + + 3. Test is completely missing: + -> return None, displayed as N/A. + """ + path = Path(path) + + if not path.exists(): + raise FileNotFoundError(f"Benchmark output not found: {path}") + + samples = {} + failed_tests = set() + all_tests = set() + + for line in path.read_text(errors="replace").splitlines(): + + # --------------------------------------------------------------- + # Successful test + # --------------------------------------------------------------- + + time_match = TIME_RE.search(line) + + if time_match: + test_name = clean_test_name(time_match.group(1)) + + # Ignore GTest summary/non-test lines. + if test_name is None: + continue + + value = float(time_match.group(2)) + unit = time_match.group(3).lower() + + value_ms = parse_time_to_ms( + value, + unit, + ) + + samples.setdefault( + test_name, + [], + ).append(value_ms) + + all_tests.add(test_name) + + continue + + # --------------------------------------------------------------- + # Failed test + # --------------------------------------------------------------- + + failed_match = FAILED_RE.search(line) + + if failed_match: + test_name = clean_test_name(failed_match.group(1)) + + # Ignore GTest summary/non-test lines. + if test_name is None: + continue + + failed_tests.add(test_name) + all_tests.add(test_name) + + # ------------------------------------------------------------------- + # Build final result. + # ------------------------------------------------------------------- + + results = {} + + for test_name in all_tests: + + # If a test failed even once, report N/A. + if test_name in failed_tests: + results[test_name] = None + continue + + values = samples.get( + test_name, + [], + ) + + if not values: + results[test_name] = None + continue + + # Same behavior as the GitHub Actions implementation: + # median of all successful iterations. + results[test_name] = statistics.median(values) + + return results + + +def get_percentage_str(value_ms, reference_ms): + """ + Return percentage change. + + Negative = faster/improvement. + Positive = slower/degradation. + """ + if value_ms is None or reference_ms is None or reference_ms == 0: + return " (N/A)" + + pct = round(((value_ms - reference_ms) / reference_ms) * 100) + + if pct > 0: + return f" (+{pct}%)" + + if pct < 0: + return f" ({pct}%)" + + return " (0%)" + + +def main(): + parser = argparse.ArgumentParser( + description="Parse ODBC performance benchmark output." + ) + + parser.add_argument( + "--existing", + required=True, + help="Existing driver benchmark output", + ) + + parser.add_argument( + "--current", + required=True, + help="Current Google driver benchmark output", + ) + + parser.add_argument( + "--main", + required=True, + help="Main Google driver benchmark output", + ) + + parser.add_argument( + "--output", + required=True, + help="Summary output file", + ) + + parser.add_argument( + "--branch-name", + default="Current", + help="Branch name used in the Google Driver column header", + ) + + args = parser.parse_args() + + # ------------------------------------------------------------------- + # Parse benchmark outputs. + # ------------------------------------------------------------------- + + existing_data = parse_gtest_output(args.existing) + + current_data = parse_gtest_output(args.current) + + main_data = parse_gtest_output(args.main) + + if not current_data: + print( + "WARNING: No current Google benchmark results were found. " + "Google Current will be shown as N/A." + ) + + if not main_data: + print( + "WARNING: No main Google benchmark results were found. " + "Google Main will be shown as N/A." + ) + + if not existing_data: + print( + "WARNING: No Existing benchmark results were found. " + "Existing Driver will be shown as N/A." + ) + + # ------------------------------------------------------------------- + # Union of all test names. + # ------------------------------------------------------------------- + + all_tests = ( + set(existing_data.keys()) | set(current_data.keys()) | set(main_data.keys()) + ) + + sorted_tests = sorted(all_tests) + + rows = [] + + for test_name in sorted_tests: + + existing_ms = existing_data.get(test_name) + + current_ms = current_data.get(test_name) + + main_ms = main_data.get(test_name) + + existing_raw = format_ms(existing_ms) + + current_raw = format_ms(current_ms) + + main_raw = format_ms(main_ms) + + # --------------------------------------------------------------- + # Current vs Existing + # --------------------------------------------------------------- + + current_pct = "" + + if current_ms is not None: + current_pct = get_percentage_str( + current_ms, + existing_ms, + ) + + # --------------------------------------------------------------- + # Main vs Current + # --------------------------------------------------------------- + + main_pct = "" + + if main_ms is not None: + main_pct = get_percentage_str( + main_ms, + current_ms, + ) + + current_value = f"{current_raw}{current_pct}" + + main_value = f"{main_raw}{main_pct}" + + rows.append( + ( + test_name, + existing_raw, + current_value, + main_value, + ) + ) + + # ------------------------------------------------------------------- + # Generate Markdown table. + # + # This intentionally follows the existing GHA table format. + # ------------------------------------------------------------------- + + h1 = "Test Case" + h2 = "Existing Driver (Current)" + h3 = f"Google Driver ({args.branch_name})" + h4 = "Google Driver (Main)" + + w1 = max([len(h1)] + [len(row[0]) for row in rows]) if rows else len(h1) + + w2 = max([len(h2)] + [len(row[1]) for row in rows]) if rows else len(h2) + + w3 = max([len(h3)] + [len(row[2]) for row in rows]) if rows else len(h3) + + w4 = max([len(h4)] + [len(row[3]) for row in rows]) if rows else len(h4) + + table = ( + f"*Percentages in **{h3}** show change relative to " + f"**{h2}**. Percentages in **{h4}** show change relative " + f"to **{h3}**. Negative values indicate improvement " + f"(faster test execution), positive values indicate " + f"degradation (slower).*" + "\n\n" + ) + + table += ( + f"| {h1.ljust(w1)} " + f"| {h2.ljust(w2)} " + f"| {h3.ljust(w3)} " + f"| {h4.ljust(w4)} |\n" + ) + + table += ( + "|-" + + ("-" * w1) + + "-|-" + + ("-" * w2) + + "-|-" + + ("-" * w3) + + "-|-" + + ("-" * w4) + + "-|\n" + ) + + for row in rows: + table += ( + f"| {row[0].ljust(w1)} " + f"| {row[1].ljust(w2)} " + f"| {row[2].ljust(w3)} " + f"| {row[3].ljust(w4)} |\n" + ) + + # ------------------------------------------------------------------- + # Write output. + # ------------------------------------------------------------------- + + output_path = Path(args.output) + + output_path.write_text(table) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh new file mode 100755 index 0000000000..66031ffced --- /dev/null +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -0,0 +1,358 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +source "$(dirname "$0")/../../lib/init.sh" +source module ci/install-dependencies.sh + +source module ci/cloudbuild/builds/lib/cmake.sh +source module ci/cloudbuild/builds/lib/secrets.sh +source module ci/lib/io.sh + +WORKSPACE_DIR=$(pwd) + +# ============================================================================ +# Configuration +# ============================================================================ + +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" + +PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" + +rm -rf "${RESULTS_DIR}" +mkdir -p "${RESULTS_DIR}" + +# ============================================================================ +# Branch +# ============================================================================ + +BRANCH_NAME="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH="$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9._-]/_/g' +)" + +echo "============================================================" +echo "Linux ODBC Performance Benchmark" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" +echo + +# ============================================================================ +# Result files +# ============================================================================ + +CURRENT_RESULT="${RESULTS_DIR}/current.txt" +MAIN_RESULT="${RESULTS_DIR}/main.txt" +EXISTING_RESULT="${RESULTS_DIR}/existing.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" + +# ============================================================================ +# Driver locations +# ============================================================================ + +DRIVER_PATH="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + +CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" +MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" + +CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" +MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" + +# ============================================================================ +# ODBC configuration +# ============================================================================ + +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" +EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" + +cd "$WORKSPACE_DIR" + +# This is the name of DSN set in odbc.ini. +mapfile -t cmake_args < <(cmake::common_args) + +GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" +EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" + +export ODBC_TESTS_DSN="SampleDSNGoogleDriver" + +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} +export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini +export ODBCINI="${GOOGLE_ODBCINI}" + +# ============================================================================ +# Validate ODBC configuration +# ============================================================================ + +if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then + echo "ERROR: Google ODBC configuration not found:" + echo " ${GOOGLE_ODBCINI}" + exit 1 +fi + +if [[ ! -f "${EXISTING_ODBCINI}" ]]; then + echo "ERROR: Existing ODBC configuration not found:" + echo " ${EXISTING_ODBCINI}" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Download Google Current driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google Current driver" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_SO_GCS}" \ + "${CURRENT_SO}" + +# --------------------------------------------------------------------------- +# Download Google Main driver +# --------------------------------------------------------------------------- + +echo +echo "Downloading Google driver from main" +echo "------------------------------------------------------------" + +if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then + echo "Main Google driver downloaded successfully." + HAS_MAIN_DRIVER=true +else + echo "WARNING: Main Google driver was not found." + echo "WARNING: Google Main benchmark will be skipped." + HAS_MAIN_DRIVER=false +fi + +mkdir -p "$(dirname "${DRIVER_PATH}")" + +cp \ + /opt/odbc-driver/roots.pem \ + "${DRIVER_PATH%/*}/roots.pem" + +# ============================================================================ +# Run benchmark +# ============================================================================ + +run_benchmark() { + local name="$1" + local output_file="$2" + local dsn="$3" + local bq_tests_flag="$4" + + echo + echo "Reconfiguring performance_test for ${name}" + echo "BQ_DRIVER_INTEGRATION_TESTS = ${bq_tests_flag}" + echo "------------------------------------------------------------" + + io::run cmake -S "${WORKSPACE_DIR}" \ + -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS="${bq_tests_flag}" + + io::run cmake --build "${BUILD_DIR}" \ + --target performance_test \ + --parallel "$(nproc)" + + PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" + + if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" + fi + + if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + echo "ERROR: performance_test was not found." + + find "${BUILD_DIR}" \ + -type f \ + -name "performance_test" \ + -print 2>/dev/null || true + + return 1 + fi + + echo "performance_test:" + echo " ${PERFORMANCE_TEST}" + + : >"${output_file}" + + local failed_iterations=0 + + for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + echo + echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" + + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >>"${output_file}" + + echo "=== ODBC tests DSN is====${ODBC_TESTS_DSN}===" + echo "=== ODBCINI is====${dsn}===" + + set +e + + ODBCINI="${dsn}" \ + ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ + "${PERFORMANCE_TEST}" \ + >>"${output_file}" 2>&1 + + run_exit=$? + + set -e + + if [[ "${run_exit}" -ne 0 ]]; then + echo "WARNING: ${name} iteration ${i} failed with exit code ${run_exit}" + echo "WARNING: Continuing with remaining iterations." + + echo "============================================================" + echo "Detailed failure for ${name}, iteration ${i}:" + echo "============================================================" + + grep -E -B 20 -A 20 \ + '\[ *FAILED *\]|Failure|FAILED|ERROR|Error|error|SQLSTATE|Diagnostic|diagnostic|SQL_ERROR|SQLConnect|SQLDriverConnect|Exception|exception' \ + "${output_file}" || true + + echo "============================================================" + + failed_iterations=$((failed_iterations + 1)) + fi + done + + echo + echo "${name} benchmark completed." + echo "Failed iterations: ${failed_iterations}/${BENCHMARK_ITERATIONS}" + echo "Raw result: ${output_file}" + + # Benchmark failures must not fail Cloud Build. + return 0 +} + +# ============================================================================ +# Google Current +# +# The existing Google DSN points to DRIVER_PATH. +# Replace the driver binary before running the benchmark. +# ============================================================================ + +echo +echo "Preparing Google Current" +echo "------------------------------------------------------------" + +cp "${CURRENT_SO}" "${DRIVER_PATH}" +ls -lh "${DRIVER_PATH}" + +run_benchmark \ + "Google Current" \ + "${CURRENT_RESULT}" \ + "${GOOGLE_ODBCINI}" \ + "ON" + +# ============================================================================ +# Google Main +# ============================================================================ + +echo +echo "Preparing Google Main" +echo "------------------------------------------------------------" + +if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then + cp "${MAIN_SO}" "${DRIVER_PATH}" + ls -lh "${DRIVER_PATH}" + + run_benchmark \ + "Google Main" \ + "${MAIN_RESULT}" \ + "${GOOGLE_ODBCINI}" \ + "ON" +else + echo "Google Main benchmark skipped: main driver artifact unavailable." + : >"$MAIN_RESULT" +fi + +# ============================================================================ +# Existing +# ============================================================================ + +echo +echo "Preparing Existing Driver" +echo "------------------------------------------------------------" + +export ODBC_TESTS_DSN="SampleDSN" +export ODBCINI="${EXISTING_ODBCINI}" +export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" + +run_benchmark \ + "Existing" \ + "${EXISTING_RESULT}" \ + "${EXISTING_ODBCINI}" \ + "OFF" + +# ============================================================================ +# Generate comparison +# ============================================================================ + +echo +echo "============================================================" +echo "Generating benchmark comparison" +echo "============================================================" + +PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" + +if [[ ! -f "${PARSER}" ]]; then + echo "ERROR: benchmark_results.py was not found:" + echo " ${PARSER}" + exit 1 +fi + +python3 "${PARSER}" \ + --existing "${EXISTING_RESULT}" \ + --current "${CURRENT_RESULT}" \ + --main "${MAIN_RESULT}" \ + --output "${SUMMARY_RESULT}" + +# ============================================================================ +# Upload results +# ============================================================================ + +RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/results" + +echo +echo "Uploading benchmark results" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${CURRENT_RESULT}" \ + "${MAIN_RESULT}" \ + "${EXISTING_RESULT}" \ + "${SUMMARY_RESULT}" \ + "${RESULTS_BUCKET}/" + +echo +echo "Results uploaded to:" +echo " ${RESULTS_BUCKET}/" + +echo +echo "============================================================" +echo "Benchmark completed successfully" +echo "============================================================" diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml new file mode 100644 index 0000000000..089990fb8a --- /dev/null +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -0,0 +1,29 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + branch: ^main$ +name: linux-bq-driver-benchmark-ci +substitutions: + _BUILD_NAME: linux-bq-driver-benchmark + _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' + _DISTRO: ubuntu-22.04-install + _TRIGGER_TYPE: ci +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +tags: +- ci diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc index d3604a8e15..264f90691a 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc @@ -24,6 +24,13 @@ #endif #include +#if (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW) && \ + defined(__GLIBCXX__) +template bool std::operator==(std::shared_ptr const&, + std::nullptr_t) noexcept; +#endif // (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW) && + // defined(__GLIBCXX__) + ////////////////////////////////////////////////////////////////// // This file has query execution related utilities which can have // statement or descriptor handles as arguments. We have some utils From 692b574b0f007fd72a4528549cbdc699eb4497c4 Mon Sep 17 00:00:00 2001 From: Kanchan Shukla Date: Wed, 26 Aug 2026 19:31:16 +0000 Subject: [PATCH 2/6] made changes after review comments --- ci/cloudbuild/builds/lib/benchmark_results.py | 67 ++------------ .../builds/linux-bq-driver-benchmark.sh | 89 ++++++++----------- .../linux-bq-driver-benchmark-ci.yaml | 2 +- 3 files changed, 47 insertions(+), 111 deletions(-) diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py index a26391a8f7..ab9caeaa56 100644 --- a/ci/cloudbuild/builds/lib/benchmark_results.py +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -19,10 +19,6 @@ TIME_RE = re.compile(r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)") -FAILED_RE = re.compile( - r"\[\s*FAILED\s*\]\s+(.+?)(?:\s+\([\d.]+\s*(?:ns|us|ms|s)\))?\s*$" -) - def clean_test_name(name): """Normalize GTest test names for comparison.""" @@ -103,14 +99,16 @@ def parse_gtest_output(path): A benchmark is run multiple times. Rules: - 1. Test passes in every iteration: - -> use median execution time. - 2. Test fails in ANY iteration: - -> return None, displayed as N/A. + 1. Only successful iterations are used. + + 2. Failed iterations are ignored. + + 3. The median execution time is calculated from all successful + iterations. - 3. Test is completely missing: - -> return None, displayed as N/A. + 4. If all iterations fail or a test is completely missing, + it is displayed as N/A. """ path = Path(path) @@ -118,11 +116,8 @@ def parse_gtest_output(path): raise FileNotFoundError(f"Benchmark output not found: {path}") samples = {} - failed_tests = set() - all_tests = set() for line in path.read_text(errors="replace").splitlines(): - # --------------------------------------------------------------- # Successful test # --------------------------------------------------------------- @@ -149,50 +144,17 @@ def parse_gtest_output(path): [], ).append(value_ms) - all_tests.add(test_name) - - continue - - # --------------------------------------------------------------- - # Failed test - # --------------------------------------------------------------- - - failed_match = FAILED_RE.search(line) - - if failed_match: - test_name = clean_test_name(failed_match.group(1)) - - # Ignore GTest summary/non-test lines. - if test_name is None: - continue - - failed_tests.add(test_name) - all_tests.add(test_name) - # ------------------------------------------------------------------- # Build final result. # ------------------------------------------------------------------- results = {} - for test_name in all_tests: - - # If a test failed even once, report N/A. - if test_name in failed_tests: - results[test_name] = None - continue - - values = samples.get( - test_name, - [], - ) - + for test_name, values in samples.items(): if not values: results[test_name] = None continue - # Same behavior as the GitHub Actions implementation: - # median of all successful iterations. results[test_name] = statistics.median(values) return results @@ -261,9 +223,7 @@ def main(): # ------------------------------------------------------------------- existing_data = parse_gtest_output(args.existing) - current_data = parse_gtest_output(args.current) - main_data = parse_gtest_output(args.main) if not current_data: @@ -299,15 +259,11 @@ def main(): for test_name in sorted_tests: existing_ms = existing_data.get(test_name) - current_ms = current_data.get(test_name) - main_ms = main_data.get(test_name) existing_raw = format_ms(existing_ms) - current_raw = format_ms(current_ms) - main_raw = format_ms(main_ms) # --------------------------------------------------------------- @@ -335,7 +291,6 @@ def main(): ) current_value = f"{current_raw}{current_pct}" - main_value = f"{main_raw}{main_pct}" rows.append( @@ -359,11 +314,8 @@ def main(): h4 = "Google Driver (Main)" w1 = max([len(h1)] + [len(row[0]) for row in rows]) if rows else len(h1) - w2 = max([len(h2)] + [len(row[1]) for row in rows]) if rows else len(h2) - w3 = max([len(h3)] + [len(row[2]) for row in rows]) if rows else len(h3) - w4 = max([len(h4)] + [len(row[3]) for row in rows]) if rows else len(h4) table = ( @@ -407,7 +359,6 @@ def main(): # ------------------------------------------------------------------- output_path = Path(args.output) - output_path.write_text(table) return 0 diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 66031ffced..6f32b0bd33 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -30,7 +30,6 @@ WORKSPACE_DIR=$(pwd) # ============================================================================ BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" - PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" BUILD_DIR="${WORKSPACE_DIR}/cmake-out" @@ -39,25 +38,12 @@ RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" rm -rf "${RESULTS_DIR}" mkdir -p "${RESULTS_DIR}" -# ============================================================================ -# Branch -# ============================================================================ - BRANCH_NAME="${BRANCH_NAME:-main}" SANITIZED_BRANCH="$( - echo "${BRANCH_NAME}" | - sed -E 's/[^a-zA-Z0-9._-]/_/g' + echo "${BRANCH_NAME}" | sed -E 's/[^a-zA-Z0-9._-]/_/g' )" -echo "============================================================" -echo "Linux ODBC Performance Benchmark" -echo "============================================================" -echo "Branch : ${BRANCH_NAME}" -echo "Iterations : ${BENCHMARK_ITERATIONS}" -echo "Workspace : ${WORKSPACE_DIR}" -echo - # ============================================================================ # Result files # ============================================================================ @@ -71,7 +57,7 @@ SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" # Driver locations # ============================================================================ -DRIVER_PATH="${WORKSPACE_DIR}/cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +DRIVER_PATH="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" @@ -86,20 +72,13 @@ MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" -cd "$WORKSPACE_DIR" - -# This is the name of DSN set in odbc.ini. -mapfile -t cmake_args < <(cmake::common_args) - -GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" -EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" - export ODBC_TESTS_DSN="SampleDSNGoogleDriver" - -export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX=${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_} -export ODBCINSTINI=/opt/odbc-driver/odbcinst.ini +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" +export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" export ODBCINI="${GOOGLE_ODBCINI}" +mapfile -t cmake_args < <(cmake::common_args) + # ============================================================================ # Validate ODBC configuration # ============================================================================ @@ -116,11 +95,22 @@ if [[ ! -f "${EXISTING_ODBCINI}" ]]; then exit 1 fi -# --------------------------------------------------------------------------- -# Download Google Current driver -# --------------------------------------------------------------------------- +# ============================================================================ +# Header +# ============================================================================ +echo "============================================================" +echo "Linux ODBC Performance Benchmark" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" echo + +# ============================================================================ +# Download drivers +# ============================================================================ + echo "Downloading Google Current driver" echo "------------------------------------------------------------" @@ -128,15 +118,11 @@ gcloud storage cp \ "${CURRENT_SO_GCS}" \ "${CURRENT_SO}" -# --------------------------------------------------------------------------- -# Download Google Main driver -# --------------------------------------------------------------------------- - echo echo "Downloading Google driver from main" echo "------------------------------------------------------------" -if gcloud storage cp "$MAIN_SO_GCS" "$MAIN_SO"; then +if gcloud storage cp "${MAIN_SO_GCS}" "${MAIN_SO}"; then echo "Main Google driver downloaded successfully." HAS_MAIN_DRIVER=true else @@ -166,23 +152,25 @@ run_benchmark() { echo "BQ_DRIVER_INTEGRATION_TESTS = ${bq_tests_flag}" echo "------------------------------------------------------------" - io::run cmake -S "${WORKSPACE_DIR}" \ + io::run cmake \ + -S "${WORKSPACE_DIR}" \ -B "${BUILD_DIR}" \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_PERFORMANCE_TEST_ONLY=ON \ -DBQ_DRIVER_INTEGRATION_TESTS="${bq_tests_flag}" - io::run cmake --build "${BUILD_DIR}" \ + io::run cmake \ + --build "${BUILD_DIR}" \ --target performance_test \ --parallel "$(nproc)" - PERFORMANCE_TEST="${BUILD_DIR}/integration_tests/performance_test" + local performance_test="${BUILD_DIR}/integration_tests/performance_test" - if [[ ! -x "${PERFORMANCE_TEST}" ]]; then - PERFORMANCE_TEST="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" + if [[ ! -x "${performance_test}" ]]; then + performance_test="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" fi - if [[ ! -x "${PERFORMANCE_TEST}" ]]; then + if [[ ! -x "${performance_test}" ]]; then echo "ERROR: performance_test was not found." find "${BUILD_DIR}" \ @@ -194,13 +182,13 @@ run_benchmark() { fi echo "performance_test:" - echo " ${PERFORMANCE_TEST}" + echo " ${performance_test}" : >"${output_file}" local failed_iterations=0 - for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do echo echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" @@ -214,10 +202,10 @@ run_benchmark() { ODBCINI="${dsn}" \ ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ - "${PERFORMANCE_TEST}" \ + "${performance_test}" \ >>"${output_file}" 2>&1 - run_exit=$? + local run_exit=$? set -e @@ -235,7 +223,7 @@ run_benchmark() { echo "============================================================" - failed_iterations=$((failed_iterations + 1)) + ((failed_iterations += 1)) fi done @@ -250,9 +238,6 @@ run_benchmark() { # ============================================================================ # Google Current -# -# The existing Google DSN points to DRIVER_PATH. -# Replace the driver binary before running the benchmark. # ============================================================================ echo @@ -276,7 +261,7 @@ echo echo "Preparing Google Main" echo "------------------------------------------------------------" -if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then +if [[ "${HAS_MAIN_DRIVER}" == "true" ]]; then cp "${MAIN_SO}" "${DRIVER_PATH}" ls -lh "${DRIVER_PATH}" @@ -287,11 +272,11 @@ if [[ "$HAS_MAIN_DRIVER" == "true" ]]; then "ON" else echo "Google Main benchmark skipped: main driver artifact unavailable." - : >"$MAIN_RESULT" + : >"${MAIN_RESULT}" fi # ============================================================================ -# Existing +# Existing Driver # ============================================================================ echo diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml index 089990fb8a..bb6d04d0d5 100644 --- a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -1,4 +1,4 @@ -# Copyright 2025 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 1df9573a3e99e9ac950c31408656c30eb5a05c00 Mon Sep 17 00:00:00 2001 From: Sachin Purohit Date: Fri, 30 Oct 2026 22:59:20 +0000 Subject: [PATCH 3/6] chore(ci): align Linux benchmark bucket paths and share benchmark_results.py with Windows --- .github/workflows/test-runner.yml | 164 +----------------- .../integration-production-bq-driver-dm.sh | 18 +- ci/cloudbuild/builds/lib/benchmark_results.py | 72 +++++++- .../builds/linux-bq-driver-benchmark.sh | 12 +- 4 files changed, 89 insertions(+), 177 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index b6ed114604..b2b6951e79 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -166,164 +166,12 @@ jobs: env: BRANCH_NAME: ${{ github.ref_name }} run: | - python3 <<'EOF' - import os - import re - - branch_name = os.environ.get('BRANCH_NAME', 'Current') - - def clean_test_name(name): - # GTest names follow [Instantiation/]TestSuite.TestCase[/Param] - # Extract everything after '.' to generically remove TestSuite/Instantiation prefix - if '.' in name: - name = name.split('.', 1)[1] - # Strip legacy parameter suffix if comparing against older baselines - name = re.sub(r'/(?:With|Without)HTAPI$', '', name) - return name - - def parse_gtest_output(filepath): - # The suite is run several times across iterations, so each test - # appears once per repetition. Take the median of its timings. - samples = {} - if not os.path.exists(filepath): - return {} - - ok_pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)') - metric_pattern = re.compile(r'\[\s+METRIC\s+\]\s+(\S+)\s+\(([^)]+)\):\s*([0-9.]+\s*ms)') - try: - with open(filepath, 'r') as f: - for line in f: - m_match = metric_pattern.search(line) - if m_match: - test_name = m_match.group(1) - metric_name = m_match.group(2) - entry_name = f'{test_name} - {metric_name}' - ms = parse_time_to_ms(m_match.group(3)) - if ms is not None: - samples.setdefault(entry_name, []).append(ms) - continue - - ok_match = ok_pattern.search(line) - if ok_match: - raw_name = ok_match.group(1) - test_name = clean_test_name(raw_name) - if 'DataFetchPerformanceParamTest' in raw_name or test_name.startswith('Benchmark/'): - base_name = test_name.split('Benchmark/', 1)[-1] - if not re.search(r'_(?:10k|100k|1M)$', base_name): - base_name = f'{base_name}_1M' - entry_name = f'{base_name} - Total execution time' - else: - entry_name = test_name - ms = parse_time_to_ms(ok_match.group(2)) - if ms is not None: - samples.setdefault(entry_name, []).append(ms) - except Exception as e: - print(f'Error reading {filepath}: {e}') - - results = {} - for entry_name, values in samples.items(): - values.sort() - n = len(values) - median = (values[n // 2] if n % 2 == 1 - else (values[n // 2 - 1] + values[n // 2]) / 2.0) - results[entry_name] = f'{median}ms' - if n > 1: - print(f'{entry_name}: median={median:.0f}ms of {n} runs ' - f'(min={values[0]:.0f}ms max={values[-1]:.0f}ms)') - return results - - def parse_time_to_ms(time_str): - if not time_str or time_str == 'N/A': - return None - time_str = time_str.strip() - match = re.match(r'^([\d.]+)\s*(\w+)$', time_str) - if not match: - return None - val = float(match.group(1)) - unit = match.group(2).lower() - if unit == 'ms': - return val - elif unit == 's': - return val * 1000 - elif unit == 'us': - return val / 1000 - elif unit == 'ns': - return val / 1000000 - return val - - def get_percentage_str(val_ms, ref_ms): - if val_ms is None or ref_ms is None or ref_ms == 0: - return ' (N/A)' - pct = round(((val_ms - ref_ms) / ref_ms) * 100) - if pct > 0: - return f' (+{pct}%)' - elif pct < 0: - return f' ({pct}%)' - else: - return ' (0%)' - - def sort_key(test_name): - match = re.match(r'^(.+?)_(10k|100k|1M) - (.+)$', test_name) - if match: - table_name, limit_label, metric_name = match.groups() - limit_order = {'10k': 1, '100k': 2, '1M': 3}.get(limit_label, 4) - metric_order = { - 'Time to first byte': 1, - 'Iteration time': 2, - 'Total execution time': 3, - }.get(metric_name, 4) - return (0, table_name, limit_order, metric_order) - return (1, test_name, 0, 0) - - existing_data = parse_gtest_output('./benchmark_results/current_core.txt') - current_bq_data = parse_gtest_output('./benchmark_results/current_bq.txt') - main_bq_data = parse_gtest_output('./benchmark_results/main_bq.txt') - - all_tests = set(existing_data.keys()).union(set(current_bq_data.keys())).union(set(main_bq_data.keys())) - sorted_tests = sorted(list(all_tests), key=sort_key) - - rows = [] - for test in sorted_tests: - existing_raw = existing_data.get(test, 'N/A') - cur_bq_raw = current_bq_data.get(test, 'N/A') - main_bq_raw = main_bq_data.get(test, 'N/A') - - existing_ms = parse_time_to_ms(existing_raw) - cur_bq_ms = parse_time_to_ms(cur_bq_raw) - main_bq_ms = parse_time_to_ms(main_bq_raw) - - cur_bq_pct = get_percentage_str(cur_bq_ms, existing_ms) if cur_bq_raw != 'N/A' else '' - main_bq_pct = get_percentage_str(main_bq_ms, cur_bq_ms) if (main_bq_raw != 'N/A' and cur_bq_raw != 'N/A') else ' (N/A)' if main_bq_raw == 'N/A' else '' - - cur_bq_val = f'{cur_bq_raw}{cur_bq_pct}' - main_bq_val = f'{main_bq_raw}{main_bq_pct}' - - rows.append((test, existing_raw, cur_bq_val, main_bq_val)) - - # Define headers - h1 = 'Test Case' - h2 = 'Existing Driver (Current)' - h3 = f'Google Driver ({branch_name})' - h4 = 'Google Driver (Main)' - - # Calculate dynamic widths for formatting - w1 = max([len(h1)] + [len(r[0]) for r in rows]) if rows else len(h1) - w2 = max([len(h2)] + [len(r[1]) for r in rows]) if rows else len(h2) - w3 = max([len(h3)] + [len(r[2]) for r in rows]) if rows else len(h3) - w4 = max([len(h4)] + [len(r[3]) for r in rows]) if rows else len(h4) - - # Construct the Markdown table - table = f"*Percentages in **{h3}** show change relative to **{h2}**. Percentages in **{h4}** show change relative to **{h3}**. Negative values indicate improvement (faster test execution), positive values indicate degradation (slower).*\n\n" - table += f'| {h1.ljust(w1)} | {h2.ljust(w2)} | {h3.ljust(w3)} | {h4.ljust(w4)} |\n' - table += '|-' + ('-' * w1) + '-|-' + ('-' * w2) + '-|-' + ('-' * w3) + '-|-' + ('-' * w4) + '-|\n' - - for r in rows: - table += f'| {r[0].ljust(w1)} | {r[1].ljust(w2)} | {r[2].ljust(w3)} | {r[3].ljust(w4)} |\n' - - # Write table to file - with open('benchmark_summary_table.txt', 'w') as f: - f.write(table) - EOF + python3 ci/cloudbuild/builds/lib/benchmark_results.py \ + --existing ./benchmark_results/current_core.txt \ + --current ./benchmark_results/current_bq.txt \ + --main ./benchmark_results/main_bq.txt \ + --branch-name "${BRANCH_NAME}" \ + --output benchmark_summary_table.txt - name: Upload Table to GCS env: diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index bd702f1094..d295365023 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -83,38 +83,38 @@ io::run cmake -B "$BUILD_DIR" \ io::run cmake --build cmake-out # --------------------------------------------------------------------------- -# Publish Google driver .so for performance benchmarks +# Publish ODBC driver .so for performance benchmarks # --------------------------------------------------------------------------- if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" if [[ ! -f "$DRIVER_SO" ]]; then - echo "ERROR: Google ODBC driver .so was not found:" + echo "ERROR: ODBC driver .so was not found:" echo " $DRIVER_SO" exit 1 fi - echo "Google driver found:" + echo "ODBC driver .so found:" ls -lh "$DRIVER_SO" # Sanitize branch name for use in GCS path. SANITIZED_BRANCH=$( echo "${BRANCH_NAME}" | - sed -E 's/[^a-zA-Z0-9._-]/_/g' + sed -E 's/[^a-zA-Z0-9-]/_/g' ) - PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + INTEGRATION_BUILDS_BUCKET="gs://odbc-integration-builds" - echo "Uploading Google driver artifact..." + echo "Uploading driver artifact..." echo "Branch: ${SANITIZED_BRANCH}" gcloud storage cp \ "$DRIVER_SO" \ - "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + "${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" - echo "Google driver benchmark artifact uploaded:" - echo "${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" + echo "Driver benchmark artifact uploaded:" + echo "${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" fi # Copy the roots.pem file to the .so directory to run test cases. diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py index ab9caeaa56..49673c4d0e 100644 --- a/ci/cloudbuild/builds/lib/benchmark_results.py +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -18,6 +18,9 @@ from pathlib import Path TIME_RE = re.compile(r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)") +METRIC_RE = re.compile( + r"\[\s*METRIC\s*\]\s+(\S+)\s+\(([^)]+)\):\s*([\d.]+)\s*(ns|us|ms|s)" +) def clean_test_name(name): @@ -113,11 +116,37 @@ def parse_gtest_output(path): path = Path(path) if not path.exists(): - raise FileNotFoundError(f"Benchmark output not found: {path}") + print(f"WARNING: Benchmark output file not found: {path}") + return {} samples = {} for line in path.read_text(errors="replace").splitlines(): + # --------------------------------------------------------------- + # Custom metric line (e.g., Time to first byte, Iteration time) + # --------------------------------------------------------------- + + metric_match = METRIC_RE.search(line) + + if metric_match: + test_name = metric_match.group(1) + metric_name = metric_match.group(2) + entry_name = f"{test_name} - {metric_name}" + + value = float(metric_match.group(3)) + unit = metric_match.group(4).lower() + + value_ms = parse_time_to_ms( + value, + unit, + ) + + samples.setdefault( + entry_name, + [], + ).append(value_ms) + continue + # --------------------------------------------------------------- # Successful test # --------------------------------------------------------------- @@ -125,12 +154,23 @@ def parse_gtest_output(path): time_match = TIME_RE.search(line) if time_match: - test_name = clean_test_name(time_match.group(1)) + raw_name = time_match.group(1) + test_name = clean_test_name(raw_name) # Ignore GTest summary/non-test lines. if test_name is None: continue + if "DataFetchPerformanceParamTest" in raw_name or test_name.startswith( + "Benchmark/" + ): + base_name = test_name.split("Benchmark/", 1)[-1] + if not re.search(r"_(?:10k|100k|1M)$", base_name): + base_name = f"{base_name}_1M" + entry_name = f"{base_name} - Total execution time" + else: + entry_name = test_name + value = float(time_match.group(2)) unit = time_match.group(3).lower() @@ -140,7 +180,7 @@ def parse_gtest_output(path): ) samples.setdefault( - test_name, + entry_name, [], ).append(value_ms) @@ -155,11 +195,33 @@ def parse_gtest_output(path): results[test_name] = None continue - results[test_name] = statistics.median(values) + values.sort() + median = statistics.median(values) + results[test_name] = median + if len(values) > 1: + print( + f"{test_name}: median={median:.0f}ms of {len(values)} runs " + f"(min={values[0]:.0f}ms max={values[-1]:.0f}ms)" + ) return results +def sort_key(test_name): + """Sort data fetch benchmarks by table, row limit (10k, 100k, 1M), and metric order.""" + match = re.match(r"^(.+?)_(10k|100k|1M) - (.+)$", test_name) + if match: + table_name, limit_label, metric_name = match.groups() + limit_order = {"10k": 1, "100k": 2, "1M": 3}.get(limit_label, 4) + metric_order = { + "Time to first byte": 1, + "Iteration time": 2, + "Total execution time": 3, + }.get(metric_name, 4) + return (0, table_name, limit_order, metric_order) + return (1, test_name, 0, 0) + + def get_percentage_str(value_ms, reference_ms): """ Return percentage change. @@ -252,7 +314,7 @@ def main(): set(existing_data.keys()) | set(current_data.keys()) | set(main_data.keys()) ) - sorted_tests = sorted(all_tests) + sorted_tests = sorted(all_tests, key=sort_key) rows = [] diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 6f32b0bd33..dc73741aac 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -30,7 +30,8 @@ WORKSPACE_DIR=$(pwd) # ============================================================================ BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" -PERF_DRIVER_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" +INTEGRATION_BUILDS_BUCKET="gs://odbc-integration-builds" +PERF_RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" BUILD_DIR="${WORKSPACE_DIR}/cmake-out" RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" @@ -41,7 +42,7 @@ mkdir -p "${RESULTS_DIR}" BRANCH_NAME="${BRANCH_NAME:-main}" SANITIZED_BRANCH="$( - echo "${BRANCH_NAME}" | sed -E 's/[^a-zA-Z0-9._-]/_/g' + echo "${BRANCH_NAME}" | sed -E 's/[^a-zA-Z0-9-]/_/g' )" # ============================================================================ @@ -62,8 +63,8 @@ DRIVER_PATH="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" -CURRENT_SO_GCS="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/libgoogle_cloud_odbc_bq_driver.so" -MAIN_SO_GCS="${PERF_DRIVER_BUCKET}/main/linux/libgoogle_cloud_odbc_bq_driver.so" +CURRENT_SO_GCS="${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" +MAIN_SO_GCS="${INTEGRATION_BUILDS_BUCKET}/main/libgoogle_cloud_odbc_bq_driver.so" # ============================================================================ # ODBC configuration @@ -314,13 +315,14 @@ python3 "${PARSER}" \ --existing "${EXISTING_RESULT}" \ --current "${CURRENT_RESULT}" \ --main "${MAIN_RESULT}" \ + --branch-name "${BRANCH_NAME}" \ --output "${SUMMARY_RESULT}" # ============================================================================ # Upload results # ============================================================================ -RESULTS_BUCKET="${PERF_DRIVER_BUCKET}/${SANITIZED_BRANCH}/linux/results" +RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/${SANITIZED_BRANCH}/linux_results" echo echo "Uploading benchmark results" From 076247562f9977819dcd8985c6e063997ab03fd2 Mon Sep 17 00:00:00 2001 From: Sachin Purohit Date: Sat, 31 Oct 2026 00:04:47 +0000 Subject: [PATCH 4/6] chore(ci): trigger linux benchmark after driver upload on main and add commit SHA to summary --- .github/workflows/test-runner.yml | 2 ++ .../integration-production-bq-driver-dm.sh | 8 ++++++++ ci/cloudbuild/builds/lib/benchmark_results.py | 20 +++++++++++++++++++ .../builds/linux-bq-driver-benchmark.sh | 1 + .../linux-bq-driver-benchmark-ci.yaml | 2 +- 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index b2b6951e79..20deea4a4d 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -165,12 +165,14 @@ jobs: - name: Parse Results and Generate Table env: BRANCH_NAME: ${{ github.ref_name }} + COMMIT_SHA: ${{ needs.pre-flight.outputs.checkout-sha }} run: | python3 ci/cloudbuild/builds/lib/benchmark_results.py \ --existing ./benchmark_results/current_core.txt \ --current ./benchmark_results/current_bq.txt \ --main ./benchmark_results/main_bq.txt \ --branch-name "${BRANCH_NAME}" \ + --commit-sha "${COMMIT_SHA}" \ --output benchmark_summary_table.txt - name: Upload Table to GCS diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index d295365023..605d823eee 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -115,6 +115,14 @@ if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then echo "Driver benchmark artifact uploaded:" echo "${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" + + if [[ "${BRANCH_NAME}" == "main" ]]; then + echo "Triggering Linux benchmark pipeline for main..." + gcloud beta builds triggers run linux-bq-driver-benchmark-ci \ + --branch=main \ + --project="${PROJECT_ID:-bigquery-devtools-drivers}" \ + --region=us-east1 + fi fi # Copy the roots.pem file to the .so directory to run test cases. diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py index 49673c4d0e..e8a45400ca 100644 --- a/ci/cloudbuild/builds/lib/benchmark_results.py +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -13,8 +13,10 @@ # limitations under the License. import argparse +import os import re import statistics +import subprocess from pathlib import Path TIME_RE = re.compile(r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)") @@ -278,6 +280,12 @@ def main(): help="Branch name used in the Google Driver column header", ) + parser.add_argument( + "--commit-sha", + default="", + help="Commit SHA that the benchmark was run on", + ) + args = parser.parse_args() # ------------------------------------------------------------------- @@ -380,7 +388,19 @@ def main(): w3 = max([len(h3)] + [len(row[2]) for row in rows]) if rows else len(h3) w4 = max([len(h4)] + [len(row[3]) for row in rows]) if rows else len(h4) + commit_sha = ( + args.commit_sha or os.environ.get("COMMIT_SHA") or os.environ.get("GITHUB_SHA") + ) + if not commit_sha: + try: + commit_sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip() + except Exception: + commit_sha = "unknown" + table = ( + f"Commit ID: `{commit_sha}`\n\n" f"*Percentages in **{h3}** show change relative to " f"**{h2}**. Percentages in **{h4}** show change relative " f"to **{h3}**. Negative values indicate improvement " diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index dc73741aac..931b30c703 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -316,6 +316,7 @@ python3 "${PARSER}" \ --current "${CURRENT_RESULT}" \ --main "${MAIN_RESULT}" \ --branch-name "${BRANCH_NAME}" \ + --commit-sha "${COMMIT_SHA:-$(git rev-parse HEAD)}" \ --output "${SUMMARY_RESULT}" # ============================================================================ diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml index bb6d04d0d5..8f6aad0c1e 100644 --- a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -17,7 +17,7 @@ github: name: cpp-bigquery-odbc owner: googleapis push: - branch: ^main$ + tag: ^benchmark- name: linux-bq-driver-benchmark-ci substitutions: _BUILD_NAME: linux-bq-driver-benchmark From f8bcaaf0e714015730002ec2e68457b27763e494 Mon Sep 17 00:00:00 2001 From: Sachin Purohit Date: Sat, 31 Oct 2026 00:31:16 +0000 Subject: [PATCH 5/6] chore(ci): align Linux benchmark flow and result filenames with Windows --- .../builds/linux-bq-driver-benchmark.sh | 125 +++++++++--------- 1 file changed, 61 insertions(+), 64 deletions(-) diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 931b30c703..35c1d9328c 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -49,10 +49,13 @@ SANITIZED_BRANCH="$( # Result files # ============================================================================ -CURRENT_RESULT="${RESULTS_DIR}/current.txt" -MAIN_RESULT="${RESULTS_DIR}/main.txt" -EXISTING_RESULT="${RESULTS_DIR}/existing.txt" -SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" +RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/${SANITIZED_BRANCH}/linux_results" +MAIN_RESULTS_GCS="${PERF_RESULTS_BUCKET}/main/linux_results/performance_benchmark_results_BqDriver.txt" + +CURRENT_RESULT="${RESULTS_DIR}/performance_benchmark_results_BqDriver.txt" +EXISTING_RESULT="${RESULTS_DIR}/performance_benchmark_results_Core.txt" +MAIN_RESULT="${RESULTS_DIR}/main_bq.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_table.txt" # ============================================================================ # Driver locations @@ -61,10 +64,7 @@ SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_linux.txt" DRIVER_PATH="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" -MAIN_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_main.so" - CURRENT_SO_GCS="${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" -MAIN_SO_GCS="${INTEGRATION_BUILDS_BUCKET}/main/libgoogle_cloud_odbc_bq_driver.so" # ============================================================================ # ODBC configuration @@ -115,21 +115,13 @@ echo echo "Downloading Google Current driver" echo "------------------------------------------------------------" -gcloud storage cp \ - "${CURRENT_SO_GCS}" \ - "${CURRENT_SO}" - -echo -echo "Downloading Google driver from main" -echo "------------------------------------------------------------" - -if gcloud storage cp "${MAIN_SO_GCS}" "${MAIN_SO}"; then - echo "Main Google driver downloaded successfully." - HAS_MAIN_DRIVER=true +if gcloud storage cp "${CURRENT_SO_GCS}" "${CURRENT_SO}"; then + echo "Current Google driver downloaded successfully." + HAS_CURRENT_DRIVER=true else - echo "WARNING: Main Google driver was not found." - echo "WARNING: Google Main benchmark will be skipped." - HAS_MAIN_DRIVER=false + echo "WARNING: Current Google driver was not found at ${CURRENT_SO_GCS}." + echo "WARNING: Google Current benchmark will be skipped." + HAS_CURRENT_DRIVER=false fi mkdir -p "$(dirname "${DRIVER_PATH}")" @@ -148,22 +140,30 @@ run_benchmark() { local dsn="$3" local bq_tests_flag="$4" + : >"${output_file}" + echo echo "Reconfiguring performance_test for ${name}" echo "BQ_DRIVER_INTEGRATION_TESTS = ${bq_tests_flag}" echo "------------------------------------------------------------" - io::run cmake \ + if ! io::run cmake \ -S "${WORKSPACE_DIR}" \ -B "${BUILD_DIR}" \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_PERFORMANCE_TEST_ONLY=ON \ - -DBQ_DRIVER_INTEGRATION_TESTS="${bq_tests_flag}" + -DBQ_DRIVER_INTEGRATION_TESTS="${bq_tests_flag}"; then + echo "WARNING: CMake configuration failed for ${name}; skipping benchmark." + return 0 + fi - io::run cmake \ + if ! io::run cmake \ --build "${BUILD_DIR}" \ --target performance_test \ - --parallel "$(nproc)" + --parallel "$(nproc)"; then + echo "WARNING: Building performance_test failed for ${name}; skipping benchmark." + return 0 + fi local performance_test="${BUILD_DIR}/integration_tests/performance_test" @@ -172,21 +172,19 @@ run_benchmark() { fi if [[ ! -x "${performance_test}" ]]; then - echo "ERROR: performance_test was not found." + echo "WARNING: performance_test was not found for ${name}; skipping benchmark." find "${BUILD_DIR}" \ -type f \ -name "performance_test" \ -print 2>/dev/null || true - return 1 + return 0 fi echo "performance_test:" echo " ${performance_test}" - : >"${output_file}" - local failed_iterations=0 for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do @@ -238,46 +236,36 @@ run_benchmark() { } # ============================================================================ -# Google Current +# Google Current (BqDriver) # ============================================================================ echo echo "Preparing Google Current" echo "------------------------------------------------------------" -cp "${CURRENT_SO}" "${DRIVER_PATH}" -ls -lh "${DRIVER_PATH}" - -run_benchmark \ - "Google Current" \ - "${CURRENT_RESULT}" \ - "${GOOGLE_ODBCINI}" \ - "ON" - -# ============================================================================ -# Google Main -# ============================================================================ - -echo -echo "Preparing Google Main" -echo "------------------------------------------------------------" - -if [[ "${HAS_MAIN_DRIVER}" == "true" ]]; then - cp "${MAIN_SO}" "${DRIVER_PATH}" +if [[ "${HAS_CURRENT_DRIVER}" == "true" ]]; then + cp "${CURRENT_SO}" "${DRIVER_PATH}" ls -lh "${DRIVER_PATH}" run_benchmark \ - "Google Main" \ - "${MAIN_RESULT}" \ + "Google Current" \ + "${CURRENT_RESULT}" \ "${GOOGLE_ODBCINI}" \ "ON" + + echo + echo "Uploading Google Current benchmark results" + echo "------------------------------------------------------------" + + gcloud storage cp "${CURRENT_RESULT}" "${RESULTS_BUCKET}/" || + echo "WARNING: Failed to upload ${CURRENT_RESULT} to ${RESULTS_BUCKET}/" else - echo "Google Main benchmark skipped: main driver artifact unavailable." - : >"${MAIN_RESULT}" + echo "Google Current benchmark skipped: current driver artifact unavailable." + : >"${CURRENT_RESULT}" fi # ============================================================================ -# Existing Driver +# Existing Driver (Core / Simba) # ============================================================================ echo @@ -294,10 +282,24 @@ run_benchmark \ "${EXISTING_ODBCINI}" \ "OFF" +echo +echo "Uploading Existing Driver benchmark results" +echo "------------------------------------------------------------" + +gcloud storage cp "${EXISTING_RESULT}" "${RESULTS_BUCKET}/" || + echo "WARNING: Failed to upload ${EXISTING_RESULT} to ${RESULTS_BUCKET}/" + # ============================================================================ -# Generate comparison +# Download Main Results & Generate Comparison # ============================================================================ +echo +echo "Downloading Google Main benchmark results from GCS" +echo "------------------------------------------------------------" + +gcloud storage cp "${MAIN_RESULTS_GCS}" "${MAIN_RESULT}" || + echo "WARNING: Main benchmark results not found at ${MAIN_RESULTS_GCS}; Google Main will be shown as N/A." + echo echo "============================================================" echo "Generating benchmark comparison" @@ -320,25 +322,20 @@ python3 "${PARSER}" \ --output "${SUMMARY_RESULT}" # ============================================================================ -# Upload results +# Upload Summary Table # ============================================================================ -RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/${SANITIZED_BRANCH}/linux_results" - echo -echo "Uploading benchmark results" +echo "Uploading benchmark summary table" echo "------------------------------------------------------------" gcloud storage cp \ - "${CURRENT_RESULT}" \ - "${MAIN_RESULT}" \ - "${EXISTING_RESULT}" \ "${SUMMARY_RESULT}" \ "${RESULTS_BUCKET}/" echo -echo "Results uploaded to:" -echo " ${RESULTS_BUCKET}/" +echo "Summary table uploaded to:" +echo " ${RESULTS_BUCKET}/benchmark_summary_table.txt" echo echo "============================================================" From 785cec5dbc296c93b209c44134e9683ff0f8eba3 Mon Sep 17 00:00:00 2001 From: Sachin Purohit Date: Sat, 31 Oct 2026 01:04:10 +0000 Subject: [PATCH 6/6] chore(ci): split Linux benchmark into separate Google Driver, Existing Driver, and Results triggers --- .../builds/linux-benchmark-results.sh | 96 ++++++++ .../builds/linux-bq-driver-benchmark.sh | 207 ++++++------------ .../builds/linux-existing-driver-benchmark.sh | 20 ++ .../triggers/linux-benchmark-results-ci.yaml | 29 +++ .../linux-bq-driver-benchmark-ci.yaml | 2 +- .../linux-existing-driver-benchmark-ci.yaml | 29 +++ 6 files changed, 240 insertions(+), 143 deletions(-) create mode 100755 ci/cloudbuild/builds/linux-benchmark-results.sh create mode 100755 ci/cloudbuild/builds/linux-existing-driver-benchmark.sh create mode 100644 ci/cloudbuild/triggers/linux-benchmark-results-ci.yaml create mode 100644 ci/cloudbuild/triggers/linux-existing-driver-benchmark-ci.yaml diff --git a/ci/cloudbuild/builds/linux-benchmark-results.sh b/ci/cloudbuild/builds/linux-benchmark-results.sh new file mode 100755 index 0000000000..e81394c3a7 --- /dev/null +++ b/ci/cloudbuild/builds/linux-benchmark-results.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +source "$(dirname "$0")/../../lib/init.sh" +source module ci/lib/io.sh + +WORKSPACE_DIR=$(pwd) + +PERF_RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" + +rm -rf "${RESULTS_DIR}" +mkdir -p "${RESULTS_DIR}" + +BRANCH_NAME="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH="$( + echo "${BRANCH_NAME}" | sed -E 's/[^a-zA-Z0-9-]/_/g' +)" + +RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/${SANITIZED_BRANCH}/linux_results" +MAIN_RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/main/linux_results" + +CURRENT_RESULT="${RESULTS_DIR}/current_bq.txt" +EXISTING_RESULT="${RESULTS_DIR}/current_core.txt" +MAIN_RESULT="${RESULTS_DIR}/main_bq.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_table.txt" + +echo "============================================================" +echo "Linux ODBC Benchmark Results Table Generator" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Results GCS : ${RESULTS_BUCKET}" +echo + +echo "Downloading benchmark results from GCS" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" \ + "${CURRENT_RESULT}" || true + +gcloud storage cp \ + "${RESULTS_BUCKET}/performance_benchmark_results_Core.txt" \ + "${EXISTING_RESULT}" || true + +gcloud storage cp \ + "${MAIN_RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" \ + "${MAIN_RESULT}" || true + +echo +echo "Generating benchmark comparison table" +echo "------------------------------------------------------------" + +PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" + +if [[ ! -f "${PARSER}" ]]; then + echo "ERROR: benchmark_results.py was not found:" + echo " ${PARSER}" + exit 1 +fi + +python3 "${PARSER}" \ + --existing "${EXISTING_RESULT}" \ + --current "${CURRENT_RESULT}" \ + --main "${MAIN_RESULT}" \ + --branch-name "${BRANCH_NAME}" \ + --commit-sha "${COMMIT_SHA:-$(git rev-parse HEAD)}" \ + --output "${SUMMARY_RESULT}" + +echo +echo "Uploading benchmark summary table" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${SUMMARY_RESULT}" \ + "${RESULTS_BUCKET}/" + +echo +echo "Uploaded benchmark summary table to:" +echo " ${RESULTS_BUCKET}/benchmark_summary_table.txt" diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh index 35c1d9328c..7f858faa5b 100755 --- a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -29,6 +29,7 @@ WORKSPACE_DIR=$(pwd) # Configuration # ============================================================================ +BUILD_SHARD="${BUILD_SHARD:-BqDriver}" BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" INTEGRATION_BUILDS_BUCKET="gs://odbc-integration-builds" PERF_RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" @@ -45,91 +46,24 @@ SANITIZED_BRANCH="$( echo "${BRANCH_NAME}" | sed -E 's/[^a-zA-Z0-9-]/_/g' )" -# ============================================================================ -# Result files -# ============================================================================ - RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/${SANITIZED_BRANCH}/linux_results" -MAIN_RESULTS_GCS="${PERF_RESULTS_BUCKET}/main/linux_results/performance_benchmark_results_BqDriver.txt" - -CURRENT_RESULT="${RESULTS_DIR}/performance_benchmark_results_BqDriver.txt" -EXISTING_RESULT="${RESULTS_DIR}/performance_benchmark_results_Core.txt" -MAIN_RESULT="${RESULTS_DIR}/main_bq.txt" -SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_table.txt" - -# ============================================================================ -# Driver locations -# ============================================================================ - -DRIVER_PATH="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" +RESULT_FILE="${RESULTS_DIR}/performance_benchmark_results_${BUILD_SHARD}.txt" -CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" -CURRENT_SO_GCS="${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" - -# ============================================================================ -# ODBC configuration -# ============================================================================ - -GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" -EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" - -export ODBC_TESTS_DSN="SampleDSNGoogleDriver" export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" -export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" -export ODBCINI="${GOOGLE_ODBCINI}" - -mapfile -t cmake_args < <(cmake::common_args) - -# ============================================================================ -# Validate ODBC configuration -# ============================================================================ - -if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then - echo "ERROR: Google ODBC configuration not found:" - echo " ${GOOGLE_ODBCINI}" - exit 1 -fi - -if [[ ! -f "${EXISTING_ODBCINI}" ]]; then - echo "ERROR: Existing ODBC configuration not found:" - echo " ${EXISTING_ODBCINI}" - exit 1 -fi # ============================================================================ # Header # ============================================================================ echo "============================================================" -echo "Linux ODBC Performance Benchmark" +echo "Linux ODBC Performance Benchmark (${BUILD_SHARD})" echo "============================================================" echo "Branch : ${BRANCH_NAME}" +echo "Shard : ${BUILD_SHARD}" echo "Iterations : ${BENCHMARK_ITERATIONS}" echo "Workspace : ${WORKSPACE_DIR}" echo -# ============================================================================ -# Download drivers -# ============================================================================ - -echo "Downloading Google Current driver" -echo "------------------------------------------------------------" - -if gcloud storage cp "${CURRENT_SO_GCS}" "${CURRENT_SO}"; then - echo "Current Google driver downloaded successfully." - HAS_CURRENT_DRIVER=true -else - echo "WARNING: Current Google driver was not found at ${CURRENT_SO_GCS}." - echo "WARNING: Google Current benchmark will be skipped." - HAS_CURRENT_DRIVER=false -fi - -mkdir -p "$(dirname "${DRIVER_PATH}")" - -cp \ - /opt/odbc-driver/roots.pem \ - "${DRIVER_PATH%/*}/roots.pem" - # ============================================================================ # Run benchmark # ============================================================================ @@ -236,108 +170,97 @@ run_benchmark() { } # ============================================================================ -# Google Current (BqDriver) +# Execute Shard # ============================================================================ -echo -echo "Preparing Google Current" -echo "------------------------------------------------------------" +if [[ "${BUILD_SHARD}" == "Core" ]]; then + EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" -if [[ "${HAS_CURRENT_DRIVER}" == "true" ]]; then - cp "${CURRENT_SO}" "${DRIVER_PATH}" - ls -lh "${DRIVER_PATH}" + if [[ ! -f "${EXISTING_ODBCINI}" ]]; then + echo "ERROR: Existing ODBC configuration not found:" + echo " ${EXISTING_ODBCINI}" + exit 1 + fi + + export ODBC_TESTS_DSN="SampleDSN" + export ODBCINI="${EXISTING_ODBCINI}" + export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" run_benchmark \ - "Google Current" \ - "${CURRENT_RESULT}" \ - "${GOOGLE_ODBCINI}" \ - "ON" + "Existing" \ + "${RESULT_FILE}" \ + "${EXISTING_ODBCINI}" \ + "OFF" +else + GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" - echo - echo "Uploading Google Current benchmark results" - echo "------------------------------------------------------------" + if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then + echo "ERROR: Google ODBC configuration not found:" + echo " ${GOOGLE_ODBCINI}" + exit 1 + fi - gcloud storage cp "${CURRENT_RESULT}" "${RESULTS_BUCKET}/" || - echo "WARNING: Failed to upload ${CURRENT_RESULT} to ${RESULTS_BUCKET}/" -else - echo "Google Current benchmark skipped: current driver artifact unavailable." - : >"${CURRENT_RESULT}" -fi + export ODBC_TESTS_DSN="SampleDSNGoogleDriver" + export ODBCINI="${GOOGLE_ODBCINI}" + export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" -# ============================================================================ -# Existing Driver (Core / Simba) -# ============================================================================ + DRIVER_PATH="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" + CURRENT_SO_GCS="${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" -echo -echo "Preparing Existing Driver" -echo "------------------------------------------------------------" + echo "Downloading Google Current driver" + echo "------------------------------------------------------------" -export ODBC_TESTS_DSN="SampleDSN" -export ODBCINI="${EXISTING_ODBCINI}" -export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" + gcloud storage cp \ + "${CURRENT_SO_GCS}" \ + "${CURRENT_SO}" -run_benchmark \ - "Existing" \ - "${EXISTING_RESULT}" \ - "${EXISTING_ODBCINI}" \ - "OFF" + mkdir -p "$(dirname "${DRIVER_PATH}")" -echo -echo "Uploading Existing Driver benchmark results" -echo "------------------------------------------------------------" + cp \ + /opt/odbc-driver/roots.pem \ + "${DRIVER_PATH%/*}/roots.pem" + + cp "${CURRENT_SO}" "${DRIVER_PATH}" + ls -lh "${DRIVER_PATH}" -gcloud storage cp "${EXISTING_RESULT}" "${RESULTS_BUCKET}/" || - echo "WARNING: Failed to upload ${EXISTING_RESULT} to ${RESULTS_BUCKET}/" + run_benchmark \ + "Google Current" \ + "${RESULT_FILE}" \ + "${GOOGLE_ODBCINI}" \ + "ON" +fi # ============================================================================ -# Download Main Results & Generate Comparison +# Upload Result # ============================================================================ echo -echo "Downloading Google Main benchmark results from GCS" +echo "Uploading benchmark results to GCS" echo "------------------------------------------------------------" -gcloud storage cp "${MAIN_RESULTS_GCS}" "${MAIN_RESULT}" || - echo "WARNING: Main benchmark results not found at ${MAIN_RESULTS_GCS}; Google Main will be shown as N/A." +gcloud storage cp \ + "${RESULT_FILE}" \ + "${RESULTS_BUCKET}/" echo -echo "============================================================" -echo "Generating benchmark comparison" -echo "============================================================" - -PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" - -if [[ ! -f "${PARSER}" ]]; then - echo "ERROR: benchmark_results.py was not found:" - echo " ${PARSER}" - exit 1 -fi - -python3 "${PARSER}" \ - --existing "${EXISTING_RESULT}" \ - --current "${CURRENT_RESULT}" \ - --main "${MAIN_RESULT}" \ - --branch-name "${BRANCH_NAME}" \ - --commit-sha "${COMMIT_SHA:-$(git rev-parse HEAD)}" \ - --output "${SUMMARY_RESULT}" +echo "Results uploaded to:" +echo " ${RESULTS_BUCKET}/$(basename "${RESULT_FILE}")" # ============================================================================ -# Upload Summary Table +# Trigger Benchmark Results Table Generator # ============================================================================ echo -echo "Uploading benchmark summary table" +echo "Triggering Linux benchmark results generator for branch ${BRANCH_NAME}..." echo "------------------------------------------------------------" -gcloud storage cp \ - "${SUMMARY_RESULT}" \ - "${RESULTS_BUCKET}/" - -echo -echo "Summary table uploaded to:" -echo " ${RESULTS_BUCKET}/benchmark_summary_table.txt" +gcloud beta builds triggers run linux-benchmark-results-ci \ + --branch="${BRANCH_NAME}" \ + --project="${PROJECT_ID:-bigquery-devtools-drivers}" \ + --region=us-east1 echo echo "============================================================" -echo "Benchmark completed successfully" +echo "Benchmark (${BUILD_SHARD}) completed successfully" echo "============================================================" diff --git a/ci/cloudbuild/builds/linux-existing-driver-benchmark.sh b/ci/cloudbuild/builds/linux-existing-driver-benchmark.sh new file mode 100755 index 0000000000..acc2645b16 --- /dev/null +++ b/ci/cloudbuild/builds/linux-existing-driver-benchmark.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +export BUILD_SHARD="Core" +exec "$(dirname "$0")/linux-bq-driver-benchmark.sh" diff --git a/ci/cloudbuild/triggers/linux-benchmark-results-ci.yaml b/ci/cloudbuild/triggers/linux-benchmark-results-ci.yaml new file mode 100644 index 0000000000..baba6706d0 --- /dev/null +++ b/ci/cloudbuild/triggers/linux-benchmark-results-ci.yaml @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + tag: ^benchmark- +name: linux-benchmark-results-ci +substitutions: + _BUILD_NAME: linux-benchmark-results + _DEPENDENCIES: '' + _DISTRO: ubuntu-22.04-install + _TRIGGER_TYPE: ci +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +tags: +- ci diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml index 8f6aad0c1e..dfc5f37c41 100644 --- a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -21,7 +21,7 @@ github: name: linux-bq-driver-benchmark-ci substitutions: _BUILD_NAME: linux-bq-driver-benchmark - _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' + _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' _DISTRO: ubuntu-22.04-install _TRIGGER_TYPE: ci includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS diff --git a/ci/cloudbuild/triggers/linux-existing-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-existing-driver-benchmark-ci.yaml new file mode 100644 index 0000000000..9410042771 --- /dev/null +++ b/ci/cloudbuild/triggers/linux-existing-driver-benchmark-ci.yaml @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + tag: ^benchmark- +name: linux-existing-driver-benchmark-ci +substitutions: + _BUILD_NAME: linux-existing-driver-benchmark + _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP' + _DISTRO: ubuntu-22.04-install + _TRIGGER_TYPE: ci +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +tags: +- ci