diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index b6ed114604..20deea4a4d 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -165,165 +165,15 @@ jobs: - name: Parse Results and Generate Table env: BRANCH_NAME: ${{ github.ref_name }} + COMMIT_SHA: ${{ needs.pre-flight.outputs.checkout-sha }} run: | - python3 <<'EOF' - import os - import re - - branch_name = os.environ.get('BRANCH_NAME', 'Current') - - def clean_test_name(name): - # GTest names follow [Instantiation/]TestSuite.TestCase[/Param] - # Extract everything after '.' to generically remove TestSuite/Instantiation prefix - if '.' in name: - name = name.split('.', 1)[1] - # Strip legacy parameter suffix if comparing against older baselines - name = re.sub(r'/(?:With|Without)HTAPI$', '', name) - return name - - def parse_gtest_output(filepath): - # The suite is run several times across iterations, so each test - # appears once per repetition. Take the median of its timings. - samples = {} - if not os.path.exists(filepath): - return {} - - ok_pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)') - metric_pattern = re.compile(r'\[\s+METRIC\s+\]\s+(\S+)\s+\(([^)]+)\):\s*([0-9.]+\s*ms)') - try: - with open(filepath, 'r') as f: - for line in f: - m_match = metric_pattern.search(line) - if m_match: - test_name = m_match.group(1) - metric_name = m_match.group(2) - entry_name = f'{test_name} - {metric_name}' - ms = parse_time_to_ms(m_match.group(3)) - if ms is not None: - samples.setdefault(entry_name, []).append(ms) - continue - - ok_match = ok_pattern.search(line) - if ok_match: - raw_name = ok_match.group(1) - test_name = clean_test_name(raw_name) - if 'DataFetchPerformanceParamTest' in raw_name or test_name.startswith('Benchmark/'): - base_name = test_name.split('Benchmark/', 1)[-1] - if not re.search(r'_(?:10k|100k|1M)$', base_name): - base_name = f'{base_name}_1M' - entry_name = f'{base_name} - Total execution time' - else: - entry_name = test_name - ms = parse_time_to_ms(ok_match.group(2)) - if ms is not None: - samples.setdefault(entry_name, []).append(ms) - except Exception as e: - print(f'Error reading {filepath}: {e}') - - results = {} - for entry_name, values in samples.items(): - values.sort() - n = len(values) - median = (values[n // 2] if n % 2 == 1 - else (values[n // 2 - 1] + values[n // 2]) / 2.0) - results[entry_name] = f'{median}ms' - if n > 1: - print(f'{entry_name}: median={median:.0f}ms of {n} runs ' - f'(min={values[0]:.0f}ms max={values[-1]:.0f}ms)') - return results - - def parse_time_to_ms(time_str): - if not time_str or time_str == 'N/A': - return None - time_str = time_str.strip() - match = re.match(r'^([\d.]+)\s*(\w+)$', time_str) - if not match: - return None - val = float(match.group(1)) - unit = match.group(2).lower() - if unit == 'ms': - return val - elif unit == 's': - return val * 1000 - elif unit == 'us': - return val / 1000 - elif unit == 'ns': - return val / 1000000 - return val - - def get_percentage_str(val_ms, ref_ms): - if val_ms is None or ref_ms is None or ref_ms == 0: - return ' (N/A)' - pct = round(((val_ms - ref_ms) / ref_ms) * 100) - if pct > 0: - return f' (+{pct}%)' - elif pct < 0: - return f' ({pct}%)' - else: - return ' (0%)' - - def sort_key(test_name): - match = re.match(r'^(.+?)_(10k|100k|1M) - (.+)$', test_name) - if match: - table_name, limit_label, metric_name = match.groups() - limit_order = {'10k': 1, '100k': 2, '1M': 3}.get(limit_label, 4) - metric_order = { - 'Time to first byte': 1, - 'Iteration time': 2, - 'Total execution time': 3, - }.get(metric_name, 4) - return (0, table_name, limit_order, metric_order) - return (1, test_name, 0, 0) - - existing_data = parse_gtest_output('./benchmark_results/current_core.txt') - current_bq_data = parse_gtest_output('./benchmark_results/current_bq.txt') - main_bq_data = parse_gtest_output('./benchmark_results/main_bq.txt') - - all_tests = set(existing_data.keys()).union(set(current_bq_data.keys())).union(set(main_bq_data.keys())) - sorted_tests = sorted(list(all_tests), key=sort_key) - - rows = [] - for test in sorted_tests: - existing_raw = existing_data.get(test, 'N/A') - cur_bq_raw = current_bq_data.get(test, 'N/A') - main_bq_raw = main_bq_data.get(test, 'N/A') - - existing_ms = parse_time_to_ms(existing_raw) - cur_bq_ms = parse_time_to_ms(cur_bq_raw) - main_bq_ms = parse_time_to_ms(main_bq_raw) - - cur_bq_pct = get_percentage_str(cur_bq_ms, existing_ms) if cur_bq_raw != 'N/A' else '' - main_bq_pct = get_percentage_str(main_bq_ms, cur_bq_ms) if (main_bq_raw != 'N/A' and cur_bq_raw != 'N/A') else ' (N/A)' if main_bq_raw == 'N/A' else '' - - cur_bq_val = f'{cur_bq_raw}{cur_bq_pct}' - main_bq_val = f'{main_bq_raw}{main_bq_pct}' - - rows.append((test, existing_raw, cur_bq_val, main_bq_val)) - - # Define headers - h1 = 'Test Case' - h2 = 'Existing Driver (Current)' - h3 = f'Google Driver ({branch_name})' - h4 = 'Google Driver (Main)' - - # Calculate dynamic widths for formatting - w1 = max([len(h1)] + [len(r[0]) for r in rows]) if rows else len(h1) - w2 = max([len(h2)] + [len(r[1]) for r in rows]) if rows else len(h2) - w3 = max([len(h3)] + [len(r[2]) for r in rows]) if rows else len(h3) - w4 = max([len(h4)] + [len(r[3]) for r in rows]) if rows else len(h4) - - # Construct the Markdown table - table = f"*Percentages in **{h3}** show change relative to **{h2}**. Percentages in **{h4}** show change relative to **{h3}**. Negative values indicate improvement (faster test execution), positive values indicate degradation (slower).*\n\n" - table += f'| {h1.ljust(w1)} | {h2.ljust(w2)} | {h3.ljust(w3)} | {h4.ljust(w4)} |\n' - table += '|-' + ('-' * w1) + '-|-' + ('-' * w2) + '-|-' + ('-' * w3) + '-|-' + ('-' * w4) + '-|\n' - - for r in rows: - table += f'| {r[0].ljust(w1)} | {r[1].ljust(w2)} | {r[2].ljust(w3)} | {r[3].ljust(w4)} |\n' - - # Write table to file - with open('benchmark_summary_table.txt', 'w') as f: - f.write(table) - EOF + python3 ci/cloudbuild/builds/lib/benchmark_results.py \ + --existing ./benchmark_results/current_core.txt \ + --current ./benchmark_results/current_bq.txt \ + --main ./benchmark_results/main_bq.txt \ + --branch-name "${BRANCH_NAME}" \ + --commit-sha "${COMMIT_SHA}" \ + --output benchmark_summary_table.txt - name: Upload Table to GCS env: diff --git a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh index b58c5ddfec..605d823eee 100755 --- a/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh +++ b/ci/cloudbuild/builds/integration-production-bq-driver-dm.sh @@ -73,6 +73,7 @@ io::run cmake -B "$BUILD_DIR" \ "${cmake_args[@]}" \ -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ -DODBC_INTEGRATION_TESTING=ON \ -DBQ_DRIVER_INTEGRATION_TESTS=ON \ -DODBC_DEMO_TESTING=ON \ @@ -81,6 +82,49 @@ io::run cmake -B "$BUILD_DIR" \ -DCLIENT_LIBRARY_INTEGRATION_TESTING=OFF io::run cmake --build cmake-out +# --------------------------------------------------------------------------- +# Publish ODBC driver .so for performance benchmarks +# --------------------------------------------------------------------------- + +if [[ "${UNIXODBC_INSTALLED}" == "false" ]]; then + DRIVER_SO="cmake-out/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + + if [[ ! -f "$DRIVER_SO" ]]; then + echo "ERROR: ODBC driver .so was not found:" + echo " $DRIVER_SO" + exit 1 + fi + + echo "ODBC driver .so found:" + ls -lh "$DRIVER_SO" + + # Sanitize branch name for use in GCS path. + SANITIZED_BRANCH=$( + echo "${BRANCH_NAME}" | + sed -E 's/[^a-zA-Z0-9-]/_/g' + ) + + INTEGRATION_BUILDS_BUCKET="gs://odbc-integration-builds" + + echo "Uploading driver artifact..." + echo "Branch: ${SANITIZED_BRANCH}" + + gcloud storage cp \ + "$DRIVER_SO" \ + "${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" + + echo "Driver benchmark artifact uploaded:" + echo "${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" + + if [[ "${BRANCH_NAME}" == "main" ]]; then + echo "Triggering Linux benchmark pipeline for main..." + gcloud beta builds triggers run linux-bq-driver-benchmark-ci \ + --branch=main \ + --project="${PROJECT_ID:-bigquery-devtools-drivers}" \ + --region=us-east1 + fi +fi + # Copy the roots.pem file to the .so directory to run test cases. cp /opt/odbc-driver/roots.pem "cmake-out/google/cloud/odbc/roots.pem" mapfile -t ctest_args < <(ctest::common_args) diff --git a/ci/cloudbuild/builds/lib/benchmark_results.py b/ci/cloudbuild/builds/lib/benchmark_results.py new file mode 100644 index 0000000000..e8a45400ca --- /dev/null +++ b/ci/cloudbuild/builds/lib/benchmark_results.py @@ -0,0 +1,450 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re +import statistics +import subprocess +from pathlib import Path + +TIME_RE = re.compile(r"\[\s*OK\s*\]\s+(.+?)\s+\(([\d.]+)\s*(ns|us|ms|s)\)") +METRIC_RE = re.compile( + r"\[\s*METRIC\s*\]\s+(\S+)\s+\(([^)]+)\):\s*([\d.]+)\s*(ns|us|ms|s)" +) + + +def clean_test_name(name): + """Normalize GTest test names for comparison.""" + name = name.strip().rstrip(",") + + # Ignore GTest summary lines such as: + # + # 19 tests, listed below: + # + if re.match(r"^\d+\s+tests?,\s+listed below:$", name): + return None + + # Remove GTest parameter description. + # + # Example: + # + # Benchmark/all_bq_types_2, where GetParam() = + # ("all_bq_types_2", "SELECT * FROM ...") + # + # becomes: + # + # Benchmark/all_bq_types_2 + # + name = re.sub( + r",\s*where\s+GetParam\(\)\s*=.*$", + "", + name, + ) + + # Remove everything before the first '.'. + # + # Example: + # + # Instantiation/TestSuite.TestCase/0 + # + # becomes: + # + # TestCase/0 + if "." in name: + name = name.split(".", 1)[1] + + # Remove legacy HTAPI suffixes. + name = re.sub( + r"/(?:With|Without)HTAPI$", + "", + name, + ) + + return name + + +def parse_time_to_ms(value, unit): + """Convert a GTest duration to milliseconds.""" + if unit == "s": + return value * 1000.0 + + if unit == "us": + return value / 1000.0 + + if unit == "ns": + return value / 1_000_000.0 + + return value + + +def format_ms(value): + """Format milliseconds for the benchmark table.""" + if value is None: + return "N/A" + + return f"{value:.0f}ms" + + +def parse_gtest_output(path): + """ + Parse repeated GTest benchmark output. + + A benchmark is run multiple times. + + Rules: + + 1. Only successful iterations are used. + + 2. Failed iterations are ignored. + + 3. The median execution time is calculated from all successful + iterations. + + 4. If all iterations fail or a test is completely missing, + it is displayed as N/A. + """ + path = Path(path) + + if not path.exists(): + print(f"WARNING: Benchmark output file not found: {path}") + return {} + + samples = {} + + for line in path.read_text(errors="replace").splitlines(): + # --------------------------------------------------------------- + # Custom metric line (e.g., Time to first byte, Iteration time) + # --------------------------------------------------------------- + + metric_match = METRIC_RE.search(line) + + if metric_match: + test_name = metric_match.group(1) + metric_name = metric_match.group(2) + entry_name = f"{test_name} - {metric_name}" + + value = float(metric_match.group(3)) + unit = metric_match.group(4).lower() + + value_ms = parse_time_to_ms( + value, + unit, + ) + + samples.setdefault( + entry_name, + [], + ).append(value_ms) + continue + + # --------------------------------------------------------------- + # Successful test + # --------------------------------------------------------------- + + time_match = TIME_RE.search(line) + + if time_match: + raw_name = time_match.group(1) + test_name = clean_test_name(raw_name) + + # Ignore GTest summary/non-test lines. + if test_name is None: + continue + + if "DataFetchPerformanceParamTest" in raw_name or test_name.startswith( + "Benchmark/" + ): + base_name = test_name.split("Benchmark/", 1)[-1] + if not re.search(r"_(?:10k|100k|1M)$", base_name): + base_name = f"{base_name}_1M" + entry_name = f"{base_name} - Total execution time" + else: + entry_name = test_name + + value = float(time_match.group(2)) + unit = time_match.group(3).lower() + + value_ms = parse_time_to_ms( + value, + unit, + ) + + samples.setdefault( + entry_name, + [], + ).append(value_ms) + + # ------------------------------------------------------------------- + # Build final result. + # ------------------------------------------------------------------- + + results = {} + + for test_name, values in samples.items(): + if not values: + results[test_name] = None + continue + + values.sort() + median = statistics.median(values) + results[test_name] = median + if len(values) > 1: + print( + f"{test_name}: median={median:.0f}ms of {len(values)} runs " + f"(min={values[0]:.0f}ms max={values[-1]:.0f}ms)" + ) + + return results + + +def sort_key(test_name): + """Sort data fetch benchmarks by table, row limit (10k, 100k, 1M), and metric order.""" + match = re.match(r"^(.+?)_(10k|100k|1M) - (.+)$", test_name) + if match: + table_name, limit_label, metric_name = match.groups() + limit_order = {"10k": 1, "100k": 2, "1M": 3}.get(limit_label, 4) + metric_order = { + "Time to first byte": 1, + "Iteration time": 2, + "Total execution time": 3, + }.get(metric_name, 4) + return (0, table_name, limit_order, metric_order) + return (1, test_name, 0, 0) + + +def get_percentage_str(value_ms, reference_ms): + """ + Return percentage change. + + Negative = faster/improvement. + Positive = slower/degradation. + """ + if value_ms is None or reference_ms is None or reference_ms == 0: + return " (N/A)" + + pct = round(((value_ms - reference_ms) / reference_ms) * 100) + + if pct > 0: + return f" (+{pct}%)" + + if pct < 0: + return f" ({pct}%)" + + return " (0%)" + + +def main(): + parser = argparse.ArgumentParser( + description="Parse ODBC performance benchmark output." + ) + + parser.add_argument( + "--existing", + required=True, + help="Existing driver benchmark output", + ) + + parser.add_argument( + "--current", + required=True, + help="Current Google driver benchmark output", + ) + + parser.add_argument( + "--main", + required=True, + help="Main Google driver benchmark output", + ) + + parser.add_argument( + "--output", + required=True, + help="Summary output file", + ) + + parser.add_argument( + "--branch-name", + default="Current", + help="Branch name used in the Google Driver column header", + ) + + parser.add_argument( + "--commit-sha", + default="", + help="Commit SHA that the benchmark was run on", + ) + + args = parser.parse_args() + + # ------------------------------------------------------------------- + # Parse benchmark outputs. + # ------------------------------------------------------------------- + + existing_data = parse_gtest_output(args.existing) + current_data = parse_gtest_output(args.current) + main_data = parse_gtest_output(args.main) + + if not current_data: + print( + "WARNING: No current Google benchmark results were found. " + "Google Current will be shown as N/A." + ) + + if not main_data: + print( + "WARNING: No main Google benchmark results were found. " + "Google Main will be shown as N/A." + ) + + if not existing_data: + print( + "WARNING: No Existing benchmark results were found. " + "Existing Driver will be shown as N/A." + ) + + # ------------------------------------------------------------------- + # Union of all test names. + # ------------------------------------------------------------------- + + all_tests = ( + set(existing_data.keys()) | set(current_data.keys()) | set(main_data.keys()) + ) + + sorted_tests = sorted(all_tests, key=sort_key) + + rows = [] + + for test_name in sorted_tests: + + existing_ms = existing_data.get(test_name) + current_ms = current_data.get(test_name) + main_ms = main_data.get(test_name) + + existing_raw = format_ms(existing_ms) + current_raw = format_ms(current_ms) + main_raw = format_ms(main_ms) + + # --------------------------------------------------------------- + # Current vs Existing + # --------------------------------------------------------------- + + current_pct = "" + + if current_ms is not None: + current_pct = get_percentage_str( + current_ms, + existing_ms, + ) + + # --------------------------------------------------------------- + # Main vs Current + # --------------------------------------------------------------- + + main_pct = "" + + if main_ms is not None: + main_pct = get_percentage_str( + main_ms, + current_ms, + ) + + current_value = f"{current_raw}{current_pct}" + main_value = f"{main_raw}{main_pct}" + + rows.append( + ( + test_name, + existing_raw, + current_value, + main_value, + ) + ) + + # ------------------------------------------------------------------- + # Generate Markdown table. + # + # This intentionally follows the existing GHA table format. + # ------------------------------------------------------------------- + + h1 = "Test Case" + h2 = "Existing Driver (Current)" + h3 = f"Google Driver ({args.branch_name})" + h4 = "Google Driver (Main)" + + w1 = max([len(h1)] + [len(row[0]) for row in rows]) if rows else len(h1) + w2 = max([len(h2)] + [len(row[1]) for row in rows]) if rows else len(h2) + w3 = max([len(h3)] + [len(row[2]) for row in rows]) if rows else len(h3) + w4 = max([len(h4)] + [len(row[3]) for row in rows]) if rows else len(h4) + + commit_sha = ( + args.commit_sha or os.environ.get("COMMIT_SHA") or os.environ.get("GITHUB_SHA") + ) + if not commit_sha: + try: + commit_sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip() + except Exception: + commit_sha = "unknown" + + table = ( + f"Commit ID: `{commit_sha}`\n\n" + f"*Percentages in **{h3}** show change relative to " + f"**{h2}**. Percentages in **{h4}** show change relative " + f"to **{h3}**. Negative values indicate improvement " + f"(faster test execution), positive values indicate " + f"degradation (slower).*" + "\n\n" + ) + + table += ( + f"| {h1.ljust(w1)} " + f"| {h2.ljust(w2)} " + f"| {h3.ljust(w3)} " + f"| {h4.ljust(w4)} |\n" + ) + + table += ( + "|-" + + ("-" * w1) + + "-|-" + + ("-" * w2) + + "-|-" + + ("-" * w3) + + "-|-" + + ("-" * w4) + + "-|\n" + ) + + for row in rows: + table += ( + f"| {row[0].ljust(w1)} " + f"| {row[1].ljust(w2)} " + f"| {row[2].ljust(w3)} " + f"| {row[3].ljust(w4)} |\n" + ) + + # ------------------------------------------------------------------- + # Write output. + # ------------------------------------------------------------------- + + output_path = Path(args.output) + output_path.write_text(table) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/cloudbuild/builds/linux-benchmark-results.sh b/ci/cloudbuild/builds/linux-benchmark-results.sh new file mode 100755 index 0000000000..e81394c3a7 --- /dev/null +++ b/ci/cloudbuild/builds/linux-benchmark-results.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +source "$(dirname "$0")/../../lib/init.sh" +source module ci/lib/io.sh + +WORKSPACE_DIR=$(pwd) + +PERF_RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" + +rm -rf "${RESULTS_DIR}" +mkdir -p "${RESULTS_DIR}" + +BRANCH_NAME="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH="$( + echo "${BRANCH_NAME}" | sed -E 's/[^a-zA-Z0-9-]/_/g' +)" + +RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/${SANITIZED_BRANCH}/linux_results" +MAIN_RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/main/linux_results" + +CURRENT_RESULT="${RESULTS_DIR}/current_bq.txt" +EXISTING_RESULT="${RESULTS_DIR}/current_core.txt" +MAIN_RESULT="${RESULTS_DIR}/main_bq.txt" +SUMMARY_RESULT="${RESULTS_DIR}/benchmark_summary_table.txt" + +echo "============================================================" +echo "Linux ODBC Benchmark Results Table Generator" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Results GCS : ${RESULTS_BUCKET}" +echo + +echo "Downloading benchmark results from GCS" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" \ + "${CURRENT_RESULT}" || true + +gcloud storage cp \ + "${RESULTS_BUCKET}/performance_benchmark_results_Core.txt" \ + "${EXISTING_RESULT}" || true + +gcloud storage cp \ + "${MAIN_RESULTS_BUCKET}/performance_benchmark_results_BqDriver.txt" \ + "${MAIN_RESULT}" || true + +echo +echo "Generating benchmark comparison table" +echo "------------------------------------------------------------" + +PARSER="${WORKSPACE_DIR}/ci/cloudbuild/builds/lib/benchmark_results.py" + +if [[ ! -f "${PARSER}" ]]; then + echo "ERROR: benchmark_results.py was not found:" + echo " ${PARSER}" + exit 1 +fi + +python3 "${PARSER}" \ + --existing "${EXISTING_RESULT}" \ + --current "${CURRENT_RESULT}" \ + --main "${MAIN_RESULT}" \ + --branch-name "${BRANCH_NAME}" \ + --commit-sha "${COMMIT_SHA:-$(git rev-parse HEAD)}" \ + --output "${SUMMARY_RESULT}" + +echo +echo "Uploading benchmark summary table" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${SUMMARY_RESULT}" \ + "${RESULTS_BUCKET}/" + +echo +echo "Uploaded benchmark summary table to:" +echo " ${RESULTS_BUCKET}/benchmark_summary_table.txt" diff --git a/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh new file mode 100755 index 0000000000..7f858faa5b --- /dev/null +++ b/ci/cloudbuild/builds/linux-bq-driver-benchmark.sh @@ -0,0 +1,266 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +source "$(dirname "$0")/../../lib/init.sh" +source module ci/install-dependencies.sh + +source module ci/cloudbuild/builds/lib/cmake.sh +source module ci/cloudbuild/builds/lib/secrets.sh +source module ci/lib/io.sh + +WORKSPACE_DIR=$(pwd) + +# ============================================================================ +# Configuration +# ============================================================================ + +BUILD_SHARD="${BUILD_SHARD:-BqDriver}" +BENCHMARK_ITERATIONS="${BENCHMARK_ITERATIONS:-3}" +INTEGRATION_BUILDS_BUCKET="gs://odbc-integration-builds" +PERF_RESULTS_BUCKET="gs://bq-dev-tools-testing-drivers/odbc-perf" + +BUILD_DIR="${WORKSPACE_DIR}/cmake-out" +RESULTS_DIR="${WORKSPACE_DIR}/benchmark-results" + +rm -rf "${RESULTS_DIR}" +mkdir -p "${RESULTS_DIR}" + +BRANCH_NAME="${BRANCH_NAME:-main}" + +SANITIZED_BRANCH="$( + echo "${BRANCH_NAME}" | sed -E 's/[^a-zA-Z0-9-]/_/g' +)" + +RESULTS_BUCKET="${PERF_RESULTS_BUCKET}/${SANITIZED_BRANCH}/linux_results" +RESULT_FILE="${RESULTS_DIR}/performance_benchmark_results_${BUILD_SHARD}.txt" + +export CPP_BIGQUERY_ODBC_TEST_TABLE_PREFIX="${TRIGGER_NAME//[-:;.,?]/_}_${BRANCH_NAME//[-:;.,?]/_}" + +# ============================================================================ +# Header +# ============================================================================ + +echo "============================================================" +echo "Linux ODBC Performance Benchmark (${BUILD_SHARD})" +echo "============================================================" +echo "Branch : ${BRANCH_NAME}" +echo "Shard : ${BUILD_SHARD}" +echo "Iterations : ${BENCHMARK_ITERATIONS}" +echo "Workspace : ${WORKSPACE_DIR}" +echo + +# ============================================================================ +# Run benchmark +# ============================================================================ + +run_benchmark() { + local name="$1" + local output_file="$2" + local dsn="$3" + local bq_tests_flag="$4" + + : >"${output_file}" + + echo + echo "Reconfiguring performance_test for ${name}" + echo "BQ_DRIVER_INTEGRATION_TESTS = ${bq_tests_flag}" + echo "------------------------------------------------------------" + + if ! io::run cmake \ + -S "${WORKSPACE_DIR}" \ + -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_PERFORMANCE_TEST_ONLY=ON \ + -DBQ_DRIVER_INTEGRATION_TESTS="${bq_tests_flag}"; then + echo "WARNING: CMake configuration failed for ${name}; skipping benchmark." + return 0 + fi + + if ! io::run cmake \ + --build "${BUILD_DIR}" \ + --target performance_test \ + --parallel "$(nproc)"; then + echo "WARNING: Building performance_test failed for ${name}; skipping benchmark." + return 0 + fi + + local performance_test="${BUILD_DIR}/integration_tests/performance_test" + + if [[ ! -x "${performance_test}" ]]; then + performance_test="${BUILD_DIR}/google/cloud/odbc/integration_tests/performance_test" + fi + + if [[ ! -x "${performance_test}" ]]; then + echo "WARNING: performance_test was not found for ${name}; skipping benchmark." + + find "${BUILD_DIR}" \ + -type f \ + -name "performance_test" \ + -print 2>/dev/null || true + + return 0 + fi + + echo "performance_test:" + echo " ${performance_test}" + + local failed_iterations=0 + + for ((i = 1; i <= BENCHMARK_ITERATIONS; i++)); do + echo + echo "=== ${name}: iteration ${i}/${BENCHMARK_ITERATIONS} ===" + + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" \ + >>"${output_file}" + + echo "=== ODBC tests DSN is====${ODBC_TESTS_DSN}===" + echo "=== ODBCINI is====${dsn}===" + + set +e + + ODBCINI="${dsn}" \ + ODBC_TESTS_DSN="${ODBC_TESTS_DSN}" \ + "${performance_test}" \ + >>"${output_file}" 2>&1 + + local run_exit=$? + + set -e + + if [[ "${run_exit}" -ne 0 ]]; then + echo "WARNING: ${name} iteration ${i} failed with exit code ${run_exit}" + echo "WARNING: Continuing with remaining iterations." + + echo "============================================================" + echo "Detailed failure for ${name}, iteration ${i}:" + echo "============================================================" + + grep -E -B 20 -A 20 \ + '\[ *FAILED *\]|Failure|FAILED|ERROR|Error|error|SQLSTATE|Diagnostic|diagnostic|SQL_ERROR|SQLConnect|SQLDriverConnect|Exception|exception' \ + "${output_file}" || true + + echo "============================================================" + + ((failed_iterations += 1)) + fi + done + + echo + echo "${name} benchmark completed." + echo "Failed iterations: ${failed_iterations}/${BENCHMARK_ITERATIONS}" + echo "Raw result: ${output_file}" + + # Benchmark failures must not fail Cloud Build. + return 0 +} + +# ============================================================================ +# Execute Shard +# ============================================================================ + +if [[ "${BUILD_SHARD}" == "Core" ]]; then + EXISTING_ODBCINI="/opt/odbc-driver/googlebigqueryodbc/odbc.ini" + + if [[ ! -f "${EXISTING_ODBCINI}" ]]; then + echo "ERROR: Existing ODBC configuration not found:" + echo " ${EXISTING_ODBCINI}" + exit 1 + fi + + export ODBC_TESTS_DSN="SampleDSN" + export ODBCINI="${EXISTING_ODBCINI}" + export ODBCINSTINI="/opt/odbc-driver/googlebigqueryodbc/odbcinst.ini" + + run_benchmark \ + "Existing" \ + "${RESULT_FILE}" \ + "${EXISTING_ODBCINI}" \ + "OFF" +else + GOOGLE_ODBCINI="/opt/odbc-driver/odbc.ini" + + if [[ ! -f "${GOOGLE_ODBCINI}" ]]; then + echo "ERROR: Google ODBC configuration not found:" + echo " ${GOOGLE_ODBCINI}" + exit 1 + fi + + export ODBC_TESTS_DSN="SampleDSNGoogleDriver" + export ODBCINI="${GOOGLE_ODBCINI}" + export ODBCINSTINI="/opt/odbc-driver/odbcinst.ini" + + DRIVER_PATH="${BUILD_DIR}/google/cloud/odbc/libgoogle_cloud_odbc_bq_driver.so" + CURRENT_SO="${RESULTS_DIR}/libgoogle_cloud_odbc_bq_driver_current.so" + CURRENT_SO_GCS="${INTEGRATION_BUILDS_BUCKET}/${SANITIZED_BRANCH}/libgoogle_cloud_odbc_bq_driver.so" + + echo "Downloading Google Current driver" + echo "------------------------------------------------------------" + + gcloud storage cp \ + "${CURRENT_SO_GCS}" \ + "${CURRENT_SO}" + + mkdir -p "$(dirname "${DRIVER_PATH}")" + + cp \ + /opt/odbc-driver/roots.pem \ + "${DRIVER_PATH%/*}/roots.pem" + + cp "${CURRENT_SO}" "${DRIVER_PATH}" + ls -lh "${DRIVER_PATH}" + + run_benchmark \ + "Google Current" \ + "${RESULT_FILE}" \ + "${GOOGLE_ODBCINI}" \ + "ON" +fi + +# ============================================================================ +# Upload Result +# ============================================================================ + +echo +echo "Uploading benchmark results to GCS" +echo "------------------------------------------------------------" + +gcloud storage cp \ + "${RESULT_FILE}" \ + "${RESULTS_BUCKET}/" + +echo +echo "Results uploaded to:" +echo " ${RESULTS_BUCKET}/$(basename "${RESULT_FILE}")" + +# ============================================================================ +# Trigger Benchmark Results Table Generator +# ============================================================================ + +echo +echo "Triggering Linux benchmark results generator for branch ${BRANCH_NAME}..." +echo "------------------------------------------------------------" + +gcloud beta builds triggers run linux-benchmark-results-ci \ + --branch="${BRANCH_NAME}" \ + --project="${PROJECT_ID:-bigquery-devtools-drivers}" \ + --region=us-east1 + +echo +echo "============================================================" +echo "Benchmark (${BUILD_SHARD}) completed successfully" +echo "============================================================" diff --git a/ci/cloudbuild/builds/linux-existing-driver-benchmark.sh b/ci/cloudbuild/builds/linux-existing-driver-benchmark.sh new file mode 100755 index 0000000000..acc2645b16 --- /dev/null +++ b/ci/cloudbuild/builds/linux-existing-driver-benchmark.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +export BUILD_SHARD="Core" +exec "$(dirname "$0")/linux-bq-driver-benchmark.sh" diff --git a/ci/cloudbuild/triggers/linux-benchmark-results-ci.yaml b/ci/cloudbuild/triggers/linux-benchmark-results-ci.yaml new file mode 100644 index 0000000000..baba6706d0 --- /dev/null +++ b/ci/cloudbuild/triggers/linux-benchmark-results-ci.yaml @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + tag: ^benchmark- +name: linux-benchmark-results-ci +substitutions: + _BUILD_NAME: linux-benchmark-results + _DEPENDENCIES: '' + _DISTRO: ubuntu-22.04-install + _TRIGGER_TYPE: ci +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +tags: +- ci diff --git a/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml new file mode 100644 index 0000000000..dfc5f37c41 --- /dev/null +++ b/ci/cloudbuild/triggers/linux-bq-driver-benchmark-ci.yaml @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + tag: ^benchmark- +name: linux-bq-driver-benchmark-ci +substitutions: + _BUILD_NAME: linux-bq-driver-benchmark + _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP_GOOGLE_DRIVER' + _DISTRO: ubuntu-22.04-install + _TRIGGER_TYPE: ci +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +tags: +- ci diff --git a/ci/cloudbuild/triggers/linux-existing-driver-benchmark-ci.yaml b/ci/cloudbuild/triggers/linux-existing-driver-benchmark-ci.yaml new file mode 100644 index 0000000000..9410042771 --- /dev/null +++ b/ci/cloudbuild/triggers/linux-existing-driver-benchmark-ci.yaml @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filename: ci/cloudbuild/cloudbuild.yaml +github: + name: cpp-bigquery-odbc + owner: googleapis + push: + tag: ^benchmark- +name: linux-existing-driver-benchmark-ci +substitutions: + _BUILD_NAME: linux-existing-driver-benchmark + _DEPENDENCIES: 'iODBC,DRIVER_MANAGER_SETUP' + _DISTRO: ubuntu-22.04-install + _TRIGGER_TYPE: ci +includeBuildLogs: INCLUDE_BUILD_LOGS_WITH_STATUS +tags: +- ci diff --git a/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc b/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc index d3604a8e15..264f90691a 100644 --- a/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc +++ b/google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.cc @@ -24,6 +24,13 @@ #endif #include +#if (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW) && \ + defined(__GLIBCXX__) +template bool std::operator==(std::shared_ptr const&, + std::nullptr_t) noexcept; +#endif // (!defined(_WIN32) || defined(_WIN64)) && !defined(NO_ARROW) && + // defined(__GLIBCXX__) + ////////////////////////////////////////////////////////////////// // This file has query execution related utilities which can have // statement or descriptor handles as arguments. We have some utils