diff --git a/.github/workflows/pr-benchmarks.yml b/.github/workflows/pr-benchmarks.yml index 0e38c2869..a044efa1f 100644 --- a/.github/workflows/pr-benchmarks.yml +++ b/.github/workflows/pr-benchmarks.yml @@ -23,25 +23,24 @@ jobs: fail-fast: false matrix: include: - - name: base - repository: ${{ github.repository }} - ref: ${{ github.event.pull_request.base.sha }} - sha: ${{ github.event.pull_request.base.sha }} - - name: head - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - sha: ${{ github.event.pull_request.head.sha }} + - topic: counter + pattern: CounterBenchmark + - topic: histogram + pattern: HistogramBenchmark + - topic: exposition + pattern: HistogramTextFormatBenchmark|TextFormatUtilBenchmark permissions: contents: read # checkout only env: HEAD_SHA: ${{ github.event.pull_request.head.sha }} PR_NUMBER: ${{ github.event.pull_request.number }} + JMH_PATTERN: ${{ matrix.pattern }} steps: - - name: Checkout ${{ matrix.name }} + - name: Checkout PR base uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - repository: ${{ matrix.repository }} - ref: ${{ matrix.ref }} + repository: ${{ github.repository }} + ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false fetch-depth: 0 @@ -55,26 +54,49 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.m2/repository - key: ${{ runner.os }}-pr-bench-maven-${{ matrix.name }}-${{ matrix.sha }} + key: ${{ runner.os }}-pr-bench-maven-${{ matrix.topic }}-${{ env.HEAD_SHA }} restore-keys: | + ${{ runner.os }}-pr-bench-maven-${{ matrix.topic }}- ${{ runner.os }}-pr-bench-maven- ${{ runner.os }}-maven- - - name: Run ${{ matrix.name }} JMH benchmarks - run: mise run benchmark:ci-json - - - name: Capture ${{ matrix.name }} runner info + - name: Capture runner info run: | python3 ./.mise/tasks/generate_benchmark_summary.py \ - --write-system-info runner-info.json + --write-system-info /tmp/runner-info.json + + - name: Run base ${{ matrix.topic }} benchmarks + run: mise run benchmark:ci-json + env: + JMH_ARGS: -f 3 -wi 3 -i 5 ${{ env.JMH_PATTERN }} + + - name: Save base ${{ matrix.topic }} results + run: mv benchmark-results.json /tmp/base-results.json - - name: Upload ${{ matrix.name }} raw benchmark results + - name: Checkout PR head + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + fetch-depth: 0 + + - name: Run head ${{ matrix.topic }} benchmarks + run: mise run benchmark:ci-json + env: + JMH_ARGS: -f 3 -wi 3 -i 5 ${{ env.JMH_PATTERN }} + + - name: Save head ${{ matrix.topic }} results + run: mv benchmark-results.json /tmp/head-results.json + + - name: Upload ${{ matrix.topic }} benchmark results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: pr-benchmark-raw-${{ matrix.name }}-${{ env.PR_NUMBER }}-${{ env.HEAD_SHA }} + name: pr-benchmark-topic-${{ matrix.topic }}-${{ env.PR_NUMBER }}-${{ env.HEAD_SHA }} path: | - benchmark-results.json - runner-info.json + /tmp/base-results.json + /tmp/head-results.json + /tmp/runner-info.json retention-days: 5 summarize: @@ -95,28 +117,42 @@ jobs: persist-credentials: false fetch-depth: 0 - - name: Download base benchmark results + - name: Download topic benchmark results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: pr-benchmark-raw-base-${{ env.PR_NUMBER }}-${{ env.HEAD_SHA }} - path: /tmp/base-benchmark-results + pattern: pr-benchmark-topic-*-${{ env.PR_NUMBER }}-${{ env.HEAD_SHA }} + path: /tmp/topic-benchmark-results - - name: Download head benchmark results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: pr-benchmark-raw-head-${{ env.PR_NUMBER }}-${{ env.HEAD_SHA }} - path: /tmp/head-benchmark-results + - name: Merge topic benchmark results + run: | + python3 - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/topic-benchmark-results') + for name in ('base', 'head'): + files = sorted(root.glob(f'*/{name}-results.json')) + if not files: + raise SystemExit(f'No {name} benchmark result files found') + merged = [] + for file in files: + with open(file, 'r') as f: + merged.extend(json.load(f)) + with open(f'/tmp/{name}-benchmark-results.json', 'w') as f: + json.dump(merged, f) + f.write('\n') + PY - name: Generate benchmark summary run: | python3 ./.mise/tasks/generate_benchmark_summary.py \ - --input /tmp/head-benchmark-results/benchmark-results.json \ - --baseline /tmp/base-benchmark-results/benchmark-results.json \ - --system-info /tmp/head-benchmark-results/runner-info.json \ - --baseline-system-info /tmp/base-benchmark-results/runner-info.json \ + --input /tmp/head-benchmark-results.json \ + --baseline /tmp/base-benchmark-results.json \ --output-dir benchmark-results \ --commit-sha "${{ github.event.pull_request.head.sha }}" \ - --baseline-sha "${{ github.event.pull_request.base.sha }}" + --baseline-sha "${{ github.event.pull_request.base.sha }}" \ + --comparison-note \ + "Base and head are compared on the same runner within each topic." env: GITHUB_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} GITHUB_BASE_REPOSITORY: ${{ github.repository }} diff --git a/.mise/tasks/generate_benchmark_summary.py b/.mise/tasks/generate_benchmark_summary.py index cd51e8ac0..e37f1e09c 100644 --- a/.mise/tasks/generate_benchmark_summary.py +++ b/.mise/tasks/generate_benchmark_summary.py @@ -60,6 +60,11 @@ def parse_args(): default=None, help="Baseline GitHub repository for commit links", ) + parser.add_argument( + "--comparison-note", + default=None, + help="Optional note to include in the benchmark comparison section", + ) parser.add_argument( "--system-info", default=None, @@ -337,6 +342,7 @@ def generate_comparison_section( baseline_sha: str, repo: str, baseline_repo: str, + comparison_note: Optional[str] = None, system_info: Optional[Dict[str, str]] = None, baseline_system_info: Optional[Dict[str, str]] = None, ) -> List[str]: @@ -353,6 +359,8 @@ def generate_comparison_section( md.append(f"- **Head:** {format_commit_link(commit_sha, repo)}") md.append(f"- **Base:** {format_commit_link(baseline_sha, baseline_repo)}") md.append("- **Change:** positive means the PR is faster than base.") + if comparison_note: + md.append(f"- **Note:** {comparison_note}") if baseline_system_info: md.append(f"- **Head runner:** {format_system_info(system_info)}") md.append(f"- **Base runner:** {format_system_info(baseline_system_info)}") @@ -406,6 +414,7 @@ def generate_markdown( baseline_results: Optional[List] = None, baseline_sha: Optional[str] = None, baseline_repo: Optional[str] = None, + comparison_note: Optional[str] = None, system_info: Optional[Dict[str, str]] = None, baseline_system_info: Optional[Dict[str, str]] = None, ) -> str: @@ -461,6 +470,7 @@ def generate_markdown( baseline_sha, repo, baseline_repo, + comparison_note=comparison_note, system_info=sysinfo, baseline_system_info=baseline_system_info, ) @@ -671,6 +681,7 @@ def main(): baseline_results=baseline_results, baseline_sha=baseline_sha, baseline_repo=baseline_repo, + comparison_note=args.comparison_note, system_info=system_info, baseline_system_info=baseline_system_info, ) diff --git a/benchmarks/src/main/java/io/prometheus/metrics/benchmarks/HistogramBenchmark.java b/benchmarks/src/main/java/io/prometheus/metrics/benchmarks/HistogramBenchmark.java index 72cba0f9f..41b7097db 100644 --- a/benchmarks/src/main/java/io/prometheus/metrics/benchmarks/HistogramBenchmark.java +++ b/benchmarks/src/main/java/io/prometheus/metrics/benchmarks/HistogramBenchmark.java @@ -47,6 +47,16 @@ public PrometheusClassicHistogram() { } } + @State(Scope.Thread) + public static class PrometheusClassicHistogramPerThread { + + final Histogram noLabels; + + public PrometheusClassicHistogramPerThread() { + noLabels = Histogram.builder().name("test").help("help").classicOnly().build(); + } + } + @State(Scope.Benchmark) public static class PrometheusNativeHistogram { @@ -143,6 +153,26 @@ public Histogram prometheusClassic( return histogram.noLabels; } + @Benchmark + @Threads(1) + public Histogram prometheusClassicSingleThread( + RandomNumbers randomNumbers, PrometheusClassicHistogram histogram) { + for (int i = 0; i < randomNumbers.randomNumbers.length; i++) { + histogram.noLabels.observe(randomNumbers.randomNumbers[i]); + } + return histogram.noLabels; + } + + @Benchmark + @Threads(4) + public Histogram prometheusClassicPerThread( + RandomNumbers randomNumbers, PrometheusClassicHistogramPerThread histogram) { + for (int i = 0; i < randomNumbers.randomNumbers.length; i++) { + histogram.noLabels.observe(randomNumbers.randomNumbers[i]); + } + return histogram.noLabels; + } + @Benchmark @Threads(4) public Histogram prometheusNative(