diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..b77956b56 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,363 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Performance regression gate for Fesod. +# +# Runs the baseline benchmark suite (fesod-benchmark/baseline package) on release tags +# (e.g. 2.1.0-incubating) and on demand (workflow_dispatch), compares the results against +# the committed baseline (fesod-benchmark/baseline/jmh-baseline.json) and: +# - appends the comparison report to the job step summary +# - emits ::error/::warning annotations for regressions (visible on the run page) +# - fails the job when a regression exceeds the fail threshold with non-overlapping +# JMH error bars (high-confidence regressions only โ€” see benchmark.md for the +# rationale of the tiered gate) +# +# On tags with a regression, a follow-up job posts the report as a comment on the +# matching GitHub Release (or opens an issue) so maintainers cannot miss it. +# A passing tag run (or a dispatch with update_baseline) opens a PR that refreshes +# the committed baseline โ€” the baseline lifecycle follows releases. +# +# The baseline can only be regenerated on a GitHub runner (same hardware/JDK). +# Never commit a baseline generated on a local machine. + +name: Benchmark + +on: + push: + tags: + - '[0-9]+.*' + workflow_dispatch: + inputs: + update_baseline: + description: 'Run the suite and open a PR refreshing the committed baseline' + type: boolean + default: false + warn_threshold: + description: 'Regression percentage that triggers a warning' + type: number + default: 10 + fail_threshold: + description: 'Regression percentage that fails the job (beyond JMH error bars)' + type: number + default: 20 + +concurrency: + group: benchmark-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + BASELINE_FILE: fesod-benchmark/baseline/jmh-baseline.json + BASELINE_META: fesod-benchmark/baseline/baseline-meta.json + CURRENT_RESULT: fesod-benchmark/target/baseline-current.json + REPORT_FILE: fesod-benchmark/target/benchmark-report.md + +jobs: + benchmark: + name: Baseline comparison (JDK 17) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + pull-requests: write + outputs: + bootstrap: ${{ steps.baseline.outputs.exists != 'true' }} + update_requested: ${{ github.event_name == 'workflow_dispatch' && inputs.update_baseline }} + is_tag: ${{ startsWith(github.ref, 'refs/tags/') }} + compare_outcome: ${{ steps.compare.outcome }} + tag: ${{ github.ref_name }} + jdk: ${{ steps.env.outputs.jdk }} + os: ${{ steps.env.outputs.os }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-benchmark-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-benchmark- + + - name: Record environment + id: env + run: | + echo "jdk=$(java -version 2>&1 | head -n 1)" >> "$GITHUB_OUTPUT" + echo "os=$(uname -sr)" >> "$GITHUB_OUTPUT" + + - name: Check baseline presence + id: baseline + run: | + if [ -f "$BASELINE_FILE" ]; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "::notice::No committed baseline found โ€” this run will bootstrap one." + fi + + - name: Build benchmark suite + run: ./mvnw -B -ntp -pl fesod-benchmark -am package -DskipTests + + - name: Run baseline suite + run: | + cd fesod-benchmark + java -cp target/benchmarks.jar org.apache.fesod.sheet.benchmark.baseline.BaselineRunner + + - name: Compare against baseline + id: compare + continue-on-error: true + run: | + java -cp fesod-benchmark/target/benchmarks.jar \ + org.apache.fesod.sheet.benchmark.baseline.BaselineComparator \ + --baseline "$BASELINE_FILE" \ + --baseline-meta "$BASELINE_META" \ + --current "$CURRENT_RESULT" \ + --warn "${{ inputs.warn_threshold || vars.BENCHMARK_WARN_PCT || '10' }}" \ + --fail "${{ inputs.fail_threshold || vars.BENCHMARK_FAIL_PCT || '20' }}" \ + --report "$REPORT_FILE" + + - name: Add report to job summary + if: always() + run: | + RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + { + echo "## ๐Ÿ“Š Fesod performance report" + echo + echo "Run \`${{ github.sha }}\` on \`ubuntu-24.04\` ยท ${{ steps.env.outputs.jdk }} ยท [full run](${RUN_URL})" + echo + if [ -f "$REPORT_FILE" ]; then + cat "$REPORT_FILE" + else + echo "_No comparison report was produced (suite run failed or was cancelled)." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + if-no-files-found: warn + path: | + ${{ env.CURRENT_RESULT }} + ${{ env.REPORT_FILE }} + + - name: Comment report on open PR (manual runs on a PR branch) + if: github.event_name == 'workflow_dispatch' && always() + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ github.ref_name }} + run: | + [ -f "$REPORT_FILE" ] || { echo "No report file โ€” skipping."; exit 0; } + PR_NUMBER=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open \ + --json number -q '.[0].number' 2>/dev/null || true) + if [ -z "$PR_NUMBER" ]; then + echo "No open PR for branch '$BRANCH' โ€” report is in the job summary only." + exit 0 + fi + MARKER='' + RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + BODY_FILE=$(mktemp) + { + echo "$MARKER" + echo "### ๐Ÿ“Š Performance vs baseline โ€” run [${{ github.run_id }}](${RUN_URL})" + echo + cat "$REPORT_FILE" + } > "$BODY_FILE" + EXISTING=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --paginate -q ".[] | select(.body | startswith(\"$MARKER\")) | .id" | head -n 1) + if [ -n "$EXISTING" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING" \ + -F body=@"$BODY_FILE" > /dev/null + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + -F body=@"$BODY_FILE" > /dev/null + fi + echo "Report commented on PR #$PR_NUMBER." + + - name: Fail on regression + if: steps.compare.outcome == 'failure' && !(github.event_name == 'workflow_dispatch' && inputs.update_baseline) + run: | + echo "::error title=Performance regression::At least one benchmark regressed beyond the fail threshold. See the benchmark report in the job summary." + exit 1 + + notify-regression: + name: Notify maintainers (tag regression) + needs: benchmark + if: >- + always() && + !cancelled() && + needs.benchmark.outputs.compare_outcome == 'failure' && + needs.benchmark.outputs.is_tag == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + issues: write + steps: + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: benchmark-results + path: fesod-benchmark/target/ + + - name: Post report on release (or open an issue) + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.benchmark.outputs.tag }} + run: | + if [ ! -f "$REPORT_FILE" ]; then + echo "No report file โ€” nothing to notify." + exit 0 + fi + BODY_FILE=$(mktemp) + { + echo "## ๐Ÿšจ Performance regression detected on tag \`${TAG}\`" + echo + echo "The [benchmark run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) found regressions beyond the fail threshold. The baseline was **not** advanced." + echo + cat "$REPORT_FILE" + } > "$BODY_FILE" + + # Best-effort notification ladder: release comment -> issue -> warning + # annotation. A missing release or a repository with issues disabled must + # never turn this job red โ€” the gate itself is already red in that case. + notified=1 + RELEASE_ID=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$TAG" -q .id 2>/dev/null || true) + # a failed lookup can leak the error body into the variable โ€” keep numeric ids only + case "$RELEASE_ID" in + ''|*[!0-9]*) RELEASE_ID='' ;; + esac + if [ -n "$RELEASE_ID" ]; then + if gh api -X POST "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/comments" \ + -F body=@"$BODY_FILE" > /dev/null 2>&1; then + echo "Commented on release $TAG." + notified=0 + else + echo "::warning title=Regression notification::Release comment failed for '$TAG' โ€” falling back to an issue." + fi + fi + if [ "$notified" -ne 0 ]; then + if gh issue create --repo "$GITHUB_REPOSITORY" \ + --title "๐Ÿšจ Performance regression on tag ${TAG}" \ + --body-file "$BODY_FILE" --label "performance" 2>/dev/null \ + || gh issue create --repo "$GITHUB_REPOSITORY" \ + --title "๐Ÿšจ Performance regression on tag ${TAG}" \ + --body-file "$BODY_FILE"; then + echo "Opened a tracking issue." + notified=0 + fi + fi + if [ "$notified" -ne 0 ]; then + echo "::warning title=Regression notification::Could not comment on release '$TAG' or open an issue (no release / issues disabled?). The report is in the job summary and run annotations." + fi + + update-baseline: + name: Refresh baseline (opens PR) + needs: benchmark + if: >- + !cancelled() && + needs.benchmark.result == 'success' && + (needs.benchmark.outputs.update_requested == 'true' || + needs.benchmark.outputs.bootstrap == 'true' || + (needs.benchmark.outputs.is_tag == 'true' && needs.benchmark.outputs.compare_outcome == 'success')) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: benchmark-results + path: fesod-benchmark/target/ + + - name: Write baseline files + env: + JDK_INFO: ${{ needs.benchmark.outputs.jdk }} + OS_INFO: ${{ needs.benchmark.outputs.os }} + SOURCE_REF: ${{ needs.benchmark.outputs.is_tag == 'true' && needs.benchmark.outputs.tag || github.ref_name }} + run: | + mkdir -p fesod-benchmark/baseline + cp "$CURRENT_RESULT" "$BASELINE_FILE" + jq -n \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg gitSha "${{ github.sha }}" \ + --arg gitRef "$SOURCE_REF" \ + --arg jdkVersion "$JDK_INFO" \ + --arg osName "$OS_INFO" \ + --arg runnerLabel "ubuntu-24.04" \ + --argjson benchmarkCount "$(jq 'length' "$CURRENT_RESULT")" \ + '{generatedAt: $generatedAt, gitSha: $gitSha, gitRef: $gitRef, jdkVersion: $jdkVersion, + osName: $osName, runnerLabel: $runnerLabel, benchmarkCount: $benchmarkCount}' \ + > "$BASELINE_META" + cat "$BASELINE_META" + + - name: Open baseline refresh PR + env: + GH_TOKEN: ${{ github.token }} + BASE_BRANCH: ${{ needs.benchmark.outputs.is_tag == 'true' && 'main' || github.ref_name }} + REASON: ${{ needs.benchmark.outputs.is_tag == 'true' && format('release tag {0}', needs.benchmark.outputs.tag) || github.ref_name }} + run: | + BRANCH="benchmark/baseline-$(date -u +%Y%m%d%H%M%S)" + PR_BODY_FILE=$(mktemp) + { + echo "## Performance baseline refresh" + echo + echo "Regenerated from \`${{ github.sha }}\` (\`$REASON\`) on a \`ubuntu-24.04\` runner with JDK 17 (Temurin)" + echo "by the [Benchmark workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + echo + if [ -f "$REPORT_FILE" ]; then + echo "### Change vs previous baseline" + echo + cat "$REPORT_FILE" + echo + fi + echo "### Files changed" + echo + echo '- `fesod-benchmark/baseline/jmh-baseline.json` โ€” fresh JMH results, the new reference' + echo '- `fesod-benchmark/baseline/baseline-meta.json` โ€” provenance (commit, runner, JDK, date)' + echo + echo "Merge this PR to accept the new performance characteristics as the project baseline." + } > "$PR_BODY_FILE" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$BRANCH" + git add fesod-benchmark/baseline + git commit -m "chore(benchmark): refresh performance baseline from ${{ github.sha }}" + git push origin "$BRANCH" + + gh pr create \ + --base "$BASE_BRANCH" \ + --head "$BRANCH" \ + --title "chore(benchmark): refresh performance baseline" \ + --body-file "$PR_BODY_FILE" diff --git a/fesod-benchmark/baseline/README.md b/fesod-benchmark/baseline/README.md new file mode 100644 index 000000000..2ae420bac --- /dev/null +++ b/fesod-benchmark/baseline/README.md @@ -0,0 +1,12 @@ +# Committed performance baseline + +This directory holds the reference results for the performance regression gate: + +- `jmh-baseline.json` โ€” JMH results of the last accepted `BaselineBenchmark` run. Generated by the + *Benchmark* workflow; **must only ever be produced on an `ubuntu-24.04` GitHub runner with JDK 17 + (Temurin)**, because comparisons are absolute. Never commit a baseline generated on a local machine. +- `baseline-meta.json` โ€” provenance of the baseline: source commit, JDK, runner, generation date. + +To refresh: *Actions โ†’ Benchmark โ†’ Run workflow* with **update_baseline** checked. The workflow opens +a PR containing the new baseline plus a delta report against the previous one. See +[`../benchmark.md`](../benchmark.md) for the full documentation of the baseline CI. diff --git a/fesod-benchmark/baseline/baseline-meta.json b/fesod-benchmark/baseline/baseline-meta.json new file mode 100644 index 000000000..a611465ef --- /dev/null +++ b/fesod-benchmark/baseline/baseline-meta.json @@ -0,0 +1,9 @@ +{ + "generatedAt": "2026-08-28T12:04:29Z", + "gitSha": "701d850a7d296434553d88076d891893abf0fcdb", + "gitRef": "feat/benchmark-comparison-workflow", + "jdkVersion": "openjdk version \"17.0.20.1\" 2026-08-18", + "osName": "Linux 6.17.0-1022-azure", + "runnerLabel": "ubuntu-24.04", + "benchmarkCount": 8 +} diff --git a/fesod-benchmark/baseline/jmh-baseline.json b/fesod-benchmark/baseline/jmh-baseline.json new file mode 100644 index 000000000..a308e27a6 --- /dev/null +++ b/fesod-benchmark/baseline/jmh-baseline.json @@ -0,0 +1,1988 @@ +[ + { + "jmhVersion" : "1.37", + "benchmark" : "org.apache.fesod.sheet.benchmark.baseline.BaselineBenchmark.read", + "mode" : "avgt", + "threads" : 1, + "forks" : 3, + "jvm" : "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java", + "jvmArgs" : [ + "-Xms1g", + "-Xmx1g", + "-XX:+UseG1GC" + ], + "jdkVersion" : "17.0.20.1", + "vmName" : "OpenJDK 64-Bit Server VM", + "vmVersion" : "17.0.20.1+1", + "warmupIterations" : 3, + "warmupTime" : "1 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "2 s", + "measurementBatchSize" : 1, + "params" : { + "datasetSize" : "SMALL", + "fileFormat" : "XLSX" + }, + "primaryMetric" : { + "score" : 26.31614848782551, + "scoreError" : 1.0237834856784767, + "scoreConfidence" : [ + 25.292365002147037, + 27.339931973503987 + ], + "scorePercentiles" : { + "0.0" : 24.483287829268292, + "50.0" : 26.702622053333332, + "90.0" : 27.618623326179602, + "95.0" : 27.786469027777777, + "99.0" : 27.786469027777777, + "99.9" : 27.786469027777777, + "99.99" : 27.786469027777777, + "99.999" : 27.786469027777777, + "99.9999" : 27.786469027777777, + "100.0" : 27.786469027777777 + }, + "scoreUnit" : "ms/op", + "rawData" : [ + [ + 27.14174727027027, + 26.48419044736842, + 25.1477271125, + 25.2647500125, + 24.483287829268292 + ], + [ + 27.50672619178082, + 26.80692768, + 26.728927986666665, + 26.884241253333332, + 25.604029 + ], + [ + 27.786469027777777, + 26.702622053333332, + 26.857012133333335, + 25.993229610389612, + 25.35033970886076 + ] + ] + }, + "secondaryMetrics" : { + "gc.alloc.rate" : { + "score" : 571.6973889050657, + "scoreError" : 22.12092751016608, + "scoreConfidence" : [ + 549.5764613948996, + 593.8183164152318 + ], + "scorePercentiles" : { + "0.0" : 541.5742290822839, + "50.0" : 563.0563745655405, + "90.0" : 603.3790590065496, + "95.0" : 613.4725922256682, + "99.0" : 613.4725922256682, + "99.9" : 613.4725922256682, + "99.99" : 613.4725922256682, + "99.999" : 613.4725922256682, + "99.9999" : 613.4725922256682, + "100.0" : 613.4725922256682 + }, + "scoreUnit" : "MB/sec", + "rawData" : [ + [ + 554.0851355753813, + 567.2857639813068, + 596.6500368604705, + 594.0309305403248, + 613.4725922256682 + ], + [ + 547.0519839116918, + 560.9801273412597, + 562.1409503433017, + 558.8241247502214, + 586.333033265347 + ], + [ + 541.5742290822839, + 563.0563745655405, + 559.5248171745116, + 577.8707792245253, + 592.5799547341529 + ] + ] + }, + "gc.alloc.rate.norm" : { + "score" : 1.576299083349771E7, + "scoreError" : 11325.494060015397, + "scoreConfidence" : [ + 1.5751665339437695E7, + 1.5774316327557726E7 + ], + "scorePercentiles" : { + "0.0" : 1.5752112878048781E7, + "50.0" : 1.5759685333333334E7, + "90.0" : 1.5781086657534245E7, + "95.0" : 1.5781371E7, + "99.0" : 1.5781371E7, + "99.9" : 1.5781371E7, + "99.99" : 1.5781371E7, + "99.999" : 1.5781371E7, + "99.9999" : 1.5781371E7, + "100.0" : 1.5781371E7 + }, + "scoreUnit" : "B/op", + "rawData" : [ + [ + 1.5778371675675675E7, + 1.576208105263158E7, + 1.57564038E7, + 1.57535218E7, + 1.5752112878048781E7 + ], + [ + 1.578089709589041E7, + 1.5770380266666668E7, + 1.5760099093333334E7, + 1.5755098986666666E7, + 1.575286805063291E7 + ], + [ + 1.5781371E7, + 1.5771230186666667E7, + 1.5759685333333334E7, + 1.5756180675324675E7, + 1.5754560607594937E7 + ] + ] + }, + "gc.count" : { + "score" : 27.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 27.0, + 27.0 + ], + "scorePercentiles" : { + "0.0" : 1.0, + "50.0" : 2.0, + "90.0" : 2.0, + "95.0" : 2.0, + "99.0" : 2.0, + "99.9" : 2.0, + "99.99" : 2.0, + "99.999" : 2.0, + "99.9999" : 2.0, + "100.0" : 2.0 + }, + "scoreUnit" : "counts", + "rawData" : [ + [ + 2.0, + 1.0, + 2.0, + 2.0, + 2.0 + ], + [ + 1.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + [ + 1.0, + 2.0, + 2.0, + 2.0, + 2.0 + ] + ] + }, + "gc.time" : { + "score" : 127.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 127.0, + 127.0 + ], + "scorePercentiles" : { + "0.0" : 4.0, + "50.0" : 9.0, + "90.0" : 10.8, + "95.0" : 12.0, + "99.0" : 12.0, + "99.9" : 12.0, + "99.99" : 12.0, + "99.999" : 12.0, + "99.9999" : 12.0, + "100.0" : 12.0 + }, + "scoreUnit" : "ms", + "rawData" : [ + [ + 10.0, + 6.0, + 9.0, + 9.0, + 9.0 + ], + [ + 4.0, + 9.0, + 12.0, + 9.0, + 9.0 + ], + [ + 5.0, + 9.0, + 10.0, + 9.0, + 8.0 + ] + ] + } + } + }, + { + "jmhVersion" : "1.37", + "benchmark" : "org.apache.fesod.sheet.benchmark.baseline.BaselineBenchmark.read", + "mode" : "avgt", + "threads" : 1, + "forks" : 3, + "jvm" : "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java", + "jvmArgs" : [ + "-Xms1g", + "-Xmx1g", + "-XX:+UseG1GC" + ], + "jdkVersion" : "17.0.20.1", + "vmName" : "OpenJDK 64-Bit Server VM", + "vmVersion" : "17.0.20.1+1", + "warmupIterations" : 3, + "warmupTime" : "1 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "2 s", + "measurementBatchSize" : 1, + "params" : { + "datasetSize" : "SMALL", + "fileFormat" : "CSV" + }, + "primaryMetric" : { + "score" : 7.573984984620764, + "scoreError" : 0.22100813735344543, + "scoreConfidence" : [ + 7.3529768472673185, + 7.794993121974209 + ], + "scorePercentiles" : { + "0.0" : 7.276923076363636, + "50.0" : 7.553406132075471, + "90.0" : 7.92170963128588, + "95.0" : 7.975900788844622, + "99.0" : 7.975900788844622, + "99.9" : 7.975900788844622, + "99.99" : 7.975900788844622, + "99.999" : 7.975900788844622, + "99.9999" : 7.975900788844622, + "100.0" : 7.975900788844622 + }, + "scoreUnit" : "ms/op", + "rawData" : [ + [ + 7.885582192913386, + 7.553406132075471, + 7.636755232824427, + 7.402206841328414, + 7.281750276363637 + ], + [ + 7.629906357414448, + 7.6158007148288975, + 7.499480018726592, + 7.276923076363636, + 7.368510102941176 + ], + [ + 7.975900788844622, + 7.784489027237354, + 7.717373730769231, + 7.470376063197026, + 7.511314213483146 + ] + ] + }, + "secondaryMetrics" : { + "gc.alloc.rate" : { + "score" : 1187.7600436276796, + "scoreError" : 34.28200695386358, + "scoreConfidence" : [ + 1153.478036673816, + 1222.0420505815432 + ], + "scorePercentiles" : { + "0.0" : 1127.4025116501, + "50.0" : 1190.1037863226231, + "90.0" : 1234.8201456232077, + "95.0" : 1235.176378876526, + "99.0" : 1235.176378876526, + "99.9" : 1235.176378876526, + "99.99" : 1235.176378876526, + "99.999" : 1235.176378876526, + "99.9999" : 1235.176378876526, + "100.0" : 1235.176378876526 + }, + "scoreUnit" : "MB/sec", + "rawData" : [ + [ + 1140.3950313722642, + 1190.1037863226231, + 1176.54707801321, + 1214.5829324857411, + 1234.582656787662 + ], + [ + 1178.5451593366547, + 1180.5410212309107, + 1198.562683845547, + 1235.176378876526, + 1220.0314666104857 + ], + [ + 1127.4025116501, + 1154.9643544710018, + 1164.8543933612984, + 1203.286140852886, + 1196.8250591982849 + ] + ] + }, + "gc.alloc.rate.norm" : { + "score" : 9428950.024588982, + "scoreError" : 788.1164486867643, + "scoreConfidence" : [ + 9428161.908140294, + 9429738.141037669 + ], + "scorePercentiles" : { + "0.0" : 9428371.911764706, + "50.0" : 9428558.861538462, + "90.0" : 9430283.206610579, + "95.0" : 9430575.149606299, + "99.0" : 9430575.149606299, + "99.9" : 9430575.149606299, + "99.99" : 9430575.149606299, + "99.999" : 9430575.149606299, + "99.9999" : 9430575.149606299, + "100.0" : 9430575.149606299 + }, + "scoreUnit" : "B/op", + "rawData" : [ + [ + 9430575.149606299, + 9429186.203773584, + 9428569.954198474, + 9428410.27306273, + 9428404.48 + ], + [ + 9430088.577946767, + 9429137.490494296, + 9428522.307116104, + 9428394.472727273, + 9428371.911764706 + ], + [ + 9430057.689243028, + 9429164.046692608, + 9428558.861538462, + 9428413.085501859, + 9428395.86516854 + ] + ] + }, + "gc.count" : { + "score" : 60.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 60.0, + 60.0 + ], + "scorePercentiles" : { + "0.0" : 4.0, + "50.0" : 4.0, + "90.0" : 4.0, + "95.0" : 4.0, + "99.0" : 4.0, + "99.9" : 4.0, + "99.99" : 4.0, + "99.999" : 4.0, + "99.9999" : 4.0, + "100.0" : 4.0 + }, + "scoreUnit" : "counts", + "rawData" : [ + [ + 4.0, + 4.0, + 4.0, + 4.0, + 4.0 + ], + [ + 4.0, + 4.0, + 4.0, + 4.0, + 4.0 + ], + [ + 4.0, + 4.0, + 4.0, + 4.0, + 4.0 + ] + ] + }, + "gc.time" : { + "score" : 113.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 113.0, + 113.0 + ], + "scorePercentiles" : { + "0.0" : 6.0, + "50.0" : 7.0, + "90.0" : 9.0, + "95.0" : 9.0, + "99.0" : 9.0, + "99.9" : 9.0, + "99.99" : 9.0, + "99.999" : 9.0, + "99.9999" : 9.0, + "100.0" : 9.0 + }, + "scoreUnit" : "ms", + "rawData" : [ + [ + 8.0, + 8.0, + 8.0, + 6.0, + 8.0 + ], + [ + 7.0, + 9.0, + 8.0, + 7.0, + 7.0 + ], + [ + 7.0, + 7.0, + 9.0, + 7.0, + 7.0 + ] + ] + } + } + }, + { + "jmhVersion" : "1.37", + "benchmark" : "org.apache.fesod.sheet.benchmark.baseline.BaselineBenchmark.read", + "mode" : "avgt", + "threads" : 1, + "forks" : 3, + "jvm" : "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java", + "jvmArgs" : [ + "-Xms1g", + "-Xmx1g", + "-XX:+UseG1GC" + ], + "jdkVersion" : "17.0.20.1", + "vmName" : "OpenJDK 64-Bit Server VM", + "vmVersion" : "17.0.20.1+1", + "warmupIterations" : 3, + "warmupTime" : "1 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "2 s", + "measurementBatchSize" : 1, + "params" : { + "datasetSize" : "MEDIUM", + "fileFormat" : "XLSX" + }, + "primaryMetric" : { + "score" : 230.7608427925926, + "scoreError" : 4.352993650581027, + "scoreConfidence" : [ + 226.40784914201157, + 235.11383644317362 + ], + "scorePercentiles" : { + "0.0" : 225.51328455555554, + "50.0" : 230.52625166666667, + "90.0" : 237.27261755555554, + "95.0" : 240.85040922222223, + "99.0" : 240.85040922222223, + "99.9" : 240.85040922222223, + "99.99" : 240.85040922222223, + "99.999" : 240.85040922222223, + "99.9999" : 240.85040922222223, + "100.0" : 240.85040922222223 + }, + "scoreUnit" : "ms/op", + "rawData" : [ + [ + 233.52516533333332, + 227.84245833333333, + 229.13108133333333, + 229.09750166666666, + 226.97711577777778 + ], + [ + 240.85040922222223, + 234.36350344444443, + 233.01068511111112, + 230.77766555555556, + 230.52625166666667 + ], + [ + 234.8874231111111, + 231.21281966666666, + 225.51328455555554, + 227.53383455555556, + 226.16344255555555 + ] + ] + }, + "secondaryMetrics" : { + "gc.alloc.rate" : { + "score" : 603.9010472722862, + "scoreError" : 11.374791793983238, + "scoreConfidence" : [ + 592.5262554783029, + 615.2758390662694 + ], + "scorePercentiles" : { + "0.0" : 578.0895807747223, + "50.0" : 603.6041824252029, + "90.0" : 616.0620493864723, + "95.0" : 617.2555180691709, + "99.0" : 617.2555180691709, + "99.9" : 617.2555180691709, + "99.99" : 617.2555180691709, + "99.999" : 617.2555180691709, + "99.9999" : 617.2555180691709, + "100.0" : 617.2555180691709 + }, + "scoreUnit" : "MB/sec", + "rawData" : [ + [ + 597.8322557515444, + 612.639181437172, + 609.1972207814339, + 608.9858205419808, + 614.555521700298 + ], + [ + 578.0895807747223, + 593.9680473523963, + 597.4041198079241, + 603.1715568271837, + 603.6041824252029 + ], + [ + 592.7503819393398, + 602.0614806951012, + 617.2555180691709, + 611.734437382816, + 615.2664035980065 + ] + ] + }, + "gc.alloc.rate.norm" : { + "score" : 1.4611673048888886E8, + "scoreError" : 209999.71099373346, + "scoreConfidence" : [ + 1.4590673077789512E8, + 1.463267301998826E8 + ], + "scorePercentiles" : { + "0.0" : 1.459679431111111E8, + "50.0" : 1.459896382222222E8, + "90.0" : 1.4639279946666667E8, + "95.0" : 1.4640863466666666E8, + "99.0" : 1.4640863466666666E8, + "99.9" : 1.4640863466666666E8, + "99.99" : 1.4640863466666666E8, + "99.999" : 1.4640863466666666E8, + "99.9999" : 1.4640863466666666E8, + "100.0" : 1.4640863466666666E8 + }, + "scoreUnit" : "B/op", + "rawData" : [ + [ + 1.4640863466666666E8, + 1.4638224266666666E8, + 1.4638220444444445E8, + 1.4637490666666666E8, + 1.463738391111111E8 + ], + [ + 1.4601157333333334E8, + 1.459798897777778E8, + 1.459800542222222E8, + 1.4597450133333334E8, + 1.459716168888889E8 + ], + [ + 1.460084968888889E8, + 1.459896382222222E8, + 1.459763271111111E8, + 1.459690888888889E8, + 1.459679431111111E8 + ] + ] + }, + "gc.count" : { + "score" : 30.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 30.0, + 30.0 + ], + "scorePercentiles" : { + "0.0" : 2.0, + "50.0" : 2.0, + "90.0" : 2.0, + "95.0" : 2.0, + "99.0" : 2.0, + "99.9" : 2.0, + "99.99" : 2.0, + "99.999" : 2.0, + "99.9999" : 2.0, + "100.0" : 2.0 + }, + "scoreUnit" : "counts", + "rawData" : [ + [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ], + [ + 2.0, + 2.0, + 2.0, + 2.0, + 2.0 + ] + ] + }, + "gc.time" : { + "score" : 134.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 134.0, + 134.0 + ], + "scorePercentiles" : { + "0.0" : 8.0, + "50.0" : 9.0, + "90.0" : 10.0, + "95.0" : 10.0, + "99.0" : 10.0, + "99.9" : 10.0, + "99.99" : 10.0, + "99.999" : 10.0, + "99.9999" : 10.0, + "100.0" : 10.0 + }, + "scoreUnit" : "ms", + "rawData" : [ + [ + 8.0, + 9.0, + 10.0, + 8.0, + 10.0 + ], + [ + 10.0, + 8.0, + 8.0, + 9.0, + 8.0 + ], + [ + 9.0, + 10.0, + 9.0, + 9.0, + 9.0 + ] + ] + } + } + }, + { + "jmhVersion" : "1.37", + "benchmark" : "org.apache.fesod.sheet.benchmark.baseline.BaselineBenchmark.read", + "mode" : "avgt", + "threads" : 1, + "forks" : 3, + "jvm" : "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java", + "jvmArgs" : [ + "-Xms1g", + "-Xmx1g", + "-XX:+UseG1GC" + ], + "jdkVersion" : "17.0.20.1", + "vmName" : "OpenJDK 64-Bit Server VM", + "vmVersion" : "17.0.20.1+1", + "warmupIterations" : 3, + "warmupTime" : "1 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "2 s", + "measurementBatchSize" : 1, + "params" : { + "datasetSize" : "MEDIUM", + "fileFormat" : "CSV" + }, + "primaryMetric" : { + "score" : 65.014355925, + "scoreError" : 1.671715792303186, + "scoreConfidence" : [ + 63.342640132696815, + 66.6860717173032 + ], + "scorePercentiles" : { + "0.0" : 63.57516996875, + "50.0" : 64.26653040625, + "90.0" : 67.21777310666667, + "95.0" : 67.27451206666667, + "99.0" : 67.27451206666667, + "99.9" : 67.27451206666667, + "99.99" : 67.27451206666667, + "99.999" : 67.27451206666667, + "99.9999" : 67.27451206666667, + "100.0" : 67.27451206666667 + }, + "scoreUnit" : "ms/op", + "rawData" : [ + [ + 64.3681089375, + 64.4112483125, + 63.952598625, + 63.69756571875, + 63.78424034375 + ], + [ + 64.26653040625, + 63.970613, + 63.5822404375, + 63.997815625, + 63.57516996875 + ], + [ + 67.27451206666667, + 67.11997923333334, + 66.9245421, + 67.17994713333333, + 67.11022696666667 + ] + ] + }, + "secondaryMetrics" : { + "gc.alloc.rate" : { + "score" : 1374.558705266136, + "scoreError" : 32.919207607009604, + "scoreConfidence" : [ + 1341.6394976591264, + 1407.4779128731457 + ], + "scorePercentiles" : { + "0.0" : 1330.9142410856707, + "50.0" : 1390.5298413059563, + "90.0" : 1408.0134624609586, + "95.0" : 1408.1417142668165, + "99.0" : 1408.1417142668165, + "99.9" : 1408.1417142668165, + "99.99" : 1408.1417142668165, + "99.999" : 1408.1417142668165, + "99.9999" : 1408.1417142668165, + "100.0" : 1408.1417142668165 + }, + "scoreUnit" : "MB/sec", + "rawData" : [ + [ + 1381.5454668838995, + 1379.8833985633466, + 1390.5298413059563, + 1395.5112799514955, + 1393.7408677001288 + ], + [ + 1393.2252162956438, + 1399.674801937291, + 1408.1417142668165, + 1399.026263634565, + 1407.9279612570535 + ], + [ + 1330.9142410856707, + 1334.002164624659, + 1337.6220733574587, + 1332.8037265823787, + 1333.8315615456768 + ] + ] + }, + "gc.alloc.rate.norm" : { + "score" : 9.368377974666667E7, + "scoreError" : 333857.9676189589, + "scoreConfidence" : [ + 9.334992177904771E7, + 9.401763771428563E7 + ], + "scorePercentiles" : { + "0.0" : 9.325626825E7, + "50.0" : 9.389642125E7, + "90.0" : 9.389762535E7, + "95.0" : 9.38976345E7, + "99.0" : 9.38976345E7, + "99.9" : 9.38976345E7, + "99.99" : 9.38976345E7, + "99.999" : 9.38976345E7, + "99.9999" : 9.38976345E7, + "100.0" : 9.38976345E7 + }, + "scoreUnit" : "B/op", + "rawData" : [ + [ + 9.325758775E7, + 9.32576035E7, + 9.325758625E7, + 9.325650375E7, + 9.325626825E7 + ], + [ + 9.389760725E7, + 9.38976345E7, + 9.389761925E7, + 9.389642125E7, + 9.389630125E7 + ], + [ + 9.38975448E7, + 9.389758133333333E7, + 9.389753573333333E7, + 9.38966664E7, + 9.389623493333334E7 + ] + ] + }, + "gc.count" : { + "score" : 68.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 68.0, + 68.0 + ], + "scorePercentiles" : { + "0.0" : 4.0, + "50.0" : 5.0, + "90.0" : 5.0, + "95.0" : 5.0, + "99.0" : 5.0, + "99.9" : 5.0, + "99.99" : 5.0, + "99.999" : 5.0, + "99.9999" : 5.0, + "100.0" : 5.0 + }, + "scoreUnit" : "counts", + "rawData" : [ + [ + 4.0, + 5.0, + 5.0, + 4.0, + 5.0 + ], + [ + 4.0, + 5.0, + 5.0, + 4.0, + 5.0 + ], + [ + 5.0, + 4.0, + 5.0, + 4.0, + 4.0 + ] + ] + }, + "gc.time" : { + "score" : 93.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 93.0, + 93.0 + ], + "scorePercentiles" : { + "0.0" : 4.0, + "50.0" : 6.0, + "90.0" : 8.4, + "95.0" : 9.0, + "99.0" : 9.0, + "99.9" : 9.0, + "99.99" : 9.0, + "99.999" : 9.0, + "99.9999" : 9.0, + "100.0" : 9.0 + }, + "scoreUnit" : "ms", + "rawData" : [ + [ + 7.0, + 7.0, + 7.0, + 5.0, + 6.0 + ], + [ + 6.0, + 9.0, + 6.0, + 5.0, + 5.0 + ], + [ + 8.0, + 6.0, + 7.0, + 5.0, + 4.0 + ] + ] + } + } + }, + { + "jmhVersion" : "1.37", + "benchmark" : "org.apache.fesod.sheet.benchmark.baseline.BaselineBenchmark.write", + "mode" : "avgt", + "threads" : 1, + "forks" : 3, + "jvm" : "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java", + "jvmArgs" : [ + "-Xms1g", + "-Xmx1g", + "-XX:+UseG1GC" + ], + "jdkVersion" : "17.0.20.1", + "vmName" : "OpenJDK 64-Bit Server VM", + "vmVersion" : "17.0.20.1+1", + "warmupIterations" : 3, + "warmupTime" : "1 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "2 s", + "measurementBatchSize" : 1, + "params" : { + "datasetSize" : "SMALL", + "fileFormat" : "XLSX" + }, + "primaryMetric" : { + "score" : 50.92110620130453, + "scoreError" : 2.680095051647972, + "scoreConfidence" : [ + 48.24101114965656, + 53.60120125295251 + ], + "scorePercentiles" : { + "0.0" : 47.724122, + "50.0" : 50.81626665, + "90.0" : 56.09068195, + "95.0" : 56.38666925, + "99.0" : 56.38666925, + "99.9" : 56.38666925, + "99.99" : 56.38666925, + "99.999" : 56.38666925, + "99.9999" : 56.38666925, + "100.0" : 56.38666925 + }, + "scoreUnit" : "ms/op", + "rawData" : [ + [ + 51.32333258974359, + 51.64395969230769, + 48.78790016666667, + 48.03560773809524, + 47.724122 + ], + [ + 56.38666925, + 55.893357083333335, + 50.24622585, + 50.837620925, + 51.48688484615385 + ], + [ + 52.30991217948718, + 49.89321036585366, + 50.81626665, + 49.127900365853655, + 49.30362331707317 + ] + ] + }, + "secondaryMetrics" : { + "gc.alloc.rate" : { + "score" : 421.9506242227614, + "scoreError" : 21.249624114651386, + "scoreConfidence" : [ + 400.70100010811, + 443.2002483374128 + ], + "scorePercentiles" : { + "0.0" : 380.09059186310986, + "50.0" : 421.7634365035191, + "90.0" : 447.4270580967219, + "95.0" : 449.11271835280155, + "99.0" : 449.11271835280155, + "99.9" : 449.11271835280155, + "99.99" : 449.11271835280155, + "99.999" : 449.11271835280155, + "99.9999" : 449.11271835280155, + "100.0" : 449.11271835280155 + }, + "scoreUnit" : "MB/sec", + "rawData" : [ + [ + 418.1157464576622, + 415.3145145333655, + 439.2882523839621, + 446.3032845926688, + 449.11271835280155 + ], + [ + 380.09059186310986, + 383.70191706574275, + 426.5777743896502, + 421.708841374545, + 416.29351986015894 + ], + [ + 410.092577217034, + 429.8946210664017, + 421.7634365035191, + 436.35131656777327, + 434.6502511130262 + ] + ] + }, + "gc.alloc.rate.norm" : { + "score" : 2.248978019077995E7, + "scoreError" : 9956.5426155935, + "scoreConfidence" : [ + 2.2479823648164358E7, + 2.2499736733395543E7 + ], + "scorePercentiles" : { + "0.0" : 2.2478518243902437E7, + "50.0" : 2.248800476190476E7, + "90.0" : 2.2505000232478634E7, + "95.0" : 2.2505736888888888E7, + "99.0" : 2.2505736888888888E7, + "99.9" : 2.2505736888888888E7, + "99.99" : 2.2505736888888888E7, + "99.999" : 2.2505736888888888E7, + "99.9999" : 2.2505736888888888E7, + "100.0" : 2.2505736888888888E7 + }, + "scoreUnit" : "B/op", + "rawData" : [ + [ + 2.2504088410256412E7, + 2.2493517128205128E7, + 2.248800476190476E7, + 2.2483082666666668E7, + 2.247910419047619E7 + ], + [ + 2.2505736888888888E7, + 2.2494005111111112E7, + 2.24891446E7, + 2.24831664E7, + 2.2479532307692308E7 + ], + [ + 2.2504509128205128E7, + 2.2493754926829267E7, + 2.2487694E7, + 2.2482844097560976E7, + 2.2478518243902437E7 + ] + ] + }, + "gc.count" : { + "score" : 20.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 20.0, + 20.0 + ], + "scorePercentiles" : { + "0.0" : 1.0, + "50.0" : 1.0, + "90.0" : 2.0, + "95.0" : 2.0, + "99.0" : 2.0, + "99.9" : 2.0, + "99.99" : 2.0, + "99.999" : 2.0, + "99.9999" : 2.0, + "100.0" : 2.0 + }, + "scoreUnit" : "counts", + "rawData" : [ + [ + 1.0, + 1.0, + 2.0, + 1.0, + 2.0 + ], + [ + 1.0, + 1.0, + 2.0, + 1.0, + 1.0 + ], + [ + 1.0, + 2.0, + 1.0, + 1.0, + 2.0 + ] + ] + }, + "gc.time" : { + "score" : 105.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 105.0, + 105.0 + ], + "scorePercentiles" : { + "0.0" : 4.0, + "50.0" : 5.0, + "90.0" : 11.4, + "95.0" : 12.0, + "99.0" : 12.0, + "99.9" : 12.0, + "99.99" : 12.0, + "99.999" : 12.0, + "99.9999" : 12.0, + "100.0" : 12.0 + }, + "scoreUnit" : "ms", + "rawData" : [ + [ + 5.0, + 7.0, + 11.0, + 5.0, + 12.0 + ], + [ + 6.0, + 4.0, + 11.0, + 5.0, + 5.0 + ], + [ + 5.0, + 11.0, + 4.0, + 5.0, + 9.0 + ] + ] + } + } + }, + { + "jmhVersion" : "1.37", + "benchmark" : "org.apache.fesod.sheet.benchmark.baseline.BaselineBenchmark.write", + "mode" : "avgt", + "threads" : 1, + "forks" : 3, + "jvm" : "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java", + "jvmArgs" : [ + "-Xms1g", + "-Xmx1g", + "-XX:+UseG1GC" + ], + "jdkVersion" : "17.0.20.1", + "vmName" : "OpenJDK 64-Bit Server VM", + "vmVersion" : "17.0.20.1+1", + "warmupIterations" : 3, + "warmupTime" : "1 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "2 s", + "measurementBatchSize" : 1, + "params" : { + "datasetSize" : "SMALL", + "fileFormat" : "CSV" + }, + "primaryMetric" : { + "score" : 9.87351593972813, + "scoreError" : 0.3159889793358319, + "scoreConfidence" : [ + 9.557526960392298, + 10.18950491906396 + ], + "scorePercentiles" : { + "0.0" : 9.518329132701421, + "50.0" : 9.720665033980582, + "90.0" : 10.312859259499827, + "95.0" : 10.484831717277487, + "99.0" : 10.484831717277487, + "99.9" : 10.484831717277487, + "99.99" : 10.484831717277487, + "99.999" : 10.484831717277487, + "99.9999" : 10.484831717277487, + "100.0" : 10.484831717277487 + }, + "scoreUnit" : "ms/op", + "rawData" : [ + [ + 10.104389914141414, + 10.141667752525253, + 9.681803768115943, + 10.484831717277487, + 9.975097119402985 + ], + [ + 10.198210954314721, + 10.15323509090909, + 9.717541339805825, + 9.720665033980582, + 9.570860785714286 + ], + [ + 10.002426795, + 9.644231740384615, + 9.635767923076923, + 9.553680028571428, + 9.518329132701421 + ] + ] + }, + "secondaryMetrics" : { + "gc.alloc.rate" : { + "score" : 1744.01717026776, + "scoreError" : 57.18785006247117, + "scoreConfidence" : [ + 1686.829320205289, + 1801.2050203302313 + ], + "scorePercentiles" : { + "0.0" : 1638.8586424963707, + "50.0" : 1765.5087572672462, + "90.0" : 1809.7245464888258, + "95.0" : 1813.5152108256518, + "99.0" : 1813.5152108256518, + "99.9" : 1813.5152108256518, + "99.99" : 1813.5152108256518, + "99.999" : 1813.5152108256518, + "99.9999" : 1813.5152108256518, + "100.0" : 1813.5152108256518 + }, + "scoreUnit" : "MB/sec", + "rawData" : [ + [ + 1701.1155871719711, + 1695.1018866391764, + 1773.85117954193, + 1638.8586424963707, + 1722.8040749509182 + ], + [ + 1683.5057981809593, + 1690.9482136118875, + 1766.5707174464756, + 1765.5087572672462, + 1793.1708204784288 + ], + [ + 1726.0494929689035, + 1790.44344384971, + 1791.6162916558296, + 1807.1974369309419, + 1813.5152108256518 + ] + ] + }, + "gc.alloc.rate.norm" : { + "score" : 1.804528926538492E7, + "scoreError" : 49660.735795691115, + "scoreConfidence" : [ + 1.799562852958923E7, + 1.809495000118061E7 + ], + "scorePercentiles" : { + "0.0" : 1.799916015238095E7, + "50.0" : 1.8026876560386475E7, + "90.0" : 1.8108782192923076E7, + "95.0" : 1.810903604E7, + "99.0" : 1.810903604E7, + "99.9" : 1.810903604E7, + "99.99" : 1.810903604E7, + "99.999" : 1.810903604E7, + "99.9999" : 1.810903604E7, + "100.0" : 1.810903604E7 + }, + "scoreUnit" : "B/op", + "rawData" : [ + [ + 1.802904905050505E7, + 1.802869983838384E7, + 1.8026876560386475E7, + 1.802421252356021E7, + 1.8023600676616915E7 + ], + [ + 1.8005050152284265E7, + 1.8004683515151516E7, + 1.8002671922330096E7, + 1.7999758796116505E7, + 1.799916015238095E7 + ], + [ + 1.810903604E7, + 1.810861296153846E7, + 1.810803803846154E7, + 1.8106310895238094E7, + 1.8103577857819904E7 + ] + ] + }, + "gc.count" : { + "score" : 86.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 86.0, + 86.0 + ], + "scorePercentiles" : { + "0.0" : 5.0, + "50.0" : 6.0, + "90.0" : 6.0, + "95.0" : 6.0, + "99.0" : 6.0, + "99.9" : 6.0, + "99.99" : 6.0, + "99.999" : 6.0, + "99.9999" : 6.0, + "100.0" : 6.0 + }, + "scoreUnit" : "counts", + "rawData" : [ + [ + 5.0, + 6.0, + 6.0, + 5.0, + 6.0 + ], + [ + 6.0, + 5.0, + 6.0, + 6.0, + 6.0 + ], + [ + 6.0, + 6.0, + 5.0, + 6.0, + 6.0 + ] + ] + }, + "gc.time" : { + "score" : 141.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 141.0, + 141.0 + ], + "scorePercentiles" : { + "0.0" : 7.0, + "50.0" : 9.0, + "90.0" : 11.4, + "95.0" : 12.0, + "99.0" : 12.0, + "99.9" : 12.0, + "99.99" : 12.0, + "99.999" : 12.0, + "99.9999" : 12.0, + "100.0" : 12.0 + }, + "scoreUnit" : "ms", + "rawData" : [ + [ + 8.0, + 10.0, + 8.0, + 9.0, + 9.0 + ], + [ + 9.0, + 9.0, + 9.0, + 11.0, + 9.0 + ], + [ + 10.0, + 12.0, + 7.0, + 10.0, + 11.0 + ] + ] + } + } + }, + { + "jmhVersion" : "1.37", + "benchmark" : "org.apache.fesod.sheet.benchmark.baseline.BaselineBenchmark.write", + "mode" : "avgt", + "threads" : 1, + "forks" : 3, + "jvm" : "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java", + "jvmArgs" : [ + "-Xms1g", + "-Xmx1g", + "-XX:+UseG1GC" + ], + "jdkVersion" : "17.0.20.1", + "vmName" : "OpenJDK 64-Bit Server VM", + "vmVersion" : "17.0.20.1+1", + "warmupIterations" : 3, + "warmupTime" : "1 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "2 s", + "measurementBatchSize" : 1, + "params" : { + "datasetSize" : "MEDIUM", + "fileFormat" : "XLSX" + }, + "primaryMetric" : { + "score" : 453.2865121866666, + "scoreError" : 13.51285787022506, + "scoreConfidence" : [ + 439.77365431644154, + 466.79937005689163 + ], + "scorePercentiles" : { + "0.0" : 439.4410048, + "50.0" : 446.218076, + "90.0" : 474.13464963999996, + "95.0" : 474.770293, + "99.0" : 474.770293, + "99.9" : 474.770293, + "99.99" : 474.770293, + "99.999" : 474.770293, + "99.9999" : 474.770293, + "100.0" : 474.770293 + }, + "scoreUnit" : "ms/op", + "rawData" : [ + [ + 455.5191974, + 445.7139904, + 442.2125374, + 439.4410048, + 443.4386324 + ], + [ + 473.7108874, + 474.770293, + 468.5277748, + 465.6975506, + 464.7233234 + ], + [ + 449.3163144, + 445.0655172, + 442.5839132, + 446.218076, + 442.3586704 + ] + ] + }, + "secondaryMetrics" : { + "gc.alloc.rate" : { + "score" : 438.58697224495285, + "scoreError" : 12.594108664364986, + "scoreConfidence" : [ + 425.9928635805879, + 451.1810809093178 + ], + "scorePercentiles" : { + "0.0" : 418.933930681255, + "50.0" : 445.08700881850456, + "90.0" : 450.1567101947839, + "95.0" : 451.95783921308254, + "99.0" : 451.95783921308254, + "99.9" : 451.95783921308254, + "99.99" : 451.95783921308254, + "99.999" : 451.95783921308254, + "99.9999" : 451.95783921308254, + "100.0" : 451.95783921308254 + }, + "scoreUnit" : "MB/sec", + "rawData" : [ + [ + 436.010146614666, + 445.4400534895038, + 448.95595751591816, + 451.95783921308254, + 447.7709493563588 + ], + [ + 419.7507575470354, + 418.933930681255, + 424.368189803136, + 426.8835214150891, + 427.7519183001302 + ], + [ + 442.0326424537865, + 446.37510469781836, + 448.6401138911782, + 445.08700881850456, + 448.8464498768307 + ] + ] + }, + "gc.alloc.rate.norm" : { + "score" : 2.0837170613333333E8, + "scoreError" : 128001.13619416619, + "scoreConfidence" : [ + 2.0824370499713916E8, + 2.084997072695275E8 + ], + "scorePercentiles" : { + "0.0" : 2.082788848E8, + "50.0" : 2.082809712E8, + "90.0" : 2.0854452896E8, + "95.0" : 2.08580048E8, + "99.0" : 2.08580048E8, + "99.9" : 2.08580048E8, + "99.99" : 2.08580048E8, + "99.999" : 2.08580048E8, + "99.9999" : 2.08580048E8, + "100.0" : 2.08580048E8 + }, + "scoreUnit" : "B/op", + "rawData" : [ + [ + 2.082809424E8, + 2.083399504E8, + 2.08279232E8, + 2.08279016E8, + 2.082788848E8 + ], + [ + 2.085208496E8, + 2.08580048E8, + 2.085193888E8, + 2.085189728E8, + 2.085191104E8 + ], + [ + 2.082809712E8, + 2.083402144E8, + 2.082794096E8, + 2.082792928E8, + 2.082793088E8 + ] + ] + }, + "gc.count" : { + "score" : 24.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 24.0, + 24.0 + ], + "scorePercentiles" : { + "0.0" : 1.0, + "50.0" : 2.0, + "90.0" : 2.0, + "95.0" : 2.0, + "99.0" : 2.0, + "99.9" : 2.0, + "99.99" : 2.0, + "99.999" : 2.0, + "99.9999" : 2.0, + "100.0" : 2.0 + }, + "scoreUnit" : "counts", + "rawData" : [ + [ + 1.0, + 2.0, + 1.0, + 2.0, + 2.0 + ], + [ + 1.0, + 2.0, + 1.0, + 2.0, + 2.0 + ], + [ + 1.0, + 2.0, + 1.0, + 2.0, + 2.0 + ] + ] + }, + "gc.time" : { + "score" : 110.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 110.0, + 110.0 + ], + "scorePercentiles" : { + "0.0" : 4.0, + "50.0" : 8.0, + "90.0" : 9.4, + "95.0" : 10.0, + "99.0" : 10.0, + "99.9" : 10.0, + "99.99" : 10.0, + "99.999" : 10.0, + "99.9999" : 10.0, + "100.0" : 10.0 + }, + "scoreUnit" : "ms", + "rawData" : [ + [ + 5.0, + 8.0, + 5.0, + 9.0, + 10.0 + ], + [ + 6.0, + 8.0, + 5.0, + 9.0, + 9.0 + ], + [ + 5.0, + 9.0, + 4.0, + 9.0, + 9.0 + ] + ] + } + } + }, + { + "jmhVersion" : "1.37", + "benchmark" : "org.apache.fesod.sheet.benchmark.baseline.BaselineBenchmark.write", + "mode" : "avgt", + "threads" : 1, + "forks" : 3, + "jvm" : "/usr/lib/jvm/temurin-17-jdk-amd64/bin/java", + "jvmArgs" : [ + "-Xms1g", + "-Xmx1g", + "-XX:+UseG1GC" + ], + "jdkVersion" : "17.0.20.1", + "vmName" : "OpenJDK 64-Bit Server VM", + "vmVersion" : "17.0.20.1+1", + "warmupIterations" : 3, + "warmupTime" : "1 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "2 s", + "measurementBatchSize" : 1, + "params" : { + "datasetSize" : "MEDIUM", + "fileFormat" : "CSV" + }, + "primaryMetric" : { + "score" : 88.76531810144928, + "scoreError" : 1.350192936663513, + "scoreConfidence" : [ + 87.41512516478576, + 90.1155110381128 + ], + "scorePercentiles" : { + "0.0" : 86.99922213043479, + "50.0" : 89.02014408695652, + "90.0" : 90.47426250434783, + "95.0" : 90.62376591304348, + "99.0" : 90.62376591304348, + "99.9" : 90.62376591304348, + "99.99" : 90.62376591304348, + "99.999" : 90.62376591304348, + "99.9999" : 90.62376591304348, + "100.0" : 90.62376591304348 + }, + "scoreUnit" : "ms/op", + "rawData" : [ + [ + 90.10029834782608, + 89.27804286956521, + 89.61417265217392, + 90.3745935652174, + 89.02014408695652 + ], + [ + 87.0570312173913, + 86.99922213043479, + 87.59747839130435, + 87.21823986956522, + 87.08750113043479 + ], + [ + 89.58155530434783, + 88.99495439130435, + 88.81200895652174, + 89.12076269565218, + 90.62376591304348 + ] + ] + }, + "secondaryMetrics" : { + "gc.alloc.rate" : { + "score" : 1936.0354358928537, + "scoreError" : 30.293825829864893, + "scoreConfidence" : [ + 1905.7416100629887, + 1966.3292617227187 + ], + "scorePercentiles" : { + "0.0" : 1896.497910140058, + "50.0" : 1928.8647099957707, + "90.0" : 1975.1856891599593, + "95.0" : 1975.9432475260974, + "99.0" : 1975.9432475260974, + "99.9" : 1975.9432475260974, + "99.99" : 1975.9432475260974, + "99.999" : 1975.9432475260974, + "99.9999" : 1975.9432475260974, + "100.0" : 1975.9432475260974 + }, + "scoreUnit" : "MB/sec", + "rawData" : [ + [ + 1905.4218861765644, + 1922.66557964535, + 1915.7365354649685, + 1899.5988915420376, + 1928.0841526020797 + ], + [ + 1974.6806502492007, + 1975.9432475260974, + 1962.448315970151, + 1970.950097543744, + 1973.3968958743073 + ], + [ + 1919.004315694062, + 1931.6383106240698, + 1935.6000393443467, + 1928.8647099957707, + 1896.497910140058 + ] + ] + }, + "gc.alloc.rate.norm" : { + "score" : 1.8019800503188407E8, + "scoreError" : 125193.65237521521, + "scoreConfidence" : [ + 1.8007281137950885E8, + 1.803231986842593E8 + ], + "scorePercentiles" : { + "0.0" : 1.800364963478261E8, + "50.0" : 1.8027833286956522E8, + "90.0" : 1.8027843568695652E8, + "95.0" : 1.8027846991304347E8, + "99.0" : 1.8027846991304347E8, + "99.9" : 1.8027846991304347E8, + "99.99" : 1.8027846991304347E8, + "99.999" : 1.8027846991304347E8, + "99.9999" : 1.8027846991304347E8, + "100.0" : 1.8027846991304347E8 + }, + "scoreUnit" : "B/op", + "rawData" : [ + [ + 1.8003846991304347E8, + 1.800383572173913E8, + 1.800383864347826E8, + 1.8003834539130434E8, + 1.800364963478261E8 + ], + [ + 1.8027839826086956E8, + 1.8027841286956522E8, + 1.8027836486956522E8, + 1.8027833286956522E8, + 1.802764873043478E8 + ], + [ + 1.8027846991304347E8, + 1.8027836E8, + 1.802783676521739E8, + 1.8027833286956522E8, + 1.802764935652174E8 + ] + ] + }, + "gc.count" : { + "score" : 96.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 96.0, + 96.0 + ], + "scorePercentiles" : { + "0.0" : 6.0, + "50.0" : 6.0, + "90.0" : 7.0, + "95.0" : 7.0, + "99.0" : 7.0, + "99.9" : 7.0, + "99.99" : 7.0, + "99.999" : 7.0, + "99.9999" : 7.0, + "100.0" : 7.0 + }, + "scoreUnit" : "counts", + "rawData" : [ + [ + 6.0, + 7.0, + 6.0, + 7.0, + 6.0 + ], + [ + 6.0, + 7.0, + 6.0, + 7.0, + 6.0 + ], + [ + 6.0, + 7.0, + 6.0, + 7.0, + 6.0 + ] + ] + }, + "gc.time" : { + "score" : 125.0, + "scoreError" : "NaN", + "scoreConfidence" : [ + 125.0, + 125.0 + ], + "scorePercentiles" : { + "0.0" : 7.0, + "50.0" : 8.0, + "90.0" : 9.0, + "95.0" : 9.0, + "99.0" : 9.0, + "99.9" : 9.0, + "99.99" : 9.0, + "99.999" : 9.0, + "99.9999" : 9.0, + "100.0" : 9.0 + }, + "scoreUnit" : "ms", + "rawData" : [ + [ + 9.0, + 8.0, + 8.0, + 9.0, + 8.0 + ], + [ + 9.0, + 9.0, + 8.0, + 9.0, + 7.0 + ], + [ + 8.0, + 9.0, + 7.0, + 8.0, + 9.0 + ] + ] + } + } + } +] + + diff --git a/fesod-benchmark/benchmark.md b/fesod-benchmark/benchmark.md new file mode 100644 index 000000000..95b05ff97 --- /dev/null +++ b/fesod-benchmark/benchmark.md @@ -0,0 +1,257 @@ +# Fesod Benchmark Guide + +This guide provides a comprehensive overview of the Fesod benchmark module: the performance regression gate that runs in CI, how to interpret its reports, how to refresh the baseline, and how to run the manual analysis suites. + +> **Note:** Benchmark code in this module is not part of the Fesod public API. + +## Overview + +The benchmark module measures the performance of Fesod for spreadsheet operations (read, write, fill) using the [Java Microbenchmark Harness (JMH)](https://openjdk.java.net/projects/code-tools/jmh/). It serves two purposes: + +1. **Performance regression gate (primary)** โ€” a small, stable baseline suite runs on **release tags** and **on demand** (`workflow_dispatch`), is compared against a committed baseline, and **fails the CI job** when a high-confidence regression is detected (see [the tiered gate](#the-tiered-gate) below). Reports are appended to the job summary, regressions surface as `::error` annotations on the run page, and a regression on a release tag additionally posts the report on the matching GitHub Release (or opens an issue) so maintainers cannot miss it. +2. **Manual analysis suites (secondary)** โ€” larger benchmark suites (`ReadBenchmark`, `WriteBenchmark`, `FillBenchmark`, and the Fesod-vs-POI comparison suite) for deep-dive performance work. These are not part of the CI gate. + +## Performance Baseline CI + +### When the gate runs + +| Trigger | What happens | +|---|---| +| `push` of a release tag (`[0-9]+.*`, e.g. `2.1.0-incubating`) | Full suite + comparison. Regression โ†’ job fails and the report is posted on the GitHub Release of that tag (or an issue is opened). Pass โ†’ a baseline refresh PR is opened automatically, so the baseline lifecycle follows releases. | +| `workflow_dispatch` (manual) | Same suite + comparison on any branch โ€” e.g. while developing a performance-sensitive change (if the branch has an open PR, the report is also commented there). Check **update_baseline** to accept a performance change and open a baseline refresh PR. | + +The gate intentionally does **not** run per pull request: GitHub shared runners fluctuate too much for every-push gating to be worth the CI time. The tag is the release-time checkpoint; manual runs cover anything in between. + +### The tiered gate + +Whether a regression should fail CI was evaluated against measured noise on real `ubuntu-24.04` runners (same code, consecutive runs): + +| Signal | Measured noise | Decision | +|---|---|---| +| `gc.alloc.rate.norm` (bytes allocated per op) | **ยฑ0.1%** โ€” deterministic, unaffected by CPU contention | Regression beyond threshold โ†’ **hard fail**. Near-zero false-positive rate. | +| Average time per op, beyond fail threshold **and** JMH confidence intervals (score ยฑ scoreError) **do not overlap** | ยฑ7% typical, ยฑ15%+ tail | **Hard fail** โ€” high confidence that it is a real regression, not runner noise. | +| Average time per op beyond threshold **but** intervals overlap | โ€” | **WARN only** โ€” noise-ambiguous; failing here would make CI flaky and train people to ignore red gates. | +| Tracked benchmark missing from a run | โ€” | **Hard fail** โ€” the tracked contract cannot silently shrink. | + +If a run's execution contract (forks/iterations) differs from the baseline's, the report carries a prominent warning โ€” deltas across contracts are indicative only. + +### How it works + +``` +release tag / manual dispatch + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ benchmark job (ubuntu-24.04, JDK 17 Temurin โ€” fixed env) โ”‚ +โ”‚ โ”‚ +โ”‚ BaselineRunner โ”€โ”€โ–บ target/baseline-current.json โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ BaselineComparator โ”‚ +โ”‚ โ”‚ vs baseline/jmh-baseline.json โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ benchmark-report.md โ”‚ +โ”‚ โ”‚ + ::error/::warning annotations โ”‚ +โ”‚ โ–ผ โ–ผ โ”‚ +โ”‚ regression? โ”€โ”€โ”€ no โ”€โ”€โ–บ step summary + (PR comment) + PASS โ”‚ +โ”‚ โ”‚ yes (tiered gate: alloc regression, or time beyond โ”‚ +โ”‚ โ”‚ threshold with non-overlapping JMH error bars) โ”‚ +โ”‚ โ–ผ โ”‚ +โ”‚ job FAILS, report + annotations show the deltas โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ โ”‚ + โ”‚ tag + regression โ”‚ tag + pass, or dispatch with + โ–ผ โ–ผ update_baseline, or no baseline +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ notify-regression job โ”‚ โ”‚ update-baseline job: โ”‚ +โ”‚ report comment on the โ”‚ โ”‚ commits new jmh-baseline.json + โ”‚ +โ”‚ GitHub Release (or โ”‚ โ”‚ meta, opens a PR a maintainer โ”‚ +โ”‚ opens an issue) โ”‚ โ”‚ reviews and merges โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +Components (package `org.apache.fesod.sheet.benchmark.baseline`): + +| Component | Role | +|---|---| +| `BaselineBenchmark` | The tracked performance contract: `write` and `read` operations, XLSX and CSV formats, 1K/10K rows ร— 20 columns, average time per operation | +| `BaselineRunner` | Runs the suite with a fixed execution contract (3 forks, 3ร—1s warmup, 5ร—2s measurement, `-Xms1g -Xmx1g -XX:+UseG1GC`, gc profiler for allocation tracking) and writes JMH JSON | +| `BaselineComparator` | Compares against the committed baseline, renders the Markdown report, returns the CI gate exit code | +| `baseline/jmh-baseline.json` | The committed reference results โ€” the only source of truth for "known good performance" | +| `baseline/baseline-meta.json` | Baseline provenance: source commit, JDK, runner, generation date | + +Workflow: [`.github/workflows/benchmark.yml`](../.github/workflows/benchmark.yml) + +### Reading the report + +Every run appends a report to the job step summary; a manual run on a branch with an open PR additionally updates a sticky PR comment. The report looks like this: + +| Benchmark | Mode | Baseline | Current | ฮ” time | ฮ” alloc | Verdict | +|---|:-:|---:|---:|---:|---:|:-:| +| `read [datasetSize=MEDIUM, fileFormat=XLSX]` | avgt | 356 ms/op | 402 ms/op | ๐Ÿ”บ +12.9% | +0.4% | :warning: WARN | +| `write [datasetSize=SMALL, fileFormat=CSV]` | avgt | 48.1 ms/op | 47.9 ms/op | โˆ’0.4% | +0.1% | :white_check_mark: OK | + +Verdicts: + +- :white_check_mark: **OK** โ€” within the warn threshold (default ยฑ10%). +- :green_circle: **FASTER** โ€” at least the warn threshold faster than the baseline. +- :warning: **WARN** โ€” slower beyond the warn threshold, *or* beyond the fail threshold but still within JMH statistical error bars (noisy CI runner). Human judgement required; does not block the PR. +- :red_circle: **FAIL** โ€” slower beyond the fail threshold (default 20%) **and** the JMH confidence intervals (score ยฑ scoreError) do not overlap. Blocks the PR. +- :new: **NEW** / :black_circle: **MISSING** โ€” benchmarks absent from the baseline / current run. NEW is informational; MISSING blocks the PR so the tracked contract cannot silently shrink. + +Two signals are tracked per benchmark: + +- **ฮ” time** โ€” change in average time per operation (ms/op). +- **ฮ” alloc** โ€” change in `gc.alloc.rate.norm` (bytes allocated per operation). Allocation is nearly noise-free and often the earliest indicator of a regression, e.g. accidental object churn in a hot loop. + +The confidence-interval rule is what keeps CI usable on shared GitHub runners: a 30% "regression" whose error bars overlap the baseline is noise-ambiguous and only warns, while a consistent regression fails regardless of the threshold. + +### Configuring thresholds + +Defaults: warn at 10%, fail at 20%. They can be changed: + +- **Per dispatch run** โ€” inputs of the *Benchmark* workflow (`warn_threshold`, `fail_threshold`). +- **Repository-wide without code changes** โ€” set repository variables `BENCHMARK_WARN_PCT` / `BENCHMARK_FAIL_PCT` (Settings โ†’ Secrets and variables โ†’ Actions โ†’ Variables). + +### Updating the baseline + +The baseline records absolute numbers and therefore must be generated on the same environment as the CI comparisons: an `ubuntu-24.04` GitHub runner with JDK 17 (Temurin). **Never commit a baseline generated on a local machine** โ€” hardware differences would invalidate every comparison. + +The baseline lifecycle follows releases: + +- **Release tag passes the gate** โ†’ the update job automatically opens a baseline refresh PR from that tag (a maintainer merges it; the baseline advances to the release). +- **Release tag regresses** โ†’ the baseline is *not* advanced; the notify job posts the regression report on the tag's GitHub Release (or opens an issue). Fix the regression and re-tag, or deliberately accept it via a manual refresh below. +- **Manual refresh** (after an intentionally accepted performance change, a new benchmark method, or a JDK/runner upgrade): *Actions โ†’ Benchmark โ†’ Run workflow* on the target branch with **update_baseline** checked. The workflow runs the suite, embeds the old-vs-new delta report in the PR body, and opens a PR updating `baseline/jmh-baseline.json` and `baseline/baseline-meta.json` for maintainer review. +- **First run on a repository** bootstraps automatically: with no baseline present, the run reports every metric as NEW, exit 0, and the update job opens the initial baseline PR. + +### Adding a benchmark to the baseline suite + +1. Add a `@Benchmark` method (or a `@Param` value) to `BaselineBenchmark`. Keep each addition under ~30s of suite runtime โ€” the gate runs on release tags and manual dispatch. +2. The method appears as :new: NEW on the next PR runs (informational only, does not block). +3. Refresh the baseline so the new metric becomes part of the tracked contract. +4. **Do not rename existing benchmark methods or params** โ€” they are the baseline keys; a rename shows up as MISSING + NEW and fails the gate until the baseline is refreshed. + +### Reproducing a CI run locally + +```bash +mvn clean package -f fesod-benchmark/pom.xml -DskipTests + +# same suite / same JVM contract as CI (results land in target/baseline-current.json) +java -cp fesod-benchmark/target/benchmarks.jar \ + org.apache.fesod.sheet.benchmark.baseline.BaselineRunner + +# compare your local run against the committed baseline +# (absolute times are NOT comparable across machines โ€” look at alloc/op and rough magnitudes) +java -cp fesod-benchmark/target/benchmarks.jar \ + org.apache.fesod.sheet.benchmark.baseline.BaselineComparator \ + --baseline fesod-benchmark/baseline/jmh-baseline.json \ + --baseline-meta fesod-benchmark/baseline/baseline-meta.json \ + --current fesod-benchmark/target/baseline-current.json \ + --report fesod-benchmark/target/benchmark-report.md + +# quick smoke run (1 fork, 1ร—1s measurement, SMALL datasets only) +java -Dbenchmark.forks=1 -Dbenchmark.warmup.iterations=1 -Dbenchmark.measurement.iterations=1 \ + -Dbenchmark.datasetSizes=SMALL \ + -cp fesod-benchmark/target/benchmarks.jar \ + org.apache.fesod.sheet.benchmark.baseline.BaselineRunner +``` + +All knobs are system properties on `BaselineRunner` (`benchmark.forks`, `benchmark.warmup.iterations`, `benchmark.warmup.seconds`, `benchmark.measurement.iterations`, `benchmark.measurement.seconds`, `benchmark.datasetSizes`, `benchmark.fileFormats`, `benchmark.result`). + +## Running the Analysis Suites + +### Using the Shade JAR (Recommended) + +Build the uber-jar and run benchmarks directly: + +```bash +mvn clean package -f fesod-benchmark/pom.xml -DskipTests +java -jar fesod-benchmark/target/benchmarks.jar +``` + +Run a specific benchmark class: + +```bash +java -jar fesod-benchmark/target/benchmarks.jar ReadBenchmark +``` + +Run with JMH GC profiler for memory analysis: + +```bash +java -jar fesod-benchmark/target/benchmarks.jar -prof gc +``` + +Export results as JSON: + +```bash +java -jar fesod-benchmark/target/benchmarks.jar -rf json -rff results.json +``` + +### Using the Comparison Runner + +The `ComparisonBenchmarkRunner` provides a pre-configured Fesod vs Apache POI comparison: + +```bash +java -cp fesod-benchmark/target/benchmarks.jar \ + org.apache.fesod.sheet.benchmark.comparison.ComparisonBenchmarkRunner +``` + +Results are written to `target/benchmark-results//`. + +### Using Maven Profiles + +```bash +# Run all analysis benchmarks via Maven +mvn verify -f fesod-benchmark/pom.xml -P benchmark -Dbenchmark.pattern=.* + +# Run a specific analysis benchmark +mvn verify -f fesod-benchmark/pom.xml -P benchmark -Dbenchmark.pattern=ReadBenchmark + +# Quick smoke test of the baseline suite (1 fork, 1 iteration, SMALL datasets) +mvn verify -f fesod-benchmark/pom.xml -P benchmark-test +``` + +## Benchmark Suites + +| Suite | Description | CI gate | +|---|---|---| +| **Baseline** | `BaselineBenchmark` โ€” write/read, XLSX/CSV, 1K/10K rows | โœ… release tags + manual dispatch, blocks high-confidence regressions | +| **Comparison** | Head-to-head comparison of Fesod vs Apache POI for read, write, and streaming operations | manual | +| **Operations** | Focused benchmarks for read (`ReadBenchmark`), write (`WriteBenchmark`), and fill (`FillBenchmark`) operations | manual | + +### Dataset Sizes + +| Size | Rows | Use Case | +|---|---|---| +| `SMALL` | 1,000 | Quick development feedback / baseline suite | +| `MEDIUM` | 10,000 | Standard CI benchmarks / baseline suite | +| `LARGE` | 100,000 | Performance analysis | +| `EXTRA_LARGE` | 1,000,000 | Stress testing (comparison benchmarks only) | + +## JMH Best Practices Applied + +This benchmark module follows JMH best practices: + +1. **No manual timing** - JMH handles all timing measurements via `@BenchmarkMode`. +2. **No `System.gc()` calls** - Avoids unpredictable pauses that distort measurements. +3. **Fixed heap size** - `-Xms` equals `-Xmx` for stable GC behavior. +4. **Pre-loaded data** - All test data is generated in `@Setup(Level.Trial)` to exclude I/O from measurements. +5. **Fixed random seed** - Ensures reproducible data generation across runs. +6. **Fair comparison** - Both Fesod and Apache POI write/read the same columns. +7. **`Blackhole.consume()`** - Prevents dead code elimination by the JIT compiler. +8. **`@OperationsPerInvocation`** - Allows JMH to correctly calculate throughput. + +## Interpreting Raw Results + +JMH produces output in the following format: + +``` +Benchmark (datasetSize) (fileFormat) Mode Cnt Score Error Units +FastExcelVsPoiBenchmark.benchmarkFesodRead SMALL XLSX avgt 5 2.345 ยฑ 0.123 ms/op +FastExcelVsPoiBenchmark.benchmarkPoiRead SMALL XLSX avgt 5 5.678 ยฑ 0.456 ms/op +``` + +Key columns: +- **Mode**: `avgt` (average time), `thrpt` (throughput), `ss` (single shot) +- **Score**: The benchmark score (lower is better for `avgt`, higher is better for `thrpt`) +- **Error**: 99.9% confidence interval โ€” the same error bars the baseline comparator uses to downgrade noise-ambiguous regressions +- **Units**: `ms/op` (milliseconds per operation), `ops/s` (operations per second) diff --git a/fesod-benchmark/pom.xml b/fesod-benchmark/pom.xml new file mode 100644 index 000000000..b858824a7 --- /dev/null +++ b/fesod-benchmark/pom.xml @@ -0,0 +1,227 @@ + + + + 4.0.0 + + + org.apache.fesod + fesod-parent + ${revision} + ../pom.xml + + + fesod-benchmark + fesod-benchmark + Comprehensive benchmark module for Fesod performance analysis + + + 1.37 + benchmarks + + + + + org.apache.fesod + fesod-sheet + ${project.version} + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + org.apache.poi + poi + + + org.apache.poi + poi-ooxml + + + + + commons-io + commons-io + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + + + com.alibaba.fastjson2 + fastjson2 + + + + + org.junit.jupiter + junit-jupiter + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + benchmark + + .* + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + run-benchmarks + integration-test + + java + + + org.openjdk.jmh.Main + + ${benchmark.pattern} + + + + + + + + + + + + benchmark-test + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + run-benchmark-smoke-test + integration-test + + exec + + + java + + -Dbenchmark.forks=1 + -Dbenchmark.warmup.iterations=1 + -Dbenchmark.warmup.seconds=1 + -Dbenchmark.measurement.iterations=1 + -Dbenchmark.measurement.seconds=1 + -Dbenchmark.datasetSizes=SMALL + -cp + target/benchmarks.jar + org.apache.fesod.sheet.benchmark.baseline.BaselineRunner + + + + + + + + + + diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineBenchmark.java new file mode 100644 index 000000000..e9d997813 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineBenchmark.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.baseline; + +import java.io.File; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.EasyExcel; +import org.apache.fesod.sheet.ExcelReader; +import org.apache.fesod.sheet.benchmark.core.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.utils.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.utils.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Performance baseline suite used by the regression CI + * ({@code .github/workflows/benchmark.yml}). + * + *

This class is the performance contract of Fesod: every benchmark method here is + * tracked against the committed baseline in {@code fesod-benchmark/baseline/} and a + * regression beyond the configured threshold fails the CI job. It is intentionally + * kept small, stable and fast so that it can run on every pull request: + * + *

+ * + *

Guidelines when evolving this suite: + * + *

+ */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 2) +@Fork( + value = BaselineBenchmark.CI_FORKS, + jvmArgs = {"-Xms1g", "-Xmx1g", "-XX:+UseG1GC"}) +public class BaselineBenchmark { + + /** Fork count used when the suite is executed directly through the JMH launcher. */ + public static final int CI_FORKS = 3; + + @Param({"SMALL", "MEDIUM"}) + private String datasetSize; + + @Param({"XLSX", "CSV"}) + private String fileFormat; + + private List data; + private File readFile; + + @Setup(Level.Trial) + public void setupTrial() { + BenchmarkConfiguration.DatasetSize size = BenchmarkConfiguration.DatasetSize.valueOf(datasetSize); + data = DataGenerator.generateTestData(size); + + String fileName = String.format( + "baseline_read_%s_%s.%s", + datasetSize.toLowerCase(), fileFormat.toLowerCase(), fileFormat.toLowerCase()); + readFile = BenchmarkFileUtil.createTestFile(fileName); + EasyExcel.write(readFile, BenchmarkData.class).sheet("Sheet1").doWrite(data); + + System.out.printf("Baseline setup: %s / %s / %d rows%n", fileFormat, datasetSize, data.size()); + } + + @TearDown(Level.Trial) + public void tearDownTrial() { + if (readFile != null && readFile.exists()) { + readFile.delete(); + } + } + + /** + * Write {@code datasetSize} rows of 20 columns to a fresh file (deleted afterwards). + */ + @Benchmark + public long write(Blackhole blackhole) { + File outputFile = BenchmarkFileUtil.createTestFile(String.format( + "baseline_write_%s_%s_%s.%s", + datasetSize.toLowerCase(), + fileFormat.toLowerCase(), + UUID.randomUUID().toString().substring(0, 8), + fileFormat.toLowerCase())); + + try { + EasyExcel.write(outputFile, BenchmarkData.class).sheet("Sheet1").doWrite(data); + blackhole.consume(outputFile.length()); + } finally { + if (outputFile.exists()) { + outputFile.delete(); + } + } + return data.size(); + } + + /** + * Read the whole pre-generated file through the streaming reader. + */ + @Benchmark + public long read(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + + ExcelReader excelReader = EasyExcel.read(readFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData row, AnalysisContext context) { + processedRows.incrementAndGet(); + blackhole.consume(row); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // no-op + } + }) + .build(); + try { + excelReader.readAll(); + } finally { + excelReader.finish(); + } + return processedRows.get(); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java new file mode 100644 index 000000000..bfe565cfa --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java @@ -0,0 +1,693 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.baseline; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * Compares a fresh {@link BaselineRunner} result file against the committed baseline and + * renders a Markdown report for humans (PR comment / job summary), plus a process exit + * code used as the CI gate. + * + *

Comparison rules: + *

    + *
  • Direction aware: for {@code avgt}/{@code ss} lower is better, for {@code thrpt} + * higher is better.
    • + *
  • Two signals per benchmark: primary metric (average time) and + * {@code gc.alloc.rate.norm} (bytes allocated per op).
    • + *
  • Noise aware: a regression is only a hard failure when it exceeds the fail + * threshold and the JMH confidence intervals (score ± scoreError) + * do not overlap; an overlapping regression is downgraded to a warning, which keeps + * CI stable on shared runners.
    • + *
  • A benchmark that exists in the baseline but not in the current run fails the gate + * (unless {@code --allow-missing}) so the tracked contract cannot silently shrink.
    • + *
+ * + *

Exit codes: {@code 0} = pass (or bootstrap mode without baseline), + * {@code 1} = regression gate failed, {@code 2} = usage or I/O error. + */ +public final class BaselineComparator { + + private static final String GC_ALLOC_RATE_NORM = "gc.alloc.rate.norm"; + + private static final String DEFAULT_BASELINE = "fesod-benchmark/baseline/jmh-baseline.json"; + private static final String DEFAULT_BASELINE_META = "fesod-benchmark/baseline/baseline-meta.json"; + private static final String DEFAULT_CURRENT = "target/baseline-current.json"; + + private static final String OK = ":white_check_mark: OK"; + private static final String IMPROVED = ":green_circle: FASTER"; + private static final String WARN = ":warning: WARN"; + private static final String FAIL = ":red_circle: FAIL"; + private static final String NEW = ":new: NEW"; + private static final String MISSING = ":black_circle: MISSING"; + + private static final String UP = ":small_red_triangle:"; + private static final String DOWN = ":small_red_triangle_down:"; + + private enum Verdict { + OK(0), + IMPROVED(1), + WARN(2), + FAIL(3), + NEW(0), + MISSING(3); + + private final int severity; + + Verdict(int severity) { + this.severity = severity; + } + } + + /** One JMH result entry: primary metric plus the gc allocation metric when present. */ + private static final class Metric { + final String key; + final String displayName; + final String mode; + final double score; + final double scoreError; + final String scoreUnit; + final Double allocScore; + final double allocError; + final String allocUnit; + + Metric( + String key, + String displayName, + String mode, + double score, + double scoreError, + String scoreUnit, + Double allocScore, + double allocError, + String allocUnit) { + this.key = key; + this.displayName = displayName; + this.mode = mode; + this.score = score; + this.scoreError = scoreError; + this.scoreUnit = scoreUnit; + this.allocScore = allocScore; + this.allocError = allocError; + this.allocUnit = allocUnit; + } + } + + /** One report row: baseline/current metric pair plus the computed verdict. */ + private static final class Row { + final Metric baseline; + final Metric current; + Verdict verdict; + String timeDetail; + String allocDetail; + + Row(Metric baseline, Metric current) { + this.baseline = baseline; + this.current = current; + } + + Metric currentOrBaseline() { + return current != null ? current : baseline; + } + } + + private final double warnPct; + private final double failPct; + private final boolean allowMissing; + + private BaselineComparator(double warnPct, double failPct, boolean allowMissing) { + this.warnPct = warnPct; + this.failPct = failPct; + this.allowMissing = allowMissing; + } + + public static void main(String[] args) { + String baselinePath = DEFAULT_BASELINE; + String baselineMetaPath = DEFAULT_BASELINE_META; + String currentPath = DEFAULT_CURRENT; + String reportPath = null; + double warnPct = 10.0d; + double failPct = 20.0d; + boolean allowMissing = false; + + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if ("--baseline".equals(arg) && i + 1 < args.length) { + baselinePath = args[++i]; + } else if ("--baseline-meta".equals(arg) && i + 1 < args.length) { + baselineMetaPath = args[++i]; + } else if ("--current".equals(arg) && i + 1 < args.length) { + currentPath = args[++i]; + } else if ("--report".equals(arg) && i + 1 < args.length) { + reportPath = args[++i]; + } else if ("--warn".equals(arg) && i + 1 < args.length) { + warnPct = Double.parseDouble(args[++i]); + } else if ("--fail".equals(arg) && i + 1 < args.length) { + failPct = Double.parseDouble(args[++i]); + } else if ("--allow-missing".equals(arg)) { + allowMissing = true; + } else { + usageAndExit("Unknown or incomplete argument: " + arg); + } + } + if (failPct <= warnPct) { + usageAndExit("--fail threshold must be greater than --warn threshold"); + } + + try { + File currentFile = new File(currentPath); + if (!currentFile.isFile()) { + System.err.println("ERROR: current result file not found: " + currentPath); + System.exit(2); + } + Map current = index(readResults(currentFile)); + String currentContract = readContract(currentFile); + + File baselineFile = new File(baselinePath); + Map baseline = + baselineFile.isFile() ? index(readResults(baselineFile)) : new TreeMap(); + String baselineContract = baselineFile.isFile() ? readContract(baselineFile) : null; + boolean contractDiffers = baselineContract != null && !baselineContract.equals(currentContract); + + BaselineComparator comparator = new BaselineComparator(warnPct, failPct, allowMissing); + List rows = comparator.compare(baseline, current); + String report = comparator.renderReport( + rows, + baseline.isEmpty(), + readMeta(new File(baselineMetaPath)), + baselineContract, + currentContract, + contractDiffers); + + System.out.println(); + System.out.println(report); + + if (reportPath != null) { + File reportFile = new File(reportPath); + if (reportFile.getParentFile() != null + && !reportFile.getParentFile().exists()) { + reportFile.getParentFile().mkdirs(); + } + Files.write(reportFile.toPath(), report.getBytes(StandardCharsets.UTF_8)); + System.out.println("Report written to: " + reportPath); + System.out.println(); + } + + boolean gateFailed = false; + for (Row row : rows) { + if (row.verdict == Verdict.FAIL || (row.verdict == Verdict.MISSING && !allowMissing)) { + gateFailed = true; + System.err.println("REGRESSION: " + row.verdict + " " + row.currentOrBaseline().displayName + + " โ€” time: " + row.timeDetail + ", alloc: " + row.allocDetail); + } + } + printAnnotations(rows); + if (contractDiffers) { + System.err.println("NOTE: execution contract differs from baseline โ€” refresh the baseline " + + "for a strictly valid comparison."); + } + + System.exit(gateFailed ? 1 : 0); + } catch (IOException e) { + System.err.println("ERROR: " + e.getMessage()); + System.exit(2); + } + } + + /** + * Emits GitHub Actions workflow commands so regressions surface as annotations on the + * run page (and any linked PR/release), not only in the log. No-op outside Actions. + */ + private static void printAnnotations(List rows) { + if (!"true".equals(System.getenv("GITHUB_ACTIONS"))) { + return; + } + for (Row row : rows) { + String detail = (row.currentOrBaseline().displayName + " โ€” time: " + row.timeDetail + ", alloc: " + + row.allocDetail) + .replace(UP + " ", "") + .replace(DOWN + " ", ""); + if (row.verdict == Verdict.FAIL || (row.verdict == Verdict.MISSING)) { + System.out.println("::error title=Performance regression::" + detail); + } else if (row.verdict == Verdict.WARN) { + System.out.println("::warning title=Performance warning::" + detail); + } + } + } + + // ------------------------------------------------------------------ + // Comparison + // ------------------------------------------------------------------ + + private List compare(Map baseline, Map current) { + List rows = new ArrayList(); + TreeSet keys = new TreeSet(); + keys.addAll(baseline.keySet()); + keys.addAll(current.keySet()); + + for (String key : keys) { + Metric base = baseline.get(key); + Metric cur = current.get(key); + Row row = new Row(base, cur); + + if (base == null) { + row.verdict = Verdict.NEW; + row.timeDetail = "not in baseline"; + row.allocDetail = "n/a"; + } else if (cur == null) { + row.verdict = Verdict.MISSING; + row.timeDetail = "missing from current run"; + row.allocDetail = "n/a"; + } else { + row.verdict = worst(metricVerdict(base, cur), allocVerdict(base, cur)); + row.timeDetail = timeDetail(base, cur); + row.allocDetail = allocDetail(base, cur); + } + rows.add(row); + } + return rows; + } + + private Verdict metricVerdict(Metric base, Metric cur) { + boolean higherIsWorse = higherIsWorse(cur.mode); + return verdict(base.score, base.scoreError, cur.score, cur.scoreError, higherIsWorse); + } + + private Verdict allocVerdict(Metric base, Metric cur) { + if (base.allocScore == null || cur.allocScore == null || base.allocScore <= 0 || cur.allocScore <= 0) { + return Verdict.OK; + } + return verdict(base.allocScore, base.allocError, cur.allocScore, cur.allocError, true); + } + + private Verdict verdict(double base, double baseErr, double cur, double curErr, boolean higherIsWorse) { + double regressionPct = higherIsWorse ? (cur - base) / base * 100.0d : (base - cur) / base * 100.0d; + boolean intervalsOverlap = + higherIsWorse ? (cur - curErr) <= (base + baseErr) : (cur + curErr) >= (base - baseErr); + + if (regressionPct >= failPct && !intervalsOverlap) { + return Verdict.FAIL; + } + if (regressionPct >= failPct || regressionPct >= warnPct) { + return Verdict.WARN; + } + if (regressionPct <= -warnPct) { + return Verdict.IMPROVED; + } + return Verdict.OK; + } + + private String timeDetail(Metric base, Metric cur) { + return changeDetail(cur.score - base.score, base.score); + } + + private String allocDetail(Metric base, Metric cur) { + if (base.allocScore == null || cur.allocScore == null || base.allocScore <= 0) { + return "n/a"; + } + return changeDetail(cur.allocScore - base.allocScore, base.allocScore); + } + + private static String changeDetail(double delta, double base) { + double pct = delta / base * 100.0d; + String icon = ""; + if (pct > 0.05d) { + icon = UP + " "; + } else if (pct < -0.05d) { + icon = DOWN + " "; + } + return String.format(Locale.ROOT, "%s%+.1f%%", icon, pct); + } + + private static boolean higherIsWorse(String mode) { + // thrpt: higher score = better; everything else JMH emits (avgt, ss, sampled) is time-like + return !"thrpt".equals(mode); + } + + private static Verdict worst(Verdict a, Verdict b) { + return a.severity >= b.severity ? a : b; + } + + // ------------------------------------------------------------------ + // Report rendering + // ------------------------------------------------------------------ + + private String renderReport( + List rows, + boolean bootstrap, + JSONObject meta, + String baselineContract, + String currentContract, + boolean contractDiffers) { + StringBuilder sb = new StringBuilder(); + + if (bootstrap) { + sb.append("## :bar_chart: Performance run โ€” baseline not yet established\n\n"); + sb.append("No committed baseline found (or it is unreadable), so no comparison was performed.\n"); + sb.append("The CI update job will open a pull request that records this run as the initial baseline.\n"); + sb.append("Every value below becomes a tracked metric once that PR is merged.\n\n"); + } else { + sb.append("## :bar_chart: Performance vs baseline\n\n"); + sb.append(baselineHeader(meta)); + sb.append("\n\n"); + if (contractDiffers) { + sb.append("> :warning: **Execution contract differs from the baseline** โ€” baseline ran `") + .append(baselineContract) + .append("`, this run used `") + .append(currentContract) + .append("`.\n") + .append("> Time deltas across different contracts are indicative only; refresh the ") + .append("baseline to restore a strictly valid comparison.\n\n"); + } + } + + sb.append("| Benchmark | Mode | Baseline | Current | "); + sb.append(bootstrap ? "ฮ” | Verdict |\n" : "ฮ” time | ฮ” alloc | Verdict |\n"); + sb.append(bootstrap ? "|---|:-:|---:|---:|---:|:-:|\n" : "|---|:-:|---:|---:|---:|---:|:-:|\n"); + + for (Row row : rows) { + Metric shown = row.current != null ? row.current : row.baseline; + sb.append("| `") + .append(shown.displayName) + .append("` | ") + .append(shown.mode) + .append(" | ") + .append(row.baseline == null ? "โ€”" : formatScore(row.baseline.score, row.baseline.scoreUnit)) + .append(" | ") + .append(row.current == null ? "โ€”" : formatScore(row.current.score, row.current.scoreUnit)) + .append(" | "); + if (bootstrap) { + sb.append(row.current == null ? "โ€”" : "new"); + } else { + sb.append(row.timeDetail).append(" | ").append(row.allocDetail); + } + sb.append(" | ").append(verdictLabel(row)).append(" |\n"); + } + + sb.append("\n").append(summaryLine(rows, bootstrap)).append("\n"); + sb.append(legend()); + return sb.toString(); + } + + private String baselineHeader(JSONObject meta) { + StringBuilder sb = new StringBuilder(); + sb.append("Baseline: "); + if (meta == null) { + sb.append("`").append(DEFAULT_BASELINE).append("` (no metadata)"); + } else { + String sha = meta.getString("gitSha"); + sb.append("`") + .append(sha == null ? "unknown" : sha.substring(0, Math.min(7, sha.length()))) + .append("`"); + String generatedAt = meta.getString("generatedAt"); + if (generatedAt != null) { + sb.append(" ยท recorded ").append(generatedAt); + } + String jdk = meta.getString("jdkVersion"); + if (jdk != null) { + sb.append(" ยท ").append(jdk); + } + String runner = meta.getString("runnerLabel"); + if (runner != null) { + sb.append(" ยท ").append(runner); + } + } + sb.append(String.format(Locale.ROOT, " ยท thresholds: warn โ‰ฅ %.0f%% / fail โ‰ฅ %.0f%%", warnPct, failPct)); + return sb.toString(); + } + + private String summaryLine(List rows, boolean bootstrap) { + int ok = 0; + int improved = 0; + int warn = 0; + int fail = 0; + int missing = 0; + int newCount = 0; + for (Row row : rows) { + switch (row.verdict) { + case OK: + ok++; + break; + case IMPROVED: + improved++; + break; + case WARN: + warn++; + break; + case FAIL: + fail++; + break; + case MISSING: + missing++; + break; + case NEW: + newCount++; + break; + default: + break; + } + } + + StringBuilder sb = new StringBuilder("**Summary**: "); + if (bootstrap) { + sb.append(newCount).append(" metric(s) recorded, none compared yet"); + return sb.toString(); + } + sb.append(ok) + .append(" ") + .append(OK) + .append(" ยท ") + .append(improved) + .append(" ") + .append(IMPROVED) + .append(" ยท ") + .append(warn) + .append(" ") + .append(WARN) + .append(" ยท ") + .append(fail) + .append(" ") + .append(FAIL); + if (missing > 0) { + sb.append(" ยท ").append(missing).append(" ").append(MISSING); + } + if (newCount > 0) { + sb.append(" ยท ").append(newCount).append(" ").append(NEW); + } + + boolean gateFailed = fail > 0 || (missing > 0 && !allowMissing); + sb.append(" โ€” gate: **").append(gateFailed ? "FAILED" : "PASSED").append("**"); + if (gateFailed) { + sb.append(" (regression beyond fail threshold or missing benchmark)"); + } + return sb.toString(); + } + + private String legend() { + return "\n

\nVerdict legend & notes\n\n" + + "- " + + OK + + ": within the warn threshold; " + + IMPROVED + + ": at least the warn threshold faster.\n" + + "- " + + WARN + + ": slower beyond the warn threshold, or beyond the fail threshold but still within JMH " + + "statistical error bars (noisy CI runner) โ€” human judgement required.\n" + + "- " + + FAIL + + ": slower beyond the fail threshold with non-overlapping error bars โ€” fails the CI run.\n" + + "- " + + NEW + + ": not tracked by the baseline yet; becomes tracked at the next baseline refresh. " + + MISSING + + ": tracked benchmark absent from the current run.\n" + + "- Time is the JMH average per operation; ฮ” alloc compares `gc.alloc.rate.norm` " + + "(bytes allocated per operation) โ€” a very stable regression signal.\n" + + "- To deliberately accept a performance change, refresh the baseline: " + + "*Actions โ†’ Benchmark โ†’ Run workflow* with **update_baseline** checked " + + "(see `fesod-benchmark/benchmark.md`).\n" + + "\n
\n"; + } + + private static String verdictLabel(Row row) { + switch (row.verdict) { + case IMPROVED: + return IMPROVED; + case WARN: + return WARN; + case FAIL: + return FAIL; + case NEW: + return NEW; + case MISSING: + return MISSING; + default: + return OK; + } + } + + // ------------------------------------------------------------------ + // JMH JSON parsing + // ------------------------------------------------------------------ + + private static List readResults(File file) throws IOException { + String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + JSONArray array = JSON.parseArray(content); + List results = new ArrayList(); + for (int i = 0; i < array.size(); i++) { + JSONObject entry = array.getJSONObject(i); + JSONObject primary = entry.getJSONObject("primaryMetric"); + if (primary == null) { + continue; + } + JSONObject params = entry.getJSONObject("params"); + + Double allocScore = null; + double allocError = 0.0d; + String allocUnit = null; + JSONObject secondary = entry.getJSONObject("secondaryMetrics"); + if (secondary != null) { + JSONObject alloc = secondary.getJSONObject(GC_ALLOC_RATE_NORM); + if (alloc != null && alloc.containsKey("score")) { + allocScore = sanitize(alloc.getDouble("score")); + allocError = sanitize(alloc.containsKey("scoreError") ? alloc.getDouble("scoreError") : 0.0d); + allocUnit = alloc.getString("scoreUnit"); + } + } + + String benchmark = entry.getString("benchmark"); + results.add(new Metric( + keyOf(benchmark, params), + displayNameOf(benchmark, params), + entry.getString("mode"), + primary.getDoubleValue("score"), + sanitize(primary.containsKey("scoreError") ? primary.getDouble("scoreError") : 0.0d), + primary.getString("scoreUnit"), + allocScore, + allocError, + allocUnit)); + } + return results; + } + + private static JSONObject readMeta(File file) { + if (!file.isFile()) { + return null; + } + try { + return JSON.parseObject(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8)); + } catch (IOException e) { + return null; + } + } + + /** + * Reads the JMH execution contract (forks, warmup, measurement) from the first result + * entry, e.g. {@code "3 forks, 3x1 s warmup, 5x2 s measurement"}. Comparisons are only + * strictly valid when the baseline and the current run share the same contract. + */ + private static String readContract(File file) throws IOException { + JSONArray array = JSON.parseArray(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8)); + if (array.isEmpty()) { + return null; + } + JSONObject entry = array.getJSONObject(0); + return String.format( + Locale.ROOT, + "%d fork%s, %dx%s warmup, %dx%s measurement", + entry.getIntValue("forks"), + entry.getIntValue("forks") == 1 ? "" : "s", + entry.getIntValue("warmupIterations"), + entry.getString("warmupTime"), + entry.getIntValue("measurementIterations"), + entry.getString("measurementTime")); + } + + private static Map index(List metrics) { + Map byKey = new TreeMap(); + for (Metric metric : metrics) { + byKey.put(metric.key, metric); + } + return byKey; + } + + private static String keyOf(String benchmark, JSONObject params) { + return benchmark + paramsSuffix(params); + } + + private static String displayNameOf(String benchmark, JSONObject params) { + String simple = benchmark.substring(benchmark.lastIndexOf('.') + 1); + return simple + paramsSuffix(params); + } + + private static String paramsSuffix(JSONObject params) { + if (params == null || params.isEmpty()) { + return ""; + } + List parts = new ArrayList(); + for (String key : new TreeSet(params.keySet())) { + parts.add(key + "=" + params.getString(key)); + } + return " [" + String.join(", ", parts) + "]"; + } + + /** JMH emits "NaN" errors for short runs; treat those as "no error information". */ + private static double sanitize(Double value) { + if (value == null || value.isNaN() || value.isInfinite()) { + return 0.0d; + } + return value; + } + + private static String formatScore(double score, String unit) { + String number; + double abs = Math.abs(score); + if (abs >= 100.0d) { + number = String.format(Locale.ROOT, "%.0f", score); + } else if (abs >= 10.0d) { + number = String.format(Locale.ROOT, "%.1f", score); + } else { + number = String.format(Locale.ROOT, "%.2f", score); + } + return unit == null || unit.isEmpty() ? number : number + " " + unit; + } + + private static void usageAndExit(String message) { + System.err.println("ERROR: " + message); + System.err.println(); + System.err.println("Usage: BaselineComparator [--baseline ] [--baseline-meta ]"); + System.err.println(" --current [--report ]"); + System.err.println(" [--warn ] [--fail ] [--allow-missing]"); + System.exit(2); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java new file mode 100644 index 000000000..ccf1afcbd --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.baseline; + +import java.io.File; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; + +/** + * Single entry point for the performance baseline suite used by the regression CI. + * + *

Runs {@link BaselineBenchmark} with a fixed execution contract (JVM settings, forks, + * iterations, gc profiler) so that a run recorded today is comparable with the committed + * baseline. Every knob can be overridden with system properties for local experiments, + * but CI always uses the defaults: + * + * 

+ *   benchmark.forks                   (default 2)
+ *   benchmark.warmup.iterations       (default 3)
+ *   benchmark.warmup.seconds          (default 1)
+ *   benchmark.measurement.iterations  (default 5)
+ *   benchmark.measurement.seconds     (default 1)
+ *   benchmark.datasetSizes            (default SMALL,MEDIUM)
+ *   benchmark.fileFormats             (default XLSX,CSV)
+ *   benchmark.result                  (default target/baseline-current.json)
+ *
+ * + *

Usage with the shaded jar: + * 

+ *   java -cp fesod-benchmark/target/benchmarks.jar \
+ *       org.apache.fesod.sheet.benchmark.baseline.BaselineRunner
+ *
+ * + *

The gc profiler is always enabled: allocation per operation is far less noisy than + * wall-clock time on shared CI runners and is tracked in the baseline as a second signal. + */ +public final class BaselineRunner { + + private static final String[] FIXED_JVM_ARGS = {"-Xms1g", "-Xmx1g", "-XX:+UseG1GC"}; + + private BaselineRunner() {} + + public static void main(String[] args) throws RunnerException { + int forks = intProperty("benchmark.forks", 3); + int warmupIterations = intProperty("benchmark.warmup.iterations", 3); + int warmupSeconds = intProperty("benchmark.warmup.seconds", 1); + int measurementIterations = intProperty("benchmark.measurement.iterations", 5); + int measurementSeconds = intProperty("benchmark.measurement.seconds", 2); + String[] datasetSizes = + property("benchmark.datasetSizes", "SMALL,MEDIUM").split(","); + String[] fileFormats = property("benchmark.fileFormats", "XLSX,CSV").split(","); + String resultFile = property("benchmark.result", "target/baseline-current.json"); + + printEnvironment(forks, warmupIterations, warmupSeconds, measurementIterations, measurementSeconds, resultFile); + + File resultParent = new File(resultFile).getAbsoluteFile().getParentFile(); + if (resultParent != null && !resultParent.exists()) { + resultParent.mkdirs(); + } + + Options opt = new OptionsBuilder() + .include(BaselineBenchmark.class.getSimpleName()) + .param("datasetSize", trim(datasetSizes)) + .param("fileFormat", trim(fileFormats)) + .forks(forks) + .warmupIterations(warmupIterations) + .warmupTime(TimeValue.seconds(warmupSeconds)) + .measurementIterations(measurementIterations) + .measurementTime(TimeValue.seconds(measurementSeconds)) + .addProfiler("gc") + .shouldFailOnError(true) + .jvmArgs(FIXED_JVM_ARGS) + .result(resultFile) + .resultFormat(ResultFormatType.JSON) + .build(); + + new Runner(opt).run(); + + System.out.println(); + System.out.println("====================================================="); + System.out.println("Baseline run finished. Results written to: " + resultFile); + System.out.println("Compare with the committed baseline via BaselineComparator."); + System.out.println("====================================================="); + } + + private static void printEnvironment( + int forks, + int warmupIterations, + int warmupSeconds, + int measurementIterations, + int measurementSeconds, + String resultFile) { + System.out.println("====================================================="); + System.out.println("Fesod baseline suite"); + System.out.println(" JVM : " + System.getProperty("java.vm.name") + " " + + System.getProperty("java.version") + " (" + System.getProperty("java.vendor") + ")"); + System.out.println(" OS : " + System.getProperty("os.name") + " " + System.getProperty("os.version") + + " " + System.getProperty("os.arch")); + System.out.println(" Forks : " + forks); + System.out.println(" Warmup : " + warmupIterations + " x " + warmupSeconds + "s"); + System.out.println(" Measurement: " + measurementIterations + " x " + measurementSeconds + "s"); + System.out.println(" JVM args : " + String.join(" ", FIXED_JVM_ARGS)); + System.out.println(" Profiler : gc"); + System.out.println(" Result : " + resultFile); + System.out.println("====================================================="); + } + + private static String[] trim(String[] values) { + String[] trimmed = new String[values.length]; + for (int i = 0; i < values.length; i++) { + trimmed[i] = values[i].trim(); + } + return trimmed; + } + + private static String property(String name, String defaultValue) { + String value = System.getProperty(name); + return value == null || value.trim().isEmpty() ? defaultValue : value.trim(); + } + + private static int intProperty(String name, int defaultValue) { + String value = System.getProperty(name); + if (value == null || value.trim().isEmpty()) { + return defaultValue; + } + try { + return Integer.parseInt(value.trim()); + } catch (NumberFormatException e) { + System.out.println("WARN: cannot parse -D" + name + "=" + value + ", using default " + defaultValue); + return defaultValue; + } + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/comparison/ComparisonBenchmarkRunner.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/comparison/ComparisonBenchmarkRunner.java new file mode 100644 index 000000000..f0c147a6c --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/comparison/ComparisonBenchmarkRunner.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.comparison; + +import java.io.File; +import java.util.UUID; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** + * Comparison benchmark runner with file-based result collection. + * Runs Fesod vs Apache POI comparison benchmarks and exports results as JSON. + */ +public class ComparisonBenchmarkRunner { + + public static void main(String[] args) throws RunnerException { + System.out.println("Starting Fesod vs Apache POI Comparison Benchmark..."); + + // Generate unique session ID for this benchmark run + String sessionId = UUID.randomUUID().toString().substring(0, 8) + "_" + System.currentTimeMillis(); + String resultDirPath = "target/benchmark-results"; + File resultDir = new File(resultDirPath, sessionId); + + System.out.println("Session ID: " + sessionId); + System.out.println("Result directory: " + resultDir.getAbsolutePath()); + + // Ensure target directory exists + File targetDir = new File("target"); + if (!targetDir.exists()) { + targetDir.mkdirs(); + } + + // Configure benchmark options with session ID as system property + Options opt = new OptionsBuilder() + .include(FastExcelVsPoiBenchmark.class.getSimpleName()) + .param("datasetSize", "SMALL", "MEDIUM", "LARGE") + .param("fileFormat", "XLSX") + .forks(1) + .warmupIterations(3) + .measurementIterations(5) + .jvmArgs( + "-Xms2g", + "-Xmx2g", + "-XX:+UseG1GC", + "-Dbenchmark.session.id=" + sessionId, + "-Dbenchmark.result.dir=" + resultDirPath) + .result("target/jmh-results-" + sessionId + ".json") + .resultFormat(ResultFormatType.JSON) + .build(); + + // Run benchmarks + System.out.println("Starting benchmark execution..."); + new Runner(opt).run(); + System.out.println("Benchmark completed successfully!"); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/comparison/FastExcelVsPoiBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/comparison/FastExcelVsPoiBenchmark.java new file mode 100644 index 000000000..02246c9c5 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/comparison/FastExcelVsPoiBenchmark.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.comparison; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.EasyExcel; +import org.apache.fesod.sheet.ExcelReader; +import org.apache.fesod.sheet.ExcelWriter; +import org.apache.fesod.sheet.benchmark.core.AbstractBenchmark; +import org.apache.fesod.sheet.benchmark.core.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.utils.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.utils.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.apache.fesod.sheet.write.metadata.WriteSheet; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.util.IOUtils; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Comprehensive comparison benchmarks between Fesod (EasyExcel) and Apache POI. + * Tests performance across different operations and dataset sizes. + * + *

Benchmark best practices applied: + *

    + *
  • No manual timing - JMH handles all timing measurements
    • + *
  • No System.gc() calls - avoids unpredictable pauses
    • + *
  • Fixed heap size via @Fork JVM args for stable GC behavior
    • + *
  • Fair comparison - both libraries write/read the same columns
    • + *
  • Pre-loaded data in @Setup to exclude I/O from measurements
    • + *
+ */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) +@Fork( + value = 1, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +public class FastExcelVsPoiBenchmark extends AbstractBenchmark { + + @Param({"SMALL", "MEDIUM", "LARGE"}) + private String datasetSize; + + @Param({"XLSX", "XLS"}) + private String fileFormat; + + private File testFile; + private List testDataList; + + @Setup(Level.Trial) + public void setupTrial() throws Exception { + super.setupTrial(); + + // Configure Apache POI to handle large files + IOUtils.setByteArrayMaxOverride(1024 * 1024 * 1024); // 1GB + + // Generate test data using fixed seed for reproducibility + BenchmarkConfiguration.DatasetSize size = BenchmarkConfiguration.DatasetSize.valueOf(datasetSize); + int rowCount = size.getRowCount(); + testDataList = DataGenerator.generateTestData(size); + + BenchmarkConfiguration.FileFormat format = BenchmarkConfiguration.FileFormat.valueOf(fileFormat); + if (format == BenchmarkConfiguration.FileFormat.XLS && rowCount > 65535) { + System.out.printf( + "WARN: XLS format supports max 65536 rows, but dataset size is %d. Truncating to 65534 rows.%n", + rowCount); + testDataList = testDataList.subList(0, 65534); + rowCount = testDataList.size(); + } + + // Create test file for read benchmarks + String fileName = String.format("comparison_%s.%s", datasetSize.toLowerCase(), fileFormat.toLowerCase()); + testFile = BenchmarkFileUtil.createTestFile(fileName); + + // Pre-populate test file using Fesod + writeTestFile(); + + System.out.printf("Setup comparison benchmark: %s format, %d rows%n", fileFormat, rowCount); + } + + @TearDown(Level.Trial) + public void tearDownTrial() throws Exception { + if (testFile != null && testFile.exists()) { + testFile.delete(); + } + + super.tearDownTrial(); + } + + @Override + protected void setupBenchmark() throws Exception { + // No additional setup needed + } + + @Override + protected void tearDownBenchmark() throws Exception { + // No additional teardown needed + } + + // ============================================================================ + // WRITE OPERATION BENCHMARKS + // ============================================================================ + + /** + * Fesod (EasyExcel) write benchmark + */ + @Benchmark + @OperationsPerInvocation(1) + public long benchmarkFesodWrite(Blackhole blackhole) { + File outputFile = BenchmarkFileUtil.createTestFile(String.format( + "fesod_write_%s_%s.%s", + datasetSize.toLowerCase(), + java.util.UUID.randomUUID().toString().substring(0, 8), + fileFormat.toLowerCase())); + + try { + ExcelWriter excelWriter = + EasyExcel.write(outputFile, BenchmarkData.class).build(); + WriteSheet writeSheet = EasyExcel.writerSheet("TestData").build(); + + excelWriter.write(testDataList, writeSheet); + excelWriter.finish(); + + blackhole.consume(outputFile); + } catch (Exception e) { + throw new RuntimeException("Fesod write failed", e); + } finally { + if (outputFile.exists()) { + outputFile.delete(); + } + } + + return testDataList.size(); + } + + /** + * Apache POI write benchmark - writes same columns as Fesod for fair comparison + */ + @Benchmark + @OperationsPerInvocation(1) + public long benchmarkPoiWrite(Blackhole blackhole) { + File outputFile = BenchmarkFileUtil.createTestFile(String.format( + "poi_write_%s_%s.%s", + datasetSize.toLowerCase(), + java.util.UUID.randomUUID().toString().substring(0, 8), + fileFormat.toLowerCase())); + + try (FileOutputStream fos = new FileOutputStream(outputFile)) { + Workbook workbook = createWorkbook(); + Sheet sheet = workbook.createSheet("TestData"); + + // Create header row matching BenchmarkData columns + Row headerRow = sheet.createRow(0); + String[] headers = { + "ID", "String Data", "Integer Value", "Long Value", "Double Value", + "BigDecimal Value", "Boolean Flag", "Date Value", "DateTime Value", "Category", + "Description", "Status", "Float Value", "Short Value", "Byte Value", + "Extra Data 1", "Extra Data 2", "Extra Data 3", "Extra Data 4", "Extra Data 5" + }; + for (int i = 0; i < headers.length; i++) { + headerRow.createCell(i).setCellValue(headers[i]); + } + + // Write data rows - same 20 columns as Fesod + for (int i = 0; i < testDataList.size(); i++) { + BenchmarkData data = testDataList.get(i); + Row row = sheet.createRow(i + 1); + + row.createCell(0).setCellValue(data.getId() != null ? data.getId() : 0); + row.createCell(1).setCellValue(data.getStringData() != null ? data.getStringData() : ""); + row.createCell(2).setCellValue(data.getIntValue() != null ? data.getIntValue() : 0); + row.createCell(3).setCellValue(data.getLongValue() != null ? data.getLongValue() : 0L); + row.createCell(4).setCellValue(data.getDoubleValue() != null ? data.getDoubleValue() : 0.0); + row.createCell(5) + .setCellValue( + data.getBigDecimalValue() != null + ? data.getBigDecimalValue().doubleValue() + : 0.0); + row.createCell(6).setCellValue(data.getBooleanFlag() != null ? data.getBooleanFlag() : false); + if (data.getDateValue() != null) { + row.createCell(7).setCellValue(data.getDateValue().toString()); + } + if (data.getDateTimeValue() != null) { + row.createCell(8).setCellValue(data.getDateTimeValue().toString()); + } + row.createCell(9).setCellValue(data.getCategory() != null ? data.getCategory() : ""); + row.createCell(10).setCellValue(data.getDescription() != null ? data.getDescription() : ""); + row.createCell(11).setCellValue(data.getStatus() != null ? data.getStatus() : ""); + row.createCell(12).setCellValue(data.getFloatValue() != null ? data.getFloatValue() : 0.0f); + row.createCell(13).setCellValue(data.getShortValue() != null ? data.getShortValue() : 0); + row.createCell(14).setCellValue(data.getByteValue() != null ? data.getByteValue() : 0); + row.createCell(15).setCellValue(data.getExtraData1() != null ? data.getExtraData1() : ""); + row.createCell(16).setCellValue(data.getExtraData2() != null ? data.getExtraData2() : ""); + row.createCell(17).setCellValue(data.getExtraData3() != null ? data.getExtraData3() : ""); + row.createCell(18).setCellValue(data.getExtraData4() != null ? data.getExtraData4() : ""); + row.createCell(19).setCellValue(data.getExtraData5() != null ? data.getExtraData5() : ""); + } + + workbook.write(fos); + workbook.close(); + + blackhole.consume(outputFile); + + } catch (Exception e) { + throw new RuntimeException("POI write failed", e); + } finally { + if (outputFile.exists()) { + outputFile.delete(); + } + } + + return testDataList.size(); + } + + // ============================================================================ + // READ OPERATION BENCHMARKS + // ============================================================================ + + /** + * Fesod (EasyExcel) read benchmark + */ + @Benchmark + @OperationsPerInvocation(1) + public long benchmarkFesodRead(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + + try { + ExcelReader excelReader = EasyExcel.read(testFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData data, AnalysisContext context) { + processedRows.incrementAndGet(); + blackhole.consume(data); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // Processing complete + } + }) + .build(); + + excelReader.readAll(); + excelReader.finish(); + + } catch (Exception e) { + throw new RuntimeException("Fesod read failed", e); + } + + return processedRows.get(); + } + + /** + * Apache POI read benchmark + */ + @Benchmark + @OperationsPerInvocation(1) + public long benchmarkPoiRead(Blackhole blackhole) { + long processedRows = 0; + + try (FileInputStream fis = new FileInputStream(testFile)) { + Workbook workbook = WorkbookFactory.create(fis); + Sheet sheet = workbook.getSheetAt(0); + + for (Row row : sheet) { + if (row.getRowNum() == 0) continue; // Skip header + + for (Cell cell : row) { + blackhole.consume(cell.toString()); + } + + processedRows++; + } + + workbook.close(); + + } catch (Exception e) { + throw new RuntimeException("POI read failed", e); + } + + return processedRows; + } + + // ============================================================================ + // STREAMING OPERATION BENCHMARKS + // ============================================================================ + + /** + * Fesod (EasyExcel) streaming read benchmark with batch processing + */ + @Benchmark + @OperationsPerInvocation(1) + public long benchmarkFesodStreamingRead(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + List batch = new ArrayList<>(); + int batchSize = 1000; + + try { + ExcelReader excelReader = EasyExcel.read(testFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData data, AnalysisContext context) { + batch.add(data); + processedRows.incrementAndGet(); + + if (batch.size() >= batchSize) { + blackhole.consume(new ArrayList<>(batch)); + batch.clear(); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + if (!batch.isEmpty()) { + blackhole.consume(batch); + batch.clear(); + } + } + }) + .build(); + + excelReader.readAll(); + excelReader.finish(); + + } catch (Exception e) { + throw new RuntimeException("Fesod streaming read failed", e); + } + + return processedRows.get(); + } + + /** + * Apache POI streaming read benchmark using batch processing approach + */ + @Benchmark + @OperationsPerInvocation(1) + public long benchmarkPoiStreamingRead(Blackhole blackhole) { + long processedRows = 0; + + try (FileInputStream fis = new FileInputStream(testFile)) { + Workbook workbook = WorkbookFactory.create(fis); + Sheet sheet = workbook.getSheetAt(0); + + for (Row row : sheet) { + if (row.getRowNum() == 0) continue; // Skip header + + for (Cell cell : row) { + blackhole.consume(cell.toString()); + } + + processedRows++; + } + + workbook.close(); + + } catch (Exception e) { + throw new RuntimeException("POI streaming read failed", e); + } + + return processedRows; + } + + // ============================================================================ + // UTILITY METHODS + // ============================================================================ + + /** + * Create appropriate workbook based on file format + */ + private Workbook createWorkbook() { + return "XLSX".equals(fileFormat) ? new XSSFWorkbook() : new HSSFWorkbook(); + } + + /** + * Write test data to file for read benchmarks + */ + private void writeTestFile() { + try { + EasyExcel.write(testFile, BenchmarkData.class).sheet("TestData").doWrite(testDataList); + } catch (Exception e) { + throw new RuntimeException("Failed to write test file", e); + } + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/core/AbstractBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/core/AbstractBenchmark.java new file mode 100644 index 000000000..4f5d90aeb --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/core/AbstractBenchmark.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.core; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; +import org.apache.fesod.sheet.benchmark.utils.MemoryProfiler; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract base class for all benchmarks providing common functionality + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = BenchmarkConfiguration.DEFAULT_WARMUP_ITERATIONS, time = 1) +@Measurement(iterations = BenchmarkConfiguration.DEFAULT_MEASUREMENT_ITERATIONS, time = 1) +@Fork(BenchmarkConfiguration.DEFAULT_FORK_COUNT) +public abstract class AbstractBenchmark { + + protected static final Logger logger = LoggerFactory.getLogger(AbstractBenchmark.class); + + protected MemoryProfiler memoryProfiler; + protected String outputDirectory; + protected String benchmarkName; + + @Setup(Level.Trial) + public void setupTrial() throws Exception { + benchmarkName = this.getClass().getSimpleName(); + outputDirectory = BenchmarkConfiguration.DEFAULT_OUTPUT_DIR + File.separator + benchmarkName; + + // Create output directories + createDirectories(); + + // Initialize memory profiler if enabled + if (BenchmarkConfiguration.ENABLE_MEMORY_PROFILING) { + memoryProfiler = new MemoryProfiler(); + } + + logger.info("Setting up benchmark: {}", benchmarkName); + setupBenchmark(); + } + + @TearDown(Level.Trial) + public void tearDownTrial() throws Exception { + logger.info("Tearing down benchmark: {}", benchmarkName); + tearDownBenchmark(); + + if (memoryProfiler != null) { + memoryProfiler.stop(); + } + } + + @Setup(Level.Iteration) + public void setupIteration() throws Exception { + if (memoryProfiler != null) { + try { + memoryProfiler.reset(); + memoryProfiler.start(); + } catch (Exception e) { + logger.warn("Failed to start memory profiler: {}", e.getMessage()); + // Continue without memory profiling + } + } + setupIteration0(); + } + + @TearDown(Level.Iteration) + public void tearDownIteration() throws Exception { + tearDownIteration0(); + + if (memoryProfiler != null) { + try { + memoryProfiler.stop(); + logMemoryUsage(); + } catch (Exception e) { + logger.warn("Failed to stop memory profiler: {}", e.getMessage()); + // Continue without memory profiling + } + } + } + + /** + * Template method for benchmark-specific setup + */ + protected abstract void setupBenchmark() throws Exception; + + /** + * Template method for benchmark-specific teardown + */ + protected abstract void tearDownBenchmark() throws Exception; + + /** + * Template method for iteration-specific setup + */ + protected void setupIteration0() throws Exception { + // Default implementation does nothing + } + + /** + * Template method for iteration-specific teardown + */ + protected void tearDownIteration0() throws Exception { + // Default implementation does nothing + } + + /** + * Create necessary output directories + */ + private void createDirectories() throws IOException { + Path outputPath = Paths.get(outputDirectory); + if (!Files.exists(outputPath)) { + Files.createDirectories(outputPath); + } + } + + /** + * Log memory usage information + */ + private void logMemoryUsage() { + if (memoryProfiler != null) { + MemoryProfiler.MemorySnapshot snapshot = memoryProfiler.getSnapshot(); + logger.info( + "Memory usage - Max: {} MB, Avg: {} MB, Allocated: {} MB, GC Count: {}, GC Time: {} ms", + snapshot.getMaxUsedMemoryMB(), + snapshot.getAvgUsedMemoryMB(), + snapshot.getAllocatedMemoryMB(), + snapshot.getGcCount(), + snapshot.getGcTime()); + } + } + + /** + * Get a temporary file path for the given format and size + */ + protected String getTempFilePath( + BenchmarkConfiguration.FileFormat format, BenchmarkConfiguration.DatasetSize size) { + return outputDirectory + File.separator + "temp_" + size.getLabel() + "." + format.getExtension(); + } + + /** + * Clean up temporary files + */ + protected void cleanupTempFiles() { + try { + Path outputPath = Paths.get(outputDirectory); + if (Files.exists(outputPath)) { + Files.walk(outputPath) + .filter(path -> path.getFileName().toString().startsWith("temp_")) + .forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + logger.warn("Failed to delete temp file: {}", path, e); + } + }); + } + } catch (IOException e) { + logger.warn("Failed to cleanup temp files", e); + } + } + + /** + * Consume data to prevent JVM optimizations + */ + protected void consumeData(Object data, Blackhole blackhole) { + blackhole.consume(data); + } + + /** + * Get current memory usage in bytes + */ + protected long getCurrentMemoryUsage() { + Runtime runtime = Runtime.getRuntime(); + return runtime.totalMemory() - runtime.freeMemory(); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/core/BenchmarkConfiguration.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/core/BenchmarkConfiguration.java new file mode 100644 index 000000000..333794d73 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/core/BenchmarkConfiguration.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.core; + +/** + * Configuration class for benchmark parameters + */ +public class BenchmarkConfiguration { + + /** + * Dataset sizes for benchmarks + */ + public enum DatasetSize { + SMALL(1000, "1K"), + MEDIUM(10000, "10K"), + LARGE(100000, "100K"), + EXTRA_LARGE(1000000, "1M"); + + private final int rowCount; + private final String label; + + DatasetSize(int rowCount, String label) { + this.rowCount = rowCount; + this.label = label; + } + + public int getRowCount() { + return rowCount; + } + + public String getLabel() { + return label; + } + } + + /** + * File formats supported for benchmarking + */ + public enum FileFormat { + XLSX("xlsx"), + XLS("xls"), + CSV("csv"); + + private final String extension; + + FileFormat(String extension) { + this.extension = extension; + } + + public String getExtension() { + return extension; + } + } + + /** + * Benchmark operation types + */ + public enum OperationType { + READ, + WRITE, + FILL + } + + // Default benchmark configuration + public static final int DEFAULT_WARMUP_ITERATIONS = 3; + public static final int DEFAULT_MEASUREMENT_ITERATIONS = 5; + public static final int DEFAULT_FORK_COUNT = 1; + public static final String DEFAULT_OUTPUT_DIR = "target/benchmark-results"; + + // Memory monitoring configuration + public static final boolean ENABLE_MEMORY_PROFILING = true; + public static final long MEMORY_SAMPLING_INTERVAL_MS = 100; +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java new file mode 100644 index 000000000..95800b8fc --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.data; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import org.apache.fesod.sheet.annotation.ExcelProperty; +import org.apache.fesod.sheet.annotation.format.DateTimeFormat; + +/** + * Standard benchmark data model with various data types for comprehensive testing + */ +public class BenchmarkData { + + @ExcelProperty(value = "ID", index = 0) + private Long id; + + @ExcelProperty(value = "String Data", index = 1) + private String stringData; + + @ExcelProperty(value = "Integer Value", index = 2) + private Integer intValue; + + @ExcelProperty(value = "Long Value", index = 3) + private Long longValue; + + @ExcelProperty(value = "Double Value", index = 4) + private Double doubleValue; + + @ExcelProperty(value = "BigDecimal Value", index = 5) + private BigDecimal bigDecimalValue; + + @ExcelProperty(value = "Boolean Flag", index = 6) + private Boolean booleanFlag; + + @ExcelProperty(value = "Date Value", index = 7) + @DateTimeFormat("yyyy-MM-dd") + private LocalDate dateValue; + + @ExcelProperty(value = "DateTime Value", index = 8) + @DateTimeFormat("yyyy-MM-dd HH:mm:ss") + private LocalDateTime dateTimeValue; + + @ExcelProperty(value = "Category", index = 9) + private String category; + + @ExcelProperty(value = "Description", index = 10) + private String description; + + @ExcelProperty(value = "Status", index = 11) + private String status; + + @ExcelProperty(value = "Float Value", index = 12) + private Float floatValue; + + @ExcelProperty(value = "Short Value", index = 13) + private Short shortValue; + + @ExcelProperty(value = "Byte Value", index = 14) + private Byte byteValue; + + @ExcelProperty(value = "Extra Data 1", index = 15) + private String extraData1; + + @ExcelProperty(value = "Extra Data 2", index = 16) + private String extraData2; + + @ExcelProperty(value = "Extra Data 3", index = 17) + private String extraData3; + + @ExcelProperty(value = "Extra Data 4", index = 18) + private String extraData4; + + @ExcelProperty(value = "Extra Data 5", index = 19) + private String extraData5; + + // Default constructor + public BenchmarkData() {} + + // Full constructor + public BenchmarkData( + Long id, + String stringData, + Integer intValue, + Long longValue, + Double doubleValue, + BigDecimal bigDecimalValue, + Boolean booleanFlag, + LocalDate dateValue, + LocalDateTime dateTimeValue, + String category, + String description, + String status, + Float floatValue, + Short shortValue, + Byte byteValue, + String extraData1, + String extraData2, + String extraData3, + String extraData4, + String extraData5) { + this.id = id; + this.stringData = stringData; + this.intValue = intValue; + this.longValue = longValue; + this.doubleValue = doubleValue; + this.bigDecimalValue = bigDecimalValue; + this.booleanFlag = booleanFlag; + this.dateValue = dateValue; + this.dateTimeValue = dateTimeValue; + this.category = category; + this.description = description; + this.status = status; + this.floatValue = floatValue; + this.shortValue = shortValue; + this.byteValue = byteValue; + this.extraData1 = extraData1; + this.extraData2 = extraData2; + this.extraData3 = extraData3; + this.extraData4 = extraData4; + this.extraData5 = extraData5; + } + + // Getters and setters + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getStringData() { + return stringData; + } + + public void setStringData(String stringData) { + this.stringData = stringData; + } + + public Integer getIntValue() { + return intValue; + } + + public void setIntValue(Integer intValue) { + this.intValue = intValue; + } + + public Long getLongValue() { + return longValue; + } + + public void setLongValue(Long longValue) { + this.longValue = longValue; + } + + public Double getDoubleValue() { + return doubleValue; + } + + public void setDoubleValue(Double doubleValue) { + this.doubleValue = doubleValue; + } + + public BigDecimal getBigDecimalValue() { + return bigDecimalValue; + } + + public void setBigDecimalValue(BigDecimal bigDecimalValue) { + this.bigDecimalValue = bigDecimalValue; + } + + public Boolean getBooleanFlag() { + return booleanFlag; + } + + public void setBooleanFlag(Boolean booleanFlag) { + this.booleanFlag = booleanFlag; + } + + public LocalDate getDateValue() { + return dateValue; + } + + public void setDateValue(LocalDate dateValue) { + this.dateValue = dateValue; + } + + public LocalDateTime getDateTimeValue() { + return dateTimeValue; + } + + public void setDateTimeValue(LocalDateTime dateTimeValue) { + this.dateTimeValue = dateTimeValue; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Float getFloatValue() { + return floatValue; + } + + public void setFloatValue(Float floatValue) { + this.floatValue = floatValue; + } + + public Short getShortValue() { + return shortValue; + } + + public void setShortValue(Short shortValue) { + this.shortValue = shortValue; + } + + public Byte getByteValue() { + return byteValue; + } + + public void setByteValue(Byte byteValue) { + this.byteValue = byteValue; + } + + public String getExtraData1() { + return extraData1; + } + + public void setExtraData1(String extraData1) { + this.extraData1 = extraData1; + } + + public String getExtraData2() { + return extraData2; + } + + public void setExtraData2(String extraData2) { + this.extraData2 = extraData2; + } + + public String getExtraData3() { + return extraData3; + } + + public void setExtraData3(String extraData3) { + this.extraData3 = extraData3; + } + + public String getExtraData4() { + return extraData4; + } + + public void setExtraData4(String extraData4) { + this.extraData4 = extraData4; + } + + public String getExtraData5() { + return extraData5; + } + + public void setExtraData5(String extraData5) { + this.extraData5 = extraData5; + } + + @Override + public String toString() { + return "BenchmarkData{" + "id=" + + id + ", stringData='" + + stringData + '\'' + ", intValue=" + + intValue + ", longValue=" + + longValue + ", doubleValue=" + + doubleValue + ", bigDecimalValue=" + + bigDecimalValue + ", booleanFlag=" + + booleanFlag + ", dateValue=" + + dateValue + ", dateTimeValue=" + + dateTimeValue + ", category='" + + category + '\'' + ", description='" + + description + '\'' + ", status='" + + status + '\'' + ", floatValue=" + + floatValue + ", shortValue=" + + shortValue + ", byteValue=" + + byteValue + ", extraData1='" + + extraData1 + '\'' + ", extraData2='" + + extraData2 + '\'' + ", extraData3='" + + extraData3 + '\'' + ", extraData4='" + + extraData4 + '\'' + ", extraData5='" + + extraData5 + '\'' + '}'; + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/FillBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/FillBenchmark.java new file mode 100644 index 000000000..e898b2c5a --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/FillBenchmark.java @@ -0,0 +1,590 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.operations; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.fesod.sheet.EasyExcel; +import org.apache.fesod.sheet.ExcelWriter; +import org.apache.fesod.sheet.benchmark.core.AbstractBenchmark; +import org.apache.fesod.sheet.benchmark.core.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.utils.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.utils.DataGenerator; +import org.apache.fesod.sheet.enums.WriteDirectionEnum; +import org.apache.fesod.sheet.write.metadata.WriteSheet; +import org.apache.fesod.sheet.write.metadata.fill.FillConfig; +import org.apache.fesod.sheet.write.metadata.fill.FillWrapper; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Comprehensive benchmarks for FastExcel fill operations + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork( + value = 1, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +public class FillBenchmark extends AbstractBenchmark { + + // Template files for different scenarios + private String simpleTemplateFile; + private String complexTemplateFile; + private String horizontalTemplateFile; + private String verticalTemplateFile; + private String multiListTemplateFile; + + // Test data for different sizes + private List smallData; + private List mediumData; + private List largeData; + + // Single objects for simple fills + private BenchmarkData singleData; + private Map simpleMap; + private Map complexMap; + + // Fill configurations + private FillConfig verticalConfig; + private FillConfig horizontalConfig; + private FillConfig forceNewRowConfig; + + // Data generator + private DataGenerator dataGenerator; + + @Override + protected void setupBenchmark() throws Exception { + logger.info("Setting up fill benchmark templates and data..."); + + dataGenerator = new DataGenerator(); + + // Generate test data + generateTestData(); + + // Create template files + createTemplateFiles(); + + // Setup fill configurations + setupFillConfigurations(); + + logger.info("Fill benchmark setup completed"); + } + + @Override + protected void tearDownBenchmark() throws Exception { + // Clean up temporary files + BenchmarkFileUtil.cleanupTempFiles(); + logger.info("Fill benchmark cleanup completed"); + } + + private void generateTestData() { + // Generate data for different sizes + smallData = dataGenerator.generateData(BenchmarkConfiguration.DatasetSize.SMALL); + mediumData = dataGenerator.generateData(BenchmarkConfiguration.DatasetSize.MEDIUM); + largeData = dataGenerator.generateData(BenchmarkConfiguration.DatasetSize.LARGE); + + // Single object for simple fills + singleData = smallData.get(0); + + // Simple map for template variable filling + simpleMap = new HashMap<>(); + simpleMap.put("title", "Benchmark Report"); + simpleMap.put("date", LocalDate.now().toString()); + simpleMap.put("dateTime", LocalDateTime.now().toString()); + simpleMap.put("total", 12345.67); + simpleMap.put("count", 1000); + simpleMap.put("author", "FastExcel Benchmark"); + + // Complex map with nested data + complexMap = new HashMap<>(); + complexMap.put("reportTitle", "Performance Analysis Report"); + complexMap.put("generatedDate", LocalDate.now()); + complexMap.put("generatedTime", LocalDateTime.now()); + complexMap.put("totalRecords", largeData.size()); + complexMap.put("avgProcessingTime", 123.45); + complexMap.put("maxMemoryUsage", "256MB"); + complexMap.put( + "summary", "This is a comprehensive performance analysis report generated by FastExcel benchmarks."); + + logger.debug( + "Generated test data - Small: {}, Medium: {}, Large: {} rows", + smallData.size(), + mediumData.size(), + largeData.size()); + } + + private void createTemplateFiles() { + // Create simple template with basic placeholders + simpleTemplateFile = createSimpleTemplate(); + + // Create complex template with multiple data types + complexTemplateFile = createComplexTemplate(); + + // Create horizontal fill template + horizontalTemplateFile = createHorizontalTemplate(); + + // Create vertical fill template + verticalTemplateFile = createVerticalTemplate(); + + // Create multi-list template + multiListTemplateFile = createMultiListTemplate(); + } + + private String createSimpleTemplate() { + String templatePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.SMALL, "SimpleTemplate"); + + // Create a simple template with placeholder rows + Map row1 = new HashMap<>(); + row1.put("name", "Simple Fill Test"); + row1.put("date", "2023-01-01"); + row1.put("version", "1.0"); + + Map row2 = new HashMap<>(); + row2.put("description", "This is a simple fill test"); + row2.put("author", "Test Author"); + row2.put("status", "Active"); + + List> templateData = new ArrayList<>(); + templateData.add(row1); + templateData.add(row2); + + try { + // Write template structure + EasyExcel.write(templatePath).sheet("Template").doWrite(templateData); + + logger.debug("Created simple template: {}", templatePath); + return templatePath; + } catch (Exception e) { + throw new RuntimeException("Failed to create simple template", e); + } + } + + private String createComplexTemplate() { + String templatePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, "ComplexTemplate"); + + // Create a more complex template with data list placeholders + List> templateData = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("A", "{reportTitle}"); + row1.put("B", ""); + row1.put("C", ""); + templateData.add(row1); + + Map row2 = new HashMap<>(); + row2.put("A", "Generated on: {generatedDate}"); + row2.put("B", "Time: {generatedTime}"); + row2.put("C", ""); + templateData.add(row2); + + Map row3 = new HashMap<>(); + row3.put("A", "Total Records: {totalRecords}"); + row3.put("B", "Avg Time: {avgProcessingTime}ms"); + row3.put("C", "Max Memory: {maxMemoryUsage}"); + templateData.add(row3); + + Map row4 = new HashMap<>(); + row4.put("A", ""); + row4.put("B", ""); + row4.put("C", ""); + templateData.add(row4); + + Map row5 = new HashMap<>(); + row5.put("A", "Summary: {summary}"); + row5.put("B", ""); + row5.put("C", ""); + templateData.add(row5); + + Map row6 = new HashMap<>(); + row6.put("A", ""); + row6.put("B", ""); + row6.put("C", ""); + templateData.add(row6); + + Map row7 = new HashMap<>(); + row7.put("A", "ID"); + row7.put("B", "String Data"); + row7.put("C", "Value"); + templateData.add(row7); + + Map row8 = new HashMap<>(); + row8.put("A", "{.id}"); + row8.put("B", "{.stringData}"); + row8.put("C", "{.intValue}"); + templateData.add(row8); + + try { + EasyExcel.write(templatePath).sheet("ComplexTemplate").doWrite(templateData); + + logger.debug("Created complex template: {}", templatePath); + return templatePath; + } catch (Exception e) { + throw new RuntimeException("Failed to create complex template", e); + } + } + + private String createHorizontalTemplate() { + String templatePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.MEDIUM, + "HorizontalTemplate"); + + // Create horizontal fill template + Map row1 = new HashMap<>(); + row1.put("A", "Horizontal Fill Demo"); + row1.put("B", ""); + row1.put("C", ""); + + Map row2 = new HashMap<>(); + row2.put("A", "{.id}"); + row2.put("B", "{.stringData}"); + row2.put("C", "{.intValue}"); + + List> templateData = new ArrayList<>(); + templateData.add(row1); + templateData.add(row2); + + try { + EasyExcel.write(templatePath).sheet("HorizontalTemplate").doWrite(templateData); + + logger.debug("Created horizontal template: {}", templatePath); + return templatePath; + } catch (Exception e) { + throw new RuntimeException("Failed to create horizontal template", e); + } + } + + private String createVerticalTemplate() { + String templatePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, "VerticalTemplate"); + + // Create vertical fill template + Map row1 = new HashMap<>(); + row1.put("A", "Dynamic Fill Test"); + row1.put("B", "Status"); + row1.put("C", "Priority"); + + Map row2 = new HashMap<>(); + row2.put("A", "{.id}"); + row2.put("B", "{.status}"); + row2.put("C", "{.priority}"); + + List> templateData = new ArrayList<>(); + templateData.add(row1); + templateData.add(row2); + + try { + EasyExcel.write(templatePath).sheet("VerticalTemplate").doWrite(templateData); + + logger.debug("Created vertical template: {}", templatePath); + return templatePath; + } catch (Exception e) { + throw new RuntimeException("Failed to create vertical template", e); + } + } + + private String createMultiListTemplate() { + String templatePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.LARGE, "MultiListTemplate"); + + // Create multi-list template + Map row1 = new HashMap<>(); + row1.put("Report", "Performance Report"); + row1.put("Date", "{date}"); + row1.put("Version", "{version}"); + + Map row2 = new HashMap<>(); + row2.put("Metric", "Value"); + row2.put("Status", "Threshold"); + row2.put("Notes", "Comments"); + + List> templateData = new ArrayList<>(); + templateData.add(row1); + templateData.add(row2); + + try { + EasyExcel.write(templatePath).sheet("MultiListTemplate").doWrite(templateData); + + logger.debug("Created multi-list template: {}", templatePath); + return templatePath; + } catch (Exception e) { + throw new RuntimeException("Failed to create multi-list template", e); + } + } + + private void setupFillConfigurations() { + verticalConfig = + FillConfig.builder().direction(WriteDirectionEnum.VERTICAL).build(); + + horizontalConfig = + FillConfig.builder().direction(WriteDirectionEnum.HORIZONTAL).build(); + + forceNewRowConfig = FillConfig.builder() + .direction(WriteDirectionEnum.VERTICAL) + .forceNewRow(true) + .build(); + } + + // Simple fill benchmarks + @Benchmark + public void fillSimpleMap(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.SMALL, "FillSimpleMap"); + + EasyExcel.write(outputFile).withTemplate(simpleTemplateFile).sheet().doFill(simpleMap); + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void fillSingleObject(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.SMALL, "FillSingleObject"); + + EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(simpleTemplateFile) + .sheet() + .doFill(singleData); + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + // List fill benchmarks - different sizes + @Benchmark + public void fillSmallList(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.SMALL, "FillSmallList"); + + EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(complexTemplateFile) + .sheet() + .doFill(smallData); + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void fillMediumList(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, "FillMediumList"); + + EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(complexTemplateFile) + .sheet() + .doFill(mediumData); + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void fillLargeList(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.LARGE, "FillLargeList"); + + EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(complexTemplateFile) + .sheet() + .doFill(largeData); + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + // Directional fill benchmarks + @Benchmark + public void fillHorizontal(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, "FillHorizontal"); + + try (ExcelWriter excelWriter = EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(horizontalTemplateFile) + .build()) { + WriteSheet writeSheet = EasyExcel.writerSheet().build(); + excelWriter.fill(mediumData, horizontalConfig, writeSheet); + } + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void fillVertical(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, "FillVertical"); + + try (ExcelWriter excelWriter = EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(verticalTemplateFile) + .build()) { + WriteSheet writeSheet = EasyExcel.writerSheet().build(); + excelWriter.fill(mediumData, verticalConfig, writeSheet); + } + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void fillForceNewRow(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, "FillForceNewRow"); + + try (ExcelWriter excelWriter = EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(verticalTemplateFile) + .build()) { + WriteSheet writeSheet = EasyExcel.writerSheet().build(); + excelWriter.fill(mediumData, forceNewRowConfig, writeSheet); + } + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + // Multi-list fill benchmarks + @Benchmark + public void fillMultipleLists(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, "FillMultipleLists"); + + try (ExcelWriter excelWriter = + EasyExcel.write(outputFile).withTemplate(multiListTemplateFile).build()) { + WriteSheet writeSheet = EasyExcel.writerSheet().build(); + + // Fill multiple lists with different prefixes + excelWriter.fill(new FillWrapper("data1", smallData), writeSheet); + excelWriter.fill(new FillWrapper("data2", mediumData), writeSheet); + + // Fill summary data + Map summary = new HashMap<>(); + summary.put("total", smallData.size() + mediumData.size()); + summary.put("date", LocalDate.now().toString()); + excelWriter.fill(summary, writeSheet); + } + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + // Complex fill scenarios + @Benchmark + public void fillComplexMixed(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.LARGE, "FillComplexMixed"); + + try (ExcelWriter excelWriter = + EasyExcel.write(outputFile).withTemplate(complexTemplateFile).build()) { + WriteSheet writeSheet = EasyExcel.writerSheet().build(); + + // Fill header variables + excelWriter.fill(complexMap, writeSheet); + + // Fill data list + excelWriter.fill(largeData, forceNewRowConfig, writeSheet); + } + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + // Streaming fill benchmark + @Benchmark + public void fillStreaming(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.LARGE, "FillStreaming"); + + try (ExcelWriter excelWriter = EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(complexTemplateFile) + .build()) { + WriteSheet writeSheet = EasyExcel.writerSheet().build(); + + // Fill in batches to test streaming behavior + int batchSize = 1000; + for (int i = 0; i < largeData.size(); i += batchSize) { + int endIndex = Math.min(i + batchSize, largeData.size()); + List batch = largeData.subList(i, endIndex); + excelWriter.fill(batch, verticalConfig, writeSheet); + } + } + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } + + // Memory efficient fill benchmark + @Benchmark + public void fillMemoryEfficient(Blackhole blackhole) throws Exception { + String outputFile = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.LARGE, + "FillMemoryEfficient"); + + try (ExcelWriter excelWriter = EasyExcel.write(outputFile, BenchmarkData.class) + .withTemplate(complexTemplateFile) + .build()) { + WriteSheet writeSheet = EasyExcel.writerSheet().build(); + + // Generate and fill data on-the-fly to test memory efficiency + DataGenerator.DataStream dataStream = + dataGenerator.generateStreamingData(BenchmarkConfiguration.DatasetSize.LARGE.getRowCount()); + + List batch = new ArrayList<>(); + int batchSize = 500; + + for (BenchmarkData data : dataStream) { + batch.add(data); + + if (batch.size() >= batchSize) { + excelWriter.fill(batch, verticalConfig, writeSheet); + batch.clear(); + } + } + + // Fill remaining data + if (!batch.isEmpty()) { + excelWriter.fill(batch, verticalConfig, writeSheet); + } + } + + long fileSize = BenchmarkFileUtil.getFileSize(outputFile); + consumeData(fileSize, blackhole); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/ReadBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/ReadBenchmark.java new file mode 100644 index 000000000..c50fe50e9 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/ReadBenchmark.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.operations; + +import java.io.FileInputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.EasyExcel; +import org.apache.fesod.sheet.benchmark.core.AbstractBenchmark; +import org.apache.fesod.sheet.benchmark.core.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.utils.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.utils.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Comprehensive benchmarks for FastExcel read operations + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork( + value = 1, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +public class ReadBenchmark extends AbstractBenchmark { + + // Test files for different sizes and formats + private String xlsxSmallFile; + private String xlsxMediumFile; + private String xlsxLargeFile; + private String xlsxExtraLargeFile; + + private String csvSmallFile; + private String csvMediumFile; + private String csvLargeFile; + private String csvExtraLargeFile; + + @Override + protected void setupBenchmark() throws Exception { + logger.info("Setting up read benchmark test files..."); + + // Generate test files for all sizes and formats + generateTestFiles(); + + logger.info("Read benchmark setup completed"); + } + + @Override + protected void tearDownBenchmark() throws Exception { + // Clean up temporary files + BenchmarkFileUtil.cleanupTempFiles(); + logger.info("Read benchmark cleanup completed"); + } + + private void generateTestFiles() { + DataGenerator generator = new DataGenerator(); + + // Generate XLSX files + xlsxSmallFile = generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.SMALL, generator); + xlsxMediumFile = generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, generator); + xlsxLargeFile = generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.LARGE, generator); + xlsxExtraLargeFile = generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.EXTRA_LARGE, generator); + + // Generate CSV files + csvSmallFile = generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat.CSV, BenchmarkConfiguration.DatasetSize.SMALL, generator); + csvMediumFile = generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat.CSV, BenchmarkConfiguration.DatasetSize.MEDIUM, generator); + csvLargeFile = generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat.CSV, BenchmarkConfiguration.DatasetSize.LARGE, generator); + csvExtraLargeFile = generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat.CSV, BenchmarkConfiguration.DatasetSize.EXTRA_LARGE, generator); + } + + private String generateAndWriteTestFile( + BenchmarkConfiguration.FileFormat format, + BenchmarkConfiguration.DatasetSize size, + DataGenerator generator) { + String filePath = BenchmarkFileUtil.getTempFilePath(format, size, "ReadBenchmark"); + List data = generator.generateData(size); + + try { + EasyExcel.write(filePath, BenchmarkData.class) + .sheet("BenchmarkData") + .doWrite(data); + + logger.debug( + "Generated test file: {} ({} rows, {})", + filePath, + size.getRowCount(), + BenchmarkFileUtil.getFileSizeFormatted(filePath)); + return filePath; + } catch (Exception e) { + throw new RuntimeException("Failed to generate test file: " + filePath, e); + } + } + + // XLSX Read Benchmarks - Different sizes + @Benchmark + public void readXlsxSmall(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(xlsxSmallFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + + @Benchmark + public void readXlsxMedium(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(xlsxMediumFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + + @Benchmark + public void readXlsxLarge(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(xlsxLargeFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + + @Benchmark + public void readXlsxExtraLarge(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(xlsxExtraLargeFile, BenchmarkData.class, listener) + .sheet() + .doRead(); + consumeData(listener.getCount(), blackhole); + } + + // CSV Read Benchmarks - Different sizes + @Benchmark + public void readCsvSmall(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(csvSmallFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + + @Benchmark + public void readCsvMedium(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(csvMediumFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + + @Benchmark + public void readCsvLarge(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(csvLargeFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + + @Benchmark + public void readCsvExtraLarge(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(csvExtraLargeFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + + // Stream reading benchmarks + @Benchmark + public void readXlsxLargeWithStreaming(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + try (FileInputStream fis = new FileInputStream(xlsxLargeFile)) { + EasyExcel.read(fis, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + } + + @Benchmark + public void readCsvLargeWithStreaming(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + try (FileInputStream fis = new FileInputStream(csvLargeFile)) { + EasyExcel.read(fis, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + } + + // Different listener types benchmarks + @Benchmark + public void readXlsxLargeCountingOnly(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(xlsxLargeFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getCount(), blackhole); + } + + @Benchmark + public void readXlsxLargeCollecting(Blackhole blackhole) throws Exception { + CollectingReadListener listener = new CollectingReadListener(); + EasyExcel.read(xlsxLargeFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getData().size(), blackhole); + } + + @Benchmark + public void readXlsxLargeProcessing(Blackhole blackhole) throws Exception { + ProcessingReadListener listener = new ProcessingReadListener(); + EasyExcel.read(xlsxLargeFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getProcessedCount(), blackhole); + } + + // Head configuration benchmarks + @Benchmark + public void readXlsxLargeWithHeadRowNumber(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(xlsxLargeFile, BenchmarkData.class, listener) + .headRowNumber(1) + .sheet() + .doRead(); + consumeData(listener.getCount(), blackhole); + } + + @Benchmark + public void readXlsxLargeSkipRows(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + EasyExcel.read(xlsxLargeFile, BenchmarkData.class, listener) + .headRowNumber(2) // Skip first row + .sheet() + .doRead(); + consumeData(listener.getCount(), blackhole); + } + + // Multiple sheets reading (using same file) + @Benchmark + public void readXlsxMultipleSheets(Blackhole blackhole) throws Exception { + CountingReadListener listener = new CountingReadListener(); + // Read the same sheet 3 times to simulate multi-sheet processing + for (int i = 0; i < 3; i++) { + EasyExcel.read(xlsxMediumFile, BenchmarkData.class, listener) + .sheet(0) // Always read first sheet since our test files have only one + .doRead(); + } + consumeData(listener.getCount(), blackhole); + } + + // Memory efficient reading with limited collections + @Benchmark + public void readXlsxLargeMemoryEfficient(Blackhole blackhole) throws Exception { + LimitedCollectingReadListener listener = new LimitedCollectingReadListener(1000); + EasyExcel.read(xlsxLargeFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getData().size(), blackhole); + } + + // Error handling benchmark + @Benchmark + public void readXlsxWithErrorHandling(Blackhole blackhole) throws Exception { + ErrorHandlingReadListener listener = new ErrorHandlingReadListener(); + EasyExcel.read(xlsxLargeFile, BenchmarkData.class, listener).sheet().doRead(); + consumeData(listener.getProcessedCount(), blackhole); + } + + // Read Listeners + private static class CountingReadListener implements ReadListener { + private final AtomicLong count = new AtomicLong(0); + + @Override + public void invoke(BenchmarkData data, AnalysisContext context) { + count.incrementAndGet(); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // Nothing to do + } + + public long getCount() { + return count.get(); + } + + public void reset() { + count.set(0); + } + } + + private static class CollectingReadListener implements ReadListener { + private final List data = new ArrayList<>(); + + @Override + public void invoke(BenchmarkData item, AnalysisContext context) { + data.add(item); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // Nothing to do + } + + public List getData() { + return data; + } + + public void reset() { + data.clear(); + } + } + + private static class ProcessingReadListener implements ReadListener { + private final AtomicLong processedCount = new AtomicLong(0); + + @Override + public void invoke(BenchmarkData data, AnalysisContext context) { + processedCount.incrementAndGet(); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // Nothing to do + } + + public long getProcessedCount() { + return processedCount.get(); + } + + public void reset() { + processedCount.set(0); + } + } + + private static class LimitedCollectingReadListener implements ReadListener { + private final List data = new ArrayList<>(); + private final int maxSize; + + public LimitedCollectingReadListener(int maxSize) { + this.maxSize = maxSize; + } + + @Override + public void invoke(BenchmarkData item, AnalysisContext context) { + if (data.size() < maxSize) { + data.add(item); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // Nothing to do + } + + public List getData() { + return data; + } + + public void reset() { + data.clear(); + } + } + + private static class ErrorHandlingReadListener implements ReadListener { + private final AtomicLong processedCount = new AtomicLong(0); + private final AtomicLong errorCount = new AtomicLong(0); + + @Override + public void invoke(BenchmarkData data, AnalysisContext context) { + try { + // Simulate processing that might fail + if (data.getStringData() != null) { + processedCount.incrementAndGet(); + } + } catch (Exception e) { + errorCount.incrementAndGet(); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // Nothing to do + } + + public long getProcessedCount() { + return processedCount.get(); + } + + public long getErrorCount() { + return errorCount.get(); + } + + public void reset() { + processedCount.set(0); + errorCount.set(0); + } + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/WriteBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/WriteBenchmark.java new file mode 100644 index 000000000..d636d5d0a --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/operations/WriteBenchmark.java @@ -0,0 +1,444 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.operations; + +import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.fesod.sheet.EasyExcel; +import org.apache.fesod.sheet.ExcelWriter; +import org.apache.fesod.sheet.benchmark.core.AbstractBenchmark; +import org.apache.fesod.sheet.benchmark.core.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.utils.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.utils.DataGenerator; +import org.apache.fesod.sheet.write.metadata.WriteSheet; +import org.apache.fesod.sheet.write.metadata.WriteTable; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Comprehensive benchmarks for FastExcel write operations + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 3) +@Fork( + value = 1, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +public class WriteBenchmark extends AbstractBenchmark { + + // Test data for different sizes + private List smallData; + private List mediumData; + private List largeData; + private List extraLargeData; + + // Batch data for streaming tests + private List> smallBatches; + private List> mediumBatches; + private List> largeBatches; + + // Data generator + private DataGenerator dataGenerator; + + @Override + protected void setupBenchmark() throws Exception { + logger.info("Setting up write benchmark test data..."); + + dataGenerator = new DataGenerator(); + + // Generate test data sets + generateTestData(); + + logger.info("Write benchmark setup completed"); + } + + @Override + protected void tearDownBenchmark() throws Exception { + // Clean up temporary files + BenchmarkFileUtil.cleanupTempFiles(); + logger.info("Write benchmark cleanup completed"); + } + + private void generateTestData() { + // Generate data for different sizes + smallData = dataGenerator.generateData(BenchmarkConfiguration.DatasetSize.SMALL); + mediumData = dataGenerator.generateData(BenchmarkConfiguration.DatasetSize.MEDIUM); + largeData = dataGenerator.generateData(BenchmarkConfiguration.DatasetSize.LARGE); + extraLargeData = dataGenerator.generateData(BenchmarkConfiguration.DatasetSize.EXTRA_LARGE); + + // Generate batch data for streaming + smallBatches = dataGenerator.generateDataInBatches(BenchmarkConfiguration.DatasetSize.SMALL.getRowCount(), 100); + mediumBatches = + dataGenerator.generateDataInBatches(BenchmarkConfiguration.DatasetSize.MEDIUM.getRowCount(), 1000); + largeBatches = + dataGenerator.generateDataInBatches(BenchmarkConfiguration.DatasetSize.LARGE.getRowCount(), 5000); + + logger.debug( + "Generated test data - Small: {}, Medium: {}, Large: {}, Extra Large: {} rows", + smallData.size(), + mediumData.size(), + largeData.size(), + extraLargeData.size()); + } + + // XLSX Write Benchmarks - Different sizes + @Benchmark + public void writeXlsxSmall(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.SMALL, "WriteBenchmark"); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("BenchmarkData").doWrite(smallData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsxMedium(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.MEDIUM, "WriteBenchmark"); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("BenchmarkData").doWrite(mediumData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsxLarge(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, BenchmarkConfiguration.DatasetSize.LARGE, "WriteBenchmark"); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("BenchmarkData").doWrite(largeData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsxExtraLarge(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.EXTRA_LARGE, + "WriteBenchmark"); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("BenchmarkData").doWrite(extraLargeData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsSmall(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLS, BenchmarkConfiguration.DatasetSize.SMALL, "WriteBenchmark"); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("BenchmarkData").doWrite(smallData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeCsvMedium(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.CSV, BenchmarkConfiguration.DatasetSize.MEDIUM, "WriteBenchmark"); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("BenchmarkData").doWrite(mediumData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeCsvLarge(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.CSV, BenchmarkConfiguration.DatasetSize.LARGE, "WriteBenchmark"); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("BenchmarkData").doWrite(largeData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeCsvExtraLarge(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.CSV, + BenchmarkConfiguration.DatasetSize.EXTRA_LARGE, + "WriteBenchmark"); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("BenchmarkData").doWrite(extraLargeData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + // Streaming write benchmarks using ExcelWriter + @Benchmark + public void writeXlsxLargeStreaming(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.LARGE, + "StreamingWriteBenchmark"); + + try (ExcelWriter excelWriter = + EasyExcel.write(filePath, BenchmarkData.class).build()) { + WriteSheet writeSheet = EasyExcel.writerSheet("StreamingData").build(); + + for (List batch : largeBatches) { + excelWriter.write(batch, writeSheet); + } + } + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeCsvLargeStreaming(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.CSV, + BenchmarkConfiguration.DatasetSize.LARGE, + "StreamingWriteBenchmark"); + + try (ExcelWriter excelWriter = + EasyExcel.write(filePath, BenchmarkData.class).build()) { + WriteSheet writeSheet = EasyExcel.writerSheet("StreamingData").build(); + + for (List batch : largeBatches) { + excelWriter.write(batch, writeSheet); + } + } + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + // Multiple sheets writing + @Benchmark + public void writeXlsxMultipleSheets(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.MEDIUM, + "MultiSheetWriteBenchmark"); + + try (ExcelWriter excelWriter = + EasyExcel.write(filePath, BenchmarkData.class).build()) { + // Write to 3 different sheets + for (int i = 0; i < 3; i++) { + WriteSheet writeSheet = + EasyExcel.writerSheet(i, "Sheet" + (i + 1)).build(); + excelWriter.write(mediumData, writeSheet); + } + } + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsxToOutputStream(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.MEDIUM, + "OutputStreamWriteBenchmark"); + + try (FileOutputStream fos = new FileOutputStream(filePath)) { + EasyExcel.write(fos, BenchmarkData.class).sheet("OutputStreamData").doWrite(mediumData); + } + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsxTableFormat(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.LARGE, + "TableFormatWriteBenchmark"); + + try (ExcelWriter excelWriter = + EasyExcel.write(filePath, BenchmarkData.class).build()) { + WriteSheet writeSheet = EasyExcel.writerSheet("TableData").build(); + WriteTable writeTable = EasyExcel.writerTable(0).build(); + + // Write data in table format + for (List batch : largeBatches) { + excelWriter.write(batch, writeSheet, writeTable); + } + } + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsxMemoryEfficientBatches(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.LARGE, + "MemoryEfficientWriteBenchmark"); + + try (ExcelWriter excelWriter = + EasyExcel.write(filePath, BenchmarkData.class).build()) { + WriteSheet writeSheet = EasyExcel.writerSheet("BatchData").build(); + + // Write in small batches to reduce memory usage + int batchSize = 1000; + for (int i = 0; i < largeData.size(); i += batchSize) { + int endIndex = Math.min(i + batchSize, largeData.size()); + List batch = largeData.subList(i, endIndex); + excelWriter.write(batch, writeSheet); + } + } + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + // Dynamic data generation and writing + @Benchmark + public void writeXlsxDynamicGeneration(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.MEDIUM, + "DynamicWriteBenchmark"); + + try (ExcelWriter excelWriter = + EasyExcel.write(filePath, BenchmarkData.class).build()) { + WriteSheet writeSheet = EasyExcel.writerSheet("DynamicData").build(); + + // Generate and write data on-the-fly + DataGenerator.DataStream dataStream = + dataGenerator.generateStreamingData(BenchmarkConfiguration.DatasetSize.MEDIUM.getRowCount()); + + List batch = new ArrayList<>(); + int batchSize = 1000; + + for (BenchmarkData data : dataStream) { + batch.add(data); + + if (batch.size() >= batchSize) { + excelWriter.write(batch, writeSheet); + batch.clear(); + } + } + + // Write remaining data + if (!batch.isEmpty()) { + excelWriter.write(batch, writeSheet); + } + } + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + // Write with different data characteristics + @Benchmark + public void writeXlsxLargeStrings(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.MEDIUM, + "LargeStringsWriteBenchmark"); + + List largeStringData = dataGenerator.generateDataWithCharacteristics( + BenchmarkConfiguration.DatasetSize.MEDIUM.getRowCount(), + DataGenerator.DataCharacteristics.defaults().withLargeStrings()); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("LargeStringData").doWrite(largeStringData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsxRepeatedValues(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.MEDIUM, + "RepeatedValuesWriteBenchmark"); + + List repeatedData = dataGenerator.generateDataWithCharacteristics( + BenchmarkConfiguration.DatasetSize.MEDIUM.getRowCount(), + DataGenerator.DataCharacteristics.defaults().withRepeatedValues()); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("RepeatedData").doWrite(repeatedData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + @Benchmark + public void writeXlsxNullValues(Blackhole blackhole) throws Exception { + String filePath = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.MEDIUM, + "NullValuesWriteBenchmark"); + + List nullData = dataGenerator.generateDataWithCharacteristics( + BenchmarkConfiguration.DatasetSize.MEDIUM.getRowCount(), + DataGenerator.DataCharacteristics.defaults().withNullValues()); + + EasyExcel.write(filePath, BenchmarkData.class).sheet("NullData").doWrite(nullData); + + long fileSize = BenchmarkFileUtil.getFileSize(filePath); + consumeData(fileSize, blackhole); + } + + // Sequential writing of multiple files + @Benchmark + public void writeXlsxMultipleFiles(Blackhole blackhole) throws Exception { + // Write multiple files sequentially + String[] filePaths = new String[3]; + List[] dataSets = new List[] {smallData, mediumData, smallData}; + + for (int i = 0; i < 3; i++) { + filePaths[i] = BenchmarkFileUtil.getTempFilePath( + BenchmarkConfiguration.FileFormat.XLSX, + BenchmarkConfiguration.DatasetSize.SMALL, + "MultiFileWriteBenchmark_" + i); + + EasyExcel.write(filePaths[i], BenchmarkData.class).sheet("Data" + i).doWrite(dataSets[i]); + } + + long totalSize = 0; + for (String filePath : filePaths) { + totalSize += BenchmarkFileUtil.getFileSize(filePath); + } + + consumeData(totalSize, blackhole); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/BenchmarkFileUtil.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/BenchmarkFileUtil.java new file mode 100644 index 000000000..970150f55 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/BenchmarkFileUtil.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.utils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; +import org.apache.fesod.sheet.benchmark.core.BenchmarkConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for managing benchmark test files + */ +public class BenchmarkFileUtil { + + private static final Logger logger = LoggerFactory.getLogger(BenchmarkFileUtil.class); + + private static final String TEST_DATA_DIR = "target/benchmark-testdata"; + + /** + * Create test data directory if it doesn't exist + */ + public static void createTestDataDirectory() { + try { + Path testDataPath = Paths.get(TEST_DATA_DIR); + if (!Files.exists(testDataPath)) { + Files.createDirectories(testDataPath); + logger.debug("Created test data directory: {}", testDataPath); + } + } catch (IOException e) { + logger.error("Failed to create test data directory", e); + throw new RuntimeException("Failed to create test data directory", e); + } + } + + /** + * Generate a temporary file path for benchmarks + */ + public static String getTempFilePath( + BenchmarkConfiguration.FileFormat format, BenchmarkConfiguration.DatasetSize size, String benchmarkName) { + createTestDataDirectory(); + + String fileName = String.format( + "temp_%s_%s_%s_%s.%s", + benchmarkName, + size.getLabel(), + format.name().toLowerCase(), + UUID.randomUUID().toString().substring(0, 8), + format.getExtension()); + return TEST_DATA_DIR + File.separator + fileName; + } + + /** + * Clean up temporary files created during benchmarks + */ + public static void cleanupTempFiles() { + try { + Path testDataPath = Paths.get(TEST_DATA_DIR); + if (Files.exists(testDataPath)) { + Files.walk(testDataPath) + .filter(path -> path.getFileName().toString().startsWith("temp_")) + .forEach(path -> { + try { + Files.deleteIfExists(path); + logger.debug("Deleted temp file: {}", path); + } catch (IOException e) { + logger.warn("Failed to delete temp file: {}", path, e); + } + }); + } + } catch (IOException e) { + logger.warn("Failed to cleanup temp files", e); + } + } + + /** + * Get file size in bytes + */ + public static long getFileSize(String filePath) { + try { + return Files.size(Paths.get(filePath)); + } catch (IOException e) { + logger.warn("Failed to get file size for: {}", filePath, e); + return 0; + } + } + + /** + * Get file size in human readable format + */ + public static String getFileSizeFormatted(String filePath) { + long bytes = getFileSize(filePath); + return formatBytes(bytes); + } + + /** + * Format bytes into human readable format + */ + public static String formatBytes(long bytes) { + if (bytes < 1024) return bytes + " B"; + int exp = (int) (Math.log(bytes) / Math.log(1024)); + String pre = "KMGTPE".charAt(exp - 1) + ""; + return String.format("%.1f %sB", bytes / Math.pow(1024, exp), pre); + } + + /** + * Create a test file with the specified name + */ + public static File createTestFile(String fileName) { + createTestDataDirectory(); + return new File(TEST_DATA_DIR, fileName); + } + + /** + * Read a string from a file using UTF-8 encoding + * @param path the path to the file + * @return the content of the file as a string + * @throws IOException if an I/O error occurs + */ + public static String readString(Path path) throws IOException { + return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/DataGenerator.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/DataGenerator.java new file mode 100644 index 000000000..58b2bfca8 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/DataGenerator.java @@ -0,0 +1,433 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.utils; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.apache.fesod.sheet.benchmark.core.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for generating test data for benchmarks + */ +public class DataGenerator { + + private static final Logger logger = LoggerFactory.getLogger(DataGenerator.class); + + // Predefined data sets for realistic data generation + private static final String[] CATEGORIES = { + "Electronics", + "Books", + "Clothing", + "Home & Garden", + "Sports", + "Automotive", + "Health & Beauty", + "Toys & Games", + "Food & Beverage", + "Office Supplies" + }; + + private static final String[] STATUSES = { + "Active", "Inactive", "Pending", "Processing", "Completed", "Cancelled", "On Hold" + }; + + private static final String[] SAMPLE_WORDS = { + "Lorem", + "ipsum", + "dolor", + "sit", + "amet", + "consectetur", + "adipiscing", + "elit", + "sed", + "do", + "eiusmod", + "tempor", + "incididunt", + "ut", + "labore", + "et", + "dolore", + "magna", + "aliqua", + "enim", + "ad", + "minim", + "veniam", + "quis", + "nostrud", + "exercitation", + "ullamco", + "laboris", + "nisi", + "aliquip", + "ex", + "ea", + "commodo" + }; + + private final Random random; + + public DataGenerator() { + this(42L); // Fixed seed for reproducible benchmark results + } + + public DataGenerator(long seed) { + this.random = new Random(seed); + } + + /** + * Generate a list of benchmark data with the specified size + */ + public List generateData(BenchmarkConfiguration.DatasetSize size) { + return generateData(size.getRowCount()); + } + + /** + * Generate a list of benchmark data with the specified row count + */ + public List generateData(int rowCount) { + logger.info("Generating {} rows of benchmark data", rowCount); + + List data = new ArrayList<>(rowCount); + long startTime = System.currentTimeMillis(); + + for (int i = 0; i < rowCount; i++) { + data.add(generateSingleRow(i + 1)); + + // Log progress for large datasets + if (rowCount > 10000 && i > 0 && i % 10000 == 0) { + logger.debug("Generated {} rows", i); + } + } + + long duration = System.currentTimeMillis() - startTime; + logger.info( + "Generated {} rows in {} ms ({} rows/sec)", + rowCount, + duration, + duration > 0 ? (rowCount * 1000 / duration) : "N/A"); + + return data; + } + + /** + * Generate benchmark data in batches to control memory usage + */ + public List> generateDataInBatches(int totalRows, int batchSize) { + logger.info("Generating {} rows in batches of {}", totalRows, batchSize); + + List> batches = new ArrayList<>(); + int remainingRows = totalRows; + int currentBatch = 1; + int startId = 1; + + while (remainingRows > 0) { + int currentBatchSize = Math.min(batchSize, remainingRows); + List batch = new ArrayList<>(currentBatchSize); + + for (int i = 0; i < currentBatchSize; i++) { + batch.add(generateSingleRow(startId + i)); + } + + batches.add(batch); + remainingRows -= currentBatchSize; + startId += currentBatchSize; + + logger.debug("Generated batch {} with {} rows", currentBatch++, currentBatchSize); + } + + logger.info("Generated {} batches totaling {} rows", batches.size(), totalRows); + return batches; + } + + /** + * Generate a single row of benchmark data + */ + private BenchmarkData generateSingleRow(long id) { + BenchmarkData data = new BenchmarkData(); + + data.setId(id); + data.setStringData(generateRandomString(10, 50)); + data.setIntValue(random.nextInt(1000000)); + data.setLongValue(random.nextLong()); + data.setDoubleValue(random.nextDouble() * 1000000); + data.setBigDecimalValue( + BigDecimal.valueOf(random.nextDouble() * 1000000).setScale(2, RoundingMode.HALF_UP)); + data.setBooleanFlag(random.nextBoolean()); + data.setDateValue(generateRandomDate()); + data.setDateTimeValue(generateRandomDateTime()); + data.setCategory(CATEGORIES[random.nextInt(CATEGORIES.length)]); + data.setDescription(generateRandomDescription()); + data.setStatus(STATUSES[random.nextInt(STATUSES.length)]); + data.setFloatValue(random.nextFloat() * 1000); + data.setShortValue((short) random.nextInt(Short.MAX_VALUE)); + data.setByteValue((byte) random.nextInt(Byte.MAX_VALUE)); + data.setExtraData1(generateRandomString(5, 20)); + data.setExtraData2(generateRandomString(5, 20)); + data.setExtraData3(generateRandomString(5, 20)); + data.setExtraData4(generateRandomString(5, 20)); + data.setExtraData5(generateRandomString(5, 20)); + + return data; + } + + /** + * Generate random string with variable length + */ + private String generateRandomString(int minLength, int maxLength) { + int length = random.nextInt(maxLength - minLength + 1) + minLength; + StringBuilder sb = new StringBuilder(length); + + for (int i = 0; i < length; i++) { + if (random.nextBoolean()) { + // Add random letter + sb.append((char) ('a' + random.nextInt(26))); + } else { + // Add random digit + sb.append((char) ('0' + random.nextInt(10))); + } + } + + return sb.toString(); + } + + /** + * Generate random description using sample words + */ + private String generateRandomDescription() { + int wordCount = random.nextInt(8) + 3; // 3-10 words + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < wordCount; i++) { + if (i > 0) { + sb.append(" "); + } + sb.append(SAMPLE_WORDS[random.nextInt(SAMPLE_WORDS.length)]); + } + + return sb.toString(); + } + + /** + * Generate random date within the last 5 years + */ + private LocalDate generateRandomDate() { + LocalDate now = LocalDate.now(); + LocalDate fiveYearsAgo = now.minusYears(5); + long daysBetween = java.time.temporal.ChronoUnit.DAYS.between(fiveYearsAgo, now); + long randomDays = Math.floorMod(random.nextLong(), daysBetween); + return fiveYearsAgo.plusDays(randomDays); + } + + /** + * Generate random datetime within the last year + */ + private LocalDateTime generateRandomDateTime() { + LocalDateTime now = LocalDateTime.now(); + LocalDateTime oneYearAgo = now.minusYears(1); + long secondsBetween = java.time.temporal.ChronoUnit.SECONDS.between(oneYearAgo, now); + long randomSeconds = Math.floorMod(random.nextLong(), secondsBetween); + return oneYearAgo.plusSeconds(randomSeconds); + } + + /** + * Generate memory-efficient streaming data + */ + public DataStream generateStreamingData(int totalRows) { + return new DataStream(totalRows, this); + } + + /** + * Iterator-based data stream for memory-efficient data generation + */ + public static class DataStream implements Iterable { + private final int totalRows; + private final DataGenerator generator; + + public DataStream(int totalRows, DataGenerator generator) { + this.totalRows = totalRows; + this.generator = generator; + } + + @Override + public java.util.Iterator iterator() { + return new java.util.Iterator() { + private int currentRow = 0; + + @Override + public boolean hasNext() { + return currentRow < totalRows; + } + + @Override + public BenchmarkData next() { + if (!hasNext()) { + throw new java.util.NoSuchElementException(); + } + return generator.generateSingleRow(++currentRow); + } + }; + } + + public int getTotalRows() { + return totalRows; + } + } + + /** + * Generate data with specific characteristics for performance testing + */ + public List generateDataWithCharacteristics(int rowCount, DataCharacteristics characteristics) { + logger.info("Generating {} rows with specific characteristics: {}", rowCount, characteristics); + + List data = new ArrayList<>(rowCount); + + for (int i = 0; i < rowCount; i++) { + BenchmarkData row = generateSingleRow(i + 1); + + // Apply characteristics + if (characteristics.isLargeStrings()) { + row.setStringData(generateRandomString(100, 500)); + row.setDescription(generateLargeDescription()); + } + + if (characteristics.isRepeatedValues()) { + // Use limited set of values to create repetition + row.setCategory(CATEGORIES[i % 3]); + row.setStatus(STATUSES[i % 2]); + } + + if (characteristics.isNullValues()) { + // Randomly nullify some fields + if (random.nextFloat() < 0.1) { // 10% chance + row.setExtraData1(null); + row.setExtraData2(null); + } + } + + data.add(row); + } + + return data; + } + + private String generateLargeDescription() { + int sentenceCount = random.nextInt(10) + 5; // 5-14 sentences + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < sentenceCount; i++) { + if (i > 0) { + sb.append(". "); + } + + int wordsInSentence = random.nextInt(15) + 5; // 5-19 words per sentence + for (int j = 0; j < wordsInSentence; j++) { + if (j > 0) { + sb.append(" "); + } + sb.append(SAMPLE_WORDS[random.nextInt(SAMPLE_WORDS.length)]); + } + } + + return sb.toString(); + } + + /** + * Configuration class for data characteristics + */ + public static class DataCharacteristics { + private boolean largeStrings = false; + private boolean repeatedValues = false; + private boolean nullValues = false; + + public static DataCharacteristics defaults() { + return new DataCharacteristics(); + } + + public DataCharacteristics withLargeStrings() { + this.largeStrings = true; + return this; + } + + public DataCharacteristics withRepeatedValues() { + this.repeatedValues = true; + return this; + } + + public DataCharacteristics withNullValues() { + this.nullValues = true; + return this; + } + + public boolean isLargeStrings() { + return largeStrings; + } + + public boolean isRepeatedValues() { + return repeatedValues; + } + + public boolean isNullValues() { + return nullValues; + } + + @Override + public String toString() { + return "DataCharacteristics{" + "largeStrings=" + + largeStrings + ", repeatedValues=" + + repeatedValues + ", nullValues=" + + nullValues + '}'; + } + } + + // Static convenience methods for backward compatibility + private static final DataGenerator defaultGenerator = new DataGenerator(42L); + + /** + * Generate test data list using default generator + */ + public static List generateTestDataList(int rowCount) { + return defaultGenerator.generateData(rowCount); + } + + /** + * Generate test data with specific size using default generator + */ + public static List generateTestDataList(BenchmarkConfiguration.DatasetSize size) { + return defaultGenerator.generateData(size); + } + + /** + * Generate test data with specific size (alias method) + */ + public static List generateTestData(BenchmarkConfiguration.DatasetSize size) { + return defaultGenerator.generateData(size); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/MemoryProfiler.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/MemoryProfiler.java new file mode 100644 index 000000000..c86db50c1 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/utils/MemoryProfiler.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.utils; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for profiling memory usage during benchmark execution + */ +public class MemoryProfiler { + + private static final Logger logger = LoggerFactory.getLogger(MemoryProfiler.class); + + private final MemoryMXBean memoryBean; + private final List gcBeans; + private volatile ScheduledExecutorService scheduler; + private final AtomicBoolean running; + private final Object schedulerLock = new Object(); + + // Memory tracking variables + private final AtomicLong maxUsedMemory; + private final AtomicLong totalMemorySamples; + private final AtomicLong sumMemoryUsage; + private final List memorySnapshots; + + // GC tracking variables + private long initialGcCount; + private long initialGcTime; + private long startTime; + + public MemoryProfiler() { + this.memoryBean = ManagementFactory.getMemoryMXBean(); + this.gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + this.scheduler = createScheduler(); + this.running = new AtomicBoolean(false); + this.maxUsedMemory = new AtomicLong(0); + this.totalMemorySamples = new AtomicLong(0); + this.sumMemoryUsage = new AtomicLong(0); + this.memorySnapshots = new ArrayList<>(); + } + + /** + * Start memory profiling + */ + public void start() { + if (running.compareAndSet(false, true)) { + reset(); + startTime = System.currentTimeMillis(); + + // Record initial GC stats + initialGcCount = getTotalGcCount(); + initialGcTime = getTotalGcTime(); + + // Create a new scheduler if needed + synchronized (schedulerLock) { + if (scheduler.isShutdown() || scheduler.isTerminated()) { + scheduler = createScheduler(); + } + } + + try { + // Start memory sampling + scheduler.scheduleAtFixedRate( + this::sampleMemory, + 0, + org.apache.fesod.sheet.benchmark.core.BenchmarkConfiguration.MEMORY_SAMPLING_INTERVAL_MS, + TimeUnit.MILLISECONDS); + + logger.debug("Memory profiling started"); + } catch (Exception e) { + logger.warn("Failed to start memory sampling: {}", e.getMessage()); + running.set(false); + } + } + } + + /** + * Stop memory profiling + */ + public void stop() { + if (running.compareAndSet(true, false)) { + synchronized (schedulerLock) { + if (scheduler != null && !scheduler.isShutdown()) { + scheduler.shutdown(); + try { + if (!scheduler.awaitTermination(1, TimeUnit.SECONDS)) { + scheduler.shutdownNow(); + } + } catch (InterruptedException e) { + scheduler.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + } + logger.debug("Memory profiling stopped"); + } + } + + /** + * Reset all memory tracking variables + */ + public void reset() { + maxUsedMemory.set(0); + totalMemorySamples.set(0); + sumMemoryUsage.set(0); + synchronized (memorySnapshots) { + memorySnapshots.clear(); + } + } + + /** + * Sample current memory usage + */ + private void sampleMemory() { + try { + MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage(); + long currentUsed = heapUsage.getUsed(); + + // Update max memory usage + maxUsedMemory.updateAndGet(current -> Math.max(current, currentUsed)); + + // Update average calculation + totalMemorySamples.incrementAndGet(); + sumMemoryUsage.addAndGet(currentUsed); + + // Store snapshot for detailed analysis + synchronized (memorySnapshots) { + memorySnapshots.add(currentUsed); + } + + } catch (Exception e) { + logger.warn("Error sampling memory usage", e); + } + } + + /** + * Get current memory snapshot + */ + public MemorySnapshot getSnapshot() { + MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage(); + + long maxUsed = maxUsedMemory.get(); + long samples = totalMemorySamples.get(); + long avgUsed = samples > 0 ? sumMemoryUsage.get() / samples : 0; + + long currentGcCount = getTotalGcCount(); + long currentGcTime = getTotalGcTime(); + + return new MemorySnapshot( + maxUsed, + avgUsed, + heapUsage.getCommitted(), + currentGcCount - initialGcCount, + currentGcTime - initialGcTime, + System.currentTimeMillis() - startTime); + } + + /** + * Shutdown the profiler + */ + public void shutdown() { + stop(); + synchronized (schedulerLock) { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + } + + /** + * Create a new scheduler + */ + private ScheduledExecutorService createScheduler() { + return Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "MemoryProfiler"); + t.setDaemon(true); + return t; + }); + } + + /** + * Get total GC time across all collectors + */ + private long getTotalGcTime() { + return gcBeans.stream() + .mapToLong(bean -> bean.getCollectionTime() > 0 ? bean.getCollectionTime() : 0) + .sum(); + } + + /** + * Get total GC count across all collectors + */ + private long getTotalGcCount() { + return gcBeans.stream() + .mapToLong(bean -> bean.getCollectionCount() > 0 ? bean.getCollectionCount() : 0) + .sum(); + } + + /** + * Get current memory usage + */ + public long getUsedMemory() { + return memoryBean.getHeapMemoryUsage().getUsed(); + } + + /** + * Get peak memory usage + */ + public long getPeakMemoryUsage() { + return maxUsedMemory.get(); + } + + /** + * Get detailed memory statistics using a single pass over snapshot data + */ + public MemoryStatistics getDetailedStatistics() { + List snapshots; + synchronized (memorySnapshots) { + snapshots = new ArrayList<>(memorySnapshots); + } + + if (snapshots.isEmpty()) { + return new MemoryStatistics(0, 0, 0, 0, 0); + } + + // Single-pass calculation for min, max, sum, sum-of-squares + long min = Long.MAX_VALUE; + long max = Long.MIN_VALUE; + long sum = 0; + int count = snapshots.size(); + + for (Long value : snapshots) { + long v = value; + if (v < min) min = v; + if (v > max) max = v; + sum += v; + } + + double avg = (double) sum / count; + + // Second pass for variance (needed for accurate stddev) + double varianceSum = 0; + for (Long value : snapshots) { + double diff = value - avg; + varianceSum += diff * diff; + } + double stdDev = Math.sqrt(varianceSum / count); + + // Calculate 95th percentile + Collections.sort(snapshots); + int p95Index = (int) Math.ceil(0.95 * count) - 1; + long p95 = snapshots.get(Math.max(0, p95Index)); + + return new MemoryStatistics(min, max, (long) avg, (long) stdDev, p95); + } + + /** + * Memory snapshot data class + */ + public static class MemorySnapshot { + private final long maxUsedMemory; + private final long avgUsedMemory; + private final long allocatedMemory; + private final long gcCount; + private final long gcTime; + private final long durationMs; + + public MemorySnapshot( + long maxUsedMemory, + long avgUsedMemory, + long allocatedMemory, + long gcCount, + long gcTime, + long durationMs) { + this.maxUsedMemory = maxUsedMemory; + this.avgUsedMemory = avgUsedMemory; + this.allocatedMemory = allocatedMemory; + this.gcCount = gcCount; + this.gcTime = gcTime; + this.durationMs = durationMs; + } + + public long getMaxUsedMemory() { + return maxUsedMemory; + } + + public long getAvgUsedMemory() { + return avgUsedMemory; + } + + public long getAllocatedMemory() { + return allocatedMemory; + } + + public long getGcCount() { + return gcCount; + } + + public long getGcTime() { + return gcTime; + } + + public long getDurationMs() { + return durationMs; + } + + public double getMaxUsedMemoryMB() { + return maxUsedMemory / (1024.0 * 1024.0); + } + + public double getAvgUsedMemoryMB() { + return avgUsedMemory / (1024.0 * 1024.0); + } + + public double getAllocatedMemoryMB() { + return allocatedMemory / (1024.0 * 1024.0); + } + + @Override + public String toString() { + return String.format( + "MemorySnapshot{maxUsed=%.2f MB, avgUsed=%.2f MB, allocated=%.2f MB, gcCount=%d, gcTime=%d ms, duration=%d ms}", + getMaxUsedMemoryMB(), getAvgUsedMemoryMB(), getAllocatedMemoryMB(), gcCount, gcTime, durationMs); + } + } + + /** + * Detailed memory statistics data class + */ + public static class MemoryStatistics { + private final long minMemory; + private final long maxMemory; + private final long avgMemory; + private final long stdDevMemory; + private final long p95Memory; + + public MemoryStatistics(long minMemory, long maxMemory, long avgMemory, long stdDevMemory, long p95Memory) { + this.minMemory = minMemory; + this.maxMemory = maxMemory; + this.avgMemory = avgMemory; + this.stdDevMemory = stdDevMemory; + this.p95Memory = p95Memory; + } + + public long getMinMemory() { + return minMemory; + } + + public long getMaxMemory() { + return maxMemory; + } + + public long getAvgMemory() { + return avgMemory; + } + + public long getStdDevMemory() { + return stdDevMemory; + } + + public long getP95Memory() { + return p95Memory; + } + + public double getMinMemoryMB() { + return minMemory / (1024.0 * 1024.0); + } + + public double getMaxMemoryMB() { + return maxMemory / (1024.0 * 1024.0); + } + + public double getAvgMemoryMB() { + return avgMemory / (1024.0 * 1024.0); + } + + public double getStdDevMemoryMB() { + return stdDevMemory / (1024.0 * 1024.0); + } + + public double getP95MemoryMB() { + return p95Memory / (1024.0 * 1024.0); + } + } +} diff --git a/pom.xml b/pom.xml index c4bcff65f..2731e7261 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,7 @@ fesod-shaded fesod-examples fesod-sheet + fesod-benchmark