diff --git a/README.md b/README.md index e572bd3..d613455 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,10 @@ result. Independent inputs share immutable header scans and directive tokens; macro definitions, include guards, counters, and dependency lists reset per input. Parameter references are indexed when a macro is defined, expanded arguments are memoized within an invocation, and header/dependency membership uses hash tables. +Cached dependency directives are borrowed until line remapping requires a copy. +Macro definitions record which arguments need their original spelling and how +often expanded arguments are used. Expansion moves argument tokens on their last +use and consumes token-paste inputs instead of copying them again. The same implementation can preprocess C compilation and `-E` without invoking Clang: diff --git a/src/preprocessor/native.rs b/src/preprocessor/native.rs index b2ea3a5..ed95373 100644 --- a/src/preprocessor/native.rs +++ b/src/preprocessor/native.rs @@ -11,6 +11,7 @@ use lexer::{lex, needs_space, spelling}; pub(super) use options::{preprocess, preprocess_many}; use smol_str::SmolStr; use std::{ + borrow::Cow, collections::{HashMap, HashSet}, fs, path::{Path, PathBuf}, @@ -70,10 +71,17 @@ struct Macro { variadic: bool, body: Vec, parameter_indices: Vec>, + parameter_uses: Vec, has_paste: bool, query: Option, } +#[derive(Clone, Default)] +struct ParameterUses { + expanded: usize, + raw: bool, +} + struct Conditional { parent: bool, active: bool, @@ -199,12 +207,24 @@ impl CleanSource { }) } - fn directive_tokens(&self, index: usize, text: &str) -> Result, String> { + fn directive_tokens( + &self, + index: usize, + text: &str, + physical_line: usize, + ) -> Result>, String> { let mut cache = self.directives.borrow_mut(); if cache[index].is_none() { - cache[index] = Some(Rc::new(lex(text)?)); + let mut tokens = lex(text)?; + if let Some(first) = tokens.first_mut() { + first.space = true; + } + for token in &mut tokens { + token.line += physical_line - 1; + } + cache[index] = Some(Rc::new(tokens)); } - Ok(cache[index].as_ref().unwrap().as_ref().clone()) + Ok(Rc::clone(cache[index].as_ref().unwrap())) } } @@ -573,16 +593,27 @@ impl Preprocessor { dependency_lines.push((physical_line.saturating_add_signed(line_offset), text)); continue; } - let mut line = if self.dependency_only { - source.directive_tokens(line_index, text)? - } else { - lex(text)? + let cached = self + .dependency_only + .then(|| source.directive_tokens(line_index, text, physical_line)) + .transpose()?; + let mut line = match &cached { + Some(tokens) => Cow::Borrowed(tokens.as_slice()), + None => Cow::Owned(lex(text)?), }; - if let Some(first) = line.first_mut() { - first.space = true; - } - for token in &mut line { - token.line = (physical_line + token.line - 1).saturating_add_signed(line_offset); + if !self.dependency_only { + if let Some(first) = line.to_mut().first_mut() { + first.space = true; + } + for token in line.to_mut() { + token.line = + (physical_line + token.line - 1).saturating_add_signed(line_offset); + } + } else if line_offset != 0 { + // A #line directive must not change tokens shared with another input. + for token in line.to_mut() { + token.line = token.line.saturating_add_signed(line_offset); + } } if line.first().is_some_and(|t| t.is("#")) { self.flush_dependency(&mut dependency_lines, &file)?; @@ -740,7 +771,7 @@ impl Preprocessor { } } } else if active { - pending.extend(line); + pending.extend(line.into_owned()); pending.push(Token::new( "\n", false, @@ -796,6 +827,10 @@ mod tests { "#define V(args...) args\nV(1,2)\n", "#define V(x,...) f(x,##__VA_ARGS__)\nV(1,) V(1)\n", "#define S(x) #x\nS(a\n b)\n", + "#define F(x) x x\nF(__COUNTER__) __COUNTER__\n", + "#define F(x) x #x x x##tail\nF(__COUNTER__) __COUNTER__\n", + "#define F(x) #x x##tail x x\nF(__COUNTER__) __COUNTER__\n", + "#define DROP(x) 42\nDROP(__COUNTER__) __COUNTER__\n", "#define x 3\n#define f(a) f(x * (a))\n#undef x\n#define x 2\n#define g f\n#define z z[0]\n#define h g(~\n#define m(a) a(w)\n#define w 0,1\n#define t(a) a\nf(y+1)+ f(f(z)) % t(t(g)(0)+t)(1);\ng(x+(3,4)-w)| h 5) & m(f)^m(m);\n", "#define x 7\n#if defined(x) && !defined missing\nx\n#else\n#error wrong\n#endif\n", "#if 0 && 1/0\n#error wrong\n#elif 1 || 1/0\nyes\n#endif\n", @@ -957,6 +992,36 @@ mod tests { ); } + #[test] + fn shared_directives_keep_physical_lines_between_dependency_scans() { + let directory = super::super::IncludeDirectory::new().unwrap(); + fs::write(directory.0.join("mapped.h"), "").unwrap(); + fs::write(directory.0.join("physical.h"), "").unwrap(); + fs::write( + directory.0.join("shared.h"), + "#ifdef REMAP\n#line 200\n#endif\n#if __LINE__ == 201\n#include \"mapped.h\"\n#elif __LINE__ == 6\n#include \"physical.h\"\n#else\n#error incorrect cached line\n#endif\n", + ) + .unwrap(); + let mut headers = HashMap::new(); + for (code, expected, absent) in [ + ( + "#define REMAP\n#include \"shared.h\"\n", + "mapped.h", + "physical.h", + ), + ("#include \"shared.h\"\n", "physical.h", "mapped.h"), + ] { + let mut pp = Preprocessor::new(vec![directory.0.clone()], vec![]); + pp.dependency_only = true; + pp.headers = headers; + pp.process(code, &directory.0.join(expected).with_extension("c")) + .unwrap(); + assert!(pp.dependencies.contains(&directory.0.join(expected))); + assert!(!pp.dependencies.contains(&directory.0.join(absent))); + headers = pp.headers; + } + } + #[test] fn source_splices_preserve_builtin_lines() { for code in [ diff --git a/src/preprocessor/native/expansion.rs b/src/preprocessor/native/expansion.rs index c23d29c..b78cc0f 100644 --- a/src/preprocessor/native/expansion.rs +++ b/src/preprocessor/native/expansion.rs @@ -1,7 +1,7 @@ //! Define and expand macros while preserving hide sets and stateful effects. use super::lexer::identifier; -use super::{Hidden, Macro, Preprocessor, Query, Token, lex, spelling}; +use super::{Hidden, Macro, ParameterUses, Preprocessor, Query, Token, lex, spelling}; use std::{collections::VecDeque, rc::Rc}; impl Preprocessor { @@ -50,7 +50,7 @@ impl Preprocessor { parameters = Some(names); } let body = tokens[index..].to_vec(); - let parameter_indices = body + let parameter_indices: Vec<_> = body .iter() .map(|t| { parameters @@ -58,6 +58,19 @@ impl Preprocessor { .and_then(|names| names.iter().position(|p| p == &t.text)) }) .collect(); + let mut parameter_uses = + vec![ParameterUses::default(); parameters.as_ref().map_or(0, Vec::len)]; + for (index, parameter) in parameter_indices.iter().enumerate() { + if let Some(parameter) = parameter { + let raw = index > 0 && matches!(body[index - 1].text.as_str(), "#" | "##") + || body.get(index + 1).is_some_and(|t| t.is("##")); + if raw { + parameter_uses[*parameter].raw = true; + } else { + parameter_uses[*parameter].expanded += 1; + } + } + } let has_paste = body.iter().any(|t| t.is("##")); if self.dependency_only { let mut effectful = has_paste; @@ -93,6 +106,7 @@ impl Preprocessor { variadic, body, parameter_indices, + parameter_uses, has_paste, query: None, }), @@ -318,7 +332,7 @@ impl Preprocessor { continue; } let mut omitted_variadic = false; - let arguments = if let Some(parameters) = &definition.parameters { + let mut arguments = if let Some(parameters) = &definition.parameters { let n = pending.iter().take_while(|t| t.is("\n")).count(); if !pending.get(n).is_some_and(|t| t.is("(")) { if !discard { @@ -407,10 +421,10 @@ impl Preprocessor { } continue; } - let mut substituted: Vec = vec![]; + let mut substituted: Vec = Vec::with_capacity(definition.body.len()); let body = &definition.body; let mut index = 0; - let mut expanded_arguments: Vec>> = + let mut expanded_arguments: Vec, usize)>> = vec![None; arguments.as_ref().map_or(0, Vec::len)]; while index < body.len() { let current = &body[index]; @@ -445,25 +459,38 @@ impl Preprocessor { } else if let Some(arg) = definition.parameter_indices[index] { let pasted = index > 0 && body[index - 1].is("##") || body.get(index + 1).is_some_and(|t| t.is("##")); - let raw = &arguments.as_ref().unwrap()[arg]; - let values = if pasted { - raw + let start = substituted.len(); + if pasted { + let raw = &arguments.as_ref().unwrap()[arg]; + if raw.is_empty() { + substituted.push(Token::new("", current.space, token.line)); + } else { + substituted.extend_from_slice(raw); + } } else { if expanded_arguments[arg].is_none() { - expanded_arguments[arg] = - Some(self.expand(raw.clone(), file, depth + 1)?); + let raw = &mut arguments.as_mut().unwrap()[arg]; + let input = if definition.parameter_uses[arg].raw { + raw.clone() + } else { + std::mem::take(raw) + }; + expanded_arguments[arg] = Some(( + self.expand(input, file, depth + 1)?, + definition.parameter_uses[arg].expanded, + )); } - expanded_arguments[arg].as_ref().unwrap() - }; - if values.is_empty() && pasted { - substituted.push(Token::new("", current.space, token.line)); - } else { - let start = substituted.len(); - substituted.extend_from_slice(values); - if let Some(first) = substituted.get_mut(start) { - first.space = current.space; + let (values, remaining) = expanded_arguments[arg].as_mut().unwrap(); + *remaining -= 1; + if *remaining == 0 { + substituted.append(values); + } else { + substituted.extend_from_slice(values); } } + if let Some(first) = substituted.get_mut(start) { + first.space = current.space; + } } else { substituted.push(current.clone()); } @@ -473,21 +500,20 @@ impl Preprocessor { substituted } else { let mut replacement: Vec = Vec::with_capacity(substituted.len()); - let mut index = 0; - while index < substituted.len() { - if substituted[index].is("##") || substituted[index].is("##gnu") { - let gnu = substituted[index].is("##gnu"); + let mut substituted = substituted.into_iter(); + while let Some(current) = substituted.next() { + if current.is("##") || current.is("##gnu") { + let gnu = current.is("##gnu"); let left = replacement .pop() .ok_or("## cannot start a macro replacement")?; - index += 1; let right = substituted - .get(index) + .next() .ok_or("## cannot end a macro replacement")?; // GNU variadic macros permit comma elision for an omitted tail. if gnu { if !omitted_variadic { - replacement.extend([left, right.clone()]); + replacement.extend([left, right]); } } else { let joined = format!("{}{}", left.text, right.text); @@ -505,9 +531,8 @@ impl Preprocessor { } } } else { - replacement.push(substituted[index].clone()); + replacement.push(current); } - index += 1; } replacement.retain(|t| !t.text.is_empty()); replacement diff --git a/tools/benchmarks/README.md b/tools/benchmarks/README.md index 0ec7a0b..d5e96c0 100644 --- a/tools/benchmarks/README.md +++ b/tools/benchmarks/README.md @@ -99,6 +99,46 @@ allocation APIs are not intercepted. Counter runs add atomic-operation overhead and must not supply performance timings. The interposer uses glibc internals and is an optional Linux profiling tool, not part of the compiler runtime. +## Profile-guided optimization + +Replay successful C compilations recorded by `cc.py` to train a PGO build of +`dynamite-cc`. Install `llvm-tools-preview` for the Rust toolchain first: + +```sh +rustup component add llvm-tools-preview +python3 tools/benchmarks/pgo.py \ + --records temp/benchmark-run/redis-dynamite-1-records \ + --records temp/benchmark-run/git-dynamite-1-records \ + --output temp/pgo-run --clang clang-15 --jobs 4 +``` + +`--toolchain VERSION` selects a Rustup toolchain. The matching `llvm-profdata` +is located in that toolchain's sysroot. The output directory must be new. +Training sources, generated headers and working directories referenced by the +records must still exist. Use trusted records: their compiler arguments are +replayed as commands, with object outputs redirected and dependency generation +disabled. Commands, build/training logs, raw and merged profiles, toolchain +metadata and both binaries remain in the output directory. + +The resulting `temp/pgo-run/dynamite-pgo` is an optional compiler executable; +the normal release binary is not replaced. Only Dynamite is trained: the external +Clang backend is unchanged. Normal `RUSTFLAGS` apply equally to the instrumented +and profile-use builds; do not supply other PGO flags. Encoded Rust flags are +rejected rather than silently overriding these settings. + +Compare the regular and PGO binaries with the same flags and clean-build workload. +Keep training and timing separate. Train on several projects and evaluate held-out +inputs before drawing conclusions about general compiler performance. Profiles +are specific to the source and Rust toolchain; regenerate them after changes. +See the [curl-only versus mixed-corpus measurements](results/linux-x86_64-2026-10-18-mixed-pgo.md) +for a three-application comparison using the same compiler source and different +training corpora. +See the [mixed-PGO versus Clang measurements](results/linux-x86_64-2026-10-18-pgo-clang.md) +for three clean builds per compiler/application with the same `-O2` settings. +The [matching `-O0` comparison](results/linux-x86_64-2026-10-18-pgo-clang-o0.md) +keeps those compiler binaries and the Clang backend fixed while changing the +application optimization level. + For the experimental LLVM tools backend, set `DYNAMITE_BACKEND=llvm` and select matching `DYNAMITE_LLC`, `DYNAMITE_OPT`, and `DYNAMITE_LLD` executables. Use `--no-trace`, since Clang time-trace options are unsupported on that path. diff --git a/tools/benchmarks/pgo.py b/tools/benchmarks/pgo.py new file mode 100644 index 0000000..79b3872 --- /dev/null +++ b/tools/benchmarks/pgo.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Build a PGO driver by replaying successful C compilations recorded by cc.py.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess + + +REPO = Path(__file__).resolve().parents[2] + + +def training_arguments(arguments, output): + # Training must not overwrite the original objects or dependency files. + result = [] + arguments = iter(arguments) + for argument in arguments: + if argument in ("-o", "-MF", "-MT", "-MQ"): + next(arguments) + elif argument not in ("-MD", "-MMD", "-MP") and not argument.startswith("-ftime-trace"): + result.append(argument) + return [*result, "-o", str(output)] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--records", type=Path, action="append", required=True, + help="cc.py record directory; repeat for different projects") + parser.add_argument("--output", type=Path, required=True, + help="New directory for binaries, profiles, commands and logs") + parser.add_argument("--toolchain", help="Rustup toolchain, e.g. 1.86.0") + parser.add_argument("--clang", default="clang") + parser.add_argument("--jobs", type=int, default=4) + args = parser.parse_args() + if args.jobs < 1: + parser.error("--jobs must be positive") + cargo = ["cargo"] + ([f"+{args.toolchain}"] if args.toolchain else []) + rustc = ["rustc"] + ([f"+{args.toolchain}"] if args.toolchain else []) + sysroot = Path(subprocess.check_output([*rustc, "--print", "sysroot"], text=True).strip()) + version = subprocess.check_output([*rustc, "-vV"], text=True) + host = next(line.removeprefix("host: ") for line in version.splitlines() + if line.startswith("host: ")) + profdata = sysroot / "lib" / "rustlib" / host / "bin" / "llvm-profdata" + if not profdata.is_file(): + parser.error("install llvm-tools-preview for the selected Rust toolchain") + if "CARGO_ENCODED_RUSTFLAGS" in os.environ: + parser.error("unset CARGO_ENCODED_RUSTFLAGS; use RUSTFLAGS for common build flags") + records = [] + for directory in args.records: + for path in sorted(directory.glob("*.json")): + if path.name.endswith(".trace.json"): + continue + record = json.loads(path.read_text()) + if record["kind"] == "compile" and record["returncode"] == 0: + records.append(record) + if not records: + parser.error("no successful C compilation records found") + clang = shutil.which(args.clang) + if not clang: + parser.error("Clang executable not found") + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=False) + profiles = root / "profiles" + profiles.mkdir() + objects = root / "training-objects" + objects.mkdir() + env = dict(os.environ, DYNAMITE_CLANG=clang) + env.pop("DYNAMITE_TIMINGS", None) + env.pop("DYNAMITE_BACKEND", None) + base_flags = env.get("RUSTFLAGS", "") + metadata = { + "rustc": version, "base_rustflags": base_flags, "clang": clang, + "record_directories": [str(p.resolve()) for p in args.records], + "training_compile_count": len(records), "jobs": args.jobs, + "commit": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip(), + "diff": subprocess.check_output(["git", "diff"], cwd=REPO, text=True), + } + + def build(name, flag): + target = root / f"{name}-target" + command = [*cargo, "build", "--release", "--bin", "dynamite-cc", + "--target", host, "--target-dir", str(target)] + build_env = dict(env, RUSTFLAGS=f"{base_flags} {flag}".strip()) + with (root / f"{name}-build.log").open("w") as log: + subprocess.run(command, cwd=REPO, env=build_env, + stdout=log, stderr=subprocess.STDOUT, check=True) + binary = root / f"dynamite-{name}" + shutil.copy2(target / host / "release" / "dynamite-cc", binary) + return binary + + instrumented = build("instrumented", f"-Cprofile-generate={profiles}") + training = [] + for index, record in enumerate(records): + training.append({"cwd": record["cwd"], "command": [str(instrumented), + *training_arguments(record["args"], objects / f"{index}.o")]}) + (root / "training.json").write_text(json.dumps(training, indent=2) + "\n") + train_env = dict(env, LLVM_PROFILE_FILE=str(profiles / "%m-%p.profraw")) + + def train(item): + index, invocation = item + with (root / f"training-{index}.log").open("w") as log: + subprocess.run(invocation["command"], cwd=invocation["cwd"], env=train_env, + stdout=log, stderr=subprocess.STDOUT, check=True) + + with ThreadPoolExecutor(max_workers=args.jobs) as workers: + list(workers.map(train, enumerate(training))) + merged = root / "merged.profdata" + raw_profiles = sorted(profiles.glob("*.profraw")) + if not raw_profiles: + raise RuntimeError("training produced no profiles") + subprocess.run([str(profdata), "merge", "-o", str(merged), + *map(str, raw_profiles)], check=True) + optimized = build("pgo", f"-Cprofile-use={merged}") + metadata["binary"] = str(optimized) + metadata["binary_sha256"] = hashlib.sha256(optimized.read_bytes()).hexdigest() + (root / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + print(optimized) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/results/linux-x86_64-2026-10-18-curl-copy-pgo.md b/tools/benchmarks/results/linux-x86_64-2026-10-18-curl-copy-pgo.md new file mode 100644 index 0000000..b8f03d6 --- /dev/null +++ b/tools/benchmarks/results/linux-x86_64-2026-10-18-curl-copy-pgo.md @@ -0,0 +1,68 @@ +# Curl token-copy and PGO measurements + +Measured on Linux x86_64 with Rust 1.86.0 and Clang 15.0.7, relative to +compiler commit `7804347c97ee021bfac6c7b1a3e6558ec8fb3f40`. The copy changes +borrow cached dependency directives, move expanded macro arguments on their +last use, and consume token-paste inputs. PGO uses the same changed source. + +## Clean builds + +Curl 8.23.0 and static libcurl, `-O2 -g0`, `make -j4`, native preprocessing +and the Clang IR backend. OpenSSL and zlib are enabled; libpsl and documentation +generation are disabled. All compilers use the same configured source tree. +Configure, cleaning and validation are outside the timer. Each run removes objects +and stale `.deps/*.Po`/`.Plo` files; the latter avoids an upstream make race +between a removed `tool_hugehelp.c` and its absolute dependency entry. + +| Compiler | Run 1 | Run 2 | Run 3 | Median | +| --- | ---: | ---: | ---: | ---: | +| Baseline | 26.50 s | 26.38 s | 26.45 s | 26.45 s | +| Token-copy changes | 26.50 s | 26.42 s | 26.48 s | 26.48 s | +| Token-copy changes + PGO | 25.91 s | 26.06 s | 25.94 s | 25.94 s | + +Orders were baseline/copy/PGO, PGO/baseline/copy, and copy/PGO/baseline. +Filesystem caches were not cleared; no compiler cache or timing instrumentation +was enabled. Every executable passed `curl --version`. The combined changes +reduce the median by 1.9%; copying changes alone do not measurably improve the +full build in these runs. + +## Preprocessing and dependencies + +For individual `easy.c`, `http.c`, `multi.c` and `vtls/openssl.c` preprocessing, +copy changes reduce whole-process allocation requests by 3.5–4.3%, with little +timing change. Expanded output matches the baseline after removing line markers. + +A single `-M` invocation over those four files reuses cached header directives. +Six alternating-order timing samples per binary gave: + +| Compiler | Median dependency scan | Allocation requests | +| --- | ---: | ---: | +| Baseline | 155.81 ms | 381,163 | +| Token-copy changes | 145.84 ms | 320,065 | +| Token-copy changes + PGO | 138.05 ms | 320,065 | + +Dependency output is byte-identical. Allocation probes use the existing glibc +`LD_PRELOAD` counter and run separately from timing. Counts include process +initialization; requested bytes are not peak memory. The dependency workload is +a batch and does not represent curl's usual per-file compilation path. + +## PGO and validation + +`tools/benchmarks/pgo.py` replayed 477 successful curl compilation records with +`-Cprofile-generate`, merged profiles using Rust 1.86.0's `llvm-profdata`, and +rebuilt with `-Cprofile-use`. External Clang is unchanged. Curl was both the +training and evaluation workload; these results do not establish improvements +on Git, Redis or held-out workloads. The normal release binary is not replaced. +Subsequent held-out [Git and Redis measurements](linux-x86_64-2026-10-18-git-redis-copy-pgo.md) +use this same PGO binary without retraining. + +The instrumented/trained build, logs, replay commands and profiles are under +`temp/curl-opt/pgo/`; build and preprocessing measurements are under +`temp/curl-opt/`. These local artifacts are ignored by Git. + +Validation passed: rustfmt, Clippy with all targets/features and warnings denied, +52 library tests, 2,101 integration tests (two ignored), and one doctest. +Tests used Clang 15 through PATH; the host's default Clang 14 cannot consume the +opaque-pointer IR. Baseline, changed and PGO binaries also compiled and ran a +macro regression using repeated `__COUNTER__`, stringification and token pasting. +macOS and AArch64 were not tested in this measurement. diff --git a/tools/benchmarks/results/linux-x86_64-2026-10-18-git-redis-copy-pgo.md b/tools/benchmarks/results/linux-x86_64-2026-10-18-git-redis-copy-pgo.md new file mode 100644 index 0000000..59e79ac --- /dev/null +++ b/tools/benchmarks/results/linux-x86_64-2026-10-18-git-redis-copy-pgo.md @@ -0,0 +1,81 @@ +# Git and Redis token-copy and PGO measurements + +Measured on Linux x86_64 with four available CPUs, using the same three Rust +1.86.0 compiler binaries as the [curl experiment](linux-x86_64-2026-10-18-curl-copy-pgo.md). +The baseline is compiler commit `7804347c97ee021bfac6c7b1a3e6558ec8fb3f40`. +The changed binary reduces preprocessing token copies. The PGO binary includes +those changes and was trained on 477 curl 8.23.0 compilation invocations. +It was not retrained on Git or Redis, so these are held-out application builds. +All three invoke the same external Clang 15.0.7 for IR lowering and linking. + +## Conditions + +- Git v2.56.0 and Redis 8.10.2, downloaded from their upstream GitHub tag archives. +- `-O2 -g0 -std=gnu99`, `make -j4`, native Dynamite preprocessing. +- A fresh copy of the pristine source repository for every build, including + bundled dependencies. Source copying and smoke checks are outside the timer. +- No compiler cache, stage timers or Clang time traces. Filesystem caches are + not cleared; these are clean builds rather than cold-cache builds. +- Three runs per project/compiler, with orders baseline/copy/PGO, + PGO/baseline/copy, and copy/PGO/baseline. Builds run sequentially. +- Git builds `all` with Rust, precompiled headers, gettext, Perl/Python and Tcl/Tk + integrations disabled using the existing benchmark's `NO_*` flags. +- Redis builds its six native tools with the libc allocator, TLS and Vector Sets + disabled, and `__STDC_NO_ATOMICS__=1`. Lua receives explicit matching `-O2 -g0` + and GNU99 flags; bundled C dependencies use the selected compiler. + +The downloaded archives were imported into local Git repositories before timing. +This gives Redis's release-header script repository context, avoiding an expensive +implicit directory comparison outside a repository. An initial archive-only +attempt was discarded. None of the 18 successful builds changes tracked upstream +source contents. Full make commands and logs are in `temp/git-redis-opt/`. + +## Results + +Times are build wall-clock seconds. Reductions compare each median to its +project's baseline median; positive percentages indicate shorter builds. + +| Project | Baseline | Copy changes | Copy + PGO | Copy reduction | Copy + PGO reduction | +| --- | ---: | ---: | ---: | ---: | ---: | +| Git v2.56.0 | 51.54 s | 51.18 s | 49.06 s | 0.7% | 4.8% | +| Redis 8.10.2 | 19.66 s | 19.44 s | 19.04 s | 1.1% | 3.2% | + +| Project | Compiler | Run 1 | Run 2 | Run 3 | +| --- | --- | ---: | ---: | ---: | +| Git | Baseline | 51.5429 | 51.0809 | 51.7368 | +| Git | Copy changes | 51.1789 | 51.0761 | 51.2434 | +| Git | Copy + PGO | 49.0618 | 49.0799 | 48.9405 | +| Redis | Baseline | 19.6642 | 19.6834 | 19.6286 | +| Redis | Copy changes | 19.4026 | 19.4391 | 19.4840 | +| Redis | Copy + PGO | 19.0380 | 19.0434 | 19.2455 | + +The small copy-only differences should be interpreted with the run-to-run +variation in mind. The PGO binary is faster than the baseline in every measured +build on these two applications. These results do not predict other targets, +optimization levels or runtime performance of the generated programs. + +## Validation and provenance + +All 18 builds completed and passed smoke validation outside the timer. Each Git +binary performed init/add/commit/show/status operations in a temporary repository. +Each Redis build passed PING, SET, GET, Lua EVAL and clean shutdown using its own +server and client binaries over a local Unix socket. Full upstream test suites +were not run for this timing comparison. + +Compiler SHA-256 values: + +| Compiler | SHA-256 | +| --- | --- | +| Baseline | `f7eb618308e2c478f93c67ab96487a2b0bc32fa699ce6bc6ab7780b5492a15bc` | +| Copy changes | `cfa32afc68f84a6e489604a0cd35c3cca436be97a11a96b26fa6d2f4996326f2` | +| Copy + PGO | `7f84bbade893ef0d2e5f586f66db679a4a87a87259f96bd303b93dee22d163e8` | + +Source archive SHA-256 values: + +- Git: `d761232b81394f7d4c3ef1a99fa804ffbe10d278ae1ad302833a0892050ca9fc` +- Redis: `541a374b753a8405683dd88560465791acc98cf09445ef10ec1bd286904273e8` + +The comparison script, metadata, exact timings and compiler hashes, source +repositories, build logs, resource measurements and smoke results remain under +`temp/git-redis-opt/`. Those local artifacts are ignored by Git. macOS and +AArch64 were not measured. diff --git a/tools/benchmarks/results/linux-x86_64-2026-10-18-mixed-pgo.md b/tools/benchmarks/results/linux-x86_64-2026-10-18-mixed-pgo.md new file mode 100644 index 0000000..a8a49d7 --- /dev/null +++ b/tools/benchmarks/results/linux-x86_64-2026-10-18-mixed-pgo.md @@ -0,0 +1,98 @@ +# Curl-only versus mixed-corpus PGO + +Measured on Linux x86_64 with Rust 1.86.0, four available CPUs, and external +Clang 15.0.7 for LLVM IR lowering and linking. Both compiler binaries contain +the same preprocessing token-copy changes on top of compiler commit +`7804347c97ee021bfac6c7b1a3e6558ec8fb3f40`. Their Rust source changes were +checked against the original curl-only build metadata before training. +These saved binaries predate the subsequent record-member optimization on main; +the numbers are not measurements of those newer compiler changes. + +## Training + +| Corpus | Curl 8.23.0 | Git v2.56.0 | Redis 8.10.2 | Total | +| --- | ---: | ---: | ---: | ---: | +| Curl-only | 477 | 0 | 0 | 477 | +| Mixed | 477 | 568 | 184 | 1,229 | + +Counts are successful C compilation invocations from one build of each included +application, including bundled C dependencies and build helpers. Each invocation +is replayed once; there is no per-project reweighting. The mixed build uses the +same curl records as the original experiment. Missing input files and failed +training commands were checked; all 1,229 invocations completed successfully. + +The mixed compiler was produced with the existing replay tool: + +```sh +python3 tools/benchmarks/pgo.py \ + --records temp/curl-stages/1-dynamite \ + --records temp/mixed-pgo/git-records \ + --records temp/mixed-pgo/redis-records \ + --output temp/mixed-pgo/pgo --toolchain 1.86.0 --clang clang-15 --jobs 4 +``` + +The output directory was new when this command ran. Instrumented builds, +training, merging and profile-use builds are excluded from evaluation timings. +The original curl-only PGO executable was preserved and measured again alongside +the mixed build; previous timing samples were not reused. + +## Clean-build results + +Each compiler was measured three times per application with `make -j4` and +`-O2 -g0`, without compiler caches, stage timers or Clang traces. Orders were +curl-only/mixed, mixed/curl-only, curl-only/mixed. Builds ran sequentially. +Filesystem caches were not flushed. Source copying, cleaning and smoke checks +were excluded from the timer. + +Git and Redis used fresh pristine source copies and the same GNU99 make commands +as the [previous comparison](linux-x86_64-2026-10-18-git-redis-copy-pgo.md). +Git disables Rust, PCH, gettext and Perl/Python/Tcl/Tk integrations. Redis builds +its six native tools with libc allocation, TLS/Vector Sets/C11 atomics disabled, +and explicit matching Lua flags. Curl uses copies of the same configured tree +as the [curl experiment](linux-x86_64-2026-10-18-curl-copy-pgo.md), with static +libcurl, OpenSSL/zlib enabled and libpsl/documentation disabled. Every curl copy +is cleaned and stale dependency files are removed before timing. + +| Application | Curl-only PGO median | Mixed PGO median | Reduction | +| --- | ---: | ---: | ---: | +| Curl 8.23.0 | 25.8992 s | 25.8822 s | 0.07% | +| Git v2.56.0 | 49.4387 s | 49.3617 s | 0.16% | +| Redis 8.10.2 | 19.1529 s | 19.0196 s | 0.70% | + +| Application | Compiler | Run 1 | Run 2 | Run 3 | +| --- | --- | ---: | ---: | ---: | +| Curl | Curl-only | 25.8992 | 26.3708 | 25.8910 | +| Curl | Mixed | 26.1164 | 25.8812 | 25.8822 | +| Git | Curl-only | 49.7431 | 49.4387 | 49.4355 | +| Git | Mixed | 49.4339 | 49.1469 | 49.3617 | +| Redis | Curl-only | 19.1498 | 19.1764 | 19.1529 | +| Redis | Mixed | 19.0149 | 19.0196 | 19.0831 | + +Additional training does not produce a substantial improvement in these builds. +Curl and Git median differences are smaller than run-to-run variation. Redis is +slightly faster with mixed PGO in all three paired comparisons. These are three +samples per configuration, not evidence of a universal improvement. All evaluated +applications are included in mixed training; no fourth held-out application was +evaluated. + +A subsequent [Clang comparison](linux-x86_64-2026-10-18-pgo-clang.md) measures +this same mixed-PGO executable against Clang 15 with matching `-O2` build settings. + +## Validation and artifacts + +All 18 builds succeeded. Each Git binary passed temporary repository operations; +each Redis server/client pair passed PING, SET/GET, Lua EVAL and clean shutdown. +Each curl binary passed HTTP and HTTPS loopback requests, with certificate +verification enabled. Git and Redis tracked source contents remained unchanged. +Full upstream test suites were not run for this timing comparison. + +| Compiler | SHA-256 | +| --- | --- | +| Curl-only PGO | `7f84bbade893ef0d2e5f586f66db679a4a87a87259f96bd303b93dee22d163e8` | +| Mixed PGO | `eeb2b4f70798b22a9c64f6f7eb366939a7b4735cd1010009bba4253f446c4d50` | + +`temp/mixed-pgo/` contains collection and training records, raw/merged profiles, +both build logs and exact commands for each evaluation, smoke results, metadata, +the comparison script and summary. The mixed executable is +`temp/mixed-pgo/pgo/dynamite-pgo`. These local artifacts are ignored by Git; +the normal release executable is unchanged. diff --git a/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang-o0.json b/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang-o0.json new file mode 100644 index 0000000..94db550 --- /dev/null +++ b/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang-o0.json @@ -0,0 +1,670 @@ +{ + "platform": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35", + "jobs": 4, + "versions": { + "curl": "8.23.0", + "git": "v2.56.0", + "redis": "8.10.2" + }, + "compiler_binaries": { + "mixed_pgo": { + "path": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "sha256": "eeb2b4f70798b22a9c64f6f7eb366939a7b4735cd1010009bba4253f446c4d50" + }, + "clang": { + "path": "/usr/bin/clang-15", + "sha256": "22f8651e4eba296325daed391a432b11c3db82d8166da127ccb43b4f6f229e18" + } + }, + "training": { + "mixed_pgo": { + "curl": 477, + "git": 568, + "redis": 184 + } + }, + "timing_scope": "make -j4 only; copies, cleaning, dependency removal and validation excluded. No compiler cache, timers or traces. Filesystem caches not flushed.", + "order": [ + [ + "mixed_pgo", + "clang" + ], + [ + "clang", + "mixed_pgo" + ], + [ + "mixed_pgo", + "clang" + ] + ], + "builds": [ + { + "project": "curl", + "round": 1, + "compiler": "mixed_pgo", + "seconds": 18.23002382554114, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "CFLAGS=-O0 -g0 -Werror-implicit-function-declaration -Wno-system-headers" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/curl-1-mixed_pgo", + "resource": "18.22 42.23 20.56 108544", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "curl", + "round": 1, + "compiler": "clang", + "seconds": 28.52669501118362, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "CFLAGS=-O0 -g0 -Werror-implicit-function-declaration -Wno-system-headers" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/curl-1-clang", + "resource": "28.52 47.76 20.16 114368", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "redis", + "round": 1, + "compiler": "mixed_pgo", + "seconds": 11.627391956746578, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O0", + "OPTIMIZATION=-O0", + "DEBUG=", + "CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/redis-1-mixed_pgo", + "resource": "11.62 17.91 5.42 107264", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=47b723fbdb06e1ac", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "redis", + "round": 1, + "compiler": "clang", + "seconds": 12.438752811402082, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O0", + "OPTIMIZATION=-O0", + "DEBUG=", + "CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/redis-1-clang", + "resource": "12.43 20.34 6.15 119232", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=8190ee04e10778", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 1, + "compiler": "mixed_pgo", + "seconds": 32.621819137595594, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O0 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/git-1-mixed_pgo", + "resource": "32.62 91.02 22.35 109824", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 1, + "compiler": "clang", + "seconds": 34.81821136176586, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O0 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/git-1-clang", + "resource": "34.81 108.00 17.65 120196", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "curl", + "round": 2, + "compiler": "clang", + "seconds": 18.837173684500158, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "CFLAGS=-O0 -g0 -Werror-implicit-function-declaration -Wno-system-headers" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/curl-2-clang", + "resource": "18.83 45.84 19.66 114616", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "curl", + "round": 2, + "compiler": "mixed_pgo", + "seconds": 18.416057834401727, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "CFLAGS=-O0 -g0 -Werror-implicit-function-declaration -Wno-system-headers" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/curl-2-mixed_pgo", + "resource": "18.41 42.15 20.69 108544", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "redis", + "round": 2, + "compiler": "clang", + "seconds": 11.762709741480649, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O0", + "OPTIMIZATION=-O0", + "DEBUG=", + "CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/redis-2-clang", + "resource": "11.76 20.04 6.18 119136", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=b415e5fcdff37859", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "redis", + "round": 2, + "compiler": "mixed_pgo", + "seconds": 8.35491410177201, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O0", + "OPTIMIZATION=-O0", + "DEBUG=", + "CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/redis-2-mixed_pgo", + "resource": "8.35 17.25 5.16 107264", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=9c9798866d7762a", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 2, + "compiler": "clang", + "seconds": 34.88775200396776, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O0 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/git-2-clang", + "resource": "34.88 107.77 17.98 120296", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 2, + "compiler": "mixed_pgo", + "seconds": 31.630191639997065, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O0 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/git-2-mixed_pgo", + "resource": "31.62 91.04 22.48 109824", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "curl", + "round": 3, + "compiler": "mixed_pgo", + "seconds": 18.433820882812142, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "CFLAGS=-O0 -g0 -Werror-implicit-function-declaration -Wno-system-headers" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/curl-3-mixed_pgo", + "resource": "18.43 42.47 20.53 108544", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "curl", + "round": 3, + "compiler": "clang", + "seconds": 19.896921730600297, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "CFLAGS=-O0 -g0 -Werror-implicit-function-declaration -Wno-system-headers" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/curl-3-clang", + "resource": "19.89 46.01 19.97 114620", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "redis", + "round": 3, + "compiler": "mixed_pgo", + "seconds": 8.305498275905848, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O0", + "OPTIMIZATION=-O0", + "DEBUG=", + "CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/redis-3-mixed_pgo", + "resource": "8.30 17.35 5.14 107264", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=259e2ca4d1c6f427", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "redis", + "round": 3, + "compiler": "clang", + "seconds": 11.691808973439038, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O0", + "OPTIMIZATION=-O0", + "DEBUG=", + "CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O0 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/redis-3-clang", + "resource": "11.69 20.17 6.11 119236", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=22c33acafdf35c5a", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 3, + "compiler": "mixed_pgo", + "seconds": 31.478255317546427, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O0 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/git-3-mixed_pgo", + "resource": "31.47 90.78 22.29 109824", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 3, + "compiler": "clang", + "seconds": 34.58248379081488, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O0 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang-o0/git-3-clang", + "resource": "34.58 107.70 17.90 120324", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + } + ], + "optimization": "-O0 -g0; all Git/Redis optimization make variables and bundled Lua flags overridden; curl CFLAGS overridden with other configured flags preserved", + "clang_version": "Ubuntu clang version 15.0.7\nTarget: x86_64-pc-linux-gnu\nThread model: posix\nInstalledDir: /usr/bin\n", + "compiler_source": "7804347c97ee021bfac6c7b1a3e6558ec8fb3f40 plus token-copy changes; saved mixed-corpus PGO binary, predates rebased PR main", + "summary": { + "redis": { + "mixed_pgo": { + "seconds": [ + 11.627391956746578, + 8.35491410177201, + 8.305498275905848 + ], + "median": 8.35491410177201 + }, + "clang": { + "seconds": [ + 12.438752811402082, + 11.762709741480649, + 11.691808973439038 + ], + "median": 11.762709741480649 + }, + "reduction_vs_clang_percent": 28.97117853457869 + }, + "git": { + "mixed_pgo": { + "seconds": [ + 32.621819137595594, + 31.630191639997065, + 31.478255317546427 + ], + "median": 31.630191639997065 + }, + "clang": { + "seconds": [ + 34.81821136176586, + 34.88775200396776, + 34.58248379081488 + ], + "median": 34.81821136176586 + }, + "reduction_vs_clang_percent": 9.156184643274312 + }, + "curl": { + "mixed_pgo": { + "seconds": [ + 18.23002382554114, + 18.416057834401727, + 18.433820882812142 + ], + "median": 18.416057834401727 + }, + "clang": { + "seconds": [ + 28.52669501118362, + 18.837173684500158, + 19.896921730600297 + ], + "median": 19.896921730600297 + }, + "reduction_vs_clang_percent": 7.4426784014589025 + } + }, + "cpu": "Intel Core i5-5575R @ 2.80 GHz, four logical CPUs", + "archives": { + "redis": { + "path": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/git-redis-opt/sources/redis-8.10.2.tar.gz", + "sha256": "541a374b753a8405683dd88560465791acc98cf09445ef10ec1bd286904273e8" + }, + "git": { + "path": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/git-redis-opt/sources/git-v2.56.0.tar.gz", + "sha256": "d761232b81394f7d4c3ef1a99fa804ffbe10d278ae1ad302833a0892050ca9fc" + }, + "curl": { + "path": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/curl-build/curl-8.23.0.tar.xz", + "sha256": "8dadd7d2536a314f982ad05a651b713f426bf7d02cc2c76cffb9796f2218e6c5" + } + }, + "curl_configuration": "Shared Dynamite-configured tree: --with-openssl --without-libpsl --disable-shared --enable-static --disable-docs original configure CFLAGS=-O2 -g0; timed make CFLAGS=-O0 -g0 -Werror-implicit-function-declaration -Wno-system-headers; configure excluded; clean and dependency file removal before each build", + "optimization_comparison": { + "O2_report": "linux-x86_64-2026-10-18-pgo-clang.json", + "O2_summary": { + "redis": { + "mixed_pgo": { + "seconds": [ + 18.994099450297654, + 19.056856021285057, + 19.242257386446 + ], + "median": 19.056856021285057 + }, + "clang": { + "seconds": [ + 23.104350995272398, + 23.056384819559753, + 23.271829958073795 + ], + "median": 23.104350995272398 + }, + "reduction_vs_clang_percent": 17.518323604136455 + }, + "git": { + "mixed_pgo": { + "seconds": [ + 49.34677276853472, + 49.10266811773181, + 49.002385552972555 + ], + "median": 49.10266811773181 + }, + "clang": { + "seconds": [ + 52.99491017591208, + 53.124948042444885, + 52.49328352790326 + ], + "median": 52.99491017591208 + }, + "reduction_vs_clang_percent": 7.34455827033258 + }, + "curl": { + "mixed_pgo": { + "seconds": [ + 25.99200871773064, + 25.98153691459447, + 26.103314482606947 + ], + "median": 25.99200871773064 + }, + "clang": { + "seconds": [ + 26.541769503615797, + 26.51101283915341, + 26.383728100918233 + ], + "median": 26.51101283915341 + }, + "reduction_vs_clang_percent": 1.9576925429882652 + } + } + }, + "limits": "Saved PGO compiler predates rebased PR head; default Clang IR backend. Application toolchains, not identical preprocessed inputs. Build time only, no runtime performance comparison. No full upstream test suites or held-out project." +} diff --git a/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang-o0.md b/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang-o0.md new file mode 100644 index 0000000..39c0e24 --- /dev/null +++ b/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang-o0.md @@ -0,0 +1,100 @@ +# Mixed-corpus PGO versus Clang at -O0 + +This repeats the [-O2 comparison](linux-x86_64-2026-10-18-pgo-clang.md) +with `-O0 -g0`, using the same saved mixed-corpus PGO executable and installed +Clang 15.0.7. The compiler binary hashes are unchanged. Dynamite uses its +**default external Clang IR lowering/linking backend**. The opt-in LLVM tools +backend used by the older 10.73-second Redis measurement is not selected here. + +The host is Linux x86_64 on an Intel Core i5-5575R with four logical CPUs. +Dynamite was built with Rust 1.86.0 and trained on curl/Git/Redis (1,229 C +compilations). Its source is `7804347c97ee021bfac6c7b1a3e6558ec8fb3f40` plus +preprocessing token-copy changes; it predates the newer record-member changes +on the rebased PR base. This measures the saved compiler, not the latest PR head. + +## Method + +Three fresh builds per compiler/application run sequentially with `make -j4`. +Compiler order is PGO/Clang, Clang/PGO, PGO/Clang. Source copying, cleaning and +smoke validation are outside the timer; bundled dependencies, build helpers and +linking are inside. Filesystem caches are not flushed. Compiler caches, stage +timers and Clang time traces are disabled. The PGO compiler is not retrained. + +Git v2.56.0 and Redis 8.10.2 use the same pristine source copies and make +arguments as the preceding -O2 comparison, replacing every `-O2` with `-O0`. +This includes Redis `OPT`, `OPTIMIZATION`, `CFLAGS` and bundled `LUA_CFLAGS`. +Git disables Rust, PCH, localization and scripting frontends. Redis builds its +six native tools with libc allocation and TLS/Vector Sets/C11 atomics disabled. +Compiler probes can select different warning flags or capability-dependent C +paths; these are application toolchain comparisons, not identical preprocessed C. + +Curl 8.23.0 shares the same configured tree as before: static libcurl, +OpenSSL/zlib enabled, libpsl/documentation disabled. The original configure step +used Dynamite with -O2 and is not repeated or timed. Each copy is cleaned and +stale dependency files are removed. Timed make overrides `CFLAGS` with +`-O0 -g0 -Werror-implicit-function-declaration -Wno-system-headers`, preserving +all other configured CFLAGS. Both compilers share the generated configuration. + +## Clean-build results + +| Application | Mixed PGO median | Clang median | Reduction vs Clang | +| --- | ---: | ---: | ---: | +| Curl 8.23.0 | 18.4161 s | 19.8969 s | 7.44% | +| Git v2.56.0 | 31.6302 s | 34.8182 s | 9.16% | +| Redis 8.10.2 | 8.3549 s | 11.7627 s | 28.97% | + +| Application | Compiler | Run 1 (s) | Run 2 (s) | Run 3 (s) | +| --- | --- | ---: | ---: | ---: | +| Curl 8.23.0 | Mixed PGO | 18.2300 | 18.4161 | 18.4338 | +| Curl 8.23.0 | Clang | 28.5267 | 18.8372 | 19.8969 | +| Git v2.56.0 | Mixed PGO | 32.6218 | 31.6302 | 31.4783 | +| Git v2.56.0 | Clang | 34.8182 | 34.8878 | 34.5825 | +| Redis 8.10.2 | Mixed PGO | 11.6274 | 8.3549 | 8.3055 | +| Redis 8.10.2 | Clang | 12.4388 | 11.7627 | 11.6918 | + +Reduction is `100 * (1 - PGO median / Clang median)`. The PGO compiler was +faster in every paired build in this snapshot. Three repetitions per +configuration do not establish a general speedup across other applications, +toolchains or platforms. All three applications were included in PGO training. + +The first Clang curl build took 28.527 seconds, versus about 19-20 seconds in +subsequent runs. The first PGO Redis build took 11.627 seconds, versus about +8.3 seconds in subsequent runs. Their user/system CPU totals vary much less +than wall time, so scheduling or other wall-time effects are not ruled out. +The cause was not isolated. All samples are retained; the median is not a +guarantee that every clean build takes that long. + +## Comparison with -O2 + +| Application | PGO -O0 | PGO -O2 | Clang -O0 | Clang -O2 | +| --- | ---: | ---: | ---: | ---: | +| Curl 8.23.0 | 18.4161 s | 25.9920 s | 19.8969 s | 26.5110 s | +| Git v2.56.0 | 31.6302 s | 49.1027 s | 34.8182 s | 52.9949 s | +| Redis 8.10.2 | 8.3549 s | 19.0569 s | 11.7627 s | 23.1044 s | + +These -O2 samples are from the preceding comparison, not interleaved with the +new -O0 builds. Source versions, binaries, backend, make parallelism and optional +features match, but the measurement session differs. Optimization flags can also +change preprocessor capability macros and selected source paths. This is a +whole-build comparison, not isolated optimizer pass timing. Equal flags do not +imply equal executable performance; runtime speed is not measured. + +The older Redis result (10.731 seconds, Clang 14.008 seconds) used -O0 with the +LLVM tools backend and a different compiler revision. The new -O0 results use +the Clang backend, so any remaining difference from 10.731 seconds cannot be +attributed solely to the optimization level or solely to PGO. + +## Validation and artifacts + +All 18 builds succeeded. Every curl binary passed HTTP and HTTPS loopback +requests with certificate verification enabled. Every Git binary passed +init/add/commit/show/clean-status checks in a temporary repository. Every Redis +server/client pair passed PING, SET/GET, Lua EVAL and clean shutdown. Git/Redis +tracked source contents were unchanged after all builds. Full upstream test +suites were not run. macOS and AArch64 were not measured. + +The [measurement snapshot](linux-x86_64-2026-10-18-pgo-clang-o0.json) records +compiler hashes, source archive hashes, exact commands, all samples and smoke +results, and the preceding -O2 medians. Local `temp/pgo-clang-o0/` contains the +comparison script, logs, source copies and smoke artifacts; it is ignored by Git. +No compiler source, normal release binary or training profile was changed. diff --git a/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang.json b/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang.json new file mode 100644 index 0000000..1e29841 --- /dev/null +++ b/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang.json @@ -0,0 +1,602 @@ +{ + "platform": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35", + "jobs": 4, + "versions": { + "curl": "8.23.0", + "git": "v2.56.0", + "redis": "8.10.2" + }, + "compiler_binaries": { + "mixed_pgo": { + "path": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "sha256": "eeb2b4f70798b22a9c64f6f7eb366939a7b4735cd1010009bba4253f446c4d50" + }, + "clang": { + "path": "/usr/bin/clang-15", + "sha256": "22f8651e4eba296325daed391a432b11c3db82d8166da127ccb43b4f6f229e18" + } + }, + "training": { + "mixed_pgo": { + "curl": 477, + "git": 568, + "redis": 184 + }, + "clang": "Installed Clang 15.0.7; no application-derived retraining" + }, + "timing_scope": "make -j4 only; copies, cleaning, dependency removal and validation excluded. No compiler cache, timers or traces. Filesystem caches not flushed.", + "order": [ + [ + "mixed_pgo", + "clang" + ], + [ + "clang", + "mixed_pgo" + ], + [ + "mixed_pgo", + "clang" + ] + ], + "builds": [ + { + "project": "curl", + "round": 1, + "compiler": "mixed_pgo", + "seconds": 25.99200871773064, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/curl-1-mixed_pgo", + "resource": "25.99 71.65 21.27 108544", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "curl", + "round": 1, + "compiler": "clang", + "seconds": 26.541769503615797, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/curl-1-clang", + "resource": "26.54 76.25 20.16 121232", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "redis", + "round": 1, + "compiler": "mixed_pgo", + "seconds": 18.994099450297654, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O2", + "OPTIMIZATION=-O2", + "DEBUG=", + "CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/redis-1-mixed_pgo", + "resource": "18.99 52.64 5.41 111556", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=2d0f8ce463e8424f", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "redis", + "round": 1, + "compiler": "clang", + "seconds": 23.104350995272398, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O2", + "OPTIMIZATION=-O2", + "DEBUG=", + "CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/redis-1-clang", + "resource": "23.10 57.90 6.48 125968", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=6fdc39a630de1883", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 1, + "compiler": "mixed_pgo", + "seconds": 49.34677276853472, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O2 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/git-1-mixed_pgo", + "resource": "49.34 161.00 22.81 121896", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 1, + "compiler": "clang", + "seconds": 52.99491017591208, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O2 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/git-1-clang", + "resource": "52.99 179.42 18.37 139548", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "curl", + "round": 2, + "compiler": "clang", + "seconds": 26.51101283915341, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/curl-2-clang", + "resource": "26.50 76.05 20.21 121232", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "curl", + "round": 2, + "compiler": "mixed_pgo", + "seconds": 25.98153691459447, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/curl-2-mixed_pgo", + "resource": "25.98 72.12 20.90 108544", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "redis", + "round": 2, + "compiler": "clang", + "seconds": 23.056384819559753, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O2", + "OPTIMIZATION=-O2", + "DEBUG=", + "CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/redis-2-clang", + "resource": "23.05 58.05 6.35 125948", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=cfeda812c8745cb", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "redis", + "round": 2, + "compiler": "mixed_pgo", + "seconds": 19.056856021285057, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O2", + "OPTIMIZATION=-O2", + "DEBUG=", + "CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/redis-2-mixed_pgo", + "resource": "19.05 52.87 5.50 109676", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=a6f4eb5a609b22e1", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 2, + "compiler": "clang", + "seconds": 53.124948042444885, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O2 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/git-2-clang", + "resource": "53.12 179.60 18.52 139296", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 2, + "compiler": "mixed_pgo", + "seconds": 49.10266811773181, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O2 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/git-2-mixed_pgo", + "resource": "49.10 160.46 23.24 121360", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "curl", + "round": 3, + "compiler": "mixed_pgo", + "seconds": 26.103314482606947, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/curl-3-mixed_pgo", + "resource": "26.10 71.72 21.33 108544", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "curl", + "round": 3, + "compiler": "clang", + "seconds": 26.383728100918233, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/curl-3-clang", + "resource": "26.38 76.17 20.09 121232", + "smoke": { + "version": "curl 8.23.0 (x86_64-pc-linux-gnu) libcurl/8.23.0 OpenSSL/3.0.2 zlib/1.2.11", + "requests": "HTTP and HTTPS over loopback; certificate verification enabled" + } + }, + { + "project": "redis", + "round": 3, + "compiler": "mixed_pgo", + "seconds": 19.242257386446, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O2", + "OPTIMIZATION=-O2", + "DEBUG=", + "CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/redis-3-mixed_pgo", + "resource": "19.24 52.85 5.47 110492", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=4f6f718982345bdd", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "redis", + "round": 3, + "compiler": "clang", + "seconds": 23.271829958073795, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "-C", + "src", + "MALLOC=libc", + "BUILD_TLS=no", + "SKIP_VEC_SETS=yes", + "OPT=-O2", + "OPTIMIZATION=-O2", + "DEBUG=", + "CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1", + "LUA_CFLAGS=-O2 -g0 -std=gnu99 -D__STDC_NO_ATOMICS__=1 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC= -DLUA_USE_MKSTEMP", + "redis-server", + "redis-cli", + "redis-benchmark", + "redis-check-rdb", + "redis-check-aof", + "redis-sentinel" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/redis-3-clang", + "resource": "23.27 58.07 6.45 125948", + "smoke": { + "version": "Redis server v=8.10.2 sha=a459aa8c:0 malloc=libc bits=64 build=c4795f70ccff8c6f", + "commands": "PING, SET, GET, EVAL, SHUTDOWN NOSAVE" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 3, + "compiler": "mixed_pgo", + "seconds": 49.002385552972555, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/mixed-pgo/pgo/dynamite-pgo", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O2 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/git-3-mixed_pgo", + "resource": "49.00 160.55 23.00 121468", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + }, + { + "project": "git", + "round": 3, + "compiler": "clang", + "seconds": 52.49328352790326, + "returncode": 0, + "command": [ + "make", + "-j4", + "CC=/usr/bin/clang-15", + "NO_RUST=YesPlease", + "NO_PRECOMPILED_HEADER=YesPlease", + "NO_GETTEXT=YesPlease", + "NO_TCLTK=YesPlease", + "NO_PERL=YesPlease", + "NO_PYTHON=YesPlease", + "CFLAGS=-O2 -g0 -std=gnu99", + "all" + ], + "cwd": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/pgo-clang/git-3-clang", + "resource": "52.49 179.24 18.45 139812", + "smoke": { + "version": "git version 2.56.0", + "repository_operations": "init, add, commit, show, clean status" + }, + "tracked_source_unchanged": true + } + ], + "clang_version": "Ubuntu clang version 15.0.7\nTarget: x86_64-pc-linux-gnu\nThread model: posix\nInstalledDir: /usr/bin\n", + "compiler_source": "7804347c97ee021bfac6c7b1a3e6558ec8fb3f40 plus token-copy changes; saved mixed-corpus PGO binary, predates rebased PR main", + "summary": { + "redis": { + "mixed_pgo": { + "seconds": [ + 18.994099450297654, + 19.056856021285057, + 19.242257386446 + ], + "median": 19.056856021285057 + }, + "clang": { + "seconds": [ + 23.104350995272398, + 23.056384819559753, + 23.271829958073795 + ], + "median": 23.104350995272398 + }, + "reduction_vs_clang_percent": 17.518323604136455 + }, + "git": { + "mixed_pgo": { + "seconds": [ + 49.34677276853472, + 49.10266811773181, + 49.002385552972555 + ], + "median": 49.10266811773181 + }, + "clang": { + "seconds": [ + 52.99491017591208, + 53.124948042444885, + 52.49328352790326 + ], + "median": 52.99491017591208 + }, + "reduction_vs_clang_percent": 7.34455827033258 + }, + "curl": { + "mixed_pgo": { + "seconds": [ + 25.99200871773064, + 25.98153691459447, + 26.103314482606947 + ], + "median": 25.99200871773064 + }, + "clang": { + "seconds": [ + 26.541769503615797, + 26.51101283915341, + 26.383728100918233 + ], + "median": 26.51101283915341 + }, + "reduction_vs_clang_percent": 1.9576925429882652 + } + }, + "cpu": "Intel Core i5-5575R @ 2.80 GHz, four logical CPUs", + "archives": { + "redis": { + "path": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/git-redis-opt/sources/redis-8.10.2.tar.gz", + "sha256": "541a374b753a8405683dd88560465791acc98cf09445ef10ec1bd286904273e8" + }, + "git": { + "path": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/git-redis-opt/sources/git-v2.56.0.tar.gz", + "sha256": "d761232b81394f7d4c3ef1a99fa804ffbe10d278ae1ad302833a0892050ca9fc" + }, + "curl": { + "path": "/home/ubuntu/.codex/worktrees/fafc/dynamite-compiler/temp/curl-build/curl-8.23.0.tar.xz", + "sha256": "8dadd7d2536a314f982ad05a651b713f426bf7d02cc2c76cffb9796f2218e6c5" + } + }, + "curl_configuration": "Shared Dynamite-configured tree: --with-openssl --without-libpsl --disable-shared --enable-static --disable-docs CFLAGS=-O2 -g0; configure excluded; clean and dependency file removal before each build", + "limits": "Saved PGO compiler predates rebased PR head; default Clang IR backend. Application toolchains, not identical preprocessed inputs. Build time only, no runtime performance comparison. No full upstream test suites or held-out project." +} diff --git a/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang.md b/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang.md new file mode 100644 index 0000000..d46c0b3 --- /dev/null +++ b/tools/benchmarks/results/linux-x86_64-2026-10-18-pgo-clang.md @@ -0,0 +1,89 @@ +# Mixed-corpus PGO versus Clang + +Measured on Linux x86_64, Intel Core i5-5575R (four logical CPUs), with +Clang 15.0.7. Dynamite is the saved Rust 1.86.0 mixed-corpus PGO executable +from the [training comparison](linux-x86_64-2026-10-18-mixed-pgo.md): compiler +commit `7804347c97ee021bfac6c7b1a3e6558ec8fb3f40` plus the preprocessing +copy changes. It predates the newer record-member changes on the rebased PR +base. This measures that saved PGO compiler, not the rebased PR head. + +## Method + +Each application/compiler pair receives three sequential clean builds with +`make -j4`, `-O2 -g0`, no compiler caches, stage timers or Clang time traces. +Orders are PGO/Clang, Clang/PGO, PGO/Clang. Filesystem caches are not flushed. +Source copies, cleaning and validation are excluded from elapsed time; dependency +compilation, build helpers and linking are included. Both compilers are measured +again together; earlier timing samples are not reused. + +Dynamite uses native preprocessing, parsing and LLVM IR generation, then the +same external Clang 15 for IR lowering and linking. Clang compiles C directly. +`DYNAMITE_BACKEND` is unset: this is the default Clang backend, not the opt-in +LLVM tools backend used in the earlier +[Git/Redis comparison](linux-x86_64-2026-10-18-full-builds.md), which also used +`-O0`. Those timings must not be combined with this `-O2` comparison. +Only Dynamite has application-derived PGO; the installed Clang is unchanged. +Equal optimization flags do not imply identical generated code or executable +performance. Runtime performance is not measured here. + +Git v2.56.0 and Redis 8.10.2 use fresh pristine source copies and the +[same make arguments](linux-x86_64-2026-10-18-git-redis-copy-pgo.md) as the +PGO experiment, with only `CC` changed between compilers. Git disables Rust, +PCH, gettext and Perl/Python/Tcl/Tk integrations. Redis builds its six native +tools, uses libc allocation, disables TLS/Vector Sets/C11 atomics, and sets +matching optimization flags for bundled Lua. Compiler probes remain in the +build timer and can select different warning options and portable/intrinsic +source paths; this compares application toolchains, not identical preprocessed C. + +Curl 8.23.0 uses copies of one tree configured by Dynamite, with static libcurl, +OpenSSL 3.0.2/zlib 1.2.11 enabled, and libpsl/documentation disabled. Each copy +is cleaned and stale dependency files are removed before timing. The configure +step is excluded. Both compilers share its generated configuration; Clang is +not separately configured to enable additional capabilities. + +## Clean-build results + +| Application | Mixed PGO median | Clang median | Reduction vs Clang | +| --- | ---: | ---: | ---: | +| Curl 8.23.0 | 25.9920 s | 26.5110 s | 1.96% | +| Git v2.56.0 | 49.1027 s | 52.9949 s | 7.34% | +| Redis 8.10.2 | 19.0569 s | 23.1044 s | 17.52% | + +| Application | Compiler | Run 1 (s) | Run 2 (s) | Run 3 (s) | +| --- | --- | ---: | ---: | ---: | +| Curl 8.23.0 | Mixed PGO | 25.9920 | 25.9815 | 26.1033 | +| Curl 8.23.0 | Clang | 26.5418 | 26.5110 | 26.3837 | +| Git v2.56.0 | Mixed PGO | 49.3468 | 49.1027 | 49.0024 | +| Git v2.56.0 | Clang | 52.9949 | 53.1249 | 52.4933 | +| Redis 8.10.2 | Mixed PGO | 18.9941 | 19.0569 | 19.2423 | +| Redis 8.10.2 | Clang | 23.1044 | 23.0564 | 23.2718 | + +Reduction is `100 * (1 - PGO median / Clang median)`. The PGO compiler was +faster in every paired build in this snapshot. Three repetitions per +configuration do not establish a general speedup across other applications, +toolchains or platforms. All three applications were included in PGO training. + +The [subsequent -O0 measurements](linux-x86_64-2026-10-18-pgo-clang-o0.md) +use these same compiler binaries and backend, changing the application +optimization level, and include a side-by-side table with the -O2 results. + +## Validation and artifacts + +All 18 builds succeeded. Every curl binary passed HTTP and HTTPS loopback +requests with certificate verification enabled. Every Git binary passed +init/add/commit/show/clean-status checks in a temporary repository. Every Redis +server/client pair passed PING, SET/GET, Lua EVAL and clean shutdown. Git/Redis +tracked source contents were unchanged after all builds. Full upstream test +suites were not run. macOS and AArch64 were not measured. + +Compiler hashes, exact make commands, per-run elapsed/resource measurements +and smoke results are preserved in the +[measurement snapshot](linux-x86_64-2026-10-18-pgo-clang.json). +The source archive identities are documented in the preceding +[curl](linux-x86_64-2026-10-18-curl-copy-pgo.md) and +[Git/Redis](linux-x86_64-2026-10-18-git-redis-copy-pgo.md) reports. +Local `temp/pgo-clang/compare.py` reuses the preceding mixed-PGO comparison, +selecting the saved mixed compiler and `/usr/bin/clang-15` instead of two PGO +compilers. `temp/pgo-clang/` retains all build logs, resource measurements, +source copies and smoke artifacts; it is ignored by Git. No compiler code, +normal release binary or training profiles were changed for this comparison.