Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,10 @@ result. Independent inputs share immutable header scans and directive tokens;
macro definitions, include guards, counters, and dependency lists reset per input.
Parameter references are indexed when a macro is defined, expanded arguments are
memoized within an invocation, and header/dependency membership uses hash tables.
Cached dependency directives are borrowed until line remapping requires a copy.
Macro definitions record which arguments need their original spelling and how
often expanded arguments are used. Expansion moves argument tokens on their last
use and consumes token-paste inputs instead of copying them again.

The same implementation can preprocess C compilation and `-E` without invoking
Clang:
Expand Down
91 changes: 78 additions & 13 deletions src/preprocessor/native.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use lexer::{lex, needs_space, spelling};
pub(super) use options::{preprocess, preprocess_many};
use smol_str::SmolStr;
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
fs,
path::{Path, PathBuf},
Expand Down Expand Up @@ -70,10 +71,17 @@ struct Macro {
variadic: bool,
body: Vec<Token>,
parameter_indices: Vec<Option<usize>>,
parameter_uses: Vec<ParameterUses>,
has_paste: bool,
query: Option<Query>,
}

#[derive(Clone, Default)]
struct ParameterUses {
expanded: usize,
raw: bool,
}

struct Conditional {
parent: bool,
active: bool,
Expand Down Expand Up @@ -199,12 +207,24 @@ impl CleanSource {
})
}

fn directive_tokens(&self, index: usize, text: &str) -> Result<Vec<Token>, String> {
fn directive_tokens(
&self,
index: usize,
text: &str,
physical_line: usize,
) -> Result<Rc<Vec<Token>>, String> {
let mut cache = self.directives.borrow_mut();
if cache[index].is_none() {
cache[index] = Some(Rc::new(lex(text)?));
let mut tokens = lex(text)?;
if let Some(first) = tokens.first_mut() {
first.space = true;
}
for token in &mut tokens {
token.line += physical_line - 1;
}
cache[index] = Some(Rc::new(tokens));
}
Ok(cache[index].as_ref().unwrap().as_ref().clone())
Ok(Rc::clone(cache[index].as_ref().unwrap()))
}
}

Expand Down Expand Up @@ -573,16 +593,27 @@ impl Preprocessor {
dependency_lines.push((physical_line.saturating_add_signed(line_offset), text));
continue;
}
let mut line = if self.dependency_only {
source.directive_tokens(line_index, text)?
} else {
lex(text)?
let cached = self
.dependency_only
.then(|| source.directive_tokens(line_index, text, physical_line))
.transpose()?;
let mut line = match &cached {
Some(tokens) => Cow::Borrowed(tokens.as_slice()),
None => Cow::Owned(lex(text)?),
};
if let Some(first) = line.first_mut() {
first.space = true;
}
for token in &mut line {
token.line = (physical_line + token.line - 1).saturating_add_signed(line_offset);
if !self.dependency_only {
if let Some(first) = line.to_mut().first_mut() {
first.space = true;
}
for token in line.to_mut() {
token.line =
(physical_line + token.line - 1).saturating_add_signed(line_offset);
}
} else if line_offset != 0 {
// A #line directive must not change tokens shared with another input.
for token in line.to_mut() {
token.line = token.line.saturating_add_signed(line_offset);
}
}
if line.first().is_some_and(|t| t.is("#")) {
self.flush_dependency(&mut dependency_lines, &file)?;
Expand Down Expand Up @@ -740,7 +771,7 @@ impl Preprocessor {
}
}
} else if active {
pending.extend(line);
pending.extend(line.into_owned());
pending.push(Token::new(
"\n",
false,
Expand Down Expand Up @@ -796,6 +827,10 @@ mod tests {
"#define V(args...) args\nV(1,2)\n",
"#define V(x,...) f(x,##__VA_ARGS__)\nV(1,) V(1)\n",
"#define S(x) #x\nS(a\n b)\n",
"#define F(x) x x\nF(__COUNTER__) __COUNTER__\n",
"#define F(x) x #x x x##tail\nF(__COUNTER__) __COUNTER__\n",
"#define F(x) #x x##tail x x\nF(__COUNTER__) __COUNTER__\n",
"#define DROP(x) 42\nDROP(__COUNTER__) __COUNTER__\n",
"#define x 3\n#define f(a) f(x * (a))\n#undef x\n#define x 2\n#define g f\n#define z z[0]\n#define h g(~\n#define m(a) a(w)\n#define w 0,1\n#define t(a) a\nf(y+1)+ f(f(z)) % t(t(g)(0)+t)(1);\ng(x+(3,4)-w)| h 5) & m(f)^m(m);\n",
"#define x 7\n#if defined(x) && !defined missing\nx\n#else\n#error wrong\n#endif\n",
"#if 0 && 1/0\n#error wrong\n#elif 1 || 1/0\nyes\n#endif\n",
Expand Down Expand Up @@ -957,6 +992,36 @@ mod tests {
);
}

#[test]
fn shared_directives_keep_physical_lines_between_dependency_scans() {
let directory = super::super::IncludeDirectory::new().unwrap();
fs::write(directory.0.join("mapped.h"), "").unwrap();
fs::write(directory.0.join("physical.h"), "").unwrap();
fs::write(
directory.0.join("shared.h"),
"#ifdef REMAP\n#line 200\n#endif\n#if __LINE__ == 201\n#include \"mapped.h\"\n#elif __LINE__ == 6\n#include \"physical.h\"\n#else\n#error incorrect cached line\n#endif\n",
)
.unwrap();
let mut headers = HashMap::new();
for (code, expected, absent) in [
(
"#define REMAP\n#include \"shared.h\"\n",
"mapped.h",
"physical.h",
),
("#include \"shared.h\"\n", "physical.h", "mapped.h"),
] {
let mut pp = Preprocessor::new(vec![directory.0.clone()], vec![]);
pp.dependency_only = true;
pp.headers = headers;
pp.process(code, &directory.0.join(expected).with_extension("c"))
.unwrap();
assert!(pp.dependencies.contains(&directory.0.join(expected)));
assert!(!pp.dependencies.contains(&directory.0.join(absent)));
headers = pp.headers;
}
}

#[test]
fn source_splices_preserve_builtin_lines() {
for code in [
Expand Down
81 changes: 53 additions & 28 deletions src/preprocessor/native/expansion.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Define and expand macros while preserving hide sets and stateful effects.

use super::lexer::identifier;
use super::{Hidden, Macro, Preprocessor, Query, Token, lex, spelling};
use super::{Hidden, Macro, ParameterUses, Preprocessor, Query, Token, lex, spelling};
use std::{collections::VecDeque, rc::Rc};

impl Preprocessor {
Expand Down Expand Up @@ -50,14 +50,27 @@ impl Preprocessor {
parameters = Some(names);
}
let body = tokens[index..].to_vec();
let parameter_indices = body
let parameter_indices: Vec<_> = body
.iter()
.map(|t| {
parameters
.as_ref()
.and_then(|names| names.iter().position(|p| p == &t.text))
})
.collect();
let mut parameter_uses =
vec![ParameterUses::default(); parameters.as_ref().map_or(0, Vec::len)];
for (index, parameter) in parameter_indices.iter().enumerate() {
if let Some(parameter) = parameter {
let raw = index > 0 && matches!(body[index - 1].text.as_str(), "#" | "##")
|| body.get(index + 1).is_some_and(|t| t.is("##"));
if raw {
parameter_uses[*parameter].raw = true;
} else {
parameter_uses[*parameter].expanded += 1;
}
}
}
let has_paste = body.iter().any(|t| t.is("##"));
if self.dependency_only {
let mut effectful = has_paste;
Expand Down Expand Up @@ -93,6 +106,7 @@ impl Preprocessor {
variadic,
body,
parameter_indices,
parameter_uses,
has_paste,
query: None,
}),
Expand Down Expand Up @@ -318,7 +332,7 @@ impl Preprocessor {
continue;
}
let mut omitted_variadic = false;
let arguments = if let Some(parameters) = &definition.parameters {
let mut arguments = if let Some(parameters) = &definition.parameters {
let n = pending.iter().take_while(|t| t.is("\n")).count();
if !pending.get(n).is_some_and(|t| t.is("(")) {
if !discard {
Expand Down Expand Up @@ -407,10 +421,10 @@ impl Preprocessor {
}
continue;
}
let mut substituted: Vec<Token> = vec![];
let mut substituted: Vec<Token> = Vec::with_capacity(definition.body.len());
let body = &definition.body;
let mut index = 0;
let mut expanded_arguments: Vec<Option<Vec<Token>>> =
let mut expanded_arguments: Vec<Option<(Vec<Token>, usize)>> =
vec![None; arguments.as_ref().map_or(0, Vec::len)];
while index < body.len() {
let current = &body[index];
Expand Down Expand Up @@ -445,25 +459,38 @@ impl Preprocessor {
} else if let Some(arg) = definition.parameter_indices[index] {
let pasted = index > 0 && body[index - 1].is("##")
|| body.get(index + 1).is_some_and(|t| t.is("##"));
let raw = &arguments.as_ref().unwrap()[arg];
let values = if pasted {
raw
let start = substituted.len();
if pasted {
let raw = &arguments.as_ref().unwrap()[arg];
if raw.is_empty() {
substituted.push(Token::new("", current.space, token.line));
} else {
substituted.extend_from_slice(raw);
}
} else {
if expanded_arguments[arg].is_none() {
expanded_arguments[arg] =
Some(self.expand(raw.clone(), file, depth + 1)?);
let raw = &mut arguments.as_mut().unwrap()[arg];
let input = if definition.parameter_uses[arg].raw {
raw.clone()
} else {
std::mem::take(raw)
};
expanded_arguments[arg] = Some((
self.expand(input, file, depth + 1)?,
definition.parameter_uses[arg].expanded,
));
}
expanded_arguments[arg].as_ref().unwrap()
};
if values.is_empty() && pasted {
substituted.push(Token::new("", current.space, token.line));
} else {
let start = substituted.len();
substituted.extend_from_slice(values);
if let Some(first) = substituted.get_mut(start) {
first.space = current.space;
let (values, remaining) = expanded_arguments[arg].as_mut().unwrap();
*remaining -= 1;
if *remaining == 0 {
substituted.append(values);
} else {
substituted.extend_from_slice(values);
}
}
if let Some(first) = substituted.get_mut(start) {
first.space = current.space;
}
} else {
substituted.push(current.clone());
}
Expand All @@ -473,21 +500,20 @@ impl Preprocessor {
substituted
} else {
let mut replacement: Vec<Token> = Vec::with_capacity(substituted.len());
let mut index = 0;
while index < substituted.len() {
if substituted[index].is("##") || substituted[index].is("##gnu") {
let gnu = substituted[index].is("##gnu");
let mut substituted = substituted.into_iter();
while let Some(current) = substituted.next() {
if current.is("##") || current.is("##gnu") {
let gnu = current.is("##gnu");
let left = replacement
.pop()
.ok_or("## cannot start a macro replacement")?;
index += 1;
let right = substituted
.get(index)
.next()
.ok_or("## cannot end a macro replacement")?;
// GNU variadic macros permit comma elision for an omitted tail.
if gnu {
if !omitted_variadic {
replacement.extend([left, right.clone()]);
replacement.extend([left, right]);
}
} else {
let joined = format!("{}{}", left.text, right.text);
Expand All @@ -505,9 +531,8 @@ impl Preprocessor {
}
}
} else {
replacement.push(substituted[index].clone());
replacement.push(current);
}
index += 1;
}
replacement.retain(|t| !t.text.is_empty());
replacement
Expand Down
40 changes: 40 additions & 0 deletions tools/benchmarks/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,46 @@ allocation APIs are not intercepted. Counter runs add atomic-operation overhead
and must not supply performance timings. The interposer uses glibc internals
and is an optional Linux profiling tool, not part of the compiler runtime.

## Profile-guided optimization

Replay successful C compilations recorded by `cc.py` to train a PGO build of
`dynamite-cc`. Install `llvm-tools-preview` for the Rust toolchain first:

```sh
rustup component add llvm-tools-preview
python3 tools/benchmarks/pgo.py \
--records temp/benchmark-run/redis-dynamite-1-records \
--records temp/benchmark-run/git-dynamite-1-records \
--output temp/pgo-run --clang clang-15 --jobs 4
```

`--toolchain VERSION` selects a Rustup toolchain. The matching `llvm-profdata`
is located in that toolchain's sysroot. The output directory must be new.
Training sources, generated headers and working directories referenced by the
records must still exist. Use trusted records: their compiler arguments are
replayed as commands, with object outputs redirected and dependency generation
disabled. Commands, build/training logs, raw and merged profiles, toolchain
metadata and both binaries remain in the output directory.

The resulting `temp/pgo-run/dynamite-pgo` is an optional compiler executable;
the normal release binary is not replaced. Only Dynamite is trained: the external
Clang backend is unchanged. Normal `RUSTFLAGS` apply equally to the instrumented
and profile-use builds; do not supply other PGO flags. Encoded Rust flags are
rejected rather than silently overriding these settings.

Compare the regular and PGO binaries with the same flags and clean-build workload.
Keep training and timing separate. Train on several projects and evaluate held-out
inputs before drawing conclusions about general compiler performance. Profiles
are specific to the source and Rust toolchain; regenerate them after changes.
See the [curl-only versus mixed-corpus measurements](results/linux-x86_64-2026-10-18-mixed-pgo.md)
for a three-application comparison using the same compiler source and different
training corpora.
See the [mixed-PGO versus Clang measurements](results/linux-x86_64-2026-10-18-pgo-clang.md)
for three clean builds per compiler/application with the same `-O2` settings.
The [matching `-O0` comparison](results/linux-x86_64-2026-10-18-pgo-clang-o0.md)
keeps those compiler binaries and the Clang backend fixed while changing the
application optimization level.

For the experimental LLVM tools backend, set `DYNAMITE_BACKEND=llvm` and select
matching `DYNAMITE_LLC`, `DYNAMITE_OPT`, and `DYNAMITE_LLD` executables. Use
`--no-trace`, since Clang time-trace options are unsupported on that path.
Expand Down
Loading
Loading