Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ Notable changes are grouped by release. See [all releases](https://github.com/Co
- Added an authors file and linked the repository documents from the README.
- SARIF rules carry a title (`shortDescription`) and a description (`fullDescription`) instead
of their id, and `describeRule` (`RuleCatalog.hpp`) gives them to programs linking the analyzer.
- In the `bc` format of `--compile-ir-format`, the default, each module comes from the
compilation itself, through coretrace-compiler's `compileToModule` (#273). The analyzer
compiled each source to a temporary `.bc` file, then read and parsed it. With `--timing`, a
source compiled that way logs `Compiled to a module in N ms`, in place of `Compilation done`
and `Bitcode parse done`. coretrace-compiler is updated to v0.13.0, which adds
`compileToModule`.

## v0.22.0

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ IR that another compiler produced (another coretrace-compiler revision or LLVM v
The cache is not used when coretrace-compiler's sources are not a clean checkout FetchContent fetched itself
(`FETCHCONTENT_SOURCE_DIR_CC`, for instance): the analyzer cannot then tell which revision it was built with.
`--compile-ir-format=bc|ll` controls source compilation output format before module load:
- `bc` (default): compile to LLVM bitcode then parse bitcode.
- `bc` (default): take the module from the compilation itself; no bitcode is written or parsed. The compile IR cache stores that module as bitcode.
- `ll`: compile to textual LLVM IR then parse text IR.
`--timing` now also includes pipeline traversal estimates per step
(`module/function/instruction` estimates) and by execution model
Expand Down
2 changes: 1 addition & 1 deletion cmake/compiler/coretrace-compiler.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ endif()
FetchContent_Declare(
cc
GIT_REPOSITORY https://github.com/CoreTrace/coretrace-compiler.git
GIT_TAG b4ad33638e9020a93f052a7a299e4105bd48f975 # v0.10.0
GIT_TAG e21305389a2e639460b2bf619b3a800c15c94fe1 # v0.13.0
)
FetchContent_MakeAvailable(cc)

Expand Down
24 changes: 18 additions & 6 deletions run_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1681,6 +1681,9 @@ def check_compile_ir_format_switch() -> bool:
"""
Validate that --compile-ir-format selects the expected source compile IR path
and that cache keys invalidate correctly when switching format.

In the bc format, a compilation gives its module itself: nothing is parsed. Only a
module read from the compile IR cache is parsed, from the bitcode the cache stored.
"""
print("=== Testing compile IR format switch ===")
sample_c = RUN_CONFIG.test_dir / "alloca/oversized-constant.c"
Expand All @@ -1703,12 +1706,16 @@ def run_and_capture(fmt: str) -> tuple[bool, str]:
bc_ok, bc_output = run_and_capture("bc")
ok = ok and bc_ok
if bc_ok:
if "Bitcode parse done in" not in bc_output:
print(" ❌ --compile-ir-format=bc did not use bitcode parse path")
if "Compiled to a module in" not in bc_output:
print(" ❌ --compile-ir-format=bc did not take the module from the compilation")
print(bc_output)
ok = False
elif "Bitcode parse done in" in bc_output or "IR parse done in" in bc_output:
print(" ❌ --compile-ir-format=bc parsed the module it had compiled")
print(bc_output)
ok = False
else:
print(" ✅ --compile-ir-format=bc uses bitcode parse path")
print(" ✅ --compile-ir-format=bc takes the module from the compilation")

ll_ok, ll_output = run_and_capture("ll")
ok = ok and ll_ok
Expand All @@ -1717,8 +1724,8 @@ def run_and_capture(fmt: str) -> tuple[bool, str]:
print(" ❌ --compile-ir-format=ll did not use textual IR parse path")
print(ll_output)
ok = False
elif "Bitcode parse done in" in ll_output:
print(" ❌ --compile-ir-format=ll unexpectedly used bitcode parse path")
elif "Bitcode parse done in" in ll_output or "Compiled to a module in" in ll_output:
print(" ❌ --compile-ir-format=ll unexpectedly used the bitcode path")
print(ll_output)
ok = False
else:
Expand All @@ -1729,7 +1736,12 @@ def run_and_capture(fmt: str) -> tuple[bool, str]:
bc2_ok, bc2_output = run_and_capture("bc")
ok = ok and bc2_ok
if bc2_ok:
if "Bitcode parse done in" not in bc2_output:
# With the compile IR cache, the bitcode of the first run is parsed; without
# it, the source is compiled to a module again.
if (
"Bitcode parse done in" not in bc2_output
and "Compiled to a module in" not in bc2_output
):
print(" ❌ cache invalidation failed when switching back to bc")
print(bc2_output)
ok = False
Expand Down
134 changes: 45 additions & 89 deletions src/analysis/InputPipeline.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include <llvm/Support/SourceMgr.h>
#include <llvm/Support/raw_ostream.h>

#include <compilerlib/compile_module.hpp>
#include <compilerlib/compiler.h>
#include <coretrace/logger.hpp>

Expand Down Expand Up @@ -78,17 +79,16 @@
args.swap(filtered);
}

// The arguments of a bitcode compilation, whose module coretrace-compiler gives
// without writing an output file.
static std::vector<std::string>
buildBitcodeCompileArgs(const std::vector<std::string>& baseArgs,
const std::filesystem::path& outputBitcodePath)
buildBitcodeCompileArgs(const std::vector<std::string>& baseArgs)
{
std::vector<std::string> args = baseArgs;
args.erase(std::remove(args.begin(), args.end(), "-S"), args.end());
removeOutputPathArgs(args);
appendIfMissing(args, "-emit-llvm");
appendIfMissing(args, "-c");
args.push_back("-o");
args.push_back(outputBitcodePath.string());
return args;
}

Expand Down Expand Up @@ -781,22 +781,6 @@
std::uint64_t active_ : 1 = false;
std::uint64_t reservedFlags_ : 63 = 0;
};

class ScopedFileCleanup
{
public:
explicit ScopedFileCleanup(const std::filesystem::path& path) : path_(path) {}

~ScopedFileCleanup()
{
std::error_code ec;
if (!path_.empty())
std::filesystem::remove(path_, ec);
}

private:
std::filesystem::path path_;
};
} // namespace

static std::string extractExtension(const std::string& path)
Expand Down Expand Up @@ -904,30 +888,17 @@
const CompileIRCachePaths cachePaths = buildCompileIRCachePaths(
config, inputPath, result.language, args, workingDir, coretraceCompilerRevision);

std::error_code tempDirErr;
std::filesystem::path tempDir = std::filesystem::temp_directory_path(tempDirErr);
if (tempDirErr)
tempDir = std::filesystem::path(".");
const std::string tempBitcodeName =
"coretrace-compile-ir-" +
md5Hex(filename + "|" + std::to_string(compileStart.time_since_epoch().count())) +
".bc";
const std::filesystem::path tempBitcodePath = tempDir / tempBitcodeName;
const ScopedFileCleanup tempBitcodeCleanup(tempBitcodePath);
const std::vector<std::string> bitcodeArgs =
buildBitcodeCompileArgs(args, tempBitcodePath);
const std::vector<std::string> bitcodeArgs = buildBitcodeCompileArgs(args);

// A compile command's relative paths are relative to its directory: compile there,
// even when the current directory also resolves them, to another header.
auto compileInWorkingDir =
[&](const std::vector<std::string>& compileArgs,
compilerlib::OutputMode outputMode) -> std::optional<compilerlib::CompileResult>
auto inWorkingDir = [&](auto&& compileHere) -> std::optional<compilerlib::CompileResult>
{
std::lock_guard<std::mutex> compileLock(gCompileInvokeMutex);
if (workingDir.empty())
{
const ScopedHotspot hotspot(config.timing, "input.compiler.invoke");
return compilerlib::compile(compileArgs, outputMode);
return compileHere();
}

std::string cwdError;
Expand All @@ -938,7 +909,24 @@
return std::nullopt;
}
const ScopedHotspot hotspot(config.timing, "input.compiler.invoke.cwd");
return compilerlib::compile(compileArgs, outputMode);
return compileHere();
};
auto compileInWorkingDir =
[&](const std::vector<std::string>& compileArgs, compilerlib::OutputMode outputMode)
{ return inWorkingDir([&] { return compilerlib::compile(compileArgs, outputMode); }); };
// A bitcode compilation gives its module itself, created in `ctx`: no bitcode is
// written to a file and read back.
std::unique_ptr<llvm::Module> compiledModule;
auto compileModuleInWorkingDir = [&](const std::vector<std::string>& compileArgs)
{
return inWorkingDir(
[&]() -> compilerlib::CompileResult
{
compilerlib::ModuleResult compiled =
compilerlib::compileToModule(compileArgs, ctx);
compiledModule = std::move(compiled.module);
return {compiled.success, std::move(compiled.diagnostics), {}, {}};
});
};

if (cachePaths.enabled)
Expand Down Expand Up @@ -1074,9 +1062,9 @@
preferBitcodeCompile ? bitcodeArgs : args;
appendDependencyCaptureArgs(cacheCompileArgs, cachePaths.depFile);

res = compileInWorkingDir(cacheCompileArgs,
preferBitcodeCompile ? compilerlib::OutputMode::ToFile
: compilerlib::OutputMode::ToMemory);
res = preferBitcodeCompile ? compileModuleInWorkingDir(cacheCompileArgs)
: compileInWorkingDir(cacheCompileArgs,
compilerlib::OutputMode::ToMemory);

if (preferBitcodeCompile && (!res || !res->success))
{
Expand Down Expand Up @@ -1111,9 +1099,9 @@
}
else
{
res = compileInWorkingDir(preferBitcodeCompile ? bitcodeArgs : args,
preferBitcodeCompile ? compilerlib::OutputMode::ToFile
: compilerlib::OutputMode::ToMemory);
res = preferBitcodeCompile
? compileModuleInWorkingDir(bitcodeArgs)
: compileInWorkingDir(args, compilerlib::OutputMode::ToMemory);
if (res && res->success)
{
compiledViaBitcode = preferBitcodeCompile;
Expand Down Expand Up @@ -1141,61 +1129,24 @@
}
compileDiagnosticsText = res->diagnostics;

if (compiledViaBitcode)
result.module = std::move(compiledModule);

if (config.timing)
{
auto compileEnd = Clock::now();
auto ms =
std::chrono::duration_cast<std::chrono::milliseconds>(compileEnd - compileStart)
.count();
coretrace::log(coretrace::Level::Info, "Compilation done in {} ms\n", ms);
// No parse follows a compilation that gave its module.
if (result.module)
coretrace::log(coretrace::Level::Info, "Compiled to a module in {} ms\n", ms);
else
coretrace::log(coretrace::Level::Info, "Compilation done in {} ms\n", ms);
}

std::string llvmIRForCache;
std::string llvmBitcodeForCache;
if (compiledViaBitcode)
{
if (readTextFile(tempBitcodePath, llvmBitcodeForCache) &&
!llvmBitcodeForCache.empty())
{
const auto parseStart = Clock::now();
auto bcBuffer = llvm::MemoryBuffer::getMemBufferCopy(
llvm::StringRef(llvmBitcodeForCache.data(), llvmBitcodeForCache.size()),
"compiled_bc");
auto bitcodeModule = [&]()
{
const ScopedHotspot hotspot(config.timing,
"input.compile.parse_bitcode_output");
return llvm::parseBitcodeFile(bcBuffer->getMemBufferRef(), ctx);
}();
if (bitcodeModule)
{
result.module = std::move(*bitcodeModule);
if (config.timing)
{
const auto parseEnd = Clock::now();
const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
parseEnd - parseStart)
.count();
coretrace::log(coretrace::Level::Info, "Bitcode parse done in {} ms\n",
ms);
}
}
else if (config.timing)
{
std::string bitcodeError = llvm::toString(bitcodeModule.takeError());
coretrace::log(coretrace::Level::Warn,
"Bitcode output invalid for {}; "
"falling back to textual IR ({})\n",
filename, bitcodeError);
}
}
else if (config.timing)
{
coretrace::log(coretrace::Level::Warn,
"Missing bitcode output for {}; falling back to textual IR\n",
filename);
}
}

if (!result.module)
{
Expand Down Expand Up @@ -1259,8 +1210,13 @@
{
if (llvmBitcodeForCache.empty())
{
// The order of each value's uses is written too. Some analyses depend on it:
// the first match of a walk, the order of a variable's reports. The reader
// otherwise rebuilds that order its own way, and a run that reads the cache
// would not walk the uses as the run that compiled the source did.
llvm::raw_string_ostream bitcodeStream(llvmBitcodeForCache);
llvm::WriteBitcodeToFile(*result.module, bitcodeStream);
llvm::WriteBitcodeToFile(*result.module, bitcodeStream,
/*ShouldPreserveUseListOrder=*/true);
bitcodeStream.flush();
}

Expand Down
Loading
Loading