From d9a2437b871ac92f6363b184c1b216f3aa8ac38e Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 13 Sep 2026 00:38:17 +0100 Subject: [PATCH 1/4] Let tslang choose the garbage collector's linkage (Windows, -mm=gc) A user's shared library and the program that imports it still linked the static gc.lib unless --gc-lib-path was swapped by hand, so they got two collectors and one freed what the other held. Now --emit=dll, and --emit=exe for a program that imports a tslang shared library (CompileOptions::importsSharedLibrary, found as LoadLibraryPermanentlyOp before lowering), link gc.dll's import library and copy gc.dll beside the output. The shared collector is found through --gc-shared-lib-path, GC_SHARED_LIB_PATH, /gcdll (the release package), or --gc-lib-path itself when it is a shared build; with none found the build stops instead of linking a collector of its own. A lone program keeps the static gc.lib. Linux unchanged. test-compile-gc-shared-auto drives tslang --emit=dll/exe itself (five cases). With the previous compiler a static-only library + importer fail the gc_single_collector assertion. Suite: 2,705 of 2,705. Co-Authored-By: Claude Opus 5 --- docs/memory-models.md | 22 +++- tslang/docs/single-gc-collector-design.md | 35 +++++- tslang/include/TypeScript/DataStructs.h | 7 ++ tslang/test/tester/CMakeLists.txt | 20 ++++ tslang/tslang/exe.cpp | 129 +++++++++++++++++++++- tslang/tslang/tslang.cpp | 6 + 6 files changed, 208 insertions(+), 11 deletions(-) diff --git a/docs/memory-models.md b/docs/memory-models.md index ea7cb9646..66ec7d9c7 100644 --- a/docs/memory-models.md +++ b/docs/memory-models.md @@ -128,13 +128,23 @@ the executable's roots, so it frees objects the executable is still holding. The a crash: the freed memory is reallocated and the program reads a plausible wrong value, which only shows up when what was written over it differs from what was there. -The Windows release package ships the shared collector in its `gcdll` folder, beside the static -`gc.lib` at its root — both files are named `gc.lib`, so the folder is what tells them apart. -Compile the executable **and** every shared library with `--gc-lib-path=/gcdll`, and -ship `gcdll/gc.dll` beside the executable. +On Windows `tslang` makes the choice itself: -From a source build, the same files come from `scripts/build_gc_release_shared_vs.bat`: link -against `3rdParty/gcdll/x64/release/lib/gc.lib` and ship `3rdParty/gcdll/x64/release/bin/gc.dll`. +- `--emit=dll`, and `--emit=exe` for a program that imports a tslang shared library, link the + shared collector and copy `gc.dll` beside the output. +- `--emit=exe` for a program that imports none keeps the static `gc.lib`, and ships as one file. + +It finds the shared collector through `--gc-shared-lib-path` (or `GC_SHARED_LIB_PATH`), else the +`gcdll` folder inside `--gc-lib-path` — which is where the Windows release package ships it, beside +the static `gc.lib` at its root; both files are named `gc.lib`, so the folder is what tells them +apart — else `--gc-lib-path` itself when that already names a shared build. If none of those has +one, the build stops with an error rather than linking a collector of its own. + +From a source build, the shared collector comes from `scripts/build_gc_release_shared_vs.bat`: +`--gc-shared-lib-path=3rdParty/gcdll/x64/release/lib` (with `gc.dll` in its `../bin`). + +Linking by hand (`--emit=obj` and your own linker) makes no choice for you: link the executable +**and** every shared library against the shared `gc.lib`, and ship `gc.dll` beside the executable. Statically linked programs are unaffected and keep the static `gc.lib` — one binary already means one collector. `-mm=rc` and `-mm=none` are unaffected either way: neither has a collector. diff --git a/tslang/docs/single-gc-collector-design.md b/tslang/docs/single-gc-collector-design.md index 91925528a..ff22c55dd 100644 --- a/tslang/docs/single-gc-collector-design.md +++ b/tslang/docs/single-gc-collector-design.md @@ -1,7 +1,7 @@ # One collector per process: design proposal -Status: **PR 1 implemented** (steps 1, 2, 5 - Windows) on branch `fix-single-gc-collector`; -steps 3, 4, 6 and Linux still open. See [Progress](#progress) at the end. +Status: **PR 1 merged** (steps 1, 2, 5 - Windows); **PR 2** (step 3) on branch +`gc-shared-lib-auto`; steps 4, 6 and Linux still open. See [Progress](#progress) at the end. ## Problem @@ -222,5 +222,32 @@ suppressed): | JIT + default-lib DLL repro | 2000 / 2000 bad | 0 bad | | exe + user DLL + default-lib DLL repro | 1984 / 2000 bad | 0 bad | -Still open: the tests that would include the default library itself (the suite still passes -`--no-default-lib`), steps 3, 4 and 6, the debug default-lib build, and all of Linux. +Merged as #309 (compiler) and TypeScriptCompilerDefaultLib #6. + +### PR 2 - `tslang` chooses the collector's linkage (step 3, Windows) + +- `CompileOptions::importsSharedLibrary`, set in `tslang.cpp` by walking the generated module for + `LoadLibraryPermanentlyOp` before the passes lower it. Every `import` that resolves to a DLL - + dynamic or `@static` - goes through `mlirGenImportSharedLib`, which emits that op. +- `exe.cpp`: under `-mm=gc` on Windows, `--emit=dll` or an importing `--emit=exe` links + `-L` instead of the static directory, and after a successful link copies `gc.dll` beside + the output (a warning names the file to ship if it cannot). +- The shared directory: `--gc-shared-lib-path`, else `GC_SHARED_LIB_PATH`, else + `/gcdll` (the release package), else `--gc-lib-path` itself when `gc.dll` sits + beside it or in `../bin` (so the default library's `build_core.bat` keeps working). Nothing found + is an **error**, not a fallback to the static `gc.lib`. +- A lone program is unchanged: static `gc.lib`, no `gc.dll` copied. +- `test-compile-gc-shared-auto` (`gc-shared-auto.cmake`) drives `tslang --emit=dll/exe` itself - + `test-runner` links with lld directly and never exercised the compiler's choice. Five cases: the + library gets `gc.dll`; its importer runs clean; `--gc-lib-path` at a shared build alone is + enough; a lone exe runs with no `gc.dll` near it and gets none copied; a library with only a + static collector fails and names `gc.dll`. +- Teeth, with the previous compiler: a library and its importer built with only + `--gc-lib-path=` link two collectors and fail the gc_single_collector assertion; with + collection suppressed the same binary prints 0 bad. +- The VS Code template's `--emit=dll` task passes `--gc-lib-path=`, so it resolves + `/gcdll` without a change. + +Still open: steps 4 and 6, the CMake project template (it links `gc` for every executable and does +not build shared libraries), tests that include the default library itself (the suite still passes +`--no-default-lib`), the debug default-lib build, and all of Linux. diff --git a/tslang/include/TypeScript/DataStructs.h b/tslang/include/TypeScript/DataStructs.h index df3709972..a90a5aea5 100644 --- a/tslang/include/TypeScript/DataStructs.h +++ b/tslang/include/TypeScript/DataStructs.h @@ -36,6 +36,13 @@ struct CompileOptions bool strictNullChecks; bool enableFastMath; + // Whether the module imports a tslang shared library (`import './lib'` resolving to a DLL). + // Set once the module is generated, and read when linking: under `-mm=gc` such a program must + // take its collector from gc.dll, like the library does, because two statically linked + // collectors in one process each free what only the other's memory references. + // See docs/single-gc-collector-design.md. + bool importsSharedLibrary = false; + // Whether the Boehm runtime has to be present. Only `gc` needs it: it is the model whose // reclamation *is* the collector. `rc` frees through the reference counts it maintains and // `none` frees nothing, so both allocate straight from `malloc` and neither links libgc. diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index 6cae407c4..505fb82f1 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -1881,6 +1881,26 @@ add_test(NAME test-compile-rc-debug-info -DTESTS_DIR=${PROJECT_SOURCE_DIR}/test/tester/tests -P ${CMAKE_CURRENT_SOURCE_DIR}/debug-info-rc.cmake) +# tslang choosing the collector's linkage itself under -mm=gc - see gc-shared-auto.cmake. +if (WIN32) + if ("${CMAKE_BUILD_TYPE}" STREQUAL "Release") + set(gc_shared_auto_opt "--opt;--opt_level=3") + else() + set(gc_shared_auto_opt "--di;--opt_level=0") + endif() + add_test(NAME test-compile-gc-shared-auto + COMMAND ${CMAKE_COMMAND} + "-DTSLANG=$" + "-DTESTS_DIR=${PROJECT_SOURCE_DIR}/test/tester/tests" + "-DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}/gc-shared-auto" + "-DGC_LIB=${TEST_GC_LIBDIR}" + "-DGC_SHARED_LIB=${TEST_GC_SHARED_PREFIX}/lib" + "-DLLVM_LIB=${PROJECT_SOURCE_DIR}/../3rdParty/llvm/x64/${CMAKE_BUILD_TYPE_LOWERCASE}/lib" + "-DTSLANG_LIB=${CMAKE_BINARY_DIR}/lib" + "-DOPT=${gc_shared_auto_opt}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/gc-shared-auto.cmake") +endif() + set(TSLANG_OWNERSHIP_SHARDS 8) math(EXPR ownership_last "${TSLANG_OWNERSHIP_SHARDS} - 1") foreach(ownership_shard RANGE ${ownership_last}) diff --git a/tslang/tslang/exe.cpp b/tslang/tslang/exe.cpp index 500e5fa89..91e51155c 100644 --- a/tslang/tslang/exe.cpp +++ b/tslang/tslang/exe.cpp @@ -14,6 +14,7 @@ #include "llvm/Support/Process.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/FileSystem.h" #include "llvm/Support/WithColor.h" #include "llvm/CodeGen/CommandFlags.h" @@ -28,6 +29,7 @@ extern cl::opt outputFilename; extern cl::opt TargetTriple; extern cl::opt defaultlibpath; extern cl::opt gclibpath; +extern cl::opt gcsharedlibpath; extern cl::opt llvmlibpath; extern cl::opt tslanglibpath; extern cl::opt emsdksysrootpath; @@ -159,6 +161,81 @@ std::string getGCLibPath() return ""; } +// Boehm built as a DLL: a directory holding its import library `gc.lib`, with `gc.dll` beside it +// (the release package's gcdll folder) or in `../bin` (a CMake install). +static std::string findGCDll(llvm::StringRef libDir) +{ + llvm::SmallString<256> beside(libDir); + llvm::sys::path::append(beside, "gc.dll"); + if (llvm::sys::fs::exists(beside)) + { + return beside.str().str(); + } + + llvm::SmallString<256> bin(libDir); + llvm::sys::path::append(bin, "..", "bin", "gc.dll"); + if (llvm::sys::fs::exists(bin)) + { + llvm::sys::path::remove_dots(bin, /*remove_dot_dot=*/true); + return bin.str().str(); + } + + return ""; +} + +static bool isSharedGCLibDir(llvm::StringRef libDir) +{ + llvm::SmallString<256> lib(libDir); + llvm::sys::path::append(lib, "gc.lib"); + return llvm::sys::fs::exists(lib) && !findGCDll(libDir).empty(); +} + +// Where the shared collector is, or "" when there is none to be found. Not a fallback to the +// static gc.lib: that is exactly the build this exists to avoid, and it links fine and then frees +// live objects, which is far harder to find than a link that stops here. +std::string getGCSharedLibPath() +{ + if (!gcsharedlibpath.empty()) + { + checkGCLibPath(gcsharedlibpath); + return gcsharedlibpath; + } + + if (auto gcSharedLibEnvValue = llvm::sys::Process::GetEnv("GC_SHARED_LIB_PATH")) + { + if (!gcSharedLibEnvValue->empty()) + { + checkGCLibPath(gcSharedLibEnvValue.value()); + return gcSharedLibEnvValue.value(); + } + } + + std::string staticPath = gclibpath; + if (staticPath.empty()) + { + staticPath = llvm::sys::Process::GetEnv("GC_LIB_PATH").value_or(""); + } + + if (!staticPath.empty()) + { + // the release package: the shared build in a gcdll folder beside the static gc.lib + llvm::SmallString<256> gcdll(staticPath); + llvm::sys::path::append(gcdll, "gcdll"); + if (isSharedGCLibDir(gcdll)) + { + return gcdll.str().str(); + } + + // --gc-lib-path already names a shared build + if (isSharedGCLibDir(staticPath)) + { + return staticPath; + } + } + + return ""; +} + std::string getLLVMLibPath() { if (!llvmlibpath.empty()) @@ -425,9 +502,34 @@ int buildExe(int argc, char **argv, std::string objFileName, std::string additio } } + // Which Boehm. A shared library, and a program that loads one, share a process with other gc + // code, so they take the collector from gc.dll: linked statically, each binary brings a + // collector of its own, and one frees objects only the other's memory references. A program + // that is alone keeps the static gc.lib and ships as one file. Windows only for now - Linux has + // not been measured. See docs/single-gc-collector-design.md. + auto useSharedGC = win && compileOptions.needsGCRuntime() && (shared || compileOptions.importsSharedLibrary); + std::string gcSharedLibPath; + std::string gcDllPath; + if (useSharedGC) + { + gcSharedLibPath = getGCSharedLibPath(); + if (gcSharedLibPath.empty()) + { + llvm::WithColor::error(llvm::errs(), "tslang") + << (shared ? "a shared library" : "a program that imports a shared library") + << " built with -mm=gc links the garbage collector from gc.dll, so that the process has only one collector" + " - linked statically, it would free objects another module still holds. Point --gc-shared-lib-path" + " (or GC_SHARED_LIB_PATH) at the directory with gc.dll's import library 'gc.lib'; the release package" + " ships it as 'gcdll'.\n"; + return 1; + } + + gcDllPath = findGCDll(gcSharedLibPath); + } + if (compileOptions.needsGCRuntime()) { - gcLibPathOpt = getLibsPathOpt(getGCLibPath()); + gcLibPathOpt = getLibsPathOpt(useSharedGC ? gcSharedLibPath : getGCLibPath()); if (!gcLibPathOpt.empty()) { args.push_back(gcLibPathOpt.c_str()); @@ -657,6 +759,31 @@ int buildExe(int argc, char **argv, std::string objFileName, std::string additio diags.getClient()->finish(); + // A binary linked against gc.dll does not start without it, so put it beside the output. + if (res == 0 && useSharedGC) + { + auto outputDir = llvm::sys::path::parent_path(outputFilename.getValue()); + llvm::SmallString<256> destination(outputDir.empty() ? "." : outputDir); + llvm::sys::path::append(destination, "gc.dll"); + + bool sameFile = false; + if (gcDllPath.empty()) + { + llvm::WithColor::warning(llvm::errs(), "tslang") + << "linked against gc.dll, but no gc.dll was found next to '" << gcSharedLibPath + << "' or in its '../bin'; ship gc.dll beside '" << outputFilename.getValue() << "'\n"; + } + else if (llvm::sys::fs::equivalent(gcDllPath, destination, sameFile) || !sameFile) + { + if (auto error = llvm::sys::fs::copy_file(gcDllPath, destination)) + { + llvm::WithColor::warning(llvm::errs(), "tslang") + << "could not copy '" << gcDllPath << "' to '" << destination << "': " << error.message() + << "; ship gc.dll beside '" << outputFilename.getValue() << "'\n"; + } + } + } + // If we have multiple failing commands, we return the result of the first // failing command. return res; diff --git a/tslang/tslang/tslang.cpp b/tslang/tslang/tslang.cpp index bf788f9d9..f6b295dd5 100644 --- a/tslang/tslang/tslang.cpp +++ b/tslang/tslang/tslang.cpp @@ -1,6 +1,7 @@ #include "TypeScript/Version.h" #include "TypeScript/Config.h" #include "TypeScript/TypeScriptDialect.h" +#include "TypeScript/TypeScriptOps.h" #include "mlir/InitAllDialects.h" #include "mlir/InitAllExtensions.h" @@ -138,6 +139,7 @@ cl::opt embedExportDeclarationsAction("embed-declarations", cl::desc("Embe cl::opt defaultlibpath("default-lib-path", cl::desc("JS library path. Should point to folder/directory with subfolder '" DEFAULT_LIB_DIR "' or DEFAULT_LIB_PATH environmental variable"), cl::value_desc("defaultlibpath"), cl::cat(TypeScriptCompilerBuildCategory)); cl::opt gclibpath("gc-lib-path", cl::desc("GC library path. Should point to file 'gc.lib' or GC_LIB_PATH environmental variable"), cl::value_desc("gclibpath"), cl::cat(TypeScriptCompilerBuildCategory)); +cl::opt gcsharedlibpath("gc-shared-lib-path", cl::desc("Shared GC library path: the directory with gc.dll's import library 'gc.lib' (gc.dll beside it or in '../bin'). Used under -mm=gc for --emit=dll and for executables that import a shared library, so the process has one collector. Or GC_SHARED_LIB_PATH environmental variable; defaults to '/gcdll'"), cl::value_desc("gcsharedlibpath"), cl::cat(TypeScriptCompilerBuildCategory)); cl::opt llvmlibpath("llvm-lib-path", cl::desc("LLVM library path. Should point to file 'LLVMSupport.lib' and 'LLVMDemangle' in linux or LLVM_LIB_PATH environmental variable"), cl::value_desc("llvmlibpath"), cl::cat(TypeScriptCompilerBuildCategory)); cl::opt tslanglibpath("tslang-lib-path", cl::desc("TypeScript Compiler Runtime library path. Should point to file 'TypeScriptAsyncRuntime.lib' or TSLANG_LIB_PATH environmental variable"), cl::value_desc("tslanglibpath"), cl::cat(TypeScriptCompilerBuildCategory)); cl::opt emsdksysrootpath("emsdk-sysroot-path", cl::desc("TypeScript Compiler Runtime library path. Should point to dir '<...>/emsdk/upstream/emscripten/cache/sysroot' or EMSDK_SYSROOT_PATH environmental variable. (used when '-mtriple=wasm32-pc-emscripten')"), cl::value_desc("emsdksysrootpath"), cl::cat(TypeScriptCompilerBuildCategory)); @@ -396,6 +398,10 @@ int main(int argc, char **argv) return error; } + // Read before the passes lower it away: a program that loads a tslang shared library has to + // share that library's garbage collector, so linking needs to know. See exe.cpp. + module->walk([&](mlir::typescript::LoadLibraryPermanentlyOp) { compileOptions.importsSharedLibrary = true; }); + if (auto error = runMLIRPasses(mlirContext, sourceMgr, module, compileOptions)) { return error; From 63b9aed3189137da48a7526e28b108227fbfc0c6 Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 13 Sep 2026 00:39:16 +0100 Subject: [PATCH 2/4] Add the test-compile-gc-shared-auto script, renamed past .gitignore's gc-* The previous commit registered test-compile-gc-shared-auto but its script, gc-shared-auto.cmake, matched the root .gitignore's gc-* (meant for the gc-8.x source folders) and was never added. Renamed to shared-collector-auto.cmake. Co-Authored-By: Claude Opus 5 --- tslang/docs/single-gc-collector-design.md | 2 +- tslang/test/tester/CMakeLists.txt | 4 +- .../test/tester/shared-collector-auto.cmake | 96 +++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 tslang/test/tester/shared-collector-auto.cmake diff --git a/tslang/docs/single-gc-collector-design.md b/tslang/docs/single-gc-collector-design.md index ff22c55dd..13db07231 100644 --- a/tslang/docs/single-gc-collector-design.md +++ b/tslang/docs/single-gc-collector-design.md @@ -237,7 +237,7 @@ Merged as #309 (compiler) and TypeScriptCompilerDefaultLib #6. beside it or in `../bin` (so the default library's `build_core.bat` keeps working). Nothing found is an **error**, not a fallback to the static `gc.lib`. - A lone program is unchanged: static `gc.lib`, no `gc.dll` copied. -- `test-compile-gc-shared-auto` (`gc-shared-auto.cmake`) drives `tslang --emit=dll/exe` itself - +- `test-compile-gc-shared-auto` (`shared-collector-auto.cmake`) drives `tslang --emit=dll/exe` itself - `test-runner` links with lld directly and never exercised the compiler's choice. Five cases: the library gets `gc.dll`; its importer runs clean; `--gc-lib-path` at a shared build alone is enough; a lone exe runs with no `gc.dll` near it and gets none copied; a library with only a diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index 505fb82f1..fd8d99a2b 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -1881,7 +1881,7 @@ add_test(NAME test-compile-rc-debug-info -DTESTS_DIR=${PROJECT_SOURCE_DIR}/test/tester/tests -P ${CMAKE_CURRENT_SOURCE_DIR}/debug-info-rc.cmake) -# tslang choosing the collector's linkage itself under -mm=gc - see gc-shared-auto.cmake. +# tslang choosing the collector's linkage itself under -mm=gc - see shared-collector-auto.cmake. if (WIN32) if ("${CMAKE_BUILD_TYPE}" STREQUAL "Release") set(gc_shared_auto_opt "--opt;--opt_level=3") @@ -1898,7 +1898,7 @@ if (WIN32) "-DLLVM_LIB=${PROJECT_SOURCE_DIR}/../3rdParty/llvm/x64/${CMAKE_BUILD_TYPE_LOWERCASE}/lib" "-DTSLANG_LIB=${CMAKE_BINARY_DIR}/lib" "-DOPT=${gc_shared_auto_opt}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/gc-shared-auto.cmake") + -P "${CMAKE_CURRENT_SOURCE_DIR}/shared-collector-auto.cmake") endif() set(TSLANG_OWNERSHIP_SHARDS 8) diff --git a/tslang/test/tester/shared-collector-auto.cmake b/tslang/test/tester/shared-collector-auto.cmake new file mode 100644 index 000000000..fbf4a6c7a --- /dev/null +++ b/tslang/test/tester/shared-collector-auto.cmake @@ -0,0 +1,96 @@ +# tslang chooses the garbage collector's linkage by itself (Windows, -mm=gc). +# +# A shared library, and a program that imports one, must take Boehm from gc.dll: two statically +# linked collectors in one process each free what only the other's memory references (see +# docs/single-gc-collector-design.md). A program that is alone keeps the static gc.lib. +# +# This drives `tslang --emit=dll` / `--emit=exe` itself, which `test-runner` does not: its shared +# tests link with lld directly and pick the collector in the test harness, so they never exercise +# the compiler's own choice. +# +# The pair is the gc_single_collector one: the library builds the strings the program holds, then +# churns different ones, so a second collector in the library frees them. + +cmake_minimum_required(VERSION 3.17.3) + +foreach(var TSLANG TESTS_DIR WORK_DIR GC_LIB GC_SHARED_LIB LLVM_LIB TSLANG_LIB OPT) + if(NOT DEFINED ${var}) + message(FATAL_ERROR "${var} is required") + endif() +endforeach() + +# Only what the command line says: an inherited GC_SHARED_LIB_PATH / GC_LIB_PATH would hide +# whether the compiler found the collector itself. +set(ENV{GC_SHARED_LIB_PATH} "") +set(ENV{GC_LIB_PATH} "") + +file(REMOVE_RECURSE "${WORK_DIR}") +file(MAKE_DIRECTORY "${WORK_DIR}/lone") + +set(common --no-default-lib ${OPT} "--llvm-lib-path=${LLVM_LIB}" "--tslang-lib-path=${TSLANG_LIB}") + +# run( ) - leaves the output in run_output +function(run what dir expect_success) + execute_process(COMMAND ${ARGN} + WORKING_DIRECTORY "${dir}" + OUTPUT_VARIABLE out + ERROR_VARIABLE err + RESULT_VARIABLE status) + set(run_output "${out}${err}" PARENT_SCOPE) + if(expect_success AND NOT status EQUAL 0) + message(FATAL_ERROR "${what}: exit ${status}\n${out}\n${err}") + endif() + if(NOT expect_success AND status EQUAL 0) + message(FATAL_ERROR "${what}: expected to fail, but succeeded\n${out}\n${err}") + endif() +endfunction() + +set(library "${TESTS_DIR}/export_gc_single_collector.ts") +set(program "${TESTS_DIR}/import_gc_single_collector.ts") + +# 1. A shared library links gc.dll and gets gc.dll copied beside it. +run("--emit=dll" "${WORK_DIR}" TRUE + "${TSLANG}" --emit=dll ${common} "--gc-lib-path=${GC_LIB}" "--gc-shared-lib-path=${GC_SHARED_LIB}" + "${library}" -o export_gc_single_collector.dll) +if(NOT EXISTS "${WORK_DIR}/gc.dll") + message(FATAL_ERROR "--emit=dll did not put gc.dll beside the library") +endif() + +# 2. A program that imports it links gc.dll too, and the two share one collector. +run("--emit=exe importing a shared library" "${WORK_DIR}" TRUE + "${TSLANG}" --emit=exe ${common} "--gc-lib-path=${GC_LIB}" "--gc-shared-lib-path=${GC_SHARED_LIB}" + "${program}" -o main.exe) +run("the program" "${WORK_DIR}" TRUE "${WORK_DIR}/main.exe") +if(NOT run_output MATCHES "done\\.") + message(FATAL_ERROR "the program did not finish:\n${run_output}") +endif() + +# 3. --gc-lib-path naming a shared build is enough on its own (the default library's build script). +file(REMOVE "${WORK_DIR}/export_gc_single_collector.dll") +run("--emit=dll with --gc-lib-path at the shared build" "${WORK_DIR}" TRUE + "${TSLANG}" --emit=dll ${common} "--gc-lib-path=${GC_SHARED_LIB}" + "${library}" -o export_gc_single_collector.dll) +run("the program, against that library" "${WORK_DIR}" TRUE "${WORK_DIR}/main.exe") +if(NOT run_output MATCHES "done\\.") + message(FATAL_ERROR "the program did not finish against the rebuilt library:\n${run_output}") +endif() + +# 4. A program that is alone keeps the static collector: it runs with no gc.dll anywhere near it. +run("--emit=exe alone" "${WORK_DIR}/lone" TRUE + "${TSLANG}" --emit=exe ${common} "--gc-lib-path=${GC_LIB}" "--gc-shared-lib-path=${GC_SHARED_LIB}" + "${TESTS_DIR}/00funcs.ts" -o lone.exe) +if(EXISTS "${WORK_DIR}/lone/gc.dll") + message(FATAL_ERROR "--emit=exe copied gc.dll beside a program that imports no shared library") +endif() +run("the lone program" "${WORK_DIR}/lone" TRUE "${WORK_DIR}/lone/lone.exe") + +# 5. With no shared collector to be found, a shared library does not link rather than silently +# getting a collector of its own. +run("--emit=dll with only a static collector" "${WORK_DIR}/lone" FALSE + "${TSLANG}" --emit=dll ${common} "--gc-lib-path=${GC_LIB}" + "${library}" -o nope.dll) +if(NOT run_output MATCHES "gc\\.dll") + message(FATAL_ERROR "--emit=dll failed without naming gc.dll:\n${run_output}") +endif() + +message(STATUS "tslang picks the shared collector for a library and its importer, and the static one for a lone program") From 5b9a15b403a824c984810cadfdcf539ba43d8732 Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 13 Sep 2026 11:28:04 +0100 Subject: [PATCH 3/4] Implement shared garbage collector support for tslang shared libraries and enforce linkage rules --- docs/how/cmake_tslang/CMakeLists.txt | 19 +++++- docs/how/cmake_tslang/README.md | 19 ++++++ docs/memory-models.md | 25 ++++++++ tslang/docs/single-gc-collector-design.md | 64 +++++++++++++++++-- tslang/include/TypeScript/ObjDumper.h | 5 ++ tslang/lib/TypeScript/MLIRGenModule.cpp | 15 +++++ tslang/lib/TypeScript/ObjDumper.cpp | 43 +++++++++++++ .../test/tester/shared-collector-auto.cmake | 27 +++++++- tslang/tslang/exe.cpp | 21 +++++- tslang/tslang/jit.cpp | 21 ++++++ 10 files changed, 250 insertions(+), 9 deletions(-) diff --git a/docs/how/cmake_tslang/CMakeLists.txt b/docs/how/cmake_tslang/CMakeLists.txt index 4efb00f49..2138a6e90 100644 --- a/docs/how/cmake_tslang/CMakeLists.txt +++ b/docs/how/cmake_tslang/CMakeLists.txt @@ -24,6 +24,12 @@ endif() set(TSLANG_MEMORY_MODEL "gc" CACHE STRING "Memory model of compiled code: gc, rc or none") set_property(CACHE TSLANG_MEMORY_MODEL PROPERTY STRINGS gc rc none) +# Turn on when the program imports a tslang shared library (`import './lib'`, built with +# `tslang --emit=dll`). Under gc the process must then have one collector, shared by the program +# and the library: on Windows both take it from gc.dll, on Linux the program exports its own. +# Two collectors free each other's live objects. See docs/memory-models.md. +option(TSLANG_SHARED_GC "The program loads tslang shared libraries (gc: share one collector)" OFF) + # Lib folders link_directories(${CMAKE_TSLANG_DIR} ${CMAKE_TSLANG_DIR}/defaultlib/lib/${TSLANG_DEFAULTLIB_BUILD}/${TSLANG_MEMORY_MODEL}) @@ -56,7 +62,18 @@ set(TSLANG_LINK_LIBS "TypeScriptDefaultLib" "TypeScriptAsyncRuntime" "LLVMSuppor # Boehm is only referenced by the gc default lib; the rc and none builds allocate through the # CRT and must not drag a collector in. if (TSLANG_MEMORY_MODEL STREQUAL "gc") - list(APPEND TSLANG_LINK_LIBS "gc") + if (NOT TSLANG_SHARED_GC) + list(APPEND TSLANG_LINK_LIBS "gc") + elseif(WIN32) + # gc.dll's import library. It has the same name as the static gc.lib in the package root, + # so it is named by its full path rather than found through the link directories. + list(APPEND TSLANG_LINK_LIBS "${CMAKE_TSLANG_DIR}/gcdll/gc.lib") + add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_TSLANG_DIR}/gcdll/gc.dll" "$") + else() + # the whole collector, exported, so a shared object's GC_* calls bind to this copy + list(APPEND TSLANG_LINK_LIBS "-Wl,--whole-archive" "gc" "-Wl,--no-whole-archive" "-Wl,--export-dynamic-symbol=GC_*") + endif() endif() # ntdll provides RtlGetLastNtStatus (pulled in by LLVMSupport) on Windows diff --git a/docs/how/cmake_tslang/README.md b/docs/how/cmake_tslang/README.md index 2c5a2abc1..1ebf0926f 100644 --- a/docs/how/cmake_tslang/README.md +++ b/docs/how/cmake_tslang/README.md @@ -56,6 +56,25 @@ It selects `defaultlib/lib//` as the link directory and ad to the compile flags, so the two cannot disagree. Valid values are `gc` (default), `rc` and `none`; only `gc` links Boehm. +## Programs that load tslang shared libraries + +A program that imports a tslang shared library (`import './lib'`) has to share one garbage +collector with it under `gc`. With two, each frees objects the other still holds. This template +does not build shared libraries itself: build them with `tslang --emit=dll`, which does this on its +own. Then configure the program with: + +``` +cmake --preset default -DTSLANG_SHARED_GC=ON +``` + +- Windows: links `gcdll/gc.lib` (gc.dll's import library) instead of the static `gc.lib`, and copies + `gc.dll` beside the program. Ship it with `TypeScriptDefaultLib.dll` + (`defaultlib/dll//gc`) and your libraries. +- Linux: links the whole static collector and exports its `GC_*` symbols, so the shared library + uses the program's collector. + +Leave it `OFF` for a program that loads none: it then ships as a single file. + ## Minimal alternative If you don't need a first-class language, either: diff --git a/docs/memory-models.md b/docs/memory-models.md index 66ec7d9c7..9849e22f3 100644 --- a/docs/memory-models.md +++ b/docs/memory-models.md @@ -146,6 +146,31 @@ From a source build, the shared collector comes from `scripts/build_gc_release_s Linking by hand (`--emit=obj` and your own linker) makes no choice for you: link the executable **and** every shared library against the shared `gc.lib`, and ship `gc.dll` beside the executable. +What to ship with a Windows program that loads a tslang shared library, all in one folder: + +- the program and its shared libraries, +- `gc.dll` (from `gcdll/` in the release package), +- `TypeScriptDefaultLib.dll` (from `defaultlib/dll//gc`), which every shared library + built with the default library imports. + +A shared library that was linked against the static `gc.lib` anyway — by an older `tslang`, or by +hand — is refused when you import it, under `--emit=exe` and `--emit=jit` alike: + +``` +error: shared library 'foo.dll' links its own garbage collector (the static gc.lib). Objects +crossing between it and this module can be freed while still in use. Rebuild it with tslang +--emit=dll, which links gc.dll. +``` + +The compiler reads this from the library itself, so it holds however the library was linked. The +JIT checks the default library's DLL the same way before loading it. + +On Linux nothing needs to ship. A program that imports a tslang shared object exports its own +collector (`--whole-archive` plus `--export-dynamic-symbol=GC_*`), and the dynamic loader binds +the shared object's `GC_*` calls to it, so the copy linked into the shared object is never used. +The default library's `.so` links no collector at all. Under the JIT, `libTypeScriptRuntime.so` +exports the collector the same way. If you link a program by hand, pass those two options. + Statically linked programs are unaffected and keep the static `gc.lib` — one binary already means one collector. `-mm=rc` and `-mm=none` are unaffected either way: neither has a collector. diff --git a/tslang/docs/single-gc-collector-design.md b/tslang/docs/single-gc-collector-design.md index 13db07231..9e46e59ff 100644 --- a/tslang/docs/single-gc-collector-design.md +++ b/tslang/docs/single-gc-collector-design.md @@ -1,7 +1,8 @@ # One collector per process: design proposal -Status: **PR 1 merged** (steps 1, 2, 5 - Windows); **PR 2** (step 3) on branch -`gc-shared-lib-auto`; steps 4, 6 and Linux still open. See [Progress](#progress) at the end. +Status: **PR 1 merged** (steps 1, 2, 5 - Windows); **PR 2** on branch `gc-shared-lib-auto`: step 3, +step 4 (by fingerprint, not a marker), step 6, the CMake template, the debug default library, and +Linux. Open: suite tests that keep the default library. See [Progress](#progress) at the end. ## Problem @@ -248,6 +249,59 @@ Merged as #309 (compiler) and TypeScriptCompilerDefaultLib #6. - The VS Code template's `--emit=dll` task passes `--gc-lib-path=`, so it resolves `/gcdll` without a change. -Still open: steps 4 and 6, the CMake project template (it links `gc` for every executable and does -not build shared libraries), tests that include the default library itself (the suite still passes -`--no-default-lib`), the debug default-lib build, and all of Linux. +### PR 2, continued - step 4, step 6, template, debug default library, Linux + +**Step 4 reads the library, not a marker.** The proposed `__tsgc__...` marker would +be written at compile time, and the compile does not know how the binary gets linked: `test-runner` +builds its shared libraries with `--emit=obj` and links them with lld, the default library's DLL is +built with `--embed-declarations=false` (so it carries no markers at all), and libraries from before +the change have none. Instead `Dump::containsGarbageCollector` (ObjDumper.cpp) looks for the string +`GC_INITIAL_HEAP_SIZE` in the binary's data sections. Boehm's `GC_init` reads that environment +variable, so the name is present in every binary that contains the collector and in none that only +calls it. Checked against: stale debug `TypeScriptDefaultLib.dll` (static gc) and `gc.dll` - present; +release `gc`/`rc` default-lib DLLs and `TypeScriptRuntime.dll` (both flavours) - absent. + +- `mlirGenImportSharedLib`: importing, under `-mm=gc` on Windows, a `gc` library that contains a + collector is an **error** (AOT and JIT). Because the fingerprint is evidence rather than a + missing marker, an old library gets the error too; it is broken, not merely old. +- `jit.cpp`: the same check on `TypeScriptDefaultLib.dll` before loading it - this is what a stale + default library looks like. +- Test: `test-compile-gc-shared-auto` case 6 builds a library against the static `gc.lib` (through a + stand-in shared directory) and expects both `--emit=exe` and `--emit=jit` importing it to fail + naming the collector. + +**Linux, measured** (WSL Ubuntu, release package v0.0-pre-alpha81, the gc_single_collector pair): + +| Case | Result | No-collection control | +| --- | --- | --- | +| exe + user `.so`, as linked today | assertion failed | 0 bad | +| JIT + user `.so` | 0 bad | 0 bad | +| exe linked with `--whole-archive libgc.a` + `--export-dynamic-symbol=GC_*` | 0 bad | 0 bad | + +The user `.so` does link its own `libgc.a` copy. Under the JIT it is harmless because +`libTypeScriptRuntime.so` exports `GC_*` (482 symbols) and ELF symbol interposition sends the `.so`'s +calls there. A plain exe exports none, so the `.so` ran its own collector. The default library's +`.so` links no collector (`GC_*` undefined) and already binds to the host. So Linux needs no +`gc.so` and no fingerprint check: `exe.cpp` now links an importing `--emit=exe` with the whole +collector exported. Whole archive because the `.so` may call `GC_*` functions the program does not +(without it 478 of 486 were exported, and the repro happened to pass). Not exercised on Linux beyond +the hand link: no Linux build of this branch was available locally, so the Linux CI run is its test. + +**Step 6.** The zip already ships `gc.dll` in its root and `gcdll/` (PR 1). `docs/memory-models.md` +now lists what to ship beside a program that loads a tslang DLL (`gc.dll`, +`TypeScriptDefaultLib.dll`), the step 4 error, and the Linux behaviour. + +**CMake template** (`docs/how/cmake_tslang`): `TSLANG_SHARED_GC` (default OFF). On Windows it links +`gcdll/gc.lib` by full path (same file name as the static one in the package root) and copies +`gc.dll` beside the target; on Linux it adds the two export options. The template still builds no +shared libraries itself - those come from `tslang --emit=dll`, which chooses on its own. + +**Debug default library.** `dll/debug/gc/TypeScriptDefaultLib.dll` was still the static-gc build from +before PR 1, and the local debug `TypeScriptRuntime.dll` predated it too. Rebuilt both: the debug +compiler (reconfigured, so it picks up `TSLANG_GC_SHARED_PREFIX`) and `scripts\build_vs.bat debug gc`, +which already linked `3rdParty/gcdll/x64/debug/lib`. Both now import `gc.dll`; no script change +needed. The release workflow builds the default library with release `GC_SHARED_LIB_PATH` for every +flavour; a debug DLL imports `gc.dll` by name and binds to the one already in the process. + +Still open: tests that include the default library itself (the suite still passes +`--no-default-lib`). diff --git a/tslang/include/TypeScript/ObjDumper.h b/tslang/include/TypeScript/ObjDumper.h index 934aef3c1..8c1e6c714 100644 --- a/tslang/include/TypeScript/ObjDumper.h +++ b/tslang/include/TypeScript/ObjDumper.h @@ -72,6 +72,11 @@ static std::unique_ptr createDumperT(const ELFObjectFile &elfObjec namespace Dump { void getSymbols(llvm::StringRef, SmallVector &, BumpPtrAllocator &); + + // Whether the binary carries a Boehm collector of its own (linked the static gc library), + // rather than importing one from gc.dll / its host or having none. Read from the binary itself, + // so it answers for libraries linked by hand or by an older tslang as well. + bool containsGarbageCollector(llvm::StringRef); } std::unique_ptr createCOFFDumper(const COFFObjectFile &); diff --git a/tslang/lib/TypeScript/MLIRGenModule.cpp b/tslang/lib/TypeScript/MLIRGenModule.cpp index 142398370..ae35b9fc4 100644 --- a/tslang/lib/TypeScript/MLIRGenModule.cpp +++ b/tslang/lib/TypeScript/MLIRGenModule.cpp @@ -970,6 +970,21 @@ namespace mlirgen << memoryModelName(compileOptions.memoryModel) << ". Objects crossing between them are never reclaimed."; } + + // One collector per process. A gc library that linked Boehm statically brings a second + // collector into this one, which cannot see this module's references and frees objects it + // still holds - silently, as wrong values rather than a crash. Windows only: an ELF + // program exports its collector (exe.cpp), so a shared object's own copy is never called. + // See docs/single-gc-collector-design.md, step 4. + if (compileOptions.isWindows && compileOptions.needsGCRuntime() && libraryModel == "gc" && + Dump::containsGarbageCollector(filePath)) + { + emitError(location) << "shared library '" << filePath + << "' links its own garbage collector (the static gc.lib). Objects crossing " + "between it and this module can be freed while still in use. Rebuild it " + "with tslang --emit=dll, which links gc.dll."; + return mlir::failure(); + } #else // only 1 file to load symbols.push_back(SHARED_LIB_DECLARATIONS_2UNDERSCORE); diff --git a/tslang/lib/TypeScript/ObjDumper.cpp b/tslang/lib/TypeScript/ObjDumper.cpp index eacaa71a9..db4a68973 100644 --- a/tslang/lib/TypeScript/ObjDumper.cpp +++ b/tslang/lib/TypeScript/ObjDumper.cpp @@ -89,4 +89,47 @@ void getSymbols(StringRef filePath, SmallVector &symbols, llvm::BumpP } } +bool containsGarbageCollector(StringRef filePath) +{ + // Boehm's GC_init reads this environment variable, so its name is a string constant in every + // binary that contains the collector, in every build flavour, stripped or not - and in none + // that only calls it. Its internal symbols are no help: a DLL does not export them. + static const StringRef fingerprint = "GC_INITIAL_HEAP_SIZE"; + + auto expectedOwningBinary = createBinary(filePath); + if (!expectedOwningBinary) + { + consumeError(expectedOwningBinary.takeError()); + return false; + } + + auto *objFile = dyn_cast(expectedOwningBinary.get().getBinary()); + if (!objFile) + { + return false; + } + + for (const SectionRef §ion : objFile->sections()) + { + if (!section.isData() || section.isBSS()) + { + continue; + } + + auto contents = section.getContents(); + if (!contents) + { + consumeError(contents.takeError()); + continue; + } + + if (contents->contains(fingerprint)) + { + return true; + } + } + + return false; +} + } \ No newline at end of file diff --git a/tslang/test/tester/shared-collector-auto.cmake b/tslang/test/tester/shared-collector-auto.cmake index fbf4a6c7a..d823f2a26 100644 --- a/tslang/test/tester/shared-collector-auto.cmake +++ b/tslang/test/tester/shared-collector-auto.cmake @@ -93,4 +93,29 @@ if(NOT run_output MATCHES "gc\\.dll") message(FATAL_ERROR "--emit=dll failed without naming gc.dll:\n${run_output}") endif() -message(STATUS "tslang picks the shared collector for a library and its importer, and the static one for a lone program") +# 6. Importing a library that did link a collector of its own is a compile-time error, AOT and JIT. +# Such a library is what an older tslang, or a hand-made link, produced. To make one here, the +# shared path is pointed at the STATIC gc.lib, with a stand-in gc.dll only so tslang accepts +# the directory - nothing loads it, since the library then imports nothing from gc.dll. +file(MAKE_DIRECTORY "${WORK_DIR}/own/fake-gcdll") +file(COPY "${GC_LIB}/gc.lib" DESTINATION "${WORK_DIR}/own/fake-gcdll") +file(WRITE "${WORK_DIR}/own/fake-gcdll/gc.dll" "") +run("--emit=dll against the static gc.lib" "${WORK_DIR}/own" TRUE + "${TSLANG}" --emit=dll ${common} "--gc-shared-lib-path=${WORK_DIR}/own/fake-gcdll" + "${library}" -o export_gc_single_collector.dll) +file(REMOVE "${WORK_DIR}/own/gc.dll") + +run("--emit=exe importing a library with its own collector" "${WORK_DIR}/own" FALSE + "${TSLANG}" --emit=exe ${common} "--gc-lib-path=${GC_LIB}" "--gc-shared-lib-path=${GC_SHARED_LIB}" + "${program}" -o main.exe) +if(NOT run_output MATCHES "links its own garbage collector") + message(FATAL_ERROR "importing a library with its own collector failed for another reason:\n${run_output}") +endif() + +run("--emit=jit importing a library with its own collector" "${WORK_DIR}/own" FALSE + "${TSLANG}" --emit=jit --no-default-lib ${OPT} "${program}") +if(NOT run_output MATCHES "links its own garbage collector") + message(FATAL_ERROR "the JIT import of a library with its own collector failed for another reason:\n${run_output}") +endif() + +message(STATUS "tslang picks the shared collector for a library and its importer, the static one for a lone program, and refuses a library with a collector of its own") diff --git a/tslang/tslang/exe.cpp b/tslang/tslang/exe.cpp index 91e51155c..a6ed5585d 100644 --- a/tslang/tslang/exe.cpp +++ b/tslang/tslang/exe.cpp @@ -584,9 +584,26 @@ int buildExe(int argc, char **argv, std::string objFileName, std::string additio } // tslang libs + // ELF: a program that loads a tslang shared object exports its whole collector, so the shared + // object's calls to GC_* bind to it at load time instead of to the static copy linked into the + // shared object, and the process runs one collector. Measured: without the export, strings the + // shared object built were freed while the program held them. The JIT needs nothing here, as + // libTypeScriptRuntime.so already exports GC_*. See docs/single-gc-collector-design.md. + auto exportGC = !win && !wasm && !shared && compileOptions.needsGCRuntime() && compileOptions.importsSharedLibrary; if (compileOptions.needsGCRuntime()) - { - args.push_back("-lgc"); + { + if (exportGC) + { + // whole archive: the shared object may call GC_* functions this program never does + args.push_back("-Wl,--whole-archive"); + args.push_back("-lgc"); + args.push_back("-Wl,--no-whole-archive"); + args.push_back("-Wl,--export-dynamic-symbol=GC_*"); + } + else + { + args.push_back("-lgc"); + } } if (isTslangLibNeeded) diff --git a/tslang/tslang/jit.cpp b/tslang/tslang/jit.cpp index b9d34704b..3bc0fbbb9 100644 --- a/tslang/tslang/jit.cpp +++ b/tslang/tslang/jit.cpp @@ -37,6 +37,13 @@ #include "TypeScript/Defines.h" +// From TypeScript/ObjDumper.h, which cannot be included here: its llvm/BinaryFormat/COFF.h +// collides with the IMAGE_* macros defines above. +namespace Dump +{ + bool containsGarbageCollector(llvm::StringRef); +} + #define DEBUG_TYPE "tslang" namespace cl = llvm::cl; @@ -387,6 +394,20 @@ int runJit(int argc, char **argv, mlir::ModuleOp module, CompileOptions &compile return -1; } +#ifdef WIN32 + // A default library built before it linked gc.dll carries a collector of its own, next to + // the one TypeScriptRuntime.dll takes from gc.dll, and frees the strings it hands this + // program. See docs/single-gc-collector-design.md. + if (compileOptions.needsGCRuntime() && Dump::containsGarbageCollector(defaultLibFile)) + { + llvm::WithColor::error(llvm::errs(), "tslang") + << defaultLibFile << " links its own garbage collector (the static gc.lib), so this " + << "process would have two and one frees what the other holds. Rebuild the default " + << "library: its DLL has to link gc.dll.\n"; + return -1; + } +#endif + clSharedLibs.push_back(defaultLibFile); } From cf5ca7ab9052572bef2658060452fcbd66e515ce Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 13 Sep 2026 11:28:24 +0100 Subject: [PATCH 4/4] Add tests and workflows for default library integration with garbage collector --- .github/workflows/create-release.yml | 22 +++ tag.bat | 2 +- tag_del.bat | 4 +- tslang/docs/single-gc-collector-design.md | 43 +++++- tslang/test/tester/CMakeLists.txt | 34 +++++ tslang/test/tester/defaultlib-collector.cmake | 134 ++++++++++++++++++ .../tester/defaultlib/defaultlib_collector.ts | 36 +++++ .../defaultlib/export_defaultlib_collector.ts | 22 +++ .../defaultlib/import_defaultlib_collector.ts | 26 ++++ tslang/tslang/exe.cpp | 25 ++++ 10 files changed, 343 insertions(+), 5 deletions(-) create mode 100644 tslang/test/tester/defaultlib-collector.cmake create mode 100644 tslang/test/tester/defaultlib/defaultlib_collector.ts create mode 100644 tslang/test/tester/defaultlib/export_defaultlib_collector.ts create mode 100644 tslang/test/tester/defaultlib/import_defaultlib_collector.ts diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 963124d95..7dad22d95 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -186,6 +186,18 @@ jobs: .\build.bat shell: pwsh + # The compiler suite runs before the default library exists, so its default-library tests + # skip there; run them now against the library just built. One collector per process with + # the default library in it: tslang/test/tester/defaultlib-collector.cmake. + - name: Test Compiler with Default Library + continue-on-error: false + working-directory: ${{github.workspace}}/__build/tslang/msbuild/x64/release + env: + DEFAULT_LIB_PATH: ${{github.workspace}}/TypeScriptCompilerDefaultLib/__build + TSLANG_REQUIRE_DEFAULT_LIB: 1 + run: ctest -C ${{ env.BUILD_TYPE }} -R gc-defaultlib-collector --output-on-failure + shell: pwsh + - name: Test Default Library continue-on-error: false working-directory: ${{github.workspace}}/TypeScriptCompilerDefaultLib @@ -379,6 +391,16 @@ jobs: TSLANG_LIB_PATH: ${{github.workspace}}/__build/tslang/ninja/release/lib run: chmod +x ./build.sh ./scripts/*.sh; ./build.sh + # See the Windows job: the default-library tests skip in the compiler suite and run here. + - name: Test Compiler with Default Library + continue-on-error: false + working-directory: ${{github.workspace}}/__build/tslang/ninja/release + shell: sh + env: + DEFAULT_LIB_PATH: ${{github.workspace}}/TypeScriptCompilerDefaultLib/__build + TSLANG_REQUIRE_DEFAULT_LIB: 1 + run: ctest -C ${{ env.BUILD_TYPE }} -R gc-defaultlib-collector --output-on-failure + - name: Test Default Library continue-on-error: false working-directory: ${{github.workspace}}/TypeScriptCompilerDefaultLib diff --git a/tag.bat b/tag.bat index 497c9a7f7..33d76ce01 100644 --- a/tag.bat +++ b/tag.bat @@ -1,2 +1,2 @@ -git tag -a v0.0-pre-alpha81 -m "pre alpha v0.0-81" +git tag -a v0.0-pre-alpha82 -m "pre alpha v0.0-82" git push origin --tags diff --git a/tag_del.bat b/tag_del.bat index aa2f44e08..90df3aba1 100644 --- a/tag_del.bat +++ b/tag_del.bat @@ -1,2 +1,2 @@ -git push --delete origin v0.0-pre-alpha81 -git tag -d v0.0-pre-alpha81 +git push --delete origin v0.0-pre-alpha82 +git tag -d v0.0-pre-alpha82 diff --git a/tslang/docs/single-gc-collector-design.md b/tslang/docs/single-gc-collector-design.md index 9e46e59ff..b67943f69 100644 --- a/tslang/docs/single-gc-collector-design.md +++ b/tslang/docs/single-gc-collector-design.md @@ -303,5 +303,44 @@ which already linked `3rdParty/gcdll/x64/debug/lib`. Both now import `gc.dll`; n needed. The release workflow builds the default library with release `GC_SHARED_LIB_PATH` for every flavour; a debug DLL imports `gc.dll` by name and binds to the one already in the process. -Still open: tests that include the default library itself (the suite still passes -`--no-default-lib`). +### Tests with the default library in the process (step 5, completed) + +`test-jit-gc-defaultlib-collector` and `test-compile-gc-defaultlib-collector` +(`test/tester/defaultlib-collector.cmake`, sources in `test/tester/defaultlib/`) drive `tslang` with +the real default library, no `--no-default-lib`: + +- JIT: the program alone (strings built by `padStart`, churn by `repeat`, both in the default-lib + DLL), and the program importing a user shared library. +- AOT: the exe alone (static default library), and an exe importing a user DLL that links + `TypeScriptDefaultLib.dll`, with the library building the held strings and churning. + +The sources are outside `tests/`: they do not compile without the default library, and the ownership +verifier compiles every file in `tests/` alone with `--no-default-lib`. + +**CI.** The compiler suite runs before any default library exists, so with none found the tests print +`SKIPPED` (`SKIP_REGULAR_EXPRESSION`) instead of failing. The library is looked up in +`DEFAULT_LIB_PATH` first, then in the `TypeScriptCompilerDefaultLib/__build` sibling (local layout) or +child (CI workspace) of the repository. Both jobs of `create-release.yml` gained a step after +**Build Default Library** that runs `ctest -R gc-defaultlib-collector` with `DEFAULT_LIB_PATH` at the +library just built and `TSLANG_REQUIRE_DEFAULT_LIB=1`, which turns a missing library into a failure. +The Linux variant has not run anywhere yet. + +**A gap the tests found.** Step 4 checks the library being imported, but a user DLL links +`TypeScriptDefaultLib.dll` without importing it by name. `exe.cpp` now refuses `--emit=dll` under `gc` +on Windows when the default-lib DLL it links contains a collector. + +**Teeth** (release, a default library built by its own script with `GC_SHARED_LIB_PATH` at the static +`gc.lib`, so its DLL carries a collector): + +| Case | Result | +| --- | --- | +| both tests against the rebuilt default library | pass, `bad: 0` in all four programs | +| both tests against the static-collector library | fail: `--emit=dll` refuses the default-lib DLL | +| exe + user DLL built correctly, stale DLL swapped in beside it at run time | assertion failed | +| the same, collection suppressed (`GC_INITIAL_HEAP_SIZE=1GB`) | `bad: 0` | + +A machine whose `DEFAULT_LIB_PATH` names an installation with a default library from before PR 1 fails +these two tests - correctly: `--emit=dll` and JIT runs against that library now fail the same way. + +Nothing from this design is open on Windows. On Linux, the `exe.cpp` export change and the +default-library tests have their first run in CI. diff --git a/tslang/test/tester/CMakeLists.txt b/tslang/test/tester/CMakeLists.txt index fd8d99a2b..cbf5913eb 100644 --- a/tslang/test/tester/CMakeLists.txt +++ b/tslang/test/tester/CMakeLists.txt @@ -1901,6 +1901,40 @@ if (WIN32) -P "${CMAKE_CURRENT_SOURCE_DIR}/shared-collector-auto.cmake") endif() +# One collector per process with the real default library in it - see defaultlib-collector.cmake. +# Skipped (not failed) when no default library has been built; the release workflows run them after +# building one. The sources live outside tests/: without the default library they do not compile, +# and every file in tests/ is compiled alone with --no-default-lib by the ownership verifier. +if ("${CMAKE_BUILD_TYPE}" STREQUAL "Release") + set(defaultlib_collector_opt "--opt;--opt_level=3") +else() + set(defaultlib_collector_opt "--di;--opt_level=0") +endif() +if (WIN32) + set(defaultlib_collector_llvm_lib "${PROJECT_SOURCE_DIR}/../3rdParty/llvm/x64/${CMAKE_BUILD_TYPE_LOWERCASE}/lib") + set(defaultlib_collector_gc_shared "-DGC_SHARED_LIB=${TEST_GC_SHARED_PREFIX}/lib") +else() + set(defaultlib_collector_llvm_lib "${PROJECT_SOURCE_DIR}/../3rdParty/llvm/${CMAKE_BUILD_TYPE_LOWERCASE}/lib") + set(defaultlib_collector_gc_shared "") +endif() +foreach(defaultlib_collector_mode jit compile) + add_test(NAME test-${defaultlib_collector_mode}-gc-defaultlib-collector + COMMAND ${CMAKE_COMMAND} + "-DMODE=${defaultlib_collector_mode}" + "-DTSLANG=$" + "-DTSLANG_BIN=${CMAKE_BINARY_DIR}/bin" + "-DTESTS_DIR=${CMAKE_CURRENT_SOURCE_DIR}/defaultlib" + "-DWORK_DIR=${CMAKE_CURRENT_BINARY_DIR}/gc-defaultlib-collector-${defaultlib_collector_mode}" + "-DGC_LIB=${TEST_GC_LIBDIR}" + ${defaultlib_collector_gc_shared} + "-DLLVM_LIB=${defaultlib_collector_llvm_lib}" + "-DTSLANG_LIB=${CMAKE_BINARY_DIR}/lib" + "-DOPT=${defaultlib_collector_opt}" + "-DDEFAULT_LIB_CANDIDATES=${PROJECT_SOURCE_DIR}/../TypeScriptCompilerDefaultLib/__build;${PROJECT_SOURCE_DIR}/../../TypeScriptCompilerDefaultLib/__build" + -P "${CMAKE_CURRENT_SOURCE_DIR}/defaultlib-collector.cmake") + set_tests_properties(test-${defaultlib_collector_mode}-gc-defaultlib-collector PROPERTIES SKIP_REGULAR_EXPRESSION "SKIPPED:") +endforeach() + set(TSLANG_OWNERSHIP_SHARDS 8) math(EXPR ownership_last "${TSLANG_OWNERSHIP_SHARDS} - 1") foreach(ownership_shard RANGE ${ownership_last}) diff --git a/tslang/test/tester/defaultlib-collector.cmake b/tslang/test/tester/defaultlib-collector.cmake new file mode 100644 index 000000000..adacb38c1 --- /dev/null +++ b/tslang/test/tester/defaultlib-collector.cmake @@ -0,0 +1,134 @@ +# One garbage collector per process, with the default library in the process (-mm=gc). +# +# Every test-runner test passes --no-default-lib, so the binary that is in almost every real +# process - the default library - was never part of one. A default library whose DLL links its own +# collector frees the strings it hands out: 2000 of 2000 under the JIT, 1984 of 2000 for an exe with +# a user shared library (docs/single-gc-collector-design.md). This drives tslang with the real +# default library instead: +# +# MODE=jit the JIT, the program alone and importing a shared library +# MODE=compile an exe alone, and an exe importing a shared library that links the default-library DLL +# +# The default library is a separate build (TypeScriptCompilerDefaultLib). With none found the test +# prints SKIPPED and CTest reports it skipped; the release workflows run it after building one. + +cmake_minimum_required(VERSION 3.17.3) + +foreach(var MODE TSLANG TSLANG_BIN TESTS_DIR WORK_DIR GC_LIB LLVM_LIB TSLANG_LIB OPT DEFAULT_LIB_CANDIDATES) + if(NOT DEFINED ${var}) + message(FATAL_ERROR "${var} is required") + endif() +endforeach() + +if(WIN32) + set(exe_suffix ".exe") + set(dll_prefix "") + set(dll_suffix ".dll") + set(runtime "${TSLANG_BIN}/TypeScriptRuntime.dll") + set(pic "") +else() + set(exe_suffix "") + set(dll_prefix "lib") + set(dll_suffix ".so") + set(runtime "${TSLANG_LIB}/libTypeScriptRuntime.so") + set(pic "-relocation-model=pic") +endif() + +if("--di" IN_LIST OPT) + set(build "debug") +else() + set(build "release") +endif() + +# DEFAULT_LIB_PATH at run time wins, so a workflow can point at the library it just built +set(default_lib "") +foreach(candidate "$ENV{DEFAULT_LIB_PATH}" ${DEFAULT_LIB_CANDIDATES}) + if(NOT candidate STREQUAL "" AND IS_DIRECTORY "${candidate}/defaultlib/dll/${build}/gc") + set(default_lib "${candidate}") + break() + endif() +endforeach() + +if(default_lib STREQUAL "") + # a workflow that has just built the library sets this, so a wrong path fails instead of skipping + if("$ENV{TSLANG_REQUIRE_DEFAULT_LIB}" STREQUAL "1") + message(FATAL_ERROR "no default library built for ${build}/gc under DEFAULT_LIB_PATH='$ENV{DEFAULT_LIB_PATH}'") + endif() + message("SKIPPED: no default library built for ${build}/gc (set DEFAULT_LIB_PATH to the folder holding 'defaultlib')") + return() +endif() + +set(default_lib_dll_dir "${default_lib}/defaultlib/dll/${build}/gc") +message(STATUS "default library: ${default_lib}") + +set(ENV{GC_SHARED_LIB_PATH} "") +set(ENV{GC_LIB_PATH} "") +set(ENV{DEFAULT_LIB_PATH} "") + +file(REMOVE_RECURSE "${WORK_DIR}") +file(MAKE_DIRECTORY "${WORK_DIR}/alone") + +set(gc_opts "--gc-lib-path=${GC_LIB}") +if(DEFINED GC_SHARED_LIB) + list(APPEND gc_opts "--gc-shared-lib-path=${GC_SHARED_LIB}") +endif() + +set(common ${OPT} ${pic} "--default-lib-path=${default_lib}" "--llvm-lib-path=${LLVM_LIB}" "--tslang-lib-path=${TSLANG_LIB}" ${gc_opts}) + +# the loader has to find the default-library shared object (Linux) and gc.dll / the library (both) +if(WIN32) + set(run_env "PATH=${WORK_DIR};$ENV{PATH}") +else() + set(run_env "LD_LIBRARY_PATH=${WORK_DIR}:${default_lib_dll_dir}:$ENV{LD_LIBRARY_PATH}") +endif() + +# run( ) - fails unless it exits 0 and prints "bad: 0" and "done." +function(run what dir) + execute_process(COMMAND ${CMAKE_COMMAND} -E env "${run_env}" ${ARGN} + WORKING_DIRECTORY "${dir}" + OUTPUT_VARIABLE out + ERROR_VARIABLE err + RESULT_VARIABLE status) + if(NOT status EQUAL 0) + message(FATAL_ERROR "${what}: exit ${status}\n${out}\n${err}") + endif() +endfunction() + +function(run_program what dir) + execute_process(COMMAND ${CMAKE_COMMAND} -E env "${run_env}" ${ARGN} + WORKING_DIRECTORY "${dir}" + OUTPUT_VARIABLE out + ERROR_VARIABLE err + RESULT_VARIABLE status) + if(NOT status EQUAL 0 OR NOT out MATCHES "bad: 0" OR NOT out MATCHES "done\\.") + message(FATAL_ERROR "${what}: exit ${status}\n${out}\n${err}") + endif() + message(STATUS "${what}: bad: 0") +endfunction() + +set(alone "${TESTS_DIR}/defaultlib_collector.ts") +set(library "${TESTS_DIR}/export_defaultlib_collector.ts") +set(program "${TESTS_DIR}/import_defaultlib_collector.ts") +set(library_file "${dll_prefix}export_defaultlib_collector${dll_suffix}") + +# The shared library links the default-library DLL; both modes import it. +run("--emit=dll with the default library" "${WORK_DIR}" + "${TSLANG}" --emit=dll ${common} "${library}" -o "${library_file}") +file(COPY "${default_lib_dll_dir}/${dll_prefix}TypeScriptDefaultLib${dll_suffix}" DESTINATION "${WORK_DIR}") + +if(MODE STREQUAL "jit") + run_program("JIT, default library" "${WORK_DIR}/alone" + "${TSLANG}" --emit=jit ${common} "--shared-libs=${runtime}" "${alone}") + run_program("JIT importing a shared library, default library" "${WORK_DIR}" + "${TSLANG}" --emit=jit ${common} "--shared-libs=${runtime}" "${program}") +elseif(MODE STREQUAL "compile") + run("--emit=exe alone, default library" "${WORK_DIR}/alone" + "${TSLANG}" --emit=exe ${common} "${alone}" -o "alone${exe_suffix}") + run_program("exe, default library" "${WORK_DIR}/alone" "${WORK_DIR}/alone/alone${exe_suffix}") + + run("--emit=exe importing a shared library, default library" "${WORK_DIR}" + "${TSLANG}" --emit=exe ${common} "${program}" -o "main${exe_suffix}") + run_program("exe importing a shared library, default library" "${WORK_DIR}" "${WORK_DIR}/main${exe_suffix}") +else() + message(FATAL_ERROR "MODE must be jit or compile, not '${MODE}'") +endif() diff --git a/tslang/test/tester/defaultlib/defaultlib_collector.ts b/tslang/test/tester/defaultlib/defaultlib_collector.ts new file mode 100644 index 000000000..8b43d6dc7 --- /dev/null +++ b/tslang/test/tester/defaultlib/defaultlib_collector.ts @@ -0,0 +1,36 @@ +// One collector per process, with the default library in it. The strings this program holds are +// built by the default library (padStart), and the churn is too (repeat), so under the JIT they +// are allocated by TypeScriptDefaultLib.dll while the array holding them belongs to the JIT'd +// code. If that DLL carries a collector of its own it cannot see the array, and frees them. +// The churn differs from what is held, and the expectation is built with the same method. +// See tslang/docs/single-gc-collector-design.md. + +function makeKey(i: number): string { + return `${i}`.padStart(12, "k"); +} + +function main() { + const N = 2000; + + let held: string[] = []; + for (let i = 0; i < N; i++) { + held.push(makeKey(i)); + } + + let total = 0; + for (let j = 0; j < 300000; j++) { + let s = "z".repeat(40 + (j % 7)); + total = total + s.length; + } + + let bad = 0; + for (let i = 0; i < N; i++) { + if (held[i] != makeKey(i)) bad = bad + 1; + } + + print("bad:", bad); + assert(bad == 0, "strings built by the default library were freed while still held"); + assert(total > 0, "churn"); + + print("done."); +} diff --git a/tslang/test/tester/defaultlib/export_defaultlib_collector.ts b/tslang/test/tester/defaultlib/export_defaultlib_collector.ts new file mode 100644 index 000000000..cb845068b --- /dev/null +++ b/tslang/test/tester/defaultlib/export_defaultlib_collector.ts @@ -0,0 +1,22 @@ +namespace D { + + // The library side of import_defaultlib_collector.ts. It links the default library as a DLL, + // so the held strings and the churn are allocated by TypeScriptDefaultLib.dll, while the + // program that holds them links the default library statically. One collector only if that + // DLL takes it from the same gc.dll. See tslang/docs/single-gc-collector-design.md. + + export function makeKey(i: number): string { + return `${i}`.padStart(12, "k"); + } + + export function churn(n: number): number { + let total = 0; + for (let j = 0; j < n; j++) { + // `let`: a `const` in this loop fails to resolve when compiled as an imported module + let s = "z".repeat(40 + (j % 7)); + total = total + s.length; + } + + return total; + } +} diff --git a/tslang/test/tester/defaultlib/import_defaultlib_collector.ts b/tslang/test/tester/defaultlib/import_defaultlib_collector.ts new file mode 100644 index 000000000..19ff0a33a --- /dev/null +++ b/tslang/test/tester/defaultlib/import_defaultlib_collector.ts @@ -0,0 +1,26 @@ +import './export_defaultlib_collector' + +// Strings built by the library through the default-library DLL, held only by this program's +// array, then churned by the library with different content. See export_defaultlib_collector.ts. + +function main() { + const N = 2000; + + let held: string[] = []; + for (let i = 0; i < N; i++) { + held.push(D.makeKey(i)); + } + + const total = D.churn(300000); + + let bad = 0; + for (let i = 0; i < N; i++) { + if (held[i] != `${i}`.padStart(12, "k")) bad = bad + 1; + } + + print("bad:", bad); + assert(bad == 0, "strings built by the default-library DLL were freed while still held"); + assert(total > 0, "churn"); + + print("done."); +} diff --git a/tslang/tslang/exe.cpp b/tslang/tslang/exe.cpp index a6ed5585d..27dfc06c0 100644 --- a/tslang/tslang/exe.cpp +++ b/tslang/tslang/exe.cpp @@ -30,6 +30,13 @@ extern cl::opt TargetTriple; extern cl::opt defaultlibpath; extern cl::opt gclibpath; extern cl::opt gcsharedlibpath; + +// From TypeScript/ObjDumper.h, declared here for the same reason jit.cpp does: that header's +// llvm/BinaryFormat/COFF.h collides with macros. +namespace Dump +{ + bool containsGarbageCollector(llvm::StringRef); +} extern cl::opt llvmlibpath; extern cl::opt tslanglibpath; extern cl::opt emsdksysrootpath; @@ -495,6 +502,24 @@ int buildExe(int argc, char **argv, std::string objFileName, std::string additio return 1; } + // A shared library links the default library's DLL and shares a process with it. If that + // DLL predates linking gc.dll it brings a second collector, which frees what the library + // and its host hold - the case step 4's import check cannot see, since the library does + // not import it by name. See docs/single-gc-collector-design.md. + if (win && shared && compileOptions.needsGCRuntime() && !defaultLibDir.empty()) + { + llvm::SmallString<256> defaultLibDll(defaultLibDir); + llvm::sys::path::append(defaultLibDll, DEFAULT_LIB_NAME ".dll"); + if (llvm::sys::fs::exists(defaultLibDll) && Dump::containsGarbageCollector(defaultLibDll)) + { + llvm::WithColor::error(llvm::errs(), "tslang") + << defaultLibDll << " links its own garbage collector (the static gc.lib), so a shared library " + << "linked with it would run two collectors in one process. Rebuild the default library: its " + << "DLL has to link gc.dll.\n"; + return 1; + } + } + defaultLibPathOpt = getLibsPathOpt(defaultLibDir); if (!defaultLibPathOpt.empty()) {