feat: extend runtime memory admission to compile edges

Author: ZheFeng7110

docs/04-mcpp-toml.md

@@ -937,6 +937,19 @@ with two behaviours has to be.
 
 `bmi_schedule` decides when importers are unblocked.
 
+`memory_threshold_percent` is a machine setting in `~/.mcpp/config.toml`, not a
+project-manifest field. It defaults to `80` and limits the compiler pool using
+the available memory observed at build start. `MCPP_MEMORY_THRESHOLD_PERCENT`
+overrides it for one invocation. Ordinary C/C++ edges and both detached GCC and
+two-phase Clang module edges use the same admission check; unsupported memory
+metrics leave the configured job/compiler cap in force.
+
+```toml
+# ~/.mcpp/config.toml
+[build]
+memory_threshold_percent = 80
+```
+
 | value | |
 |---|---|
 | `"auto"` | **the default, and it currently means OFF** |

docs/zh/04-mcpp-toml.md

@@ -878,6 +878,18 @@ bmi_schedule = "off"     # auto (default, = off) | on | off
 
 `bmi_schedule` 决定 importer 何时被解除阻塞。
 
+`memory_threshold_percent` 是 `~/.mcpp/config.toml` 中的本机设置，不是
+工程 manifest 字段。默认值为 `80`，它依据构建开始时观测到的可用内存限制
+编译器池；单次调用可以用 `MCPP_MEMORY_THRESHOLD_PERCENT` 覆盖。普通
+C/C++ 编译边、GCC 分离式模块边和 Clang 两阶段模块边使用同一条准入检查；
+不受支持的内存指标不会取消已经配置的 jobs/编译器上限。
+
+```toml
+# ~/.mcpp/config.toml
+[build]
+memory_threshold_percent = 80
+```
+
 | 取值 | |
 |---|---|
 | `"auto"` | **默认值，目前意味着关闭** |

src/build/ninja_backend.cppm

@@ -1839,6 +1839,21 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements,
         append(std::format("  rspfile_content ={}{}\n",
                            msvcDeps ? " " + std::string(kUtf8ByteOrderMark) : "", payload));
     };
+    // Keep ordinary compile edges inside the same admission controller as the
+    // detached GCC path. The compiler command remains argv-based; only the
+    // existing shell-free depfile filter is moved into the wrapper.
+    const bool memoryAdmission = plan.memoryCompilerCap > 0
+                              && plan.memoryBaselineBytes != 0;
+    auto admission_prefix = [&]() {
+        return std::format("$mcpp compile-admit --sem .mcpp-sched --cap {}"
+                           " --mem-baseline {} --mem-threshold {}",
+                           plan.memoryCompilerCap, plan.memoryBaselineBytes,
+                           plan.memoryThresholdPercent);
+    };
+    auto admission_dep = [&]() {
+        return needsGnuModuleFilter
+            ? std::string(" --dep-raw $out.d.raw --dep-out $out.d") : std::string{};
+    };
 
     // Tell the driver, every time, that this TU is a module interface.
     //
@@ -1957,26 +1972,39 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements,
         append("rule cxx_precompile\n");
         if constexpr (mcpp::platform::is_windows) {
             const std::string payload = " $local_includes";
-            append(std::format(
-                "  command = {}$cxx{} $cxxflags $unit_cxxflags{}{} {}$in {}$out\n",
-                win_depfile_filter, rsp_ref(payload), traits.bmiOnlyFlags,
-                module_src_flags, mmd_flag, dial.outputObjPrefix));
+            if (memoryAdmission)
+                append(std::format(
+                    "  command = {} -- $cxx{} $cxxflags $unit_cxxflags{}{} {}$in {}$out\n",
+                    admission_prefix(), rsp_ref(payload), traits.bmiOnlyFlags,
+                    module_src_flags, mmd_flag, dial.outputObjPrefix));
+            else
+                append(std::format(
+                    "  command = {}$cxx{} $cxxflags $unit_cxxflags{}{} {}$in {}$out\n",
+                    win_depfile_filter, rsp_ref(payload), traits.bmiOnlyFlags,
+                    module_src_flags, mmd_flag, dial.outputObjPrefix));
             append_rspfile(payload);
             append_cxx_deps();
         } else {
             // Same bak / bmi-equal / restore dance as cxx_module, and for the
             // same reason: ninja's `restat` compares the output's MTIME, and a
             // compiler that rewrites a byte-identical BMI still moves it. What
             // suppresses the cascade is putting the old file back.
-            append(std::format(
-                "  command = "
-                "if [ -f \"$out\" ]; then cp -p \"$out\" \"$out.bak\"; fi && "
-                "$cxx $local_includes $cxxflags $unit_cxxflags{}{} {}$in {}$out && "
-                "if [ -f \"$out.bak\" ] && $mcpp bmi-equal \"$out\" \"$out.bak\"; then "
-                  "mv \"$out.bak\" \"$out\"; "
-                "else rm -f \"$out.bak\"; fi\n",
-                traits.bmiOnlyFlags, module_src_flags, mmd_flag,
-                dial.outputObjPrefix));
+            if (memoryAdmission)
+                append(std::format("  command = {}{} -- $cxx $local_includes"
+                                   " $cxxflags $unit_cxxflags{}{} {}$in {}$out\n",
+                                   admission_prefix(), admission_dep(),
+                                   traits.bmiOnlyFlags, module_src_flags, mmd_flag,
+                                   dial.outputObjPrefix));
+            else
+                append(std::format(
+                    "  command = "
+                    "if [ -f \"$out\" ]; then cp -p \"$out\" \"$out.bak\"; fi && "
+                    "$cxx $local_includes $cxxflags $unit_cxxflags{}{} {}$in {}$out && "
+                    "if [ -f \"$out.bak\" ] && $mcpp bmi-equal \"$out\" \"$out.bak\"; then "
+                      "mv \"$out.bak\" \"$out\"; "
+                    "else rm -f \"$out.bak\"; fi\n",
+                    traits.bmiOnlyFlags, module_src_flags, mmd_flag,
+                    dial.outputObjPrefix));
             append_cxx_deps();
         }
         append("  description = BMI $out\n");
@@ -1989,15 +2017,26 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements,
         append("rule cxx_module_object\n");
         if constexpr (mcpp::platform::is_windows) {
             const std::string payload = " $local_includes";
-            append(std::format("  command = {}$cxx{} $cxxflags $unit_cxxflags{} {}{}\n",
-                               win_depfile_filter, rsp_ref(payload), module_src_flags,
-                               mmd_flag, compile_tail));
+            if (memoryAdmission)
+                append(std::format("  command = {}{} -- $cxx{} $cxxflags $unit_cxxflags{} {}{}\n",
+                                   admission_prefix(), admission_dep(), rsp_ref(payload),
+                                   module_src_flags, mmd_flag, compile_tail));
+            else
+                append(std::format("  command = {}$cxx{} $cxxflags $unit_cxxflags{} {}{}\n",
+                                   win_depfile_filter, rsp_ref(payload), module_src_flags,
+                                   mmd_flag, compile_tail));
             append_rspfile(payload);
             append_cxx_deps();
         } else {
-            append(std::format(
-                "  command = $cxx $local_includes $cxxflags $unit_cxxflags{} {}{}{}\n",
-                module_src_flags, mmd_flag, compile_tail, mmd_filter));
+            if (memoryAdmission)
+                append(std::format("  command = {}{} -- $cxx $local_includes"
+                                   " $cxxflags $unit_cxxflags{} {}{}\n",
+                                   admission_prefix(), admission_dep(),
+                                   module_src_flags, mmd_flag, compile_tail));
+            else
+                append(std::format(
+                    "  command = $cxx $local_includes $cxxflags $unit_cxxflags{} {}{}{}\n",
+                    module_src_flags, mmd_flag, compile_tail, mmd_filter));
             append_cxx_deps();
         }
         append("  description = OBJ $out\n");
@@ -2007,14 +2046,24 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements,
     append("rule cxx_object\n");
     if constexpr (mcpp::platform::is_windows) {
         const std::string payload = " $local_includes";
-        append(std::format("  command = {}$cxx{} $cxxflags $unit_cxxflags {}{}\n",
-                           win_depfile_filter, rsp_ref(payload), mmd_flag,
-                           compile_tail));
+        if (memoryAdmission)
+            append(std::format("  command = {}{} -- $cxx{} $cxxflags $unit_cxxflags {}{}\n",
+                               admission_prefix(), admission_dep(), rsp_ref(payload),
+                               mmd_flag, compile_tail));
+        else
+            append(std::format("  command = {}$cxx{} $cxxflags $unit_cxxflags {}{}\n",
+                               win_depfile_filter, rsp_ref(payload), mmd_flag, compile_tail));
         append_rspfile(payload);
     } else {
-        append(std::format(
-            "  command = $cxx $local_includes $cxxflags $unit_cxxflags {}{}{}\n",
-            mmd_flag, compile_tail, mmd_filter));
+        if (memoryAdmission)
+            append(std::format("  command = {}{} -- $cxx $local_includes"
+                               " $cxxflags $unit_cxxflags {}{}\n",
+                               admission_prefix(), admission_dep(),
+                               mmd_flag, compile_tail));
+        else
+            append(std::format(
+                "  command = $cxx $local_includes $cxxflags $unit_cxxflags {}{}{}\n",
+                mmd_flag, compile_tail, mmd_filter));
     }
     append("  description = OBJ $out\n");
     append_cxx_deps();
@@ -2025,8 +2074,13 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements,
     if (need_c_rule) {
         append("rule c_object\n");
         const std::string payload = " $local_includes";
-        append(std::format("  command = $cc{} $cflags $unit_cflags {}{}\n",
-                           rsp_ref(payload), c_mmd_flag, compile_tail));
+        if (memoryAdmission)
+            append(std::format("  command = {} -- $cc{} $cflags $unit_cflags {}{}\n",
+                               admission_prefix(), rsp_ref(payload), c_mmd_flag,
+                               compile_tail));
+        else
+            append(std::format("  command = $cc{} $cflags $unit_cflags {}{}\n",
+                               rsp_ref(payload), c_mmd_flag, compile_tail));
         append_rspfile(payload);
         append("  description = CC $out\n");
         append_cxx_deps();

src/build/schedule/detach_codegen.cppm

@@ -83,6 +83,7 @@ export module mcpp.build.schedule.detach_codegen;
 import std;
 import mcpp.platform.process;
 import mcpp.platform.capacity;
+import mcpp.build.depfile;
 import mcpp.build.stage;
 
 export namespace mcpp::build::schedule::detach {
@@ -151,6 +152,17 @@ int supervise(const std::filesystem::path& slot,
 // must not pass on.
 int await_unit(const std::filesystem::path& slot, const std::filesystem::path& object);
 
+// Run an ordinary compiler edge while holding the same admission token used
+// by detached compilation. The command is argv-based so compiler arguments do
+// not acquire a second quoting language at the admission boundary.
+int run_admitted(const std::vector<std::string>& command,
+                 const std::filesystem::path& semaphore,
+                 int maxCompilers,
+                 unsigned long long memoryBaselineBytes,
+                 int memoryThresholdPercent,
+                 const std::filesystem::path& depFrom = {},
+                 const std::filesystem::path& depTo = {});
+
 }  // namespace mcpp::build::schedule::detach
 
 // ---------------------------------------------------------------------------
@@ -690,4 +702,35 @@ int await_unit(const std::filesystem::path& slot, const std::filesystem::path& o
     return 0;
 }
 
+int run_admitted(const std::vector<std::string>& command,
+                 const std::filesystem::path& semaphore,
+                 int maxCompilers,
+                 unsigned long long memoryBaselineBytes,
+                 int memoryThresholdPercent,
+                 const std::filesystem::path& depFrom,
+                 const std::filesystem::path& depTo) {
+    if (command.empty()) return 2;
+    const auto token = acquire_token(semaphore, maxCompilers,
+                                     memoryBaselineBytes, memoryThresholdPercent);
+    bool timedOut = false;
+    const int rc = mcpp::platform::process::run_exec_deadline(
+        command, {}, std::chrono::milliseconds(0), &timedOut);
+    if (!token.empty()) {
+        std::error_code ec;
+        std::filesystem::remove(token, ec);
+    }
+    if (rc != 0 || depFrom.empty() || depTo.empty()) return rc;
+
+    std::ifstream in(depFrom, std::ios::binary);
+    if (!in) return 1;
+    std::string raw{std::istreambuf_iterator<char>{in}, {}};
+    std::ofstream out(depTo, std::ios::binary | std::ios::trunc);
+    if (!out) return 1;
+    out << mcpp::build::depfile::first_record(raw);
+    if (!out) return 1;
+    std::error_code ec;
+    std::filesystem::remove(depFrom, ec);
+    return 0;
+}
+
 }  // namespace mcpp::build::schedule::detach

src/cli.cppm

@@ -35,6 +35,7 @@ import mcpp.platform;                // is_windows — the PATH separator of `__
 import mcpp.platform.env;            // --offline → MCPP_OFFLINE
 import mcpp.platform.process;        // __action-stamp runs the checked command
 import mcpp.platform.fs;             // __action-stamp writes its stamps
+import mcpp.build.schedule.detach_codegen;
 import mcpp.build.progress;          // __action reports its start
 import mcpp.platform.runtime_search; // linker-wrapper path-injection opt-out
 import mcpp.ui;
@@ -1136,6 +1137,39 @@ int run(int argc, char** argv) {
         return 0;
     }
 
+    if (std::string_view(argv[1]) == "compile-admit") {
+        std::string sem, capText, baselineText, thresholdText, depRaw, depOut;
+        int i = 2;
+        for (; i < argc && std::string_view(argv[i]) != "--"; ++i) {
+            const std::string_view a = argv[i];
+            if (i + 1 >= argc) return 2;
+            if (a == "--sem") sem = argv[++i];
+            else if (a == "--cap") capText = argv[++i];
+            else if (a == "--mem-baseline") baselineText = argv[++i];
+            else if (a == "--mem-threshold") thresholdText = argv[++i];
+            else if (a == "--dep-raw") depRaw = argv[++i];
+            else if (a == "--dep-out") depOut = argv[++i];
+            else {
+                std::println(stderr, "error: compile-admit: unknown option '{}'", a);
+                return 2;
+            }
+        }
+        if (i + 1 >= argc || sem.empty()) {
+            std::println(stderr, "error: compile-admit requires --sem and -- <compiler argv...>");
+            return 2;
+        }
+        int cap = 0, threshold = 80;
+        unsigned long long baseline = 0;
+        std::from_chars(capText.data(), capText.data() + capText.size(), cap);
+        std::from_chars(baselineText.data(), baselineText.data() + baselineText.size(), baseline);
+        std::from_chars(thresholdText.data(), thresholdText.data() + thresholdText.size(), threshold);
+        std::vector<std::string> command;
+        for (++i; i < argc; ++i) command.emplace_back(argv[i]);
+        return mcpp::build::schedule::detach::run_admitted(
+            command, std::filesystem::path{sem}, cap, baseline, threshold,
+            std::filesystem::path{depRaw}, std::filesystem::path{depOut});
+    }
+
     if (std::string_view(argv[1]) == "__action-stamp"
         || std::string_view(argv[1]) == "__action") {
         const bool named = std::string_view(argv[1]) == "__action";