lint: the compiles of one run share their modules

CMakeLists.txt

@@ -2537,7 +2537,7 @@ if(TARGET daslang)
         COMMAND ${CMAKE_COMMAND} -E copy ${_identity} ${DAS_TEST_CACHE_DIR})
     add_dependencies(dastest_cache_guard ${_das_test_binaries})
 endif()
-foreach(_t check_lint_tree check_vulkan run_tests_vulkan run_tests_imgui_integration
+foreach(_t check_lint_tree check_lint_dasllama check_vulkan run_tests_vulkan run_tests_imgui_integration
         build_site_wasm check_playground_samples run_imgui2rst run_vulkan2rst)
     if(TARGET ${_t} AND _das_shared_modules)
         add_dependencies(${_t} ${_das_shared_modules})

ci/CMakeLists.txt

@@ -77,7 +77,7 @@ set(DAS_EXTENDED_CORE
     check_workflow_matrix check_shipped_skills check_md_ascii check_wheel_repack check_manifest_render
     check_example_games check_dasllama_site run_tests_bench_updater
     run_tests_pr_babysit check_dasgen run_tests_review_md run_tests_dastest
-    check_lint_tree check_ast_verify_changed check_line_length check_gated_das check_review_gates
+    check_lint_tree check_lint_dasllama check_ast_verify_changed check_line_length check_gated_das check_review_gates
     check_standalone_exes check_format check_format_exe run_utils_tests)
 if(NOT WIN32)
     # check_model_mint: its test stands in a shell script for daslang, which Windows cannot exec;

daslib/ast_verify.das

@@ -1702,10 +1702,23 @@ class AstInferPassVerifyMacro : AstPassMacro {
     }
 }
 
+//! a promoted copy's post-compile pass reaches every later program of
+//! the process (the walk is over the global module list), so only a
+//! program holding this module in its own library is verified
+def private program_asked_for_verify(prog : ProgramPtr) : bool {
+    var asked = false
+    program_for_each_module(prog) $(m) {
+        if (m.name == "ast_verify") {
+            asked = true
+        }
+    }
+    return asked
+}
+
 [post_compile_macro]
 class AstPostCompileVerifyMacro : AstPassMacro {
     def override apply(var prog : ProgramPtr; mod : Module?) : bool {
-        if (mod != null && !mod.moduleFlags.builtIn) {
+        if (mod != null && !mod.moduleFlags.builtIn && program_asked_for_verify(prog)) {
             verify_module(prog, mod)
             if (batch_mode()) {
                 verify_module_after_infer(prog, mod)

doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst

@@ -113,3 +113,4 @@ Path to linker, which is used in JIT.
 compile_file from a script reads and refreshes the default module cache around this compile (keyed by the file, the running binary, the host arguments and these policies).
 Interpreter only: caps how deep fastcall (frameless) calls may nest, panicking past the cap instead of exhausting the native stack; 0 leaves fastcall unchecked. Host-side counterpart of ``options max_fast_call_depth`` (the option overrides the policy).
 Keeps the used-function set and the function and variable indices alive after the main simulate. Off by default, they are freed when rtti is off; a host that calls ``is_used``, ``function_index`` or ``variable_index`` on the program after simulate sets it.
+A compile under ``no_optimizations`` promotes the shared modules it requires into the process, as an optimizing compile does, so a later compile in the process reuses them; for a process that compiles many programs under one policy set and runs none of them (the lint). Off by default.

include/daScript/ast/ast_serializer.h

@@ -365,7 +365,7 @@ namespace das {
         AstSerializer & serializeModule ( Module & module, bool already_exists );
 
         static constexpr uint32_t getVersion () {
-            return 232;   // 232: an expression whose source type is unresolved stays untyped until the type resolves
+            return 233;   // 233: CodeOfPolicies carries promote_shared_unoptimized
         }
 
         void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept;

include/daScript/simulate/REVIEW.md

@@ -24,6 +24,12 @@ annotation appends it after that type's last member, or bumps `LLVM_JIT_CODEGEN_
 members it read by offset; a member inserted before them moves every later one under code that
 still uses the old offsets.
 
+**A diff that changes `CodeOfPolicies`' layout or a field's meaning without changing its size - a
+reorder, a repurpose, a member that lands in a padding pocket, the appended `bool` among them -
+bumps `DAS_POLICIES_VERSION` (`code_of_policies.h`) in the same change.** The ABI stamp folds the
+version and the size alone, so a host built on the old headers passes the stamp check and the
+library reads the new field off the host's padding.
+
 **A diff that hashes a table key hashes a builtin key type - one in `heap.h`'s
 `makeTableKeyValueNode` list - as itself through `hash_function(context, key)` (`hash.h`),
 and a handled key type as the value type its annotation's `makeValueType()` returns.** A

include/daScript/simulate/code_of_policies.h

@@ -7,7 +7,7 @@ namespace das {
 
     // bump when CodeOfPolicies changes layout or meaning without changing size — a field
     // reorder/repurpose, or an insert that lands in a padding pocket (sizeof can't see those)
-    #define DAS_POLICIES_VERSION    3
+    #define DAS_POLICIES_VERSION    4
 
     // per-binary linkage for the ABI-stamp functions: at -O0 gcc/clang emit them as weak
     // default-visibility symbols, and the dynamic linker would bind every binary to the FIRST
@@ -174,6 +174,7 @@ namespace das {
         bool module_cache = false;               // a script's compile_file runs under the default module cache
         /*option*/ uint32_t max_fast_call_depth = 0;  // interpreter only; 0 = unchecked.
         bool keep_symbol_tables = false;         // keep usedFunctions and the function/variable indices past the main simulate
+        bool promote_shared_unoptimized = false; // src/ast/ARCHITECTURE_SHARED.md#shared-promotion
 
         // the abi_stamp truth (a member so the NSDMI above can call it - complete-class context).
         // Low byte 0 keeps a pre-stamp libDaScript reading this word seeing aot == false -

skills/perf_lint.md

@@ -13,11 +13,14 @@ message - fix the text in `daslib/perf_lint.das`, don't document around it.
 ```bash
 bin/daslang utils/lint/main.das -- file.das [dir ...] \
   [--quiet] [--silent] [--perf-only|--style-only|--paranoid-only] \
-  [--disable CODE,...] [--enable CODE,...] [--workers N] [--fix]
+  [--disable CODE,...] [--enable CODE,...] [--workers N] [--fix] \
+  [--no-shared-modules] [--exclude PATH ...]
 ```
 
 The MCP `lint` tool runs the same three passes. `--enable` is a whitelist (only the listed rules
-run); on overlap `--disable` wins. `--fix` writes the edits rules attach to their findings and
+run); on overlap `--disable` wins. A run compiles each shared module once per process and lints
+the files in require order; `--no-shared-modules` compiles every file's modules in its own group.
+`--exclude PATH` leaves a folder out of a directory walk. `--fix` writes the edits rules attach to their findings and
 re-lints; a fix that breaks the compile is rolled back. Pair it with `--enable CODE` to sweep one
 rule, and review the diff - a fix only proves the file still compiles.
 

src/ast/ARCHITECTURE_SHARED.md

@@ -20,5 +20,11 @@ program requires a daslib module first is the filesystem's choice: a lint test t
 fixture with optimizations off runs before `tests/linq` on APFS and after it on ext4 and NTFS.
 A compile with optimizations off still reuses a module promoted earlier, and `requireModuleNow`
 (`ARCHITECTURE.md#require-after-walk`) promotes its target whatever the policy, since a late
-require asks for a process module by name. The lint runner, the MCP server and the LSP set
-`ignore_shared_modules` beside `no_optimizations`, so they neither promote nor reuse.
+require asks for a process module by name. The MCP server, the LSP and the lint runner outside its
+shared-modules mode set `ignore_shared_modules` beside `no_optimizations`, so they neither promote
+nor reuse. `promote_shared_unoptimized` is the override: a compile under `no_optimizations` that sets it
+promotes its shared dependencies as an optimizing compile would, for a process that compiles many
+programs under one policy set and never serializes them - the lint's shared-modules mode, where
+the unoptimized body serves every later file and the AOT-stub mismatch has nothing to reach. A
+program the process compiles AFTER such a promotion reuses that unoptimized body, so the override
+belongs to a process that runs nothing.

src/ast/ast_parse.cpp

@@ -2356,7 +2356,7 @@ namespace das {
                 }
             }
             // src/ast/ARCHITECTURE_SHARED.md#shared-promotion
-            if ( auto failed = parseRequiredModules(walk.req, access, logs, libGroup, policies, !policies.no_optimizations) ) {
+            if ( auto failed = parseRequiredModules(walk.req, access, logs, libGroup, policies, !policies.no_optimizations || policies.promote_shared_unoptimized) ) {
                 return failed;
             }
             auto & serializer_read = daScriptEnvironment::getBound()->serializer_read;

src/builtin/module_builtin_ast_serialize.cpp

@@ -3252,7 +3252,7 @@ namespace das {
         X(no_global_variables) X(no_global_variables_at_all) X(no_global_heap) X(only_fast_aot) \
         X(aot_order_side_effects) X(no_unused_function_arguments) X(no_unused_block_arguments) \
         X(allow_block_variable_shadowing) X(allow_local_variable_shadowing) X(allow_shared_lambda) \
-        X(ignore_shared_modules) X(default_module_public) X(no_deprecated) X(no_aliasing) \
+        X(ignore_shared_modules) X(promote_shared_unoptimized) X(default_module_public) X(no_deprecated) X(no_aliasing) \
         X(strict_smart_pointers) X(no_init) X(strict_unsafe_delete) X(no_members_functions_in_struct) \
         X(no_local_class_members) X(no_unsafe_uninitialized_structures) X(default_init_containers) \
         X(strict_properties) X(no_writing_to_nameless) X(no_optimizations) X(no_fold_unsafe) X(fast_math) X(disable_dse) \

src/builtin/module_builtin_rtti.cpp

@@ -1028,6 +1028,7 @@ namespace das {
             addField<DAS_BIND_MANAGED_FIELD(allow_local_variable_shadowing)>("allow_local_variable_shadowing");
             addField<DAS_BIND_MANAGED_FIELD(allow_shared_lambda)>("allow_shared_lambda");
             addField<DAS_BIND_MANAGED_FIELD(ignore_shared_modules)>("ignore_shared_modules");
+            addField<DAS_BIND_MANAGED_FIELD(promote_shared_unoptimized)>("promote_shared_unoptimized");
             addField<DAS_BIND_MANAGED_FIELD(default_module_public)>("default_module_public");
             addField<DAS_BIND_MANAGED_FIELD(no_deprecated)>("no_deprecated");
             addField<DAS_BIND_MANAGED_FIELD(no_aliasing)>("no_aliasing");

tests/language/optional_require.das

@@ -1,4 +1,5 @@
 options gen2
+options _nolint = "STYLE030"   // every guarded require here IS the test, used or not
 require dastest/testing_boost public
 
 // Optional require: `require ?guard target` loads *target* when *guard* is available. A
@@ -9,7 +10,7 @@ require dastest/testing_boost public
 // guard-present-but-target-missing case still errors with 20605 (verified by hand;
 // module-resolution failures fire before the `expect` line parses, so that case is not
 // expressible as a cant_ test).
-require ?math math                                  // nolint:STYLE030 — the guarded-load mechanics ARE the test: guard present (math is a core builtin) -> target loaded
+require ?math math                                  // guard present (a builtin) -> target loaded
 require ?no_such_guard_xyz no_such_target_also_xyz  // guard absent, target unresolvable -> skipped
 require ?no_such_guard_pkg optional_require_fixture // guard absent -> SKIPPED even though the target resolves (strict plain guards); the non-load IS the test
 // PATH guard (contains '/'): availability = the guard's OWN file resolves; the rail for
@@ -18,7 +19,9 @@ require ?daslib/fio optional_require_fixture_pathguard        // path guard reso
 require ?daslib/no_such_file_xyz optional_require_fixture_skipme  // path guard unresolvable -> SKIPPED even though the target file resolves
 // %/-rooted guards: grammar-legal require_module_name forms the pre-parse require scanner must
 // scan identically to ast_requireModule (it once stopped at '%', mangling guard AND target)
-require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard  // nolint:STYLE030 — %/ guard resolves -> target loaded; referenced only under static_if
+// %/ guard resolves -> loaded; referenced under static_if alone,
+// whose builtin_module_exists is true once a run promoted the fixture
+require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard
 require ?%/tests/language/no_such_file_xyz.das optional_require_fixture_pctskip  // %/ guard unresolvable -> SKIPPED, target file resolves
 require optional_require_fixture_private   // requires optional_require_fixture_hidden PRIVATELY: the module_exists visibility pin below
 

utils/CMakeLists.txt

@@ -21,12 +21,15 @@ add_custom_target(check_dasgen
     COMMENT "Checking generated builtin .inc files are up to date (dasgen)"
 )
 
-# check_lint_tree lints DAS_CHUNKS of the tree in parallel. LINT019 needs every compile's nolint
+# check_lint_tree lints DAS_CHUNKS of the tree in parallel, each chunk one process that compiles the
+# shared modules its files require once. LINT019 needs every compile's nolint
 set(_lint_roots daslib utils modules tutorials examples tests src)
 list(TRANSFORM _lint_roots PREPEND "${PROJECT_SOURCE_DIR}/" OUTPUT_VARIABLE _lint_globs)
 list(TRANSFORM _lint_globs APPEND "/*.das")
 file(GLOB_RECURSE _lint_files RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS ${_lint_globs})
 list(FILTER _lint_files EXCLUDE REGEX "(^|/)(\\.[^/]*|node_modules|_[^/]*)(/|$)")
+# modules/dasLLAMA lints in check_lint_dasllama - one process over the module instead of its engine graph compiled per chunk
+list(FILTER _lint_files EXCLUDE REGEX "^modules/dasLLAMA/")
 list(LENGTH _lint_files _lint_count)
 math(EXPR _lint_size "(${_lint_count} + 127) / 128")
 DAS_CHUNKS(_lint_chunks ${_lint_size} ${_lint_files})
@@ -38,6 +41,10 @@ foreach(_n ${_lint_chunks})
     list(APPEND _lint_targets lint_tree_chunk_${_n})
 endforeach()
 add_custom_target(check_lint_tree DEPENDS ${_lint_targets})
+add_custom_target(check_lint_dasllama
+    COMMAND $<TARGET_FILE:daslang> ./utils/lint/main.das -- modules/dasLLAMA -q --silent
+    DEPENDS daslang WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+    COMMENT "Linting modules/dasLLAMA in one process, its engine graph compiled once")
 
 set(_dupe_paths -p daslib -p utils -p src)
 file(GLOB _dupe_entries LIST_DIRECTORIES true RELATIVE ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/modules/*/*)

utils/internal/ast-fuzz/test_ast_fuzz.das

@@ -107,6 +107,22 @@ def test_ast_verify_guesses_a_location_for_an_unset_at(t : T?) {
         "the search walked into an assume alias it should not see\n{out}")
 }
 
+// A promoted verify's post-compile pass reaches every later program
+// of the process; one that never required it is left alone. The lint
+// runner shares modules, so the verify fixture ahead of the breaking
+// one in one run is the shape.
+[test]
+def test_ast_verify_leaves_a_program_that_did_not_ask(t : T?) {
+    var out : string
+    let d = "utils/internal/ast-fuzz/selftest"
+    let argv <- [BIN, "utils/lint/main.das", "--",
+        "{d}/verify_breaker.das", "{d}/at_unset.das", "--no-cache"]
+    let ec = run(argv, out, 300.0)
+    t |> success(!died(ec, out), "the lint crashed (exit {ec})\n{out}")
+    t |> success(find(out, "{d}/at_unset.das") >= 0 && find(out, "FAIL") < 0,
+        "the promoted verify reached a program that did not require it\n{out}")
+}
+
 [test]
 def test_ast_verify_clean_on_valid_code(t : T?) {
     var out : string

utils/internal/doc-verify/main.das

@@ -1050,9 +1050,13 @@ def compile_file_rc(cfg : Config; path : string; var output : string&) : int {
     //! -dasroot names the tree whose modules the page's requires resolve from: the audited
     //! root, not the binary's own tree, so a worktree session checks its own sources
     let args <- [normalize(cfg.daslang), "-dasroot", normalize(cfg.root), "-compile-only", normalize(path)]
-    return run_and_capture(args, output, 120.0)
+    return run_and_capture(args, output, PROBE_TIMEOUT_SEC)
 }
 
+//! a companion probe compiles an engine's whole module graph cold:
+//! the dasLLAMA facade takes a minute on a fast box, more on a runner
+let PROBE_TIMEOUT_SEC = 300.0
+
 def spawn_failure(cfg : Config) : string =>
     "cannot spawn '{cfg.daslang}' - pass --daslang <path to the daslang binary>"
 
@@ -1179,7 +1183,10 @@ def run_probes(require_lines : table<string; bool>; file_mods : table<string>; c
         var out : string
         let prc = compile_file_rc(cfg, pfile, out)
         if (prc != 0) {
-            let why = prc == -1 ? " ({spawn_failure(cfg)})" : ""
+            var why = prc == -1 ? " ({spawn_failure(cfg)})" : ""
+            if (prc == popen_timed_out) {
+                why = " (timed out after {int(PROBE_TIMEOUT_SEC)} s: a slow box, no module missing)"
+            }
             failures |> push((origins[idx] ? "[companion] " : "[page] ") + trim(fread(pfile)) + why)
         }
     }

utils/lint/README.md

@@ -30,6 +30,26 @@ build of the runner never compiles, cannot check that version, and runs without
 those alone re-lints nothing until `--no-cache`. CI keeps the folder between runs, one per platform
 (`actions/cache` in `.github/workflows/extended_checks.yml`).
 
+The compiles of one run share their modules: each promotes the shared modules it requires into the
+process (`promote_shared_unoptimized`, `src/ast/ARCHITECTURE_SHARED.md#shared-promotion`), so every
+module compiles once and each later file reuses it, and a status line carries the count the compile
+took promoted (`PASS file (promoted 77)`). The files lint in require order, a module ahead of every
+file that requires it, read off the sources' `require` lines (a require names a file by the file's
+name): a module promoted before its own compile would collide with it. A parallel run shares inside
+each worker, over the chunk it holds. The result cache's entries hash the same module sources either
+way. `--no-shared-modules` compiles every file's modules in its own group, as the runner did before.
+A file whose code keys on `builtin_module_exists` of a `shared` module - a `static_if` over the
+guard - compiles differently once the run promoted that module, as it does under dastest, so a
+`nolint` that one mode needs lists `LINT019` for the other, or the file suppresses the rule for
+itself (`options _nolint`). A promoted module's post-compile
+macros run on every later program of the process (the walk is over the global module list), so
+such a macro guards on the program's own library, as `daslib/ast_verify` does.
+CI's chunked `check_lint_tree` leaves `modules/dasLLAMA` out, whose every file compiles the engine's
+whole graph, and `check_lint_dasllama` lints that module as one process.
+
+`--exclude <path>` (repeatable) leaves every file and folder under the path out of a run: the
+directory walk, the runner's folder rules and a positional root under it alike.
+
 Five rules are the runner's own, because they are about folders rather than code. Each runs
 once per invocation, over a walk of the directory roots the run was given. A directory whose
 `.lint_config` carries `[docs] rule_docs_only = true` may hold only rule documents