From f3b6e6a2ca110746a89fbc46ed60160e2e19e44f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Thu, 22 Oct 2026 11:55:05 -0700 Subject: [PATCH 1/5] lint: the compiles of one run share their modules. The lint compiled every file cold - no module cache under a compile-only policy set, the shared modules neither promoted nor reused since the runner set ignore_shared_modules beside no_optimizations - so a dasLLAMA file paid its whole graph (18 s for the Vulkan classes file, 60 s for the resident llama test) and the module walk ran near an hour on one core. CodeOfPolicies gains promote_shared_unoptimized: a compile under no_optimizations that sets it promotes its shared dependencies as an optimizing compile would, for a process that compiles many programs under one policy set and runs none of them (src/ast/ARCHITECTURE_SHARED.md). The lint sets it by default and drops ignore_shared_modules, so every shared module compiles once per process and each later file reuses it; the files lint in require order - a module ahead of every file that requires it, since a module promoted before its own compile would collide with it - the require paths read through the review gates' das_requires and module_group_module_name (a require names a file by the file's name), the order daslib/algorithm's topological_sort; a parallel run orders each worker's chunk the same way; every status line carries the count of modules the compile took promoted, builtin left out since it promotes under every policy; the result cache works as before, since its entries hash the same module sources. --no-shared-modules compiles every file's modules in its own group, as the runner did. On this box, the same findings either way: six dasLLAMA files 88.7 s cold against 46.0 s shared, twelve (the CUDA home among them) 175.8 against 61.1, the whole module (493 files, 33 Metal skips) 2 min 16 s against a cold serial walk stopped at 94 minutes, at a peak working set of 1.9 GB; one of the tree sweep's 128 chunks (33 files) 16 s against 3 s at a peak of 247 against 237 MB. CI's chunked check_lint_tree leaves modules/dasLLAMA out, whose every file compiles the engine's whole graph, and the new check_lint_dasllama target lints that module as one process, in the extended checks' core lane set. --exclude (repeatable) leaves a folder out of a run - the directory walk, the runner's folder rules and a positional root under it alike - the lever a whole-tree sweep needs to carve a module tree out by hand. The lint's own tests cover the mode and the flags (lint_shared_modules.das: the same findings with sharing and without and the cache hit on the next run, the promoted count past a file that requires nothing of the set, the module ahead of its users by name and by file path, a worker run at two workers, the excluded folder never linted) over a fixture the cache-family tests share (_lint_child.das: the planted tree under test_temp_dir, the lint child, the status and line readers), where each test carried its own copy. The policy field sits last in CodeOfPolicies, since a cached JIT DLL reads the members before it by offset, and DAS_POLICIES_VERSION moves to 4, since the appended bool lands in a padding pocket the ABI stamp's size cannot see - a host built on the earlier headers fails the stamp at its first compile instead of handing the library its padding; the simulate checklist gains that duty. The README and skills/perf_lint.md carry the flags, the handmade CodeOfPolicies page the field's line; the serializer version is 233, since CodeOfPolicies rides the module cache. Co-Authored-By: Claude Fable 5.1 --- CMakeLists.txt | 2 +- ci/CMakeLists.txt | 2 +- ...ructure_annotation-rtti-CodeOfPolicies.rst | 1 + include/daScript/ast/ast_serializer.h | 2 +- include/daScript/simulate/REVIEW.md | 6 + include/daScript/simulate/code_of_policies.h | 3 +- skills/perf_lint.md | 7 +- src/ast/ARCHITECTURE_SHARED.md | 10 +- src/ast/ast_parse.cpp | 2 +- src/builtin/module_builtin_ast_serialize.cpp | 2 +- src/builtin/module_builtin_rtti.cpp | 1 + utils/CMakeLists.txt | 9 +- utils/lint/README.md | 14 ++ utils/lint/main.das | 154 +++++++++++++----- utils/lint/tests/_lint_child.das | 73 +++++++++ utils/lint/tests/lint_result_cache.das | 78 +-------- utils/lint/tests/lint_shared_modules.das | 114 +++++++++++++ 17 files changed, 354 insertions(+), 126 deletions(-) create mode 100644 utils/lint/tests/_lint_child.das create mode 100644 utils/lint/tests/lint_shared_modules.das diff --git a/CMakeLists.txt b/CMakeLists.txt index c80de2775d..4609ef46b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2537,7 +2537,7 @@ if(TARGET daslang) COMMAND ${CMAKE_COMMAND} -E copy ${_identity} ${DAS_TEST_CACHE_DIR}) add_dependencies(dastest_cache_guard ${_das_test_binaries}) endif() -foreach(_t check_lint_tree check_vulkan run_tests_vulkan run_tests_imgui_integration +foreach(_t check_lint_tree check_lint_dasllama check_vulkan run_tests_vulkan run_tests_imgui_integration build_site_wasm check_playground_samples run_imgui2rst run_vulkan2rst) if(TARGET ${_t} AND _das_shared_modules) add_dependencies(${_t} ${_das_shared_modules}) diff --git a/ci/CMakeLists.txt b/ci/CMakeLists.txt index 51741beaab..5537a4e989 100644 --- a/ci/CMakeLists.txt +++ b/ci/CMakeLists.txt @@ -77,7 +77,7 @@ set(DAS_EXTENDED_CORE check_workflow_matrix check_shipped_skills check_md_ascii check_wheel_repack check_manifest_render check_example_games check_dasllama_site run_tests_bench_updater run_tests_pr_babysit check_dasgen run_tests_review_md run_tests_dastest - check_lint_tree check_ast_verify_changed check_line_length check_gated_das check_review_gates + check_lint_tree check_lint_dasllama check_ast_verify_changed check_line_length check_gated_das check_review_gates check_standalone_exes check_format check_format_exe run_utils_tests) if(NOT WIN32) # check_model_mint: its test stands in a shell script for daslang, which Windows cannot exec; diff --git a/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst b/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst index 0a7f0c7808..eb93ba4e17 100644 --- a/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst +++ b/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst @@ -113,3 +113,4 @@ Path to linker, which is used in JIT. compile_file from a script reads and refreshes the default module cache around this compile (keyed by the file, the running binary, the host arguments and these policies). Interpreter only: caps how deep fastcall (frameless) calls may nest, panicking past the cap instead of exhausting the native stack; 0 leaves fastcall unchecked. Host-side counterpart of ``options max_fast_call_depth`` (the option overrides the policy). Keeps the used-function set and the function and variable indices alive after the main simulate. Off by default, they are freed when rtti is off; a host that calls ``is_used``, ``function_index`` or ``variable_index`` on the program after simulate sets it. +A compile under ``no_optimizations`` promotes the shared modules it requires into the process, as an optimizing compile does, so a later compile in the process reuses them; for a process that compiles many programs under one policy set and runs none of them (the lint). Off by default. diff --git a/include/daScript/ast/ast_serializer.h b/include/daScript/ast/ast_serializer.h index 6d5e753e5e..34a780bcc2 100644 --- a/include/daScript/ast/ast_serializer.h +++ b/include/daScript/ast/ast_serializer.h @@ -365,7 +365,7 @@ namespace das { AstSerializer & serializeModule ( Module & module, bool already_exists ); static constexpr uint32_t getVersion () { - return 232; // 232: an expression whose source type is unresolved stays untyped until the type resolves + return 233; // 233: CodeOfPolicies carries promote_shared_unoptimized } void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept; diff --git a/include/daScript/simulate/REVIEW.md b/include/daScript/simulate/REVIEW.md index 1a6ff7b827..a948b192e1 100644 --- a/include/daScript/simulate/REVIEW.md +++ b/include/daScript/simulate/REVIEW.md @@ -24,6 +24,12 @@ annotation appends it after that type's last member, or bumps `LLVM_JIT_CODEGEN_ members it read by offset; a member inserted before them moves every later one under code that still uses the old offsets. +**A diff that changes `CodeOfPolicies`' layout or a field's meaning without changing its size - a +reorder, a repurpose, a member that lands in a padding pocket, the appended `bool` among them - +bumps `DAS_POLICIES_VERSION` (`code_of_policies.h`) in the same change.** The ABI stamp folds the +version and the size alone, so a host built on the old headers passes the stamp check and the +library reads the new field off the host's padding. + **A diff that hashes a table key hashes a builtin key type - one in `heap.h`'s `makeTableKeyValueNode` list - as itself through `hash_function(context, key)` (`hash.h`), and a handled key type as the value type its annotation's `makeValueType()` returns.** A diff --git a/include/daScript/simulate/code_of_policies.h b/include/daScript/simulate/code_of_policies.h index 2bf2c766db..76e0ddf1cb 100644 --- a/include/daScript/simulate/code_of_policies.h +++ b/include/daScript/simulate/code_of_policies.h @@ -7,7 +7,7 @@ namespace das { // bump when CodeOfPolicies changes layout or meaning without changing size — a field // reorder/repurpose, or an insert that lands in a padding pocket (sizeof can't see those) - #define DAS_POLICIES_VERSION 3 + #define DAS_POLICIES_VERSION 4 // per-binary linkage for the ABI-stamp functions: at -O0 gcc/clang emit them as weak // default-visibility symbols, and the dynamic linker would bind every binary to the FIRST @@ -174,6 +174,7 @@ namespace das { bool module_cache = false; // a script's compile_file runs under the default module cache /*option*/ uint32_t max_fast_call_depth = 0; // interpreter only; 0 = unchecked. bool keep_symbol_tables = false; // keep usedFunctions and the function/variable indices past the main simulate + bool promote_shared_unoptimized = false; // src/ast/ARCHITECTURE_SHARED.md#shared-promotion // the abi_stamp truth (a member so the NSDMI above can call it - complete-class context). // Low byte 0 keeps a pre-stamp libDaScript reading this word seeing aot == false - diff --git a/skills/perf_lint.md b/skills/perf_lint.md index 156cf8eba2..8cdf6e862b 100644 --- a/skills/perf_lint.md +++ b/skills/perf_lint.md @@ -13,11 +13,14 @@ message - fix the text in `daslib/perf_lint.das`, don't document around it. ```bash bin/daslang utils/lint/main.das -- file.das [dir ...] \ [--quiet] [--silent] [--perf-only|--style-only|--paranoid-only] \ - [--disable CODE,...] [--enable CODE,...] [--workers N] [--fix] + [--disable CODE,...] [--enable CODE,...] [--workers N] [--fix] \ + [--no-shared-modules] [--exclude PATH ...] ``` The MCP `lint` tool runs the same three passes. `--enable` is a whitelist (only the listed rules -run); on overlap `--disable` wins. `--fix` writes the edits rules attach to their findings and +run); on overlap `--disable` wins. A run compiles each shared module once per process and lints +the files in require order; `--no-shared-modules` compiles every file's modules in its own group. +`--exclude PATH` leaves a folder out of a directory walk. `--fix` writes the edits rules attach to their findings and re-lints; a fix that breaks the compile is rolled back. Pair it with `--enable CODE` to sweep one rule, and review the diff - a fix only proves the file still compiles. diff --git a/src/ast/ARCHITECTURE_SHARED.md b/src/ast/ARCHITECTURE_SHARED.md index 4cdb6a2740..e84db538e1 100644 --- a/src/ast/ARCHITECTURE_SHARED.md +++ b/src/ast/ARCHITECTURE_SHARED.md @@ -20,5 +20,11 @@ program requires a daslib module first is the filesystem's choice: a lint test t fixture with optimizations off runs before `tests/linq` on APFS and after it on ext4 and NTFS. A compile with optimizations off still reuses a module promoted earlier, and `requireModuleNow` (`ARCHITECTURE.md#require-after-walk`) promotes its target whatever the policy, since a late -require asks for a process module by name. The lint runner, the MCP server and the LSP set -`ignore_shared_modules` beside `no_optimizations`, so they neither promote nor reuse. +require asks for a process module by name. The MCP server, the LSP and the lint runner outside its +shared-modules mode set `ignore_shared_modules` beside `no_optimizations`, so they neither promote +nor reuse. `promote_shared_unoptimized` is the override: a compile under `no_optimizations` that sets it +promotes its shared dependencies as an optimizing compile would, for a process that compiles many +programs under one policy set and never serializes them - the lint's shared-modules mode, where +the unoptimized body serves every later file and the AOT-stub mismatch has nothing to reach. A +program the process compiles AFTER such a promotion reuses that unoptimized body, so the override +belongs to a process that runs nothing. diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index e0ba3a678a..99c63cfe2f 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -2356,7 +2356,7 @@ namespace das { } } // src/ast/ARCHITECTURE_SHARED.md#shared-promotion - if ( auto failed = parseRequiredModules(walk.req, access, logs, libGroup, policies, !policies.no_optimizations) ) { + if ( auto failed = parseRequiredModules(walk.req, access, logs, libGroup, policies, !policies.no_optimizations || policies.promote_shared_unoptimized) ) { return failed; } auto & serializer_read = daScriptEnvironment::getBound()->serializer_read; diff --git a/src/builtin/module_builtin_ast_serialize.cpp b/src/builtin/module_builtin_ast_serialize.cpp index a479c602fe..1ea49eaf8a 100644 --- a/src/builtin/module_builtin_ast_serialize.cpp +++ b/src/builtin/module_builtin_ast_serialize.cpp @@ -3252,7 +3252,7 @@ namespace das { X(no_global_variables) X(no_global_variables_at_all) X(no_global_heap) X(only_fast_aot) \ X(aot_order_side_effects) X(no_unused_function_arguments) X(no_unused_block_arguments) \ X(allow_block_variable_shadowing) X(allow_local_variable_shadowing) X(allow_shared_lambda) \ - X(ignore_shared_modules) X(default_module_public) X(no_deprecated) X(no_aliasing) \ + X(ignore_shared_modules) X(promote_shared_unoptimized) X(default_module_public) X(no_deprecated) X(no_aliasing) \ X(strict_smart_pointers) X(no_init) X(strict_unsafe_delete) X(no_members_functions_in_struct) \ X(no_local_class_members) X(no_unsafe_uninitialized_structures) X(default_init_containers) \ X(strict_properties) X(no_writing_to_nameless) X(no_optimizations) X(no_fold_unsafe) X(fast_math) X(disable_dse) \ diff --git a/src/builtin/module_builtin_rtti.cpp b/src/builtin/module_builtin_rtti.cpp index d56b7ad20d..e17c9acd76 100644 --- a/src/builtin/module_builtin_rtti.cpp +++ b/src/builtin/module_builtin_rtti.cpp @@ -1028,6 +1028,7 @@ namespace das { addField("allow_local_variable_shadowing"); addField("allow_shared_lambda"); addField("ignore_shared_modules"); + addField("promote_shared_unoptimized"); addField("default_module_public"); addField("no_deprecated"); addField("no_aliasing"); diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 3ec785e0b1..8364ec42ab 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -21,12 +21,15 @@ add_custom_target(check_dasgen COMMENT "Checking generated builtin .inc files are up to date (dasgen)" ) -# check_lint_tree lints DAS_CHUNKS of the tree in parallel. LINT019 needs every compile's nolint +# check_lint_tree lints DAS_CHUNKS of the tree in parallel, each chunk one process that compiles the +# shared modules its files require once. LINT019 needs every compile's nolint set(_lint_roots daslib utils modules tutorials examples tests src) list(TRANSFORM _lint_roots PREPEND "${PROJECT_SOURCE_DIR}/" OUTPUT_VARIABLE _lint_globs) list(TRANSFORM _lint_globs APPEND "/*.das") file(GLOB_RECURSE _lint_files RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS ${_lint_globs}) list(FILTER _lint_files EXCLUDE REGEX "(^|/)(\\.[^/]*|node_modules|_[^/]*)(/|$)") +# modules/dasLLAMA lints in check_lint_dasllama - one process over the module instead of its engine graph compiled per chunk +list(FILTER _lint_files EXCLUDE REGEX "^modules/dasLLAMA/") list(LENGTH _lint_files _lint_count) math(EXPR _lint_size "(${_lint_count} + 127) / 128") DAS_CHUNKS(_lint_chunks ${_lint_size} ${_lint_files}) @@ -38,6 +41,10 @@ foreach(_n ${_lint_chunks}) list(APPEND _lint_targets lint_tree_chunk_${_n}) endforeach() add_custom_target(check_lint_tree DEPENDS ${_lint_targets}) +add_custom_target(check_lint_dasllama + COMMAND $ ./utils/lint/main.das -- modules/dasLLAMA -q --silent + DEPENDS daslang WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMENT "Linting modules/dasLLAMA in one process, its engine graph compiled once") set(_dupe_paths -p daslib -p utils -p src) file(GLOB _dupe_entries LIST_DIRECTORIES true RELATIVE ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/modules/*/*) diff --git a/utils/lint/README.md b/utils/lint/README.md index 60d3c56d81..d1958a7787 100644 --- a/utils/lint/README.md +++ b/utils/lint/README.md @@ -30,6 +30,20 @@ build of the runner never compiles, cannot check that version, and runs without those alone re-lints nothing until `--no-cache`. CI keeps the folder between runs, one per platform (`actions/cache` in `.github/workflows/extended_checks.yml`). +The compiles of one run share their modules: each promotes the shared modules it requires into the +process (`promote_shared_unoptimized`, `src/ast/ARCHITECTURE_SHARED.md#shared-promotion`), so every +module compiles once and each later file reuses it, and a status line carries the count the compile +took promoted (`PASS file (promoted 77)`). The files lint in require order, a module ahead of every +file that requires it, read off the sources' `require` lines (a require names a file by the file's +name): a module promoted before its own compile would collide with it. A parallel run shares inside +each worker, over the chunk it holds. The result cache's entries hash the same module sources either +way. `--no-shared-modules` compiles every file's modules in its own group, as the runner did before. +CI's chunked `check_lint_tree` leaves `modules/dasLLAMA` out, whose every file compiles the engine's +whole graph, and `check_lint_dasllama` lints that module as one process. + +`--exclude ` (repeatable) leaves every file and folder under the path out of a run: the +directory walk, the runner's folder rules and a positional root under it alike. + Five rules are the runner's own, because they are about folders rather than code. Each runs once per invocation, over a walk of the directory roots the run was given. A directory whose `.lint_config` carries `[docs] rule_docs_only = true` may hold only rule documents diff --git a/utils/lint/main.das b/utils/lint/main.das index d17f6409f4..715285df86 100644 --- a/utils/lint/main.das +++ b/utils/lint/main.das @@ -16,6 +16,9 @@ require math require strings require daslib/strings_boost require daslib/rtti +require daslib/algorithm // topological_sort: the --shared-modules file order +require daslib/module_group // module_group_module_name: a require path's module +require dastest/review_gate // das_requires: a file's require paths require ../common/parallel_workers.das require ../common/arch_citations.das @@ -31,6 +34,7 @@ struct LintResult { excluded : int // findings dropped by the .lint_config [paths] exclude policy fixed : int cached : bool // a clean result the cache vouched for: nothing was compiled + promoted_modules : int // counted under --shared-modules alone } [CommandLineArgs] @@ -64,6 +68,10 @@ struct Config { fix : bool @clarg_doc = "Skip the result cache: compile and lint every file, record nothing" no_cache : bool + @clarg_doc = "Compile each file's modules in its own group; the default shares them" + no_shared_modules : bool + @clarg_doc = "Leave every file and folder under this path out of the run. Repeatable" + exclude : array @clarg_doc = "Where the result cache lives (one entry per clean file: the content hash of every input its compile read)" cache_dir : string = LINT_CACHE_DEFAULT_DIR @@ -152,6 +160,13 @@ def is_skip_dir(name : string) : bool => name |> starts_with(".") || name == "no def is_build_tree(path : string) : bool => stat("{path}/CMakeCache.txt").is_valid || stat("{path}/.git").is_valid +var lint_exclude_roots : array //! --exclude's roots, absolute generic paths + +def is_excluded_path(path : string) : bool => + owning_root(to_generic_path(path), lint_exclude_roots) != "." + +def is_skip_tree(path : string) : bool => is_build_tree(path) || is_excluded_path(path) + def scan_das_files(path : string; var files : array; var cache : table; var skipped : int&) { let st = stat(path) return if (!st.is_valid) @@ -176,7 +191,7 @@ def scan_das_files(path : string; var files : array; var cache : table ends_with(".das") && !cache |> key_exists(full)) { if (is_skip_file(name)) { @@ -223,7 +238,7 @@ def scan_docs_folders(path : string; want_rule_docs, want_doc_lines : bool; return if (name == "." || name == ".." || name |> starts_with("_") || is_skip_dir(name)) let full = "{path}/{name}" let fst = stat(full) - if (fst.is_valid && fst.is_dir && !is_build_tree(full)) { + if (fst.is_valid && fst.is_dir && !is_skip_tree(full)) { scan_docs_folders(full, want_rule_docs, want_doc_lines, folders, line_folders, seen) } } @@ -360,7 +375,7 @@ def collect_basenames(path : string; var out : table) { let fst = stat(full) return if (!fst.is_valid) if (fst.is_dir) { - return if (is_build_tree(full)) + return if (is_skip_tree(full)) collect_basenames(full, out) } else { out |> insert(name, null) @@ -404,7 +419,7 @@ def collect_test_cells(path : string; var out : table) { fio::dir(path) $(name) { return if (name == "." || name == ".." || is_skip_dir(name)) let full = "{path}/{name}" - return if (is_build_tree(full)) + return if (is_skip_tree(full)) collect_test_cells(full, out) } } @@ -556,7 +571,7 @@ def private scan_arch_folder(path : string; var mds, dases : array; var let fst = stat(full) return if (!fst.is_valid) if (fst.is_dir) { - return if (is_build_tree(full)) + return if (is_skip_tree(full)) scan_arch_folder(full, mds, dases, seen) } elif (fst.is_reg && name |> ends_with(".md")) { mds |> push(full) @@ -723,7 +738,8 @@ def private file_nolint_codes(prog : ProgramPtr; base : table) : table @@ -821,6 +838,27 @@ def lint_cache_store(entry : string; inputs : array) { }) } +//! an environment failure - a missing prerequisite (error[20605]) +//! or an [extern] dlopen miss - skips the file, read off the first +//! diagnostic's block alone, since such a failure aborts the compile +//! before any other error and the dlopen miss puts "can't load +//! library" on the message's second line; anything else fails it +def private note_compile_failure(issue_str : string; var result : LintResult) { + var first_diag = "" + let err_start = find(issue_str, "error[") + if (err_start >= 0) { + first_diag = slice(issue_str, err_start, min(err_start + 240, length(issue_str))) + } + result.compile_errors = issue_str + if (find(first_diag, "error[20605]") >= 0) { + result.skip_reason = "(missing prerequisite — install daspkg deps to lint)" + } elif (find(first_diag, "can't load library") >= 0) { + result.skip_reason = "(native library unavailable — install it to lint this file)" + } else { + result.failed = true + } +} + def lint_file(file : string; run_paranoid, run_perf, run_style, comment_hygiene, lint_fixtures, lint_excluded_paths : bool; disabled_codes, enabled_codes : table) : LintResult { var result = LintResult(file = file) @@ -839,37 +877,17 @@ def lint_file(file : string; run_paranoid, run_perf, run_style, comment_hygiene, lint_policies(cop) compile_file(file, access, unsafe(addr(mg)), cop) $(ok; program; issues) { if (!ok) { - let issue_str = string(issues) - // Environment classes downgrade to SKIP only off the FIRST diagnostic's header - // line: a genuine environment failure aborts the compile before any other error, - // while a whole-string substring match would let one env line mask real errors - // (or match the literal echoed from an erroring source line). - var first_diag = "" - let err_start = find(issue_str, "error[") - if (err_start >= 0) { - // the first diagnostic's BLOCK, not just its header: the [extern] dlopen - // miss puts "can't load library" on the message's second line - first_diag = slice(issue_str, err_start, min(err_start + 240, length(issue_str))) - } - // error[20605] = missing prerequisite (e.g. daspkg-installed `anthropic/anthropic`) — - // out-of-tree as far as a stock daslang build can tell. - if (find(first_diag, "error[20605]") >= 0) { - result.skip_reason = "(missing prerequisite — install daspkg deps to lint)" - result.compile_errors = issue_str - return - } - // A [extern] dlopen miss (e.g. dasSMT without libz3) is the environment, not the - // code — skip with a visible reason so nightly coverage counting still sees it. - if (find(first_diag, "can't load library") >= 0) { - result.skip_reason = "(native library unavailable — install it to lint this file)" - result.compile_errors = issue_str - return - } - result.failed = true - result.compile_errors = issue_str + note_compile_failure(string(issues), result) return } inputs |> push_from(lint_cache_inputs(program, file)) if (!empty(entry)) + if (lint_shared_modules) { + program_for_each_module(program) $(mod) { + return if (!mod.moduleFlags.promoted || empty(mod.fileName)) + // builtin promotes under every policy + result.promoted_modules++ if (mod.name != "builtin") + } + } let file_disabled <- file_nolint_codes(program, disabled_codes) if (run_paranoid) { var paranoid_issues : array @@ -953,6 +971,9 @@ def apply_path_excludes(var result : LintResult) { } } +def private promoted_suffix(result : LintResult) : string => + lint_shared_modules ? " (promoted {result.promoted_modules})" : "" + // Print a single LintResult in the user-facing format. Shared by serial and // parallel paths; per-file output matches, though serial mode appends its // LINT019 findings as a trailing block while parallel reports them inline. @@ -972,12 +993,14 @@ def print_result(result : LintResult; quiet, silent : bool) { print(" {result.compile_errors}\n") } } elif (result.count > 0) { - print("WARN {result.file} ({result.count})\n") + print("WARN {result.file} ({result.count}){promoted_suffix(result)}\n") print(build_string() $(var w) { write_deduped(w, result.issues, " ", !silent) }) } elif (!quiet) { - print(result.cached ? "CACHED {result.file}\n" : "PASS {result.file}\n") + let line = (result.cached ? "CACHED {result.file}" + : "PASS {result.file}{promoted_suffix(result)}") + print("{line}\n") } } @@ -1017,8 +1040,10 @@ def maybe_collect(var since_collect : int&) { def run_worker(paths : array; run_paranoid, run_perf, run_style, comment_hygiene, lint_fixtures, lint_excluded_paths : bool; disabled_codes, enabled_codes : table; fix : bool) { var since_collect = 0 - for (f in paths) { - let result = lint_file_fixing(f, run_paranoid, run_perf, run_style, comment_hygiene, lint_fixtures, lint_excluded_paths, disabled_codes, enabled_codes, fix) + var inscope order <- lint_order(paths) + for (idx in order) { + let result = lint_file_fixing(paths[idx], run_paranoid, run_perf, run_style, + comment_hygiene, lint_fixtures, lint_excluded_paths, disabled_codes, enabled_codes, fix) print("{JSON_PREFIX}{write_json(JV(result))}\n{JSON_PREFIX}") maybe_collect(since_collect) } @@ -1035,6 +1060,9 @@ def build_worker_argv(spawn_prefix : array; chunk_path : string; cfg : C if (cfg.quiet) { argv |> push("--quiet") } + if (cfg.no_shared_modules) { + argv |> push("--no-shared-modules") + } if (cfg.comment_hygiene) { argv |> push("--comment-hygiene") } @@ -1212,6 +1240,7 @@ def collect_input_files(var files : array&; var dirs : array&) : // absolute, tripping error 20605 "several modules with the same name" (e.g. dasPEG) let full = get_full_file_name(p) let target = empty(full) ? p : replace(full, "\\", "/") + continue if (is_excluded_path(target)) let st = stat(target) if (st.is_valid && st.is_dir) { dirs |> push(target) @@ -1303,6 +1332,45 @@ def print_lint_help(rc : int) : int { return rc } +struct private RequireNode { + id : int + before : table +} + +//! the files' indices in require order - a module ahead of every +//! file that requires it. A file's compile promotes the shared +//! modules it requires, and a module promoted before its own compile +//! would collide with it, so each file lints before anything that +//! requires it. A require names a file by the file's name (the module +//! it declares is that name wherever the compiler resolves it), the +//! first file keeps a repeated name, and a name no file of the set +//! carries is left to the compiler +def private require_order(files : array) : array { + var name_of : table + for (file, i in files, count()) { + let name = module_group_module_name(file) + if (!key_exists(name_of, name)) { + name_of[name] = i + } + } + var nodes : array + nodes |> reserve(length(files)) + for (file, i in files, count()) { + var node <- RequireNode(id = i) + for (path in das_requires(file)) { + let j = name_of?[module_group_module_name(path)] ?? -1 + node.before |> insert(j) if (j >= 0 && j != i) + } + nodes |> emplace(node) + } + return <- [for (node in topological_sort(nodes)); node.id] +} + +//! the order a run lints `files` in: the require order under sharing, +//! the given one without +def private lint_order(files : array) : array => + <- lint_shared_modules ? require_order(files) : [for (i in range(length(files))); i] + [export, unsafe_heap_collect, lint_tool_entry] def main() : int { var r <- parse_args(type) @@ -1322,6 +1390,12 @@ def main() : int { var disabled_codes, enabled_codes : table if (!resolve_rule_filters(cfg, disabled_codes, enabled_codes)) return 1 lint_cache_dir = cfg.no_cache || is_standalone_exe() || jit_enabled() ? "" : cfg.cache_dir + lint_shared_modules = !cfg.no_shared_modules + lint_exclude_roots |> reserve(length(cfg.exclude)) + for (x in cfg.exclude) { + let full = get_full_file_name(x) + lint_exclude_roots |> push(to_generic_path(empty(full) ? x : full)) + } // Worker mode: lint a pre-resolved list from a paths-from file and emit // JSON-wrapped results. No banner, no summary, no exit-code derivation. @@ -1358,7 +1432,9 @@ def main() : int { print("Linting {length(files)} file(s)...\n\n") } var since_collect = 0 - for (file in files) { + var inscope order <- lint_order(files) + for (idx in order) { + let file = files[idx] if (!cfg.quiet) { print("checking {file}...\n") } diff --git a/utils/lint/tests/_lint_child.das b/utils/lint/tests/_lint_child.das new file mode 100644 index 0000000000..6579777f95 --- /dev/null +++ b/utils/lint/tests/_lint_child.das @@ -0,0 +1,73 @@ +options gen2 +options indenting = 4 + +//! the lint CLI as a child over a planted tree: a cell plants files +//! under `test_temp_dir()`, runs the lint with its cache under the +//! tree, and reads a file's line off the output +module _lint_child shared public + +require dastest/testing_boost public +require strings +require daslib/strings_boost +require daslib/fio +require daslib/command_line // get_das_exe + +def plant(dir, name, content : string) { + fopen(path_join(dir, name), "wb") $(f) { + if (f != null) { + fwrite(f, content) + } + } +} + +//! the lint over `root` with its cache under `root/cache`; `host`: +//! daslang's flags before the script, `extra`: after `--` +def run_lint(root : string; extra, host : array) : string { + var argv <- [get_das_exe(), "-dasroot", get_das_root()] + argv |> push_from(host) + argv |> push("{get_das_root()}/utils/lint/main.das") + argv |> push("--") + argv |> push(root) + argv |> push("--cache-dir") + argv |> push(path_join(root, "cache")) + argv |> push_from(extra) + var output : string + unsafe(popen_argv(argv, 0.0, $(f) { + if (f != null) { + output = fread(f) + } + })) + return output +} + +//! the result-cache entries under `root/cache`, the stamp aside +def entry_count(root : string) : int { + var n = 0 + dir(path_join(root, "cache")) $(name) { + n++ if (name != "." && name != ".." && name != "lint_hash.txt") + } + return n +} + +//! the first line of `out` about `file` under `root` that opens with +//! one of `prefixes`: its index, -1 when none, the line in `line` +def line_of(out, root, file : string; prefixes : array; var line : string&) : int { + let path = to_generic_path(path_join(root, file)) + for (candidate, i in split(out, "\n"), count()) { + for (prefix in prefixes) { + if (candidate |> starts_with(prefix) && find(to_generic_path(candidate), path) >= 0) { + line = candidate + return i + } + } + } + return -1 +} + +//! the status word on `file`'s line - PASS, CACHED, WARN or FAIL - +//! or "?" when the run printed none +def status_of(out, root, file : string) : string { + var line = "" + return "?" if (line_of(out, root, file, ["PASS ", "CACHED ", "WARN ", "FAIL "], line) < 0) + return slice(line, 0, find(line, " ")) +} diff --git a/utils/lint/tests/lint_result_cache.das b/utils/lint/tests/lint_result_cache.das index 65a46668d3..c00e70d030 100644 --- a/utils/lint/tests/lint_result_cache.das +++ b/utils/lint/tests/lint_result_cache.das @@ -1,45 +1,13 @@ options gen2 options indenting = 4 -require dastest/testing_boost public +require _lint_child public -require strings -require daslib/strings_boost require daslib/fio -var private _tmp_dir : string -var private _tmp_dir_inited = false - -def private tmp_root() : string { - if (!_tmp_dir_inited) { - let r = create_temp_directory_result("lint_result_cache") - if (!(r is value)) { - panic("could not create temp dir for the lint cache test: {unsafe(r.error)}") - } - _tmp_dir = unsafe(r.value) - _tmp_dir_inited = true - } - return _tmp_dir -} - -[finalize] -def cleanup_temp_dir { - if (_tmp_dir_inited && !empty(_tmp_dir)) { - rmdir_rec(_tmp_dir) - } -} - -def private plant(dir, name, content : string) { - fopen(path_join(dir, name), "wb") $(f) { - if (f != null) { - fwrite(f, content) - } - } -} - //! A tree of three files: `a.das` requires the module `b.das`, `c.das` carries a LINT003 finding def private build_tree(name : string) : string { - let root = path_join(tmp_root(), name) + let root = path_join(test_temp_dir(), name) mkdir_rec(root) plant(root, "b.das", "options gen2\n\nmodule b public\n\ndef public twice(x : int) : int => x * 2\n") plant(root, "a.das", "options gen2\n\nrequire b\n\n[export]\ndef main \{\n print(\"\{twice(2)\}\\n\")\n\}\n") @@ -47,48 +15,6 @@ def private build_tree(name : string) : string { return root } -def private das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} - -//! `host`: daslang's flags before the script; `extra`: after `--` -def private run_lint(root : string; extra, host : array) : string { - var argv <- [das_exe(), "-dasroot", get_das_root()] - argv |> push_from(host) - argv |> push("{get_das_root()}/utils/lint/main.das") - argv |> push("--") - argv |> push(root) - argv |> push("--cache-dir") - argv |> push(path_join(root, "cache")) - argv |> push_from(extra) - var output : string - unsafe(popen_argv(argv, 0.0, $(f) { - if (f != null) { - output = fread(f) - } - })) - return output -} - -def private entry_count(root : string) : int { - var n = 0 - dir(path_join(root, "cache")) $(name) { - n++ if (name != "." && name != ".." && name != "lint_hash.txt") - } - return n -} - -def private status_of(out, root, file : string) : string { - let path = to_generic_path(path_join(root, file)) - for (line in split(out, "\n")) { - for (status in ["PASS", "CACHED", "WARN"]) { - return status if (line |> starts_with("{status} ") && find(to_generic_path(line), path) >= 0) - } - } - return "?" -} - [test] def test_lint_result_cache(t : T?) { t |> run("a second run takes clean files from the cache, never a file with findings") @(t : T?) { diff --git a/utils/lint/tests/lint_shared_modules.das b/utils/lint/tests/lint_shared_modules.das new file mode 100644 index 0000000000..5774639c10 --- /dev/null +++ b/utils/lint/tests/lint_shared_modules.das @@ -0,0 +1,114 @@ +options gen2 +options indenting = 4 + +require _lint_child public + +require strings +require daslib/fio + +//! A tree of five files over one shared module: `a.das` and `c.das` +//! require the shared `b.das`, `sub/e.das` requires it by file path, +//! `c.das` carries a LINT003 finding, `d.das` requires only a builtin +//! module; the directory walk visits `a` before `b`, so the require +//! order has to move `b` ahead +def private build_tree(name : string) : string { + let root = path_join(test_temp_dir(), name) + mkdir_rec(root) + plant(root, "b.das", "options gen2\n\nmodule b shared public\n\n" + + "def public twice(x : int) : int => x * 2\n") + plant(root, "a.das", "options gen2\n\nrequire b\n\n[export]\ndef main \{\n" + + " print(\"\{twice(2)\}\\n\")\n\}\n") + plant(root, "c.das", "options gen2\n\nrequire b public\n\n[export]\ndef main \{\n" + + " var x = twice(13)\n print(\"\{x\}\\n\")\n\}\n") + plant(root, "d.das", "options gen2\n\nrequire strings\n\n[export]\ndef main \{\n" + + " print(\"\{to_upper(\"d\")\}\\n\")\n\}\n") + mkdir_rec(path_join(root, "sub")) + plant(path_join(root, "sub"), "e.das", "options gen2\n\nrequire ../b.das\n\n[export]\n" + + "def main \{\n print(\"\{twice(3)\}\\n\")\n\}\n") + return root +} + +//! the promoted-module count on a file's status line, -1 where the +//! line carries none +def private promoted_of(out, root, file : string) : int { + var line = "" + return -1 if (line_of(out, root, file, ["PASS ", "WARN "], line) < 0) + let at = find(line, "(promoted ") + return at < 0 ? -1 : to_int(slice(line, at + 10, find(line, ")", at))) +} + +//! the position of `file`'s "checking" line in the run's output, -1 +//! when the run never reached it +def private checked_at(out, root, file : string) : int { + var line = "" + return line_of(out, root, file, ["checking "], line) +} + +def private run_lint(root : string; extra : array) : string => run_lint(root, extra, []) + +[test] +def test_lint_shared_modules(t : T?) { + t |> run("a run reads as one without sharing, counts its promoted modules, caches") @(t : T?) { + let root = build_tree("same") + let plain = run_lint(root, ["--no-cache", "--no-shared-modules"]) + let pooled = run_lint(root, []) + for (file in ["a.das", "b.das", "c.das", "d.das"]) { + t |> equal(status_of(pooled, root, file), status_of(plain, root, file), + "{file} reads alike\nplain:\n{plain}\nshared:\n{pooled}") + } + t |> equal(status_of(pooled, root, "c.das"), "WARN", "the finding survives\n{pooled}") + t |> success(find(pooled, "LINT003") >= 0, "the finding is the same rule\n{pooled}") + t |> equal(entry_count(root), 4, "the clean files' entries land: a, b, d and sub/e") + let again = run_lint(root, []) + t |> equal(status_of(again, root, "a.das"), "CACHED", "a clean file is a hit\n{again}") + t |> equal(status_of(again, root, "c.das"), "WARN", "a finding lints every run\n{again}") + let d_took = promoted_of(pooled, root, "d.das") + t |> equal(d_took, 0, "d.das requires none of the set; builtin counts nothing\n{pooled}") + t |> success(promoted_of(pooled, root, "a.das") > d_took, + "a.das's compile took its module promoted from the process\n{pooled}") + t |> success(promoted_of(pooled, root, "c.das") > d_took, + "c.das's compile took its module promoted from the process\n{pooled}") + t |> equal(promoted_of(plain, root, "a.das"), -1, "no sharing, no count\n{plain}") + } + t |> run("a module lints ahead of the files that require it") @(t : T?) { + let root = build_tree("order") + let pooled = run_lint(root, []) + let b_at = checked_at(pooled, root, "b.das") + t |> success(b_at >= 0, "b.das was linted\n{pooled}") + t |> success(b_at < checked_at(pooled, root, "a.das"), "b.das before a.das\n{pooled}") + t |> success(b_at < checked_at(pooled, root, "c.das"), "b.das before c.das\n{pooled}") + t |> success(b_at < checked_at(pooled, root, "sub/e.das"), + "b.das before sub/e.das, which requires it by file path\n{pooled}") + t |> equal(status_of(pooled, root, "sub/e.das"), "PASS", "the file-path require\n{pooled}") + t |> equal(status_of(pooled, root, "b.das"), "PASS", + "the module lints clean after its users' compiles promote it\n{pooled}") + } + t |> run("--exclude leaves a folder out of the walk") @(t : T?) { + let root = build_tree("exclude") + mkdir_rec(path_join(root, "skip")) + plant(path_join(root, "skip"), "e.das", "options gen2\n\n[export]\ndef main \{\n" + + " var x = 13\n print(\"\{x\}\\n\")\n\}\n") + let walked = run_lint(root, ["--no-cache"]) + t |> equal(status_of(walked, root, "skip/e.das"), "WARN", "the finding is seen\n{walked}") + let left_out = run_lint(root, ["--no-cache", "--exclude", path_join(root, "skip")]) + t |> equal(status_of(left_out, root, "skip/e.das"), "?", "never linted\n{left_out}") + t |> equal(status_of(left_out, root, "c.das"), "WARN", "the rest lints\n{left_out}") + } + t |> run("a parallel run shares inside each worker") @(t : T?) { + let root = path_join(test_temp_dir(), "wide") + mkdir_rec(root) + plant(root, "b.das", "options gen2\n\nmodule b shared public\n\n" + + "def public twice(x : int) : int => x * 2\n") + for (i in range(16)) { // past the floor below which a run stays serial + plant(root, "a{i}.das", "options gen2\n\nrequire b\n\n[export]\ndef main \{\n" + + " print(\"\{twice({i})\}\\n\")\n\}\n") + } + let out = run_lint(root, ["--no-cache", "--workers", "2"]) + t |> equal(status_of(out, root, "b.das"), "PASS", "the module in its worker\n{out}") + for (i in range(16)) { + t |> equal(status_of(out, root, "a{i}.das"), "PASS", "a{i}.das in its worker\n{out}") + t |> success(promoted_of(out, root, "a{i}.das") >= 1, + "a{i}.das's compile took its module promoted from its worker's process\n{out}") + } + } +} From 6e0108dd45551007c44bce75f89341df611e5b23 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Thu, 22 Oct 2026 15:35:19 -0700 Subject: [PATCH 2/5] lint: the optional-require test's nolint survives a run that promoted its fixture. The test requires a fixture module declared shared under a %/ guard and references it only under static_if over builtin_module_exists; a lint run that already promoted the fixture answers that guard true, the reference counts, STYLE030 stays silent and LINT019 reports the nolint stale, which the CI tree chunk showed once the compiles of one run shared their modules. The nolint lists LINT019 for that mode, as the finding text names, and the lint README states the rule for a file whose code keys on builtin_module_exists of a shared module. Reproduced and verified on the five files in both modes. Co-Authored-By: Claude Fable 5.1 --- tests/language/optional_require.das | 2 +- utils/lint/README.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/language/optional_require.das b/tests/language/optional_require.das index 1584ff7d20..8b0b031421 100644 --- a/tests/language/optional_require.das +++ b/tests/language/optional_require.das @@ -18,7 +18,7 @@ require ?daslib/fio optional_require_fixture_pathguard // path guard reso require ?daslib/no_such_file_xyz optional_require_fixture_skipme // path guard unresolvable -> SKIPPED even though the target file resolves // %/-rooted guards: grammar-legal require_module_name forms the pre-parse require scanner must // scan identically to ast_requireModule (it once stopped at '%', mangling guard AND target) -require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard // nolint:STYLE030 — %/ guard resolves -> target loaded; referenced only under static_if +require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard // nolint:STYLE030,LINT019 — %/ guard resolves -> target loaded; referenced only under static_if, whose builtin_module_exists guard is true once a lint run promoted the shared fixture require ?%/tests/language/no_such_file_xyz.das optional_require_fixture_pctskip // %/ guard unresolvable -> SKIPPED, target file resolves require optional_require_fixture_private // requires optional_require_fixture_hidden PRIVATELY: the module_exists visibility pin below diff --git a/utils/lint/README.md b/utils/lint/README.md index d1958a7787..6bcd88788a 100644 --- a/utils/lint/README.md +++ b/utils/lint/README.md @@ -38,6 +38,9 @@ file that requires it, read off the sources' `require` lines (a require names a name): a module promoted before its own compile would collide with it. A parallel run shares inside each worker, over the chunk it holds. The result cache's entries hash the same module sources either way. `--no-shared-modules` compiles every file's modules in its own group, as the runner did before. +A file whose code keys on `builtin_module_exists` of a `shared` module - a `static_if` over the +guard - compiles differently once the run promoted that module, as it does under dastest, so a +`nolint` that one mode needs lists `LINT019` for the other. CI's chunked `check_lint_tree` leaves `modules/dasLLAMA` out, whose every file compiles the engine's whole graph, and `check_lint_dasllama` lints that module as one process. From 977959b27645c8c58e2bda9d6fdbb35c65cf4c97 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Thu, 22 Oct 2026 16:06:10 -0700 Subject: [PATCH 3/5] lint: a promoted verify module leaves a program that did not ask for it alone. The post-compile macros run over the process's global module list with no visibility guard, so once one compile of a run promoted daslib/ast_verify (the ast-fuzz self-test fixtures require it), its verify ran on every later program of the process - the CI tree chunk hit the self-test fixture that strips its own locations on purpose, which the verify then reported. The verify's post-compile pass now runs only where the program's own library holds the module, a require or --ast-verify's extra dependency; the fuzz tests still see every report under the flag, and a new cell lints the verify fixture ahead of the breaking one in one shared run. The lint README states the mechanism. Co-Authored-By: Claude Fable 5.1 --- daslib/ast_verify.das | 15 ++++++++++++++- utils/internal/ast-fuzz/test_ast_fuzz.das | 13 +++++++++++++ utils/lint/README.md | 4 +++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/daslib/ast_verify.das b/daslib/ast_verify.das index 80feef2d8a..8b9b6f5136 100644 --- a/daslib/ast_verify.das +++ b/daslib/ast_verify.das @@ -1702,10 +1702,23 @@ class AstInferPassVerifyMacro : AstPassMacro { } } +//! the post-compile pass of a promoted copy reaches every later program of the process - the +//! walk is over the global module list - so only a program that holds this module in its own +//! library (a `require`, or `--ast-verify`'s extra dependency) is verified +def private program_asked_for_verify(prog : ProgramPtr) : bool { + var asked = false + program_for_each_module(prog) $(m) { + if (m.name == "ast_verify") { + asked = true + } + } + return asked +} + [post_compile_macro] class AstPostCompileVerifyMacro : AstPassMacro { def override apply(var prog : ProgramPtr; mod : Module?) : bool { - if (mod != null && !mod.moduleFlags.builtIn) { + if (mod != null && !mod.moduleFlags.builtIn && program_asked_for_verify(prog)) { verify_module(prog, mod) if (batch_mode()) { verify_module_after_infer(prog, mod) diff --git a/utils/internal/ast-fuzz/test_ast_fuzz.das b/utils/internal/ast-fuzz/test_ast_fuzz.das index 14201eb21b..e029526832 100644 --- a/utils/internal/ast-fuzz/test_ast_fuzz.das +++ b/utils/internal/ast-fuzz/test_ast_fuzz.das @@ -107,6 +107,19 @@ def test_ast_verify_guesses_a_location_for_an_unset_at(t : T?) { "the search walked into an assume alias it should not see\n{out}") } +// A promoted verify module's post-compile pass reaches every later program of the process; a +// program that never required it is left alone. The lint runner shares its compiles' modules, so +// linting a fixture that requires the verify ahead of one that breaks its own tree is the shape. +[test] +def test_ast_verify_leaves_a_program_that_did_not_ask(t : T?) { + var out : string + let d = "utils/internal/ast-fuzz/selftest" + let ec = run([BIN, "utils/lint/main.das", "--", "{d}/verify_breaker.das", "{d}/at_unset.das", "--no-cache"], out, 300.0) + t |> success(!died(ec, out), "the lint crashed (exit {ec})\n{out}") + t |> success(find(out, "{d}/at_unset.das") >= 0 && find(out, "FAIL") < 0, + "the promoted verify reached a program that did not require it\n{out}") +} + [test] def test_ast_verify_clean_on_valid_code(t : T?) { var out : string diff --git a/utils/lint/README.md b/utils/lint/README.md index 6bcd88788a..6476fd067c 100644 --- a/utils/lint/README.md +++ b/utils/lint/README.md @@ -40,7 +40,9 @@ each worker, over the chunk it holds. The result cache's entries hash the same m way. `--no-shared-modules` compiles every file's modules in its own group, as the runner did before. A file whose code keys on `builtin_module_exists` of a `shared` module - a `static_if` over the guard - compiles differently once the run promoted that module, as it does under dastest, so a -`nolint` that one mode needs lists `LINT019` for the other. +`nolint` that one mode needs lists `LINT019` for the other. A promoted module's post-compile +macros run on every later program of the process (the walk is over the global module list), so +such a macro guards on the program's own library, as `daslib/ast_verify` does. CI's chunked `check_lint_tree` leaves `modules/dasLLAMA` out, whose every file compiles the engine's whole graph, and `check_lint_dasllama` lints that module as one process. From 5f9c0410350109e1ae81d995b0058f0487dae0a0 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Thu, 22 Oct 2026 17:11:36 -0700 Subject: [PATCH 4/5] lint: the require order releases a cycle instead of panicking. The order the shared run lints files in is read off the sources' require lines, which spell requires the compiler never takes - a guarded one, a commented one - so daslib read as a cycle and topological_sort's panic took the whole run down (the bundle smoke's lint of the installed daslib). The walk is the runner's own Kahn's now: a node is free once nothing it waits on is left, and a cycle releases the lowest index left, since a true cycle compiles nowhere. The optional-require test suppresses STYLE030 for itself - every guarded require there is the test - in place of the per-line nolints the two modes read apart, and the comments the line-length gate and STYLE014 reported on the verify guard and the fuzz cell are trimmed. Co-Authored-By: Claude Fable 5.1 --- daslib/ast_verify.das | 6 ++--- tests/language/optional_require.das | 7 ++++-- utils/internal/ast-fuzz/test_ast_fuzz.das | 11 +++++---- utils/lint/README.md | 3 ++- utils/lint/main.das | 27 +++++++++++++++++++++-- 5 files changed, 42 insertions(+), 12 deletions(-) diff --git a/daslib/ast_verify.das b/daslib/ast_verify.das index 8b9b6f5136..ca20601099 100644 --- a/daslib/ast_verify.das +++ b/daslib/ast_verify.das @@ -1702,9 +1702,9 @@ class AstInferPassVerifyMacro : AstPassMacro { } } -//! the post-compile pass of a promoted copy reaches every later program of the process - the -//! walk is over the global module list - so only a program that holds this module in its own -//! library (a `require`, or `--ast-verify`'s extra dependency) is verified +//! a promoted copy's post-compile pass reaches every later program of +//! the process (the walk is over the global module list), so only a +//! program holding this module in its own library is verified def private program_asked_for_verify(prog : ProgramPtr) : bool { var asked = false program_for_each_module(prog) $(m) { diff --git a/tests/language/optional_require.das b/tests/language/optional_require.das index 8b0b031421..979393791b 100644 --- a/tests/language/optional_require.das +++ b/tests/language/optional_require.das @@ -1,4 +1,5 @@ options gen2 +options _nolint = "STYLE030" // every guarded require here IS the test, used or not require dastest/testing_boost public // Optional require: `require ?guard target` loads *target* when *guard* is available. A @@ -9,7 +10,7 @@ require dastest/testing_boost public // guard-present-but-target-missing case still errors with 20605 (verified by hand; // module-resolution failures fire before the `expect` line parses, so that case is not // expressible as a cant_ test). -require ?math math // nolint:STYLE030 — the guarded-load mechanics ARE the test: guard present (math is a core builtin) -> target loaded +require ?math math // guard present (a builtin) -> target loaded require ?no_such_guard_xyz no_such_target_also_xyz // guard absent, target unresolvable -> skipped require ?no_such_guard_pkg optional_require_fixture // guard absent -> SKIPPED even though the target resolves (strict plain guards); the non-load IS the test // PATH guard (contains '/'): availability = the guard's OWN file resolves; the rail for @@ -18,7 +19,9 @@ require ?daslib/fio optional_require_fixture_pathguard // path guard reso require ?daslib/no_such_file_xyz optional_require_fixture_skipme // path guard unresolvable -> SKIPPED even though the target file resolves // %/-rooted guards: grammar-legal require_module_name forms the pre-parse require scanner must // scan identically to ast_requireModule (it once stopped at '%', mangling guard AND target) -require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard // nolint:STYLE030,LINT019 — %/ guard resolves -> target loaded; referenced only under static_if, whose builtin_module_exists guard is true once a lint run promoted the shared fixture +// %/ guard resolves -> loaded; referenced under static_if alone, +// whose builtin_module_exists is true once a run promoted the fixture +require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard require ?%/tests/language/no_such_file_xyz.das optional_require_fixture_pctskip // %/ guard unresolvable -> SKIPPED, target file resolves require optional_require_fixture_private // requires optional_require_fixture_hidden PRIVATELY: the module_exists visibility pin below diff --git a/utils/internal/ast-fuzz/test_ast_fuzz.das b/utils/internal/ast-fuzz/test_ast_fuzz.das index e029526832..5ead78de51 100644 --- a/utils/internal/ast-fuzz/test_ast_fuzz.das +++ b/utils/internal/ast-fuzz/test_ast_fuzz.das @@ -107,14 +107,17 @@ def test_ast_verify_guesses_a_location_for_an_unset_at(t : T?) { "the search walked into an assume alias it should not see\n{out}") } -// A promoted verify module's post-compile pass reaches every later program of the process; a -// program that never required it is left alone. The lint runner shares its compiles' modules, so -// linting a fixture that requires the verify ahead of one that breaks its own tree is the shape. +// A promoted verify's post-compile pass reaches every later program +// of the process; one that never required it is left alone. The lint +// runner shares modules, so the verify fixture ahead of the breaking +// one in one run is the shape. [test] def test_ast_verify_leaves_a_program_that_did_not_ask(t : T?) { var out : string let d = "utils/internal/ast-fuzz/selftest" - let ec = run([BIN, "utils/lint/main.das", "--", "{d}/verify_breaker.das", "{d}/at_unset.das", "--no-cache"], out, 300.0) + let argv <- [BIN, "utils/lint/main.das", "--", + "{d}/verify_breaker.das", "{d}/at_unset.das", "--no-cache"] + let ec = run(argv, out, 300.0) t |> success(!died(ec, out), "the lint crashed (exit {ec})\n{out}") t |> success(find(out, "{d}/at_unset.das") >= 0 && find(out, "FAIL") < 0, "the promoted verify reached a program that did not require it\n{out}") diff --git a/utils/lint/README.md b/utils/lint/README.md index 6476fd067c..6f9fd28b27 100644 --- a/utils/lint/README.md +++ b/utils/lint/README.md @@ -40,7 +40,8 @@ each worker, over the chunk it holds. The result cache's entries hash the same m way. `--no-shared-modules` compiles every file's modules in its own group, as the runner did before. A file whose code keys on `builtin_module_exists` of a `shared` module - a `static_if` over the guard - compiles differently once the run promoted that module, as it does under dastest, so a -`nolint` that one mode needs lists `LINT019` for the other. A promoted module's post-compile +`nolint` that one mode needs lists `LINT019` for the other, or the file suppresses the rule for +itself (`options _nolint`). A promoted module's post-compile macros run on every later program of the process (the walk is over the global module list), so such a macro guards on the program's own library, as `daslib/ast_verify` does. CI's chunked `check_lint_tree` leaves `modules/dasLLAMA` out, whose every file compiles the engine's diff --git a/utils/lint/main.das b/utils/lint/main.das index 715285df86..e7d23046c7 100644 --- a/utils/lint/main.das +++ b/utils/lint/main.das @@ -16,7 +16,6 @@ require math require strings require daslib/strings_boost require daslib/rtti -require daslib/algorithm // topological_sort: the --shared-modules file order require daslib/module_group // module_group_module_name: a require path's module require dastest/review_gate // das_requires: a file's require paths require ../common/parallel_workers.das @@ -1363,7 +1362,31 @@ def private require_order(files : array) : array { } nodes |> emplace(node) } - return <- [for (node in topological_sort(nodes)); node.id] + return <- release_order(nodes) +} + +//! Kahn's walk over `nodes`, a node free once nothing it waits on is +//! left; a cycle (a require the sources spell and the compiler never +//! takes; a true one compiles nowhere) releases the lowest index left +def private release_order(var nodes : array) : array { + var order : array + order |> reserve(length(nodes)) + while (!empty(nodes)) { + var pick = 0 + for (node, k in nodes, count()) { + if (empty(node.before)) { + pick = k + break + } + } + let id = nodes[pick].id + nodes |> erase(pick) + for (other in nodes) { + other.before |> erase(id) + } + order |> push(id) + } + return <- order } //! the order a run lints `files` in: the require order under sharing, From 028ba74cb007fd1c0c71744670812024d08b1dd1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Thu, 22 Oct 2026 18:24:00 -0700 Subject: [PATCH 5/5] doc-verify: a probe that times out says so. A companion probe compiles an engine's whole module graph cold - the dasLLAMA facade takes 56 s on a 3990X - against a 120 s timeout, and a timeout read as a plain compile failure, which the tool then reported as a binary gap, "build the missing modules and re-run", on a macOS runner that had every module. The probe timeout is 300 s, and a timeout names itself with the seconds. Co-Authored-By: Claude Fable 5.1 --- utils/internal/doc-verify/main.das | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/utils/internal/doc-verify/main.das b/utils/internal/doc-verify/main.das index 278c01c2a8..039c880200 100644 --- a/utils/internal/doc-verify/main.das +++ b/utils/internal/doc-verify/main.das @@ -1050,9 +1050,13 @@ def compile_file_rc(cfg : Config; path : string; var output : string&) : int { //! -dasroot names the tree whose modules the page's requires resolve from: the audited //! root, not the binary's own tree, so a worktree session checks its own sources let args <- [normalize(cfg.daslang), "-dasroot", normalize(cfg.root), "-compile-only", normalize(path)] - return run_and_capture(args, output, 120.0) + return run_and_capture(args, output, PROBE_TIMEOUT_SEC) } +//! a companion probe compiles an engine's whole module graph cold: +//! the dasLLAMA facade takes a minute on a fast box, more on a runner +let PROBE_TIMEOUT_SEC = 300.0 + def spawn_failure(cfg : Config) : string => "cannot spawn '{cfg.daslang}' - pass --daslang " @@ -1179,7 +1183,10 @@ def run_probes(require_lines : table; file_mods : table; c var out : string let prc = compile_file_rc(cfg, pfile, out) if (prc != 0) { - let why = prc == -1 ? " ({spawn_failure(cfg)})" : "" + var why = prc == -1 ? " ({spawn_failure(cfg)})" : "" + if (prc == popen_timed_out) { + why = " (timed out after {int(PROBE_TIMEOUT_SEC)} s: a slow box, no module missing)" + } failures |> push((origins[idx] ? "[companion] " : "[page] ") + trim(fread(pfile)) + why) } }