diff --git a/CMakeLists.txt b/CMakeLists.txt index c80de2775d..4609ef46b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2537,7 +2537,7 @@ if(TARGET daslang) COMMAND ${CMAKE_COMMAND} -E copy ${_identity} ${DAS_TEST_CACHE_DIR}) add_dependencies(dastest_cache_guard ${_das_test_binaries}) endif() -foreach(_t check_lint_tree check_vulkan run_tests_vulkan run_tests_imgui_integration +foreach(_t check_lint_tree check_lint_dasllama check_vulkan run_tests_vulkan run_tests_imgui_integration build_site_wasm check_playground_samples run_imgui2rst run_vulkan2rst) if(TARGET ${_t} AND _das_shared_modules) add_dependencies(${_t} ${_das_shared_modules}) diff --git a/ci/CMakeLists.txt b/ci/CMakeLists.txt index 51741beaab..5537a4e989 100644 --- a/ci/CMakeLists.txt +++ b/ci/CMakeLists.txt @@ -77,7 +77,7 @@ set(DAS_EXTENDED_CORE check_workflow_matrix check_shipped_skills check_md_ascii check_wheel_repack check_manifest_render check_example_games check_dasllama_site run_tests_bench_updater run_tests_pr_babysit check_dasgen run_tests_review_md run_tests_dastest - check_lint_tree check_ast_verify_changed check_line_length check_gated_das check_review_gates + check_lint_tree check_lint_dasllama check_ast_verify_changed check_line_length check_gated_das check_review_gates check_standalone_exes check_format check_format_exe run_utils_tests) if(NOT WIN32) # check_model_mint: its test stands in a shell script for daslang, which Windows cannot exec; diff --git a/daslib/ast_verify.das b/daslib/ast_verify.das index 80feef2d8a..ca20601099 100644 --- a/daslib/ast_verify.das +++ b/daslib/ast_verify.das @@ -1702,10 +1702,23 @@ class AstInferPassVerifyMacro : AstPassMacro { } } +//! a promoted copy's post-compile pass reaches every later program of +//! the process (the walk is over the global module list), so only a +//! program holding this module in its own library is verified +def private program_asked_for_verify(prog : ProgramPtr) : bool { + var asked = false + program_for_each_module(prog) $(m) { + if (m.name == "ast_verify") { + asked = true + } + } + return asked +} + [post_compile_macro] class AstPostCompileVerifyMacro : AstPassMacro { def override apply(var prog : ProgramPtr; mod : Module?) : bool { - if (mod != null && !mod.moduleFlags.builtIn) { + if (mod != null && !mod.moduleFlags.builtIn && program_asked_for_verify(prog)) { verify_module(prog, mod) if (batch_mode()) { verify_module_after_infer(prog, mod) diff --git a/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst b/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst index 0a7f0c7808..eb93ba4e17 100644 --- a/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst +++ b/doc/source/stdlib/handmade/structure_annotation-rtti-CodeOfPolicies.rst @@ -113,3 +113,4 @@ Path to linker, which is used in JIT. compile_file from a script reads and refreshes the default module cache around this compile (keyed by the file, the running binary, the host arguments and these policies). Interpreter only: caps how deep fastcall (frameless) calls may nest, panicking past the cap instead of exhausting the native stack; 0 leaves fastcall unchecked. Host-side counterpart of ``options max_fast_call_depth`` (the option overrides the policy). Keeps the used-function set and the function and variable indices alive after the main simulate. Off by default, they are freed when rtti is off; a host that calls ``is_used``, ``function_index`` or ``variable_index`` on the program after simulate sets it. +A compile under ``no_optimizations`` promotes the shared modules it requires into the process, as an optimizing compile does, so a later compile in the process reuses them; for a process that compiles many programs under one policy set and runs none of them (the lint). Off by default. diff --git a/include/daScript/ast/ast_serializer.h b/include/daScript/ast/ast_serializer.h index 6d5e753e5e..34a780bcc2 100644 --- a/include/daScript/ast/ast_serializer.h +++ b/include/daScript/ast/ast_serializer.h @@ -365,7 +365,7 @@ namespace das { AstSerializer & serializeModule ( Module & module, bool already_exists ); static constexpr uint32_t getVersion () { - return 232; // 232: an expression whose source type is unresolved stays untyped until the type resolves + return 233; // 233: CodeOfPolicies carries promote_shared_unoptimized } void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept; diff --git a/include/daScript/simulate/REVIEW.md b/include/daScript/simulate/REVIEW.md index 1a6ff7b827..a948b192e1 100644 --- a/include/daScript/simulate/REVIEW.md +++ b/include/daScript/simulate/REVIEW.md @@ -24,6 +24,12 @@ annotation appends it after that type's last member, or bumps `LLVM_JIT_CODEGEN_ members it read by offset; a member inserted before them moves every later one under code that still uses the old offsets. +**A diff that changes `CodeOfPolicies`' layout or a field's meaning without changing its size - a +reorder, a repurpose, a member that lands in a padding pocket, the appended `bool` among them - +bumps `DAS_POLICIES_VERSION` (`code_of_policies.h`) in the same change.** The ABI stamp folds the +version and the size alone, so a host built on the old headers passes the stamp check and the +library reads the new field off the host's padding. + **A diff that hashes a table key hashes a builtin key type - one in `heap.h`'s `makeTableKeyValueNode` list - as itself through `hash_function(context, key)` (`hash.h`), and a handled key type as the value type its annotation's `makeValueType()` returns.** A diff --git a/include/daScript/simulate/code_of_policies.h b/include/daScript/simulate/code_of_policies.h index 2bf2c766db..76e0ddf1cb 100644 --- a/include/daScript/simulate/code_of_policies.h +++ b/include/daScript/simulate/code_of_policies.h @@ -7,7 +7,7 @@ namespace das { // bump when CodeOfPolicies changes layout or meaning without changing size — a field // reorder/repurpose, or an insert that lands in a padding pocket (sizeof can't see those) - #define DAS_POLICIES_VERSION 3 + #define DAS_POLICIES_VERSION 4 // per-binary linkage for the ABI-stamp functions: at -O0 gcc/clang emit them as weak // default-visibility symbols, and the dynamic linker would bind every binary to the FIRST @@ -174,6 +174,7 @@ namespace das { bool module_cache = false; // a script's compile_file runs under the default module cache /*option*/ uint32_t max_fast_call_depth = 0; // interpreter only; 0 = unchecked. bool keep_symbol_tables = false; // keep usedFunctions and the function/variable indices past the main simulate + bool promote_shared_unoptimized = false; // src/ast/ARCHITECTURE_SHARED.md#shared-promotion // the abi_stamp truth (a member so the NSDMI above can call it - complete-class context). // Low byte 0 keeps a pre-stamp libDaScript reading this word seeing aot == false - diff --git a/skills/perf_lint.md b/skills/perf_lint.md index 156cf8eba2..8cdf6e862b 100644 --- a/skills/perf_lint.md +++ b/skills/perf_lint.md @@ -13,11 +13,14 @@ message - fix the text in `daslib/perf_lint.das`, don't document around it. ```bash bin/daslang utils/lint/main.das -- file.das [dir ...] \ [--quiet] [--silent] [--perf-only|--style-only|--paranoid-only] \ - [--disable CODE,...] [--enable CODE,...] [--workers N] [--fix] + [--disable CODE,...] [--enable CODE,...] [--workers N] [--fix] \ + [--no-shared-modules] [--exclude PATH ...] ``` The MCP `lint` tool runs the same three passes. `--enable` is a whitelist (only the listed rules -run); on overlap `--disable` wins. `--fix` writes the edits rules attach to their findings and +run); on overlap `--disable` wins. A run compiles each shared module once per process and lints +the files in require order; `--no-shared-modules` compiles every file's modules in its own group. +`--exclude PATH` leaves a folder out of a directory walk. `--fix` writes the edits rules attach to their findings and re-lints; a fix that breaks the compile is rolled back. Pair it with `--enable CODE` to sweep one rule, and review the diff - a fix only proves the file still compiles. diff --git a/src/ast/ARCHITECTURE_SHARED.md b/src/ast/ARCHITECTURE_SHARED.md index 4cdb6a2740..e84db538e1 100644 --- a/src/ast/ARCHITECTURE_SHARED.md +++ b/src/ast/ARCHITECTURE_SHARED.md @@ -20,5 +20,11 @@ program requires a daslib module first is the filesystem's choice: a lint test t fixture with optimizations off runs before `tests/linq` on APFS and after it on ext4 and NTFS. A compile with optimizations off still reuses a module promoted earlier, and `requireModuleNow` (`ARCHITECTURE.md#require-after-walk`) promotes its target whatever the policy, since a late -require asks for a process module by name. The lint runner, the MCP server and the LSP set -`ignore_shared_modules` beside `no_optimizations`, so they neither promote nor reuse. +require asks for a process module by name. The MCP server, the LSP and the lint runner outside its +shared-modules mode set `ignore_shared_modules` beside `no_optimizations`, so they neither promote +nor reuse. `promote_shared_unoptimized` is the override: a compile under `no_optimizations` that sets it +promotes its shared dependencies as an optimizing compile would, for a process that compiles many +programs under one policy set and never serializes them - the lint's shared-modules mode, where +the unoptimized body serves every later file and the AOT-stub mismatch has nothing to reach. A +program the process compiles AFTER such a promotion reuses that unoptimized body, so the override +belongs to a process that runs nothing. diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index e0ba3a678a..99c63cfe2f 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -2356,7 +2356,7 @@ namespace das { } } // src/ast/ARCHITECTURE_SHARED.md#shared-promotion - if ( auto failed = parseRequiredModules(walk.req, access, logs, libGroup, policies, !policies.no_optimizations) ) { + if ( auto failed = parseRequiredModules(walk.req, access, logs, libGroup, policies, !policies.no_optimizations || policies.promote_shared_unoptimized) ) { return failed; } auto & serializer_read = daScriptEnvironment::getBound()->serializer_read; diff --git a/src/builtin/module_builtin_ast_serialize.cpp b/src/builtin/module_builtin_ast_serialize.cpp index a479c602fe..1ea49eaf8a 100644 --- a/src/builtin/module_builtin_ast_serialize.cpp +++ b/src/builtin/module_builtin_ast_serialize.cpp @@ -3252,7 +3252,7 @@ namespace das { X(no_global_variables) X(no_global_variables_at_all) X(no_global_heap) X(only_fast_aot) \ X(aot_order_side_effects) X(no_unused_function_arguments) X(no_unused_block_arguments) \ X(allow_block_variable_shadowing) X(allow_local_variable_shadowing) X(allow_shared_lambda) \ - X(ignore_shared_modules) X(default_module_public) X(no_deprecated) X(no_aliasing) \ + X(ignore_shared_modules) X(promote_shared_unoptimized) X(default_module_public) X(no_deprecated) X(no_aliasing) \ X(strict_smart_pointers) X(no_init) X(strict_unsafe_delete) X(no_members_functions_in_struct) \ X(no_local_class_members) X(no_unsafe_uninitialized_structures) X(default_init_containers) \ X(strict_properties) X(no_writing_to_nameless) X(no_optimizations) X(no_fold_unsafe) X(fast_math) X(disable_dse) \ diff --git a/src/builtin/module_builtin_rtti.cpp b/src/builtin/module_builtin_rtti.cpp index d56b7ad20d..e17c9acd76 100644 --- a/src/builtin/module_builtin_rtti.cpp +++ b/src/builtin/module_builtin_rtti.cpp @@ -1028,6 +1028,7 @@ namespace das { addField("allow_local_variable_shadowing"); addField("allow_shared_lambda"); addField("ignore_shared_modules"); + addField("promote_shared_unoptimized"); addField("default_module_public"); addField("no_deprecated"); addField("no_aliasing"); diff --git a/tests/language/optional_require.das b/tests/language/optional_require.das index 1584ff7d20..979393791b 100644 --- a/tests/language/optional_require.das +++ b/tests/language/optional_require.das @@ -1,4 +1,5 @@ options gen2 +options _nolint = "STYLE030" // every guarded require here IS the test, used or not require dastest/testing_boost public // Optional require: `require ?guard target` loads *target* when *guard* is available. A @@ -9,7 +10,7 @@ require dastest/testing_boost public // guard-present-but-target-missing case still errors with 20605 (verified by hand; // module-resolution failures fire before the `expect` line parses, so that case is not // expressible as a cant_ test). -require ?math math // nolint:STYLE030 — the guarded-load mechanics ARE the test: guard present (math is a core builtin) -> target loaded +require ?math math // guard present (a builtin) -> target loaded require ?no_such_guard_xyz no_such_target_also_xyz // guard absent, target unresolvable -> skipped require ?no_such_guard_pkg optional_require_fixture // guard absent -> SKIPPED even though the target resolves (strict plain guards); the non-load IS the test // PATH guard (contains '/'): availability = the guard's OWN file resolves; the rail for @@ -18,7 +19,9 @@ require ?daslib/fio optional_require_fixture_pathguard // path guard reso require ?daslib/no_such_file_xyz optional_require_fixture_skipme // path guard unresolvable -> SKIPPED even though the target file resolves // %/-rooted guards: grammar-legal require_module_name forms the pre-parse require scanner must // scan identically to ast_requireModule (it once stopped at '%', mangling guard AND target) -require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard // nolint:STYLE030 — %/ guard resolves -> target loaded; referenced only under static_if +// %/ guard resolves -> loaded; referenced under static_if alone, +// whose builtin_module_exists is true once a run promoted the fixture +require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard require ?%/tests/language/no_such_file_xyz.das optional_require_fixture_pctskip // %/ guard unresolvable -> SKIPPED, target file resolves require optional_require_fixture_private // requires optional_require_fixture_hidden PRIVATELY: the module_exists visibility pin below diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 3ec785e0b1..8364ec42ab 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -21,12 +21,15 @@ add_custom_target(check_dasgen COMMENT "Checking generated builtin .inc files are up to date (dasgen)" ) -# check_lint_tree lints DAS_CHUNKS of the tree in parallel. LINT019 needs every compile's nolint +# check_lint_tree lints DAS_CHUNKS of the tree in parallel, each chunk one process that compiles the +# shared modules its files require once. LINT019 needs every compile's nolint set(_lint_roots daslib utils modules tutorials examples tests src) list(TRANSFORM _lint_roots PREPEND "${PROJECT_SOURCE_DIR}/" OUTPUT_VARIABLE _lint_globs) list(TRANSFORM _lint_globs APPEND "/*.das") file(GLOB_RECURSE _lint_files RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS ${_lint_globs}) list(FILTER _lint_files EXCLUDE REGEX "(^|/)(\\.[^/]*|node_modules|_[^/]*)(/|$)") +# modules/dasLLAMA lints in check_lint_dasllama - one process over the module instead of its engine graph compiled per chunk +list(FILTER _lint_files EXCLUDE REGEX "^modules/dasLLAMA/") list(LENGTH _lint_files _lint_count) math(EXPR _lint_size "(${_lint_count} + 127) / 128") DAS_CHUNKS(_lint_chunks ${_lint_size} ${_lint_files}) @@ -38,6 +41,10 @@ foreach(_n ${_lint_chunks}) list(APPEND _lint_targets lint_tree_chunk_${_n}) endforeach() add_custom_target(check_lint_tree DEPENDS ${_lint_targets}) +add_custom_target(check_lint_dasllama + COMMAND $ ./utils/lint/main.das -- modules/dasLLAMA -q --silent + DEPENDS daslang WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMENT "Linting modules/dasLLAMA in one process, its engine graph compiled once") set(_dupe_paths -p daslib -p utils -p src) file(GLOB _dupe_entries LIST_DIRECTORIES true RELATIVE ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/modules/*/*) diff --git a/utils/internal/ast-fuzz/test_ast_fuzz.das b/utils/internal/ast-fuzz/test_ast_fuzz.das index 14201eb21b..5ead78de51 100644 --- a/utils/internal/ast-fuzz/test_ast_fuzz.das +++ b/utils/internal/ast-fuzz/test_ast_fuzz.das @@ -107,6 +107,22 @@ def test_ast_verify_guesses_a_location_for_an_unset_at(t : T?) { "the search walked into an assume alias it should not see\n{out}") } +// A promoted verify's post-compile pass reaches every later program +// of the process; one that never required it is left alone. The lint +// runner shares modules, so the verify fixture ahead of the breaking +// one in one run is the shape. +[test] +def test_ast_verify_leaves_a_program_that_did_not_ask(t : T?) { + var out : string + let d = "utils/internal/ast-fuzz/selftest" + let argv <- [BIN, "utils/lint/main.das", "--", + "{d}/verify_breaker.das", "{d}/at_unset.das", "--no-cache"] + let ec = run(argv, out, 300.0) + t |> success(!died(ec, out), "the lint crashed (exit {ec})\n{out}") + t |> success(find(out, "{d}/at_unset.das") >= 0 && find(out, "FAIL") < 0, + "the promoted verify reached a program that did not require it\n{out}") +} + [test] def test_ast_verify_clean_on_valid_code(t : T?) { var out : string diff --git a/utils/internal/doc-verify/main.das b/utils/internal/doc-verify/main.das index 278c01c2a8..039c880200 100644 --- a/utils/internal/doc-verify/main.das +++ b/utils/internal/doc-verify/main.das @@ -1050,9 +1050,13 @@ def compile_file_rc(cfg : Config; path : string; var output : string&) : int { //! -dasroot names the tree whose modules the page's requires resolve from: the audited //! root, not the binary's own tree, so a worktree session checks its own sources let args <- [normalize(cfg.daslang), "-dasroot", normalize(cfg.root), "-compile-only", normalize(path)] - return run_and_capture(args, output, 120.0) + return run_and_capture(args, output, PROBE_TIMEOUT_SEC) } +//! a companion probe compiles an engine's whole module graph cold: +//! the dasLLAMA facade takes a minute on a fast box, more on a runner +let PROBE_TIMEOUT_SEC = 300.0 + def spawn_failure(cfg : Config) : string => "cannot spawn '{cfg.daslang}' - pass --daslang " @@ -1179,7 +1183,10 @@ def run_probes(require_lines : table; file_mods : table; c var out : string let prc = compile_file_rc(cfg, pfile, out) if (prc != 0) { - let why = prc == -1 ? " ({spawn_failure(cfg)})" : "" + var why = prc == -1 ? " ({spawn_failure(cfg)})" : "" + if (prc == popen_timed_out) { + why = " (timed out after {int(PROBE_TIMEOUT_SEC)} s: a slow box, no module missing)" + } failures |> push((origins[idx] ? "[companion] " : "[page] ") + trim(fread(pfile)) + why) } } diff --git a/utils/lint/README.md b/utils/lint/README.md index 60d3c56d81..6f9fd28b27 100644 --- a/utils/lint/README.md +++ b/utils/lint/README.md @@ -30,6 +30,26 @@ build of the runner never compiles, cannot check that version, and runs without those alone re-lints nothing until `--no-cache`. CI keeps the folder between runs, one per platform (`actions/cache` in `.github/workflows/extended_checks.yml`). +The compiles of one run share their modules: each promotes the shared modules it requires into the +process (`promote_shared_unoptimized`, `src/ast/ARCHITECTURE_SHARED.md#shared-promotion`), so every +module compiles once and each later file reuses it, and a status line carries the count the compile +took promoted (`PASS file (promoted 77)`). The files lint in require order, a module ahead of every +file that requires it, read off the sources' `require` lines (a require names a file by the file's +name): a module promoted before its own compile would collide with it. A parallel run shares inside +each worker, over the chunk it holds. The result cache's entries hash the same module sources either +way. `--no-shared-modules` compiles every file's modules in its own group, as the runner did before. +A file whose code keys on `builtin_module_exists` of a `shared` module - a `static_if` over the +guard - compiles differently once the run promoted that module, as it does under dastest, so a +`nolint` that one mode needs lists `LINT019` for the other, or the file suppresses the rule for +itself (`options _nolint`). A promoted module's post-compile +macros run on every later program of the process (the walk is over the global module list), so +such a macro guards on the program's own library, as `daslib/ast_verify` does. +CI's chunked `check_lint_tree` leaves `modules/dasLLAMA` out, whose every file compiles the engine's +whole graph, and `check_lint_dasllama` lints that module as one process. + +`--exclude ` (repeatable) leaves every file and folder under the path out of a run: the +directory walk, the runner's folder rules and a positional root under it alike. + Five rules are the runner's own, because they are about folders rather than code. Each runs once per invocation, over a walk of the directory roots the run was given. A directory whose `.lint_config` carries `[docs] rule_docs_only = true` may hold only rule documents diff --git a/utils/lint/main.das b/utils/lint/main.das index d17f6409f4..e7d23046c7 100644 --- a/utils/lint/main.das +++ b/utils/lint/main.das @@ -16,6 +16,8 @@ require math require strings require daslib/strings_boost require daslib/rtti +require daslib/module_group // module_group_module_name: a require path's module +require dastest/review_gate // das_requires: a file's require paths require ../common/parallel_workers.das require ../common/arch_citations.das @@ -31,6 +33,7 @@ struct LintResult { excluded : int // findings dropped by the .lint_config [paths] exclude policy fixed : int cached : bool // a clean result the cache vouched for: nothing was compiled + promoted_modules : int // counted under --shared-modules alone } [CommandLineArgs] @@ -64,6 +67,10 @@ struct Config { fix : bool @clarg_doc = "Skip the result cache: compile and lint every file, record nothing" no_cache : bool + @clarg_doc = "Compile each file's modules in its own group; the default shares them" + no_shared_modules : bool + @clarg_doc = "Leave every file and folder under this path out of the run. Repeatable" + exclude : array @clarg_doc = "Where the result cache lives (one entry per clean file: the content hash of every input its compile read)" cache_dir : string = LINT_CACHE_DEFAULT_DIR @@ -152,6 +159,13 @@ def is_skip_dir(name : string) : bool => name |> starts_with(".") || name == "no def is_build_tree(path : string) : bool => stat("{path}/CMakeCache.txt").is_valid || stat("{path}/.git").is_valid +var lint_exclude_roots : array //! --exclude's roots, absolute generic paths + +def is_excluded_path(path : string) : bool => + owning_root(to_generic_path(path), lint_exclude_roots) != "." + +def is_skip_tree(path : string) : bool => is_build_tree(path) || is_excluded_path(path) + def scan_das_files(path : string; var files : array; var cache : table; var skipped : int&) { let st = stat(path) return if (!st.is_valid) @@ -176,7 +190,7 @@ def scan_das_files(path : string; var files : array; var cache : table ends_with(".das") && !cache |> key_exists(full)) { if (is_skip_file(name)) { @@ -223,7 +237,7 @@ def scan_docs_folders(path : string; want_rule_docs, want_doc_lines : bool; return if (name == "." || name == ".." || name |> starts_with("_") || is_skip_dir(name)) let full = "{path}/{name}" let fst = stat(full) - if (fst.is_valid && fst.is_dir && !is_build_tree(full)) { + if (fst.is_valid && fst.is_dir && !is_skip_tree(full)) { scan_docs_folders(full, want_rule_docs, want_doc_lines, folders, line_folders, seen) } } @@ -360,7 +374,7 @@ def collect_basenames(path : string; var out : table) { let fst = stat(full) return if (!fst.is_valid) if (fst.is_dir) { - return if (is_build_tree(full)) + return if (is_skip_tree(full)) collect_basenames(full, out) } else { out |> insert(name, null) @@ -404,7 +418,7 @@ def collect_test_cells(path : string; var out : table) { fio::dir(path) $(name) { return if (name == "." || name == ".." || is_skip_dir(name)) let full = "{path}/{name}" - return if (is_build_tree(full)) + return if (is_skip_tree(full)) collect_test_cells(full, out) } } @@ -556,7 +570,7 @@ def private scan_arch_folder(path : string; var mds, dases : array; var let fst = stat(full) return if (!fst.is_valid) if (fst.is_dir) { - return if (is_build_tree(full)) + return if (is_skip_tree(full)) scan_arch_folder(full, mds, dases, seen) } elif (fst.is_reg && name |> ends_with(".md")) { mds |> push(full) @@ -723,7 +737,8 @@ def private file_nolint_codes(prog : ProgramPtr; base : table) : table @@ -821,6 +837,27 @@ def lint_cache_store(entry : string; inputs : array) { }) } +//! an environment failure - a missing prerequisite (error[20605]) +//! or an [extern] dlopen miss - skips the file, read off the first +//! diagnostic's block alone, since such a failure aborts the compile +//! before any other error and the dlopen miss puts "can't load +//! library" on the message's second line; anything else fails it +def private note_compile_failure(issue_str : string; var result : LintResult) { + var first_diag = "" + let err_start = find(issue_str, "error[") + if (err_start >= 0) { + first_diag = slice(issue_str, err_start, min(err_start + 240, length(issue_str))) + } + result.compile_errors = issue_str + if (find(first_diag, "error[20605]") >= 0) { + result.skip_reason = "(missing prerequisite — install daspkg deps to lint)" + } elif (find(first_diag, "can't load library") >= 0) { + result.skip_reason = "(native library unavailable — install it to lint this file)" + } else { + result.failed = true + } +} + def lint_file(file : string; run_paranoid, run_perf, run_style, comment_hygiene, lint_fixtures, lint_excluded_paths : bool; disabled_codes, enabled_codes : table) : LintResult { var result = LintResult(file = file) @@ -839,37 +876,17 @@ def lint_file(file : string; run_paranoid, run_perf, run_style, comment_hygiene, lint_policies(cop) compile_file(file, access, unsafe(addr(mg)), cop) $(ok; program; issues) { if (!ok) { - let issue_str = string(issues) - // Environment classes downgrade to SKIP only off the FIRST diagnostic's header - // line: a genuine environment failure aborts the compile before any other error, - // while a whole-string substring match would let one env line mask real errors - // (or match the literal echoed from an erroring source line). - var first_diag = "" - let err_start = find(issue_str, "error[") - if (err_start >= 0) { - // the first diagnostic's BLOCK, not just its header: the [extern] dlopen - // miss puts "can't load library" on the message's second line - first_diag = slice(issue_str, err_start, min(err_start + 240, length(issue_str))) - } - // error[20605] = missing prerequisite (e.g. daspkg-installed `anthropic/anthropic`) — - // out-of-tree as far as a stock daslang build can tell. - if (find(first_diag, "error[20605]") >= 0) { - result.skip_reason = "(missing prerequisite — install daspkg deps to lint)" - result.compile_errors = issue_str - return - } - // A [extern] dlopen miss (e.g. dasSMT without libz3) is the environment, not the - // code — skip with a visible reason so nightly coverage counting still sees it. - if (find(first_diag, "can't load library") >= 0) { - result.skip_reason = "(native library unavailable — install it to lint this file)" - result.compile_errors = issue_str - return - } - result.failed = true - result.compile_errors = issue_str + note_compile_failure(string(issues), result) return } inputs |> push_from(lint_cache_inputs(program, file)) if (!empty(entry)) + if (lint_shared_modules) { + program_for_each_module(program) $(mod) { + return if (!mod.moduleFlags.promoted || empty(mod.fileName)) + // builtin promotes under every policy + result.promoted_modules++ if (mod.name != "builtin") + } + } let file_disabled <- file_nolint_codes(program, disabled_codes) if (run_paranoid) { var paranoid_issues : array @@ -953,6 +970,9 @@ def apply_path_excludes(var result : LintResult) { } } +def private promoted_suffix(result : LintResult) : string => + lint_shared_modules ? " (promoted {result.promoted_modules})" : "" + // Print a single LintResult in the user-facing format. Shared by serial and // parallel paths; per-file output matches, though serial mode appends its // LINT019 findings as a trailing block while parallel reports them inline. @@ -972,12 +992,14 @@ def print_result(result : LintResult; quiet, silent : bool) { print(" {result.compile_errors}\n") } } elif (result.count > 0) { - print("WARN {result.file} ({result.count})\n") + print("WARN {result.file} ({result.count}){promoted_suffix(result)}\n") print(build_string() $(var w) { write_deduped(w, result.issues, " ", !silent) }) } elif (!quiet) { - print(result.cached ? "CACHED {result.file}\n" : "PASS {result.file}\n") + let line = (result.cached ? "CACHED {result.file}" + : "PASS {result.file}{promoted_suffix(result)}") + print("{line}\n") } } @@ -1017,8 +1039,10 @@ def maybe_collect(var since_collect : int&) { def run_worker(paths : array; run_paranoid, run_perf, run_style, comment_hygiene, lint_fixtures, lint_excluded_paths : bool; disabled_codes, enabled_codes : table; fix : bool) { var since_collect = 0 - for (f in paths) { - let result = lint_file_fixing(f, run_paranoid, run_perf, run_style, comment_hygiene, lint_fixtures, lint_excluded_paths, disabled_codes, enabled_codes, fix) + var inscope order <- lint_order(paths) + for (idx in order) { + let result = lint_file_fixing(paths[idx], run_paranoid, run_perf, run_style, + comment_hygiene, lint_fixtures, lint_excluded_paths, disabled_codes, enabled_codes, fix) print("{JSON_PREFIX}{write_json(JV(result))}\n{JSON_PREFIX}") maybe_collect(since_collect) } @@ -1035,6 +1059,9 @@ def build_worker_argv(spawn_prefix : array; chunk_path : string; cfg : C if (cfg.quiet) { argv |> push("--quiet") } + if (cfg.no_shared_modules) { + argv |> push("--no-shared-modules") + } if (cfg.comment_hygiene) { argv |> push("--comment-hygiene") } @@ -1212,6 +1239,7 @@ def collect_input_files(var files : array&; var dirs : array&) : // absolute, tripping error 20605 "several modules with the same name" (e.g. dasPEG) let full = get_full_file_name(p) let target = empty(full) ? p : replace(full, "\\", "/") + continue if (is_excluded_path(target)) let st = stat(target) if (st.is_valid && st.is_dir) { dirs |> push(target) @@ -1303,6 +1331,69 @@ def print_lint_help(rc : int) : int { return rc } +struct private RequireNode { + id : int + before : table +} + +//! the files' indices in require order - a module ahead of every +//! file that requires it. A file's compile promotes the shared +//! modules it requires, and a module promoted before its own compile +//! would collide with it, so each file lints before anything that +//! requires it. A require names a file by the file's name (the module +//! it declares is that name wherever the compiler resolves it), the +//! first file keeps a repeated name, and a name no file of the set +//! carries is left to the compiler +def private require_order(files : array) : array { + var name_of : table + for (file, i in files, count()) { + let name = module_group_module_name(file) + if (!key_exists(name_of, name)) { + name_of[name] = i + } + } + var nodes : array + nodes |> reserve(length(files)) + for (file, i in files, count()) { + var node <- RequireNode(id = i) + for (path in das_requires(file)) { + let j = name_of?[module_group_module_name(path)] ?? -1 + node.before |> insert(j) if (j >= 0 && j != i) + } + nodes |> emplace(node) + } + return <- release_order(nodes) +} + +//! Kahn's walk over `nodes`, a node free once nothing it waits on is +//! left; a cycle (a require the sources spell and the compiler never +//! takes; a true one compiles nowhere) releases the lowest index left +def private release_order(var nodes : array) : array { + var order : array + order |> reserve(length(nodes)) + while (!empty(nodes)) { + var pick = 0 + for (node, k in nodes, count()) { + if (empty(node.before)) { + pick = k + break + } + } + let id = nodes[pick].id + nodes |> erase(pick) + for (other in nodes) { + other.before |> erase(id) + } + order |> push(id) + } + return <- order +} + +//! the order a run lints `files` in: the require order under sharing, +//! the given one without +def private lint_order(files : array) : array => + <- lint_shared_modules ? require_order(files) : [for (i in range(length(files))); i] + [export, unsafe_heap_collect, lint_tool_entry] def main() : int { var r <- parse_args(type) @@ -1322,6 +1413,12 @@ def main() : int { var disabled_codes, enabled_codes : table if (!resolve_rule_filters(cfg, disabled_codes, enabled_codes)) return 1 lint_cache_dir = cfg.no_cache || is_standalone_exe() || jit_enabled() ? "" : cfg.cache_dir + lint_shared_modules = !cfg.no_shared_modules + lint_exclude_roots |> reserve(length(cfg.exclude)) + for (x in cfg.exclude) { + let full = get_full_file_name(x) + lint_exclude_roots |> push(to_generic_path(empty(full) ? x : full)) + } // Worker mode: lint a pre-resolved list from a paths-from file and emit // JSON-wrapped results. No banner, no summary, no exit-code derivation. @@ -1358,7 +1455,9 @@ def main() : int { print("Linting {length(files)} file(s)...\n\n") } var since_collect = 0 - for (file in files) { + var inscope order <- lint_order(files) + for (idx in order) { + let file = files[idx] if (!cfg.quiet) { print("checking {file}...\n") } diff --git a/utils/lint/tests/_lint_child.das b/utils/lint/tests/_lint_child.das new file mode 100644 index 0000000000..6579777f95 --- /dev/null +++ b/utils/lint/tests/_lint_child.das @@ -0,0 +1,73 @@ +options gen2 +options indenting = 4 + +//! the lint CLI as a child over a planted tree: a cell plants files +//! under `test_temp_dir()`, runs the lint with its cache under the +//! tree, and reads a file's line off the output +module _lint_child shared public + +require dastest/testing_boost public +require strings +require daslib/strings_boost +require daslib/fio +require daslib/command_line // get_das_exe + +def plant(dir, name, content : string) { + fopen(path_join(dir, name), "wb") $(f) { + if (f != null) { + fwrite(f, content) + } + } +} + +//! the lint over `root` with its cache under `root/cache`; `host`: +//! daslang's flags before the script, `extra`: after `--` +def run_lint(root : string; extra, host : array) : string { + var argv <- [get_das_exe(), "-dasroot", get_das_root()] + argv |> push_from(host) + argv |> push("{get_das_root()}/utils/lint/main.das") + argv |> push("--") + argv |> push(root) + argv |> push("--cache-dir") + argv |> push(path_join(root, "cache")) + argv |> push_from(extra) + var output : string + unsafe(popen_argv(argv, 0.0, $(f) { + if (f != null) { + output = fread(f) + } + })) + return output +} + +//! the result-cache entries under `root/cache`, the stamp aside +def entry_count(root : string) : int { + var n = 0 + dir(path_join(root, "cache")) $(name) { + n++ if (name != "." && name != ".." && name != "lint_hash.txt") + } + return n +} + +//! the first line of `out` about `file` under `root` that opens with +//! one of `prefixes`: its index, -1 when none, the line in `line` +def line_of(out, root, file : string; prefixes : array; var line : string&) : int { + let path = to_generic_path(path_join(root, file)) + for (candidate, i in split(out, "\n"), count()) { + for (prefix in prefixes) { + if (candidate |> starts_with(prefix) && find(to_generic_path(candidate), path) >= 0) { + line = candidate + return i + } + } + } + return -1 +} + +//! the status word on `file`'s line - PASS, CACHED, WARN or FAIL - +//! or "?" when the run printed none +def status_of(out, root, file : string) : string { + var line = "" + return "?" if (line_of(out, root, file, ["PASS ", "CACHED ", "WARN ", "FAIL "], line) < 0) + return slice(line, 0, find(line, " ")) +} diff --git a/utils/lint/tests/lint_result_cache.das b/utils/lint/tests/lint_result_cache.das index 65a46668d3..c00e70d030 100644 --- a/utils/lint/tests/lint_result_cache.das +++ b/utils/lint/tests/lint_result_cache.das @@ -1,45 +1,13 @@ options gen2 options indenting = 4 -require dastest/testing_boost public +require _lint_child public -require strings -require daslib/strings_boost require daslib/fio -var private _tmp_dir : string -var private _tmp_dir_inited = false - -def private tmp_root() : string { - if (!_tmp_dir_inited) { - let r = create_temp_directory_result("lint_result_cache") - if (!(r is value)) { - panic("could not create temp dir for the lint cache test: {unsafe(r.error)}") - } - _tmp_dir = unsafe(r.value) - _tmp_dir_inited = true - } - return _tmp_dir -} - -[finalize] -def cleanup_temp_dir { - if (_tmp_dir_inited && !empty(_tmp_dir)) { - rmdir_rec(_tmp_dir) - } -} - -def private plant(dir, name, content : string) { - fopen(path_join(dir, name), "wb") $(f) { - if (f != null) { - fwrite(f, content) - } - } -} - //! A tree of three files: `a.das` requires the module `b.das`, `c.das` carries a LINT003 finding def private build_tree(name : string) : string { - let root = path_join(tmp_root(), name) + let root = path_join(test_temp_dir(), name) mkdir_rec(root) plant(root, "b.das", "options gen2\n\nmodule b public\n\ndef public twice(x : int) : int => x * 2\n") plant(root, "a.das", "options gen2\n\nrequire b\n\n[export]\ndef main \{\n print(\"\{twice(2)\}\\n\")\n\}\n") @@ -47,48 +15,6 @@ def private build_tree(name : string) : string { return root } -def private das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} - -//! `host`: daslang's flags before the script; `extra`: after `--` -def private run_lint(root : string; extra, host : array) : string { - var argv <- [das_exe(), "-dasroot", get_das_root()] - argv |> push_from(host) - argv |> push("{get_das_root()}/utils/lint/main.das") - argv |> push("--") - argv |> push(root) - argv |> push("--cache-dir") - argv |> push(path_join(root, "cache")) - argv |> push_from(extra) - var output : string - unsafe(popen_argv(argv, 0.0, $(f) { - if (f != null) { - output = fread(f) - } - })) - return output -} - -def private entry_count(root : string) : int { - var n = 0 - dir(path_join(root, "cache")) $(name) { - n++ if (name != "." && name != ".." && name != "lint_hash.txt") - } - return n -} - -def private status_of(out, root, file : string) : string { - let path = to_generic_path(path_join(root, file)) - for (line in split(out, "\n")) { - for (status in ["PASS", "CACHED", "WARN"]) { - return status if (line |> starts_with("{status} ") && find(to_generic_path(line), path) >= 0) - } - } - return "?" -} - [test] def test_lint_result_cache(t : T?) { t |> run("a second run takes clean files from the cache, never a file with findings") @(t : T?) { diff --git a/utils/lint/tests/lint_shared_modules.das b/utils/lint/tests/lint_shared_modules.das new file mode 100644 index 0000000000..5774639c10 --- /dev/null +++ b/utils/lint/tests/lint_shared_modules.das @@ -0,0 +1,114 @@ +options gen2 +options indenting = 4 + +require _lint_child public + +require strings +require daslib/fio + +//! A tree of five files over one shared module: `a.das` and `c.das` +//! require the shared `b.das`, `sub/e.das` requires it by file path, +//! `c.das` carries a LINT003 finding, `d.das` requires only a builtin +//! module; the directory walk visits `a` before `b`, so the require +//! order has to move `b` ahead +def private build_tree(name : string) : string { + let root = path_join(test_temp_dir(), name) + mkdir_rec(root) + plant(root, "b.das", "options gen2\n\nmodule b shared public\n\n" + + "def public twice(x : int) : int => x * 2\n") + plant(root, "a.das", "options gen2\n\nrequire b\n\n[export]\ndef main \{\n" + + " print(\"\{twice(2)\}\\n\")\n\}\n") + plant(root, "c.das", "options gen2\n\nrequire b public\n\n[export]\ndef main \{\n" + + " var x = twice(13)\n print(\"\{x\}\\n\")\n\}\n") + plant(root, "d.das", "options gen2\n\nrequire strings\n\n[export]\ndef main \{\n" + + " print(\"\{to_upper(\"d\")\}\\n\")\n\}\n") + mkdir_rec(path_join(root, "sub")) + plant(path_join(root, "sub"), "e.das", "options gen2\n\nrequire ../b.das\n\n[export]\n" + + "def main \{\n print(\"\{twice(3)\}\\n\")\n\}\n") + return root +} + +//! the promoted-module count on a file's status line, -1 where the +//! line carries none +def private promoted_of(out, root, file : string) : int { + var line = "" + return -1 if (line_of(out, root, file, ["PASS ", "WARN "], line) < 0) + let at = find(line, "(promoted ") + return at < 0 ? -1 : to_int(slice(line, at + 10, find(line, ")", at))) +} + +//! the position of `file`'s "checking" line in the run's output, -1 +//! when the run never reached it +def private checked_at(out, root, file : string) : int { + var line = "" + return line_of(out, root, file, ["checking "], line) +} + +def private run_lint(root : string; extra : array) : string => run_lint(root, extra, []) + +[test] +def test_lint_shared_modules(t : T?) { + t |> run("a run reads as one without sharing, counts its promoted modules, caches") @(t : T?) { + let root = build_tree("same") + let plain = run_lint(root, ["--no-cache", "--no-shared-modules"]) + let pooled = run_lint(root, []) + for (file in ["a.das", "b.das", "c.das", "d.das"]) { + t |> equal(status_of(pooled, root, file), status_of(plain, root, file), + "{file} reads alike\nplain:\n{plain}\nshared:\n{pooled}") + } + t |> equal(status_of(pooled, root, "c.das"), "WARN", "the finding survives\n{pooled}") + t |> success(find(pooled, "LINT003") >= 0, "the finding is the same rule\n{pooled}") + t |> equal(entry_count(root), 4, "the clean files' entries land: a, b, d and sub/e") + let again = run_lint(root, []) + t |> equal(status_of(again, root, "a.das"), "CACHED", "a clean file is a hit\n{again}") + t |> equal(status_of(again, root, "c.das"), "WARN", "a finding lints every run\n{again}") + let d_took = promoted_of(pooled, root, "d.das") + t |> equal(d_took, 0, "d.das requires none of the set; builtin counts nothing\n{pooled}") + t |> success(promoted_of(pooled, root, "a.das") > d_took, + "a.das's compile took its module promoted from the process\n{pooled}") + t |> success(promoted_of(pooled, root, "c.das") > d_took, + "c.das's compile took its module promoted from the process\n{pooled}") + t |> equal(promoted_of(plain, root, "a.das"), -1, "no sharing, no count\n{plain}") + } + t |> run("a module lints ahead of the files that require it") @(t : T?) { + let root = build_tree("order") + let pooled = run_lint(root, []) + let b_at = checked_at(pooled, root, "b.das") + t |> success(b_at >= 0, "b.das was linted\n{pooled}") + t |> success(b_at < checked_at(pooled, root, "a.das"), "b.das before a.das\n{pooled}") + t |> success(b_at < checked_at(pooled, root, "c.das"), "b.das before c.das\n{pooled}") + t |> success(b_at < checked_at(pooled, root, "sub/e.das"), + "b.das before sub/e.das, which requires it by file path\n{pooled}") + t |> equal(status_of(pooled, root, "sub/e.das"), "PASS", "the file-path require\n{pooled}") + t |> equal(status_of(pooled, root, "b.das"), "PASS", + "the module lints clean after its users' compiles promote it\n{pooled}") + } + t |> run("--exclude leaves a folder out of the walk") @(t : T?) { + let root = build_tree("exclude") + mkdir_rec(path_join(root, "skip")) + plant(path_join(root, "skip"), "e.das", "options gen2\n\n[export]\ndef main \{\n" + + " var x = 13\n print(\"\{x\}\\n\")\n\}\n") + let walked = run_lint(root, ["--no-cache"]) + t |> equal(status_of(walked, root, "skip/e.das"), "WARN", "the finding is seen\n{walked}") + let left_out = run_lint(root, ["--no-cache", "--exclude", path_join(root, "skip")]) + t |> equal(status_of(left_out, root, "skip/e.das"), "?", "never linted\n{left_out}") + t |> equal(status_of(left_out, root, "c.das"), "WARN", "the rest lints\n{left_out}") + } + t |> run("a parallel run shares inside each worker") @(t : T?) { + let root = path_join(test_temp_dir(), "wide") + mkdir_rec(root) + plant(root, "b.das", "options gen2\n\nmodule b shared public\n\n" + + "def public twice(x : int) : int => x * 2\n") + for (i in range(16)) { // past the floor below which a run stays serial + plant(root, "a{i}.das", "options gen2\n\nrequire b\n\n[export]\ndef main \{\n" + + " print(\"\{twice({i})\}\\n\")\n\}\n") + } + let out = run_lint(root, ["--no-cache", "--workers", "2"]) + t |> equal(status_of(out, root, "b.das"), "PASS", "the module in its worker\n{out}") + for (i in range(16)) { + t |> equal(status_of(out, root, "a{i}.das"), "PASS", "a{i}.das in its worker\n{out}") + t |> success(promoted_of(out, root, "a{i}.das") >= 1, + "a{i}.das's compile took its module promoted from its worker's process\n{out}") + } + } +}