From 77e489d3cc730e65fb8b853b0f24a94c2f44ad44 Mon Sep 17 00:00:00 2001 From: Dryade Date: Fri, 13 Nov 2026 18:11:13 +0100 Subject: [PATCH 1/2] fix(python): bind function-local imports in the scope of their def (#2285) An import inside a def body was never recorded (parse_python_imports walks the module's direct statements only) and py_process_statement had no case for an import statement. Inside the def, `Store` fell through to the module scope: `store = Store(); store.load_local()` stayed unresolved and the weak-member guard refused a name guess, so the call got no CALLS edge. When the module bound the same name, the call resolved against the module-level class instead: a wrong edge. - Extraction records every import inside a def body, at any depth, flagged CBMImport.function_scope. A local name a module-level import binds as well stays unrecorded: per-name consumers cannot tell the two bindings apart, so the def's import then binds UNKNOWN (fail closed). - cbm_import_binds_runtime_name() is false for a flagged import: no file-wide CALLS import map binds it. The LSP maps already include it. - py_process_statement replays an import statement where it stands, into the current scope, matching the import entries afresh per statement. - python_receiver_is_exempt ignores flagged imports, so a def's import does not exempt a same-named receiver elsewhere in the file. Tests: extraction flags (def, try in a method, nested def, shadowed alias, module-level if); pipeline sequential + parallel: module-level control, import in the def, import in a try block, an unrelated class with the same method name, a sibling def without the import, and a def whose import shadows a module-level name. Co-Authored-By: Dammerzone Signed-off-by: Dryade --- internal/cbm/cbm.h | 14 ++-- internal/cbm/extract_calls.c | 4 +- internal/cbm/extract_imports.c | 46 +++++++++++ internal/cbm/lsp/py_lsp.c | 17 ++++ tests/test_extraction.c | 45 +++++++++++ tests/test_pipeline.c | 137 +++++++++++++++++++++++++++++++++ 6 files changed, 257 insertions(+), 6 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 9c29cbabb1..6fbba6c3bf 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -449,16 +449,20 @@ typedef struct { bool is_default; // ES default import (`import X from "Y"`), JS/TS only (#1916) bool type_checking_only; // Python: inside a module-level `if TYPE_CHECKING:` block -- // binds the name for annotations, never at runtime (#1277) + bool function_scope; // Python: inside a def body -- binds the name in that + // function's scope only, never in the module's (#2285) CBMImportKind kind; // CBM_IMPORT_KIND_DEFAULT unless the syntax names a member kind } CBMImport; -/* True when an import binds its local name at runtime. A Python import under - * `if TYPE_CHECKING:` exists for annotations only: a call through that name - * cannot happen, so no CALLS import map may bind it (an import_map edge +/* True when an import binds its local name in the module scope at runtime. A + * Python import under `if TYPE_CHECKING:` exists for annotations only: a call + * through that name cannot happen. A Python import inside a def binds a local + * of that function only: a call elsewhere in the file never goes through it. + * Either way no file-wide CALLS import map may bind it (an import_map edge * through it would be fabricated by construction). THE one rule every CALLS - * import-map builder applies. */ + * import-map builder applies; the Python LSP binds both where they stand. */ static inline bool cbm_import_binds_runtime_name(const CBMImport *imp) { - return imp && !imp->type_checking_only; + return imp && !imp->type_checking_only && !imp->function_scope; } /* Content-only ordered Python namespace facts; no resolved target identities. diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 127340c440..7c722830fa 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -3903,7 +3903,9 @@ static bool python_receiver_is_exempt(CBMExtractCtx *ctx, TSNode receiver) { * number (tens), never the corpus, so this stays O(file), not O(corpus). */ for (int i = 0; i < ctx->result->imports.count; i++) { const char *local_name = ctx->result->imports.items[i].local_name; - if (local_name && strcmp(local_name, name) == 0) { + /* A def's own import (#2285) binds no root elsewhere in the file. */ + if (local_name && !ctx->result->imports.items[i].function_scope && + strcmp(local_name, name) == 0) { return true; } } diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 62985df7c7..eba986cdfd 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -466,6 +466,51 @@ static void process_py_type_checking_block(CBMExtractCtx *ctx, TSNode if_node) { py_drop_rebound_imports(&ctx->result->imports, first, mark, star); } +/* An import inside a def body binds a local of that function (#2285): `def f(): + * from pkg.store import Store; Store().load()`. Record every such import, at + * any depth of the body (try/if/with, nested defs), flagged function_scope: + * no module-scope import map binds it, and the Python LSP binds it where the + * statement stands. A local name a module-level import binds as well stays + * unrecorded: every per-name consumer would conflate the two bindings, and + * the LSP then binds the function's own import as UNKNOWN (fail closed). An + * iterative walk: adversarially deep nesting must not exhaust the stack. */ +static void py_record_function_imports(CBMExtractCtx *ctx) { + CBMImportArray *imports = &ctx->result->imports; + int module_count = imports->count; + TSTreeCursor cursor = ts_tree_cursor_new(ctx->root); + for (bool walking = true; walking;) { + TSNode node = ts_tree_cursor_current_node(&cursor); + bool is_def = strcmp(ts_node_type(node), "function_definition") == 0; + if (is_def) { + TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body")); + if (!ts_node_is_null(body)) { + (void)py_record_subtree_imports(ctx, body); + } + } + if (!is_def && ts_tree_cursor_goto_first_child(&cursor)) { + continue; + } + while (walking && !ts_tree_cursor_goto_next_sibling(&cursor)) { + walking = ts_tree_cursor_goto_parent(&cursor); + } + } + ts_tree_cursor_delete(&cursor); + int keep = module_count; + for (int i = module_count; i < imports->count; i++) { + CBMImport imp = imports->items[i]; + bool module_bound = false; + for (int j = 0; j < module_count && !module_bound; j++) { + module_bound = imp.local_name && imports->items[j].local_name && + strcmp(imp.local_name, imports->items[j].local_name) == 0; + } + if (!module_bound) { + imp.function_scope = true; + imports->items[keep++] = imp; + } + } + imports->count = keep; +} + static void parse_python_imports(CBMExtractCtx *ctx) { TSTreeCursor cursor = ts_tree_cursor_new(ctx->root); if (!ts_tree_cursor_goto_first_child(&cursor)) { @@ -481,6 +526,7 @@ static void parse_python_imports(CBMExtractCtx *ctx) { } } while (ts_tree_cursor_goto_next_sibling(&cursor)); ts_tree_cursor_delete(&cursor); + py_record_function_imports(ctx); } /* A proven same-module function (its OWN_DEF fact and definition), and diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index b7309bc335..2a115f2dc6 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -230,6 +230,7 @@ static void py_resolve_calls_in(PyLSPContext *ctx, TSNode node) { static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node); static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node); static void py_process_statement(PyLSPContext *ctx, TSNode node); +static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt, unsigned char *consumed); static void py_exec_class_definition(PyLSPContext *ctx, TSNode class_node); static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int depth); static const CBMRegisteredFunc *py_lookup_attribute(PyLSPContext *ctx, const char *type_qn, @@ -2834,6 +2835,22 @@ static void py_process_statement(PyLSPContext *ctx, TSNode node) { return; const char *k = ts_node_type(node); + /* An import inside a def binds in the current scope, where it stands + * (#2285); the module-level replay never reaches it. Extraction records it + * flagged function_scope, so it reaches this walk through the LSP import + * map only. Several defs may import one name: each statement matches the + * entries afresh instead of consuming one. Unmatched, it binds UNKNOWN. */ + if (strcmp(k, "import_statement") == 0 || strcmp(k, "import_from_statement") == 0) { + unsigned char *fresh = NULL; + if (ctx->import_count > 0) { + fresh = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); + if (fresh) + memset(fresh, 0, (size_t)ctx->import_count); + } + py_replay_import_statement(ctx, node, fresh); + return; + } + if (strcmp(k, "assignment") == 0) { TSNode left = ts_node_child_by_field_name(node, "left", 4); TSNode right = ts_node_child_by_field_name(node, "right", 5); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 041c6db513..59ae03e4e3 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1989,6 +1989,50 @@ TEST(python_type_checking_rebound_in_else_unrecorded_issue1277) { PASS(); } +/* #2285: an import inside a def, at any depth of its body, is recorded flagged + * function_scope; a module-level import is not. A def's import of a name the + * module binds as well stays unrecorded (the module-level one remains), and a + * module-level compound statement's import stays unrecorded as before. */ +TEST(python_function_local_imports_flagged_issue2285) { + const char *src = "from pkg.other import Other\n" + "def use_local():\n" + " from pkg.store import Store\n" + " return Store().load_local(1)\n" + "class Holder:\n" + " def method(self):\n" + " try:\n" + " import pkg.sub.lib as lib\n" + " except ImportError:\n" + " return None\n" + " def inner():\n" + " from pkg.deep import Deep\n" + " return lib\n" + "def shadowed():\n" + " from pkg.store import Store as Other\n" + "if FLAG:\n" + " from pkg.flag import Conditional\n"; + CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "t", "use.py"); + ASSERT_NOT_NULL(r); + const CBMImport *other = py_find_import(r, "Other"); + const CBMImport *store = py_find_import(r, "Store"); + const CBMImport *lib = py_find_import(r, "lib"); + const CBMImport *deep = py_find_import(r, "Deep"); + ASSERT_NOT_NULL(other); + ASSERT_FALSE(other->function_scope); + ASSERT_STR_EQ(other->module_path, "pkg.other.Other"); /* the def's alias is unrecorded */ + ASSERT_NOT_NULL(store); + ASSERT_TRUE(store->function_scope); + ASSERT_FALSE(cbm_import_binds_runtime_name(store)); + ASSERT_NOT_NULL(lib); + ASSERT_TRUE(lib->function_scope); + ASSERT_NOT_NULL(deep); + ASSERT_TRUE(deep->function_scope); + ASSERT_NULL(py_find_import(r, "Conditional")); + ASSERT_EQ(r->imports.count, 4); + cbm_free_result(r); + PASS(); +} + TEST(python_function) { CBMFileResult *r = extract( "def greet(name):\n return f\"Hello {name}\"\n\ndef main():\n greet(\"World\")\n", @@ -15818,6 +15862,7 @@ SUITE(extraction) { RUN_TEST(python_annotated_instance_fields_exported_issue1277); RUN_TEST(python_type_checking_imports_flagged_issue1277); RUN_TEST(python_type_checking_rebound_in_else_unrecorded_issue1277); + RUN_TEST(python_function_local_imports_flagged_issue2285); RUN_TEST(js_function); RUN_TEST(js_class); RUN_TEST(ts_function); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 23fabaf561..3e20a20437 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -21683,6 +21683,141 @@ TEST(python_type_checking_typed_field_issue1277_parallel) { return check_python_type_checking_typed_field_issue1277(true); } +/* #2285: a class imported inside a def types its constructor result there, so + * `store = Store(); store.load_local(1)` resolves to Store.load_local exactly + * as with the module-level import (the control). The import stays a local of + * its def: a sibling def without it gets no edge, an unrelated class with the + * same method name gets none, and a local import that shadows a module-level + * name keeps the module-level class out of that def only. */ +static int check_python_function_local_import_issue2285(bool parallel) { + const char *store = "class Store:\n" + " def load_top(self, key):\n" + " return key\n\n" + " def load_local(self, key):\n" + " return key\n\n" + " def load_local_try(self, key):\n" + " return key\n"; + const char *other = "class Other:\n" + " def load_local(self, key):\n" + " return key\n"; + const char *top = "from store import Store\n\n" + "def use_top():\n" + " store = Store()\n" + " return store.load_top(1)\n"; + const char *local = "def use_local():\n" + " from store import Store\n\n" + " store = Store()\n" + " return store.load_local(1)\n\n" + "def use_local_try():\n" + " try:\n" + " from store import Store\n\n" + " store = Store()\n" + " items = store.load_local_try(1)\n" + " except Exception:\n" + " return None\n" + " return items\n\n" + "def no_local_import():\n" + " store = Store()\n" + " return store.load_local(1)\n"; + const char *shadow = "from other import Other\n\n" + "def module_level():\n" + " return Other().load_local(1)\n\n" + "def shadowed():\n" + " from store import Store as Other\n" + " return Other().load_local(1)\n"; + if (setup_usages_repo("store.py", store, "other.py", other) != 0) { + FAIL("failed to create temp dir"); + } + const char *names[] = {"top.py", "local.py", "shadow.py"}; + const char *bodies[] = {top, local, shadow}; + char path[512]; + for (int i = 0; i < 3; i++) { + snprintf(path, sizeof(path), "%s/%s", g_usages_tmpdir, names[i]); + if (th_write_file(path, bodies[i]) != 0) { + teardown_usages_repo(); + FAIL("failed to write fixture"); + } + } + if (parallel) { + for (int i = 0; i < 50; i++) { + snprintf(path, sizeof(path), "%s/fl_pad_%02d.py", g_usages_tmpdir, i); + if (th_write_file(path, "fl_padding = 0\n") != 0) { + teardown_usages_repo(); + FAIL("failed to write parallel-selection fixture"); + } + } + } + + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + if ((old_workers && !saved_workers) || (old_single && !saved_single)) { + free(saved_workers); + free(saved_single); + teardown_usages_repo(); + FAIL("failed to save pipeline environment"); + } + int pin_workers_rc = parallel ? cbm_setenv("CBM_WORKERS", "4", 1) : 0; + int pin_single_rc = parallel ? cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") : 0; + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/test_2285.db", g_usages_tmpdir); + cbm_pipeline_t *p = pin_workers_rc == 0 && pin_single_rc == 0 + ? cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL) + : NULL; + int run_rc = p ? cbm_pipeline_run(p) : -1; + cbm_store_t *s = run_rc == 0 ? cbm_store_open_path(db_path) : NULL; + bool store_opened = s != NULL; + int top_ok = -1, local_ok = -1, try_ok = -1, unrelated = -1, leaked = -1; + int module_ok = -1, shadow_kept = -1; + if (s) { + const char *project = cbm_pipeline_project_name(p); + top_ok = count_lsp_calls(s, project, "top.use_top", "store.Store.load_top"); + local_ok = count_lsp_calls(s, project, "local.use_local", "store.Store.load_local"); + try_ok = count_lsp_calls(s, project, "local.use_local_try", "store.Store.load_local_try"); + unrelated = count_calls_with(s, project, "local.use_local", "other.Other.load_local", NULL); + leaked = + count_calls_with(s, project, "local.no_local_import", "store.Store.load_local", NULL); + module_ok = count_lsp_calls(s, project, "shadow.module_level", "other.Other.load_local"); + shadow_kept = + count_calls_with(s, project, "shadow.shadowed", "other.Other.load_local", NULL); + cbm_store_close(s); + } + + cbm_pipeline_free(p); + int restore_workers_rc = + saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS"); + int restore_single_rc = saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1) + : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + free(saved_workers); + free(saved_single); + teardown_usages_repo(); + + ASSERT_EQ(pin_workers_rc, 0); + ASSERT_EQ(pin_single_rc, 0); + ASSERT_EQ(restore_workers_rc, 0); + ASSERT_EQ(restore_single_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_TRUE(store_opened); + ASSERT_EQ(top_ok, 1); /* control: module-level import */ + ASSERT_EQ(local_ok, 1); /* import inside the def */ + ASSERT_EQ(try_ok, 1); /* import inside a try block of the def */ + ASSERT_EQ(unrelated, 0); /* same method name, unrelated class */ + ASSERT_EQ(leaked, 0); /* a sibling def does not see the import */ + ASSERT_EQ(module_ok, 1); /* control: module-level binding */ + ASSERT_EQ(shadow_kept, 0); /* the local import shadows it in its own def */ + PASS(); +} + +TEST(python_function_local_import_issue2285) { + return check_python_function_local_import_issue2285(false); +} + +TEST(python_function_local_import_issue2285_parallel) { + return check_python_function_local_import_issue2285(true); +} + /* A base class spelled through a root-binding import (`import pkg.mod.nodes` * binds `pkg`; `class X(pkg.mod.nodes.Context)`) is the same class as the * from-imported spelling. The Python LSP's inheritance view never resolved the @@ -34907,6 +35042,8 @@ SUITE(pipeline) { RUN_TEST(python_crossfile_typed_field_calls_issue1277_parallel); RUN_TEST(python_type_checking_typed_field_issue1277); RUN_TEST(python_type_checking_typed_field_issue1277_parallel); + RUN_TEST(python_function_local_import_issue2285); + RUN_TEST(python_function_local_import_issue2285_parallel); RUN_TEST(python_dotted_base_inheritance); RUN_TEST(python_dotted_base_inheritance_parallel); RUN_TEST(pipeline_python_package_reexports_resolve_by_lsp); From ab6aa4c4df0fad130c90e76189cd22314b5f1d54 Mon Sep 17 00:00:00 2001 From: Dryade Date: Sat, 14 Nov 2026 16:35:48 +0100 Subject: [PATCH 2/2] fix(python): def-local imports keep name fallback, re-export rebinding and driver parity (#2285 review) - External-import guard: cbm_suppress_external_import_shadow skips a def's own import. It never enters the file-wide import map, so its missing key is no evidence of an external binding; a call the LSP cannot type (a decorated function) keeps its name-based edge again. - Re-exported names: py_lsp keeps the module path each import row was added with (import_source_qns) and matches every import statement against it. An earlier replay canonicalizes import_module_qns to the re-export target, so the same statement in a later def no longer matched and bound UNKNOWN. - Driver parity: the sequential INHERITS / DECORATES import map (pass_semantic build_import_map) skips function_scope imports like the parallel driver's map, so a class nested in a def gets the same base in both drivers. Its IMPORTS-edge fallback runs only for a file with no extracted import, so no def's import reaches it. - py_process_statement reuses one per-context consumed buffer per import statement instead of an arena allocation each time. Tests: pipeline twins for the decorated-function fallback, both re-export shapes and the dotted import inside a method (edge kept), a sequential/parallel INHERITS parity test, an assertion documenting that a bare constructor call in the shadowing def still binds the module-level class, and a registry unit row. Co-Authored-By: Dammerzone Signed-off-by: Dryade --- internal/cbm/lsp/py_lsp.c | 26 ++- internal/cbm/lsp/py_lsp.h | 8 + src/pipeline/pass_semantic.c | 8 +- src/pipeline/registry.c | 3 + tests/test_pipeline.c | 343 ++++++++++++++++++++++++++++++++++- tests/test_registry.c | 7 +- 6 files changed, 383 insertions(+), 12 deletions(-) diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index c990c6fb7b..9da92e4cf1 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -410,23 +410,28 @@ void py_lsp_add_import(PyLSPContext *ctx, const char *local_name, const char *mo sizeof(const char *)); const char **qns = (const char **)cbm_arena_alloc(ctx->arena, (size_t)(capacity + 1) * sizeof(const char *)); + const char **sources = + (const char **)cbm_arena_alloc(ctx->arena, (size_t)capacity * sizeof(const char *)); unsigned char *kinds = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)capacity * sizeof(unsigned char)); - if (!names || !qns || !kinds) + if (!names || !qns || !sources || !kinds) return; for (int i = 0; i < ctx->import_count; i++) { CBM_LSP_WORK(1); names[i] = ctx->import_local_names[i]; qns[i] = ctx->import_module_qns[i]; + sources[i] = ctx->import_source_qns[i]; kinds[i] = ctx->import_kinds ? ctx->import_kinds[i] : 0; } ctx->import_local_names = names; ctx->import_module_qns = qns; + ctx->import_source_qns = sources; ctx->import_kinds = kinds; ctx->import_capacity = capacity; } ctx->import_local_names[ctx->import_count] = cbm_arena_strdup(ctx->arena, local_name); ctx->import_module_qns[ctx->import_count] = cbm_arena_strdup(ctx->arena, module_qn); + ctx->import_source_qns[ctx->import_count] = ctx->import_module_qns[ctx->import_count]; ctx->import_kinds[ctx->import_count] = 0; ctx->import_local_names[new_count] = NULL; ctx->import_module_qns[new_count] = NULL; @@ -3002,12 +3007,14 @@ static void py_process_statement(PyLSPContext *ctx, TSNode node) { * map only. Several defs may import one name: each statement matches the * entries afresh instead of consuming one. Unmatched, it binds UNKNOWN. */ if (strcmp(k, "import_statement") == 0 || strcmp(k, "import_from_statement") == 0) { - unsigned char *fresh = NULL; - if (ctx->import_count > 0) { - fresh = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); - if (fresh) - memset(fresh, 0, (size_t)ctx->import_count); - } + if (ctx->import_count > ctx->def_import_consumed_cap) { + ctx->def_import_consumed = + (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); + ctx->def_import_consumed_cap = ctx->def_import_consumed ? ctx->import_count : 0; + } + unsigned char *fresh = ctx->import_count > 0 ? ctx->def_import_consumed : NULL; + if (fresh) + memset(fresh, 0, (size_t)ctx->import_count); py_replay_import_statement(ctx, node, fresh); return; } @@ -5871,7 +5878,10 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l !ctx->import_module_qns[i] || strcmp(ctx->import_local_names[i], local) != 0) { continue; } - const char *candidate_qn = ctx->import_module_qns[i]; + /* Match the path the row was added with: an earlier replay may have + * canonicalized import_module_qns[i] to a re-export target, which the + * same statement in another def no longer spells (#2285). */ + const char *candidate_qn = ctx->import_source_qns[i]; PyDirectImportKind kind = py_import_kind_from_statement(ctx, stmt, local, &candidate_qn); if (!py_replayable_import_kind(kind)) continue; diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index a34a0bb4cb..347d109340 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -70,7 +70,15 @@ typedef struct { // Built from CBMFileResult.imports. const char **import_local_names; const char **import_module_qns; + // The module_qn each row was added with, never rewritten: an import + // statement is matched against it, while import_module_qns carries the + // canonical QN a replay bound (a re-export target, #2285). + const char **import_source_qns; unsigned char *import_kinds; // internal PyDirectImportKind, classified once from the AST + // A def's import statement matches the rows afresh: their consumed flags, + // cleared per statement (py_process_statement), grown with import_count. + unsigned char *def_import_consumed; + int def_import_consumed_cap; int import_count; int import_capacity; // rows the three arrays hold (py_lsp_add_import) CBMNameRows import_rows; // import_local_names by name, rows in table order diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 9ad910852e..11200787e0 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -100,7 +100,10 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path, for (int i = 0; i < result->imports.count; i++) { const CBMImport *imp = &result->imports.items[i]; - if (!imp->local_name || !imp->local_name[0] || !imp->module_path) { + /* A def's own import (#2285) binds no file-wide name: the parallel + * driver's map (cbm_import_binds_runtime_name) skips it as well. */ + if (!imp->local_name || !imp->local_name[0] || !imp->module_path || + imp->function_scope) { continue; } char *target_qn = cbm_pipeline_fqn_module(ctx->project_name, imp->module_path); @@ -120,7 +123,8 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path, return 0; } - /* Slow path: scan graph buffer IMPORTS edges + parse JSON properties */ + /* Slow path: scan graph buffer IMPORTS edges + parse JSON properties. Only a + * file with no extracted import reaches it, so no def's own import does. */ char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); free(file_qn); diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 73c2a41481..f11d05c3b0 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -1008,6 +1008,9 @@ bool cbm_suppress_external_import_shadow(const char *callee_name, const char *st if (!imp->local_name || !imp->module_path || strcmp(imp->local_name, callee_name) != 0) { continue; } + if (imp->function_scope) { + continue; /* a def's own import never enters the file-wide import map */ + } if (specifier_is_relative(imp->module_path) || specifier_names_indexed_package(indexed_packages, imp->module_path) || specifier_names_declared_package(declared_packages, imp->module_path)) { diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index f9e9c07319..c17d4a869b 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -22075,7 +22075,7 @@ static int check_python_function_local_import_issue2285(bool parallel) { cbm_store_t *s = run_rc == 0 ? cbm_store_open_path(db_path) : NULL; bool store_opened = s != NULL; int top_ok = -1, local_ok = -1, try_ok = -1, unrelated = -1, leaked = -1; - int module_ok = -1, shadow_kept = -1; + int module_ok = -1, shadow_kept = -1, shadow_ctor = -1; if (s) { const char *project = cbm_pipeline_project_name(p); top_ok = count_lsp_calls(s, project, "top.use_top", "store.Store.load_top"); @@ -22087,6 +22087,7 @@ static int check_python_function_local_import_issue2285(bool parallel) { module_ok = count_lsp_calls(s, project, "shadow.module_level", "other.Other.load_local"); shadow_kept = count_calls_with(s, project, "shadow.shadowed", "other.Other.load_local", NULL); + shadow_ctor = count_calls_with(s, project, "shadow.shadowed", "other.Other", NULL); cbm_store_close(s); } @@ -22112,6 +22113,9 @@ static int check_python_function_local_import_issue2285(bool parallel) { ASSERT_EQ(leaked, 0); /* a sibling def does not see the import */ ASSERT_EQ(module_ok, 1); /* control: module-level binding */ ASSERT_EQ(shadow_kept, 0); /* the local import shadows it in its own def */ + /* Fail closed holds for member calls only: the bare constructor call still + * binds the module-level class through the file-wide import map. */ + ASSERT_EQ(shadow_ctor, 1); PASS(); } @@ -22123,6 +22127,336 @@ TEST(python_function_local_import_issue2285_parallel) { return check_python_function_local_import_issue2285(true); } +/* One indexed fixture of the #2285 def-import tests: files[i] = {path, body}. */ +typedef struct { + char tmp[256]; + py_workers_pin_t pin; + cbm_pipeline_t *p; + cbm_store_t *s; + int run_rc; +} py_def_import_run_t; + +static void py_def_import_open(py_def_import_run_t *r, const char *const (*files)[2], int n, + bool parallel) { + memset(r, 0, sizeof(*r)); + r->run_rc = -1; + snprintf(r->tmp, sizeof(r->tmp), "/tmp/cbm_py_2285_XXXXXX"); + if (!cbm_mkdtemp(r->tmp)) { + r->tmp[0] = '\0'; + return; + } + for (int i = 0; i < n; i++) { + write_temp_file(r->tmp, files[i][0], files[i][1]); + } + py_write_padding(r->tmp, "defimp_pad", parallel); + r->pin = py_pin_workers(parallel); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/defimp.db", r->tmp); + r->p = r->pin.rc == 0 ? cbm_pipeline_new(r->tmp, db_path, CBM_MODE_FULL) : NULL; + r->run_rc = r->p ? cbm_pipeline_run(r->p) : -1; + r->s = r->run_rc == 0 ? cbm_store_open_path(db_path) : NULL; +} + +/* Closes everything; returns the unpin rc. */ +static int py_def_import_close(py_def_import_run_t *r, bool parallel) { + if (r->s) { + cbm_store_close(r->s); + } + cbm_pipeline_free(r->p); + int rc = py_unpin_workers(parallel, &r->pin); + if (r->tmp[0]) { + th_rmtree(r->tmp); + } + return rc; +} + +/* #2285 review: a def's own import of a project name the LSP cannot type (a + * decorated function) keeps the name-based fallback. The import never enters + * the file-wide import map, which the external-import guard must not read as + * "this import resolved outside the project". */ +static int check_python_def_import_name_fallback_issue2285(bool parallel) { + static const char *const files[][2] = { + {"pkg/__init__.py", ""}, + {"pkg/outcomes.py", "def _with_exception(t):\n" + " def decorate(func):\n" + " func.Exception = t\n" + " return func\n" + " return decorate\n\n" + "@_with_exception(RuntimeError)\n" + "def fail(reason=\"\"):\n" + " raise RuntimeError(reason)\n"}, + {"pkg/compat.py", "def getfuncargnames(function):\n" + " try:\n" + " return function.__code__.co_varnames\n" + " except AttributeError:\n" + " from pkg.outcomes import fail\n" + " fail(\"no code object\")\n"}, + }; + py_def_import_run_t r; + py_def_import_open(&r, files, 3, parallel); + int fallback = -1; + if (r.s) { + fallback = count_calls_with(r.s, cbm_pipeline_project_name(r.p), + "pkg.compat.getfuncargnames", "pkg.outcomes.fail", NULL); + } + bool opened = r.s != NULL; + int unpin_rc = py_def_import_close(&r, parallel); + ASSERT_EQ(unpin_rc, 0); + ASSERT_EQ(r.run_rc, 0); + ASSERT_TRUE(opened); + ASSERT_EQ(fallback, 1); /* decorated: no LSP type, the name fallback binds it */ + PASS(); +} + +TEST(python_def_import_name_fallback_issue2285) { + return check_python_def_import_name_fallback_issue2285(false); +} + +TEST(python_def_import_name_fallback_issue2285_parallel) { + return check_python_def_import_name_fallback_issue2285(true); +} + +/* #2285 review: a re-exported name (`from pkg.forms import CharField`, which + * pkg/forms/__init__.py re-exports) binds its definition in every def that + * imports it, after the module or an earlier def already replayed the same + * import. A same-named class elsewhere never gets the call. */ +static int check_python_def_import_reexport_issue2285(bool parallel) { + static const char char_field[] = "class CharField:\n" + " def __init__(self, max_length):\n" + " self.max_length = max_length\n\n" + " def clean(self, value):\n" + " return value\n"; + static const char *const files[][2] = { + {"pkg/__init__.py", ""}, + {"pkg/models/__init__.py", ""}, + {"pkg/forms/fields.py", char_field}, + {"pkg/models/fields.py", char_field}, + {"pkg/forms/__init__.py", "from pkg.forms.fields import CharField\n"}, + {"pkg/use.py", "from pkg.forms import CharField\n\n" + "def reimported_in_def():\n" + " from pkg.forms import CharField\n" + " field = CharField(20)\n" + " return field.clean(2)\n"}, + {"pkg/use3.py", "def first_def():\n" + " from pkg.forms import CharField\n" + " return CharField(40).clean(4)\n\n" + "def second_def():\n" + " from pkg.forms import CharField\n" + " return CharField(50).clean(5)\n"}, + }; + py_def_import_run_t r; + py_def_import_open(&r, files, 7, parallel); + int re_init = -1, re_clean = -1, re_wrong = -1, first = -1, second = -1, second_wrong = -1; + if (r.s) { + const char *project = cbm_pipeline_project_name(r.p); + re_init = count_lsp_calls(r.s, project, "pkg.use.reimported_in_def", + "pkg.forms.fields.CharField.__init__"); + re_clean = count_lsp_calls(r.s, project, "pkg.use.reimported_in_def", + "pkg.forms.fields.CharField.clean"); + re_wrong = count_calls_with(r.s, project, "pkg.use.reimported_in_def", + "pkg.models.fields.CharField", NULL); + first = + count_lsp_calls(r.s, project, "pkg.use3.first_def", "pkg.forms.fields.CharField.clean"); + second = count_lsp_calls(r.s, project, "pkg.use3.second_def", + "pkg.forms.fields.CharField.clean"); + second_wrong = count_calls_with(r.s, project, "pkg.use3.second_def", + "pkg.models.fields.CharField.clean", NULL); + } + bool opened = r.s != NULL; + int unpin_rc = py_def_import_close(&r, parallel); + ASSERT_EQ(unpin_rc, 0); + ASSERT_EQ(r.run_rc, 0); + ASSERT_TRUE(opened); + ASSERT_EQ(re_init, 1); /* after the module-level import of the same name */ + ASSERT_EQ(re_clean, 1); /* typed by the re-exported class */ + ASSERT_EQ(re_wrong, 0); /* never the same-named class elsewhere */ + ASSERT_EQ(first, 1); /* the first def importing it */ + ASSERT_EQ(second, 1); /* a later def importing it again */ + ASSERT_EQ(second_wrong, 0); + PASS(); +} + +TEST(python_def_import_reexport_issue2285) { + return check_python_def_import_reexport_issue2285(false); +} + +TEST(python_def_import_reexport_issue2285_parallel) { + return check_python_def_import_reexport_issue2285(true); +} + +/* #2285 review: `import p.config` inside a method, then + * `p.config.create_writer(...)`, keeps its one CALLS edge to the submodule's + * function: with the def import only, when the module also imports the root + * (`import p`) or a name from it (`from p import helper`), and in a src layout + * with a decorated method. The Python LSP does not type an unaliased dotted + * import here even at module level (the control), on main as on this branch, + * so the edge is a name-based one; this pins that the def's own import never + * drops the call. */ +static int check_python_def_dotted_import_issue2285(bool parallel) { + static const char *const files[][2] = { + {"p/__init__.py", "def helper():\n return 0\n"}, + {"p/config/__init__.py", "def create_writer(arg):\n return arg\n"}, + {"p/only_def.py", "class Debugger:\n" + " def init_pdb(self):\n" + " import p.config\n" + " return p.config.create_writer(1)\n"}, + {"p/module_level.py", "import p.config\n\n" + "class Debugger:\n" + " def init_pdb(self):\n" + " return p.config.create_writer(0)\n"}, + {"p/root_import.py", "import p\n\n" + "class Debugger:\n" + " def init_pdb(self):\n" + " import p.config\n" + " return p.config.create_writer(2)\n"}, + {"p/from_root.py", "from p import helper\n\n" + "class Debugger:\n" + " def init_pdb(self):\n" + " import p.config\n" + " return p.config.create_writer(3)\n"}, + /* The reported shape: src layout, a decorated method, the module + * importing a name from the same submodule. */ + {"src/q/__init__.py", ""}, + {"src/q/config/__init__.py", "class Config:\n" + " pass\n\n" + "def create_writer(arg):\n" + " return arg\n"}, + {"src/q/debugging.py", "from q.config import Config\n\n" + "class Debugger:\n" + " @classmethod\n" + " def init_pdb(cls):\n" + " import q.config\n" + " return q.config.create_writer(4)\n"}, + }; + static const char *const callers[][2] = { + {"p.module_level.Debugger.init_pdb", "p.config.create_writer"}, + {"p.only_def.Debugger.init_pdb", "p.config.create_writer"}, + {"p.root_import.Debugger.init_pdb", "p.config.create_writer"}, + {"p.from_root.Debugger.init_pdb", "p.config.create_writer"}, + {"q.debugging.Debugger.init_pdb", "q.config.create_writer"}, + }; + py_def_import_run_t r; + py_def_import_open(&r, files, 9, parallel); + int edges[5] = {-1, -1, -1, -1, -1}; + for (int i = 0; r.s && i < 5; i++) { + edges[i] = count_calls_with(r.s, cbm_pipeline_project_name(r.p), callers[i][0], + callers[i][1], NULL); + } + bool opened = r.s != NULL; + int unpin_rc = py_def_import_close(&r, parallel); + ASSERT_EQ(unpin_rc, 0); + ASSERT_EQ(r.run_rc, 0); + ASSERT_TRUE(opened); + ASSERT_EQ(edges[0], 1); /* control: module-level import p.config */ + ASSERT_EQ(edges[1], 1); /* the def's import only */ + ASSERT_EQ(edges[2], 1); /* the module also imports the root */ + ASSERT_EQ(edges[3], 1); /* the module imports a name from the root */ + ASSERT_EQ(edges[4], 1); /* src layout, decorated method */ + PASS(); +} + +TEST(python_def_dotted_import_issue2285) { + return check_python_def_dotted_import_issue2285(false); +} + +TEST(python_def_dotted_import_issue2285_parallel) { + return check_python_def_dotted_import_issue2285(true); +} + +/* The target QN of the one INHERITS edge whose source QN ends in src_suffix: + * "" when there is none, "" when there are more. Caller frees. */ +static char *py_inherits_target(cbm_store_t *s, const char *project, const char *src_suffix) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + cbm_store_find_edges_by_type(s, project, "INHERITS", &edges, &edge_count); + char *found = NULL; + size_t ss = strlen(src_suffix); + for (int i = 0; i < edge_count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK) { + size_t sl = strlen(src.qualified_name); + if (sl >= ss && strcmp(src.qualified_name + sl - ss, src_suffix) == 0) { + free(found); + found = strdup(found ? "" : tgt.qualified_name); + } + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) + cbm_store_free_edges(edges, edge_count); + return found ? found : strdup(""); +} + +/* #2285 review: sequential and parallel indexing resolve the base of a class + * nested in a def the same way. Neither driver's file-wide import map binds a + * def's own import, so the base name never resolves through it in one driver + * and by a name guess in the other. */ +TEST(python_def_import_nested_base_driver_parity_issue2285) { + static const char wrapper[] = "class Wrapper:\n" + " def ping(self):\n" + " return 1\n"; + static const char *const files[][2] = { + {"pkg/__init__.py", ""}, + {"pkg/base_a.py", wrapper}, + {"pkg/base_b.py", wrapper}, + {"pkg/def_only.py", "def build_one():\n" + " from pkg.base_b import Wrapper\n\n" + " class CustomOne(Wrapper):\n" + " pass\n" + " return CustomOne\n"}, + {"pkg/both.py", "from pkg.base_a import Wrapper\n\n" + "def build_two():\n" + " from pkg.base_b import Wrapper as Base\n\n" + " class CustomTwo(Base):\n" + " pass\n" + " return CustomTwo\n\n" + "def build_three():\n" + " from pkg.base_b import Wrapper\n\n" + " class CustomThree(Wrapper):\n" + " pass\n" + " return CustomThree\n"}, + }; + static const char *const nested[] = {".CustomOne", ".CustomTwo", ".CustomThree"}; + char *base[2][3] = {{NULL}}; + int run_rc[2] = {-1, -1}; + int unpin_rc[2] = {-1, -1}; + for (int mode = 0; mode < 2; mode++) { + py_def_import_run_t r; + py_def_import_open(&r, files, 5, mode == 1); + run_rc[mode] = r.s ? r.run_rc : -1; + for (int i = 0; r.s && i < 3; i++) { + base[mode][i] = py_inherits_target(r.s, cbm_pipeline_project_name(r.p), nested[i]); + } + unpin_rc[mode] = py_def_import_close(&r, mode == 1); + } + bool agree = true; + for (int i = 0; i < 3; i++) { + /* The project name is the temp dir's, so compare from the package on. */ + const char *seq = base[0][i] ? strstr(base[0][i], "pkg.") : NULL; + const char *par = base[1][i] ? strstr(base[1][i], "pkg.") : NULL; + if (!base[0][i] || !base[1][i] || (seq == NULL) != (par == NULL) || + (seq && strcmp(seq, par) != 0)) { + printf(" %s: sequential=\"%s\" parallel=\"%s\"\n", nested[i] + 1, + base[0][i] ? base[0][i] : "(not run)", base[1][i] ? base[1][i] : "(not run)"); + agree = false; + } + } + for (int m = 0; m < 2; m++) { + for (int i = 0; i < 3; i++) { + free(base[m][i]); + } + } + ASSERT_EQ(run_rc[0], 0); + ASSERT_EQ(run_rc[1], 0); + ASSERT_EQ(unpin_rc[0], 0); + ASSERT_EQ(unpin_rc[1], 0); + ASSERT_TRUE(agree); + PASS(); +} + /* A base class spelled through a root-binding import (`import pkg.mod.nodes` * binds `pkg`; `class X(pkg.mod.nodes.Context)`) is the same class as the * from-imported spelling. The Python LSP's inheritance view never resolved the @@ -35354,6 +35688,13 @@ SUITE(pipeline) { RUN_TEST(python_type_checking_typed_field_issue1277_parallel); RUN_TEST(python_function_local_import_issue2285); RUN_TEST(python_function_local_import_issue2285_parallel); + RUN_TEST(python_def_import_name_fallback_issue2285); + RUN_TEST(python_def_import_name_fallback_issue2285_parallel); + RUN_TEST(python_def_import_reexport_issue2285); + RUN_TEST(python_def_import_reexport_issue2285_parallel); + RUN_TEST(python_def_dotted_import_issue2285); + RUN_TEST(python_def_dotted_import_issue2285_parallel); + RUN_TEST(python_def_import_nested_base_driver_parity_issue2285); RUN_TEST(python_dotted_base_inheritance); RUN_TEST(python_dotted_base_inheritance_parallel); RUN_TEST(pipeline_python_package_reexports_resolve_by_lsp); diff --git a/tests/test_registry.c b/tests/test_registry.c index 2baa5819a8..01048fa7ea 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -2376,8 +2376,9 @@ TEST(external_import_shadow_keeps_everything_else) { {.local_name = "eq", .module_path = "drizzle-orm"}, {.local_name = "normalize", .module_path = "./text-utils"}, {.local_name = "getRootContainer", .module_path = "@repo/shared"}, + {.local_name = "fail", .module_path = "pkg.outcomes.fail", .function_scope = true}, }; - CBMImportArray arr = {.items = imports, .count = 3, .cap = 3}; + CBMImportArray arr = {.items = imports, .count = 4, .cap = 4}; /* The relative import and the workspace package both materialized. */ const char *keys[] = {"normalize", "getRootContainer"}; const int nkeys = 2; @@ -2406,6 +2407,10 @@ TEST(external_import_shadow_keeps_everything_else) { /* A package the import map DOES bind resolved in-graph (workspace package). */ ASSERT_FALSE(cbm_suppress_external_import_shadow("getRootContainer", "unique_name", &arr, keys, nkeys, NULL, NULL)); + /* A Python def's own import (#2285) never enters the file-wide import map: + * its missing key is no evidence of an external binding. */ + ASSERT_FALSE( + cbm_suppress_external_import_shadow("fail", "unique_name", &arr, keys, nkeys, NULL, NULL)); /* A name the file never imports (called with no import at all). */ ASSERT_FALSE(cbm_suppress_external_import_shadow("helper", "unique_name", &arr, keys, nkeys, NULL, NULL));