From 9b5139d73da34fc882beabeae403c3eee94c56b6 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 18 Oct 2026 03:41:31 +0200 Subject: [PATCH] fix(python): bound the function-local walks at the symbol's own file A package __init__.py can define a function whose QN equals a submodule's (poetry: layouts/__init__.py `def layout` is the QN prefix of every symbol in layouts/layout.py). The prefix walks that recognise a def declared inside a function body accepted that other file's function as the enclosing def, so class Layout and its methods read as locals of layout(): they left the short-name index and the import symbol fallback, and the answer depended on which file registered first (discovery order on a full index, node-id order on an incremental one). Every walk now stops at the symbol's own module: - registration (pass_definitions.c, pass_parallel.c, and the incremental seed) passes the defining file's symbol prefix length, cbm_pipeline_fqn_symbol_prefix_len, computed without allocating, to cbm_registry_add_lang_in_module; cbm_registry_add_lang keeps its signature and does no local detection; - cbm_registry_resolve_lexical stops at the caller's module QN; - the import fallback's local test only counts an enclosing def in the same file. Also moves the #1548 registry tests out of the test they had been inserted into. Tests: registry_python_local_bound_at_own_file_issue1548 (both registration orders), fqn_symbol_prefix_len_matches_fqn_compute, pipeline_python_init_function_shares_submodule_qn{,_parallel}_issue1548 (src/ layout: the import and calls onto Layout survive, then again after an incremental re-index). Signed-off-by: Martin Vogel --- src/pipeline/fqn.c | 24 +++++ src/pipeline/pass_calls.c | 2 +- src/pipeline/pass_definitions.c | 13 ++- src/pipeline/pass_parallel.c | 18 +++- src/pipeline/pass_pkgmap.c | 10 +- src/pipeline/pipeline.h | 41 +++++--- src/pipeline/pipeline_incremental.c | 17 +++- src/pipeline/registry.c | 75 ++++++++++++--- tests/test_pipeline.c | 99 +++++++++++++++++++ tests/test_registry.c | 144 +++++++++++++++++++++------- 10 files changed, 366 insertions(+), 77 deletions(-) diff --git a/src/pipeline/fqn.c b/src/pipeline/fqn.c index f391ce1e9..f4603dc80 100644 --- a/src/pipeline/fqn.c +++ b/src/pipeline/fqn.c @@ -355,6 +355,30 @@ char *cbm_pipeline_resolve_relative_import(const char *source_rel, const char *m return resolve_js_relative(buf, sizeof(buf), module_path); } +size_t cbm_pipeline_fqn_symbol_prefix_len(const char *project, const char *rel_path) { + if (!project) { + return 0; + } + char path[FQN_PATH_BUF]; + int n = snprintf(path, sizeof(path), "%s", rel_path ? rel_path : ""); + if (n < 0 || (size_t)n >= sizeof(path)) { + return 0; + } + cbm_normalize_path_sep(path); + strip_file_extension(path); + const char *segments[CBM_SZ_256]; + int seg_count = 0; + segments[seg_count++] = project; + seg_count += tokenize_path(path, segments + seg_count, FQN_MAX_PATH_SEGS); + /* Any symbol name: the prefix cbm_pipeline_fqn_compute puts before it. */ + strip_init_or_index(segments, &seg_count, "symbol"); + size_t total = 0; + for (int i = 0; i < seg_count; i++) { + total += strlen(segments[i]) + (i > 0 ? FQN_SEP_LEN : 0); + } + return total; +} + char *cbm_pipeline_fqn_folder(const char *project, const char *rel_dir) { if (!project) { return strdup(""); diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 80b2e69b8..dfbb8f7fb 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -593,7 +593,7 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *ca cbm_resolution_t hres = {0}; if (lang == CBM_LANG_PYTHON) { hres = cbm_registry_resolve_lexical(ctx->registry, call->second_arg_name, - call->enclosing_func_qn); + call->enclosing_func_qn, module_qn); } if (!hres.qualified_name) { hres = cbm_registry_resolve_handler(ctx->registry, call->second_arg_name, module_qn, diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index aecd91d99..80bf0a8a9 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -416,7 +416,7 @@ static void pd_add_file_doc(const cbm_gbuf_node_t *file_node, const char *doc) { * `file_doc`, the file's own doc (passed with its first def only), goes on the * File node. */ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel, - const char *file_doc, CBMLanguage lang) { + const char *file_doc, CBMLanguage lang, size_t module_prefix_len) { if (!def->qualified_name || !def->name) { return; } @@ -442,7 +442,8 @@ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const * (SQL FROM/JOIN lineage). pass_parallel.c and pipeline_incremental.c seed * through the same predicate, so the three registries cannot diverge. */ if (node_id > 0 && cbm_label_is_registry_symbol(def->label)) { - cbm_registry_add_lang(ctx->registry, def->name, def->qualified_name, def->label, lang); + cbm_registry_add_lang_in_module(ctx->registry, def->name, def->qualified_name, def->label, + lang, module_prefix_len); } char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); @@ -1019,10 +1020,14 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t "test_declarations"); } - /* Create nodes for each definition */ + /* Create nodes for each definition. A Python file's module QN bounds + * the registry's function-local detection to this file. */ const char *file_doc = result->module_doc; /* goes with the first def */ + size_t py_prefix_len = lang == CBM_LANG_PYTHON + ? cbm_pipeline_fqn_symbol_prefix_len(ctx->project_name, rel) + : 0; for (int d = 0; d < result->defs.count; d++) { - process_def(ctx, &result->defs.items[d], rel, file_doc, lang); + process_def(ctx, &result->defs.items[d], rel, file_doc, lang, py_prefix_len); file_doc = NULL; total_defs++; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 501320dd6..f908de590 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -1773,7 +1773,8 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, * once per file by the caller: computing the file QN and finding its node for * every definition was 700 k allocations and lookups on the Go corpus. */ static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, - int64_t file_node_id, CBMLanguage lang, int *reg_entries) { + int64_t file_node_id, CBMLanguage lang, size_t module_prefix_len, + int *reg_entries) { int edges = 0; if (!def->name || !def->qualified_name || !def->label) { return 0; @@ -1781,7 +1782,8 @@ static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *d /* Registry membership is defined ONCE by cbm_label_is_registry_symbol * (helpers.c) — see pass_definitions.c for the per-label rationale. */ if (cbm_label_is_registry_symbol(def->label)) { - cbm_registry_add_lang(ctx->registry, def->name, def->qualified_name, def->label, lang); + cbm_registry_add_lang_in_module(ctx->registry, def->name, def->qualified_name, def->label, + lang, module_prefix_len); (*reg_entries)++; } char marked[CBM_SZ_4K]; @@ -1988,9 +1990,14 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t int64_t file_node_id = file_node ? file_node->id : 0; free(file_qn); pp_add_file_doc(file_node, result->module_doc); + /* Bounds the function-local detection to this file (pass_definitions.c). */ + size_t py_prefix_len = files[i].language == CBM_LANG_PYTHON + ? cbm_pipeline_fqn_symbol_prefix_len(ctx->project_name, rel) + : 0; for (int d = 0; d < result->defs.count; d++) { - defines_edges += register_and_link_def(ctx, &result->defs.items[d], file_node_id, - files[i].language, ®_entries); + defines_edges += + register_and_link_def(ctx, &result->defs.items[d], file_node_id, + files[i].language, py_prefix_len, ®_entries); } } @@ -2415,7 +2422,8 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou * lexical lookup sees a function's locals. MUST match pass_calls.c. */ cbm_resolution_t hres = {0}; if (lang == CBM_LANG_PYTHON) { - hres = cbm_registry_resolve_lexical(registry, handler_ref, call->enclosing_func_qn); + hres = cbm_registry_resolve_lexical(registry, handler_ref, call->enclosing_func_qn, + module_qn); } if (!hres.qualified_name) { hres = cbm_registry_resolve_handler(registry, handler_ref, module_qn, ik, iv, ic, diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index e30a0c231..303b293b9 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -1824,8 +1824,11 @@ bool cbm_import_symbol_fallback_allowed(CBMLanguage lang) { * no import can name it. Its QN extends a Function/Method node's * (proj.tests.mock_inputs.inner.wrapper.mock_getpass.getpass), which is how it * is recognised; the symbol-name fallback bound `import getpass` to exactly - * that def. Mirrors cbm_registry_add_lang, which keeps the same - * symbols out of the call resolver's short-name index. */ + * that def. Only an enclosing def of the SAME file counts: a package + * __init__.py function can share a submodule's QN (poetry: layouts/__init__.py + * `def layout` vs layouts/layout.py `class Layout`). Mirrors + * cbm_registry_add_lang_in_module, which keeps the same symbols out of the call + * resolver's short-name index. */ static bool import_candidate_is_python_local(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *cand) { const char *qn = cand->qualified_name; const char *slash = cand->file_path ? strrchr(cand->file_path, '/') : NULL; @@ -1839,7 +1842,8 @@ static bool import_candidate_is_python_local(const cbm_gbuf_t *gbuf, const cbm_g for (char *dot = strrchr(scope, '.'); dot; dot = strrchr(scope, '.')) { *dot = '\0'; const cbm_gbuf_node_t *outer = cbm_gbuf_find_by_qn(gbuf, scope); - if (outer && outer->label && + if (outer && outer->label && outer->file_path && + strcmp(outer->file_path, cand->file_path) == 0 && (strcmp(outer->label, "Function") == 0 || strcmp(outer->label, "Method") == 0)) { return true; } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index e03c0dbe6..072674d8d 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -200,6 +200,12 @@ char *cbm_pipeline_fqn_compute(const char *project, const char *rel_path, const /* Module QN: project.dir.parts (no name). Caller must free(). */ char *cbm_pipeline_fqn_module(const char *project, const char *rel_path); +/* Length of the prefix cbm_pipeline_fqn_compute(project, rel_path, name) puts + * before a symbol `name` (the joining dot excluded): the QN prefix every + * definition of that file carries, __init__ / index already dropped. Computed + * without allocating; 0 when unavailable. */ +size_t cbm_pipeline_fqn_symbol_prefix_len(const char *project, const char *rel_path); + /* Language-aware module QN. When `module_is_dir` is true (Java/Go package * semantics) the module is derived from the CONTAINING DIRECTORY (the filename * stem is dropped), so it agrees with the extraction-side def QNs; when false @@ -253,14 +259,24 @@ void cbm_registry_add(cbm_registry_t *r, const char *name, const char *qualified /* As cbm_registry_add, recording `lang`: the DETECTED language of the file * that defines the symbol (CBM_LANG_COUNT = unknown), which the cross-language - * veto (cbm_registry_name_guess_vetoed) checks against the caller. A Python - * symbol declared inside a function body (see cbm_registry_qn_is_function_local) - * is registered for exact lookups only, never offered to the short-name - * strategies; every registration site seeds through this, so full and - * incremental indexes agree. */ + * veto (cbm_registry_name_guess_vetoed) checks against the caller. Detects no + * function-local symbols: that needs the defining file's module QN, see + * cbm_registry_add_lang_in_module. */ void cbm_registry_add_lang(cbm_registry_t *r, const char *name, const char *qualified_name, const char *label, CBMLanguage lang); +/* As cbm_registry_add_lang. A Python symbol declared inside a function body of + * its own file (cbm_registry_qn_is_function_local, bounded by + * `module_prefix_len`: cbm_pipeline_fqn_symbol_prefix_len of the defining file) + * is registered for exact lookups only, never offered to the short-name + * strategies. The three graph + * registration sites (pass_definitions.c, pass_parallel.c, + * pipeline_incremental.c) all register through this, so full and incremental + * indexes agree. 0 = no local detection. */ +void cbm_registry_add_lang_in_module(cbm_registry_t *r, const char *name, + const char *qualified_name, const char *label, + CBMLanguage lang, size_t module_prefix_len); + /* True when code in `caller` can call a symbol defined in `target` by bare * name: same language, same interop family (JS/TS + script hosts, C/C++/CUDA/ * ObjC, JVM, .NET, ...; the table lives in registry.c), or Swift with the C @@ -272,16 +288,19 @@ bool cbm_lang_resolution_compatible(CBMLanguage caller, CBMLanguage target); * CBM_LANG_COUNT for anything else. */ CBMLanguage cbm_registry_synthetic_path_language(const char *file_path); -/* True when a proper prefix of `qualified_name` is a registered Function or - * Method, i.e. the symbol was declared inside a function body. */ -bool cbm_registry_qn_is_function_local(const cbm_registry_t *r, const char *qualified_name); +/* True when a proper prefix of `qualified_name` longer than + * `module_prefix_len` (cbm_pipeline_fqn_symbol_prefix_len of the defining file) + * is a registered Function or Method, i.e. the symbol was declared inside a + * function body of its own file. False when module_prefix_len is 0. */ +bool cbm_registry_qn_is_function_local(const cbm_registry_t *r, const char *qualified_name, + size_t module_prefix_len); /* Python bare-name lexical lookup: the def or class named `callee_name` bound * in the nearest function scope enclosing `scope_qn` (the caller's QN), - * skipping class bodies, stopping at the module. Strategy "lexical_scope"; - * qualified_name is NULL when no such local exists. */ + * skipping class bodies, stopping at the caller's module (`module_qn`). + * Strategy "lexical_scope"; qualified_name is NULL when no such local exists. */ cbm_resolution_t cbm_registry_resolve_lexical(const cbm_registry_t *r, const char *callee_name, - const char *scope_qn); + const char *scope_qn, const char *module_qn); /* Resolve a callee name using prioritized strategies. * import_map: NULL-terminated array of {local_name, resolved_qn} pairs, or NULL. diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 4d0d667f5..a860b5e32 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1340,6 +1340,7 @@ static bool incr_label_is_registry_symbol(const char *label) { typedef struct { cbm_registry_t *registry; CBMHashTable *lang_by_path; /* rel_path -> (CBMLanguage + 1); keys borrowed */ + const char *project; /* bounds Python function-local detection per file */ } registry_seed_t; static void registry_visitor(const cbm_gbuf_node_t *node, void *userdata) { @@ -1354,14 +1355,20 @@ static void registry_visitor(const cbm_gbuf_node_t *node, void *userdata) { lang = (CBMLanguage)((uintptr_t)v - SKIP_ONE); } } - cbm_registry_add_lang(seed->registry, node->name, node->qualified_name, node->label, lang); + /* Same per-file bound as the full index (pass_definitions.c). */ + size_t prefix_len = lang == CBM_LANG_PYTHON + ? cbm_pipeline_fqn_symbol_prefix_len(seed->project, node->file_path) + : 0; + cbm_registry_add_lang_in_module(seed->registry, node->name, node->qualified_name, node->label, + lang, prefix_len); } /* Seed the registry from every registry symbol in the graph buffer, with * languages taken from the discovered files (see registry_visitor). */ static void registry_seed_from_gbuf(cbm_registry_t *registry, const cbm_gbuf_t *gbuf, - const cbm_file_info_t *files, int file_count) { - registry_seed_t seed = {.registry = registry, .lang_by_path = NULL}; + const char *project, const cbm_file_info_t *files, + int file_count) { + registry_seed_t seed = {.registry = registry, .lang_by_path = NULL, .project = project}; if (files && file_count > 0) { seed.lang_by_path = cbm_ht_create((size_t)file_count * PAIR_LEN); } @@ -3305,7 +3312,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char if (!registry) { goto out; } - registry_seed_from_gbuf(registry, gbuf, files, file_count); + registry_seed_from_gbuf(registry, gbuf, project, files, file_count); cbm_log_info("delta.preseed_done", "registry", itoa_buf(cbm_registry_size(registry)), "elapsed_ms", itoa_buf((int)elapsed_ms(t))); @@ -4155,7 +4162,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil /* Step 3-5: Registry + extract + resolve */ cbm_registry_t *registry = cbm_registry_new(); cbm_clock_gettime(CLOCK_MONOTONIC, &t); - registry_seed_from_gbuf(registry, existing, files, file_count); + registry_seed_from_gbuf(registry, existing, project, files, file_count); cbm_log_info("incremental.registry_seed", "symbols", itoa_buf(cbm_registry_size(registry)), "elapsed_ms", itoa_buf((int)elapsed_ms(t))); diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 8a1122ddf..5dda868a4 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -1187,14 +1187,45 @@ static bool registry_label_is_callable(const char *label) { return label && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0); } -bool cbm_registry_qn_is_function_local(const cbm_registry_t *r, const char *qualified_name) { +/* Length of the QN prefix the defs of the caller's file carry: its module QN + * (cbm_pipeline_fqn_module), minus the trailing `__init__` / `index` segment + * that symbol QNs drop (pkg/__init__.py defines proj.pkg.f). 0 when there is + * no module QN or `qualified_name` does not extend that prefix. */ +static size_t python_def_prefix_len(const char *module_qn, const char *qualified_name) { + static const char *const dropped[] = {".__init__", ".index"}; + size_t n = module_qn ? strlen(module_qn) : 0; + for (size_t i = 0; i < sizeof(dropped) / sizeof(dropped[0]); i++) { + size_t k = strlen(dropped[i]); + if (n > k && strcmp(module_qn + n - k, dropped[i]) == 0) { + n -= k; + break; + } + } + if (n == 0 || !qualified_name || strncmp(qualified_name, module_qn, n) != 0 || + qualified_name[n] != '.') { + return 0; + } + return n; +} + +/* Whether `qualified_name` sits inside a def of its OWN file: only prefixes + * strictly longer than `module_prefix_len` (cbm_pipeline_fqn_symbol_prefix_len + * of the defining file) count. A package's __init__.py can define a function + * whose QN equals a submodule's (poetry: layouts/__init__.py `def layout` vs + * layouts/layout.py), so an unbounded walk read the submodule's classes as + * that function's locals -- and the answer depended on which file registered + * first. */ +bool cbm_registry_qn_is_function_local(const cbm_registry_t *r, const char *qualified_name, + size_t module_prefix_len) { char scope[CBM_SZ_512]; size_t len = qualified_name ? strlen(qualified_name) : 0; - if (!r || len == 0 || len >= sizeof(scope)) { + size_t floor = module_prefix_len; + if (!r || floor == 0 || len <= floor || len >= sizeof(scope) || qualified_name[floor] != '.') { return false; } memcpy(scope, qualified_name, len + SKIP_ONE); - for (char *dot = strrchr(scope, '.'); dot; dot = strrchr(scope, '.')) { + for (char *dot = strrchr(scope, '.'); dot && (size_t)(dot - scope) > floor; + dot = strrchr(scope, '.')) { *dot = '\0'; if (registry_label_is_callable(cbm_ht_get(r->exact, scope))) { return true; @@ -1238,7 +1269,12 @@ static const char *registry_add_exact(cbm_registry_t *r, const char *qualified_n void cbm_registry_add(cbm_registry_t *r, const char *name, const char *qualified_name, const char *label) { - cbm_registry_add_lang(r, name, qualified_name, label, CBM_LANG_COUNT); + cbm_registry_add_lang_in_module(r, name, qualified_name, label, CBM_LANG_COUNT, 0); +} + +void cbm_registry_add_lang(cbm_registry_t *r, const char *name, const char *qualified_name, + const char *label, CBMLanguage lang) { + cbm_registry_add_lang_in_module(r, name, qualified_name, label, lang, 0); } /* Python nests defs: a def or class declared in a function body -- a closure, @@ -1250,14 +1286,19 @@ void cbm_registry_add(cbm_registry_t *r, const char *name, const char *qualified * decorator's `def wrapper` to a caller in another function or file. The exact * map still holds it, for LSP targets and cbm_registry_resolve_lexical. One * file's defs register parent first, so the enclosing function is already in - * the exact map when its locals arrive. Gated to Python: probing every - * prefix of every symbol would cost a kernel index seconds for nothing. */ -void cbm_registry_add_lang(cbm_registry_t *r, const char *name, const char *qualified_name, - const char *label, CBMLanguage lang) { + * the exact map when its locals arrive; the walk never leaves the symbol's own + * file (`module_prefix_len`, cbm_pipeline_fqn_symbol_prefix_len of the defining + * file; 0 = no local detection). + * Gated to Python: probing every prefix of every symbol would cost a kernel + * index seconds for nothing. */ +void cbm_registry_add_lang_in_module(cbm_registry_t *r, const char *name, + const char *qualified_name, const char *label, + CBMLanguage lang, size_t module_prefix_len) { if (!r || !qualified_name || !label) { return; } - if (lang == CBM_LANG_PYTHON && cbm_registry_qn_is_function_local(r, qualified_name)) { + if (lang == CBM_LANG_PYTHON && + cbm_registry_qn_is_function_local(r, qualified_name, module_prefix_len)) { (void)registry_add_exact(r, qualified_name, label); return; } @@ -2176,19 +2217,22 @@ cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const cha * enclosing def -- create_app.index, Tree.visit.visit -- before anything a * module-wide strategy would find. Walks scope_qn outward while its prefixes * are registered defs or classes (skipping the classes) and stops at the - * module, which the other strategies own. Empty unless such a local exists. - * Uncached: the answer depends on the caller, not only on the name. */ + * caller's module (`module_qn`), which the other strategies own -- a + * same-QN function of a package __init__.py is never an enclosing scope. + * Empty unless such a local exists. Uncached: the answer depends on the + * caller, not only on the name. */ cbm_resolution_t cbm_registry_resolve_lexical(const cbm_registry_t *r, const char *callee_name, - const char *scope_qn) { + const char *scope_qn, const char *module_qn) { char scope[CBM_SZ_512]; char candidate[CBM_SZ_512]; size_t len = scope_qn ? strlen(scope_qn) : 0; + size_t floor = python_def_prefix_len(module_qn, scope_qn); if (!r || !callee_name || !callee_name[0] || strchr(callee_name, '.') || - strstr(callee_name, "::") || len == 0 || len >= sizeof(scope)) { + strstr(callee_name, "::") || len == 0 || len >= sizeof(scope) || floor == 0) { return empty_result(); } memcpy(scope, scope_qn, len + SKIP_ONE); - for (;;) { + while (strlen(scope) > floor) { const char *label = cbm_ht_get(r->exact, scope); if (!label) { break; @@ -2223,7 +2267,8 @@ cbm_resolution_t cbm_registry_resolve_call_scoped(const cbm_registry_t *r, const int import_map_count, CBMLanguage lang, const char *scope_qn) { if (lang == CBM_LANG_PYTHON) { - cbm_resolution_t lexical = cbm_registry_resolve_lexical(r, callee_name, scope_qn); + cbm_resolution_t lexical = + cbm_registry_resolve_lexical(r, callee_name, scope_qn, module_qn); if (lexical.qualified_name && lexical.qualified_name[0]) { return lexical; } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 3141ab32d..2cb51cfd6 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14713,6 +14713,103 @@ TEST(pipeline_python_local_def_shadows_module_def_parallel_issue1548) { return pyn_run_shadowing(52); } +/* L7 review of #1548: a package __init__.py function can share a submodule's QN + * (poetry: layouts/__init__.py `def layout` is the QN prefix of every symbol in + * layouts/layout.py). The function-local detection is bounded at the symbol's + * own file, so class Layout stays a module-level, importable class. src/ layout + * without a manifest: the module resolver misses `rowpkg.layout`, so the import + * relies on the symbol-name fallback, which skips function locals -- unbounded, + * Layout read as a local of layout() and its import and calls vanished. Both + * resolvers; the second run re-indexes incrementally after an edit of + * layout.py, so the incremental registry seed is held to the same bound. */ +static const char pyn_layout_init_py[] = "def layout(name):\n" + " return name\n"; +static const char pyn_layout_mod_py[] = "class Layout:\n" + " def __init__(self, name):\n" + " self.name = name\n" + "\n" + " def create(self):\n" + " return self.name\n"; +static const char pyn_layout_app_py[] = "from rowpkg.layout import Layout\n" + "\n" + "def test_build():\n" + " return Layout(\"demo\").create()\n"; + +static int pyn_run_init_collision(int fillers) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_py_initqn_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "src/rowpkg/__init__.py", pyn_layout_init_py); + write_temp_file(tmp, "src/rowpkg/layout.py", pyn_layout_mod_py); + write_temp_file(tmp, "tests/test_app.py", pyn_layout_app_py); + for (int i = 0; i < fillers; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "filler%d.py", i); + snprintf(body, sizeof(body), "def filler%d():\n return %d\n", i, i); + write_temp_file(tmp, name, body); + } + char *old_workers = getenv("CBM_WORKERS"); + char *saved = old_workers ? strdup(old_workers) : NULL; + if (fillers > 0) { + cbm_setenv("CBM_WORKERS", "4", 1); + } + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/initqn.db", tmp); + int rc[2] = {-1, -1}; + bool imports[2] = {false, false}; + bool calls[2] = {false, false}; + bool method[2] = {false, false}; + for (int run = 0; run < 2; run++) { + if (run == 1) { + char path[512]; + snprintf(path, sizeof(path), "%s/src/rowpkg/layout.py", tmp); + th_append_file(path, "\n# edited: the second run re-indexes incrementally\n"); + } + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + rc[run] = p ? cbm_pipeline_run(p) : -1; + cbm_store_t *s = rc[run] == 0 ? cbm_store_open_path(db_path) : NULL; + if (s) { + const char *project = cbm_pipeline_project_name(p); + imports[run] = pyn_edge(s, project, "tests.test_app.py.__file__", + "src.rowpkg.layout.Layout", "IMPORTS"); + calls[run] = + pyn_edge(s, project, "tests.test_app.test_build", "src.rowpkg.layout.Layout", + "CALLS"); + method[run] = pyn_edge(s, project, "tests.test_app.test_build", + "src.rowpkg.layout.Layout.create", "CALLS"); + cbm_store_close(s); + } + if (p) { + cbm_pipeline_free(p); + } + } + if (saved) { + cbm_setenv("CBM_WORKERS", saved, 1); + free(saved); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + th_rmtree(tmp); + for (int run = 0; run < 2; run++) { + ASSERT_EQ(rc[run], 0); + ASSERT_TRUE(imports[run]); /* the class stays importable */ + ASSERT_TRUE(calls[run]); /* Layout("demo") */ + ASSERT_TRUE(method[run]); /* .create() through the constructed type */ + } + PASS(); +} + +TEST(pipeline_python_init_function_shares_submodule_qn_issue1548) { + return pyn_run_init_collision(0); +} + +TEST(pipeline_python_init_function_shares_submodule_qn_parallel_issue1548) { + return pyn_run_init_collision(52); +} + TEST(pipeline_python_project) { /* Port of TestPipelinePythonProject */ const char *files[] = {"main.py", "utils.py"}; @@ -24214,6 +24311,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_python_nested_def_local_after_incremental_issue1548); RUN_TEST(pipeline_python_local_def_shadows_module_def_issue1548); RUN_TEST(pipeline_python_local_def_shadows_module_def_parallel_issue1548); + RUN_TEST(pipeline_python_init_function_shares_submodule_qn_issue1548); + RUN_TEST(pipeline_python_init_function_shares_submodule_qn_parallel_issue1548); RUN_TEST(pipeline_header_include_target_is_independent_of_registration_order); RUN_TEST(pipeline_imports_multi_symbol_edges); RUN_TEST(pipeline_go_cross_package_call); diff --git a/tests/test_registry.c b/tests/test_registry.c index 055de4d78..0f3e805f2 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -1962,6 +1962,9 @@ TEST(registry_python_scope_qualified_terminal_bucket_uncapped) { bool seeded = cbm_registry_size(r) == 302; cbm_registry_free(r); ASSERT_TRUE(shape && seeded && resolved); + PASS(); +} + /* #1548: a Python def or class declared in a function body is a local of it. * It is registered for exact lookups but kept out of the short-name index, so * no weak strategy offers it to a caller elsewhere. A class nested in a class @@ -1969,24 +1972,29 @@ TEST(registry_python_scope_qualified_terminal_bucket_uncapped) { * does the same QN shape from another language. */ TEST(registry_python_function_local_is_exact_only_issue1548) { cbm_registry_t *r = cbm_registry_new(); - cbm_registry_add_lang(r, "create_app", "proj.web.create_app", "Function", - CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "index", "proj.web.create_app.index", "Function", - CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "Settings", "proj.web.create_app.Settings", "Class", - CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "value", "proj.web.create_app.Settings.value", "Method", - CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "Outer", "proj.web.Outer", "Class", CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "Inner", "proj.web.Outer.Inner", "Class", CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "f", "proj.ts.f", "Function", CBM_LANG_TYPESCRIPT); - cbm_registry_add_lang(r, "g", "proj.ts.f.g", "Function", CBM_LANG_TYPESCRIPT); + const size_t web = strlen("proj.web"); /* symbol prefix of web.py */ + static const char *const defs[][3] = { + {"create_app", "proj.web.create_app", "Function"}, + {"index", "proj.web.create_app.index", "Function"}, + {"Settings", "proj.web.create_app.Settings", "Class"}, + {"value", "proj.web.create_app.Settings.value", "Method"}, + {"Outer", "proj.web.Outer", "Class"}, + {"Inner", "proj.web.Outer.Inner", "Class"}, + }; + for (size_t i = 0; i < sizeof(defs) / sizeof(defs[0]); i++) { + cbm_registry_add_lang_in_module(r, defs[i][0], defs[i][1], defs[i][2], CBM_LANG_PYTHON, + web); + } + cbm_registry_add_lang_in_module(r, "f", "proj.ts.f", "Function", CBM_LANG_TYPESCRIPT, + strlen("proj.ts")); + cbm_registry_add_lang_in_module(r, "g", "proj.ts.f.g", "Function", CBM_LANG_TYPESCRIPT, + strlen("proj.ts")); static const char *const local[] = {"proj.web.create_app.index", "proj.web.create_app.Settings", "proj.web.create_app.Settings.value"}; for (int i = 0; i < 3; i++) { ASSERT_TRUE(cbm_registry_exists(r, local[i])); - ASSERT_TRUE(cbm_registry_qn_is_function_local(r, local[i])); + ASSERT_TRUE(cbm_registry_qn_is_function_local(r, local[i], web)); const char *simple = strrchr(local[i], '.') + 1; const char **out = NULL; int count = -1; @@ -2003,7 +2011,9 @@ TEST(registry_python_function_local_is_exact_only_issue1548) { ASSERT_EQ(count, 1); ASSERT_STR_EQ(out[0], nameable[i][1]); } - ASSERT_FALSE(cbm_registry_qn_is_function_local(r, "proj.web.Outer.Inner")); + ASSERT_FALSE(cbm_registry_qn_is_function_local(r, "proj.web.Outer.Inner", web)); + /* No module prefix: no local detection. */ + ASSERT_FALSE(cbm_registry_qn_is_function_local(r, "proj.web.create_app.index", 0)); /* A weak strategy cannot reach the local; before, index() anywhere in * the project resolved to it by unique_name. */ cbm_resolution_t res = cbm_registry_resolve(r, "index", "proj.other", NULL, NULL, 0); @@ -2017,44 +2027,110 @@ TEST(registry_python_function_local_is_exact_only_issue1548) { * to the other strategies. A dotted callee is not a bare name. */ TEST(registry_resolve_lexical_walks_enclosing_defs_issue1548) { cbm_registry_t *r = cbm_registry_new(); - cbm_registry_add_lang(r, "Tree", "proj.m.Tree", "Class", CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "visit", "proj.m.Tree.visit", "Method", CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "visit", "proj.m.Tree.visit.visit", "Function", - CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "helper", "proj.m.Tree.helper", "Method", CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "factory", "proj.m.factory", "Function", CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "helper", "proj.m.factory.helper", "Function", - CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "Local", "proj.m.factory.Local", "Class", CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "run", "proj.m.factory.Local.run", "Method", CBM_LANG_PYTHON); - cbm_registry_add_lang(r, "top", "proj.m.top", "Function", CBM_LANG_PYTHON); + static const char *const defs[][3] = { + {"Tree", "proj.m.Tree", "Class"}, + {"visit", "proj.m.Tree.visit", "Method"}, + {"visit", "proj.m.Tree.visit.visit", "Function"}, + {"helper", "proj.m.Tree.helper", "Method"}, + {"factory", "proj.m.factory", "Function"}, + {"helper", "proj.m.factory.helper", "Function"}, + {"Local", "proj.m.factory.Local", "Class"}, + {"run", "proj.m.factory.Local.run", "Method"}, + {"top", "proj.m.top", "Function"}, + }; + for (size_t i = 0; i < sizeof(defs) / sizeof(defs[0]); i++) { + cbm_registry_add_lang_in_module(r, defs[i][0], defs[i][1], defs[i][2], CBM_LANG_PYTHON, + strlen("proj.m")); + } /* In method Tree.visit, `visit` is its nested def; so it is inside that * nested def (its own recursion). */ - cbm_resolution_t res = cbm_registry_resolve_lexical(r, "visit", "proj.m.Tree.visit"); + cbm_resolution_t res = cbm_registry_resolve_lexical(r, "visit", "proj.m.Tree.visit", "proj.m"); ASSERT_STR_EQ(res.qualified_name, "proj.m.Tree.visit.visit"); ASSERT_STR_EQ(res.strategy, "lexical_scope"); - res = cbm_registry_resolve_lexical(r, "visit", "proj.m.Tree.visit.visit"); + res = cbm_registry_resolve_lexical(r, "visit", "proj.m.Tree.visit.visit", "proj.m"); ASSERT_STR_EQ(res.qualified_name, "proj.m.Tree.visit.visit"); /* The class body is skipped: Tree.helper is not a bare name in a method. */ - res = cbm_registry_resolve_lexical(r, "helper", "proj.m.Tree.visit.visit"); + res = cbm_registry_resolve_lexical(r, "helper", "proj.m.Tree.visit.visit", "proj.m"); ASSERT_NULL(res.qualified_name); /* A method of a local class sees the def enclosing its class. */ - res = cbm_registry_resolve_lexical(r, "helper", "proj.m.factory.Local.run"); + res = cbm_registry_resolve_lexical(r, "helper", "proj.m.factory.Local.run", "proj.m"); ASSERT_STR_EQ(res.qualified_name, "proj.m.factory.helper"); - res = cbm_registry_resolve_lexical(r, "Local", "proj.m.factory"); + res = cbm_registry_resolve_lexical(r, "Local", "proj.m.factory", "proj.m"); ASSERT_STR_EQ(res.qualified_name, "proj.m.factory.Local"); /* Module-level names and dotted callees are not this strategy's. */ - res = cbm_registry_resolve_lexical(r, "top", "proj.m.factory"); + res = cbm_registry_resolve_lexical(r, "top", "proj.m.factory", "proj.m"); ASSERT_NULL(res.qualified_name); - res = cbm_registry_resolve_lexical(r, "self.helper", "proj.m.factory"); + res = cbm_registry_resolve_lexical(r, "self.helper", "proj.m.factory", "proj.m"); ASSERT_NULL(res.qualified_name); - res = cbm_registry_resolve_lexical(r, "helper", "proj.m"); + res = cbm_registry_resolve_lexical(r, "helper", "proj.m", "proj.m"); ASSERT_NULL(res.qualified_name); cbm_registry_free(r); PASS(); } +/* A package __init__.py function can share a submodule's QN: poetry's + * layouts/__init__.py `def layout` is proj.pkg.layout, the QN prefix of every + * symbol in layouts/layout.py. Bounded at its own file, class Layout and its + * methods are no locals of that function, in either registration order (full + * index: discovery order; incremental: node-id order), and the lexical walk of + * a caller in layout.py never enters the __init__ function. */ +TEST(registry_python_local_bound_at_own_file_issue1548) { + const size_t init_prefix = strlen("proj.pkg"); /* pkg/__init__.py */ + const size_t mod_prefix = strlen("proj.pkg.layout"); /* pkg/layout.py */ + for (int order = 0; order < 2; order++) { + cbm_registry_t *r = cbm_registry_new(); + if (order == 0) { + cbm_registry_add_lang_in_module(r, "layout", "proj.pkg.layout", "Function", + CBM_LANG_PYTHON, init_prefix); + } + cbm_registry_add_lang_in_module(r, "Layout", "proj.pkg.layout.Layout", "Class", + CBM_LANG_PYTHON, mod_prefix); + cbm_registry_add_lang_in_module(r, "create", "proj.pkg.layout.Layout.create", "Method", + CBM_LANG_PYTHON, mod_prefix); + cbm_registry_add_lang_in_module(r, "make", "proj.pkg.layout.make", "Function", + CBM_LANG_PYTHON, mod_prefix); + if (order == 1) { + cbm_registry_add_lang_in_module(r, "layout", "proj.pkg.layout", "Function", + CBM_LANG_PYTHON, init_prefix); + } + static const char *const nameable[][2] = {{"Layout", "proj.pkg.layout.Layout"}, + {"create", "proj.pkg.layout.Layout.create"}, + {"make", "proj.pkg.layout.make"}}; + for (size_t i = 0; i < sizeof(nameable) / sizeof(nameable[0]); i++) { + const char **out = NULL; + int count = 0; + cbm_registry_find_by_name(r, nameable[i][0], &out, &count); + ASSERT_EQ(count, 1); + ASSERT_STR_EQ(out[0], nameable[i][1]); + ASSERT_FALSE(cbm_registry_qn_is_function_local(r, nameable[i][1], mod_prefix)); + } + /* `Layout()` in layout.py's make(): no lexical local; module strategies own it. */ + cbm_resolution_t res = + cbm_registry_resolve_lexical(r, "Layout", "proj.pkg.layout.make", "proj.pkg.layout"); + ASSERT_NULL(res.qualified_name); + cbm_registry_free(r); + } + PASS(); +} + +/* The prefix every def of a file carries, without allocating; the same as + * cbm_pipeline_fqn_compute(project, path, name) minus ".name". */ +TEST(fqn_symbol_prefix_len_matches_fqn_compute) { + static const char *const paths[] = {"pkg/layout.py", "pkg/__init__.py", "a.py", + "deep/x/y/mod.py", "web/index.py", "noext"}; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + char *qn = cbm_pipeline_fqn_compute("proj", paths[i], "sym"); + ASSERT_NOT_NULL(qn); + size_t want = strlen(qn) - strlen(".sym"); + free(qn); + ASSERT_EQ(cbm_pipeline_fqn_symbol_prefix_len("proj", paths[i]), want); + } + ASSERT_EQ(cbm_pipeline_fqn_symbol_prefix_len("proj", "pkg/__init__.py"), strlen("proj.pkg")); + ASSERT_EQ(cbm_pipeline_fqn_symbol_prefix_len(NULL, "a.py"), 0); + PASS(); +} + SUITE(registry) { RUN_TEST(python_symbol_scope_len_preserves_file_identity); RUN_TEST(registry_python_package_scope_is_explicit); @@ -2161,4 +2237,6 @@ SUITE(registry) { RUN_TEST(weak_call_guards_share_one_drop_list); RUN_TEST(registry_python_function_local_is_exact_only_issue1548); RUN_TEST(registry_resolve_lexical_walks_enclosing_defs_issue1548); + RUN_TEST(registry_python_local_bound_at_own_file_issue1548); + RUN_TEST(fqn_symbol_prefix_len_matches_fqn_compute); }