diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 2ab97c94f..7417ebd52 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -877,6 +877,97 @@ static bool py_ns_delete(PyNamespaceBuilder *b, TSNode node, int depth) { return ok; } +static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node); + +/* `if TYPE_CHECKING:` / `if .TYPE_CHECKING:` */ +static bool py_ns_is_type_checking_if(PyNamespaceBuilder *b, TSNode node) { + TSNode cond = ts_node_child_by_field_name(node, TS_FIELD("condition")); + if (ts_node_is_null(cond)) + return false; + TSNode name = cond; + if (strcmp(ts_node_type(cond), "attribute") == 0) + name = ts_node_child_by_field_name(cond, TS_FIELD("attribute")); + const char *text = NULL; + size_t len = 0; + return !ts_node_is_null(name) && strcmp(ts_node_type(name), "identifier") == 0 && + py_ns_slice(b, name, &text, &len) && len == strlen("TYPE_CHECKING") && + memcmp(text, "TYPE_CHECKING", len) == 0; +} + +/* An import in an elif/else branch of a TYPE_CHECKING block rebinds its names + * at runtime from wherever that branch says (the compat shape: `else: try: + * from runtime.place import name`). A static reader cannot tell which source + * the module ends up exporting, so every name such an import binds becomes a + * SHADOW (reason UNSUPPORTED: the re-export is withdrawn, as for a later + * assignment); a star import, or an import form the builder does not read, + * may bind any name, so the whole namespace becomes unknown. An iterative + * walk of the branch: deep nesting must not exhaust the stack. */ +static bool py_ns_alternative_imports(PyNamespaceBuilder *b, TSNode root) { + TSTreeCursor cursor = ts_tree_cursor_new(root); + bool done = false; + while (!done && !b->failure) { + TSNode node = ts_tree_cursor_current_node(&cursor); + const char *kind = ts_node_type(node); + bool from = strcmp(kind, "import_from_statement") == 0; + bool is_import = from || strcmp(kind, "import_statement") == 0; + if (is_import) { + int start = b->facts.count; + bool star = !py_ns_import(b, node, from); + for (int i = start; !star && i < b->facts.count; i++) { + star = b->facts.items[i].kind == CBM_PY_NS_IMPORT_STAR; + b->facts.items[i] = (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = b->facts.items[i].local_name, + .reason = CBM_PY_NS_REASON_UNSUPPORTED}; + } + if (star && !b->failure) { + b->facts.count = start; + py_ns_unknown(b, CBM_PY_NS_REASON_COMPOUND); + } + } + if (!is_import && ts_tree_cursor_goto_first_child(&cursor)) + continue; + while (!ts_tree_cursor_goto_next_sibling(&cursor)) { + if (!ts_tree_cursor_goto_parent(&cursor)) { + done = true; + break; + } + } + } + ts_tree_cursor_delete(&cursor); + return !b->failure; +} + +/* The type checker view of a module-level `if TYPE_CHECKING:`: its + * consequence runs, its elif/else branches never do. Packages that load + * their public names lazily at runtime (`else: sys.modules[...] = + * _LazyModule(...)`) publish them exactly there, so a static reader of the + * namespace must take this branch to see them; every other compound + * statement stays a whole-namespace unknown. A name an elif/else branch + * imports again is withdrawn (py_ns_alternative_imports). */ +static bool py_ns_type_checking_block(PyNamespaceBuilder *b, TSNode node) { + TSNode body = ts_node_child_by_field_name(node, TS_FIELD("consequence")); + if (ts_node_is_null(body)) + return py_ns_unknown(b, CBM_PY_NS_REASON_COMPOUND); + uint32_t count = ts_node_named_child_count(body); + for (uint32_t i = 0; i < count && !b->failure; i++) { + TSNode stmt = ts_node_named_child(body, i); + int start = b->facts.count; + if (!py_ns_statement(b, stmt) && !b->failure) { + b->facts.count = start; + if (!py_ns_unknown(b, CBM_PY_NS_REASON_UNSUPPORTED)) + return false; + } + } + uint32_t children = ts_node_child_count(node); + for (uint32_t i = 0; i < children && !b->failure; i++) { + TSNode alt = ts_node_child(node, i); + const char *kind = ts_node_type(alt); + if (strcmp(kind, "elif_clause") == 0 || strcmp(kind, "else_clause") == 0) + py_ns_alternative_imports(b, alt); + } + return !b->failure; +} + static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { const char *kind = ts_node_type(node); if (strcmp(kind, "comment") == 0 || strcmp(kind, "pass_statement") == 0) @@ -922,6 +1013,8 @@ static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { return py_ns_assignment(b, expr); return py_ns_safe_expr(b, expr, 0) || py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); } + if (strcmp(kind, "if_statement") == 0 && py_ns_is_type_checking_if(b, node)) + return py_ns_type_checking_block(b, node); if (strcmp(kind, "if_statement") == 0 || strcmp(kind, "try_statement") == 0 || strcmp(kind, "for_statement") == 0 || strcmp(kind, "while_statement") == 0 || strcmp(kind, "with_statement") == 0 || strcmp(kind, "match_statement") == 0) diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index c63a6cfc1..9d3cf486e 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -13,6 +13,7 @@ #include "py_lsp.h" #include "../cbm.h" #include "../helpers.h" +#include "../../../src/foundation/hash_table.h" /* module-scope suffix index (re-exports) */ #include "lsp_node_iter.h" #include "tree_sitter/api.h" #include @@ -252,6 +253,15 @@ static void py_process_class_in(PyLSPContext *ctx, TSNode class_node, const char typedef enum { PY_LOCAL_DEFS_BIND, PY_LOCAL_DEFS_PROCESS } PyLocalDefsMode; static void py_visit_local_defs(PyLSPContext *ctx, TSNode block, PyLocalDefsMode mode, int depth); +/* A name looked up in a module namespace: the module's own function or + * class, or the definition the module re-exports under that name. At most + * one of func/type is set. */ +typedef struct { + const CBMRegisteredFunc *func; + const CBMRegisteredType *type; +} PyModuleMember; +static PyModuleMember py_module_member(PyLSPContext *ctx, const char *module_qn, const char *name); + /* ── Scope mutation wrappers ───────────────────────────────────────────── * * py_eval_expr_type memoizes per-node results (see the cache block above @@ -739,6 +749,38 @@ static void py_classify_imports_for_root(PyLSPContext *ctx, TSNode root) { } } +/* `from X import Y` names X.Y, which need not be a definition: + * - X re-exports Y (`from .fields import *` in X/__init__.py): bind the + * definition Y resolves to, and record that canonical QN for the import + * so every later consumer of the import metadata agrees; + * - Y is a submodule (`from django.db import models`): bind the MODULE, so + * `models.CharField` is a module-namespace lookup, not a class attribute. + * A registered X.Y (the ordinary case) keeps its NAMED binding. */ +static const CBMType *py_canonical_named_import(PyLSPContext *ctx, int index, const char **qn_io, + const CBMType *named) { + const char *qn = *qn_io; + const CBMTypeRegistry *reg = ctx->registry; + if (!reg || cbm_registry_lookup_type(reg, qn) || cbm_registry_lookup_func(reg, qn)) + return named; + const char *dot = strrchr(qn, '.'); + if (dot && dot > qn) { + /* the export table is keyed by symbol scope: a package import value + * may still carry the package file identity (P.pkg.__init__.Y) */ + char *module = cbm_arena_strndup(ctx->arena, qn, (size_t)(dot - qn)); + if (module) + module[cbm_fqn_symbol_scope_len(module)] = '\0'; + const char *target = module ? cbm_registry_lookup_export(reg, module, dot + 1) : NULL; + if (target) { + *qn_io = target; + ctx->import_module_qns[index] = target; + return cbm_type_named(ctx->arena, target); + } + } + if (cbm_registry_is_module(reg, qn)) + return cbm_type_module(ctx->arena, qn); + return named; +} + static void py_bind_import_index(PyLSPContext *ctx, int index, bool synthetic_fallback) { if (!ctx || !ctx->current_scope || index < 0 || index >= ctx->import_count) return; @@ -777,6 +819,8 @@ static void py_bind_import_index(PyLSPContext *ctx, int index, bool synthetic_fa // `import X` / `import X as Y` (single segment) — MODULE(X). t = cbm_type_module(ctx->arena, qn); } + if (t->kind == CBM_TYPE_NAMED) + t = py_canonical_named_import(ctx, index, &qn, t); const CBMRegisteredFunc *imported = from_style && ctx->registry ? cbm_registry_lookup_func(ctx->registry, qn) : NULL; if (py_func_is_exact_callable_value(imported)) { @@ -1132,7 +1176,7 @@ static const char *py_exact_callable_target_ex(PyLSPContext *ctx, TSNode node, receiver = cbm_type_resolve_alias(receiver); if (receiver->kind == CBM_TYPE_MODULE) { const CBMRegisteredFunc *f = - cbm_registry_lookup_symbol(ctx->registry, receiver->data.module.module_qn, member); + py_module_member(ctx, receiver->data.module.module_qn, member).func; return py_func_is_exact_callable_value(f) ? f->qualified_name : NULL; } if (receiver->kind == CBM_TYPE_NAMED) { @@ -1224,6 +1268,85 @@ static const CBMRegisteredFunc *py_lookup_attribute_depth(PyLSPContext *ctx, con return NULL; } +static PyModuleMember py_module_member_exact(PyLSPContext *ctx, const char *module_qn, + const char *name) { + PyModuleMember m = {0}; + m.func = cbm_registry_lookup_symbol(ctx->registry, module_qn, name); + if (m.func) + return m; + const char *qn = cbm_arena_sprintf(ctx->arena, "%s.%s", module_qn, name); + m.type = qn ? cbm_registry_lookup_type(ctx->registry, qn) : NULL; + if (m.type) + return m; + /* Re-export: `module.name` is bound by the module's imports (the export + * table the cross registry materialized once, before its seal), keyed by + * the module symbol scope (a package without its `.__init__`). */ + size_t scope_len = cbm_fqn_symbol_scope_len(module_qn); + const char *scope = + module_qn[scope_len] ? cbm_arena_strndup(ctx->arena, module_qn, scope_len) : module_qn; + const char *target = scope ? cbm_registry_lookup_export(ctx->registry, scope, name) : NULL; + if (target) { + m.func = cbm_registry_lookup_func(ctx->registry, target); + if (!m.func) + m.type = cbm_registry_lookup_type(ctx->registry, target); + } + return m; +} + +/* `prefix.module_qn`, or NULL when module_qn already starts with prefix. */ +static const char *py_qualify_module(PyLSPContext *ctx, const char *prefix, size_t prefix_len, + const char *module_qn) { + if (prefix_len == 0 || + (strncmp(module_qn, prefix, prefix_len) == 0 && module_qn[prefix_len] == '.')) + return NULL; + return cbm_arena_sprintf(ctx->arena, "%.*s.%s", (int)prefix_len, prefix, module_qn); +} + +/* An in-project `import sibling` / `import pkg` binds the module by its + * source spelling ("pkg"), while its defs are registered project-qualified. + * Try the spelling as written, then qualified by the importing file's + * package, then by the project root (the file module's first segment). */ +static PyModuleMember py_module_member(PyLSPContext *ctx, const char *module_qn, const char *name) { + PyModuleMember none = {0}; + if (!ctx || !ctx->registry || !module_qn || !name) + return none; + PyModuleMember m = py_module_member_exact(ctx, module_qn, name); + if (m.func || m.type || !ctx->file_module_qn) + return m; + const char *file_mod = ctx->file_module_qn; + const char *last_dot = strrchr(file_mod, '.'); + const char *first_dot = strchr(file_mod, '.'); + if (last_dot && last_dot > file_mod) { + const char *qual = + py_qualify_module(ctx, file_mod, (size_t)(last_dot - file_mod), module_qn); + m = qual ? py_module_member_exact(ctx, qual, name) : none; + if (m.func || m.type) + return m; + } + if (first_dot && first_dot != last_dot) { + const char *qual = + py_qualify_module(ctx, file_mod, (size_t)(first_dot - file_mod), module_qn); + m = qual ? py_module_member_exact(ctx, qual, name) : none; + } + return m; +} + +/* Emit a call resolved through a module namespace. A re-export may bind the + * definition under another name (`from .x import Orig as Renamed`): that is + * an import alias, whose join reads the written name from `reason`. */ +static void py_emit_module_member_call(PyLSPContext *ctx, const char *callee_qn, + const char *strategy, float confidence, + const char *written_name, TSNode site) { + const char *dot = strrchr(callee_qn, '.'); + const char *short_name = dot ? dot + 1 : callee_qn; + if (strcmp(short_name, written_name) != 0) { + py_emit_resolved_call_reason(ctx, callee_qn, "lsp_import_alias", confidence, written_name, + site); + } else { + py_emit_resolved_call(ctx, callee_qn, strategy, confidence, site); + } +} + /* ── The C3 method resolution order ──────────────────────────────── */ /* Longest linearization followed. A longer one (or more bases than this) keeps @@ -1939,16 +2062,17 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) return cbm_type_unknown(); if (obj_type->kind == CBM_TYPE_MODULE) { - // module.attr — return type of the registered symbol if known. + // module.attr — its own function or class, or what it re-exports + // under the name. const char *mod = obj_type->data.module.module_qn; - const CBMRegisteredFunc *f = cbm_registry_lookup_symbol(ctx->registry, mod, attr_name); - if (f) - return py_func_return_type(ctx, f->qualified_name); - // Could also be a class — return NAMED("mod.attr") + PyModuleMember mm = py_module_member(ctx, mod, attr_name); + if (mm.func) + return py_func_return_type(ctx, mm.func->qualified_name); + if (mm.type) + return cbm_type_named(ctx->arena, mm.type->qualified_name); const char *qn = cbm_arena_sprintf(ctx->arena, "%s.%s", mod, attr_name); - const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, qn); - if (rt) - return cbm_type_named(ctx->arena, qn); + if (qn && cbm_registry_is_module(ctx->registry, qn)) + return cbm_type_module(ctx->arena, qn); if (py_names_submodule(ctx, qn)) return cbm_type_module(ctx->arena, qn); return cbm_type_unknown(); @@ -2229,10 +2353,11 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) if (obj_type->kind == CBM_TYPE_MODULE) { const char *mod = obj_type->data.module.module_qn; - const CBMRegisteredFunc *f = - cbm_registry_lookup_symbol(ctx->registry, mod, attr_name); - if (f) - return py_func_return_type(ctx, f->qualified_name); + PyModuleMember mm = py_module_member(ctx, mod, attr_name); + if (mm.func) + return py_func_return_type(ctx, mm.func->qualified_name); + if (mm.type) // `mod.Class()` constructs an instance + return cbm_type_named(ctx->arena, mm.type->qualified_name); } if (obj_type->kind == CBM_TYPE_NAMED) { const CBMRegisteredFunc *f = @@ -3246,46 +3371,21 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { if (obj_type->kind == CBM_TYPE_MODULE) { const char *mod = obj_type->data.module.module_qn; - const CBMRegisteredFunc *f = cbm_registry_lookup_symbol(ctx->registry, mod, attr_name); - if (f) { - py_emit_resolved_call(ctx, f->qualified_name, "lsp_module_attr", 0.92f, call_node); + // py_module_member also retries an in-project module written in its + // short source form ("helpers") project-qualified; a genuinely + // external module (requests, os) has no project def, so it stays + // lsp_module_attr_unresolved below. It also answers a name the + // module re-exports (`models.CharField`). + PyModuleMember mm = py_module_member(ctx, mod, attr_name); + if (mm.func) { + py_emit_module_member_call(ctx, mm.func->qualified_name, "lsp_module_attr", 0.92f, + attr_name, call_node); return; } - // An `import sibling` of an IN-PROJECT module records the module's QN - // in its short, source-written form ("helpers"), but the sibling's - // defs are registered project-qualified (".helpers.do_work"). - // So the lookup above misses for in-project modules even though the - // target IS resolvable, and the call used to drop to - // lsp_module_attr_unresolved @0.55 (below the join's 0.6 floor) — no - // edge. Retry against the project-qualified module: derive the - // project root from the current file's module_qn (strip its last - // segment) and look up ".". A genuinely-external module - // (requests, os) has no such project def, so it correctly stays - // lsp_module_attr_unresolved. - if (mod && ctx->file_module_qn) { - const char *last_dot = strrchr(ctx->file_module_qn, '.'); - if (last_dot && last_dot > ctx->file_module_qn) { - size_t root_len = (size_t)(last_dot - ctx->file_module_qn); - // Skip if mod is already rooted under the project to avoid - // "..mod". - if (!(strncmp(mod, ctx->file_module_qn, root_len) == 0 && - mod[root_len] == '.')) { - char *qual_mod = - (char *)cbm_arena_alloc(ctx->arena, root_len + 1 + strlen(mod) + 1); - if (qual_mod) { - memcpy(qual_mod, ctx->file_module_qn, root_len); - qual_mod[root_len] = '.'; - strcpy(qual_mod + root_len + 1, mod); - const CBMRegisteredFunc *qf = - cbm_registry_lookup_symbol(ctx->registry, qual_mod, attr_name); - if (qf) { - py_emit_resolved_call(ctx, qf->qualified_name, "lsp_module_attr", - 0.92f, call_node); - return; - } - } - } - } + if (mm.type) { + py_emit_module_member_call(ctx, mm.type->qualified_name, "lsp_constructor", 0.88f, + attr_name, call_node); + return; } // Best-effort: emit "module.attr" QN — Phase 9 cross-file may fix up. const char *qn = cbm_arena_sprintf(ctx->arena, "%s.%s", mod, attr_name); @@ -6171,6 +6271,453 @@ bool cbm_py_lsp_test_register_defs(CBMArena *arena, CBMTypeRegistry *reg, CBMLSP } #endif +/* ── Module namespaces: package re-exports ──────────────────────────────── + * + * cbm_pxc_collect_all_defs carries each Python module's namespace records + * (PyReexport / PyStarImport / PyAll, built from its ordered namespace facts) + * next to its definitions. py_build_module_namespaces folds them into the + * registry's export table ONCE, after the func/type indexes are finalized + * (every probe below is a hashed lookup or a binary search) and before the + * seal. Per module it computes the namespace its imports bind: a re-export + * resolves its target through further re-exports (chains); a star copies the + * source module's namespace, restricted to its literal `__all__` when it has + * one and otherwise to names without a leading `_`; the module's own + * definitions are never replaced. Modules are visited in sorted order and + * records in source order; a module reached again while its own namespace is + * being computed (an import cycle) contributes what it has so far, so every + * cycle ends with the same answer on every run. PY_NS_MAX_DEPTH bounds only + * the native recursion (no project re-exports through 64 modules in a row). */ +enum { PY_NS_MAX_DEPTH = 64, PY_NS_MAP_MIN_CAP = 8 }; + +typedef struct { + const char *name; /* short name (borrowed); NULL = empty slot */ + const char *target; /* definition QN, registry-owned */ + bool own; /* the module's own definition: imports never replace it */ +} PyNsSlot; + +typedef struct { + PyNsSlot *slots; + int cap; /* power of two */ + int count; +} PyNsMap; + +typedef struct { + const char *scope; + int *records; /* def indices of PyReexport / PyStarImport, source order */ + int record_count; + const char **all_names; /* NULL-terminated when has_all */ + bool has_all; + bool star_source; + int state; /* 0 = pending, 1 = computing, 2 = done */ + PyNsMap ns; +} PyNsModule; + +typedef struct { + CBMArena *scratch; + const CBMTypeRegistry *reg; + const CBMLSPDef *defs; + PyNsModule *mods; + const char **scopes; /* sorted, parallel to mods */ + int mod_count; + CBMHashTable *by_suffix; /* dot-boundary scope tail -> module index + 1, or PY_NS_AMBIGUOUS */ +} PyNsBuild; + +#define PY_NS_AMBIGUOUS ((void *)(uintptr_t)UINTPTR_MAX) + +static bool py_ns_is_record(const CBMLSPDef *d) { + return strcmp(d->label, "PyReexport") == 0 || strcmp(d->label, "PyStarImport") == 0 || + strcmp(d->label, "PyAll") == 0; +} + +static uint32_t py_ns_hash(const char *s) { + uint32_t h = 2166136261u; + for (; *s; s++) + h = (h ^ (unsigned char)*s) * 16777619u; + return h; +} + +static PyNsSlot *py_ns_find(const PyNsMap *map, const char *name) { + if (!map->slots) + return NULL; + uint32_t mask = (uint32_t)map->cap - 1; + for (uint32_t i = py_ns_hash(name) & mask;; i = (i + 1) & mask) { + PyNsSlot *s = &map->slots[i]; + if (!s->name || strcmp(s->name, name) == 0) + return s; + } +} + +static bool py_ns_grow(CBMArena *scratch, PyNsMap *map) { + int cap = map->cap ? map->cap * 2 : PY_NS_MAP_MIN_CAP; + PyNsSlot *slots = (PyNsSlot *)cbm_arena_alloc(scratch, (size_t)cap * sizeof(PyNsSlot)); + if (!slots) + return false; + memset(slots, 0, (size_t)cap * sizeof(PyNsSlot)); + PyNsMap grown = {slots, cap, 0}; + for (int i = 0; i < map->cap; i++) { + if (map->slots[i].name) { + *py_ns_find(&grown, map->slots[i].name) = map->slots[i]; + grown.count++; + } + } + *map = grown; + return true; +} + +/* Bind name -> target. A later import rebinds an earlier one (Python's + * semantics); nothing rebinds the module's own definition. */ +static void py_ns_put(CBMArena *scratch, PyNsMap *map, const char *name, const char *target, + bool own) { + if ((map->count + 1) * 2 > map->cap && !py_ns_grow(scratch, map)) + return; + PyNsSlot *s = py_ns_find(map, name); + if (s->name && s->own) + return; + if (!s->name) + map->count++; + s->name = name; + s->target = target; + s->own = own; +} + +static int py_ns_module_index(const PyNsBuild *b, const char *scope) { + int lo = 0; + int hi = b->mod_count - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + int cmp = strcmp(b->scopes[mid], scope); + if (cmp == 0) + return mid; + if (cmp < 0) + lo = mid + 1; + else + hi = mid - 1; + } + return -1; +} + +static void py_ns_compute(PyNsBuild *b, int m, int depth); + +/* The definition `qn` names: itself when registered, else the name its + * module's namespace binds (a chained re-export), else NULL. */ +static const char *py_ns_resolve(PyNsBuild *b, const char *qn, int depth) { + const CBMRegisteredFunc *f = cbm_registry_lookup_func(b->reg, qn); + if (f) + return f->qualified_name; + const CBMRegisteredType *t = cbm_registry_lookup_type(b->reg, qn); + if (t) + return t->qualified_name; + const char *dot = strrchr(qn, '.'); + if (!dot || dot == qn || depth > PY_NS_MAX_DEPTH) + return NULL; + char *scope = cbm_arena_strndup(b->scratch, qn, (size_t)(dot - qn)); + int m = scope ? py_ns_module_index(b, scope) : -1; + if (m < 0) + return NULL; + py_ns_compute(b, m, depth + 1); + const PyNsSlot *s = py_ns_find(&b->mods[m].ns, dot + 1); + return s && s->name ? s->target : NULL; /* NULL target = tombstone */ +} + +/* The parent of a dotted scope ("a.b.c" -> "a.b"), or NULL at the root. */ +static char *py_ns_parent(CBMArena *scratch, const char *scope) { + const char *dot = strrchr(scope, '.'); + return dot && dot > scope ? cbm_arena_strndup(scratch, scope, (size_t)(dot - scope)) : NULL; +} + +/* Module index a record raw module spec names, seen from the module that + * wrote it (importer_qn: its raw module QN, `.__init__` kept), or -1. + * Relative (`..pkg.mod`): the importer package -- the importer itself for a + * package `__init__`, else its parent -- ascended once per further dot, then + * the dotted rest; exact. Absolute (`app.db.models`): the one project module + * scope whose dot-boundary tail spells it; none or several = unknown (a + * vendored copy, an external module), never a guess between candidates. */ +static int py_ns_resolve_spec(PyNsBuild *b, const char *spec, const char *importer_qn) { + if (!spec || !importer_qn) + return -1; + size_t level = 0; + while (spec[level] == '.') + level++; + const char *rest = spec + level; + if (level == 0) { + void *v = b->by_suffix ? cbm_ht_get(b->by_suffix, rest) : NULL; + if (!v || v == PY_NS_AMBIGUOUS) + return -1; + return (int)((uintptr_t)v - 1); + } + size_t scope_len = cbm_fqn_symbol_scope_len(importer_qn); + char *pkg = cbm_arena_strndup(b->scratch, importer_qn, scope_len); + bool is_package = importer_qn[scope_len] != '\0'; + if (pkg && !is_package) + pkg = py_ns_parent(b->scratch, pkg); + for (size_t up = 1; pkg && up < level; up++) + pkg = py_ns_parent(b->scratch, pkg); + if (!pkg) + return -1; + const char *module = rest[0] ? cbm_arena_sprintf(b->scratch, "%s.%s", pkg, rest) : pkg; + return module ? py_ns_module_index(b, module) : -1; +} + +/* Index every dot-boundary tail of every module scope ("p.app.db" -> "app.db", + * "db"); a tail two modules share is ambiguous. */ +static bool py_ns_build_suffix_index(PyNsBuild *b) { + b->by_suffix = cbm_ht_create((uint32_t)(b->mod_count * 4 + 16)); + if (!b->by_suffix) + return false; + for (int m = 0; m < b->mod_count; m++) { + const char *s = b->scopes[m]; + for (const char *tail = s; tail; tail = strchr(tail, '.') ? strchr(tail, '.') + 1 : NULL) { + void *prev = cbm_ht_get(b->by_suffix, tail); + cbm_ht_set(b->by_suffix, tail, prev ? PY_NS_AMBIGUOUS : (void *)(uintptr_t)(m + 1)); + } + } + return true; +} + +/* An unresolvable star is a whole-namespace barrier: every name the + * module's imports bound so far becomes unknown (a tombstone, so the + * open-addressing chains stay intact); own definitions stay. */ +static void py_ns_barrier(PyNsModule *mod) { + for (int i = 0; i < mod->ns.cap; i++) { + if (mod->ns.slots[i].name && !mod->ns.slots[i].own) + mod->ns.slots[i].target = NULL; + } +} + +/* `from src import *` into dst. */ +static void py_ns_star_merge(PyNsBuild *b, PyNsModule *dst, const PyNsModule *src) { + if (src->has_all) { + for (int i = 0; src->all_names && src->all_names[i]; i++) { + const PyNsSlot *s = py_ns_find(&src->ns, src->all_names[i]); + if (s && s->name && s->target) + py_ns_put(b->scratch, &dst->ns, s->name, s->target, false); + } + return; + } + for (int i = 0; i < src->ns.cap; i++) { + const PyNsSlot *s = &src->ns.slots[i]; + if (s->name && s->target && s->name[0] != '_') + py_ns_put(b->scratch, &dst->ns, s->name, s->target, false); + } +} + +static void py_ns_compute(PyNsBuild *b, int m, int depth) { + PyNsModule *mod = &b->mods[m]; + if (mod->state != 0 || depth > PY_NS_MAX_DEPTH) + return; + mod->state = 1; + for (int r = 0; r < mod->record_count; r++) { + const CBMLSPDef *d = &b->defs[mod->records[r]]; + if (!d->receiver_type) + continue; + int src = py_ns_resolve_spec(b, d->receiver_type, d->def_module_qn); + if (strcmp(d->label, "PyReexport") == 0) { + const char *qn = + src >= 0 && d->method_names_str + ? cbm_arena_sprintf(b->scratch, "%s.%s", b->scopes[src], d->method_names_str) + : NULL; + const char *target = qn ? py_ns_resolve(b, qn, depth + 1) : NULL; + /* unresolved: the name is still rebound, to something unknown */ + py_ns_put(b->scratch, &mod->ns, d->short_name, target, false); + continue; + } + if (src < 0) { + py_ns_barrier(mod); + continue; + } + if (src == m) + continue; + py_ns_compute(b, src, depth + 1); + py_ns_star_merge(b, mod, &b->mods[src]); + } + mod->state = 2; +} + +static int py_ns_cmp_str(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +typedef struct { + const char *key; + const char *target; +} PyNsExport; + +static int py_ns_cmp_export(const void *a, const void *b) { + return strcmp(((const PyNsExport *)a)->key, ((const PyNsExport *)b)->key); +} + +/* The symbol scope of a module QN (a package's `.__init__` dropped). */ +static const char *py_ns_scope(CBMArena *scratch, const char *module_qn) { + return cbm_arena_strndup(scratch, module_qn, cbm_fqn_symbol_scope_len(module_qn)); +} + +/* Sorted, de-duplicated module scopes of every Python def and record (and + * every star source). Returns the count; def_scope[i] is def i's scope + * (NULL when def i is skipped). */ +static int py_ns_collect_scopes(PyNsBuild *b, int def_count, bool check_lang, + const char **def_scope) { + const char **all = + (const char **)cbm_arena_alloc(b->scratch, (size_t)(def_count * 2 + 1) * sizeof(char *)); + if (!all) + return -1; + int n = 0; + for (int i = 0; i < def_count; i++) { + const CBMLSPDef *d = &b->defs[i]; + def_scope[i] = NULL; + if ((check_lang && d->lang != CBM_LANG_PYTHON) || !d->label || !d->qualified_name || + !d->def_module_qn) + continue; + def_scope[i] = py_ns_scope(b->scratch, d->def_module_qn); + if (!def_scope[i]) + continue; + all[n++] = def_scope[i]; + } + qsort(all, (size_t)n, sizeof(char *), py_ns_cmp_str); + int u = 0; + for (int i = 0; i < n; i++) { + if (u == 0 || strcmp(all[u - 1], all[i]) != 0) + all[u++] = all[i]; + } + b->scopes = all; + return u; +} + +/* Attach records, __all__ lists and (for star sources) own top-level + * definitions to their modules. */ +static bool py_ns_attach(PyNsBuild *b, int def_count, const char **def_scope) { + int *def_mod = (int *)cbm_arena_alloc(b->scratch, (size_t)(def_count + 1) * sizeof(int)); + if (!def_mod) + return false; + for (int i = 0; i < def_count; i++) { + const CBMLSPDef *d = &b->defs[i]; + def_mod[i] = def_scope[i] ? py_ns_module_index(b, def_scope[i]) : -1; + if (def_mod[i] < 0 || !py_ns_is_record(d)) + continue; + PyNsModule *mod = &b->mods[def_mod[i]]; + if (strcmp(d->label, "PyAll") == 0) { + mod->has_all = true; + mod->all_names = py_split_pipe(b->scratch, d->method_names_str); + continue; + } + mod->record_count++; + if (strcmp(d->label, "PyStarImport") == 0 && d->receiver_type) { + int src = py_ns_resolve_spec(b, d->receiver_type, d->def_module_qn); + if (src >= 0) + b->mods[src].star_source = true; + } + } + for (int m = 0; m < b->mod_count; m++) { + PyNsModule *mod = &b->mods[m]; + if (mod->record_count > 0) { + mod->records = + (int *)cbm_arena_alloc(b->scratch, (size_t)mod->record_count * sizeof(int)); + if (!mod->records) + return false; + mod->record_count = 0; + } + } + for (int i = 0; i < def_count; i++) { + if (def_mod[i] < 0) + continue; + const CBMLSPDef *d = &b->defs[i]; + PyNsModule *mod = &b->mods[def_mod[i]]; + if (py_ns_is_record(d)) { + if (strcmp(d->label, "PyAll") != 0) + mod->records[mod->record_count++] = i; + continue; + } + /* A star source exports its own top-level classes and functions + * (direct children of the module scope, never methods). */ + size_t scope_len = strlen(def_scope[i]); + if (!mod->star_source || !d->short_name || d->receiver_type || + strncmp(d->qualified_name, def_scope[i], scope_len) != 0 || + d->qualified_name[scope_len] != '.' || + strcmp(d->qualified_name + scope_len + 1, d->short_name) != 0) + continue; + const CBMRegisteredFunc *f = cbm_registry_lookup_func(b->reg, d->qualified_name); + const CBMRegisteredType *t = f ? NULL : cbm_registry_lookup_type(b->reg, d->qualified_name); + const char *target = f ? f->qualified_name : t ? t->qualified_name : NULL; + if (target) + py_ns_put(b->scratch, &mod->ns, d->short_name, target, true); + } + return true; +} + +/* Copy the computed namespaces (import-bound names only) and the module set + * into the registry's arena, sorted for binary search. */ +static void py_ns_publish(PyNsBuild *b, CBMTypeRegistry *reg) { + int total = 0; + for (int m = 0; m < b->mod_count; m++) { + for (int i = 0; i < b->mods[m].ns.cap; i++) + total += b->mods[m].ns.slots[i].name && !b->mods[m].ns.slots[i].own && + b->mods[m].ns.slots[i].target; + } + const char **module_qns = + (const char **)cbm_arena_alloc(reg->arena, (size_t)(b->mod_count + 1) * sizeof(char *)); + PyNsExport *exports = + (PyNsExport *)cbm_arena_alloc(b->scratch, (size_t)(total + 1) * sizeof(PyNsExport)); + const char **keys = + (const char **)cbm_arena_alloc(reg->arena, (size_t)(total + 1) * sizeof(char *)); + const char **targets = + (const char **)cbm_arena_alloc(reg->arena, (size_t)(total + 1) * sizeof(char *)); + if (!module_qns || !exports || !keys || !targets) + return; + int n = 0; + for (int m = 0; m < b->mod_count; m++) { + module_qns[m] = cbm_arena_strdup(reg->arena, b->scopes[m]); + const PyNsMap *ns = &b->mods[m].ns; + for (int i = 0; i < ns->cap; i++) { + const PyNsSlot *s = &ns->slots[i]; + if (!s->name || s->own || !s->target) + continue; + exports[n].key = cbm_arena_sprintf(reg->arena, "%s.%s", b->scopes[m], s->name); + exports[n].target = s->target; + if (exports[n].key) + n++; + } + } + qsort(exports, (size_t)n, sizeof(PyNsExport), py_ns_cmp_export); + for (int i = 0; i < n; i++) { + keys[i] = exports[i].key; + targets[i] = exports[i].target; + } + reg->module_qns = module_qns; + reg->module_count = b->mod_count; + reg->export_keys = keys; + reg->export_targets = targets; + reg->export_count = n; +} + +/* Build reg's module set and export table from defs[]. reg must be + * finalized and not yet sealed. check_lang: defs[] is mixed-language. */ +static void py_build_module_namespaces(CBMTypeRegistry *reg, CBMLSPDef *defs, int def_count, + bool check_lang) { + if (!reg || !defs || def_count <= 0 || reg->read_only) + return; + CBMArena scratch; + cbm_arena_init(&scratch); + PyNsBuild b = {.scratch = &scratch, .reg = reg, .defs = defs}; + const char **def_scope = + (const char **)cbm_arena_alloc(&scratch, (size_t)def_count * sizeof(char *)); + int mod_count = def_scope ? py_ns_collect_scopes(&b, def_count, check_lang, def_scope) : -1; + if (mod_count > 0) { + b.mod_count = mod_count; + b.mods = (PyNsModule *)cbm_arena_alloc(&scratch, (size_t)mod_count * sizeof(PyNsModule)); + } + if (b.mods) { + memset(b.mods, 0, (size_t)mod_count * sizeof(PyNsModule)); + for (int m = 0; m < mod_count; m++) + b.mods[m].scope = b.scopes[m]; + if (py_ns_build_suffix_index(&b) && py_ns_attach(&b, def_count, def_scope)) { + for (int m = 0; m < mod_count; m++) + py_ns_compute(&b, m, 0); + py_ns_publish(&b, reg); + } + } + cbm_ht_free(b.by_suffix); + cbm_arena_destroy(&scratch); +} + bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, @@ -6226,6 +6773,7 @@ CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, in /* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize" * comment for the rationale. */ cbm_registry_finalize_into(®, &idx_arena); + py_build_module_namespaces(®, defs, def_count, false); ctx.syn_calls = synthetic_calls; for (int i = 0; i < import_count; i++) { @@ -6268,6 +6816,8 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i py_mark_ambiguous_callable_bindings(reg, arena); cbm_registry_build_type_qn_order(reg, arena); cbm_registry_finalize(reg); + /* Export table: after finalize (hashed probes), before the seal. */ + py_build_module_namespaces(reg, defs, def_count, true); reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */ return reg; } diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index e943c2522..d381b5696 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -964,6 +964,64 @@ const CBMRegisteredFunc *cbm_registry_lookup_method_aliased(const CBMTypeRegistr return NULL; } +/* Binary search of a sorted string array; -1 when absent. The key is `prefix` + * alone (name == NULL) or `prefix.name`, compared without building it. */ +static int registry_bsearch_qn(const char *const *arr, int count, const char *prefix, + const char *name) { + size_t prefix_len = strlen(prefix); + int lo = 0; + int hi = count - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + const char *s = arr[mid]; + int cmp = strncmp(s, prefix, prefix_len); + if (cmp == 0) { + const char *rest = s + prefix_len; + if (!name) { + cmp = *rest ? 1 : 0; + } else if (*rest != '.') { + cmp = (unsigned char)*rest - (unsigned char)'.'; + } else { + cmp = strcmp(rest + 1, name); + } + } + if (cmp == 0) + return mid; + if (cmp < 0) + lo = mid + 1; + else + hi = mid - 1; + } + return -1; +} + +bool cbm_registry_is_module(const CBMTypeRegistry *reg, const char *module_qn) { + if (!module_qn) + return false; + for (const CBMTypeRegistry *r = reg; r; r = r->fallback) { + CBM_LSP_WORK(1); + if (r->module_qns && + registry_bsearch_qn(r->module_qns, r->module_count, module_qn, NULL) >= 0) + return true; + } + return false; +} + +const char *cbm_registry_lookup_export(const CBMTypeRegistry *reg, const char *module_qn, + const char *name) { + if (!module_qn || !name) + return NULL; + for (const CBMTypeRegistry *r = reg; r; r = r->fallback) { + CBM_LSP_WORK(1); + if (!r->export_keys) + continue; + int i = registry_bsearch_qn(r->export_keys, r->export_count, module_qn, name); + if (i >= 0) + return r->export_targets[i]; + } + return NULL; +} + const CBMRegisteredFunc *cbm_registry_lookup_symbol(const CBMTypeRegistry *reg, const char *package_qn, const char *name) { if (!reg || !package_qn || !name) diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index 836f65f87..29bc0a351 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -181,6 +181,19 @@ typedef struct CBMTypeRegistry { CBMRegistryTypeQNEntry *type_qn_sorted; int type_qn_sorted_count; int type_qn_sorted_upto; + + /* Module namespaces (Python re-exports). module_qns: every module symbol + * scope with definitions or namespace facts, sorted (strcmp). export_keys / + * export_targets: `module.name` -> the definition QN the module's imports + * bind that name to (re-exports), sorted by key. Built once by the language + * builder after finalize and before the seal; lookups are binary searches + * and never mutate. Separate from the func/type arrays by design: a + * re-export is an alias of a definition, never a definition of its own. */ + const char **module_qns; + int module_count; + const char **export_keys; + const char **export_targets; + int export_count; } CBMTypeRegistry; // Initialize a registry. @@ -229,6 +242,13 @@ const CBMRegisteredFunc *cbm_registry_lookup_func(const CBMTypeRegistry *reg, const CBMRegisteredFunc *cbm_registry_lookup_symbol(const CBMTypeRegistry *reg, const char *package_qn, const char *name); +// Module-namespace lookups (CBMTypeRegistry.module_qns / export_keys); both +// chain to reg->fallback. cbm_registry_lookup_export returns the definition QN +// `module_qn.name` is re-exported as, or NULL. +bool cbm_registry_is_module(const CBMTypeRegistry *reg, const char *module_qn); +const char *cbm_registry_lookup_export(const CBMTypeRegistry *reg, const char *module_qn, + const char *name); + /* Build in an arena that outlives every use (owner arena for shared registries, * walk scratch for per-file registries). QN strings and registered identities * must remain immutable while a snapshot is used; end its use before changing diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index fd56d1ff8..5aa955112 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -958,6 +958,220 @@ static int pxc_build_rust_impl_relation(CBMArena *arena, const CBMImplTrait *imp return 0; } +/* ── Python namespace records (package re-exports) ──────────────────────── + * + * A package's namespace holds what its __init__ imports, not only what it + * defines: `models.CharField` exists because django/db/models/__init__.py + * star-imports its fields module. This turns a module's ordered namespace + * facts (cbm_extract_python_namespace_facts) into records next to its + * definitions: + * PyReexport a name the module's imports bind at the end of its top + * level: short_name = the bound local, receiver_type = the + * import's RAW module spec (one '.' per relative level, then + * the dotted module), method_names_str = the imported member + * PyStarImport `from X import *`: receiver_type = X's raw module spec + * PyAll a literal `__all__`: method_names_str = "a|b|c" + * Records are content-only -- built from the facts alone, with no import map + * and no graph -- so every driver, the incremental surface probe and the + * stored surface rows produce the same bytes for the same source, and the + * records of an unchanged file travel through its surface row. The Python + * cross registry resolves the specs to project modules and folds the records + * into its export table (py_build_module_namespaces). The replay is + * conservative: a later own definition, module import, shadow or delete drops + * a name; a whole-namespace UNKNOWN fact (an effect, a compound statement) + * drops everything recorded before it, and later exact facts record again; + * `__all__` is published only as a literal. Facts that are not COMPLETE + * publish nothing. */ +typedef struct { + bool star; + bool live; + const char *name; /* bound local (PyReexport) */ + const char *spec; /* raw module spec */ + const char *member; /* imported member (PyReexport) */ +} PxcPyNsOp; + +/* `level` dots followed by `module` ("..pkg.mod", ".", "app.db"). */ +static const char *pxc_py_module_spec(CBMArena *arena, uint32_t level, const char *module) { + const char *mod = module ? module : ""; + if (level == 0) + return mod[0] ? cbm_arena_strdup(arena, mod) : NULL; + if (level > 64) + return NULL; + char dots[65]; + memset(dots, '.', level); + dots[level] = '\0'; + return cbm_arena_sprintf(arena, "%s%s", dots, mod); +} + +/* Number of records pxc_build_py_namespace_records can write for `fr` (one + * per fact plus the __all__ record). */ +static int pxc_py_namespace_record_bound(const CBMFileResult *fr) { + if (!fr || fr->py_namespace.status != CBM_PY_NS_COMPLETE || + fr->py_namespace.language != CBM_LANG_PYTHON) + return 0; + return fr->py_namespace.count + 1; +} + +/* Replay fr's namespace facts and write the surviving records to out[] + * (capacity pxc_py_namespace_record_bound(fr)). Every string is copied into + * `arena`: fr may be a reloaded spill freed right after. Returns the count. */ +static int pxc_build_py_namespace_records(CBMArena *arena, const CBMFileResult *fr, + const char *module_qn, CBMLSPDef *out) { + int bound = pxc_py_namespace_record_bound(fr); + if (!arena || !module_qn || bound <= 1) + return 0; + const CBMPyNamespaceFacts *facts = &fr->py_namespace; + PxcPyNsOp *ops = (PxcPyNsOp *)cbm_arena_alloc(arena, (size_t)facts->count * sizeof(PxcPyNsOp)); + CBMHashTable *by_name = cbm_ht_create((uint32_t)facts->count); + const char *scope = cbm_arena_strndup(arena, module_qn, cbm_fqn_symbol_scope_len(module_qn)); + if (!ops || !by_name || !scope) { + cbm_ht_free(by_name); + return 0; + } + int n = 0; + enum { PXC_ALL_ABSENT, PXC_ALL_LITERAL, PXC_ALL_UNKNOWN } all_state = PXC_ALL_ABSENT; + const char **all_names = NULL; + int all_count = 0; + int all_cap = 0; + for (int i = 0; i < facts->count; i++) { + const CBMPyNamespaceFact *f = &facts->items[i]; + bool barrier = false; + const char *bind_name = NULL; /* a name this fact (re)binds */ + const char *bind_spec = NULL; /* set when the binding is a re-export */ + switch (f->kind) { + case CBM_PY_NS_IMPORT_NAME: + bind_name = f->local_name; + bind_spec = f->member_name + ? pxc_py_module_spec(arena, f->relative_level, f->module_name) + : NULL; + break; + case CBM_PY_NS_OWN_DEF: + case CBM_PY_NS_IMPORT_MODULE: + case CBM_PY_NS_SHADOW: + case CBM_PY_NS_DELETE: + bind_name = f->local_name; + break; + case CBM_PY_NS_IMPORT_STAR: { + const char *spec = pxc_py_module_spec(arena, f->relative_level, f->module_name); + if (!spec) { + barrier = true; + } else { + ops[n++] = (PxcPyNsOp){.star = true, .live = true, .spec = spec}; + } + break; + } + case CBM_PY_NS_ALL_SET: + case CBM_PY_NS_ALL_APPEND: + if (f->kind == CBM_PY_NS_ALL_SET) { + all_state = PXC_ALL_LITERAL; + all_count = 0; + } else if (all_state != PXC_ALL_LITERAL) { + all_state = PXC_ALL_UNKNOWN; + break; + } + for (int k = 0; k < f->name_count && all_state == PXC_ALL_LITERAL; k++) { + if (all_count == all_cap) { + int cap = all_cap ? all_cap * 2 : 16; + const char **grown = + (const char **)cbm_arena_alloc(arena, (size_t)cap * sizeof(char *)); + if (!grown) { + all_state = PXC_ALL_UNKNOWN; + break; + } + if (all_count) + memcpy(grown, all_names, (size_t)all_count * sizeof(char *)); + all_names = grown; + all_cap = cap; + } + all_names[all_count++] = f->names[k]; + } + break; + case CBM_PY_NS_ALL_UNKNOWN: + all_state = PXC_ALL_UNKNOWN; + break; + case CBM_PY_NS_ALL_DELETE: + all_state = PXC_ALL_ABSENT; + all_count = 0; + break; + case CBM_PY_NS_UNKNOWN: + default: + barrier = true; + break; + } + if (barrier) { + for (int k = 0; k < n; k++) + ops[k].live = false; + all_state = PXC_ALL_UNKNOWN; + continue; + } + if (!bind_name) + continue; + void *prev = cbm_ht_get(by_name, bind_name); + if (prev) + ops[(uintptr_t)prev - 1].live = false; + if (bind_spec) { + ops[n] = (PxcPyNsOp){ + .name = bind_name, .live = true, .spec = bind_spec, .member = f->member_name}; + cbm_ht_set(by_name, bind_name, (void *)(uintptr_t)(n + 1)); + n++; + } else if (prev) { + cbm_ht_delete(by_name, bind_name); + } + } + cbm_ht_free(by_name); + + int written = 0; + for (int k = 0; k < n; k++) { + if (!ops[k].live) + continue; + CBMLSPDef *d = &out[written]; + memset(d, 0, sizeof(*d)); + d->def_module_qn = module_qn; + d->receiver_type = ops[k].spec; + d->lang = CBM_LANG_PYTHON; + if (ops[k].star) { + d->label = "PyStarImport"; + d->qualified_name = scope; + d->short_name = "*"; + } else { + d->label = "PyReexport"; + d->short_name = cbm_arena_strdup(arena, ops[k].name); + d->method_names_str = cbm_arena_strdup(arena, ops[k].member); + d->qualified_name = cbm_arena_sprintf(arena, "%s.%s", scope, ops[k].name); + if (!d->short_name || !d->method_names_str) + continue; + } + if (d->qualified_name) + written++; + } + if (all_state == PXC_ALL_LITERAL) { + size_t len = 1; + for (int k = 0; k < all_count; k++) + len += strlen(all_names[k]) + 1; + char *names = (char *)cbm_arena_alloc(arena, len); + if (names) { + size_t pos = 0; + for (int k = 0; k < all_count; k++) { + if (pos > 0) + names[pos++] = '|'; + size_t l = strlen(all_names[k]); + memcpy(names + pos, all_names[k], l); + pos += l; + } + names[pos] = '\0'; + CBMLSPDef *d = &out[written++]; + memset(d, 0, sizeof(*d)); + d->qualified_name = scope; + d->short_name = "__all__"; + d->label = "PyAll"; + d->def_module_qn = module_qn; + d->method_names_str = names; + d->lang = CBM_LANG_PYTHON; + } + } + return written; +} + /* Collect a project-wide CBMLSPDef[] from all cached results. Returns a * malloc'd array (caller frees) of length *out_count. String fields are * borrowed from cache[i]->arena and from def_modules[i] (also borrowed). @@ -973,17 +1187,29 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are for (int i = 0; i < file_count; i++) { int defs = 0; int impls = 0; + int records = 0; /* a Python module's namespace records (re-exports) */ if (cache[i]) { owner_walk_required = owner_walk_required || cache[i]->has_test_definition_owners; defs = cache[i]->defs.count; impls = cache[i]->impl_traits.count; + records = pxc_py_namespace_record_bound(cache[i]); } else if (spill && cbm_result_spill_has(spill, i)) { - cbm_result_spill_peek_counts(spill, i, &defs, &impls); + /* The header carries the counts (and the fact stream's status and + * size), so the bound needs no full load. */ + CBMFileResult hdr; + if (cbm_result_spill_peek_header(spill, i, &hdr)) { + defs = hdr.defs.count; + impls = hdr.impl_traits.count; + records = pxc_py_namespace_record_bound(&hdr); + } } total += defs; if (files[i].language == CBM_LANG_RUST) { total += impls; } + if (files[i].language == CBM_LANG_PYTHON) { + total += records; + } } if (total == 0) { *out_count = 0; @@ -1000,6 +1226,7 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are } return NULL; } + int capacity = total; int idx = 0; for (int fi = 0; fi < file_count; fi++) { if (out_def_starts) { @@ -1066,6 +1293,13 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are } if (files[fi].language == CBM_LANG_PYTHON) { pxc_fold_py_field_types(arena, fr, defs, file_start, idx, &bs, def_modules[fi]); + /* Package namespace records (re-exports), reserved in the bound + * above; written only while they fit it (a result whose loaded + * stream outgrew its header would otherwise overflow). */ + int need = pxc_py_namespace_record_bound(fr); + if (need > 0 && idx + need <= capacity) { + idx += pxc_build_py_namespace_records(arena, fr, def_modules[fi], &defs[idx]); + } } cbm_pxc_free_import_map(bs.imp_keys, bs.imp_vals, bs.imp_count); /* NULL-safe */ if (files[fi].language == CBM_LANG_GO) { @@ -2740,6 +2974,123 @@ void cbm_pxc_free_module_def_index(CBMModuleDefIndex *idx) { free(idx); } +/* A Python namespace record that points at another module (a re-export or a + * star import; `__all__` points nowhere). */ +static bool pxc_is_py_reexport_record(const CBMLSPDef *d) { + return d->lang == CBM_LANG_PYTHON && d->label && d->receiver_type && + (strcmp(d->label, "PyReexport") == 0 || strcmp(d->label, "PyStarImport") == 0); +} + +/* The index entry of the Python module `qn` (a module file, or a package via + * its `__init__`), or NULL. */ +static const pxc_module_entry_t *pxc_py_module_entry(const CBMModuleDefIndex *idx, const char *qn, + size_t len) { + char key[CBM_SZ_1K]; + if (len + sizeof(".__init__") > sizeof(key)) + return NULL; + memcpy(key, qn, len); + key[len] = '\0'; + const pxc_module_entry_t *e = (const pxc_module_entry_t *)cbm_ht_get(idx->ht, key); + if (e) + return e; + memcpy(key + len, ".__init__", sizeof(".__init__")); + return (const pxc_module_entry_t *)cbm_ht_get(idx->ht, key); +} + +/* The entry of the module `spec` names from inside the module `module_qn`: + * a relative spec (leading dots) against that module's package, exactly; an + * absolute spec under the longest prefix of module_qn that has it (the + * project and any source root precede the import path). `member`, when set, + * also names a submodule (`from . import base`). NULL when nothing matches. */ +static const pxc_module_entry_t *pxc_py_spec_entry(const CBMModuleDefIndex *idx, + const char *module_qn, const char *spec, + const char *member) { + char qn[CBM_SZ_1K]; + size_t level = strspn(spec, "."); + const char *rest = spec + level; + if (level > 0) { + size_t base = strlen(module_qn); /* drop the module's own segment first */ + for (size_t up = 0; up < level; up++) { + while (base > 0 && module_qn[base - 1] != '.') + base--; + if (base == 0) + return NULL; + base--; + } + int n = rest[0] ? snprintf(qn, sizeof(qn), "%.*s.%s", (int)base, module_qn, rest) + : snprintf(qn, sizeof(qn), "%.*s", (int)base, module_qn); + if (n < 0 || (size_t)n >= sizeof(qn)) + return NULL; + const pxc_module_entry_t *e = pxc_py_module_entry(idx, qn, (size_t)n); + if (!e && member) { + int m = snprintf(qn + n, sizeof(qn) - (size_t)n, ".%s", member); + if (m > 0 && (size_t)(n + m) < sizeof(qn)) + e = pxc_py_module_entry(idx, qn, (size_t)(n + m)); + } + return e; + } + for (size_t cut = strlen(module_qn); cut > 0; cut--) { + if (module_qn[cut - 1] != '.') + continue; + int n = snprintf(qn, sizeof(qn), "%.*s%s", (int)cut, module_qn, rest); + if (n < 0 || (size_t)n >= sizeof(qn)) + continue; + const pxc_module_entry_t *e = pxc_py_module_entry(idx, qn, (size_t)n); + if (e) + return e; + } + return NULL; +} + +/* A per-file selection holds the defs of the caller's own and imported + * modules. A Python package that re-exports a name keeps only the record; + * the definition lives in the module the record points at, possibly through + * further re-exports. Select those modules' defs too, following every + * selected record once (a worklist over def indices: linear in the defs it + * reaches), so the per-file LSP can fold the chain as the shared registry + * does. */ +static void pxc_mark_reexport_targets(const CBMModuleDefIndex *idx, bool *selected, + const CBMLSPDef *all_defs, int *total) { + int cap = idx->def_count; + int *work = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)cap * sizeof(int)); + bool *queued = (bool *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)cap * sizeof(bool)); + if (!work || !queued) { + cbm_free(CBM_MEM_CLASS_EXTRACT, work); + cbm_free(CBM_MEM_CLASS_EXTRACT, queued); + return; + } + int n = 0; + for (int i = 0; i < cap; i++) { + if (selected[i] && pxc_is_py_reexport_record(&all_defs[i])) { + queued[i] = true; + work[n++] = i; + } + } + for (int w = 0; w < n; w++) { + const CBMLSPDef *r = &all_defs[work[w]]; + bool star = strcmp(r->label, "PyStarImport") == 0; + const pxc_module_entry_t *e = + r->def_module_qn ? pxc_py_spec_entry(idx, r->def_module_qn, r->receiver_type, + star ? NULL : r->method_names_str) + : NULL; + for (int j = 0; e && j < e->count; j++) { + int di = e->indices[j]; + if (di < 0 || di >= cap || all_defs[di].lang != CBM_LANG_PYTHON) + continue; + if (!selected[di]) { + selected[di] = true; + (*total)++; + } + if (!queued[di] && pxc_is_py_reexport_record(&all_defs[di])) { + queued[di] = true; + work[n++] = di; + } + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, work); + cbm_free(CBM_MEM_CLASS_EXTRACT, queued); +} + CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef *all_defs, CBMLanguage caller_lang, const char *caller_namespace, const char *own_module, const char *const *imp_qns, @@ -2772,6 +3123,9 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(idx->namespace_ht, namespace_key); total += pxc_mark_entry_defs(selected, e, all_defs, caller_lang); } + if (caller_lang == CBM_LANG_PYTHON) { + pxc_mark_reexport_targets(idx, selected, all_defs, &total); + } if (total == 0) { cbm_free(CBM_MEM_CLASS_EXTRACT, selected); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 3b5ad5785..3b82d05be 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -12283,6 +12283,41 @@ TEST(python_namespace_facts_preserve_order_and_import_provenance) { PASS(); } +/* The type checker view of `if TYPE_CHECKING:` publishes its consequence (a + * lazy-module package: the else branch only installs the module). A name an + * elif/else import binds again is withdrawn by a SHADOW: its runtime source is + * the alternative. Both streams stay COMPLETE and valid. */ +TEST(python_namespace_facts_type_checking_alternatives) { + CBMFileResult *lazy = extract("from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n from .lazy_mod import Lazy\n" + "else:\n import sys\n", + CBM_LANG_PYTHON, "t", "lz/__init__.py"); + CBMFileResult *compat = extract("from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n from .types_only import Shim\n" + "else:\n try:\n from .runtime_impl import Shim\n" + " except ImportError:\n Shim = None\n", + CBM_LANG_PYTHON, "t", "cmp/__init__.py"); + bool ok = lazy && compat && lazy->py_namespace.status == CBM_PY_NS_COMPLETE && + compat->py_namespace.status == CBM_PY_NS_COMPLETE && + cbm_py_namespace_facts_valid(&lazy->py_namespace) && + cbm_py_namespace_facts_valid(&compat->py_namespace) && + lazy->py_namespace.count == 3 && compat->py_namespace.count == 3; + if (ok) { + const CBMPyNamespaceFact *l = lazy->py_namespace.items; + const CBMPyNamespaceFact *c = compat->py_namespace.items; + ok = l[1].kind == CBM_PY_NS_IMPORT_NAME && strcmp(l[1].local_name, "Lazy") == 0 && + l[2].kind == CBM_PY_NS_SHADOW && strcmp(l[2].local_name, "sys") == 0 && + c[1].kind == CBM_PY_NS_IMPORT_NAME && strcmp(c[1].local_name, "Shim") == 0 && + c[2].kind == CBM_PY_NS_SHADOW && strcmp(c[2].local_name, "Shim") == 0; + } + if (lazy) + cbm_free_result(lazy); + if (compat) + cbm_free_result(compat); + ASSERT_TRUE(ok); + PASS(); +} + TEST(python_namespace_facts_distinguish_all_operations_and_unknowns) { static const char source[] = "__all__ = ['old']; __all__ = []\n" "__all__ = dynamic\n" @@ -12669,6 +12704,7 @@ SUITE(extraction) { RUN_TEST(python_namespace_facts_distinguish_all_operations_and_unknowns); RUN_TEST(python_namespace_facts_unknown_effects_and_body_stability); RUN_TEST(python_namespace_facts_parse_errors_are_complete_unknown); + RUN_TEST(python_namespace_facts_type_checking_alternatives); RUN_TEST(python_namespace_facts_capture_and_copy_fail_atomically); RUN_TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema); RUN_TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index cd5810138..2b80f51bd 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14575,6 +14575,259 @@ TEST(python_dotted_base_inheritance_parallel) { return check_python_dotted_base_inheritance(true); } +/* Properties of the `type` edge src_qn -> tgt_qn (strdup'd; caller frees), or NULL. */ +static char *py_reexp_edge_props(cbm_store_t *s, const char *project, const char *src_qn, + const char *tgt_qn, const char *type) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + char *props = NULL; + if (cbm_store_find_node_by_qn(s, project, src_qn, &src) == CBM_STORE_OK && + cbm_store_find_node_by_qn(s, project, tgt_qn, &tgt) == CBM_STORE_OK) { + cbm_edge_t *edges = NULL; + int ec = 0; + cbm_store_find_edges_by_source_type(s, src.id, type, &edges, &ec); + for (int i = 0; i < ec && !props; i++) { + if (edges[i].target_id == tgt.id) + props = strdup(edges[i].properties_json ? edges[i].properties_json : "{}"); + } + if (edges) + cbm_store_free_edges(edges, ec); + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + return props; +} + +/* Package re-exports, end to end: extraction records each module's ordered + * namespace facts, the collector turns them into re-export records, the + * shared Python registry folds them into its export table, and the LSP + * consults it. `models.CharField` with `from .fields import *` in + * models/__init__.py is the django shape. Each app function makes one call so + * every edge's strategy is attributable. */ +static const char *const py_reexport_calls[][2] = { + {"attr_explicit_cls", "pkg.explicit_mod.Explicit"}, + {"attr_explicit_fn", "pkg.explicit_mod.explicit_fn"}, + {"attr_star_cls", "pkg.star_mod.Star"}, + {"attr_star_fn", "pkg.star_mod.star_fn"}, + {"attr_all_listed", "pkg.all_mod.Listed"}, + {"attr_chain", "pkg.chain.inner.Deep"}, + {"attr_renamed", "pkg.alias_mod.Orig"}, + {"from_explicit", "pkg.explicit_mod.Explicit"}, + {"from_star_fn", "pkg.star_mod.star_fn"}, + {"from_all_listed", "pkg.all_mod.Listed"}, + {"from_chain", "pkg.chain.inner.Deep"}, + {"from_renamed", "pkg.alias_mod.Orig"}, + {"cycle_a", "cyc.a.a_fn"}, + {"cycle_b", "cyc.b.b_fn"}, + {"sub_models", "db.models.fields.CharField"}, + {"chain_var", "pkg.chain.inner.Deep.dive"}, + {"ctl_from_cls", "direct.impl.Direct"}, + {"ctl_from_fn", "direct.impl.direct_fn"}, + {"lazy_tc", "lz.lazy_mod.Lazy"}, +}; +enum { PY_REEXPORT_N_CALLS = sizeof(py_reexport_calls) / sizeof(py_reexport_calls[0]) }; + +/* Calls that must NOT reach their target by the LSP: an `__all__`-unlisted + * name, a private name under a star, a re-export the module shadows later, + * and one made before a whole-namespace barrier (a top-level `try:`). */ +static const char *const py_reexport_excluded[][2] = { + {"excl_unlisted", "pkg.all_mod.Unlisted"}, {"excl_private", "pkg.star_mod._private_fn"}, + {"excl_shadowed", "shad.impl.Gone"}, {"excl_barrier", "barr.impl.Before"}, + {"excl_compat", "cmp.types_only.Shim"}, +}; +enum { PY_REEXPORT_N_EXCLUDED = sizeof(py_reexport_excluded) / sizeof(py_reexport_excluded[0]) }; + +/* Index the fixture; lsp[c] = call c's edge has an lsp_* strategy; excluded[e] + * = excluded call e reached its target by the LSP anyway. */ +static int py_reexport_run(bool parallel, bool incremental, bool *lsp, bool *excluded) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_py_reexport_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + return -1; + } + write_temp_file(tmp, "app/__init__.py", ""); + write_temp_file(tmp, "app/pkg/__init__.py", + "from .explicit_mod import Explicit, explicit_fn\n" + "from .star_mod import *\n" + "from .all_mod import *\n" + "from .chain import *\n" + "from .alias_mod import Orig as Renamed\n"); + write_temp_file(tmp, "app/pkg/explicit_mod.py", + "class Explicit:\n pass\n\n\ndef explicit_fn():\n return 1\n"); + write_temp_file(tmp, "app/pkg/star_mod.py", + "class Star:\n pass\n\n\n" + "def star_fn():\n return 1\n\n\ndef _private_fn():\n return 0\n"); + write_temp_file(tmp, "app/pkg/all_mod.py", + "__all__ = [\"Listed\"]\n\n\nclass Listed:\n pass\n\n\n" + "class Unlisted:\n pass\n"); + write_temp_file(tmp, "app/pkg/chain/__init__.py", "from .inner import Deep\n"); + write_temp_file(tmp, "app/pkg/chain/inner.py", + "class Deep:\n def dive(self):\n return 1\n"); + write_temp_file(tmp, "app/pkg/alias_mod.py", "class Orig:\n pass\n"); + write_temp_file(tmp, "app/cyc/__init__.py", "from .a import *\nfrom .b import *\n"); + write_temp_file(tmp, "app/cyc/a.py", "from .b import *\n\n\ndef a_fn():\n return 1\n"); + write_temp_file(tmp, "app/cyc/b.py", "from .a import *\n\n\ndef b_fn():\n return 2\n"); + write_temp_file(tmp, "app/db/__init__.py", ""); + write_temp_file(tmp, "app/db/models/__init__.py", "from app.db.models.fields import *\n"); + write_temp_file(tmp, "app/db/models/fields/__init__.py", + "__all__ = [\"CharField\"]\n\n\nclass CharField:\n pass\n"); + write_temp_file(tmp, "app/direct/__init__.py", ""); + write_temp_file(tmp, "app/direct/impl.py", + "class Direct:\n pass\n\n\ndef direct_fn():\n return 1\n"); + write_temp_file(tmp, "app/shad/__init__.py", "from .impl import Gone\nGone = None\n"); + write_temp_file(tmp, "app/shad/impl.py", "class Gone:\n pass\n"); + write_temp_file( + tmp, "app/barr/__init__.py", + "from .impl import Before\n" + "try:\n import optional_dep\nexcept ImportError:\n optional_dep = None\n"); + write_temp_file(tmp, "app/barr/impl.py", "class Before:\n pass\n"); + write_temp_file(tmp, "app/lz/__init__.py", + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n from .lazy_mod import Lazy\n" + "else:\n import sys\n"); + write_temp_file(tmp, "app/lz/lazy_mod.py", "class Lazy:\n pass\n"); + write_temp_file(tmp, "app/cmp/__init__.py", + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n from .types_only import Shim\n" + "else:\n try:\n from .runtime_impl import Shim\n" + " except ImportError:\n Shim = None\n"); + write_temp_file(tmp, "app/cmp/types_only.py", "class Shim:\n pass\n"); + write_temp_file(tmp, "app/cmp/runtime_impl.py", "class Shim:\n pass\n"); + write_temp_file(tmp, "app/main.py", + "from app import pkg\n" + "from app import cyc\n" + "from app import shad\n" + "from app import barr\n" + "from app import lz\n" + "from app import cmp\n" + "from app.db import models\n" + "from app.pkg import Explicit, Listed, Deep, star_fn, Renamed\n" + "from app.direct.impl import Direct, direct_fn\n\n\n" + "def attr_explicit_cls():\n pkg.Explicit()\n\n\n" + "def attr_explicit_fn():\n pkg.explicit_fn()\n\n\n" + "def attr_star_cls():\n pkg.Star()\n\n\n" + "def attr_star_fn():\n pkg.star_fn()\n\n\n" + "def attr_all_listed():\n pkg.Listed()\n\n\n" + "def attr_chain():\n pkg.Deep()\n\n\n" + "def attr_renamed():\n pkg.Renamed()\n\n\n" + "def from_explicit():\n Explicit()\n\n\n" + "def from_star_fn():\n star_fn()\n\n\n" + "def from_all_listed():\n Listed()\n\n\n" + "def from_chain():\n Deep()\n\n\n" + "def from_renamed():\n Renamed()\n\n\n" + "def cycle_a():\n cyc.a_fn()\n\n\n" + "def cycle_b():\n cyc.b_fn()\n\n\n" + "def sub_models():\n models.CharField()\n\n\n" + "def chain_var():\n x = pkg.Deep()\n x.dive()\n\n\n" + "def excl_unlisted():\n pkg.Unlisted()\n\n\n" + "def excl_private():\n pkg._private_fn()\n\n\n" + "def excl_shadowed():\n shad.Gone()\n\n\n" + "def excl_barrier():\n barr.Before()\n\n\n" + "def excl_compat():\n cmp.Shim()\n\n\n" + "def lazy_tc():\n lz.Lazy()\n\n\n" + "def ctl_from_cls():\n Direct()\n\n\n" + "def ctl_from_fn():\n direct_fn()\n"); + for (int i = 0; parallel && i < 52; i++) { + char name[64]; + char pad[128]; + snprintf(name, sizeof(name), "pad/py_pad_%02d.py", i); + snprintf(pad, sizeof(pad), "def py_pad_%02d():\n return %d\n", i, i); + write_temp_file(tmp, name, pad); + } + + char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + cbm_setenv("CBM_WORKERS", parallel ? "4" : "1", 1); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/py_reexport.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + int run_rc = p ? cbm_pipeline_run(p) : -1; + if (incremental && run_rc == 0) { + /* Touch only the caller: re-indexed alone, it must still find the + * re-exports of the unchanged packages. */ + cbm_pipeline_free(p); + char main_path[512]; + snprintf(main_path, sizeof(main_path), "%s/app/main.py", tmp); + th_append_file(main_path, "\n# edited: the second run re-indexes the caller\n"); + p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + run_rc = p ? cbm_pipeline_run(p) : -1; + } + const char *project = p ? cbm_pipeline_project_name(p) : NULL; + cbm_store_t *s = project ? cbm_store_open_path(db_path) : NULL; + if (!s && run_rc == 0) { + run_rc = -1; + } + if (s) { + char src[512]; + char tgt[512]; + for (int c = 0; c < PY_REEXPORT_N_CALLS; c++) { + snprintf(src, sizeof(src), "%s.app.main.%s", project, py_reexport_calls[c][0]); + snprintf(tgt, sizeof(tgt), "%s.app.%s", project, py_reexport_calls[c][1]); + char *props = py_reexp_edge_props(s, project, src, tgt, "CALLS"); + lsp[c] = props && strstr(props, "\"strategy\":\"lsp_") != NULL; + if (!lsp[c]) { + printf(" [%s] %s -> %s: %s\n", parallel ? "parallel" : "sequential", src, tgt, + props ? props : "(no CALLS edge)"); + } + free(props); + } + for (int e = 0; e < PY_REEXPORT_N_EXCLUDED; e++) { + snprintf(src, sizeof(src), "%s.app.main.%s", project, py_reexport_excluded[e][0]); + snprintf(tgt, sizeof(tgt), "%s.app.%s", project, py_reexport_excluded[e][1]); + char *props = py_reexp_edge_props(s, project, src, tgt, "CALLS"); + excluded[e] = props && strstr(props, "\"strategy\":\"lsp_") != NULL; + if (excluded[e]) { + printf(" [%s] excluded %s -> %s: %s\n", parallel ? "parallel" : "sequential", src, + tgt, props); + } + free(props); + } + cbm_store_close(s); + } + if (p) { + cbm_pipeline_free(p); + } + if (saved_workers) { + cbm_setenv("CBM_WORKERS", saved_workers, 1); + free(saved_workers); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + th_rmtree(tmp); + return run_rc; +} + +/* Sequential driver (few files, one worker) and parallel driver (fillers + + * four workers), each as a full index and as an incremental re-index that + * touched only the caller. */ +TEST(pipeline_python_package_reexports_resolve_by_lsp) { + static const char *const mode_names[] = {"sequential", "parallel", "sequential incremental", + "parallel incremental"}; + for (int mode = 0; mode < 4; mode++) { + bool lsp[PY_REEXPORT_N_CALLS]; + bool excluded[PY_REEXPORT_N_EXCLUDED]; + memset(lsp, 0, sizeof(lsp)); + memset(excluded, 1, sizeof(excluded)); + ASSERT_EQ(py_reexport_run((mode & 1) != 0, mode >= 2, lsp, excluded), 0); + for (int c = 0; c < PY_REEXPORT_N_CALLS; c++) { + if (!lsp[c]) { + fprintf(stderr, " re-export call %s (%s) did not resolve by the LSP\n", + py_reexport_calls[c][0], mode_names[mode]); + } + ASSERT_TRUE(lsp[c]); /* RED for every re-export shape; controls stay green */ + } + for (int e = 0; e < PY_REEXPORT_N_EXCLUDED; e++) { + if (excluded[e]) { + fprintf(stderr, " excluded call %s (%s) resolved by the LSP anyway\n", + py_reexport_excluded[e][0], mode_names[mode]); + } + ASSERT_FALSE(excluded[e]); /* __all__, `_`, shadow, barrier */ + } + } + PASS(); +} + /* Bind the external-import veto specifically to #1277's typed-field fold. * No base classes or CALLS edges participate: a name-only call fallback * cannot turn this metadata assertion into a false positive. The unguarded @@ -25994,6 +26247,7 @@ SUITE(pipeline) { RUN_TEST(python_type_checking_typed_field_issue1277_parallel); RUN_TEST(python_dotted_base_inheritance); RUN_TEST(python_dotted_base_inheritance_parallel); + RUN_TEST(pipeline_python_package_reexports_resolve_by_lsp); RUN_TEST(python_typed_field_external_import_stays_unresolved); RUN_TEST(python_typed_field_project_class_resolves); RUN_TEST(usages_creates_edges);