diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index b11436091..c19debd80 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -484,12 +484,32 @@ static void parse_python_imports(CBMExtractCtx *ctx) { ts_tree_cursor_delete(&cursor); } +/* A proven same-module function (its OWN_DEF fact and definition), and + * whether a top-level call of it leaves the namespace intact (see + * py_ns_allowed_call). */ +typedef enum { + PY_NS_CALL_UNCHECKED = 0, + PY_NS_CALL_ALLOWED = 1, + PY_NS_CALL_EFFECT = 2, +} PyNsCallVerdict; + +typedef struct { + int fact; + TSNode node; + PyNsCallVerdict verdict; + const char **shadows; /* submodules the body imports into the package */ + int shadow_count; +} PyNsFunction; + /* Ordered Python namespace facts are a separate content-only pass. The older * import records keep their existing graph/heuristic meaning. */ typedef struct { CBMExtractCtx *ctx; CBMPyNamespaceFacts facts; CBMPyNamespaceFailure failure; + PyNsFunction *functions; /* in fact order; scratch, never part of the result */ + int function_count; + int function_cap; } PyNamespaceBuilder; #ifdef CBM_ENABLE_TEST_SEAMS @@ -516,6 +536,18 @@ static void *py_ns_alloc(PyNamespaceBuilder *b, size_t size) { return p; } +/* Builder bookkeeping nothing in the result points at: the extraction scratch + * when the context has one. */ +static void *py_ns_scratch_alloc(PyNamespaceBuilder *b, size_t size) { + if (b->failure) + return NULL; + CBMArena *arena = b->ctx->scratch ? b->ctx->scratch : b->ctx->arena; + void *p = cbm_arena_alloc(arena, size); + if (!p) + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return p; +} + static const char *py_ns_copy(PyNamespaceBuilder *b, const char *text, size_t len) { if (len == SIZE_MAX) { b->failure = CBM_PY_NS_FAILURE_LIMIT; @@ -1024,6 +1056,477 @@ static bool py_ns_future_import(PyNamespaceBuilder *b, TSNode node) { return ok && emitted > 0; } +/* ── A top-level call the namespace survives ────────────────────────────── + * Any top-level call may rebind any name of the module, so it withdraws the + * whole namespace (UNKNOWN, reason EFFECT). sqlalchemy's sql/__init__.py ends + * with `__go(locals())`, which withdrew all of its 108 re-exports. A call is + * allowed when its callee provably cannot rebind the caller's names: + * - the callee is the latest binding of its name, a proven (undecorated) + * same-module function, with no star import or unknown effect since; + * - every argument is a safe expression or a bare `locals()` / `globals()` + * the module does not rebind; + * - the body, nested scopes included, has no `global` / `nonlocal` and never + * names a parameter, a function or class this module defined earlier (it + * could rebind through `global`), the module's own name, or a name that + * reaches a namespace (py_ns_namespace_names). + * The body may import: each submodule of this package it imports becomes a + * package attribute, so in a package `__init__` those names are shadowed at + * the call. Every other call stays a whole-namespace effect. */ + +static const char *const py_ns_namespace_names[] = { + "globals", "vars", "locals", "exec", "eval", "setattr", "delattr", + "__import__", "sys", "importlib", "__name__", "__dict__", "__globals__", "__builtins__", + "builtins", "__self__", "inspect", "f_globals", "f_locals", "f_back", "gc", +}; + +typedef struct { + const char *text; + size_t len; +} PyNsSlice; + +typedef struct { + PyNamespaceBuilder *b; + PyNsSlice own; + bool package; + PyNsSlice *params; + int param_count; + const char **defs; /* earlier same-module functions and classes, sorted */ + int def_count; + const char **shadows; + int shadow_count; + int shadow_cap; +} PyNsCallCheck; + +static bool py_ns_text_is(const char *text, size_t len, const char *s) { + return strlen(s) == len && memcmp(text, s, len) == 0; +} + +/* strcmp of a NUL-terminated string against a slice. */ +static int py_ns_slice_cmp(const char *s, const char *text, size_t len) { + int r = strncmp(s, text, len); + if (r != 0) + return r; + return s[len] ? 1 : 0; +} + +static int py_ns_cmp_cstr(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +/* `pkg/sub/__init__.py` -> "sub", a package; `pkg/mod.py` -> "mod". */ +static bool py_ns_own_name(const char *rel_path, PyNsSlice *own, bool *package) { + if (!rel_path) + return false; + const char *slash = strrchr(rel_path, '/'); + const char *file = slash ? slash + 1 : rel_path; + const char *ext = strrchr(file, '.'); + if (!ext || ext == file) + return false; + size_t stem = (size_t)(ext - file); + *package = py_ns_text_is(file, stem, "__init__"); + if (!*package) { + *own = (PyNsSlice){file, stem}; + } else { + if (!slash) + return false; + const char *dir = slash; + while (dir > rel_path && dir[-1] != '/') + dir--; + *own = (PyNsSlice){dir, (size_t)(slash - dir)}; + } + return py_ns_identifier_bytes(own->text, own->len, false); +} + +static bool py_ns_reaches_namespace(const char *text, size_t len) { + for (size_t i = 0; i < sizeof(py_ns_namespace_names) / sizeof(py_ns_namespace_names[0]); i++) { + if (py_ns_text_is(text, len, py_ns_namespace_names[i])) + return true; + } + return false; +} + +/* A name of the body (not a module path) the call cannot allow. */ +static bool py_ns_forbidden_name(const PyNsCallCheck *c, const char *text, size_t len) { + if (py_ns_reaches_namespace(text, len) || + (len == c->own.len && memcmp(text, c->own.text, len) == 0)) + return true; + for (int i = 0; i < c->param_count; i++) { + if (len == c->params[i].len && memcmp(text, c->params[i].text, len) == 0) + return true; + } + int lo = 0, hi = c->def_count - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + int r = py_ns_slice_cmp(c->defs[mid], text, len); + if (r == 0) + return true; + if (r < 0) + lo = mid + 1; + else + hi = mid - 1; + } + return false; +} + +static bool py_ns_node_forbidden(const PyNsCallCheck *c, TSNode node) { + const char *text = NULL; + size_t len = 0; + return ts_node_is_null(node) || !py_ns_slice(c->b, node, &text, &len) || + py_ns_forbidden_name(c, text, len); +} + +static bool py_ns_add_shadow(PyNsCallCheck *c, const char *text, size_t len) { + if (!py_ns_identifier_bytes(text, len, false)) + return false; + if (c->shadow_count == c->shadow_cap) { + int cap = c->shadow_cap ? c->shadow_cap * 2 : 8; + const char **items = py_ns_scratch_alloc(c->b, (size_t)cap * sizeof(*items)); + if (!items) + return false; + if (c->shadow_count) + memcpy(items, c->shadows, (size_t)c->shadow_count * sizeof(*items)); + c->shadows = items; + c->shadow_cap = cap; + } + const char *name = py_ns_copy(c->b, text, len); + if (!name) + return false; + c->shadows[c->shadow_count++] = name; + return true; +} + +/* The segments of a dotted module path: none may reach a namespace. Sets + * *own_at to the index of the segment naming this module (-1: none) and + * *next to the slice after it (empty when it is the last segment). */ +static bool py_ns_check_path(const PyNsCallCheck *c, const char *text, size_t len, int *own_at, + PyNsSlice *next) { + *own_at = -1; + *next = (PyNsSlice){NULL, 0}; + int index = 0; + size_t start = 0; + while (start <= len && len > 0) { + size_t end = start; + while (end < len && text[end] != '.') + end++; + const char *seg = text + start; + size_t seg_len = end - start; + if (py_ns_reaches_namespace(seg, seg_len)) + return false; + if (*own_at >= 0 && *own_at == index - 1) + *next = (PyNsSlice){seg, seg_len}; + if (*own_at < 0 && seg_len == c->own.len && memcmp(seg, c->own.text, seg_len) == 0) + *own_at = index; + index++; + start = end + 1; + } + return true; +} + +/* The relative level of a `from` module path: its leading dots. */ +static size_t py_ns_leading_dots(const char *text, size_t len) { + const char *end = text + len; + const char *p = text; + while (p != end && *p == '.') + p++; + return (size_t)(p - text); +} + +/* An import in the callee's body. When its module is this package, the + * submodule it installs as a package attribute is shadowed: the first + * segment after the package, or every imported member when the import names + * the package itself. A relative import of a package `__init__` is relative + * to the package (level 1) or to its parent (level 2, whose first segment + * then names this package or a sibling). Deeper relative imports that name + * this module cannot be placed and are refused. The names an import binds are + * checked like every name of the body; this module's own name is allowed in a + * module path only. */ +static bool py_ns_check_import(PyNsCallCheck *c, TSNode node, bool from) { + TSNode module = from ? ts_node_child_by_field_name(node, TS_FIELD("module_name")) : (TSNode){0}; + bool into_package = false; /* the import's module is this package */ + PyNsSlice sub = {NULL, 0}; /* the submodule after the package, if any */ + if (from) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(c->b, module, &text, &len)) + return false; + size_t level = py_ns_leading_dots(text, len); + PyNsSlice rest = {text + level, len - level}; + if (rest.len && !py_ns_identifier_bytes(rest.text, rest.len, true)) + return false; + int own_at = -1; + PyNsSlice next; + if (!py_ns_check_path(c, rest.text, rest.len, &own_at, &next)) + return false; + if (c->package && level == 1) { + /* `from . import x` names the package; `from .x.y import z` installs x */ + const char *dot = rest.len ? memchr(rest.text, '.', rest.len) : NULL; + into_package = rest.len == 0; + sub = (PyNsSlice){rest.text, dot ? (size_t)(dot - rest.text) : rest.len}; + } else if (c->package && (level == 0 || (level == 2 && own_at == 0)) && own_at >= 0) { + /* level 2: the parent's child of this name is this package; a + * later segment of the same name is a package further down */ + if (next.len) + sub = next; + else + into_package = true; + } else if (c->package && level > 2 && own_at >= 0) { + return false; + } + if (sub.len && !py_ns_add_shadow(c, sub.text, sub.len)) + return false; + } + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || ts_node_eq(child, module) || + strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *kind = ts_node_type(child); + bool alias = strcmp(kind, "aliased_import") == 0; + if (!alias && strcmp(kind, "dotted_name") != 0) { + ok = false; /* a star import, or a form this check does not read */ + break; + } + TSNode name = alias ? ts_node_child_by_field_name(child, TS_FIELD("name")) : child; + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(c->b, name, &text, &len)) { + ok = false; + break; + } + if (alias && + py_ns_node_forbidden(c, ts_node_child_by_field_name(child, TS_FIELD("alias")))) { + ok = false; + break; + } + if (from) { + /* a member is a name the import binds (or a submodule) */ + if (py_ns_forbidden_name(c, text, len) || + (into_package && !py_ns_add_shadow(c, text, len))) { + ok = false; + break; + } + continue; + } + int own_at = -1; + PyNsSlice next; + if (!py_ns_identifier_bytes(text, len, true) || + !py_ns_check_path(c, text, len, &own_at, &next) || + (alias && own_at >= 0 && !next.len)) { + ok = false; /* `import pkg.own as alias` hands out this module */ + break; + } + if (c->package && own_at >= 0 && next.len && + !py_ns_add_shadow(c, next.text, next.len)) { + ok = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok; +} + +static bool py_ns_collect_params(PyNsCallCheck *c, TSNode def) { + TSNode params = ts_node_child_by_field_name(def, TS_FIELD("parameters")); + if (ts_node_is_null(params)) + return false; + uint32_t count = ts_node_named_child_count(params); + c->params = count ? py_ns_scratch_alloc(c->b, (size_t)count * sizeof(*c->params)) : NULL; + if (count && !c->params) + return false; + for (uint32_t i = 0; i < count; i++) { + TSNode p = ts_node_named_child(params, i); + const char *kind = ts_node_type(p); + if (strcmp(kind, "comment") == 0 || strcmp(kind, "keyword_separator") == 0 || + strcmp(kind, "positional_separator") == 0) + continue; + TSNode name = p; + if (strcmp(kind, "default_parameter") == 0 || + strcmp(kind, "typed_default_parameter") == 0) { + name = ts_node_child_by_field_name(p, TS_FIELD("name")); + } else if (strcmp(kind, "typed_parameter") == 0 || + strcmp(kind, "list_splat_pattern") == 0 || + strcmp(kind, "dictionary_splat_pattern") == 0) { + name = ts_node_named_child(p, 0); + const char *inner = ts_node_is_null(name) ? "" : ts_node_type(name); + if (strcmp(inner, "list_splat_pattern") == 0 || + strcmp(inner, "dictionary_splat_pattern") == 0) + name = ts_node_named_child(name, 0); + } else if (strcmp(kind, "identifier") != 0) { + return false; + } + PyNsSlice s; + if (ts_node_is_null(name) || strcmp(ts_node_type(name), "identifier") != 0 || + !py_ns_slice(c->b, name, &s.text, &s.len)) + return false; + c->params[c->param_count++] = s; + } + return true; +} + +static bool py_ns_collect_defs(PyNsCallCheck *c, int before) { + c->defs = before ? py_ns_scratch_alloc(c->b, (size_t)before * sizeof(*c->defs)) : NULL; + if (before && !c->defs) + return false; + for (int i = 0; i < before; i++) { + const CBMPyNamespaceFact *f = &c->b->facts.items[i]; + bool def = f->kind == CBM_PY_NS_OWN_DEF || + (f->kind == CBM_PY_NS_SHADOW && (f->reason == CBM_PY_NS_REASON_DECORATED || + f->reason == CBM_PY_NS_REASON_UNPROVEN_CLASS)); + if (def && f->local_name) + c->defs[c->def_count++] = f->local_name; + } + if (c->def_count > 1) + qsort(c->defs, (size_t)c->def_count, sizeof(*c->defs), py_ns_cmp_cstr); + return true; +} + +/* Walk the callee's body (iteratively: nesting must not exhaust the stack). */ +static bool py_ns_check_body(PyNsCallCheck *c, TSNode body) { + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool ok = true; + bool done = false; + while (ok && !done && !c->b->failure) { + TSNode node = ts_tree_cursor_current_node(&cursor); + const char *kind = ts_node_type(node); + bool from = strcmp(kind, "import_from_statement") == 0; + bool is_import = from || strcmp(kind, "import_statement") == 0; + if (strcmp(kind, "global_statement") == 0 || strcmp(kind, "nonlocal_statement") == 0 || + strcmp(kind, "future_import_statement") == 0) { + ok = false; + } else if (is_import) { + ok = py_ns_check_import(c, node, from); + } else if (strcmp(kind, "identifier") == 0) { + ok = !py_ns_node_forbidden(c, node); + } + if (!ok) + break; + if (!is_import && ts_tree_cursor_goto_first_child(&cursor)) + continue; + while (!ts_tree_cursor_goto_next_sibling(&cursor)) { + if (!ts_tree_cursor_goto_parent(&cursor)) { + done = true; + break; + } + } + } + ts_tree_cursor_delete(&cursor); + return ok && !c->b->failure; +} + +static PyNsCallVerdict py_ns_check_function(PyNamespaceBuilder *b, PyNsFunction *fn) { + PyNsCallCheck c = {.b = b}; + TSNode body = ts_node_child_by_field_name(fn->node, TS_FIELD("body")); + if (ts_node_is_null(body) || !py_ns_own_name(b->ctx->rel_path, &c.own, &c.package) || + !py_ns_collect_params(&c, fn->node) || !py_ns_collect_defs(&c, fn->fact) || + !py_ns_check_body(&c, body)) + return PY_NS_CALL_EFFECT; + if (!c.package) + c.shadow_count = 0; /* a plain module's imports install nothing in it */ + fn->shadows = c.shadows; + fn->shadow_count = c.shadow_count; + return PY_NS_CALL_ALLOWED; +} + +static void py_ns_note_function(PyNamespaceBuilder *b, int fact, TSNode node) { + if (b->function_count == b->function_cap) { + int cap = b->function_cap ? b->function_cap * 2 : 16; + PyNsFunction *items = py_ns_scratch_alloc(b, (size_t)cap * sizeof(*items)); + if (!items) + return; + if (b->function_count) + memcpy(items, b->functions, (size_t)b->function_count * sizeof(*items)); + b->functions = items; + b->function_cap = cap; + } + b->functions[b->function_count++] = (PyNsFunction){.fact = fact, .node = node}; +} + +enum { PY_NS_UNBOUND = -1, PY_NS_BARRIER = -2 }; + +/* The index of the latest fact binding `name`: PY_NS_UNBOUND when none did, + * PY_NS_BARRIER when a star import or an unknown effect may have since. */ +static int py_ns_latest_binding(const PyNamespaceBuilder *b, const char *name, size_t len) { + for (int i = b->facts.count - 1; i >= 0; i--) { + const CBMPyNamespaceFact *f = &b->facts.items[i]; + if (f->kind == CBM_PY_NS_UNKNOWN || f->kind == CBM_PY_NS_IMPORT_STAR) + return PY_NS_BARRIER; + if (f->local_name && py_ns_slice_cmp(f->local_name, name, len) == 0) + return i; + } + return PY_NS_UNBOUND; +} + +/* Safe expressions, or a bare `locals()` / `globals()` of the builtins (the + * body never names the parameter that receives it). */ +static bool py_ns_call_arguments_safe(PyNamespaceBuilder *b, TSNode args) { + if (ts_node_is_null(args) || strcmp(ts_node_type(args), "argument_list") != 0) + return false; + uint32_t count = ts_node_named_child_count(args); + for (uint32_t i = 0; i < count; i++) { + TSNode arg = ts_node_named_child(args, i); + if (py_ns_safe_expr(b, arg, 0)) + continue; + if (strcmp(ts_node_type(arg), "call") != 0) + return false; + TSNode fn = ts_node_child_by_field_name(arg, TS_FIELD("function")); + TSNode inner = ts_node_child_by_field_name(arg, TS_FIELD("arguments")); + const char *text = NULL; + size_t len = 0; + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "identifier") != 0 || + !py_ns_slice(b, fn, &text, &len) || + !(py_ns_text_is(text, len, "locals") || py_ns_text_is(text, len, "globals")) || + ts_node_is_null(inner) || ts_node_named_child_count(inner) != 0 || + py_ns_latest_binding(b, text, len) != PY_NS_UNBOUND) + return false; + } + return true; +} + +/* True when the top-level call `call` cannot rebind a name of this module's + * namespace; the submodules its callee imports into the package are then + * shadowed here. False when it stays a whole-namespace effect. */ +static bool py_ns_allowed_call(PyNamespaceBuilder *b, TSNode call) { + TSNode callee = ts_node_child_by_field_name(call, TS_FIELD("function")); + const char *name = NULL; + size_t len = 0; + if (ts_node_is_null(callee) || strcmp(ts_node_type(callee), "identifier") != 0 || + !py_ns_slice(b, callee, &name, &len) || + !py_ns_call_arguments_safe(b, ts_node_child_by_field_name(call, TS_FIELD("arguments")))) + return false; + int at = py_ns_latest_binding(b, name, len); + if (at < 0 || b->facts.items[at].kind != CBM_PY_NS_OWN_DEF || + b->facts.items[at].def_kind != CBM_PY_NS_DEF_FUNCTION) + return false; + int lo = 0, hi = b->function_count - 1; + PyNsFunction *fn = NULL; + while (lo <= hi && !fn) { + int mid = lo + (hi - lo) / 2; + if (b->functions[mid].fact == at) + fn = &b->functions[mid]; + else if (b->functions[mid].fact < at) + lo = mid + 1; + else + hi = mid - 1; + } + if (!fn) + return false; + if (fn->verdict == PY_NS_CALL_UNCHECKED) + fn->verdict = py_ns_check_function(b, fn); + if (fn->verdict != PY_NS_CALL_ALLOWED || b->failure) + return false; + for (int i = 0; i < fn->shadow_count; i++) { + if (!py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = fn->shadows[i], + .reason = CBM_PY_NS_REASON_VALUE})) + return false; + } + return true; +} + static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { const char *kind = ts_node_type(node); if (strcmp(kind, "comment") == 0 || strcmp(kind, "pass_statement") == 0) @@ -1050,16 +1553,20 @@ static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) return false; } - return py_ns_push( - b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, - .local_name = name, - .def_kind = function ? CBM_PY_NS_DEF_FUNCTION - : CBM_PY_NS_DEF_CLASS} - : (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, - .local_name = name, - .reason = decorated - ? CBM_PY_NS_REASON_DECORATED - : CBM_PY_NS_REASON_UNPROVEN_CLASS}); + if (!py_ns_push( + b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, + .local_name = name, + .def_kind = function ? CBM_PY_NS_DEF_FUNCTION + : CBM_PY_NS_DEF_CLASS} + : (CBMPyNamespaceFact){ + .kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = decorated ? CBM_PY_NS_REASON_DECORATED + : CBM_PY_NS_REASON_UNPROVEN_CLASS})) + return false; + if (proven && function) + py_ns_note_function(b, b->facts.count - 1, def); + return !b->failure; } } if (strcmp(kind, "delete_statement") == 0) @@ -1069,7 +1576,9 @@ static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { const char *ek = ts_node_type(expr); if (strcmp(ek, "assignment") == 0 || strcmp(ek, "augmented_assignment") == 0) return py_ns_assignment(b, expr); - return py_ns_safe_expr(b, expr, 0) || py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); + if (py_ns_safe_expr(b, expr, 0) || (strcmp(ek, "call") == 0 && py_ns_allowed_call(b, expr))) + return true; + return py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); } if (strcmp(kind, "if_statement") == 0 && py_ns_is_type_checking_if(b, node)) return py_ns_type_checking_block(b, node); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index c404c76ae..7bfda63fc 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -12525,6 +12525,116 @@ TEST(python_namespace_facts_string_statements_and_future_imports) { PASS(); } +/* A top-level call of a same-module function is a namespace effect, unless the + * callee provably cannot rebind the caller's names: sqlalchemy's + * `sql/__init__.py` ends with `__go(locals())`. Every submodule its body + * imports becomes a package attribute, so those names are shadowed at the + * call; everything else of the namespace survives. */ +static bool py_facts_has_effect(const CBMFileResult *r) { + for (int i = 0; r && i < r->py_namespace.count; i++) { + if (r->py_namespace.items[i].kind == CBM_PY_NS_UNKNOWN && + r->py_namespace.items[i].reason == CBM_PY_NS_REASON_EFFECT) + return true; + } + return false; +} + +static int py_facts_count_kind(const CBMFileResult *r, CBMPyNamespaceFactKind kind) { + int n = 0; + for (int i = 0; r && i < r->py_namespace.count; i++) + n += r->py_namespace.items[i].kind == kind; + return n; +} + +static bool py_facts_valid_complete(const CBMFileResult *r) { + return r && !r->has_error && r->py_namespace.status == CBM_PY_NS_COMPLETE && + cbm_py_namespace_facts_valid(&r->py_namespace); +} + +TEST(python_namespace_facts_init_call_allowlist) { + static const char go[] = "from typing import TYPE_CHECKING, Any\n" + "from .impl import base\n" + "from .elements import cast\n" + "def __go(lcls: Any) -> None:\n" + " from .. import util as _sa_util\n" + " from . import base\n" + " from . import coercions, elements as _el\n" + " if not TYPE_CHECKING:\n" + " base.coercions = _el.coercions = coercions\n" + " from .annotation import _prepare_annotations\n" + " _prepare_annotations(cast, cast)\n" + " _sa_util.preloaded.import_prefix(\"sqlalchemy.sql\")\n" + "\n" + "__go(locals())\n"; + CBMFileResult *r = extract(go, CBM_LANG_PYTHON, "t", "sql/__init__.py"); + bool ok = py_facts_valid_complete(r) && !py_facts_has_effect(r) && r->py_namespace.count == 9; + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + static const char *const shadowed[] = {"base", "coercions", "elements", "annotation"}; + ok = f[2].kind == CBM_PY_NS_IMPORT_NAME && strcmp(f[2].local_name, "base") == 0 && + f[4].kind == CBM_PY_NS_OWN_DEF && strcmp(f[4].local_name, "__go") == 0; + for (int i = 0; ok && i < 4; i++) { + ok = f[5 + i].kind == CBM_PY_NS_SHADOW && f[5 + i].local_name && + strcmp(f[5 + i].local_name, shadowed[i]) == 0; + } + } + if (!ok && r) { + for (int i = 0; i < r->py_namespace.count; i++) { + const CBMPyNamespaceFact *f = &r->py_namespace.items[i]; + printf(" __go fact %d: kind %d reason %d name %s\n", i, (int)f->kind, (int)f->reason, + f->local_name ? f->local_name : "-"); + } + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); /* RED while every top-level call withdraws the namespace */ + + /* A plain module is no package: the body's imports set attributes of its + * parent package, never of the module itself. */ + r = extract("def __go(lcls):\n from . import base\n\n__go(locals())\n", CBM_LANG_PYTHON, "t", + "sql/mod.py"); + ok = py_facts_valid_complete(r) && !py_facts_has_effect(r) && + py_facts_count_kind(r, CBM_PY_NS_SHADOW) == 0; + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + + /* Each of these can rebind a name of the caller's namespace (or is not + * provably a same-module function): the call stays a whole-namespace + * effect. */ + static const char *const effects[][2] = { + {"param", "def __go(lcls):\n lcls['base'] = 1\n\n__go(locals())\n"}, + {"global", "def __go(lcls):\n global base\n base = 1\n\n__go(locals())\n"}, + {"nested global", "def __go(lcls):\n def inner():\n global base\n" + " base = 1\n inner()\n\n__go(locals())\n"}, + {"globals()", "def __go(lcls):\n globals()['base'] = 1\n\n__go(locals())\n"}, + {"sys.modules", "def __go(lcls):\n import sys\n sys.modules[__name__].base = 1\n" + "\n__go(locals())\n"}, + {"imported callee", "from .setup import __go\n__go(locals())\n"}, + {"own package name", "def __go(lcls):\n from .. import sql\n sql.base = 1\n" + "\n__go(locals())\n"}, + {"same-module helper", "def helper():\n pass\n\ndef __go(lcls):\n helper()\n" + "\n__go(locals())\n"}, + {"rebound callee", "def __go(lcls):\n pass\n\n__go = other\n__go(locals())\n"}, + {"call before def", "__go(locals())\n\ndef __go(lcls):\n pass\n"}, + {"argument effect", "def __go(lcls):\n pass\n\n__go(make())\n"}, + {"rebound locals", "def locals():\n return {}\n\ndef __go(lcls):\n pass\n" + "\n__go(locals())\n"}, + }; + for (size_t i = 0; i < sizeof(effects) / sizeof(effects[0]); i++) { + char src[512]; + snprintf(src, sizeof(src), "from .impl import base\n%s", effects[i][1]); + r = extract(src, CBM_LANG_PYTHON, "t", "sql/__init__.py"); + ok = py_facts_valid_complete(r) && py_facts_has_effect(r); + if (!ok) + printf(" control '%s' is no namespace effect\n", effects[i][0]); + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + } + PASS(); +} + TEST(python_namespace_facts_distinguish_all_operations_and_unknowns) { static const char source[] = "__all__ = ['old']; __all__ = []\n" "__all__ = dynamic\n" @@ -12915,6 +13025,7 @@ SUITE(extraction) { RUN_TEST(python_namespace_facts_parse_errors_are_complete_unknown); RUN_TEST(python_namespace_facts_type_checking_alternatives); RUN_TEST(python_namespace_facts_string_statements_and_future_imports); + RUN_TEST(python_namespace_facts_init_call_allowlist); RUN_TEST(python_namespace_facts_capture_and_copy_fail_atomically); RUN_TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema); RUN_TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 5f7dabed9..d9b327226 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14902,16 +14902,20 @@ static const char *const py_reexport_calls[][2] = { {"ctl_from_fn", "direct.impl.direct_fn"}, {"lazy_tc", "lz.lazy_mod.Lazy"}, {"doc_reexport", "docp.impl.Doc"}, + {"go_reexport", "gop.impl.Thing"}, }; enum { PY_REEXPORT_N_CALLS = sizeof(py_reexport_calls) / sizeof(py_reexport_calls[0]) }; /* Calls that must NOT reach their target by the LSP: an `__all__`-unlisted * name, a private name under a star, a re-export the module shadows later, - * and one made before a whole-namespace barrier (a top-level `try:`). */ + * one made before a whole-namespace barrier (a top-level `try:`), a + * re-export a submodule import of `__go` replaces, and one behind a `__go` + * that writes its parameter. */ static const char *const py_reexport_excluded[][2] = { {"excl_unlisted", "pkg.all_mod.Unlisted"}, {"excl_private", "pkg.star_mod._private_fn"}, {"excl_shadowed", "shad.impl.Gone"}, {"excl_barrier", "barr.impl.Before"}, {"excl_compat", "cmp.types_only.Shim"}, {"excl_fstring", "fsp.impl.Kept"}, + {"excl_go_submodule", "gop.impl.base"}, {"excl_go_param", "gox.impl.Kept"}, }; enum { PY_REEXPORT_N_EXCLUDED = sizeof(py_reexport_excluded) / sizeof(py_reexport_excluded[0]) }; @@ -14984,6 +14988,25 @@ static int py_reexport_run(bool parallel, bool incremental, bool *lsp, bool *exc "from .impl import Kept\n" "f\"{Kept}\"\n"); write_temp_file(tmp, "app/fsp/impl.py", "class Kept:\n pass\n"); + /* sqlalchemy's sql/__init__.py shape: a top-level `__go(locals())`. */ + write_temp_file(tmp, "app/gop/__init__.py", + "from typing import Any\n" + "from .impl import Thing\n" + "from .impl import base\n\n\n" + "def __go(lcls: Any) -> None:\n" + " from . import base\n" + " from .annotation import prepare\n" + " prepare(Thing)\n\n\n" + "__go(locals())\n"); + write_temp_file(tmp, "app/gop/impl.py", + "class Thing:\n pass\n\n\ndef base():\n return 1\n"); + write_temp_file(tmp, "app/gop/base.py", "def helper():\n return 1\n"); + write_temp_file(tmp, "app/gop/annotation.py", "def prepare(cls):\n return cls\n"); + write_temp_file(tmp, "app/gox/__init__.py", + "from .impl import Kept\n\n\n" + "def __go(lcls):\n lcls[\"Kept\"] = None\n\n\n" + "__go(locals())\n"); + write_temp_file(tmp, "app/gox/impl.py", "class Kept:\n pass\n"); write_temp_file(tmp, "app/main.py", "from app import pkg\n" "from app import cyc\n" @@ -14993,6 +15016,8 @@ static int py_reexport_run(bool parallel, bool incremental, bool *lsp, bool *exc "from app import cmp\n" "from app import docp\n" "from app import fsp\n" + "from app import gop\n" + "from app import gox\n" "from app.db import models\n" "from app.pkg import Explicit, Listed, Deep, star_fn, Renamed\n" "from app.direct.impl import Direct, direct_fn\n\n\n" @@ -15020,6 +15045,9 @@ static int py_reexport_run(bool parallel, bool incremental, bool *lsp, bool *exc "def lazy_tc():\n lz.Lazy()\n\n\n" "def doc_reexport():\n docp.Doc()\n\n\n" "def excl_fstring():\n fsp.Kept()\n\n\n" + "def go_reexport():\n gop.Thing()\n\n\n" + "def excl_go_submodule():\n gop.base()\n\n\n" + "def excl_go_param():\n gox.Kept()\n\n\n" "def ctl_from_cls():\n Direct()\n\n\n" "def ctl_from_fn():\n direct_fn()\n"); for (int i = 0; parallel && i < 52; i++) {