From 6bdb971c9072da6cdebe125214f2eba3f99ce77c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 10:11:03 +0200 Subject: [PATCH 01/52] fix(k8s): read manifests and kustomizations behind a leading comment A comment between `---` and the mapping -- the `# yaml-language-server: $schema=...` line editors add -- is the YAML document's first named child. Both the manifest and the kustomization walker took that child as the content, found no mapping and skipped the document. The label scan for Service selectors also stopped at a leading `---`, treating the opening separator as the end of the first document. On onedr0p/home-ops (428 manifests open this way): Resource nodes 26 -> 318, kustomization entries resolved 1 -> 348 of 348. kubermatic 286 -> 300; flux2-kustomize-helm-example and microservices-demo unchanged. Each of the three new probes fails with its fix reverted. Signed-off-by: Martin Vogel --- internal/cbm/extract_k8s.c | 27 ++++++++----- src/pipeline/pass_k8s.c | 12 +++++- tests/test_grammar_probe_f.c | 76 ++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 10 deletions(-) diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 0396ee0e4..bad926e8e 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -102,15 +102,30 @@ static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, const c } } +// First named child that is not a comment. A comment between `---` and the +// mapping (the `# yaml-language-server: $schema=...` line editors add) is the +// document's first named child; the content follows it. +static TSNode first_content_child(TSNode node) { + uint32_t n = ts_node_named_child_count(node); + for (uint32_t i = 0; i < n; i++) { + TSNode child = ts_node_named_child(node, i); + if (strcmp(ts_node_type(child), "comment") != 0) { + return child; + } + } + TSNode null_node = {0}; + return null_node; +} + // Unwrap a YAML node through optional block_node wrapper to get block_mapping. // Returns null node if not a block_mapping. static TSNode unwrap_block_mapping(TSNode doc_child) { - TSNode mapping = ts_node_named_child(doc_child, 0); + TSNode mapping = first_content_child(doc_child); if (ts_node_is_null(mapping)) { return mapping; } if (strcmp(ts_node_type(mapping), "block_node") == 0) { - mapping = ts_node_named_child(mapping, 0); + mapping = first_content_child(mapping); } if (ts_node_is_null(mapping) || strcmp(ts_node_type(mapping), "block_mapping") != 0) { TSNode null_node = {0}; @@ -307,16 +322,10 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { continue; } - TSNode mapping = ts_node_named_child(stream_child, 0); + TSNode mapping = unwrap_block_mapping(stream_child); if (ts_node_is_null(mapping)) { continue; } - if (strcmp(ts_node_type(mapping), "block_node") == 0) { - mapping = ts_node_named_child(mapping, 0); - } - if (ts_node_is_null(mapping) || strcmp(ts_node_type(mapping), "block_mapping") != 0) { - continue; - } char kind_buf[K8S_BUF_SIZE] = {0}; char meta_name_buf[K8S_BUF_SIZE] = {0}; diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 3a6982ef6..4bc665309 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -239,6 +239,7 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec) { } stack[K8S_PATH_DEPTH]; int depth = 0; bool got_name = false; + bool seen_key = false; const char *p = source; while (p && *p) { @@ -254,11 +255,20 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec) { while (*trimmed == ' ') { trimmed++; } + /* A `---` before any key opens the first document; it does not end it. */ if (strncmp(trimmed, "---", 3) == 0 && line == trimmed) { - break; + if (seen_key) { + break; + } + if (!eol) { + break; + } + p = eol + 1; + continue; } if (trimmed[0] && trimmed[0] != '#') { + seen_key = true; int ind = k8s_indent(line); while (depth > 0 && stack[depth - 1].indent >= ind) { depth--; diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index ed734f052..a0673374f 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -470,6 +470,79 @@ TEST(probe_kustomize_with_patches) { PASS(); } +/* A comment between `---` and the mapping (the `# yaml-language-server: + * $schema=...` line editors add) is the document's first named child; the + * manifest behind it is still a manifest. onedr0p/home-ops opens 428 of its + * manifests this way and got a Resource for 26 of 390 objects. */ +TEST(probe_k8s_manifest_after_leading_comment) { + GpfMetrics m = gpf_metrics("helmrelease.yaml", + "---\n" + "# yaml-language-server: $schema=https://example.com/helmrelease.json\n" + "apiVersion: helm.toolkit.fluxcd.io/v2\n" + "kind: HelmRelease\n" + "metadata:\n" + " name: echo\n" + "spec:\n" + " interval: 1h\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 1); + PASS(); +} + +/* The same comment in front of a kustomization hid its resource list: + * home-ops resolved 1 of 348 entries. */ +TEST(probe_kustomize_imports_after_leading_comment) { + static const GpfFile files[] = { + {"kustomization.yaml", + "---\n" + "# yaml-language-server: $schema=https://json.schemastore.org/kustomization\n" + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - namespace.yaml\n"}, + {"namespace.yaml", + "apiVersion: v1\n" + "kind: Namespace\n" + "metadata:\n" + " name: network\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_TRUE(m.imports >= 1); + PASS(); +} + +/* The label scan stopped at a leading `---`, so a Service written that way + * had no selector and linked to nothing. The workload's name differs from the + * selector value, so only the pod-template label can match. */ +TEST(probe_k8s_selector_after_leading_separator) { + static const GpfFile files[] = { + {"service.yaml", + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " app: web-pods\n"}, + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web-deploy\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web-pods\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * GROUP 4 — Nix (.nix) * @@ -1567,6 +1640,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_kustomize_module_node); RUN_TEST(probe_kustomize_imports_edges); RUN_TEST(probe_kustomize_with_patches); + RUN_TEST(probe_k8s_manifest_after_leading_comment); + RUN_TEST(probe_kustomize_imports_after_leading_comment); + RUN_TEST(probe_k8s_selector_after_leading_separator); /* Nix */ RUN_TEST(probe_nix_attrset_node); From ac72d189fa0775238e301fcace17c4fe33c53024 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 6 Oct 2026 05:16:47 +0200 Subject: [PATCH 02/52] routes: HANDLES no longer depend on worker order or on substring paths The same binary indexed fastapi to 3,252, 3,950, 5,541 or 1,119,241 HANDLES edges from one run to the next; every other edge type was identical. - A prefix Route took the directory of whichever registrar's CALLS edge came first -- the workers' merge order -- and linked every route-decorated function under it. Every registrar now counts (edges dedup, so the union does not depend on order). - A root mount (`app.mount("/", ...)`) names no part of a handler's path, so it linked every decorated function in the directory (702 on fastapi). It bridges nothing now. - An infra URL took the handlers of a root route through `root_svc_match`, which any prefix or ANY route at `/` satisfied for any URL. A root route stands for a URL's service only when its file is that service's. - Both path tests were substrings: a prefix `/a` matched every file path holding an "a" and every URL path starting with `/a` (github.com/acidjunk), so with the root fixed the flood moved to the next substring match. A prefix now matches whole path segments of the handler's file, and a route path matches a URL path ending at a segment boundary; a bare-host URL matches only root routes of its service. On real input: fastapi 1,198 HANDLES in five of five runs (1,189 decorator edges, 9 prefix bridges, no infra copies); every other edge type unchanged. dapr 507 -> 367 HANDLES: the 140 removed are all infra copies, five URLs (an MQTT broker, an AMQP broker, an SVG image, two bare hosts) given the same 28 HTTP handlers each; decorator and bridge edges unchanged. The new pipeline test fails with each of the four rules reverted. Signed-off-by: Martin Vogel --- src/pipeline/pass_route_nodes.c | 95 +++++++++++++++++------ tests/test_pipeline.c | 132 ++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 23 deletions(-) diff --git a/src/pipeline/pass_route_nodes.c b/src/pipeline/pass_route_nodes.c index 878dd7f50..11fe8c7d7 100644 --- a/src/pipeline/pass_route_nodes.c +++ b/src/pipeline/pass_route_nodes.c @@ -236,6 +236,44 @@ static const char *url_path(const char *url) { return path ? path : "/"; } +/* Route path `needle` inside URL path `hay`, ending where a segment does: + * `/items` is in `/api/items` and `/items/42`; `/a` is not in `/acidjunk` + * (fastapi: a `/a` prefix route matched every github.com/a... URL). */ +static bool path_has_route(const char *hay, const char *needle) { + size_t n = strlen(needle); + if (n == 0) { + return false; + } + for (const char *p = strstr(hay, needle); p; p = strstr(p + SKIP_ONE, needle)) { + char c = p[n]; + bool start = needle[0] == '/' || p == hay || p[-SKIP_ONE] == '/'; + bool end = needle[n - SKIP_ONE] == '/' || c == '\0' || c == '/' || c == '?' || c == '#'; + if (start && end) { + return true; + } + } + return false; +} + +/* A route prefix's segments (`items`, `api/v1`) as whole segments of a file + * path: `app/routers/items.py`, `items/views.py`; `a` is not in + * `docs_src/app_testing/main.py`. */ +static bool file_has_segments(const char *path, const char *segs) { + size_t n = strlen(segs); + if (n == 0) { + return false; + } + for (const char *p = strstr(path, segs); p; p = strstr(p + SKIP_ONE, segs)) { + char c = p[n]; + bool start = p == path || p[-SKIP_ONE] == '/'; + bool end = segs[n - SKIP_ONE] == '/' || c == '\0' || c == '/' || c == '.'; + if (start && end) { + return true; + } + } + return false; +} + /* Check if a trailing dash-segment is a hash/project-number (strippable). * Returns true if the segment is short alphanumeric but not a meaningful word. */ static bool is_hash_segment(const char *seg, size_t slen) { @@ -373,9 +411,12 @@ static int match_one_infra_route(cbm_gbuf_t *gb, const cbm_gbuf_node_t *infra, } int path_match = - (strlen(handler_path) > SKIP_ONE && (strstr(infra_path, handler_path) != NULL || - strstr(handler_path, infra_path) != NULL)); - int root_svc_match = (strcmp(handler_path, "/") == 0); + (strlen(handler_path) > SKIP_ONE && path_has_route(infra_path, handler_path)) || + (strlen(infra_path) > SKIP_ONE && path_has_route(handler_path, infra_path)); + /* a root route stands for a URL's service only when its file is that + * service's; any prefix or ANY route at `/` matched every URL (a docs + * site's chat link took the root handlers) */ + int root_svc_match = file_matches && strcmp(handler_path, "/") == 0; if (path_match || root_svc_match) { if (infra_candidate_is_better(handler_route, best)) { best = handler_route; @@ -564,7 +605,7 @@ static int bridge_funcs_to_prefix(cbm_gbuf_t *gb, const cbm_gbuf_node_t *prefix_ if (!func->properties_json || !strstr(func->properties_json, "\"route_path\"")) { continue; } - if (prefix_segs && prefix_segs[0] && !strstr(func->file_path, prefix_segs)) { + if (prefix_segs && prefix_segs[0] && !file_has_segments(func->file_path, prefix_segs)) { continue; } cbm_gbuf_insert_edge(gb, func->id, prefix_route->id, "HANDLES", @@ -592,29 +633,37 @@ static void connect_prefix_to_decorators(cbm_gbuf_t *gb) { continue; } - const cbm_gbuf_edge_t **calls_in = NULL; - int calls_count = 0; - cbm_gbuf_find_edges_by_target_type(gb, prefix_route->id, "CALLS", &calls_in, &calls_count); - if (calls_count == 0) { - continue; - } - - const cbm_gbuf_node_t *registrar = cbm_gbuf_find_by_id(gb, calls_in[0]->source_id); - if (!registrar || !registrar->file_path) { - continue; - } - const char *last_slash = strrchr(registrar->file_path, '/'); - if (!last_slash) { - continue; - } - int dir_len = (int)(last_slash - registrar->file_path) + SKIP_ONE; - const char *prefix_path = prefix_route->name; const char *prefix_segs = (prefix_path && prefix_path[0] == '/') ? prefix_path + SKIP_ONE : prefix_path; + /* A root mount (`app.mount("/", ...)`) names no part of a handler's + * path, so nothing tells its handlers from the rest of the directory: + * bridging it linked every decorated function there (fastapi: 702 + * handlers to `/`, then to every infra URL through it). */ + if (!prefix_segs || !prefix_segs[0]) { + continue; + } - connected += - bridge_funcs_to_prefix(gb, prefix_route, registrar->file_path, dir_len, prefix_segs); + const cbm_gbuf_edge_t **calls_in = NULL; + int calls_count = 0; + cbm_gbuf_find_edges_by_target_type(gb, prefix_route->id, "CALLS", &calls_in, &calls_count); + + /* Every registrar's directory, not the first CALLS edge's: their + * order is the workers' merge order, so the first one moved between + * runs of the same corpus. Edges dedup, so the union is order-free. */ + for (int ci = 0; ci < calls_count; ci++) { + const cbm_gbuf_node_t *registrar = cbm_gbuf_find_by_id(gb, calls_in[ci]->source_id); + if (!registrar || !registrar->file_path) { + continue; + } + const char *last_slash = strrchr(registrar->file_path, '/'); + if (!last_slash) { + continue; + } + int dir_len = (int)(last_slash - registrar->file_path) + SKIP_ONE; + connected += bridge_funcs_to_prefix(gb, prefix_route, registrar->file_path, dir_len, + prefix_segs); + } } if (connected > 0) { diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a2861d054..77647762e 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -9073,6 +9073,137 @@ TEST(pipeline_swift_nested_url_makes_route_issue1892) { PASS(); } +/* HANDLES edges from the function named `source_name` to Routes whose + * qualified name equals `route_qn`. */ +static int handles_to_route(cbm_store_t *s, const char *project, const char *source_name, + const char *route_qn) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + if (cbm_store_find_edges_by_type(s, project, "HANDLES", &edges, &edge_count) != CBM_STORE_OK) { + return -1; + } + int matches = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t source = {0}; + cbm_node_t target = {0}; + int source_ok = cbm_store_find_node_by_id(s, edges[i].source_id, &source) == CBM_STORE_OK; + int target_ok = cbm_store_find_node_by_id(s, edges[i].target_id, &target) == CBM_STORE_OK; + if (source_ok && target_ok && source.name && target.qualified_name && + (!source_name || strcmp(source.name, source_name) == 0) && + strcmp(target.qualified_name, route_qn) == 0) { + matches++; + } + cbm_node_free_fields(&source); + cbm_node_free_fields(&target); + } + if (edges) { + cbm_store_free_edges(edges, edge_count); + } + return matches; +} + +/* fastapi (2026-10-05): one binary indexed the same tree to 3,253 or to + * 1,119,241 HANDLES edges. A prefix Route took the directory of whichever + * registrar's CALLS edge came first (worker order) and linked every decorated + * handler under it, with no filter for the root prefix `/`; every infra URL + * then copied the handlers of the root route. Now: every registrar counts, a + * root mount bridges nothing, and an infra URL matches a root route only of + * its own service. */ +TEST(pipeline_route_prefix_bridge_is_order_free_and_root_bridges_nothing) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_routebridge_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + /* two apps mounted at the root and under /api, from two directories */ + write_temp_file(tmp, "svc/a/main.py", + "from fastapi import FastAPI\n" + "app = FastAPI()\n" + "app.mount(\"/\", None)\n" + "app.mount(\"/api\", None)\n" + "\n" + "@app.get(\"/items\")\n" + "def read_items():\n" + " return []\n"); + write_temp_file(tmp, "svc/a/api/routes.py", + "from fastapi import APIRouter\n" + "router = APIRouter()\n" + "\n" + "@router.get(\"/ax\")\n" + "def api_a():\n" + " return 1\n"); + write_temp_file(tmp, "svc/b/main.py", + "from fastapi import FastAPI\n" + "app = FastAPI()\n" + "app.mount(\"/\", None)\n" + "app.mount(\"/api\", None)\n" + "app.mount(\"/c\", None)\n" + "\n" + "@app.get(\"/users\")\n" + "def read_users():\n" + " return []\n"); + write_temp_file(tmp, "svc/b/api/routes.py", + "from fastapi import APIRouter\n" + "router = APIRouter()\n" + "\n" + "@router.get(\"/by\")\n" + "def api_b():\n" + " return 2\n"); + /* a real root handler, and a documentation site's link to a chat server */ + write_temp_file(tmp, "web/site.py", + "from flask import Flask\n" + "app = Flask(__name__)\n" + "\n" + "@app.route(\"/\")\n" + "def index():\n" + " return \"ok\"\n" + "\n" + "@app.route(\"/c\")\n" + "def contact():\n" + " return \"c\"\n"); + write_temp_file(tmp, "mkdocs.yml", + "site_name: demo\n" + "extra:\n" + " social:\n" + " - icon: fontawesome/brands/discord\n" + " link: https://discord.com/invite/abc123\n" + " - icon: fontawesome/brands/github\n" + " link: https://github.com/carol\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/routebridge.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* decorator routes are untouched */ + ASSERT_EQ(handles_to_route(s, project, "read_items", "__route__GET__/items"), 1); + ASSERT_EQ(handles_to_route(s, project, "index", "__route__ANY__/"), 1); + /* a root mount names no part of a handler's path: nothing is bridged */ + ASSERT_EQ(handles_to_route(s, project, "read_items", "__route__ANY__/"), 0); + ASSERT_EQ(handles_to_route(s, project, "read_users", "__route__ANY__/"), 0); + /* /api is registered from both directories: both api handlers, whatever + * order the registrations' CALLS edges landed in */ + ASSERT_EQ(handles_to_route(s, project, "api_a", "__route__ANY__/api"), 1); + ASSERT_EQ(handles_to_route(s, project, "api_b", "__route__ANY__/api"), 1); + /* a prefix and a URL path match on whole segments only: `c` is no segment + * of svc/b/main.py, and `/c` is no route of github.com/carol */ + ASSERT_EQ(handles_to_route(s, project, "read_users", "__route__ANY__/c"), 0); + ASSERT_EQ(handles_to_route(s, project, "contact", "__route__ANY__/c"), 1); + ASSERT_EQ(handles_to_route(s, project, NULL, "__route__infra__https://github.com/carol"), 0); + /* the chat link is no route of this repository's root handler */ + ASSERT_EQ( + handles_to_route(s, project, NULL, "__route__infra__https://discord.com/invite/abc123"), 0); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + TEST(pipeline_swift_http_call_makes_route_issue1892) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_swifthttp_XXXXXX"); @@ -19431,6 +19562,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_native_fetch_classified_as_http_calls); RUN_TEST(pipeline_swift_nested_url_makes_route_issue1892); RUN_TEST(pipeline_swift_http_call_makes_route_issue1892); + RUN_TEST(pipeline_route_prefix_bridge_is_order_free_and_root_bridges_nothing); RUN_TEST(pipeline_native_fetch_parallel_classified_as_http_calls); RUN_TEST(pipeline_local_fetch_shadow_not_classified_as_http); RUN_TEST(pipeline_typed_http_call_has_no_arg_url_twin); From 5fe57ea321de3db1d637d0188cbd7d59372293b8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 12:05:27 +0200 Subject: [PATCH 03/52] fix(kustomize): discover `Kustomization` and resolve entries as paths only kustomize reads three file names; the extensionless `Kustomization` (kubermatic's CSI addons) was never discovered. A kustomization entry is a path relative to its own directory -- a manifest or a directory -- or a remote base. It went through the generic import resolver, whose name guesses bound entries to whatever shared a name: - a remote base `.../deploy/kubernetes/base` to the shell function `deploy` in hack/ci/deploy.sh (kubermatic), - component paths to Go structs and Makefile targets (gardener, cluster-api), - `../bases/md.yaml` to the overlay's own patch file `md.yaml` (cluster-api), through the basename-sibling candidate. Entries now resolve exactly: remote bases are external (no edge), local entries go to the named file's node or the named directory's folder, and nothing else. On real input, kustomization IMPORTS against main: home-ops 1 -> 348 (all 348 local entries), cluster-api 115 -> 159, gardener 157 -> 161, kubermatic 0 -> 3, microservices-demo 54 -> 55, k8s.io and flux2-kustomize-helm-example unchanged; every removed edge was one of the false targets above, and no edge targets a symbol any more. The remote-base probe fails with the resolver reverted, the extensionless probe with the file-name entry reverted; a nested-overlay probe guards the sibling resolution. Signed-off-by: Martin Vogel --- src/discover/language.c | 1 + src/pipeline/pass_pkgmap.c | 76 ++++++++++++++++++++++++++++++--- tests/test_grammar_probe_f.c | 82 ++++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 7 deletions(-) diff --git a/src/discover/language.c b/src/discover/language.c index b1cd7d938..f7fff9544 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -720,6 +720,7 @@ static const filename_entry_t FILENAME_TABLE[] = { {"meson_options.txt", CBM_LANG_MESON}, {"kustomization.yaml", CBM_LANG_KUSTOMIZE}, {"kustomization.yml", CBM_LANG_KUSTOMIZE}, + {"Kustomization", CBM_LANG_KUSTOMIZE}, /* kustomize's extensionless name */ /* Note: FILENAME_TABLE uses case-sensitive strcmp, so mixed-case variants * (e.g. "Kustomization.yaml") are not matched here. They fall through to * CBM_LANG_YAML and are re-classified by cbm_is_kustomize_file() in diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index f12dd8d4e..a2014c5d3 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -2213,6 +2213,63 @@ cbm_psr4_member_t cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char return *out_qn ? CBM_PSR4_MEMBER_RESOLVED : CBM_PSR4_MEMBER_UNRESOLVED; } +/* A kustomization entry naming a remote base -- a URL, a go-getter host path, + * an ssh remote or a `?ref=` pin -- lies outside the repository. */ +static bool kustomize_remote_entry(const char *entry) { + static const char *const hosts[] = {"github.com/", "gitlab.com/", "bitbucket.org/", "git@", + NULL}; + if (strstr(entry, "://") || strstr(entry, "?ref=") || strstr(entry, ".git/")) { + return true; + } + for (int i = 0; hosts[i]; i++) { + if (strncmp(entry, hosts[i], strlen(hosts[i])) == 0) { + return true; + } + } + return false; +} + +static bool kustomize_file_label(const char *label) { + return label && (strcmp(label, "Module") == 0 || strcmp(label, "File") == 0); +} + +/* A kustomization entry is a path relative to the kustomization's own + * directory -- a manifest file or a directory -- or a remote base; it never + * names a symbol. Resolve it exactly or not at all: the name-guess strategies + * bound a remote base's URL segment `deploy` to a shell function, component + * paths to Go structs, and an entry's basename to a same-named file beside + * the importer. */ +static const cbm_gbuf_node_t *resolve_kustomize_entry(const cbm_pipeline_ctx_t *ctx, + const char *source_rel, const char *entry) { + if (!entry[0] || kustomize_remote_entry(entry)) { + return NULL; + } + /* "x.yaml" and "./x.yaml" name the same file; make every entry relative. */ + bool dotted = entry[0] == '.' && (entry[1] == '/' || (entry[1] == '.' && entry[2] == '/')); + char rel[PKGMAP_PATH_BUF]; + snprintf(rel, sizeof(rel), "%s%s", dotted ? "" : "./", entry); + char *path = cbm_pipeline_resolve_relative_import(source_rel, rel); + if (!path) { + return NULL; + } + const cbm_gbuf_node_t *found = NULL; + char *qn = cbm_pipeline_fqn_module(ctx->project_name, path); + const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(ctx->gbuf, qn) : NULL; + free(qn); + if (n && kustomize_file_label(n->label)) { + found = n; + } else { + char *fqn = cbm_pipeline_fqn_folder(ctx->project_name, path); + n = fqn ? cbm_gbuf_find_by_qn(ctx->gbuf, fqn) : NULL; + free(fqn); + if (n && n->label && strcmp(n->label, "Folder") == 0) { + found = n; + } + } + free(path); + return found; +} + const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t *ctx, const char *source_rel, const char *source_file_qn, @@ -2222,6 +2279,18 @@ const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t return NULL; } + /* The importing file's language gates the strategies below. */ + const char *src_base = source_rel ? source_rel : ""; + for (const char *pb = src_base; *pb; pb++) { + if (*pb == '/' || *pb == '\\') { + src_base = pb + SKIP_ONE; + } + } + const CBMLanguage src_lang = cbm_language_for_filename(src_base); + if (src_lang == CBM_LANG_KUSTOMIZE) { + return resolve_kustomize_entry(ctx, source_rel, imp->module_path); + } + /* Prefer exact header-file nodes for C/C++ includes so same-stem source or * module nodes do not steal the edge target. */ const cbm_gbuf_node_t *header_target = @@ -2283,13 +2352,6 @@ const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t /* Name-guess fallbacks below (Strategy 1b sibling-file, Strategy 3 * symbol-name) are gated per importing-file language — see * cbm_import_symbol_fallback_allowed (#1934). */ - const char *src_base = source_rel ? source_rel : ""; - for (const char *pb = src_base; *pb; pb++) { - if (*pb == '/' || *pb == '\\') { - src_base = pb + SKIP_ONE; - } - } - const CBMLanguage src_lang = cbm_language_for_filename(src_base); const bool symbol_fallback_allowed = cbm_import_symbol_fallback_allowed(src_lang); /* Strategy 1b: sibling-file resolution for build/markup grammars whose diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index a0673374f..e5e5314c8 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -512,6 +512,85 @@ TEST(probe_kustomize_imports_after_leading_comment) { PASS(); } +/* kustomize also reads an extensionless file named `Kustomization` + * (kubermatic's addons use it); it was never discovered as a kustomization. */ +TEST(probe_kustomize_extensionless_file) { + static const GpfFile files[] = { + {"Kustomization", + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - namespace.yaml\n"}, + {"namespace.yaml", + "apiVersion: v1\n" + "kind: Namespace\n" + "metadata:\n" + " name: addons\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_TRUE(m.modules >= 1); + ASSERT_TRUE(m.imports >= 1); + PASS(); +} + +/* A remote base is outside the repository. The import resolver's symbol-name + * guess walked its URL segments and linked kubermatic's gcp addon to the + * `deploy` function of hack/ci/deploy.sh. A kustomize entry is a path, + * never a symbol. */ +TEST(probe_kustomize_remote_base_names_no_symbol) { + static const GpfFile files[] = { + {"kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - https://github.com/example/csi-driver.git/deploy/kubernetes/base\n"}, + {"hack/deploy.sh", + "#!/usr/bin/env bash\n" + "deploy() {\n" + " echo deploying\n" + "}\n"}, + /* Same name as the URL's last segment: the sibling lookup's basename + * candidate must not bind a remote base to it either. */ + {"base.yaml", + "apiVersion: v1\n" + "kind: ConfigMap\n" + "metadata:\n" + " name: local\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 3); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.imports, 0); + PASS(); +} + +/* Entries are relative to the kustomization's own directory; in a nested + * overlay they resolve through the sibling lookup, not the repo root. */ +TEST(probe_kustomize_nested_sibling_entries) { + static const GpfFile files[] = { + {"apps/base/podinfo/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - namespace.yaml\n" + " - release.yaml\n"}, + {"apps/base/podinfo/namespace.yaml", + "apiVersion: v1\n" + "kind: Namespace\n" + "metadata:\n" + " name: podinfo\n"}, + {"apps/base/podinfo/release.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\n" + "kind: HelmRelease\n" + "metadata:\n" + " name: podinfo\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 3); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.imports, 2); + PASS(); +} + /* The label scan stopped at a leading `---`, so a Service written that way * had no selector and linked to nothing. The workload's name differs from the * selector value, so only the pod-template label can match. */ @@ -1643,6 +1722,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_manifest_after_leading_comment); RUN_TEST(probe_kustomize_imports_after_leading_comment); RUN_TEST(probe_k8s_selector_after_leading_separator); + RUN_TEST(probe_kustomize_extensionless_file); + RUN_TEST(probe_kustomize_remote_base_names_no_symbol); + RUN_TEST(probe_kustomize_nested_sibling_entries); /* Nix */ RUN_TEST(probe_nix_attrset_node); From 2929afa60b1f655a315b0baeafcef8d328ad1b5e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 12:24:14 +0200 Subject: [PATCH 04/52] fix(k8s): stop minting an edgeless second Module per kustomization The k8s pass gave every kustomization a second Module (`source: kustomize`) next to the file's own Module and tried to link it to the entries by looking each one up as a repo-root path. Entries are relative to the kustomization, so the lookup resolved nothing: in 17 infra repositories the pass produced 0 IMPORTS edges, and the Module stayed an orphan. The entries are already IMPORTS of the kustomization file, resolved by the import resolver. The pass no longer handles kustomizations (no Module, no re-extraction of uncached kustomization files). On real input: gardener 110, onedr0p/home-ops 104 and flux2-kustomize-helm-example 7 orphan Modules gone; IMPORTS and every edge count unchanged. incremental_kustomize_entries_indexed (was ..._module_indexed) asserted the orphan as its sign that an added kustomization is indexed incrementally; it now asserts the kustomization's IMPORTS edge to the manifest it names. The new single-Module probe fails with the pass restored. Signed-off-by: Martin Vogel --- src/pipeline/pass_k8s.c | 94 ++++-------------------------------- tests/test_grammar_probe_f.c | 15 ++++++ tests/test_pipeline.c | 32 +++++++----- 3 files changed, 44 insertions(+), 97 deletions(-) diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 4bc665309..aa09748ee 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -1,14 +1,12 @@ /* - * pass_k8s.c — Pipeline pass for Kubernetes manifest and Kustomize overlay processing. + * pass_k8s.c — Pipeline pass for Kubernetes manifests, Helm charts and dependency manifests. * - * For each discovered YAML file: - * 1. Check if it is a kustomize overlay (kustomization.yaml / kustomization.yml) - * → emit a Module node and IMPORTS edges for each resources/bases/patches entry - * 2. Else if it is a generic k8s manifest (apiVersion: detected) - * → emit one Resource node per file (first document only — multi-document YAML is not yet - * supported) + * For each discovered YAML file that is a generic k8s manifest (apiVersion: detected) + * → emit one Resource node per file (first document only — multi-document YAML is not yet + * supported). Kustomizations are not handled here: their entries are IMPORTS of the + * file itself, resolved by the import resolver (resolve_kustomize_entry). * - * Depends on: pass_infrascan.c (cbm_is_kustomize_file, cbm_is_k8s_manifest, cbm_infra_qn), + * Depends on: pass_infrascan.c (cbm_is_k8s_manifest, cbm_infra_qn), * extraction layer (cbm.h), graph_buffer, pipeline internals. */ #include "foundation/constants.h" @@ -86,59 +84,6 @@ static const char *k8s_basename(const char *path) { return p ? p + SKIP_ONE : path; } -/* ── Kustomize handler ───────────────────────────────────────────── */ - -/* `result` is the cached parallel-pass result or, for an uncached file, the - * re-extraction the preparation step made; the caller owns it. */ -static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *rel_path, - const CBMFileResult *result) { - /* Emit Module node for this kustomize overlay file */ - char *mod_qn = cbm_infra_qn(ctx->project_name, rel_path, "kustomize", NULL); - if (!mod_qn) { - return; - } - - int64_t mod_id = cbm_gbuf_upsert_node(ctx->gbuf, "Module", k8s_basename(rel_path), mod_qn, - rel_path, SKIP_ONE, 0, "{\"source\":\"kustomize\"}"); - free(mod_qn); - - if (mod_id <= 0) { - return; - } - - /* With an extraction result, emit IMPORTS edges for - * resources/bases/patches/components entries */ - int import_count = 0; - const CBMFileResult *res = result; - - if (res) { - for (int j = 0; j < res->imports.count; j++) { - CBMImport *imp = &res->imports.items[j]; - if (!imp->module_path) { - continue; - } - - /* Compute target file QN */ - char *target_qn = - cbm_pipeline_fqn_compute(ctx->project_name, imp->module_path, "__file__"); - if (!target_qn) { - continue; - } - - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_qn(ctx->gbuf, target_qn); - free(target_qn); - - if (target) { - cbm_gbuf_insert_edge(ctx->gbuf, mod_id, target->id, "IMPORTS", - "{\"via\":\"kustomize\"}"); - import_count++; - } - } - } - - cbm_log_info("pass.k8s.kustomize", "file", rel_path, "imports", itoa_k8s(import_count)); -} - /* ── K8s cross-manifest label-selector matching ────────────────────── * A Service routes traffic to the workload Pods whose labels match the * Service's spec.selector. We record, per manifest, the Resource node id plus @@ -623,13 +568,13 @@ static void handle_dep_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c * the sequential loop did. Sequentially the pass re-read and re-parsed every * YAML manifest on one thread -- 1.1 s of the Go corpus (6,275 manifests; * profile + waste sanitizer, 2026-09-17). */ -typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_KUSTOMIZE, K8S_HELM, K8S_MANIFEST } k8s_kind_t; +typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_HELM, K8S_MANIFEST } k8s_kind_t; typedef struct { k8s_kind_t kind; char *source; /* DEP, HELM, MANIFEST */ int src_len; - CBMFileResult *res; /* MANIFEST, or KUSTOMIZE without a cached result */ + CBMFileResult *res; /* MANIFEST */ } k8s_prep_t; typedef struct { @@ -653,19 +598,6 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { if (is_gomod_file(base) || lang == CBM_LANG_GOMOD || is_requirements_file(base)) { out->source = k8s_read_file(path, &out->src_len); out->kind = out->source ? K8S_DEP : K8S_SKIP; - } else if (cbm_is_kustomize_file(base)) { - out->kind = K8S_KUSTOMIZE; - bool cached = ctx->result_cache && ctx->result_cache[i]; - if (!cached) { - /* Fall back to re-extraction */ - int src_len = 0; - char *source = k8s_read_file(path, &src_len); - if (source) { - out->res = cbm_extract_file(source, src_len, CBM_LANG_KUSTOMIZE, ctx->project_name, - rel, CBM_EXTRACT_BUDGET, NULL, NULL); - cbm_free(CBM_MEM_CLASS_OTHER, source); - } - } } else if (lang == CBM_LANG_YAML || lang == CBM_LANG_K8S) { /* Read source once to classify (and reuse for uncached extraction). */ out->source = k8s_read_file(path, &out->src_len); @@ -711,7 +643,6 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_init(); - int kustomize_count = 0; int manifest_count = 0; int helm_count = 0; @@ -750,17 +681,11 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, if (!cancelled) { const char *rel = files[i].rel_path; const char *base = k8s_basename(rel); - CBMFileResult *cached = - (ctx->result_cache && ctx->result_cache[i]) ? ctx->result_cache[i] : NULL; switch (p->kind) { case K8S_DEP: handle_dep_manifest(ctx, rel, p->source, is_requirements_file(base) ? "pypi" : "gomod"); break; - case K8S_KUSTOMIZE: - handle_kustomize(ctx, rel, cached ? cached : p->res); - kustomize_count++; - break; case K8S_HELM: handle_helm_chart(ctx, rel, p->source); helm_count++; @@ -796,8 +721,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_selectors(ctx, &recs); cbm_free(CBM_MEM_CLASS_OTHER, recs.items); - cbm_log_info("pass.done", "pass", "k8s", "kustomize", itoa_k8s(kustomize_count), "manifests", - itoa_k8s(manifest_count)); + cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count)); (void)helm_count; return 0; } diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index e5e5314c8..c619e68d7 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -564,6 +564,20 @@ TEST(probe_kustomize_remote_base_names_no_symbol) { PASS(); } +/* The k8s pass minted a second, edgeless Module per kustomization next to + * the file's own Module: its import lookup took entries as repo-root paths + * and resolved none of them in any of 17 infra repositories. */ +TEST(probe_kustomize_single_module) { + GpfMetrics m = gpf_metrics("kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - deploy.yaml\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.modules, 1); + PASS(); +} + /* Entries are relative to the kustomization's own directory; in a nested * overlay they resolve through the sibling lookup, not the repo root. */ TEST(probe_kustomize_nested_sibling_entries) { @@ -1725,6 +1739,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_kustomize_extensionless_file); RUN_TEST(probe_kustomize_remote_base_names_no_symbol); RUN_TEST(probe_kustomize_nested_sibling_entries); + RUN_TEST(probe_kustomize_single_module); /* Nix */ RUN_TEST(probe_nix_attrset_node); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 77647762e..68166f0c4 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -16992,9 +16992,10 @@ TEST(incremental_k8s_manifest_indexed) { PASS(); } -TEST(incremental_kustomize_module_indexed) { +TEST(incremental_kustomize_entries_indexed) { /* Verifies that a kustomization.yaml added after the initial full index - * gets a Module node via the incremental k8s pass. */ + * is indexed incrementally: its entry becomes an IMPORTS edge to the + * manifest it names. */ char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_kust_incr_XXXXXX"); if (!cbm_mkdtemp(tmpdir)) { @@ -17035,20 +17036,27 @@ TEST(incremental_kustomize_module_indexed) { ASSERT_EQ(cbm_pipeline_run(p), 0); cbm_pipeline_free(p); - /* Verify Module node created for the kustomization overlay */ + /* Verify the overlay's entry was resolved: kustomization.yaml IMPORTS deploy.yaml */ cbm_store_t *s = cbm_store_open_path(dbpath); ASSERT_NOT_NULL(s); - cbm_node_t *nodes = NULL; - int count = 0; - cbm_store_find_nodes_by_label(s, project, "Module", &nodes, &count); + cbm_edge_t *edges = NULL; + int edge_count = 0; + cbm_store_find_edges_by_type(s, project, "IMPORTS", &edges, &edge_count); bool found_kust = false; - for (int i = 0; i < count; i++) { - if (nodes[i].properties_json && strstr(nodes[i].properties_json, "kustomize")) { - found_kust = true; - break; + for (int i = 0; i < edge_count && !found_kust; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK) { + found_kust = src.file_path && strcmp(src.file_path, "kustomization.yaml") == 0 && + tgt.file_path && strcmp(tgt.file_path, "deploy.yaml") == 0; } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) { + cbm_store_free_edges(edges, edge_count); } - cbm_store_free_nodes(nodes, count); cbm_store_close(s); ASSERT_TRUE(found_kust); @@ -19783,7 +19791,7 @@ SUITE(pipeline) { #endif RUN_TEST(incremental_fast_preserves_mode_skipped_tools_dir); RUN_TEST(incremental_k8s_manifest_indexed); - RUN_TEST(incremental_kustomize_module_indexed); + RUN_TEST(incremental_kustomize_entries_indexed); /* Resource management & internal helper tests */ RUN_TEST(pipeline_empty_path); RUN_TEST(pipeline_project_name_content); From 52ca6b7e1960919d759284580f53aaa131514ab3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 12:45:52 +0200 Subject: [PATCH 05/52] fix(k8s): link a Service to the pods its whole selector selects Service -> workload INFRA_MAPS edges came from a value-only guess: - any one selector VALUE matched, ignoring keys and the other pairs (`{app: web, tier: front}` selected `tier: back` pods; `component: web` selected `app: web`), - any `labels:` block counted, including the workload's own metadata.labels rather than its pod template's, - a workload whose NAME equalled a selector value was selected, - namespaces were ignored, - only the first 512 manifests took part (deckhouse has 2,003), and the match was Services x workloads. Now a Service selects a workload when every `key=value` of its spec.selector is a pod-template label (a bare Pod's metadata.labels) and their namespaces do not conflict -- how Kubernetes resolves endpoints. Records grow without a cap; an inverted index from "key=value" to workloads makes each Service scan only the workloads carrying its rarest pair. Records stay in file order, so the edge set is deterministic. On real input against main: gardener 12 -> 11 (the removed edge joined a Service to a Helm template only through the Deployment's name; its pod labels come from `.Values.labels`), k8s.io 11 -> 12, cluster-api 0 -> 1, eight other corpora unchanged. Most Services still sit in the second document of their file, which only the first document is read from. probe_k8s_infra_maps_edge passed only through the name guess (its Deployment had no pod labels); the fixture now carries the label. Six new probes, one per defect above, fail on the old matcher. Signed-off-by: Martin Vogel --- src/pipeline/pass_k8s.c | 410 ++++++++++++++++++++++++++--------- tests/test_grammar_probe_f.c | 223 ++++++++++++++++++- 2 files changed, 531 insertions(+), 102 deletions(-) diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index aa09748ee..047df4355 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -85,34 +85,40 @@ static const char *k8s_basename(const char *path) { } /* ── K8s cross-manifest label-selector matching ────────────────────── - * A Service routes traffic to the workload Pods whose labels match the - * Service's spec.selector. We record, per manifest, the Resource node id plus - * its selector label-values (Services) and pod label-values (workloads), then - * after all manifests are processed connect each Service to the workload(s) it - * targets via an INFRA_MAPS edge. This models the runtime traffic path the same - * way k8s itself resolves Service → Endpoints by label. - * - * Matching is by label *value*: a Service selector `app: frontend` matches a - * workload that either carries the pod label value "frontend" OR is named - * "frontend" (the ubiquitous app==name convention, used when the pod template - * omits explicit labels). */ -enum { K8S_MAX_LABELS = 16, K8S_MAX_RECORDS = 512, K8S_LABEL_LEN = 128 }; + * A Service routes traffic to the Pods whose labels carry EVERY pair of the + * Service's spec.selector, in the Service's own namespace -- the way k8s + * resolves Service → Endpoints. Per manifest we record the Resource node, its + * namespace and the label pairs that take part: a Service's spec.selector, a + * workload's pod-template labels, a bare Pod's metadata.labels. After all + * manifests are read, an inverted index from "key=value" to workloads turns + * each Service into a scan of the workloads carrying its rarest pair rather + * than of every workload; each match becomes an INFRA_MAPS edge + * (Service → workload). */ +enum { K8S_LABEL_LEN = 128, K8S_GROW_MIN = 64 }; + +typedef enum { + K8S_REC_OTHER = 0, + K8S_REC_SERVICE, /* pairs = spec.selector */ + K8S_REC_WORKLOAD, /* pairs = pod-template labels */ + K8S_REC_POD, /* pairs = metadata.labels */ +} k8s_rec_kind_t; typedef struct { int64_t node_id; char name[K8S_LABEL_LEN]; - bool is_service; - bool is_workload; - char selector_vals[K8S_MAX_LABELS][K8S_LABEL_LEN]; /* Service spec.selector values */ - int n_selector; - char label_vals[K8S_MAX_LABELS][K8S_LABEL_LEN]; /* workload pod-template label values */ - int n_label; + char ns[K8S_LABEL_LEN]; /* metadata.namespace; "" when unset */ + k8s_rec_kind_t kind; + int pair_off; /* this record's pairs: pairs[pair_off .. pair_off + pair_n) */ + int pair_n; } k8s_record_t; typedef struct { k8s_record_t *items; int count; int cap; + char **pairs; /* "key=value", heap-owned */ + int pair_count; + int pair_cap; } k8s_record_array_t; /* Count leading-space indentation of a line (tabs are invalid YAML indent). */ @@ -164,24 +170,125 @@ static int k8s_split_kv(const char *t, char *key, size_t key_sz, char *val, size return 1; } -static void k8s_add_val(char dst[][K8S_LABEL_LEN], int *n, const char *v) { - if (*n >= K8S_MAX_LABELS || !v || !v[0]) { - return; +/* Which label block a leaf `key: value` sits in, by its parent path. */ +typedef enum { + K8S_PAIR_NONE = 0, + K8S_PAIR_SELECTOR, /* spec.selector (a Service's selector is a flat map) */ + K8S_PAIR_TEMPLATE_LABEL, /* spec.template.metadata.labels, also under a CronJob's jobTemplate */ + K8S_PAIR_META_LABEL, /* metadata.labels */ +} k8s_pair_block_t; + +static k8s_pair_block_t k8s_pair_block(const char *parent_path) { + if (strcmp(parent_path, "spec.selector") == 0) { + return K8S_PAIR_SELECTOR; } - snprintf(dst[*n], K8S_LABEL_LEN, "%s", v); - (*n)++; + if (strcmp(parent_path, "spec.template.metadata.labels") == 0 || + strcmp(parent_path, "spec.jobTemplate.spec.template.metadata.labels") == 0) { + return K8S_PAIR_TEMPLATE_LABEL; + } + if (strcmp(parent_path, "metadata.labels") == 0) { + return K8S_PAIR_META_LABEL; + } + return K8S_PAIR_NONE; } -/* Scan a single-document k8s manifest's text for the resource name, selector - * label-values (Service) and pod-template label-values (workload). A small - * indentation path-stack distinguishes spec.selector from - * spec.template.metadata.labels and from top-level metadata.name. */ -static void k8s_scan_labels(const char *source, k8s_record_t *rec) { +static k8s_rec_kind_t k8s_rec_kind(const char *kind) { + static const char *const workloads[] = { + "Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "ReplicationController", + "Job", "CronJob", NULL}; + if (strcmp(kind, "Service") == 0) { + return K8S_REC_SERVICE; + } + if (strcmp(kind, "Pod") == 0) { + return K8S_REC_POD; + } + for (int i = 0; workloads[i]; i++) { + if (strcmp(kind, workloads[i]) == 0) { + return K8S_REC_WORKLOAD; + } + } + return K8S_REC_OTHER; +} + +/* The block whose pairs a record of this kind contributes. */ +static k8s_pair_block_t k8s_kind_block(k8s_rec_kind_t kind) { + switch (kind) { + case K8S_REC_SERVICE: + return K8S_PAIR_SELECTOR; + case K8S_REC_WORKLOAD: + return K8S_PAIR_TEMPLATE_LABEL; + case K8S_REC_POD: + return K8S_PAIR_META_LABEL; + case K8S_REC_OTHER: + default: + return K8S_PAIR_NONE; + } +} + +/* One scanned `key: value` with the block it sits in. */ +typedef struct { + k8s_pair_block_t block; + char *kv; /* "key=value", heap-owned until moved into the record array */ +} k8s_scanned_pair_t; + +typedef struct { + k8s_scanned_pair_t *items; + int count; + int cap; +} k8s_scanned_t; + +static char *k8s_join_kv(const char *key, const char *val) { + size_t kl = strlen(key); + size_t vl = strlen(val); + char *kv = cbm_alloc(CBM_MEM_CLASS_OTHER, kl + vl + 2); + if (!kv) { + return NULL; + } + memcpy(kv, key, kl); + kv[kl] = '='; + memcpy(kv + kl + 1, val, vl + 1); + return kv; +} + +static void k8s_scanned_push(k8s_scanned_t *s, k8s_pair_block_t block, const char *key, + const char *val) { + if (s->count == s->cap) { + int cap = s->cap ? s->cap * 2 : K8S_GROW_MIN; + k8s_scanned_pair_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, s->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + s->items = grown; + s->cap = cap; + } + char *kv = k8s_join_kv(key, val); + if (kv) { + s->items[s->count++] = (k8s_scanned_pair_t){.block = block, .kv = kv}; + } +} + +/* Dotted path of the open block keys, e.g. "spec.template.metadata.labels". */ +static void k8s_parent_path(char *out, size_t out_sz, const char (*keys)[64], int depth) { + out[0] = '\0'; + size_t w = 0; + for (int i = 0; i < depth && w + 1 < out_sz; i++) { + int n = snprintf(out + w, out_sz - w, "%s%s", i ? "." : "", keys[i]); + if (n < 0) { + break; + } + w += (size_t)n < out_sz - w ? (size_t)n : out_sz - w - 1; + } +} + +/* Scan a single-document k8s manifest's text for its name, namespace, kind and + * label pairs into `rec` / `out`. A small indentation path-stack gives every + * leaf its parent path, which tells spec.selector from the pod template's + * labels and from the object's own metadata.labels. */ +static void k8s_scan_labels(const char *source, k8s_record_t *rec, k8s_scanned_t *out) { enum { K8S_PATH_DEPTH = 12 }; - struct { - int indent; - char key[64]; - } stack[K8S_PATH_DEPTH]; + int indents[K8S_PATH_DEPTH]; + char keys[K8S_PATH_DEPTH][64]; int depth = 0; bool got_name = false; bool seen_key = false; @@ -215,42 +322,34 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec) { if (trimmed[0] && trimmed[0] != '#') { seen_key = true; int ind = k8s_indent(line); - while (depth > 0 && stack[depth - 1].indent >= ind) { + while (depth > 0 && indents[depth - 1] >= ind) { depth--; } char key[64]; char val[K8S_LABEL_LEN]; if (k8s_split_kv(trimmed, key, sizeof(key), val, sizeof(val))) { - /* Build the current dotted path for context decisions. */ - bool under_selector = (depth >= 1 && strcmp(stack[depth - 1].key, "selector") == 0); - bool under_labels = (depth >= 1 && strcmp(stack[depth - 1].key, "labels") == 0); - bool under_metadata = (depth >= 1 && strcmp(stack[depth - 1].key, "metadata") == 0); - if (val[0] == '\0') { /* Block-opening key: push onto the path stack. */ if (depth < K8S_PATH_DEPTH) { - stack[depth].indent = ind; - snprintf(stack[depth].key, sizeof(stack[depth].key), "%s", key); + indents[depth] = ind; + snprintf(keys[depth], sizeof(keys[depth]), "%s", key); depth++; } - } else { - /* Leaf key: value. */ - if (ind == 0 && strcmp(key, "kind") == 0) { - rec->is_service = (strcmp(val, "Service") == 0); - rec->is_workload = - (strcmp(val, "Deployment") == 0 || strcmp(val, "StatefulSet") == 0 || - strcmp(val, "DaemonSet") == 0 || strcmp(val, "ReplicaSet") == 0 || - strcmp(val, "ReplicationController") == 0 || strcmp(val, "Pod") == 0 || - strcmp(val, "Job") == 0 || strcmp(val, "CronJob") == 0); - } else if (!got_name && under_metadata && strcmp(key, "name") == 0) { + } else if (ind == 0 && strcmp(key, "kind") == 0) { + rec->kind = k8s_rec_kind(val); + } else if (depth == 1 && strcmp(keys[0], "metadata") == 0) { + if (!got_name && strcmp(key, "name") == 0) { snprintf(rec->name, sizeof(rec->name), "%s", val); got_name = true; - } else if (under_selector) { - /* Service spec.selector matchLabels values. */ - k8s_add_val(rec->selector_vals, &rec->n_selector, val); - } else if (under_labels) { - /* Pod-template / metadata labels values. */ - k8s_add_val(rec->label_vals, &rec->n_label, val); + } else if (strcmp(key, "namespace") == 0) { + snprintf(rec->ns, sizeof(rec->ns), "%s", val); + } + } else { + char parent[CBM_SZ_512]; + k8s_parent_path(parent, sizeof(parent), (const char (*)[64])keys, depth); + k8s_pair_block_t block = k8s_pair_block(parent); + if (block != K8S_PAIR_NONE) { + k8s_scanned_push(out, block, key, val); } } } @@ -263,40 +362,163 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec) { } } -/* True if any of the service's selector values matches the workload. */ -static bool k8s_selector_matches(const k8s_record_t *svc, const k8s_record_t *wl) { - for (int s = 0; s < svc->n_selector; s++) { - const char *sv = svc->selector_vals[s]; - if (!sv[0]) { - continue; +static bool k8s_records_reserve(k8s_record_array_t *recs, int pair_extra) { + if (recs->count == recs->cap) { + int cap = recs->cap ? recs->cap * 2 : K8S_GROW_MIN; + k8s_record_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, recs->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; } - if (wl->name[0] && strcmp(sv, wl->name) == 0) { - return true; + recs->items = grown; + recs->cap = cap; + } + if (recs->pair_count + pair_extra > recs->pair_cap) { + int cap = recs->pair_cap ? recs->pair_cap : K8S_GROW_MIN; + while (cap < recs->pair_count + pair_extra) { + cap *= 2; } - for (int l = 0; l < wl->n_label; l++) { - if (strcmp(sv, wl->label_vals[l]) == 0) { - return true; + char **grown = cbm_realloc(CBM_MEM_CLASS_OTHER, recs->pairs, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + recs->pairs = grown; + recs->pair_cap = cap; + } + return true; +} + +/* Record the manifest behind Resource `node_id`: scan its text, keep the + * pairs its kind contributes, drop the rest. */ +static void k8s_record_manifest(k8s_record_array_t *recs, int64_t node_id, const char *source) { + k8s_record_t rec = {.node_id = node_id}; + k8s_scanned_t scanned = {0}; + k8s_scan_labels(source, &rec, &scanned); + k8s_pair_block_t keep = k8s_kind_block(rec.kind); + int kept = 0; + for (int i = 0; i < scanned.count; i++) { + kept += scanned.items[i].block == keep; + } + if (rec.kind != K8S_REC_OTHER && k8s_records_reserve(recs, kept)) { + rec.pair_off = recs->pair_count; + for (int i = 0; i < scanned.count; i++) { + if (scanned.items[i].block == keep) { + recs->pairs[recs->pair_count++] = scanned.items[i].kv; + scanned.items[i].kv = NULL; } } + rec.pair_n = recs->pair_count - rec.pair_off; + recs->items[recs->count++] = rec; + } + for (int i = 0; i < scanned.count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, scanned.items[i].kv); + } + cbm_free(CBM_MEM_CLASS_OTHER, scanned.items); +} + +static void k8s_records_free(k8s_record_array_t *recs) { + for (int i = 0; i < recs->pair_count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, recs->pairs[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, recs->pairs); + cbm_free(CBM_MEM_CLASS_OTHER, recs->items); + *recs = (k8s_record_array_t){0}; +} + +/* Workloads (record indices, ascending) that carry one "key=value" pair. */ +typedef struct { + int *items; + int count; + int cap; +} k8s_posting_t; + +static void k8s_posting_free(const char *key, void *value, void *userdata) { + (void)key; + (void)userdata; + k8s_posting_t *posting = value; + cbm_free(CBM_MEM_CLASS_OTHER, posting->items); + cbm_free(CBM_MEM_CLASS_OTHER, posting); +} + +static bool k8s_posting_add(CBMHashTable *index, const char *kv, int rec_idx) { + k8s_posting_t *posting = cbm_ht_get(index, kv); + if (!posting) { + posting = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*posting)); + if (!posting) { + return false; + } + cbm_ht_set(index, kv, posting); + } + if (posting->count && posting->items[posting->count - 1] == rec_idx) { + return true; /* the same pair twice in one workload */ + } + if (posting->count == posting->cap) { + int cap = posting->cap ? posting->cap * 2 : K8S_GROW_MIN; + int *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, posting->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + posting->items = grown; + posting->cap = cap; + } + posting->items[posting->count++] = rec_idx; + return true; +} + +static bool k8s_has_pair(const k8s_record_array_t *recs, const k8s_record_t *rec, const char *kv) { + for (int i = 0; i < rec->pair_n; i++) { + if (strcmp(recs->pairs[rec->pair_off + i], kv) == 0) { + return true; + } } return false; } -/* After all manifests are recorded, connect each Service to the workload(s) its - * selector targets via an INFRA_MAPS edge (Service Resource → workload Resource). */ +/* Two set namespaces must agree; an unset one is the deploy-time namespace. */ +static bool k8s_same_namespace(const k8s_record_t *a, const k8s_record_t *b) { + return !a->ns[0] || !b->ns[0] || strcmp(a->ns, b->ns) == 0; +} + +/* Connect each Service to the workloads its selector targets via an INFRA_MAPS + * edge (Service Resource → workload Resource). */ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs) { + CBMHashTable *index = cbm_ht_create(0); + if (!index) { + return; + } + for (int i = 0; i < recs->count; i++) { + const k8s_record_t *wl = &recs->items[i]; + if (wl->kind == K8S_REC_WORKLOAD || wl->kind == K8S_REC_POD) { + for (int p = 0; p < wl->pair_n; p++) { + (void)k8s_posting_add(index, recs->pairs[wl->pair_off + p], i); + } + } + } int edges = 0; for (int i = 0; i < recs->count; i++) { const k8s_record_t *svc = &recs->items[i]; - if (!svc->is_service || svc->n_selector == 0 || svc->node_id <= 0) { + if (svc->kind != K8S_REC_SERVICE || svc->pair_n == 0) { continue; } - for (int j = 0; j < recs->count; j++) { - const k8s_record_t *wl = &recs->items[j]; - if (i == j || !wl->is_workload || wl->node_id <= 0) { - continue; + /* Candidates: the workloads carrying the Service's rarest pair. */ + const k8s_posting_t *rarest = NULL; + for (int p = 0; p < svc->pair_n; p++) { + const k8s_posting_t *posting = cbm_ht_get(index, recs->pairs[svc->pair_off + p]); + if (!posting) { + rarest = NULL; + break; } - if (k8s_selector_matches(svc, wl)) { + if (!rarest || posting->count < rarest->count) { + rarest = posting; + } + } + for (int c = 0; rarest && c < rarest->count; c++) { + const k8s_record_t *wl = &recs->items[rarest->items[c]]; + bool all = k8s_same_namespace(svc, wl); + for (int p = 0; all && p < svc->pair_n; p++) { + all = k8s_has_pair(recs, wl, recs->pairs[svc->pair_off + p]); + } + if (all) { char props[CBM_SZ_256]; snprintf(props, sizeof(props), "{\"kind\":\"selector\",\"service\":\"%s\",\"workload\":\"%s\"}", @@ -306,6 +528,8 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t } } } + cbm_ht_foreach(index, k8s_posting_free, NULL); + cbm_ht_free(index); if (edges > 0) { cbm_log_info("pass.k8s.selectors", "linked", itoa_k8s(edges)); } @@ -314,12 +538,12 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t /* ── K8s manifest handler ────────────────────────────────────────── */ /* source/src_len are the already-read file bytes (caller retains ownership and - * must free after this call returns). When `rec` is non-NULL it is populated - * with the first Resource's node id, name and label/selector values for later - * cross-manifest selector matching. */ + * must free after this call returns). The first Resource's manifest is + * recorded in `recs` for cross-manifest selector matching. */ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, - const CBMFileResult *res, k8s_record_t *rec) { + const CBMFileResult *res, k8s_record_array_t *recs) { int resource_count = 0; + int64_t first_node_id = 0; if (!res) { return; @@ -349,16 +573,16 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c } /* Capture the first Resource for cross-manifest selector matching. */ - if (rec && rec->node_id <= 0 && node_id > 0) { - rec->node_id = node_id; + if (first_node_id <= 0 && node_id > 0) { + first_node_id = node_id; } resource_count++; } - /* Record selector / pod-label values for later Service → workload linking. */ - if (rec && rec->node_id > 0) { - k8s_scan_labels(source, rec); + /* Record selector / pod-label pairs for later Service → workload linking. */ + if (first_node_id > 0 && source) { + k8s_record_manifest(recs, first_node_id, source); } cbm_log_info("pass.k8s.manifest", "file", rel_path, "resources", itoa_k8s(resource_count)); @@ -648,12 +872,9 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, /* Collect per-manifest selector/label records for cross-manifest matching. */ k8s_record_array_t recs = {0}; - recs.items = cbm_calloc(CBM_MEM_CLASS_OTHER, K8S_MAX_RECORDS * sizeof(*recs.items)); - recs.cap = recs.items ? K8S_MAX_RECORDS : 0; k8s_prep_t *prep = cbm_calloc(CBM_MEM_CLASS_OTHER, K8S_CHUNK * sizeof(*prep)); if (!prep) { - cbm_free(CBM_MEM_CLASS_OTHER, recs.items); return CBM_NOT_FOUND; } int workers = cbm_default_worker_count(false); @@ -662,7 +883,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int end = begin + K8S_CHUNK < file_count ? begin + K8S_CHUNK : file_count; if (cbm_pipeline_check_cancel(ctx)) { cbm_free(CBM_MEM_CLASS_OTHER, prep); - cbm_free(CBM_MEM_CLASS_OTHER, recs.items); + k8s_records_free(&recs); return CBM_NOT_FOUND; } memset(prep, 0, (size_t)K8S_CHUNK * sizeof(*prep)); @@ -690,15 +911,10 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, handle_helm_chart(ctx, rel, p->source); helm_count++; break; - case K8S_MANIFEST: { - k8s_record_t *rec = (recs.count < recs.cap) ? &recs.items[recs.count] : NULL; - handle_k8s_manifest(ctx, rel, p->source, p->res, rec); - if (rec && rec->node_id > 0) { - recs.count++; - } + case K8S_MANIFEST: + handle_k8s_manifest(ctx, rel, p->source, p->res, &recs); manifest_count++; break; - } case K8S_SKIP: default: break; @@ -711,7 +927,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, } if (cancelled) { cbm_free(CBM_MEM_CLASS_OTHER, prep); - cbm_free(CBM_MEM_CLASS_OTHER, recs.items); + k8s_records_free(&recs); return CBM_NOT_FOUND; } } @@ -719,7 +935,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, /* Connect Services to the workloads their selectors target (INFRA_MAPS). */ k8s_link_selectors(ctx, &recs); - cbm_free(CBM_MEM_CLASS_OTHER, recs.items); + k8s_records_free(&recs); cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count)); (void)helm_count; diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index c619e68d7..bdfc9c4b5 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -351,9 +351,10 @@ TEST(probe_k8s_two_manifests) { PASS(); } -/* K8s: cross-manifest reference via INFRA_MAPS. - * RED: the single-path pipeline does not emit INFRA_MAPS edges for k8s refs; - * pass_k8s only emits Resource nodes (no ref-resolution between manifests). */ +/* K8s: a Service routes to the workload whose pod template carries every + * label of its selector (INFRA_MAPS Service -> workload). The fixture had no + * pod labels and linked only through a "selector value == workload name" + * guess that Kubernetes does not make. */ TEST(probe_k8s_infra_maps_edge) { static const GpfFile files[] = { {"deploy.yaml", @@ -363,6 +364,9 @@ TEST(probe_k8s_infra_maps_edge) { " name: frontend\n" "spec:\n" " template:\n" + " metadata:\n" + " labels:\n" + " app: frontend\n" " spec:\n" " containers:\n" " - name: frontend\n" @@ -378,8 +382,211 @@ TEST(probe_k8s_infra_maps_edge) { }; GpfMetrics m = gpf_metrics_files(files, 2); ASSERT_TRUE(m.ok); - /* RED: inter-manifest INFRA_MAPS edges not modeled by the current k8s pass. */ - ASSERT_TRUE(m.infra_maps >= 1); /* expected RED — no k8s selector → INFRA_MAPS */ + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + +/* A selector is the AND of its pairs: `tier: back` pods are not selected by a + * `{app: web, tier: front}` Service. */ +TEST(probe_k8s_selector_requires_all_pairs) { + static const GpfFile files[] = { + {"front.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web-front\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n" + " tier: front\n"}, + {"back.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web-back\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n" + " tier: back\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " app: web\n" + " tier: front\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 3); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + +/* Only the pod template's labels are selectable; the workload's own + * metadata.labels are not. */ +TEST(probe_k8s_selector_ignores_workload_metadata_labels) { + static const GpfFile files[] = { + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: api\n" + " labels:\n" + " app: web\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: api\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " app: web\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 0); + PASS(); +} + +/* A workload whose NAME equals a selector value is not selected by it. */ +TEST(probe_k8s_selector_no_name_guess) { + static const GpfFile files[] = { + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: frontend\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: fe\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: frontend\n" + "spec:\n" + " selector:\n" + " app: frontend\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 0); + PASS(); +} + +/* `component: web` does not select pods labelled `app: web`. */ +TEST(probe_k8s_selector_key_matters) { + static const GpfFile files[] = { + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: api\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " component: web\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 0); + PASS(); +} + +/* A Service selects pods of its own namespace only. */ +TEST(probe_k8s_selector_namespace_scoped) { + static const GpfFile files[] = { + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + " namespace: team-b\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + " namespace: team-a\n" + "spec:\n" + " selector:\n" + " app: web\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 0); + PASS(); +} + +/* Selector linking recorded at most 512 manifests; a Service and workload + * indexed after that limit (deckhouse has 2,003 manifest files) were never + * linked. No count of manifests may decide graph content. */ +TEST(probe_k8s_selector_beyond_512_manifests) { + enum { FILLER = 600 }; + GpfFile *files = calloc(FILLER + 2, sizeof(*files)); + char (*names)[32] = calloc(FILLER, sizeof(*names)); + ASSERT_NOT_NULL(files); + ASSERT_NOT_NULL(names); + for (int i = 0; i < FILLER; i++) { + snprintf(names[i], sizeof(names[i]), "a/cm-%03d.yaml", i); + files[i].name = names[i]; + files[i].content = "apiVersion: v1\n" + "kind: ConfigMap\n" + "metadata:\n" + " name: filler\n" + "data:\n" + " k: v\n"; + } + files[FILLER].name = "z/deploy.yaml"; + files[FILLER].content = "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: late\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: late\n"; + files[FILLER + 1].name = "z/service.yaml"; + files[FILLER + 1].content = "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: late\n" + "spec:\n" + " selector:\n" + " app: late\n"; + GpfMetrics m = gpf_metrics_files(files, FILLER + 2); + free(names); + free(files); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 1); PASS(); } @@ -1728,6 +1935,12 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_deployment_resource_node); RUN_TEST(probe_k8s_two_manifests); RUN_TEST(probe_k8s_infra_maps_edge); + RUN_TEST(probe_k8s_selector_requires_all_pairs); + RUN_TEST(probe_k8s_selector_ignores_workload_metadata_labels); + RUN_TEST(probe_k8s_selector_no_name_guess); + RUN_TEST(probe_k8s_selector_key_matters); + RUN_TEST(probe_k8s_selector_namespace_scoped); + RUN_TEST(probe_k8s_selector_beyond_512_manifests); /* Kustomize */ RUN_TEST(probe_kustomize_module_node); From 29864dc0cdff7a961b2bb1f5c17c419bfba4f573 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 13:16:53 +0200 Subject: [PATCH 06/52] fix(helm): resolve file:// chart dependencies and keep their repository Chart.yaml dependencies became one external Chart node per name and nothing else: a `file://../common` dependency -- the library chart in the next directory -- was an "external" chart, every edge lost where the dependency comes from, and the parser kept at most 128 entries. - The parser reads each entry's name and repository (keys in any order; lists nested in an entry, such as import-values, name no dependency of their own) into a growing array. - A `file://` dependency that names a chart of this repository links DEPENDS_ON to that chart's node, resolved after every Chart.yaml is read so the order of the files does not matter. - Any other dependency -- remote, OCI, or a `file://` path the repository cannot resolve (deckhouse names its build container's /deckhouse/...) -- links to the shared external node as before. - Every DEPENDS_ON edge carries the entry's repository. On real input: kubermatic 15/15 and deckhouse 2/2 dependency edges now carry their repository, prometheus-community/helm-charts 8 of 10 (two entries name none); Chart nodes and edge counts unchanged. None of the 17 corpora has a relative `file://` dependency; the new pipeline probe covers it. Probes: the local-chart link fails with the file:// branch disabled, the nested-list parser test fails with a parser that takes every dash and key in the block. Signed-off-by: Martin Vogel --- src/pipeline/pass_infrascan.c | 74 +++++++++++++--- src/pipeline/pass_k8s.c | 142 ++++++++++++++++++++++++++++--- src/pipeline/pipeline_internal.h | 17 ++-- tests/test_grammar_probe_f.c | 50 +++++++++++ tests/test_pipeline.c | 53 +++++++++++- 5 files changed, 305 insertions(+), 31 deletions(-) diff --git a/src/pipeline/pass_infrascan.c b/src/pipeline/pass_infrascan.c index a9b2418fd..24af73d86 100644 --- a/src/pipeline/pass_infrascan.c +++ b/src/pipeline/pass_infrascan.c @@ -1272,14 +1272,51 @@ static const char *helm_match_key(const char *content, const char *eol, const ch return content + kl + 1; } +/* Append an empty dependency entry; returns it, or NULL when out of memory. */ +static cbm_helm_dep_t *helm_push_dep(cbm_helm_chart_t *out) { + if (out->dep_count == out->dep_cap) { + enum { HELM_DEPS_MIN = 8 }; + int cap = out->dep_cap ? out->dep_cap * 2 : HELM_DEPS_MIN; + cbm_helm_dep_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, out->deps, (size_t)cap * sizeof(*grown)); + if (!grown) { + return NULL; + } + out->deps = grown; + out->dep_cap = cap; + } + cbm_helm_dep_t *dep = &out->deps[out->dep_count++]; + dep->name[0] = '\0'; + dep->repository[0] = '\0'; + return dep; +} + +void cbm_helm_chart_free(cbm_helm_chart_t *hc) { + if (!hc) { + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, hc->deps); + hc->deps = NULL; + hc->dep_count = 0; + hc->dep_cap = 0; +} + int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out) { - if (!source || !out) { + if (!out) { return -1; } out->chart_name[0] = '\0'; + out->deps = NULL; out->dep_count = 0; + out->dep_cap = 0; + if (!source) { + return -1; + } bool in_deps = false; + cbm_helm_dep_t *cur = NULL; /* the dependency entry being read */ + int item_indent = -1; /* indent of the entries' "- " */ + int key_col = -1; /* column of an entry's keys */ const char *p = source; while (*p) { const char *eol = strchr(p, '\n'); @@ -1296,26 +1333,35 @@ int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out) { if (indent == 0) { /* New top-level key ends any dependencies block. */ in_deps = (helm_match_key(content, eol, "dependencies") != NULL); + cur = NULL; + item_indent = -1; + key_col = -1; const char *nv = helm_match_key(content, eol, "name"); if (!in_deps && nv && out->chart_name[0] == '\0') { helm_copy_scalar(nv, eol, out->chart_name, CBM_HELM_NAME_MAX); } } else if (in_deps) { - /* List item: optional leading "- ", then look for name:. */ + /* A list item ("- ") at the entries' indent opens an entry; its + * keys share the dash line or follow at the same key column. + * Deeper lines (import-values, tags) belong to the entry. */ const char *q = content; - if (*q == '-') { + bool opens = *q == '-' && (item_indent < 0 || indent == item_indent); + if (opens) { + item_indent = indent; q++; while (q < eol && *q == ' ') { q++; } + key_col = indent + (int)(q - content); + cur = helm_push_dep(out); } - const char *nv = helm_match_key(q, eol, "name"); - if (nv && out->dep_count < CBM_HELM_MAX_DEPS) { - char buf[CBM_HELM_NAME_MAX]; - helm_copy_scalar(nv, eol, buf, CBM_HELM_NAME_MAX); - if (buf[0]) { - memcpy(out->deps[out->dep_count], buf, CBM_HELM_NAME_MAX); - out->dep_count++; + if (cur && (opens || indent == key_col)) { + const char *nv = helm_match_key(q, eol, "name"); + const char *rv = helm_match_key(q, eol, "repository"); + if (nv) { + helm_copy_scalar(nv, eol, cur->name, CBM_HELM_NAME_MAX); + } else if (rv) { + helm_copy_scalar(rv, eol, cur->repository, CBM_HELM_REPO_MAX); } } } @@ -1325,5 +1371,13 @@ int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out) { } p = eol + 1; } + /* Keep only named entries (a dependency without a name names nothing). */ + int kept = 0; + for (int i = 0; i < out->dep_count; i++) { + if (out->deps[i].name[0]) { + out->deps[kept++] = out->deps[i]; + } + } + out->dep_count = kept; return (out->chart_name[0] || out->dep_count > 0) ? 0 : -1; } diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 047df4355..34a6aeb76 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -23,6 +23,7 @@ #include "helpers.h" /* cbm_kind_in_set_free_cache */ #include "foundation/slab_alloc.h" /* cbm_slab_destroy_thread */ #include "foundation/platform.h" /* cbm_default_worker_count */ +#include "foundation/str_util.h" /* cbm_json_escape */ #include "pipeline/worker_pool.h" #include @@ -30,6 +31,8 @@ #include #include +#define K8S_SLEN(s) (sizeof(s) - 1) + /* ── Internal helpers ────────────────────────────────────────────── */ /* Read entire file into heap-allocated buffer. Returns NULL on error. @@ -594,17 +597,125 @@ static bool is_helm_chart_file(const char *base) { return strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0; } -/* Emit a Chart node for a Chart.yaml and a DEPENDS_ON edge to a (shared, - * deduplicated) Chart node per declared dependency (#338). */ -static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source) { +/* A chart's `file://` dependency: the chart in that directory of this + * repository. Linked once every Chart.yaml has been read, so a dependency + * indexed after its dependent still resolves. One that names no chart of this + * repository (an absolute build-container path, a directory without a chart) + * stays a declared dependency on the external node, like a remote one. */ +typedef struct { + int64_t chart_id; + char name[CBM_HELM_NAME_MAX]; /* dependency name, for the external fallback */ + char *chart_rel; /* the depending Chart.yaml */ + char *target_qn; /* QN of the depended-on Chart node; NULL = unresolvable */ + char *props; /* edge properties */ +} k8s_local_dep_t; + +typedef struct { + k8s_local_dep_t *items; + int count; + int cap; +} k8s_local_deps_t; + +static bool k8s_is_file_repository(const char *repository) { + return strncmp(repository, "file://", K8S_SLEN("file://")) == 0; +} + +static char *k8s_dep_props(const char *repository) { + char escaped[CBM_HELM_REPO_MAX * 2]; + cbm_json_escape(escaped, (int)sizeof(escaped), repository); + size_t n = strlen(escaped) + sizeof("{\"repository\":\"\"}"); + char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, n); + if (props) { + snprintf(props, n, "{\"repository\":\"%s\"}", escaped); + } + return props; +} + +/* DEPENDS_ON from `chart_id` to the shared external Chart node of `name` + * (one per name and project, so charts depending on the same chart meet). */ +static bool k8s_external_dep(cbm_pipeline_ctx_t *ctx, int64_t chart_id, const char *name, + const char *chart_rel, const char *props) { + char dep_qn[CBM_SZ_512]; + snprintf(dep_qn, sizeof(dep_qn), "%s.__helm_dep__.%s", ctx->project_name, name); + int64_t dep_id = cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, dep_qn, chart_rel, SKIP_ONE, 0, + "{\"source\":\"helm\",\"external\":true}"); + if (dep_id <= 0) { + return false; + } + cbm_gbuf_insert_edge(ctx->gbuf, chart_id, dep_id, "DEPENDS_ON", props ? props : "{}"); + return true; +} + +static void k8s_local_deps_push(k8s_local_deps_t *deps, int64_t chart_id, const char *chart_rel, + const char *project, const cbm_helm_dep_t *dep) { + const char *dir = dep->repository + K8S_SLEN("file://"); + char *target_qn = NULL; + if (dir[0] && dir[0] != '/') { /* an absolute path names the build host, not this repo */ + char rel[CBM_HELM_REPO_MAX + K8S_SLEN("./") + K8S_SLEN("/Chart.yaml")]; + snprintf(rel, sizeof(rel), "%s%s/Chart.yaml", strncmp(dir, "../", 3) == 0 ? "" : "./", dir); + char *path = cbm_pipeline_resolve_relative_import(chart_rel, rel); + target_qn = path ? cbm_infra_qn(project, path, "helm-chart", NULL) : NULL; + free(path); + } + if (deps->count == deps->cap) { + int cap = deps->cap ? deps->cap * 2 : K8S_GROW_MIN; + k8s_local_dep_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, deps->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + free(target_qn); + return; + } + deps->items = grown; + deps->cap = cap; + } + k8s_local_dep_t *d = &deps->items[deps->count++]; + *d = (k8s_local_dep_t){.chart_id = chart_id, .target_qn = target_qn}; + snprintf(d->name, sizeof(d->name), "%s", dep->name); + d->chart_rel = strdup(chart_rel); + d->props = k8s_dep_props(dep->repository); +} + +static void k8s_local_deps_free(k8s_local_deps_t *deps) { + for (int i = 0; i < deps->count; i++) { + free(deps->items[i].target_qn); + free(deps->items[i].chart_rel); + cbm_free(CBM_MEM_CLASS_OTHER, deps->items[i].props); + } + cbm_free(CBM_MEM_CLASS_OTHER, deps->items); + *deps = (k8s_local_deps_t){0}; +} + +static void k8s_link_local_deps(cbm_pipeline_ctx_t *ctx, k8s_local_deps_t *deps) { + for (int i = 0; i < deps->count; i++) { + const k8s_local_dep_t *d = &deps->items[i]; + const cbm_gbuf_node_t *target = + d->target_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, d->target_qn) : NULL; + if (target && target->label && strcmp(target->label, "Chart") == 0 && + target->id != d->chart_id) { + cbm_gbuf_insert_edge(ctx->gbuf, d->chart_id, target->id, "DEPENDS_ON", d->props); + } else if (d->chart_rel) { + (void)k8s_external_dep(ctx, d->chart_id, d->name, d->chart_rel, d->props); + } + } + k8s_local_deps_free(deps); +} + +/* Emit a Chart node for a Chart.yaml and a DEPENDS_ON edge per declared + * dependency (#338): to the chart in this repository for a resolvable + * `file://` dependency (once all charts are read), otherwise to a shared + * external Chart node per dependency name, the repository on the edge. */ +static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, + k8s_local_deps_t *local_deps) { cbm_helm_chart_t hc; if (cbm_parse_helm_chart(source, &hc) != 0) { + cbm_helm_chart_free(&hc); return; } const char *cname = hc.chart_name[0] ? hc.chart_name : k8s_basename(rel_path); char *chart_qn = cbm_infra_qn(ctx->project_name, rel_path, "helm-chart", NULL); if (!chart_qn) { + cbm_helm_chart_free(&hc); return; } int64_t chart_id = cbm_gbuf_upsert_node(ctx->gbuf, "Chart", cname, chart_qn, rel_path, SKIP_ONE, @@ -620,18 +731,16 @@ static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, con int dep_edges = 0; for (int i = 0; i < hc.dep_count && chart_id > 0; i++) { - /* Stable per-project QN so multiple charts depending on the same chart - * link to one shared dependency node. */ - char dep_qn[CBM_SZ_512]; - snprintf(dep_qn, sizeof(dep_qn), "%s.__helm_dep__.%s", ctx->project_name, hc.deps[i]); - int64_t dep_id = - cbm_gbuf_upsert_node(ctx->gbuf, "Chart", hc.deps[i], dep_qn, rel_path, SKIP_ONE, 0, - "{\"source\":\"helm\",\"external\":true}"); - if (dep_id > 0) { - cbm_gbuf_insert_edge(ctx->gbuf, chart_id, dep_id, "DEPENDS_ON", "{}"); - dep_edges++; + const cbm_helm_dep_t *dep = &hc.deps[i]; + if (k8s_is_file_repository(dep->repository)) { + k8s_local_deps_push(local_deps, chart_id, rel_path, ctx->project_name, dep); + continue; } + char *props = dep->repository[0] ? k8s_dep_props(dep->repository) : NULL; + dep_edges += k8s_external_dep(ctx, chart_id, dep->name, rel_path, props); + cbm_free(CBM_MEM_CLASS_OTHER, props); } + cbm_helm_chart_free(&hc); cbm_log_info("pass.k8s.helm", "file", rel_path, "deps", itoa_k8s(dep_edges)); } @@ -872,6 +981,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, /* Collect per-manifest selector/label records for cross-manifest matching. */ k8s_record_array_t recs = {0}; + k8s_local_deps_t local_deps = {0}; k8s_prep_t *prep = cbm_calloc(CBM_MEM_CLASS_OTHER, K8S_CHUNK * sizeof(*prep)); if (!prep) { @@ -884,6 +994,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, if (cbm_pipeline_check_cancel(ctx)) { cbm_free(CBM_MEM_CLASS_OTHER, prep); k8s_records_free(&recs); + k8s_local_deps_free(&local_deps); return CBM_NOT_FOUND; } memset(prep, 0, (size_t)K8S_CHUNK * sizeof(*prep)); @@ -908,7 +1019,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, is_requirements_file(base) ? "pypi" : "gomod"); break; case K8S_HELM: - handle_helm_chart(ctx, rel, p->source); + handle_helm_chart(ctx, rel, p->source, &local_deps); helm_count++; break; case K8S_MANIFEST: @@ -928,6 +1039,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, if (cancelled) { cbm_free(CBM_MEM_CLASS_OTHER, prep); k8s_records_free(&recs); + k8s_local_deps_free(&local_deps); return CBM_NOT_FOUND; } } @@ -936,6 +1048,8 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, /* Connect Services to the workloads their selectors target (INFRA_MAPS). */ k8s_link_selectors(ctx, &recs); k8s_records_free(&recs); + /* Link `file://` chart dependencies now that every Chart.yaml is read. */ + k8s_link_local_deps(ctx, &local_deps); cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count)); (void)helm_count; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 87b9c160b..544bc3412 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -995,17 +995,24 @@ int cbm_parse_shell_source(const char *source, cbm_shell_result_t *out); /* Parse a Terraform file from source text. Returns 0 if parsed, -1 if empty. */ int cbm_parse_terraform_source(const char *source, cbm_terraform_result_t *out); -/* Helm Chart.yaml parse result: chart name + dependency chart names (#338). */ -enum { CBM_HELM_MAX_DEPS = 128, CBM_HELM_NAME_MAX = 128 }; +/* Helm Chart.yaml parse result: chart name + its dependencies (#338). */ +enum { CBM_HELM_NAME_MAX = 128, CBM_HELM_REPO_MAX = 512 }; +typedef struct { + char name[CBM_HELM_NAME_MAX]; + char repository[CBM_HELM_REPO_MAX]; /* "" when the entry names none */ +} cbm_helm_dep_t; typedef struct { char chart_name[CBM_HELM_NAME_MAX]; - char deps[CBM_HELM_MAX_DEPS][CBM_HELM_NAME_MAX]; + cbm_helm_dep_t *deps; /* heap-owned; release with cbm_helm_chart_free */ int dep_count; + int dep_cap; } cbm_helm_chart_t; -/* Parse a Helm Chart.yaml: top-level `name:` and `dependencies:` list names. - * Returns 0 if parsed (name or deps found), -1 otherwise. */ +/* Parse a Helm Chart.yaml: top-level `name:` and every `dependencies:` entry's + * name and repository. Returns 0 if parsed (name or deps found), -1 otherwise; + * either way the caller releases `out` with cbm_helm_chart_free. */ int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out); +void cbm_helm_chart_free(cbm_helm_chart_t *hc); /* Build an infrastructure QN. Caller must free the returned string. */ char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *infra_type, diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index bdfc9c4b5..ec569c312 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -138,6 +138,7 @@ typedef struct { int modules; /* Module label */ int classes; /* Class label */ int resources; /* Resource label (k8s) */ + int charts; /* Chart label (helm) */ int imports; /* IMPORTS edges */ int depends_on; /* DEPENDS_ON edges */ int infra_maps; /* INFRA_MAPS edges */ @@ -154,6 +155,7 @@ static GpfMetrics gpf_metrics_files(const GpfFile *files, int nfiles) { m.modules = gpf_count_label(store, lp.project, "Module"); m.classes = gpf_count_label(store, lp.project, "Class"); m.resources = gpf_count_label(store, lp.project, "Resource"); + m.charts = gpf_count_label(store, lp.project, "Chart"); m.imports = cbm_store_count_edges_by_type(store, lp.project, "IMPORTS"); m.depends_on = cbm_store_count_edges_by_type(store, lp.project, "DEPENDS_ON"); m.infra_maps = cbm_store_count_edges_by_type(store, lp.project, "INFRA_MAPS"); @@ -741,6 +743,52 @@ TEST(probe_kustomize_extensionless_file) { PASS(); } +/* A `file://` dependency is the chart in that directory of this repository, + * not a published one: `app` depends on the local `common`, and no external + * Chart node stands in for it. `common` is read after `app`. */ +TEST(probe_helm_file_dependency_links_local_chart) { + static const GpfFile files[] = { + {"charts/app/Chart.yaml", + "apiVersion: v2\n" + "name: app\n" + "version: 1.0.0\n" + "dependencies:\n" + " - name: common\n" + " repository: file://../common\n" + " version: 1.x\n"}, + {"charts/common/Chart.yaml", + "apiVersion: v2\n" + "name: common\n" + "type: library\n" + "version: 1.2.0\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.charts, 2); + ASSERT_EQ(m.depends_on, 1); + PASS(); +} + +/* A `file://` path this repository cannot resolve (deckhouse names its build + * container's `/deckhouse/...`) still declares a dependency: it stays the + * external Chart node, the repository on its edge, rather than vanishing. */ +TEST(probe_helm_unresolved_file_dependency_stays_external) { + static const GpfFile files[] = { + {"modules/policy/Chart.yaml", + "apiVersion: v2\n" + "name: policy\n" + "version: 1.0.0\n" + "dependencies:\n" + " - name: constraint-templates\n" + " repository: \"file:///deckhouse/modules/policy/charts/constraint_templates\"\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 1); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.charts, 2); + ASSERT_EQ(m.depends_on, 1); + PASS(); +} + /* A remote base is outside the repository. The import resolver's symbol-name * guess walked its URL segments and linked kubermatic's gcp addon to the * `deploy` function of hack/ci/deploy.sh. A kustomize entry is a path, @@ -1951,6 +1999,8 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_selector_after_leading_separator); RUN_TEST(probe_kustomize_extensionless_file); RUN_TEST(probe_kustomize_remote_base_names_no_symbol); + RUN_TEST(probe_helm_file_dependency_links_local_chart); + RUN_TEST(probe_helm_unresolved_file_dependency_stays_external); RUN_TEST(probe_kustomize_nested_sibling_entries); RUN_TEST(probe_kustomize_single_module); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 68166f0c4..5ac761369 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14165,8 +14165,11 @@ TEST(helm_parse_chart_dependencies_issue338) { ASSERT_EQ(cbm_parse_helm_chart(src, &hc), 0); ASSERT_STR_EQ(hc.chart_name, "mychart"); ASSERT_EQ(hc.dep_count, 2); - ASSERT_STR_EQ(hc.deps[0], "postgresql"); - ASSERT_STR_EQ(hc.deps[1], "redis"); + ASSERT_STR_EQ(hc.deps[0].name, "postgresql"); + ASSERT_STR_EQ(hc.deps[0].repository, "https://charts.bitnami.com/bitnami"); + ASSERT_STR_EQ(hc.deps[1].name, "redis"); + ASSERT_STR_EQ(hc.deps[1].repository, "https://charts.bitnami.com/bitnami"); + cbm_helm_chart_free(&hc); PASS(); } @@ -14175,6 +14178,50 @@ TEST(helm_parse_chart_no_deps_issue338) { ASSERT_EQ(cbm_parse_helm_chart("name: solo\nversion: 0.1.0\n", &hc), 0); ASSERT_STR_EQ(hc.chart_name, "solo"); ASSERT_EQ(hc.dep_count, 0); + cbm_helm_chart_free(&hc); + PASS(); +} + +/* An entry's keys come in any order; lists nested in an entry + * (import-values, tags) belong to it and name no dependency of their own. */ +TEST(helm_parse_chart_dependency_entries) { + const char *src = "name: app\n" + "dependencies:\n" + " - repository: file://../common\n" + " name: common\n" + " import-values:\n" + " - child: default.data\n" + " parent: imported\n" + " - name: not-a-dependency\n" + " - name: redis\n" + " tags:\n" + " - cache\n"; + cbm_helm_chart_t hc; + ASSERT_EQ(cbm_parse_helm_chart(src, &hc), 0); + ASSERT_EQ(hc.dep_count, 2); + ASSERT_STR_EQ(hc.deps[0].name, "common"); + ASSERT_STR_EQ(hc.deps[0].repository, "file://../common"); + ASSERT_STR_EQ(hc.deps[1].name, "redis"); + ASSERT_STR_EQ(hc.deps[1].repository, ""); + cbm_helm_chart_free(&hc); + PASS(); +} + +/* No dependency count decides what is read: the parser kept 128. */ +TEST(helm_parse_chart_many_dependencies) { + enum { DEPS = 130 }; + char *src = malloc(DEPS * 32 + 64); + ASSERT_NOT_NULL(src); + int n = snprintf(src, 64, "name: umbrella\ndependencies:\n"); + for (int i = 0; i < DEPS; i++) { + n += snprintf(src + n, 32, " - name: dep-%03d\n", i); + } + cbm_helm_chart_t hc; + ASSERT_EQ(cbm_parse_helm_chart(src, &hc), 0); + free(src); + ASSERT_EQ(hc.dep_count, DEPS); + ASSERT_STR_EQ(hc.deps[DEPS - 1].name, "dep-129"); + cbm_helm_chart_free(&hc); PASS(); } @@ -19702,6 +19749,8 @@ SUITE(pipeline) { RUN_TEST(infra_parse_terraform_empty); RUN_TEST(helm_parse_chart_dependencies_issue338); RUN_TEST(helm_parse_chart_no_deps_issue338); + RUN_TEST(helm_parse_chart_dependency_entries); + RUN_TEST(helm_parse_chart_many_dependencies); /* Infrascan: QN helper */ RUN_TEST(infra_qn_helper); /* Infrascan: pipeline integration */ From b3e66e1bf1d05656a764a0c2faa2922707f6612f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 13:26:13 +0200 Subject: [PATCH 07/52] fix(yaml): a Kubernetes manifest mints no Variables for its envelope Every top-level key of a YAML file became a Variable, so each manifest added `apiVersion`, `kind`, `metadata`, `spec` and `status` (deckhouse: `kind` 1,358, `apiVersion` 1,347, `metadata` 1,336, `spec` 1,231). They are the object's envelope, which its Resource node already carries, and as config keys they fed name-similarity CONFIGURES links -- a Chart's `apiVersion` "configured" every `*.apiVersion` helper in 46 charts. A YAML document whose top level holds both `apiVersion` and `kind` now mints no key Variables; other YAML keeps them. On real input (Variables in YAML files / key_symbol CONFIGURES edges), against main: deckhouse 11,892 -> 5,662 / 2,631 -> 2,167, gardener 2,282 -> 503 / 1,677 -> 650, kubernetes/k8s.io 1,862 -> 844, onedr0p/home-ops 1,334 -> 41, kubermatic 7,911 -> 6,692, prometheus-community/helm-charts 2,364 -> 2,301 (its templates open with `{{- if }}` and do not parse as manifests). The manifest probe fails without the check; a config-YAML probe guards that other YAML keeps its Variables. Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 32 ++++++++++++++++++++++++++++++++ tests/test_grammar_probe_f.c | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 2a3b9d14d..ddf165d8c 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -9961,6 +9961,31 @@ static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) { } } +// True when the YAML document's top level holds both `apiVersion` and `kind`: +// a Kubernetes object, whose top-level keys are its envelope. +static bool yaml_is_k8s_object(CBMExtractCtx *ctx, TSNode root) { + TSNode bm = find_yaml_toplevel_mapping(root); + if (ts_node_is_null(bm)) { + return false; + } + bool api_version = false; + bool kind = false; + uint32_t n = ts_node_named_child_count(bm); + for (uint32_t i = 0; i < n && !(api_version && kind); i++) { + TSNode pair = ts_node_named_child(bm, i); + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + TSNode key = ts_node_child_by_field_name(pair, TS_FIELD("key")); + const char *k = ts_node_is_null(key) ? NULL : cbm_node_text(ctx->arena, key, ctx->source); + if (k) { + api_version = api_version || strcmp(k, "apiVersion") == 0; + kind = kind || strcmp(k, "kind") == 0; + } + } + return api_version && kind; +} + static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) { if (!spec->variable_node_types || !spec->variable_node_types[0]) { return; @@ -9972,6 +9997,13 @@ static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec return; } + // A Kubernetes object's top-level keys (apiVersion, kind, metadata, spec, + // status) are its envelope, which its Resource node carries; as Variables + // they repeated in every manifest and fed name-similarity config links. + if (ctx->language == CBM_LANG_YAML && yaml_is_k8s_object(ctx, root)) { + return; + } + // Config languages with nested structure: use recursive walk if (ctx->language == CBM_LANG_YAML || ctx->language == CBM_LANG_TOML || ctx->language == CBM_LANG_INI || ctx->language == CBM_LANG_JSON) { diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index ec569c312..8daf6d9c9 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -139,6 +139,7 @@ typedef struct { int classes; /* Class label */ int resources; /* Resource label (k8s) */ int charts; /* Chart label (helm) */ + int variables; /* Variable label */ int imports; /* IMPORTS edges */ int depends_on; /* DEPENDS_ON edges */ int infra_maps; /* INFRA_MAPS edges */ @@ -156,6 +157,7 @@ static GpfMetrics gpf_metrics_files(const GpfFile *files, int nfiles) { m.classes = gpf_count_label(store, lp.project, "Class"); m.resources = gpf_count_label(store, lp.project, "Resource"); m.charts = gpf_count_label(store, lp.project, "Chart"); + m.variables = gpf_count_label(store, lp.project, "Variable"); m.imports = cbm_store_count_edges_by_type(store, lp.project, "IMPORTS"); m.depends_on = cbm_store_count_edges_by_type(store, lp.project, "DEPENDS_ON"); m.infra_maps = cbm_store_count_edges_by_type(store, lp.project, "INFRA_MAPS"); @@ -743,6 +745,38 @@ TEST(probe_kustomize_extensionless_file) { PASS(); } +/* A K8s object's top-level keys (apiVersion, kind, metadata, spec, status) are + * its envelope, carried by its Resource node; as Variables they repeated per + * manifest (deckhouse: `kind` 1,358, `apiVersion` 1,347, `metadata` 1,336) + * and fed name-similarity config links. */ +TEST(probe_k8s_manifest_mints_no_envelope_variables) { + GpfMetrics m = gpf_metrics("deploy.yaml", + "# a manifest\n" + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " replicas: 2\n" + "status: {}\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 1); + ASSERT_EQ(m.variables, 0); + PASS(); +} + +/* A plain config YAML keeps its top-level keys as Variables. */ +TEST(probe_yaml_config_keeps_variables) { + GpfMetrics m = gpf_metrics("config.yaml", + "server:\n" + " port: 8080\n" + "kind: production\n" + "replicas: 3\n"); + ASSERT_TRUE(m.ok); + ASSERT_TRUE(m.variables >= 3); + PASS(); +} + /* A `file://` dependency is the chart in that directory of this repository, * not a published one: `app` depends on the local `common`, and no external * Chart node stands in for it. `common` is read after `app`. */ @@ -2000,6 +2034,8 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_kustomize_extensionless_file); RUN_TEST(probe_kustomize_remote_base_names_no_symbol); RUN_TEST(probe_helm_file_dependency_links_local_chart); + RUN_TEST(probe_k8s_manifest_mints_no_envelope_variables); + RUN_TEST(probe_yaml_config_keeps_variables); RUN_TEST(probe_helm_unresolved_file_dependency_stays_external); RUN_TEST(probe_kustomize_nested_sibling_entries); RUN_TEST(probe_kustomize_single_module); From 0dac32ad0d0f51bf54145c4cb11390eba096c583 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 13:37:44 +0200 Subject: [PATCH 08/52] fix(routes): only an exposed-endpoint key turns a config URL into a Route A URL in a config file became an infra Route -- "an endpoint this service exposes" -- unless its key was on a short deny list (jwks, registry, upstream, ...). In 17 infra repositories almost every such Route was a reference instead: changelog `pull_request` links (deckhouse 6,240), docs `link`s, alert `runbook_url`s, chart `home`/`source`, kubeconfig `server`s, Argo `repoURL`s, Helm repository `url`s. Until the previous commit one root handler took a HANDLES edge to each of them. The rule is now an allow list: the key's leaf, compared ignoring case, `_` and `-`, must name an endpoint other systems call on this service -- push_endpoint, post_url, callback(_url), webhook(_url) -- the keys the deny list's own comment named as the ones to keep. On real input, infra Routes: deckhouse 6,380 -> 3 (three `webhook` keys), kubermatic 228 -> 0, prometheus-community/helm-charts 124 -> 0, gardener 26 -> 0, kubernetes/k8s.io 48 -> 0, onedr0p/home-ops 100 -> 0. HANDLES and HTTP_CALLS are unchanged in all six. A new contract test with those reference keys fails on the deny list; another pins the camelCase and _url endpoint spellings; the #999 guard (push_endpoint still mints) passes unchanged. Signed-off-by: Martin Vogel --- src/pipeline/pipeline.c | 45 +++++++++++++++++++++------------- tests/test_lang_contract.c | 50 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 179a5c981..36846c819 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1640,21 +1640,32 @@ static bool is_ci_tooling_config(const char *fp) { return false; } -/* True when a YAML key path denotes an UPSTREAM dependency, CONFIG value, or - * HEALTHCHECK target rather than an endpoint this service exposes. Such URLs - * (auth JWKS, downstream service base URLs, package-registry URLs, healthcheck - * curl targets) are NOT routes the service serves and must not mint Route nodes - * (#521). Exposed-endpoint keys (push_endpoint, post_url, callback, webhook) - * are intentionally absent here so they still produce infra Route nodes. */ -static bool is_upstream_config_key(const char *key_path) { +/* True when a config key names an endpoint THIS service exposes -- a URL other + * systems call on it: a push subscription's endpoint, a post/callback/webhook + * URL. Only such keys mint infra Route nodes. Every other config URL is a + * reference: an upstream dependency, a registry or repository, a docs or + * runbook link, a changelog's pull-request link, a kubeconfig server. A deny + * list of those (#521) could not keep up: in 17 infra repositories almost + * every infra Route came from such a key (deckhouse: 6,240 `pull_request` + * links). The leaf key decides, compared ignoring case, `_` and `-`. */ +static bool is_exposed_endpoint_key(const char *key_path) { if (!key_path) { - /* No key context (e.g. flat string) — keep prior behaviour and mint. */ return false; } - static const char *const deny[] = {"jwks", "registry", "registries", "healthcheck", - "upstream", "_service_url", "auth", NULL}; - for (int i = 0; deny[i]; i++) { - if (strstr(key_path, deny[i]) != NULL) { + const char *leaf = strrchr(key_path, '.'); + leaf = leaf ? leaf + SKIP_ONE : key_path; + char norm[CBM_SZ_64]; + size_t n = 0; + for (const char *p = leaf; *p && n + 1 < sizeof(norm); p++) { + if (*p != '_' && *p != '-') { + norm[n++] = (char)tolower((unsigned char)*p); + } + } + norm[n] = '\0'; + static const char *const endpoints[] = {"pushendpoint", "posturl", "callback", "callbackurl", + "webhook", "webhookurl", NULL}; + for (int i = 0; endpoints[i]; i++) { + if (strcmp(norm, endpoints[i]) == 0) { return true; } } @@ -1666,8 +1677,8 @@ static void try_upsert_infra_route(cbm_gbuf_t *gbuf, const CBMStringRef *sr, con if (sr->kind != CBM_STRREF_URL || !sr->value || !strstr(sr->value, "://")) { return; } - /* Skip upstream/config/healthcheck URLs — they are not exposed routes (#521). */ - if (is_upstream_config_key(sr->key_path)) { + /* Only an endpoint this service exposes is a route it serves (#521). */ + if (!is_exposed_endpoint_key(sr->key_path)) { return; } char route_qn[CBM_ROUTE_QN_SIZE]; @@ -1685,8 +1696,8 @@ static void try_upsert_infra_route(cbm_gbuf_t *gbuf, const CBMStringRef *sr, con /* A URL string_ref that does NOT denote a route the service serves: a value * containing whitespace is a command/sentence with an embedded URL (e.g. a * Docker healthcheck `curl --fail http://... || exit 1`); a NULL key_path is a - * context-less/duplicate ref; an upstream/config/healthcheck key is an external - * dependency, not an exposed route. (#521) */ + * context-less/duplicate ref; any key but an exposed-endpoint key is a + * reference, not an exposed route. (#521) */ static bool route_sr_denied(const CBMStringRef *sr) { if (!sr->value || strchr(sr->value, ' ')) { return true; @@ -1694,7 +1705,7 @@ static bool route_sr_denied(const CBMStringRef *sr) { if (!sr->key_path) { return true; } - return is_upstream_config_key(sr->key_path); + return !is_exposed_endpoint_key(sr->key_path); } static void cbm_pipeline_extract_infra_routes(const cbm_pipeline_ctx_t *ctx, cbm_gbuf_t *gbuf, diff --git a/tests/test_lang_contract.c b/tests/test_lang_contract.c index 6b3824df6..73d8c5575 100644 --- a/tests/test_lang_contract.c +++ b/tests/test_lang_contract.c @@ -1553,6 +1553,54 @@ TEST(contract_edge_infra_routes_from_deploy_configs_still_minted) { PASS(); } +/* A config URL becomes an infra Route only under a key that names an endpoint + * this service exposes. Reference links are not: across 17 infra repositories + * the infra Routes were changelog `pull_request` links (deckhouse 6,240), + * alert `runbook_url`s, chart `home`/`source`, kubeconfig `server`s, Argo + * `repoURL`s and docs `link`s -- and one root handler took all of them. */ +TEST(contract_edge_no_infra_routes_from_reference_links) { + LangProj lp; + static const LangFile f[] = { + {"CHANGELOG/v1.29.yml", "candi:\n" + " fixes:\n" + " - summary: fix a race\n" + " pull_request: https://github.com/org/repo/pull/1234\n"}, + {"modules/cni/oss.yaml", "- name: Cilium\n" + " link: https://github.com/cilium/cilium\n"}, + {"charts/app/alerts.yaml", "groups:\n" + " - name: app\n" + " rules:\n" + " - alert: Down\n" + " annotations:\n" + " runbook_url: https://runbooks.example.io/down\n"}, + {"apps/app.yaml", "spec:\n" + " source:\n" + " repoURL: https://charts.example.io/stable\n" + " home: https://app.example.io\n"}}; + cbm_store_t *store = lang_index_files(&lp, f, 4); + ASSERT_NOT_NULL(store); + ASSERT_EQ(count_infra_routes_matching(store, lp.project, "https://"), 0); + lang_cleanup(&lp, store); + PASS(); +} + +/* The endpoint keys keep minting in their camelCase / _url spellings. */ +TEST(contract_edge_infra_routes_from_endpoint_key_spellings) { + LangProj lp; + static const LangFile f[] = { + {"pubsub.yaml", "subscriptions:\n" + " - name: orders\n" + " pushEndpoint: https://orders.internal.example/push\n"}, + {"hooks.yaml", "payments:\n" + " callbackUrl: https://pay.internal.example/callback\n"}}; + cbm_store_t *store = lang_index_files(&lp, f, 2); + ASSERT_NOT_NULL(store); + ASSERT_TRUE(count_infra_routes_matching(store, lp.project, "orders.internal.example") >= 1); + ASSERT_TRUE(count_infra_routes_matching(store, lp.project, "pay.internal.example") >= 1); + lang_cleanup(&lp, store); + PASS(); +} + /* True if some CALLS edge's TARGET node carries `label` and a QN ending with * `qn_suffix` — i.e. the call resolved to that specific definition. */ static int calls_edge_targets(cbm_store_t *store, const char *project, const char *label, @@ -1897,6 +1945,8 @@ SUITE(lang_contract) { RUN_TEST(contract_edge_python_aliased_import_call_resolves_issue988); RUN_TEST(contract_edge_no_infra_routes_from_ci_configs_issue999); RUN_TEST(contract_edge_infra_routes_from_deploy_configs_still_minted); + RUN_TEST(contract_edge_no_infra_routes_from_reference_links); + RUN_TEST(contract_edge_infra_routes_from_endpoint_key_spellings); RUN_TEST(contract_edge_commonjs_require_call_resolves_issue871); RUN_TEST(contract_edge_depends_on); RUN_TEST(contract_edge_parallel_service_edges); From 75e317518c682d1dbcd3fbab7b47cb669dc619ad Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 13:53:54 +0200 Subject: [PATCH 09/52] fix(hcl): index .tofu and .tfvars; only top-level blocks are Classes - OpenTofu configuration (`.tofu`) and variable files (`.tfvars`) had no language and were never indexed (cloud-foundation-fabric: 104 `.tofu`, 208 `.tfvars`). - Every HCL `block` became a Class, nested ones included: a resource's `lifecycle`, `dynamic`/`content` and argument blocks were entities next to the resource (cloud-foundation-fabric: `dynamic` 1,342, `content` 205 beside 898 resources). Only a block directly in the file's body -- resource, data, module, variable, output, locals, provider, terraform, Terragrunt's include/dependency -- is a Class now. - A file's own assignments (a `.tfvars` file, Terragrunt `inputs`) sit one level below the root the variable walk scanned; none was extracted. They are Variables now. On real input (HCL Classes / .tofu+.tfvars files / HCL Variables), against main: cloud-foundation-fabric 6,324 -> 3,919 / 0 -> 312 / 0 -> 1,067, kubernetes/k8s.io 2,102 -> 1,590 / 0 -> 11 / 0 -> 42, kubespray 892 -> 804 / 0 -> 13 / 0 -> 174, eks-blueprints 924 -> 733, kube-hetzner 757 -> 683, deckhouse 976 -> 830; the terragrunt example keeps its 31 top-level blocks and gains 5 input Variables. Not changed here: the remaining HCL USAGE edges are generic name guesses (a `range` call resolves to a Class in , module outputs to same-named outputs of unrelated modules), and the name-similarity config links now also see the 104 OpenTofu files. The nested-block, .tofu and .tfvars probes and the extension unit test fail without the change. Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 35 ++++++++++++++++++++++++- src/discover/language.c | 4 ++- tests/test_grammar_probe_f.c | 50 ++++++++++++++++++++++++++++++++++++ tests/test_language.c | 9 +++++++ 4 files changed, 96 insertions(+), 2 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index ddf165d8c..feb64086c 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -6862,6 +6862,19 @@ static char *find_ini_section_name(CBMArena *a, TSNode node, const char *source) return NULL; } +// HCL: a top-level block (resource, data, module, variable, output, locals, +// provider, terraform; Terragrunt's include, dependency, ...) sits directly in +// the file's body. Blocks nested in one (lifecycle, dynamic, content, a +// resource's own argument blocks) are part of it, not entities of their own. +static bool hcl_block_is_top_level(TSNode node) { + TSNode body = ts_node_parent(node); + if (ts_node_is_null(body) || strcmp(ts_node_type(body), "body") != 0) { + return false; + } + TSNode file = ts_node_parent(body); + return !ts_node_is_null(file) && strcmp(ts_node_type(file), "config_file") == 0; +} + // HCL: extract block name from identifier child. static char *find_hcl_block_name(CBMArena *a, TSNode node, const char *source) { TSNode id = cbm_find_child_by_kind(node, "identifier"); @@ -6923,7 +6936,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char // #518: index what the section SAYS, not just its title. docstring = extract_markdown_section_body(a, node, ctx->source); } else if (ctx->language == CBM_LANG_HCL && strcmp(kind, "block") == 0) { - name = find_hcl_block_name(a, node, ctx->source); + name = hcl_block_is_top_level(node) ? find_hcl_block_name(a, node, ctx->source) : NULL; } else { return false; } @@ -10011,6 +10024,26 @@ static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec return; } + /* HCL: a file's own assignments -- a .tfvars file, Terragrunt's `inputs` + * -- sit in the top-level body, one level below the root the generic loop + * scans, so none was extracted. Attributes inside blocks belong to their + * block. */ + if (ctx->language == CBM_LANG_HCL) { + TSNode body = cbm_find_child_by_kind(root, "body"); + uint32_t n = ts_node_is_null(body) ? 0 : ts_node_named_child_count(body); + for (uint32_t i = 0; i < n; i++) { + TSNode attr = ts_node_named_child(body, i); + if (strcmp(ts_node_type(attr), "attribute") != 0) { + continue; + } + TSNode id = cbm_find_child_by_kind(attr, "identifier"); + if (!ts_node_is_null(id)) { + push_var_def(ctx, cbm_node_text(ctx->arena, id, ctx->source), attr); + } + } + return; + } + /* Nix: the file's top level sits behind its header lambda(s), so the root's * only child is a function_expression and the generic loop below would see * nothing. Resolve past the header to the binding container(s) that actually diff --git a/src/discover/language.c b/src/discover/language.c index f7fff9544..7b66f717a 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -152,9 +152,11 @@ static const ext_entry_t EXT_TABLE[] = { /* Haskell */ {".hs", CBM_LANG_HASKELL}, - /* HCL / Terraform */ + /* HCL / Terraform / OpenTofu */ {".hcl", CBM_LANG_HCL}, {".tf", CBM_LANG_HCL}, + {".tofu", CBM_LANG_HCL}, + {".tfvars", CBM_LANG_HCL}, /* HTML */ {".htm", CBM_LANG_HTML}, diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 8daf6d9c9..5995b8f7e 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -276,6 +276,53 @@ TEST(probe_hcl_no_spurious_imports) { PASS(); } +/* Only a top-level block (resource, data, module, variable, output, ...) is an + * entity; blocks nested in it (lifecycle, dynamic, content, the resource's own + * argument blocks) are part of it. cloud-foundation-fabric minted 1,342 + * `dynamic` and 205 `content` Classes next to 898 resources. */ +TEST(probe_hcl_nested_blocks_are_not_classes) { + GpfMetrics m = gpf_metrics("main.tf", + "resource \"google_container_cluster\" \"main\" {\n" + " name = \"main\"\n" + " addons_config {\n" + " http_load_balancing { disabled = false }\n" + " }\n" + " dynamic \"node_pool\" {\n" + " for_each = var.pools\n" + " content {\n" + " name = node_pool.value.name\n" + " }\n" + " }\n" + " lifecycle {\n" + " ignore_changes = [node_pool]\n" + " }\n" + "}\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.classes, 1); + PASS(); +} + +/* OpenTofu configuration (.tofu) is indexed like .tf. */ +TEST(probe_tofu_file_indexed) { + GpfMetrics m = gpf_metrics("main.tofu", + "resource \"aws_s3_bucket\" \"logs\" {\n" + " bucket = \"logs\"\n" + "}\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.classes, 1); + PASS(); +} + +/* A variable file's assignments are its content. */ +TEST(probe_tfvars_file_indexed) { + GpfMetrics m = gpf_metrics("prod.tfvars", + "region = \"europe-west1\"\n" + "node_count = 3\n"); + ASSERT_TRUE(m.ok); + ASSERT_TRUE(m.variables >= 2); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * GROUP 2 — Kubernetes manifests (.yaml with apiVersion:) * @@ -2011,6 +2058,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_hcl_module_block_node); RUN_TEST(probe_hcl_infra_maps_edge); RUN_TEST(probe_hcl_no_spurious_imports); + RUN_TEST(probe_hcl_nested_blocks_are_not_classes); + RUN_TEST(probe_tofu_file_indexed); + RUN_TEST(probe_tfvars_file_indexed); /* Kubernetes manifests */ RUN_TEST(probe_k8s_pod_resource_node); diff --git a/tests/test_language.c b/tests/test_language.c index 286733c7f..26c6f05c0 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -386,6 +386,14 @@ TEST(lang_ext_hcl2) { ASSERT_EQ(cbm_language_for_extension(".hcl"), CBM_LANG_HCL); PASS(); } +/* OpenTofu's own configuration files and Terraform/OpenTofu variable files + * are HCL too; neither was indexed (cloud-foundation-fabric: 104 .tofu, + * 208 .tfvars). */ +TEST(lang_ext_tofu_tfvars) { + ASSERT_EQ(cbm_language_for_extension(".tofu"), CBM_LANG_HCL); + ASSERT_EQ(cbm_language_for_extension(".tfvars"), CBM_LANG_HCL); + PASS(); +} TEST(lang_ext_sql) { ASSERT_EQ(cbm_language_for_extension(".sql"), CBM_LANG_SQL); PASS(); @@ -1588,6 +1596,7 @@ SUITE(language) { RUN_TEST(lang_ext_toml); RUN_TEST(lang_ext_hcl); RUN_TEST(lang_ext_hcl2); + RUN_TEST(lang_ext_tofu_tfvars); RUN_TEST(lang_ext_sql); RUN_TEST(lang_ext_dockerfile); RUN_TEST(lang_ext_json); From 0a65d1ec80bd5ab9814437ff3188ea881aa97a34 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 15:12:50 +0200 Subject: [PATCH 10/52] fix(configlink): exact, component-scoped key links for infra files; no caps key_symbol linked a config key to every code symbol whose normalized name equalled or merely CONTAINED it, anywhere in the repository, over at most 4,096 config keys x 8,192 code symbols -- the first ones of a QN-sorted list, so which links existed depended on what else the repository held. In 17 infrastructure repositories 4 of 40 sampled edges were right, all exact and inside one directory tree: a stage's JSON schema "configured" unrelated module variables, a chart's `apiVersion` every `*.apiVersion` helper, `config_path` every Go `*ConfigPath`. Per the decision for infrastructure files (YAML, JSON, HCL, tfvars): - a pair with either side in such a file links by exact name only, and only when the code file lies in the config file's component -- its directory tree, or the tree around a conventional config folder (config/, conf/, settings/, ...); - an HCL symbol takes part only as a Terraform/OpenTofu `variable`, by its declared name (`variable.` matches ``); outputs, modules, resources and .tfvars assignments are not what a config key sets. Other config formats keep exact + substring matching anywhere. No count decides content any more: the collectors keep every candidate, and an exact-name index plus a trigram index (a key's substring candidates are the symbols holding its rarest trigram) replace the all-pairs scan, so the cost follows the matches. On real input, key_symbol edges against main: cloud-foundation-fabric 9,892 -> 1, prometheus-community/helm-charts 4,148 -> 0, deckhouse 2,631 -> 49, gardener 1,677 -> 0, kubermatic 739 -> 7, microservices-demo 376 -> 0, kubespray 156 -> 6, k8s.io 218 -> 0, opentelemetry-demo 71 -> 26. Every infra edge left was reviewed: kubespray's gcp tfvars.json sets its module's variables (6/6), a project template's YAML its variables, flags.json its generated flags, a values.yaml key its Makefile variable; one miss -- a repository-root config sees the whole repository. The rest are TOML/INI/Cargo substring matches, which this change leaves as they were. Tests: infra exact-only, component scope, config-folder scope, the Terraform variable rule, and no config/code cap (4,200 keys; 9,000 candidates) fail on the old matcher. camel_case now matches a JSON key exactly (its substring form is what infra files no longer do); the order-independence test also asserts that all 9,000 candidates link. Signed-off-by: Martin Vogel --- src/pipeline/pass_configlink.c | 510 ++++++++++++++++++++++----------- tests/test_configlink.c | 148 +++++++++- 2 files changed, 479 insertions(+), 179 deletions(-) diff --git a/src/pipeline/pass_configlink.c b/src/pipeline/pass_configlink.c index 1ee3ecf4b..5c1b41654 100644 --- a/src/pipeline/pass_configlink.c +++ b/src/pipeline/pass_configlink.c @@ -12,6 +12,7 @@ #include "pipeline/pipeline_internal.h" #include "graph_buffer/graph_buffer.h" #include "foundation/hash_table.h" +#include "foundation/arena.h" #include "foundation/log.h" #include "foundation/compat.h" @@ -63,18 +64,17 @@ static bool is_dep_section(const char *s) { /* ── Strategy 1: Config Key → Code Symbol ───────────────────────── */ -/* Canonical candidate order (determinism). Both collectors below fill a - * fixed-capacity array and stop at max_out; the label indexes they walk are - * gbuf insertion order = parallel-extraction merge order, which varies run to - * run. On a repo with more candidates than the cap, sorting by a pure content - * key first is what keeps the surviving set — and therefore the emitted - * CONFIGURES edges — a function of the inputs rather than of worker - * scheduling. Tie-breaks stay content-only: node ids are handed out in merge - * order, so an id tie-break belongs in no canonical comparator. */ +/* Canonical candidate order (determinism). The label indexes the collectors + * walk are gbuf insertion order = parallel-extraction merge order, which + * varies run to run. Sorting by a pure content key first keeps the candidate + * order -- and so the order edges are inserted in -- a function of the inputs + * rather than of worker scheduling. (The collectors once stopped at a cap, + * where this order also decided WHICH candidates survived; no cap cuts them + * now.) Tie-breaks stay content-only: node ids are handed out in merge order, + * so an id tie-break belongs in no canonical comparator. */ enum { - CANON_CMP_LESS = -1, /* qsort: left sorts before right */ - CANON_CMP_GREATER = 1, /* qsort: left sorts after right */ - CANON_CAP_BUF = 32 /* decimal rendering of a cap value */ + CANON_CMP_LESS = -1, /* qsort: left sorts before right */ + CANON_CMP_GREATER = 1 /* qsort: left sorts after right */ }; static int cmp_node_ptr_canonical(const void *pa, const void *pb) { @@ -100,14 +100,6 @@ static int cmp_node_ptr_canonical(const void *pa, const void *pb) { return strcmp(na, nb); } -/* A filled-to-capacity collector dropped candidates; say so rather than - * truncating silently. */ -static void log_candidate_truncation(const char *side, int cap) { - char cap_buf[CANON_CAP_BUF]; - snprintf(cap_buf, sizeof(cap_buf), "%d", cap); - cbm_log_info("configlinker.truncated", "side", side, "cap", cap_buf); -} - /* Heap copy of `nodes` sorted by cmp_node_ptr_canonical. Returns NULL (and * leaves the caller on the unsorted borrowed array) only on allocation * failure, which degrades determinism but never correctness. */ @@ -124,21 +116,105 @@ static const cbm_gbuf_node_t **canonical_node_copy(const cbm_gbuf_node_t *const return sorted; } +/* Infrastructure files: YAML, JSON and HCL (Terraform, OpenTofu, tfvars). A + * key_symbol pair with either side in one links by exact name and only inside + * the config file's component (user decision 2026-10-10): in 17 infra + * repositories 4 of 40 sampled key_symbol edges were right, all of them exact + * and within one directory tree; the rest were substring and cross-component + * coincidences. Other config formats keep exact + substring matching. */ +static bool cl_is_infra_path(const char *path) { + static const char *const exts[] = {".yaml", ".yml", ".json", ".tf", + ".hcl", ".tofu", ".tfvars", NULL}; + const char *dot = path ? strrchr(path, '.') : NULL; + const char *slash = path ? strrchr(path, '/') : NULL; + if (!dot || (slash && dot < slash)) { + return false; + } + for (int i = 0; exts[i]; i++) { + if (strcmp(dot, exts[i]) == 0) { + return true; + } + } + return false; +} + +/* Length of the config file's component directory: its own directory, or the + * one around it when that is a conventional config folder; 0 = the repo root. */ +static size_t cl_component_len(const char *path) { + static const char *const config_dirs[] = {"config", "configs", "conf", "settings", "etc", NULL}; + const char *slash = path ? strrchr(path, '/') : NULL; + if (!slash) { + return 0; + } + size_t dir_len = (size_t)(slash - path); + const char *base = path; + for (const char *p = path; p < slash; p++) { + if (*p == '/') { + base = p + SKIP_ONE; + } + } + size_t base_len = (size_t)(slash - base); + for (int i = 0; config_dirs[i]; i++) { + if (base_len == strlen(config_dirs[i]) && strncmp(base, config_dirs[i], base_len) == 0) { + return base == path ? 0 : (size_t)(base - path) - SKIP_ONE; + } + } + return dir_len; +} + typedef struct { int64_t node_id; - char normalized[CBM_SZ_256]; - char name[CBM_SZ_256]; -} config_entry_t; + const char *file_path; /* borrowed from the graph buffer for the pass */ + const char *name; /* borrowed */ + char *normalized; /* in the strategy's arena */ + size_t len; + size_t component_len; /* config entries: cl_component_len(file_path) */ + bool infra; +} cl_entry_t; + +typedef struct { + cl_entry_t *items; + int count; + int cap; +} cl_entries_t; + +static bool cl_entries_push(cl_entries_t *entries, const cbm_gbuf_node_t *node, const char *norm, + CBMArena *arena) { + if (entries->count == entries->cap) { + int cap = entries->cap ? entries->cap * PAIR_LEN : CBM_SZ_1K; + cl_entry_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, entries->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + entries->items = grown; + entries->cap = cap; + } + char *normalized = cbm_arena_strdup(arena, norm); + if (!normalized) { + return false; + } + entries->items[entries->count++] = (cl_entry_t){ + .node_id = node->id, + .file_path = node->file_path ? node->file_path : "", + .name = node->name ? node->name : "", + .normalized = normalized, + .len = strlen(normalized), + .component_len = cl_component_len(node->file_path), + .infra = cl_is_infra_path(node->file_path), + }; + return true; +} /* Collect config Variable nodes with ≥2 tokens, each ≥3 chars. */ -static int collect_config_entries(const cbm_gbuf_node_t *const *vars, int var_count, - config_entry_t *out, int max_out) { - int n = 0; +static bool collect_config_entries(const cbm_gbuf_node_t *const *vars, int var_count, + cl_entries_t *out, CBMArena *arena) { const cbm_gbuf_node_t **sorted = canonical_node_copy(vars, var_count); if (sorted) { vars = sorted; } - for (int i = 0; i < var_count && n < max_out; i++) { + bool ok = true; + for (int i = 0; i < var_count && ok; i++) { if (!cbm_has_config_extension(vars[i]->file_path)) { continue; } @@ -161,78 +237,168 @@ static int collect_config_entries(const cbm_gbuf_node_t *const *vars, int var_co } p = end ? end + SKIP_ONE : p + tlen; } - if (!all_long) { - continue; + if (all_long) { + ok = cl_entries_push(out, vars[i], norm, arena); } - - out[n].node_id = vars[i]->id; - snprintf(out[n].normalized, sizeof(out[n].normalized), "%s", norm); - snprintf(out[n].name, sizeof(out[n].name), "%s", vars[i]->name); - n++; - } - if (n == max_out) { - log_candidate_truncation("config", max_out); } free((void *)sorted); - return n; + return ok; +} + +static bool cl_is_hcl_path(const char *path) { + static const char *const exts[] = {".tf", ".hcl", ".tofu", ".tfvars", NULL}; + const char *dot = path ? strrchr(path, '.') : NULL; + for (int i = 0; dot && exts[i]; i++) { + if (strcmp(dot, exts[i]) == 0) { + return true; + } + } + return false; +} + +/* The name an HCL symbol matches a config key by. A config value sets a + * Terraform/OpenTofu `variable` block, whose Class is named `variable.`: + * it matches by . Outputs, modules, resources and a .tfvars file's own + * assignments are not what a config key sets; they take no part (NULL). */ +static const char *cl_hcl_variable_name(const cbm_gbuf_node_t *node) { + static const char prefix[] = "variable."; + if (!node->label || strcmp(node->label, "Class") != 0 || !node->name || + strncmp(node->name, prefix, sizeof(prefix) - SKIP_ONE) != 0) { + return NULL; + } + return node->name + sizeof(prefix) - SKIP_ONE; } /* Collect code nodes (Function/Variable/Class/Struct) not from config files. */ -typedef struct { - int64_t node_id; - char normalized[CBM_SZ_256]; -} code_entry_t; - -static int collect_code_entries(cbm_gbuf_t *gb, code_entry_t *out, int max_out) { - int n = 0; +static bool collect_code_entries(cbm_gbuf_t *gb, cl_entries_t *out, CBMArena *arena) { /* "Struct" alongside "Class": a config key may name a Go/Rust/Swift/D struct * type, which is now labelled "Struct" — keep it linkable. */ static const char *labels[] = {"Function", "Variable", "Class", "Struct", NULL}; - for (int li = 0; labels[li] && n < max_out; li++) { + bool ok = true; + for (int li = 0; labels[li] && ok; li++) { const cbm_gbuf_node_t **nodes = NULL; int count = 0; if (cbm_gbuf_find_by_label(gb, labels[li], &nodes, &count) != 0) { continue; } - /* Canonical order before the cap — see cmp_node_ptr_canonical. The cap - * spans the whole label list, so a later label can be cut mid-group; - * sorting per group keeps that cut a pure function of content. */ + /* Canonical order -- see cmp_node_ptr_canonical. */ const cbm_gbuf_node_t **sorted = canonical_node_copy(nodes, count); const cbm_gbuf_node_t *const *scan = sorted ? sorted : nodes; - for (int i = 0; i < count && n < max_out; i++) { + for (int i = 0; i < count && ok; i++) { if (cbm_has_config_extension(scan[i]->file_path)) { continue; } + const char *match_name = scan[i]->name; + if (cl_is_hcl_path(scan[i]->file_path)) { + match_name = cl_hcl_variable_name(scan[i]); + if (!match_name) { + continue; + } + } + char norm[CBM_SZ_256]; - int tokens = cbm_normalize_config_key(scan[i]->name, norm, sizeof(norm)); + int tokens = cbm_normalize_config_key(match_name, norm, sizeof(norm)); if (tokens == 0 || norm[0] == '\0') { continue; } - - out[n].node_id = scan[i]->id; - snprintf(out[n].normalized, sizeof(out[n].normalized), "%s", norm); - n++; + ok = cl_entries_push(out, scan[i], norm, arena); } /* gbuf data is borrowed — only the sorted copy is owned */ free((void *)sorted); } - if (n == max_out) { - log_candidate_truncation("code", max_out); - } - return n; + return ok; } -/* The key x symbol comparison of strategy 1, per config key: every pair's - * verdict depends only on its two strings, so rows run in parallel and the - * edges are inserted afterwards in the original (config, code) order. One - * thread compared up to 4,096 x 8,192 pairs -- configlink's 0.93 s on the Go - * corpus (profile, 2026-09-17). A pair whose lengths rule both verdicts out - * skips the string calls: an exact match needs equal lengths, a substring one - * no longer than the code key. */ +/* Code entries (indices, ascending) under one index key. */ +typedef struct { + int *items; + int count; + int cap; +} cl_posting_t; + +static bool cl_posting_add(CBMHashTable *index, const char *key, int idx) { + cl_posting_t *posting = cbm_ht_get(index, key); + if (!posting) { + posting = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*posting)); + if (!posting) { + return false; + } + cbm_ht_set(index, key, posting); + } + if (posting->count && posting->items[posting->count - 1] == idx) { + return true; /* the same trigram twice in one name */ + } + if (posting->count == posting->cap) { + int cap = posting->cap ? posting->cap * PAIR_LEN : CBM_SZ_16; + int *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, posting->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + posting->items = grown; + posting->cap = cap; + } + posting->items[posting->count++] = idx; + return true; +} + +static void cl_posting_free(const char *key, void *value, void *userdata) { + (void)key; + (void)userdata; + cl_posting_t *posting = value; + cbm_free(CBM_MEM_CLASS_OTHER, posting->items); + cbm_free(CBM_MEM_CLASS_OTHER, posting); +} + +enum { CL_TRIGRAM = 3 }; + +/* Exact index: normalized name -> code entries. Trigram index: every + * 3-character run of a non-infra code entry's normalized name -> entries, so a + * key's substring candidates are the entries holding its rarest trigram + * instead of every code entry (cost follows the matches, no cap needed). */ +static bool cl_build_indexes(const cl_entries_t *code, CBMHashTable *exact, CBMHashTable *trigrams, + CBMArena *arena) { + for (int k = 0; k < code->count; k++) { + const cl_entry_t *e = &code->items[k]; + if (!cl_posting_add(exact, e->normalized, k)) { + return false; + } + if (e->infra) { + continue; /* infra symbols link by exact name only */ + } + for (size_t i = 0; i + CL_TRIGRAM <= e->len; i++) { + char tri[CL_TRIGRAM + SKIP_ONE]; + memcpy(tri, e->normalized + i, CL_TRIGRAM); + tri[CL_TRIGRAM] = '\0'; + const char *key = cbm_ht_get_key(trigrams, tri); + if (!key) { + key = cbm_arena_strdup(arena, tri); + if (!key) { + return false; + } + } + if (!cl_posting_add(trigrams, key, k)) { + return false; + } + } + } + return true; +} + +/* A config key's component holds the code file. */ +static bool cl_in_component(const cl_entry_t *config, const cl_entry_t *code) { + size_t n = config->component_len; + return n == 0 || + (strncmp(code->file_path, config->file_path, n) == 0 && code->file_path[n] == '/'); +} + +/* The key x symbol matches of strategy 1, per config key: exact matches from + * the exact index, substring matches among the entries holding the key's + * rarest trigram. Rows run in parallel; each row's matches are in ascending + * code order, and the edges are inserted afterwards in (config, code) order. */ typedef struct { int co; double confidence; @@ -245,29 +411,78 @@ typedef struct { } key_match_list_t; typedef struct { - const config_entry_t *config; - const size_t *config_len; - int config_count; - const code_entry_t *code; - const size_t *code_len; - int code_count; + const cl_entries_t *config; + const cl_entries_t *code; + const CBMHashTable *exact; + const CBMHashTable *trigrams; key_match_list_t *lists; _Atomic int next; _Atomic bool failed; } key_match_job_t; -static double key_pair_confidence(const char *config_norm, size_t config_len, const char *code_norm, - size_t code_len) { - if (config_len > code_len) { - return 0.0; +static bool key_match_push(key_match_list_t *list, int co, double confidence) { + if (list->count == list->cap) { + int cap = list->cap ? list->cap * PAIR_LEN : CBM_SZ_16; + key_match_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, list->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + list->items = grown; + list->cap = cap; } - if (config_len == code_len && memcmp(config_norm, code_norm, code_len) == 0) { - return CONF_KEY_EXACT; /* Exact match */ + list->items[list->count].co = co; + list->items[list->count].confidence = confidence; + list->count++; + return true; +} + +/* The trigram posting of `key` with the fewest entries; NULL when some + * trigram occurs in no entry (then no entry holds the key). */ +static const cl_posting_t *cl_rarest_trigram(const CBMHashTable *trigrams, const char *key, + size_t len) { + const cl_posting_t *rarest = NULL; + for (size_t i = 0; i + CL_TRIGRAM <= len; i++) { + char tri[CL_TRIGRAM + SKIP_ONE]; + memcpy(tri, key + i, CL_TRIGRAM); + tri[CL_TRIGRAM] = '\0'; + const cl_posting_t *posting = cbm_ht_get(trigrams, tri); + if (!posting) { + return NULL; + } + if (!rarest || posting->count < rarest->count) { + rarest = posting; + } } - if (strstr(code_norm, config_norm) != NULL) { - return CONF_KEY_SUBSTRING; /* Substring match */ + return rarest; +} + +static bool key_match_row(const key_match_job_t *job, int ci, key_match_list_t *list) { + const cl_entry_t *cfg = &job->config->items[ci]; + const cl_posting_t *exact = cbm_ht_get(job->exact, cfg->normalized); + const cl_posting_t *subs = + cfg->infra ? NULL : cl_rarest_trigram(job->trigrams, cfg->normalized, cfg->len); + int ei = 0; + int si = 0; + int ecount = exact ? exact->count : 0; + int scount = subs ? subs->count : 0; + /* Merge both ascending candidate lists so the row stays in code order. */ + while (ei < ecount || si < scount) { + bool take_exact = si >= scount || (ei < ecount && exact->items[ei] <= subs->items[si]); + int co = take_exact ? exact->items[ei++] : subs->items[si++]; + const cl_entry_t *code = &job->code->items[co]; + double confidence = 0.0; + if (take_exact) { + bool infra = cfg->infra || code->infra; + confidence = (!infra || cl_in_component(cfg, code)) ? CONF_KEY_EXACT : 0.0; + } else if (code->len > cfg->len && strstr(code->normalized, cfg->normalized)) { + confidence = CONF_KEY_SUBSTRING; + } + if (confidence > 0.0 && !key_match_push(list, co, confidence)) { + return false; + } } - return 0.0; + return true; } static void key_match_worker(int worker_id, void *arg) { @@ -275,30 +490,12 @@ static void key_match_worker(int worker_id, void *arg) { key_match_job_t *job = (key_match_job_t *)arg; while (!atomic_load_explicit(&job->failed, memory_order_relaxed)) { int ci = atomic_fetch_add_explicit(&job->next, SKIP_ONE, memory_order_relaxed); - if (ci >= job->config_count) { + if (ci >= job->config->count) { break; } - key_match_list_t *list = &job->lists[ci]; - for (int co = 0; co < job->code_count; co++) { - double confidence = key_pair_confidence(job->config[ci].normalized, job->config_len[ci], - job->code[co].normalized, job->code_len[co]); - if (confidence <= 0.0) { - continue; - } - if (list->count == list->cap) { - int cap = list->cap ? list->cap * PAIR_LEN : CBM_SZ_16; - key_match_t *grown = - cbm_realloc(CBM_MEM_CLASS_OTHER, list->items, (size_t)cap * sizeof(*grown)); - if (!grown) { - atomic_store_explicit(&job->failed, true, memory_order_relaxed); - return; - } - list->items = grown; - list->cap = cap; - } - list->items[list->count].co = co; - list->items[list->count].confidence = confidence; - list->count++; + if (!key_match_row(job, ci, &job->lists[ci])) { + atomic_store_explicit(&job->failed, true, memory_order_relaxed); + return; } } } @@ -311,38 +508,30 @@ static int strategy_key_symbols(cbm_gbuf_t *gb) { return 0; } - config_entry_t config_entries[CBM_SZ_4K]; - int config_count = collect_config_entries(vars, var_count, config_entries, CBM_SZ_4K); - - if (config_count == 0) { - return 0; - } - - code_entry_t code_entries[CBM_SZ_8K]; - int code_count = collect_code_entries(gb, code_entries, CBM_SZ_8K); - + CBMArena arena; + cbm_arena_init(&arena); + cl_entries_t config = {0}; + cl_entries_t code = {0}; + CBMHashTable *exact = NULL; + CBMHashTable *trigrams = NULL; + key_match_list_t *lists = NULL; int edge_count = 0; - size_t *config_len = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)config_count * sizeof(size_t)); - size_t *code_len = - cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)(code_count > 0 ? code_count : 1) * sizeof(size_t)); - key_match_list_t *lists = - cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)config_count * sizeof(key_match_list_t)); - bool parallel_ok = config_len && code_len && lists; - if (parallel_ok) { - for (int ci = 0; ci < config_count; ci++) { - config_len[ci] = strlen(config_entries[ci].normalized); - } - for (int co = 0; co < code_count; co++) { - code_len[co] = strlen(code_entries[co].normalized); - } + bool ok = collect_config_entries(vars, var_count, &config, &arena); + if (ok && config.count > 0) { + ok = collect_code_entries(gb, &code, &arena); + exact = ok ? cbm_ht_create(0) : NULL; + trigrams = exact ? cbm_ht_create(0) : NULL; + ok = trigrams && cl_build_indexes(&code, exact, trigrams, &arena); + lists = ok ? cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)config.count * sizeof(*lists)) : NULL; + ok = lists != NULL; + } + if (ok && lists) { key_match_job_t job = { - .config = config_entries, - .config_len = config_len, - .config_count = config_count, - .code = code_entries, - .code_len = code_len, - .code_count = code_count, + .config = &config, + .code = &code, + .exact = exact, + .trigrams = trigrams, .lists = lists, }; atomic_init(&job.next, 0); @@ -351,59 +540,44 @@ static int strategy_key_symbols(cbm_gbuf_t *gb) { cbm_parallel_for( workers, key_match_worker, &job, (cbm_parallel_for_opts_t){.max_workers = workers, .force_pthreads = false}); - parallel_ok = !atomic_load_explicit(&job.failed, memory_order_relaxed); + ok = !atomic_load_explicit(&job.failed, memory_order_relaxed); } - if (parallel_ok) { - for (int ci = 0; ci < config_count; ci++) { + if (ok && lists) { + for (int ci = 0; ci < config.count; ci++) { for (int m = 0; m < lists[ci].count; m++) { const key_match_t *km = &lists[ci].items[m]; char props[CBM_SZ_512]; snprintf(props, sizeof(props), "{\"strategy\":\"key_symbol\",\"confidence\":%.2f,\"config_key\":\"%s\"}", - km->confidence, config_entries[ci].name); - - cbm_gbuf_insert_edge(gb, code_entries[km->co].node_id, config_entries[ci].node_id, + km->confidence, config.items[ci].name); + cbm_gbuf_insert_edge(gb, code.items[km->co].node_id, config.items[ci].node_id, "CONFIGURES", props); edge_count++; } } + } else if (!ok) { + char cfg_buf[CBM_SZ_16]; + char code_buf[CBM_SZ_16]; + snprintf(cfg_buf, sizeof(cfg_buf), "%d", config.count); + snprintf(code_buf, sizeof(code_buf), "%d", code.count); + cbm_log_warn("configlinker.key_symbol.oom", "config", cfg_buf, "code", code_buf); } - for (int ci = 0; lists && ci < config_count; ci++) { + + for (int ci = 0; lists && ci < config.count; ci++) { cbm_free(CBM_MEM_CLASS_OTHER, lists[ci].items); } cbm_free(CBM_MEM_CLASS_OTHER, lists); - cbm_free(CBM_MEM_CLASS_OTHER, config_len); - cbm_free(CBM_MEM_CLASS_OTHER, code_len); - if (parallel_ok) { - return edge_count; + if (exact) { + cbm_ht_foreach(exact, cl_posting_free, NULL); + cbm_ht_free(exact); } - - /* Allocation failed: the one-thread comparison, as before. */ - for (int ci = 0; ci < config_count; ci++) { - for (int co = 0; co < code_count; co++) { - double confidence = 0.0; - - if (strcmp(config_entries[ci].normalized, code_entries[co].normalized) == 0) { - /* Exact match */ - confidence = CONF_KEY_EXACT; - } else if (strstr(code_entries[co].normalized, config_entries[ci].normalized) != NULL) { - /* Substring match */ - confidence = CONF_KEY_SUBSTRING; - } - - if (confidence > 0.0) { - char props[CBM_SZ_512]; - snprintf(props, sizeof(props), - "{\"strategy\":\"key_symbol\",\"confidence\":%.2f,\"config_key\":\"%s\"}", - confidence, config_entries[ci].name); - - cbm_gbuf_insert_edge(gb, code_entries[co].node_id, config_entries[ci].node_id, - "CONFIGURES", props); - edge_count++; - } - } + if (trigrams) { + cbm_ht_foreach(trigrams, cl_posting_free, NULL); + cbm_ht_free(trigrams); } - + cbm_free(CBM_MEM_CLASS_OTHER, config.items); + cbm_free(CBM_MEM_CLASS_OTHER, code.items); + cbm_arena_destroy(&arena); return edge_count; } diff --git a/tests/test_configlink.c b/tests/test_configlink.c index 1b1ea216a..c7248a8bc 100644 --- a/tests/test_configlink.c +++ b/tests/test_configlink.c @@ -160,14 +160,16 @@ TEST(configlink_key_symbol_short_key_skipped) { } /* Go: TestConfigKeySymbol_CamelCaseNormalization - * config.json has maxRetryCount, handler.go has getMaxRetryCount() */ + * config.json has max_retry_count, handler.go has maxRetryCount: camelCase and + * snake_case normalize to the same key. (A JSON key links by exact name only, + * see configlink_key_symbol_infra_exact_only, so the symbol names it exactly.) */ TEST(configlink_key_symbol_camel_case) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); - cbm_gbuf_upsert_node(gb, "Variable", "maxRetryCount", "test.config.maxRetryCount", + cbm_gbuf_upsert_node(gb, "Variable", "max_retry_count", "test.config.max_retry_count", "config.json", 0, 0, NULL); - cbm_gbuf_upsert_node(gb, "Function", "getMaxRetryCount", "test.handler.getMaxRetryCount", + cbm_gbuf_upsert_node(gb, "Function", "maxRetryCount", "test.handler.maxRetryCount", "handler.go", 0, 0, NULL); run_configlink(gb, "test", NULL); @@ -180,6 +182,124 @@ TEST(configlink_key_symbol_camel_case) { PASS(); } +static int count_key_symbol_edges(cbm_gbuf_t *gb) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "CONFIGURES", &edges, &count); + int n = 0; + for (int i = 0; i < count; i++) { + if (edges[i]->properties_json && + strstr(edges[i]->properties_json, "\"strategy\":\"key_symbol\"")) { + n++; + } + } + return n; +} + +/* Keys of infrastructure files (YAML, JSON, HCL, tfvars) link by exact name + * only: the substring guess linked a chart's `node_selector` to every + * function whose name contains it. In 17 infra repositories 4 of 40 sampled + * key_symbol edges were right, all exact and in the same directory tree. */ +TEST(configlink_key_symbol_infra_exact_only) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + cbm_gbuf_upsert_node(gb, "Variable", "node_selector", "test.charts.app.values.node_selector", + "charts/app/values.yaml", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Function", "GetPodNodeSelectorConfig", + "test.charts.app.gen.GetPodNodeSelectorConfig", "charts/app/gen.go", 0, 0, + NULL); + run_configlink(gb, "test", NULL); + ASSERT_EQ(count_key_symbol_edges(gb), 0); + cbm_gbuf_free(gb); + PASS(); +} + +/* ...and only within the config file's own component (its directory tree): + * a stage's JSON schema does not configure an unrelated module's variable. */ +TEST(configlink_key_symbol_infra_scoped_to_component) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + int64_t cfg = cbm_gbuf_upsert_node(gb, "Variable", "cluster_name", + "test.svc_a.config.cluster_name", "svc-a/config.yaml", 0, + 0, NULL); + int64_t inside = cbm_gbuf_upsert_node(gb, "Function", "clusterName", + "test.svc_a.cmd.main.clusterName", "svc-a/cmd/main.go", + 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Function", "clusterName", "test.svc_b.main.clusterName", + "svc-b/main.go", 0, 0, NULL); + run_configlink(gb, "test", NULL); + ASSERT_EQ(count_key_symbol_edges(gb), 1); + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "CONFIGURES", &edges, &count); + ASSERT_EQ(count, 1); + ASSERT_EQ(edges[0]->source_id, inside); + ASSERT_EQ(edges[0]->target_id, cfg); + cbm_gbuf_free(gb); + PASS(); +} + +/* A conventional config directory (config/, conf/, settings/, ...) belongs to + * the component around it. */ +TEST(configlink_key_symbol_config_dir_component) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + cbm_gbuf_upsert_node(gb, "Variable", "cluster_name", "test.config.app.cluster_name", + "config/app.yaml", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Function", "clusterName", "test.pkg.app.clusterName", "pkg/app.go", + 0, 0, NULL); + run_configlink(gb, "test", NULL); + ASSERT_EQ(count_key_symbol_edges(gb), 1); + cbm_gbuf_free(gb); + PASS(); +} + +/* A Terraform/OpenTofu variable is what a config value sets: a tfvars.json key + * links to `variable ""` of its component (the block's Class is named + * `variable.`), not to outputs, modules or resources, and not to a + * variable of another component (kubespray: contrib/terraform/gcp/tfvars.json + * sets contrib/terraform/gcp/modules/.../variables.tf). */ +TEST(configlink_key_symbol_terraform_variable) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + int64_t cfg = cbm_gbuf_upsert_node(gb, "Variable", "ssh_whitelist", + "test.gcp.tfvars.ssh_whitelist", + "contrib/terraform/gcp/tfvars.json", 0, 0, NULL); + int64_t var = cbm_gbuf_upsert_node( + gb, "Class", "variable.ssh_whitelist", "test.gcp.modules.cluster.variable.ssh_whitelist", + "contrib/terraform/gcp/modules/cluster/variables.tf", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Class", "output.ssh_whitelist", + "test.gcp.modules.cluster.output.ssh_whitelist", + "contrib/terraform/gcp/modules/cluster/outputs.tf", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Class", "variable.ssh_whitelist", + "test.hetzner.variable.ssh_whitelist", + "contrib/terraform/hetzner/variables.tf", 0, 0, NULL); + run_configlink(gb, "test", NULL); + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "CONFIGURES", &edges, &count); + ASSERT_EQ(count, 1); + ASSERT_EQ(edges[0]->source_id, var); + ASSERT_EQ(edges[0]->target_id, cfg); + cbm_gbuf_free(gb); + PASS(); +} + +/* No count of config keys decides what links: the collector stopped at 4,096. */ +TEST(configlink_key_symbol_no_config_cap) { + enum { KEYS = 4200 }; + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + for (int i = 0; i < KEYS; i++) { + char name[CBM_SZ_64]; + char qn[CBM_SZ_128]; + snprintf(name, sizeof(name), "limit_alpha_%04d", i); + snprintf(qn, sizeof(qn), "test.config.%s", name); + cbm_gbuf_upsert_node(gb, "Variable", name, qn, "config.toml", 0, 0, NULL); + snprintf(qn, sizeof(qn), "test.main.%s", name); + cbm_gbuf_upsert_node(gb, "Function", name, qn, "main.go", 0, 0, NULL); + } + run_configlink(gb, "test", NULL); + ASSERT_EQ(count_key_symbol_edges(gb), KEYS); + cbm_gbuf_free(gb); + PASS(); +} + /* Go: TestConfigKeySymbol_NoFalsePositive * config.toml has url — 1-token key, should not match parseURL */ TEST(configlink_key_symbol_no_false_positive) { @@ -366,17 +486,18 @@ static cbm_gbuf_t *build_oversized_corpus(bool reverse) { return gb; } -/* REGRESSION: collect_config_entries/collect_code_entries fill fixed-capacity - * arrays and stop at the cap, walking gbuf label indexes in insertion order — - * which under parallel extraction is worker-merge order and varies run to run. - * Sorting candidates canonically before the cap is what keeps the surviving - * set, and therefore the emitted CONFIGURES edges, a pure function of the - * inputs. This test stands in for the scheduling variance by feeding the same - * corpus in two insertion orders: the emitted edge set must be identical. */ +/* REGRESSION: the collectors walk gbuf label indexes in insertion order -- + * under parallel extraction worker-merge order, which varies run to run. + * They filled fixed-capacity arrays and stopped at a cap, so the surviving + * set depended on that order until candidates were sorted first; now no cap + * cuts the set at all. This test stands in for the scheduling variance by + * feeding the same corpus in two insertion orders: the emitted edge set must + * be identical -- and complete, all 9,000 candidates past the old 8,192 cap. */ TEST(configlink_candidate_truncation_is_order_independent) { cbm_gbuf_t *fwd = build_oversized_corpus(false); run_configlink(fwd, "test", NULL); char *fp_fwd = configures_fingerprint(fwd); + int fwd_edges = count_key_symbol_edges(fwd); cbm_gbuf_t *rev = build_oversized_corpus(true); run_configlink(rev, "test", NULL); @@ -384,7 +505,7 @@ TEST(configlink_candidate_truncation_is_order_independent) { ASSERT_TRUE(fp_fwd != NULL); ASSERT_TRUE(fp_rev != NULL); - ASSERT_TRUE(fp_fwd[0] != '\0'); /* the cap must actually have been exercised */ + ASSERT_EQ(fwd_edges, CL_DET_CANDIDATES); ASSERT_STR_EQ(fp_fwd, fp_rev); free(fp_fwd); @@ -403,6 +524,11 @@ SUITE(configlink) { RUN_TEST(configlink_key_symbol_short_key_skipped); RUN_TEST(configlink_key_symbol_camel_case); RUN_TEST(configlink_key_symbol_no_false_positive); + RUN_TEST(configlink_key_symbol_infra_exact_only); + RUN_TEST(configlink_key_symbol_infra_scoped_to_component); + RUN_TEST(configlink_key_symbol_config_dir_component); + RUN_TEST(configlink_key_symbol_no_config_cap); + RUN_TEST(configlink_key_symbol_terraform_variable); /* Strategy 2: Dependency → Import */ RUN_TEST(configlink_dep_import_package_json); From 0ac529f5f4a6e0ad7e89373ef77029ce670e7caa Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 15:24:17 +0200 Subject: [PATCH 11/52] fix(cross-repo): no row count decides which calls are matched Each of the cross-repo scans -- a project's HTTP_CALLS, async and channel edges, a target's Route templates for fuzzy matching -- stopped after 4,096 rows and treated the rest as not there. A call examined after 4,096 others never got its CROSS_HTTP_CALLS edge even though the target serves its route; which edges existed depended on how many others the project held. No count may decide graph content. The scans now read every row; only a SQLite error counts as failure, as before. The project-cache guards (CR_MAX_PROJECTS, CR_MAX_CACHE_ENTRIES) stay: they fail the run visibly instead of dropping content. cross_repo_scan_bound_counts_examined_rows_not_matches asserted that the 4,097th call stays unmatched; as cross_repo_matches_calls_past_4096_rows it asserts the edge exists, and fails on the bounded scans. None of the 17 infra corpora has 4,096 rows in one of these scans, so their graphs are unchanged. Signed-off-by: Martin Vogel --- src/pipeline/pass_cross_repo.c | 42 +++++++++++----------------------- tests/test_cross_repo.c | 9 +++++--- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/src/pipeline/pass_cross_repo.c b/src/pipeline/pass_cross_repo.c index 09384a8e9..1f57d1c25 100644 --- a/src/pipeline/pass_cross_repo.c +++ b/src/pipeline/pass_cross_repo.c @@ -33,7 +33,6 @@ enum { CR_PATH_BUF = 4096, CR_QN_BUF = 512, CR_PROPS_BUF = 2048, - CR_MAX_EDGES = 4096, CR_DB_EXT_LEN = 3, /* strlen(".db") */ CR_INIT_CAP = 32, CR_MAX_PROJECTS = 4096, @@ -481,9 +480,9 @@ static bool cr_path_matches_template(const char *concrete, const char *templ) { * the route QN into out_qn and return the handler id; returns 0 on no match. * * COST: this scans every Route node of the target project once per unmatched - * HTTP_CALLS edge — O(calls × routes) per project pair. Acceptable while both - * factors stay small (calls are capped at CR_MAX_EDGES and it only runs for - * edges the exact lookup missed); revisit with a prepared template index if + * HTTP_CALLS edge — O(calls × routes) per project pair. It only runs for edges + * the exact lookup missed; no row count stops a scan (a cap decided which + * calls were matched at all); revisit with a prepared template index if * cross-repo matching ever shows up in profiles. (#523) */ static int64_t find_route_handler_fuzzy(cbm_store_t *target_store, const char *concrete_path, const char *method, char *out_qn, size_t out_qn_sz, @@ -504,11 +503,8 @@ static int64_t find_route_handler_fuzzy(cbm_store_t *target_store, const char *c return 0; } int64_t found = 0; - int scanned = 0; int step_rc = SQLITE_DONE; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { const char *qn = (const char *)sqlite3_column_text(s, 0); if (!qn || strncmp(qn, "__route__", CR_ROUTE_PREFIX_LEN) != 0) { continue; @@ -550,7 +546,7 @@ static int64_t find_route_handler_fuzzy(cbm_store_t *target_store, const char *c break; } } - if (!cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { *failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { @@ -713,12 +709,9 @@ static cr_match_result_t match_http_routes(cbm_store_t *src_store, const char *s } int count = 0; - int scanned = 0; int step_rc = SQLITE_DONE; bool failed = false; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { int64_t caller_id = sqlite3_column_int64(s, 0); int64_t route_id = sqlite3_column_int64(s, SKIP_ONE); const char *props = (const char *)sqlite3_column_text(s, PAIR_LEN); @@ -772,7 +765,7 @@ static cr_match_result_t match_http_routes(cbm_store_t *src_store, const char *s count++; } - if (!failed && !cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!failed && !cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { @@ -804,12 +797,9 @@ static cr_match_result_t match_async_routes(cbm_store_t *src_store, const char * } int count = 0; - int scanned = 0; int step_rc = SQLITE_DONE; bool failed = false; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { int64_t caller_id = sqlite3_column_int64(s, 0); int64_t route_id = sqlite3_column_int64(s, SKIP_ONE); const char *props = (const char *)sqlite3_column_text(s, PAIR_LEN); @@ -854,7 +844,7 @@ static cr_match_result_t match_async_routes(cbm_store_t *src_store, const char * } count++; } - if (!failed && !cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!failed && !cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { @@ -969,12 +959,9 @@ static cr_match_result_t match_channels(cbm_store_t *src_store, const char *src_ } int count = 0; - int scanned = 0; int step_rc = SQLITE_DONE; bool failed = false; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { const char *channel_name = (const char *)sqlite3_column_text(s, SKIP_ONE); const char *channel_qn = (const char *)sqlite3_column_text(s, PAIR_LEN); if (!channel_name || !channel_qn) { @@ -1007,7 +994,7 @@ static cr_match_result_t match_channels(cbm_store_t *src_store, const char *src_ count++; } } - if (!failed && !cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!failed && !cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { @@ -1084,12 +1071,9 @@ static cr_match_result_t match_typed_routes(cbm_store_t *src_store, const char * } int count = 0; - int scanned = 0; int step_rc = SQLITE_DONE; bool failed = false; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { int64_t caller_id = sqlite3_column_int64(s, 0); int64_t route_id = sqlite3_column_int64(s, SKIP_ONE); const char *props = (const char *)sqlite3_column_text(s, PAIR_LEN); @@ -1139,7 +1123,7 @@ static cr_match_result_t match_typed_routes(cbm_store_t *src_store, const char * } count++; } - if (!failed && !cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!failed && !cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { diff --git a/tests/test_cross_repo.c b/tests/test_cross_repo.c index 23d74fb5b..9d318f8c0 100644 --- a/tests/test_cross_repo.c +++ b/tests/test_cross_repo.c @@ -574,7 +574,10 @@ static bool cross_repo_seed_bounded_scan(const cross_repo_fixture_t *fixture, return ok; } -TEST(cross_repo_scan_bound_counts_examined_rows_not_matches) { +/* No row count decides which calls are matched: the scans stopped after 4,096 + * rows, so a call examined after 4,096 others never got its CROSS_HTTP_CALLS + * edge even though the target serves its route. */ +TEST(cross_repo_matches_calls_past_4096_rows) { cross_repo_fixture_t fixture; bool setup = cross_repo_fixture_begin(&fixture) && cross_repo_seed_bounded_scan(&fixture, "bounded-source", "bounded-target"); @@ -589,7 +592,7 @@ TEST(cross_repo_scan_bound_counts_examined_rows_not_matches) { ASSERT_FALSE(result.failed); ASSERT_EQ(result.projects_scanned, 1); - ASSERT_EQ(result.http_edges, 0); + ASSERT_EQ(result.http_edges, 1); PASS(); } @@ -952,7 +955,7 @@ SUITE(cross_repo) { RUN_TEST(cross_repo_named_pre768_target_fails_before_cleanup_issue2133); RUN_TEST(cross_repo_status_distinguishes_never_run_from_ran_without_links); RUN_TEST(cross_repo_wildcard_reports_skipped_pre768_store); - RUN_TEST(cross_repo_scan_bound_counts_examined_rows_not_matches); + RUN_TEST(cross_repo_matches_calls_past_4096_rows); RUN_TEST(cross_repo_propagates_delete_failure); RUN_TEST(cross_repo_failed_bidirectional_insert_is_not_counted); RUN_TEST(cross_repo_cancel_mid_run_keeps_completed_target_and_stops_before_later_target); From 40c9af37373cc69f2da7ed03aaed6e5300dfc82f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 15:40:21 +0200 Subject: [PATCH 12/52] refactor(infra): delete the line-regex infra parsers nothing calls pass_infrascan.c carried line-by-line parsers for Dockerfiles, .env files, shell scripts and Terraform, five file classifiers and a JSON bracket cleaner; pass_envscan.c walked a repository for URL-valued environment assignments. No production code calls any of them: the graph gets these files from the tree-sitter extractors, and the envscan walk said so in its own header comment. Their only callers were 40 tests in test_pipeline.c that asserted the parsers' output, so the suite measured code the index never runs. Deleted: cbm_is_dockerfile, cbm_is_compose_file, cbm_is_cloudbuild_file, cbm_is_env_file, cbm_is_shell_script, cbm_clean_json_brackets, cbm_parse_{dockerfile,dotenv,shell,terraform}_source, cbm_scan_project_env_urls(_excluded), their result structs, and the tests and test helpers that existed only for them. Kept, all with production callers or a planned one: cbm_is_kustomize_file and cbm_is_k8s_manifest (pass_k8s), cbm_infra_qn, the Helm Chart.yaml parser, and the secret detectors cbm_is_secret_binding / cbm_is_secret_value (still covered by infra_secret_detection), which the secret-hygiene work reuses. The secret detector's prefix offsets now have their own names instead of borrowing the Dockerfile parser's keyword lengths. No graph change: the deleted functions were unreachable from the index. Suites green: pipeline 299 (339 minus the 40 deleted), platform, grammar_probe_f/g, infrascan, discover, language, lang_contract, incremental, integration, edge_structural, configlink. Signed-off-by: Martin Vogel --- Makefile.cbm | 1 - scripts/memory-core-baseline.txt | 1 - src/discover/language.c | 6 +- src/pipeline/pass_envscan.c | 625 ----------------- src/pipeline/pass_infrascan.c | 1071 +----------------------------- src/pipeline/pipeline_internal.h | 122 ---- tests/test_grammar_probe_g.c | 3 +- tests/test_pipeline.c | 985 +-------------------------- tests/test_platform.c | 2 +- 9 files changed, 42 insertions(+), 2774 deletions(-) delete mode 100644 src/pipeline/pass_envscan.c diff --git a/Makefile.cbm b/Makefile.cbm index cd5eb4124..76ba5b467 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -515,7 +515,6 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ - src/pipeline/pass_envscan.c \ src/pipeline/pass_compile_commands.c \ src/pipeline/pass_infrascan.c \ src/pipeline/pass_k8s.c \ diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 251920e2f..32c4a89db 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -64,7 +64,6 @@ src/pipeline/pass_cross_repo.c 8 src/pipeline/pass_definitions.c 11 src/pipeline/pass_enrichment.c 15 src/pipeline/pass_ensemble_routing.c 10 -src/pipeline/pass_envscan.c 2 src/pipeline/pass_githistory.c 27 src/pipeline/pass_importance.c 4 src/pipeline/pass_infrascan.c 2 diff --git a/src/discover/language.c b/src/discover/language.c index 7b66f717a..8307f5324 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -971,9 +971,9 @@ static CBMLanguage lang_for_filename(const cbm_userconfig_t *ucfg, const char *f /* DotEnv variant filenames (".env.local", ".env.production", …): the * filename starts with ".env." but its last "extension" (e.g. ".local") - * is not a real language extension. Match the dotenv convention used by - * pass_envscan/pass_infrascan (".env" exact, ".env." prefix, "*.env" - * suffix) so file-index routing agrees with direct extraction. */ + * is not a real language extension. Match the dotenv convention (".env" + * exact, ".env." prefix, "*.env" suffix) so file-index routing agrees with + * direct extraction. */ if (strncmp(filename, ".env.", SLEN(".env.")) == 0) { return CBM_LANG_DOTENV; } diff --git a/src/pipeline/pass_envscan.c b/src/pipeline/pass_envscan.c deleted file mode 100644 index 88bd2e685..000000000 --- a/src/pipeline/pass_envscan.c +++ /dev/null @@ -1,625 +0,0 @@ -/* - * pass_envscan.c — Environment URL scanner. - * - * Walks a project directory, scans config files (Dockerfile, .env, shell, - * YAML, TOML, Terraform, .properties) for environment variable assignments - * where the value is a URL. Filters out secrets. - * - * Port of internal/pipeline/envscan.go:ScanProjectEnvURLs(). - */ -#include "foundation/constants.h" - -enum { - ENV_REGEX_MAX = 5, - ENV_GRP_1 = 1, - ENV_GRP_2 = 2, - ENV_GRP_3 = 3, - ENV_GRP_4 = 4, - ENV_GRP_5 = 5, - ENV_EXT_LEN = 4, /* strlen(".env") */ -}; - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "foundation/log.h" - -#include -#include "foundation/compat_fs.h" -#include "foundation/compat_regex.h" -#include "foundation/platform.h" /* cbm_normalize_path_sep */ -#include -#include -#include -#include - -#ifdef _WIN32 -#include "foundation/win_utf8.h" /* cbm_path_to_wide */ -#include -/* Canonical paths on both sides come from the same resolver, so casing already - * agrees; comparing case-insensitively removes the doubt at no cost. */ -#define envscan_path_ncmp _strnicmp -#else -#include -#include -#define envscan_path_ncmp strncmp -#endif - -/* ── Regex patterns (compiled lazily) ──────────────────────────── */ - -static cbm_regex_t dockerfile_re; /* ENV|ARG KEY=VALUE or KEY VALUE */ -static cbm_regex_t yaml_kv_re; /* key: "https://..." */ -static cbm_regex_t yaml_setenv_re; /* --set-env-vars KEY=VALUE */ -static cbm_regex_t terraform_re; /* default|value = "https://..." */ -static cbm_regex_t shell_re; /* [export] KEY=https://... */ -static cbm_regex_t envfile_re; /* KEY=https://... */ -static cbm_regex_t toml_re; /* key = "https://..." */ -static cbm_regex_t properties_re; /* key=https://... */ -static int patterns_compiled = 0; - -/* POSIX ERE doesn't support \w or \S — use bracket expressions */ -#define W "[A-Za-z0-9_]" /* word char */ -#define NW "[^ \t\"']" /* non-whitespace, non-quote */ - -static void compile_patterns(void) { - if (patterns_compiled) { - return; - } - - cbm_regcomp(&dockerfile_re, "^(ENV|ARG)[[:space:]]+(" W "+)[= ](.*)", CBM_REG_EXTENDED); - cbm_regcomp(&yaml_kv_re, "(" W "+):[[:space:]]*[\"']?(https?://" NW "+)", CBM_REG_EXTENDED); - cbm_regcomp(&yaml_setenv_re, "--set-env-vars[[:space:]]+(" W "+)=([^ \t]+)", CBM_REG_EXTENDED); - cbm_regcomp(&terraform_re, "(default|value)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", - CBM_REG_EXTENDED); - cbm_regcomp(&shell_re, "(export[[:space:]]+)?(" W "+)=[\"']?(https?://" NW "+)", - CBM_REG_EXTENDED); - cbm_regcomp(&envfile_re, "^(" W "+)=(https?://[^ \t]+)", CBM_REG_EXTENDED); - cbm_regcomp(&toml_re, "(" W "+)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", - CBM_REG_EXTENDED); - cbm_regcomp(&properties_re, "(" W "+)[[:space:]]*=[[:space:]]*(https?://[^ \t]+)", - CBM_REG_EXTENDED); - - patterns_compiled = SKIP_ONE; -} - -#undef W -#undef NW - -/* ── File type detection ───────────────────────────────────────── */ - -static int is_dockerfile_name(const char *name) { - /* Case-insensitive check */ - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - if (strcmp(lower, "dockerfile") == 0) { - return SKIP_ONE; - } -#define DOCKERFILE_SUFFIX_LEN 11 /* strlen("dockerfile.") == strlen(".dockerfile") */ - if (strncmp(lower, "dockerfile.", DOCKERFILE_SUFFIX_LEN) == 0) { - return SKIP_ONE; - } - if (len > DOCKERFILE_SUFFIX_LEN && - strcmp(lower + len - DOCKERFILE_SUFFIX_LEN, ".dockerfile") == 0) { - return SKIP_ONE; - } - return 0; -} - -static int is_env_file_name(const char *name) { - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - if (strcmp(lower, ".env") == 0) { - return SKIP_ONE; - } - if (strncmp(lower, ".env.", SLEN(".env.")) == 0) { - return SKIP_ONE; - } - if (len > ENV_EXT_LEN && strcmp(lower + len - ENV_EXT_LEN, ".env") == 0) { - return SKIP_ONE; - } - return 0; -} - -static int is_secret_file(const char *name) { - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - static const char *patterns[] = { - "service_account", "credentials", "key.json", "key.pem", "id_rsa", - "id_ed25519", ".pem", ".key", NULL}; - for (int i = 0; patterns[i]; i++) { - if (strstr(lower, patterns[i])) { - return SKIP_ONE; - } - } - return 0; -} - -/* ── Ignored directories ───────────────────────────────────────── */ - -static int is_ignored_dir(const char *name) { - static const char *dirs[] = { - ".git", "node_modules", ".svn", ".hg", "__pycache__", "vendor", ".terraform", ".cache", - ".idea", ".vscode", "dist", "build", ".next", ".nuxt", "target", NULL}; - for (int i = 0; dirs[i]; i++) { - if (strcmp(name, dirs[i]) == 0) { - return SKIP_ONE; - } - } - return 0; -} - -/* ── File type enum ────────────────────────────────────────────── */ - -typedef enum { - FT_UNKNOWN = 0, - FT_DOCKERFILE, - FT_YAML, - FT_TERRAFORM, - FT_SHELL, - FT_ENVFILE, - FT_TOML, - FT_PROPERTIES, -} file_type_t; - -static file_type_t detect_file_type(const char *name) { - if (is_dockerfile_name(name)) { - return FT_DOCKERFILE; - } - if (is_env_file_name(name)) { - return FT_ENVFILE; - } - - const char *ext = strrchr(name, '.'); - if (!ext) { - return FT_UNKNOWN; - } - - if (strcmp(ext, ".yaml") == 0 || strcmp(ext, ".yml") == 0) { - return FT_YAML; - } - if (strcmp(ext, ".tf") == 0 || strcmp(ext, ".hcl") == 0) { - return FT_TERRAFORM; - } - if (strcmp(ext, ".sh") == 0 || strcmp(ext, ".bash") == 0 || strcmp(ext, ".zsh") == 0) { - return FT_SHELL; - } - if (strcmp(ext, ".toml") == 0) { - return FT_TOML; - } - if (strcmp(ext, ".properties") == 0 || strcmp(ext, ".cfg") == 0 || strcmp(ext, ".ini") == 0) { - return FT_PROPERTIES; - } - - return FT_UNKNOWN; -} - -/* ── Line scanner ──────────────────────────────────────────────── */ - -/* Extract key/value from a regex match with two capture groups. - * Returns 1 on success, 0 if groups are empty or too large. */ -static int extract_kv_groups(const char *trimmed, const cbm_regmatch_t *m, int key_grp, int val_grp, - char *key_out, size_t key_sz, char *val_out, size_t val_sz) { - int klen = (m[key_grp].rm_eo - m[key_grp].rm_so); - int vlen = (m[val_grp].rm_eo - m[val_grp].rm_so); - if (klen <= 0 || klen >= (int)key_sz || vlen <= 0 || vlen >= (int)val_sz) { - return 0; - } - memcpy(key_out, trimmed + m[key_grp].rm_so, klen); - key_out[klen] = '\0'; - memcpy(val_out, trimmed + m[val_grp].rm_so, vlen); - val_out[vlen] = '\0'; - return SKIP_ONE; -} - -/* Try to scan a Dockerfile line. */ -static int scan_dockerfile_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&dockerfile_re, line, ENV_GRP_4, m, 0) != 0) { - return 0; - } - if (!extract_kv_groups(line, m, ENV_GRP_2, ENV_GRP_3, key, ksz, val, vsz)) { - return 0; - } - size_t vl = strlen(val); - while (vl > 0 && (val[vl - SKIP_ONE] == '"' || val[vl - SKIP_ONE] == '\'')) { - val[--vl] = '\0'; - } - return SKIP_ONE; -} - -/* Try to scan a YAML line. */ -static int scan_yaml_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&yaml_kv_re, line, ENV_GRP_3, m, 0) == 0 && - extract_kv_groups(line, m, ENV_GRP_1, ENV_GRP_2, key, ksz, val, vsz)) { - return SKIP_ONE; - } - if (cbm_regexec(&yaml_setenv_re, line, ENV_GRP_3, m, 0) == 0 && - extract_kv_groups(line, m, ENV_GRP_1, ENV_GRP_2, key, ksz, val, vsz)) { - return SKIP_ONE; - } - return 0; -} - -/* Try to scan a Terraform line. */ -static int scan_terraform_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&terraform_re, line, ENV_GRP_3, m, 0) != 0) { - return 0; - } - int vlen = (m[ENV_GRP_2].rm_eo - m[ENV_GRP_2].rm_so); - if (vlen <= 0 || vlen >= (int)vsz) { - return 0; - } - strncpy(key, "_tf_default", ksz - SKIP_ONE); - key[ksz - SKIP_ONE] = '\0'; - memcpy(val, line + m[ENV_GRP_2].rm_so, vlen); - val[vlen] = '\0'; - return SKIP_ONE; -} - -/* Try single-regex scan (shell, envfile, toml, properties). */ -static int scan_regex_line(cbm_regex_t *re, const char *line, int kg, int vg, char *key, size_t ksz, - char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(re, line, ENV_GRP_5, m, 0) == 0 && - extract_kv_groups(line, m, kg, vg, key, ksz, val, vsz)) { - return SKIP_ONE; - } - return 0; -} - -static int scan_line(const char *line, file_type_t ft, char *key_out, size_t key_sz, char *val_out, - size_t val_sz) { - const char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - if (*trimmed == '#' || (trimmed[0] == '/' && trimmed[SKIP_ONE] == '/')) { - return 0; - } - - switch (ft) { - case FT_DOCKERFILE: - return scan_dockerfile_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_YAML: - return scan_yaml_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_TERRAFORM: - return scan_terraform_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_SHELL: - return scan_regex_line(&shell_re, trimmed, ENV_GRP_2, ENV_GRP_3, key_out, key_sz, val_out, - val_sz); - case FT_ENVFILE: - return scan_regex_line(&envfile_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, val_out, - val_sz); - case FT_TOML: - return scan_regex_line(&toml_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, val_out, - val_sz); - case FT_PROPERTIES: - return scan_regex_line(&properties_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, - val_out, val_sz); - default: - return 0; - } -} - -/* ── Containment ───────────────────────────────────────────────── */ - -/* Join dir + name, rejecting rather than truncating. - * - * Silent truncation here was not merely a lost path. The root-relative suffix - * used to be derived as `full_path + strlen(root_path)`, so once the real root - * was longer than this fixed buffer that pointer landed past the end of the - * array — out-of-bounds arithmetic on every entry below it. Refusing the path is - * what keeps envscan_relative_suffix in bounds by construction. */ -static bool envscan_join_path(char *out, size_t out_sz, const char *dir, const char *name) { - if (out_sz == 0) { - return false; - } - int written = snprintf(out, out_sz, "%s/%s", dir, name); - if (written <= 0 || (size_t)written >= out_sz) { - out[0] = '\0'; - cbm_log_warn("envscan.path_too_long", "dir", dir, "name", name); - return false; - } - return true; -} - -/* Root-relative suffix of a path built beneath root_path. - * - * The prefix comparison is not a formality: it succeeding is precisely what - * proves `path` holds at least root_len bytes, which is what makes - * `path + root_len` a valid pointer. Never compute this suffix from an - * unverified strlen. Returns NULL when `path` is not under the root. */ -static const char *envscan_relative_suffix(const char *path, const char *root_path, - size_t root_len) { - if (strncmp(path, root_path, root_len) != 0) { - return NULL; - } - const char *rel = path + root_len; - while (*rel == '/') { - rel++; - } - return rel; -} - -/* Stat WITHOUT following links, so a repository-controlled symlink cannot lead - * the walk outside the project. The previous stat() resolved the link, reported - * the target's S_ISDIR, and the walk happily descended wherever it pointed. - * Returns 0 on success, CBM_NOT_FOUND to skip the entry. - * Mirrors pass_pkgmap.c's pkgmap_safe_stat and discover.c's safe_stat. */ -static int envscan_safe_stat(const char *abs_path, struct stat *st) { -#ifdef _WIN32 - /* Windows has no lstat; a junction or directory symlink surfaces as a - * reparse point, which is the same escape hatch, so screen the attribute. - * The wide stat also keeps non-ASCII paths off the ANSI CRT. */ - wchar_t *wide = cbm_path_to_wide(abs_path); - if (!wide) { - return CBM_NOT_FOUND; - } - DWORD attributes = GetFileAttributesW(wide); - if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - free(wide); - return CBM_NOT_FOUND; - } - struct _stat64 wst; - int ret = _wstat64(wide, &wst); - free(wide); - if (ret != 0) { - return CBM_NOT_FOUND; - } - st->st_mode = wst.st_mode; - st->st_size = wst.st_size; - st->st_mtime = wst.st_mtime; - return 0; -#else - if (lstat(abs_path, st) != 0) { - return CBM_NOT_FOUND; - } - if (S_ISLNK(st->st_mode)) { - return CBM_NOT_FOUND; - } - return 0; -#endif -} - -/* True when `candidate` still resolves to somewhere under `canonical_root`. - * - * Belt to envscan_safe_stat's braces. The link screen refuses the symlink - * itself; this catches escapes a mode check cannot see — a resolved path that - * leaves the root for any other reason. `canonical_root` is resolved once per - * scan because cbm_canonical_path costs a syscall per component. */ -static bool envscan_within_root(const char *candidate, const char *canonical_root) { - char resolved[CBM_SZ_4K]; - if (!cbm_canonical_path(candidate, resolved, sizeof(resolved))) { - return false; - } -#ifdef _WIN32 - cbm_normalize_path_sep(resolved); -#endif - size_t root_len = strlen(canonical_root); - if (root_len == 0 || envscan_path_ncmp(resolved, canonical_root, root_len) != 0) { - return false; - } - /* Demand a real component boundary, so root "/repo" does not admit - * "/repo-elsewhere". */ - return canonical_root[root_len - 1] == '/' || resolved[root_len] == '\0' || - resolved[root_len] == '/'; -} - -/* ── Public API ────────────────────────────────────────────────── */ - -/* Open a candidate config file without following a symlink where the platform - * allows it. Mode "r" is preserved from the previous cbm_fopen call so the - * per-platform newline handling the line loop already copes with is unchanged. */ -static FILE *envscan_open_file(const char *full_path) { -#ifdef _WIN32 - /* No O_NOFOLLOW equivalent here; reparse points were screened by - * envscan_safe_stat. cbm_fopen is required for UTF-8 → _wfopen mapping. */ - return cbm_fopen(full_path, "r"); -#else - /* O_NOFOLLOW closes the window between the lstat above and this open: if the - * entry turned into a symlink in between, the open fails rather than reading - * through it. A raw open is correct on POSIX — paths are bytes there, and the - * cbm_fopen rule exists for Windows' wide-API mapping. */ - int flags = O_RDONLY | O_CLOEXEC; -#ifdef O_NOFOLLOW - flags |= O_NOFOLLOW; -#endif - int descriptor = open(full_path, flags); - if (descriptor < 0) { - return NULL; - } - FILE *stream = fdopen(descriptor, "r"); - if (!stream) { - (void)close(descriptor); - } - return stream; -#endif -} - -/* Scan a single file for env URL bindings. Returns number of bindings added. */ -static int scan_env_file(const char *full_path, const char *rel, file_type_t ft, - cbm_env_binding_t *out, int max_out) { - FILE *f = envscan_open_file(full_path); - if (!f) { - return 0; - } - - struct stat fst; - if (fstat(fileno(f), &fst) != 0 || fst.st_size > (long)CBM_SZ_1K * CBM_SZ_1K) { - (void)fclose(f); - return 0; - } - - int count = 0; - char line[CBM_SZ_2K]; - while (fgets(line, sizeof(line), f) && count < max_out) { - size_t ll = strlen(line); - while (ll > 0 && (line[ll - SKIP_ONE] == '\n' || line[ll - SKIP_ONE] == '\r')) { - line[--ll] = '\0'; - } - - char key[CBM_SZ_128]; - char value[CBM_SZ_512]; - if (!scan_line(line, ft, key, sizeof(key), value, sizeof(value))) { - continue; - } - if (strncmp(value, "http://", SLEN("http://")) != 0 && - strncmp(value, "https://", SLEN("https://")) != 0) { - continue; - } - if (cbm_is_secret_binding(key, value) || cbm_is_secret_value(value)) { - continue; - } - - strncpy(out[count].key, key, sizeof(out[count].key) - 1); - out[count].key[sizeof(out[count].key) - SKIP_ONE] = '\0'; - strncpy(out[count].value, value, sizeof(out[count].value) - 1); - out[count].value[sizeof(out[count].value) - SKIP_ONE] = '\0'; - strncpy(out[count].file_path, rel, sizeof(out[count].file_path) - 1); - out[count].file_path[sizeof(out[count].file_path) - SKIP_ONE] = '\0'; - count++; - } - (void)fclose(f); - return count; -} - -/* Process a single directory entry for env scanning. Returns bindings added. - * `root_len` is strlen of the root the walk started from, verified against - * path_stack[0] by the caller; `canonical_root` is that root resolved once. */ -static int process_env_entry(cbm_dirent_t *ent, const char *dir_path, const char *root_path, - size_t root_len, const char *canonical_root, cbm_env_binding_t *out, - int max_out, char path_stack[][CBM_SZ_512], int *stack_top, - char **excluded_dirs, int excluded_count) { - char full_path[CBM_SZ_512]; - if (!envscan_join_path(full_path, sizeof(full_path), dir_path, ent->name)) { - return 0; - } - const char *rel = envscan_relative_suffix(full_path, root_path, root_len); - if (!rel) { - return 0; - } - - struct stat st; - if (envscan_safe_stat(full_path, &st) != 0) { - return 0; - } - if (S_ISDIR(st.st_mode)) { - if (is_ignored_dir(ent->name) || - cbm_pipeline_relpath_is_excluded(rel, excluded_dirs, excluded_count) || - *stack_top >= CBM_SZ_256) { - return 0; - } - if (!envscan_within_root(full_path, canonical_root)) { - cbm_log_warn("envscan.dir_outside_root", "path", full_path, "root", canonical_root); - return 0; - } - /* Checked copy. The stack rows are the same width as full_path, so this - * cannot truncate today — but check anyway rather than lean on that - * coincidence: a truncated row is exactly what reintroduced the - * out-of-bounds suffix arithmetic. */ - int written = snprintf(path_stack[*stack_top], sizeof(path_stack[0]), "%s", full_path); - if (written <= 0 || (size_t)written >= sizeof(path_stack[0])) { - cbm_log_warn("envscan.path_too_long", "dir", dir_path, "name", ent->name); - return 0; - } - (*stack_top)++; - return 0; - } - /* Only regular files are config files. Without this, the lstat above still - * lets a FIFO through, and opening one blocks the pass until a writer - * appears. */ - if (!S_ISREG(st.st_mode)) { - return 0; - } - if (is_secret_file(ent->name)) { - return 0; - } - file_type_t ft = detect_file_type(ent->name); - if (ft == FT_UNKNOWN) { - return 0; - } - return scan_env_file(full_path, rel, ft, out, max_out); -} - -int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t *out, int max_out, - char **excluded_dirs, int excluded_count) { - if (!root_path || !out || max_out <= 0) { - return 0; - } - compile_patterns(); - - int count = 0; - char path_stack[CBM_SZ_256][CBM_SZ_512]; - int stack_top = SKIP_ONE; - /* Reject an over-long root instead of truncating it in. The old strncpy left - * path_stack[0] SHORTER than root_path, and every suffix pointer derived from - * strlen(root_path) was then past the end of its buffer. Establishing - * root_len from the copy that actually fit is what removes that class. */ - int written = snprintf(path_stack[0], sizeof(path_stack[0]), "%s", root_path); - if (written <= 0 || (size_t)written >= sizeof(path_stack[0])) { - cbm_log_warn("envscan.root_too_long", "root", root_path); - return 0; - } - size_t root_len = (size_t)written; - - /* Resolved once per scan: every directory the walk wants to descend is - * checked against this, and resolving costs a syscall per component. */ - char canonical_root[CBM_SZ_4K]; - if (!cbm_canonical_path(root_path, canonical_root, sizeof(canonical_root))) { - /* Unresolvable root — nothing to walk. Previously opendir simply failed - * on the first iteration, so the result is the same: zero bindings. */ - return 0; - } -#ifdef _WIN32 - cbm_normalize_path_sep(canonical_root); -#endif - - while (stack_top > 0 && count < max_out) { - stack_top--; - char dir_path[CBM_SZ_512]; - int copied = snprintf(dir_path, sizeof(dir_path), "%s", path_stack[stack_top]); - if (copied <= 0 || (size_t)copied >= sizeof(dir_path)) { - continue; - } - - cbm_dir_t *d = cbm_opendir(dir_path); - if (!d) { - continue; - } - cbm_dirent_t *ent; - while ((ent = cbm_readdir(d)) && count < max_out) { - count += process_env_entry(ent, dir_path, root_path, root_len, canonical_root, - out + count, max_out - count, path_stack, &stack_top, - excluded_dirs, excluded_count); - } - cbm_closedir(d); - } - return count; -} - -int cbm_scan_project_env_urls(const char *root_path, cbm_env_binding_t *out, int max_out) { - return cbm_scan_project_env_urls_excluded(root_path, out, max_out, NULL, 0); -} diff --git a/src/pipeline/pass_infrascan.c b/src/pipeline/pass_infrascan.c index 24af73d86..4b51149e3 100644 --- a/src/pipeline/pass_infrascan.c +++ b/src/pipeline/pass_infrascan.c @@ -1,35 +1,24 @@ /* - * pass_infrascan.c — Infrastructure file detection and parsing helpers. + * pass_infrascan.c — Infrastructure file helpers. * - * Pure helper functions for detecting infrastructure file types, - * secret filtering, and parsing Dockerfiles, .env files, shell scripts, - * and Terraform files. + * Kustomization / K8s manifest detection, secret filtering, the infra QN + * helper, and the Helm Chart.yaml parser. */ #include "foundation/constants.h" enum { - IS_ENV_EXT = 4, /* strlen(".env") */ IS_CONTENT_LIMIT = 4096, - IS_MAX_STAGES = 16, - IS_MAX_PORTS = 16, - IS_MAX_SOURCES = 16, - IS_TOKEN_MIN_1 = 16, /* min alnum for AWS key */ - IS_TOKEN_MIN_2 = 20, /* min alnum for GH classic token */ - IS_FROM_SKIP = 4, /* strlen("FROM") */ - IS_EXPOSE_SKIP = 6, /* strlen("EXPOSE") */ - IS_RUN_SKIP = 3, /* strlen("RUN") / strlen("ENV") / strlen("ARG") */ - IS_CMD_SKIP = 4, /* strlen("CMD ") */ - IS_USER_SKIP = 5, /* strlen("USER ") */ - IS_EXPORT_SKIP = 7, /* strlen("export ") */ - IS_WORKDIR_SKIP = 8, /* strlen("WORKDIR ") */ - IS_ENTRYPOINT_OFF = 10, /* strlen("ENTRYPOINT") */ - IS_AS_SKIP = 3, /* strlen("AS ") */ - IS_SOURCE_DOT = 2, /* strlen(". ") */ - IS_QUOTE_TRIM = 2, /* strip surrounding quotes */ + IS_TOKEN_MIN_1 = 16, /* min alnum for AWS key */ + IS_TOKEN_MIN_2 = 20, /* min alnum for GH classic token */ + SECRET_AKIA_LEN = 4, /* strlen("AKIA") */ + SECRET_SK_LEN = 3, /* strlen("sk-") */ + SECRET_GHP_LEN = 4, /* strlen("ghp_") */ + SECRET_GLPAT_LEN = 6, /* strlen("glpat-") */ + SECRET_XOX_LEN = 3, /* strlen("xox"); the token-type letter follows */ + SECRET_XOX_DASH = 4, /* offset of the '-' in "xoxb-" */ + SECRET_XOX_BODY = 5, /* strlen("xoxb-") */ }; -#define IS_NOT_FOUND (-1) -#define SLEN(s) (sizeof(s) - 1) #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" @@ -38,14 +27,6 @@ enum { #include #include -/* ── Constants ───────────────────────────────────────────────────── */ - -/* String length constants for keyword matching */ -#define LEN_DOCKERFILE 11 /* strlen("dockerfile.") == strlen(".dockerfile") */ -#define LEN_DOCKER_COMPOSE 14 /* strlen("docker-compose") */ -#define LEN_HEALTHCHECK 11 /* strlen("HEALTHCHECK") */ -#define LEN_DOCKER_COMPOSE_SKIP 15 /* strlen("docker-compose") + space */ - /* Minimum alnum chars after prefix for secret detection */ #define GITHUB_PAT_MIN_ALNUM 36 /* ghp_ + 36 alnum chars */ @@ -95,124 +76,8 @@ static int count_alnum_dash(const char *s) { return n; } -/* Skip whitespace, return pointer past it */ -static const char *skip_ws(const char *p) { - while (*p == ' ' || *p == '\t') { - p++; - } - return p; -} - -/* Extract a word ([a-zA-Z0-9_]+) into buf. Returns length. */ -static int extract_word(const char *p, char *buf, size_t buf_sz) { - int n = 0; - while ((isalnum((unsigned char)p[n]) || p[n] == '_') && (size_t)n < buf_sz - SKIP_ONE) { - buf[n] = p[n]; - n++; - } - buf[n] = '\0'; - return n; -} - -/* Extract a non-space token into buf. Returns length. */ -static int extract_token(const char *p, char *buf, size_t buf_sz) { - int n = 0; - while (p[n] && p[n] != ' ' && p[n] != '\t' && (size_t)n < buf_sz - SKIP_ONE) { - buf[n] = p[n]; - n++; - } - buf[n] = '\0'; - return n; -} - -/* Trim trailing whitespace in-place */ -static void rtrim(char *s) { - size_t len = strlen(s); - while (len > 0 && (s[len - SKIP_ONE] == ' ' || s[len - SKIP_ONE] == '\t' || - s[len - SKIP_ONE] == '\r' || s[len - SKIP_ONE] == '\n')) { - s[--len] = '\0'; - } -} - /* ── File identification ────────────────────────────────────────── */ -bool cbm_is_dockerfile(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strcmp(lower, "dockerfile") == 0) { - return true; - } - if (strncmp(lower, "dockerfile.", LEN_DOCKERFILE) == 0) { - return true; - } - size_t len = strlen(lower); - if (len > LEN_DOCKERFILE && strcmp(lower + len - LEN_DOCKERFILE, ".dockerfile") == 0) { - return true; - } - return false; -} - -bool cbm_is_compose_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - bool prefix_match = (strncmp(lower, "docker-compose", LEN_DOCKER_COMPOSE) == 0) || - (strcmp(lower, "compose.yml") == 0) || (strcmp(lower, "compose.yaml") == 0); - if (!prefix_match) { - return false; - } - - const char *ext = strrchr(lower, '.'); - if (!ext) { - return false; - } - return (strcmp(ext, ".yml") == 0 || strcmp(ext, ".yaml") == 0); -} - -bool cbm_is_cloudbuild_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strncmp(lower, "cloudbuild", SLEN("cloudbuild")) != 0) { - return false; - } - const char *ext = strrchr(lower, '.'); - if (!ext) { - return false; - } - return (strcmp(ext, ".yml") == 0 || strcmp(ext, ".yaml") == 0); -} - -bool cbm_is_env_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strcmp(lower, ".env") == 0) { - return true; - } - if (strncmp(lower, ".env.", SLEN(".env.")) == 0) { - return true; - } - size_t len = strlen(lower); - if (len > IS_ENV_EXT && strcmp(lower + len - IS_ENV_EXT, ".env") == 0) { - return true; - } - return false; -} - bool cbm_is_kustomize_file(const char *name) { if (!name) { return false; @@ -237,14 +102,6 @@ bool cbm_is_k8s_manifest(const char *name, const char *content) { return ci_strstr(buf, "apiVersion:") != NULL; } -bool cbm_is_shell_script(const char *name, const char *ext) { - (void)name; - if (!ext) { - return false; - } - return (strcmp(ext, ".sh") == 0 || strcmp(ext, ".bash") == 0 || strcmp(ext, ".zsh") == 0); -} - /* ── Secret detection ───────────────────────────────────────────── */ static bool key_is_secret(const char *key) { @@ -276,35 +133,35 @@ bool cbm_is_secret_value(const char *value) { /* AKIA + 16 alnum (AWS key) */ p = ci_strstr(value, "AKIA"); - if (p && count_alnum(p + IS_CMD_SKIP) >= IS_TOKEN_MIN_1) { + if (p && count_alnum(p + SECRET_AKIA_LEN) >= IS_TOKEN_MIN_1) { return true; } /* sk- + 20 alnum (API key) */ p = ci_strstr(value, "sk-"); - if (p && count_alnum(p + IS_RUN_SKIP) >= IS_TOKEN_MIN_2) { + if (p && count_alnum(p + SECRET_SK_LEN) >= IS_TOKEN_MIN_2) { return true; } /* ghp_ + 36 alnum (GitHub PAT) */ p = ci_strstr(value, "ghp_"); - if (p && count_alnum(p + IS_CMD_SKIP) >= GITHUB_PAT_MIN_ALNUM) { + if (p && count_alnum(p + SECRET_GHP_LEN) >= GITHUB_PAT_MIN_ALNUM) { return true; } /* glpat- + 20 alnum/dash (GitLab PAT) */ p = ci_strstr(value, "glpat-"); - if (p && count_alnum_dash(p + IS_EXPOSE_SKIP) >= IS_TOKEN_MIN_2) { + if (p && count_alnum_dash(p + SECRET_GLPAT_LEN) >= IS_TOKEN_MIN_2) { return true; } /* xox[bps]- (Slack token) */ p = ci_strstr(value, "xox"); - if (p && p[IS_RUN_SKIP] != '\0' && - (tolower((unsigned char)p[IS_RUN_SKIP]) == 'b' || - tolower((unsigned char)p[IS_RUN_SKIP]) == 'p' || - tolower((unsigned char)p[IS_RUN_SKIP]) == 's') && - p[IS_CMD_SKIP] == '-' && count_alnum_dash(p + IS_USER_SKIP) >= SKIP_ONE) { + if (p && p[SECRET_XOX_LEN] != '\0' && + (tolower((unsigned char)p[SECRET_XOX_LEN]) == 'b' || + tolower((unsigned char)p[SECRET_XOX_LEN]) == 'p' || + tolower((unsigned char)p[SECRET_XOX_LEN]) == 's') && + p[SECRET_XOX_DASH] == '-' && count_alnum_dash(p + SECRET_XOX_BODY) >= SKIP_ONE) { return true; } @@ -321,892 +178,6 @@ bool cbm_is_secret_binding(const char *key, const char *value) { return false; } -/* ── Clean JSON brackets ────────────────────────────────────────── */ - -/* Strip JSON array brackets, quotes, and normalize whitespace. */ -static void clean_json_array_inner(const char *s, size_t len, char *out, size_t out_sz) { - size_t pos = 0; - bool in_space = false; - for (size_t i = SKIP_ONE; i < len - SKIP_ONE && pos < out_sz - SKIP_ONE; i++) { - char c = s[i]; - if (c == '"') { - continue; - } - if (c == ',') { - c = ' '; - } - if (c == ' ' || c == '\t') { - if (!in_space && pos > 0) { - out[pos++] = ' '; - in_space = true; - } - } else { - out[pos++] = c; - in_space = false; - } - } - while (pos > 0 && out[pos - SKIP_ONE] == ' ') { - pos--; - } - out[pos] = '\0'; -} - -void cbm_clean_json_brackets(const char *s, char *out, size_t out_sz) { - if (!s || !out || out_sz == 0) { - return; - } - - size_t len = strlen(s); - if (len >= PAIR_LEN && s[0] == '[' && s[len - SKIP_ONE] == ']') { - clean_json_array_inner(s, len, out, out_sz); - } else { - snprintf(out, out_sz, "%s", s); - } -} - -/* ── Dockerfile parser ──────────────────────────────────────────── */ - -static void df_parse_from(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "FROM", SLEN("FROM")) != 0) { - return; - } - const char *p = line + IS_FROM_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - /* Extract image */ - if (r->stage_count >= IS_MAX_STAGES) { - return; - } - int idx = r->stage_count; - extract_token(p, r->stage_images[idx], sizeof(r->stage_images[idx])); - - /* Advance past image */ - while (*p && *p != ' ' && *p != '\t') { - p++; - } - p = skip_ws(p); - - /* Check for AS */ - r->stage_names[idx][0] = '\0'; - if (strncasecmp(p, "AS", SLEN("AS")) == 0 && (p[PAIR_LEN] == ' ' || p[PAIR_LEN] == '\t')) { - p = skip_ws(p + IS_AS_SKIP); - extract_word(p, r->stage_names[idx], sizeof(r->stage_names[idx])); - } - r->stage_count++; -} - -static void df_parse_expose(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "EXPOSE", SLEN("EXPOSE")) != 0) { - return; - } - const char *p = line + IS_EXPOSE_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - /* Parse space-separated ports */ - while (*p && r->port_count < IS_MAX_PORTS) { - char port[CBM_SZ_32]; - int n = extract_token(p, port, sizeof(port)); - if (n == 0) { - break; - } - - /* Strip protocol suffix (e.g. 8080/tcp) */ - char *slash = strchr(port, '/'); - if (slash) { - *slash = '\0'; - } - - snprintf(r->exposed_ports[r->port_count], sizeof(r->exposed_ports[0]), "%s", port); - r->port_count++; - - p += n; - p = skip_ws(p); - } -} - -static void df_parse_env(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "ENV", SLEN("ENV")) != 0) { - return; - } - const char *p = line + IS_RUN_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - if (r->env_count >= CBM_SZ_64) { - return; - } - - /* Extract key */ - char key[CBM_SZ_128]; - int klen = extract_word(p, key, sizeof(key)); - if (klen == 0) { - return; - } - p += klen; - - /* Separator: = or space */ - if (*p != '=' && *p != ' ' && *p != '\t') { - return; - } - p++; - /* Skip additional whitespace after separator */ - while (*p == ' ' || *p == '\t') { - p++; - } - - /* Value is rest of line (trimmed) */ - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", p); - rtrim(value); - - /* Filter secrets */ - if (cbm_is_secret_binding(key, value)) { - return; - } - - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void df_parse_arg(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "ARG", SLEN("ARG")) != 0) { - return; - } - const char *p = line + IS_RUN_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - if (r->build_arg_count >= CBM_SZ_32) { - return; - } - extract_word(p, r->build_args[r->build_arg_count], sizeof(r->build_args[0])); - if (r->build_args[r->build_arg_count][0]) { - r->build_arg_count++; - } -} - -/* Parse a Dockerfile CMD/ENTRYPOINT value: strip brackets, trim. */ -static void df_parse_json_cmd(const char *line, int prefix_len, char *out, size_t out_sz) { - const char *p = skip_ws(line + prefix_len); - char raw[CBM_SZ_512]; - snprintf(raw, sizeof(raw), "%s", p); - rtrim(raw); - cbm_clean_json_brackets(raw, out, out_sz); -} - -static void df_parse_directives(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "WORKDIR", SLEN("WORKDIR")) == 0 && - (line[IS_EXPORT_SKIP] == ' ' || line[IS_EXPORT_SKIP] == '\t')) { - const char *p = skip_ws(line + IS_WORKDIR_SKIP); - snprintf(r->workdir, sizeof(r->workdir), "%s", p); - rtrim(r->workdir); - return; - } - if (strncasecmp(line, "CMD", SLEN("CMD")) == 0 && - (line[IS_RUN_SKIP] == ' ' || line[IS_RUN_SKIP] == '\t')) { - df_parse_json_cmd(line, IS_CMD_SKIP, r->cmd, sizeof(r->cmd)); - return; - } - if (strncasecmp(line, "ENTRYPOINT", SLEN("ENTRYPOINT")) == 0 && - (line[IS_ENTRYPOINT_OFF] == ' ' || line[IS_ENTRYPOINT_OFF] == '\t')) { - df_parse_json_cmd(line, LEN_HEALTHCHECK, r->entrypoint, sizeof(r->entrypoint)); - return; - } - if (strncasecmp(line, "USER", SLEN("USER")) == 0 && - (line[IS_CMD_SKIP] == ' ' || line[IS_CMD_SKIP] == '\t')) { - const char *p = skip_ws(line + IS_USER_SKIP); - extract_word(p, r->user, sizeof(r->user)); - return; - } - if (strncasecmp(line, "HEALTHCHECK", LEN_HEALTHCHECK) == 0 && - (line[LEN_HEALTHCHECK] == ' ' || line[LEN_HEALTHCHECK] == '\t')) { - const char *cmd_pos = ci_strstr(line + LEN_HEALTHCHECK, "CMD"); - if (cmd_pos) { - cmd_pos = skip_ws(cmd_pos + IS_RUN_SKIP); - snprintf(r->healthcheck, sizeof(r->healthcheck), "%s", cmd_pos); - rtrim(r->healthcheck); - } - } -} - -int cbm_parse_dockerfile_source(const char *source, cbm_dockerfile_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - /* Process line by line */ - const char *p = source; - char line[CBM_SZ_4K]; - - while (*p) { - /* Extract one line */ - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - /* Trim */ - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - /* Skip empty lines and comments */ - if (*trimmed == '\0' || *trimmed == '#') { - continue; - } - - df_parse_from(trimmed, out); - df_parse_expose(trimmed, out); - df_parse_env(trimmed, out); - df_parse_arg(trimmed, out); - df_parse_directives(trimmed, out); - } - - /* No stages = empty/invalid Dockerfile */ - if (out->stage_count == 0) { - return CBM_NOT_FOUND; - } - - /* base_image = last stage's image (use intermediate copy to avoid restrict overlap) */ - { - char tmp[sizeof(out->base_image)]; - snprintf(tmp, sizeof(tmp), "%s", out->stage_images[out->stage_count - SKIP_ONE]); - memcpy(out->base_image, tmp, sizeof(out->base_image)); - } - - return 0; -} - -/* ── Dotenv parser ──────────────────────────────────────────────── */ - -/* Strip matching surrounding quotes from a value string in-place. */ -static void strip_surrounding_quotes(char *value) { - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } -} - -/* Parse a single dotenv line: KEY=VALUE. Returns 1 if env var added. */ -static int parse_dotenv_line(const char *trimmed, cbm_dotenv_result_t *out) { - char key[CBM_SZ_128]; - int klen = 0; - if (isalpha((unsigned char)trimmed[0]) || trimmed[0] == '_') { - klen = extract_word(trimmed, key, sizeof(key)); - } - if (klen == 0 || trimmed[klen] != '=') { - return 0; - } - - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", trimmed + klen + SKIP_ONE); - rtrim(value); - strip_surrounding_quotes(value); - - if (cbm_is_secret_binding(key, value) || out->env_count >= CBM_SZ_64) { - return 0; - } - snprintf(out->env_vars[out->env_count].key, sizeof(out->env_vars[0].key), "%s", key); - snprintf(out->env_vars[out->env_count].value, sizeof(out->env_vars[0].value), "%s", value); - out->env_count++; - return SKIP_ONE; -} - -int cbm_parse_dotenv_source(const char *source, cbm_dotenv_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - - while (*p) { - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - if (*trimmed == '\0' || *trimmed == '#') { - continue; - } - - parse_dotenv_line(trimmed, out); - } - - return (out->env_count > 0) ? 0 : IS_NOT_FOUND; -} - -/* ── Shell script parser ────────────────────────────────────────── */ - -static void shell_parse_export(const char *line, cbm_shell_result_t *r) { - /* export VAR=VALUE */ - if (strncmp(line, "export", SLEN("export")) != 0 || - (line[IS_EXPOSE_SKIP] != ' ' && line[IS_EXPOSE_SKIP] != '\t')) { - return; - } - const char *p = skip_ws(line + IS_EXPORT_SKIP); - - char key[CBM_SZ_128]; - int klen = extract_word(p, key, sizeof(key)); - if (klen == 0 || p[klen] != '=') { - return; - } - - const char *val_start = p + klen + SKIP_ONE; - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", val_start); - rtrim(value); - - /* Strip surrounding quotes */ - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } - - if (cbm_is_secret_binding(key, value)) { - return; - } - if (r->env_count >= CBM_SZ_64) { - return; - } - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void shell_parse_plain_var(const char *line, cbm_shell_result_t *r) { - /* VAR=VALUE (only if no spaces in line — avoids matching commands) */ - if (strchr(line, ' ') || strchr(line, '\t')) { - return; - } - - char key[CBM_SZ_128]; - int klen = extract_word(line, key, sizeof(key)); - if (klen == 0 || line[klen] != '=') { - return; - } - - const char *val_start = line + klen + SKIP_ONE; - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", val_start); - rtrim(value); - - /* Strip surrounding quotes */ - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } - - if (cbm_is_secret_binding(key, value)) { - return; - } - if (r->env_count >= CBM_SZ_64) { - return; - } - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void shell_parse_source(const char *line, cbm_shell_result_t *r) { - /* source or . */ - const char *p = NULL; - if (strncmp(line, "source", SLEN("source")) == 0 && - (line[IS_EXPOSE_SKIP] == ' ' || line[IS_EXPOSE_SKIP] == '\t')) { - p = skip_ws(line + IS_EXPORT_SKIP); - } else if (line[0] == '.' && (line[SKIP_ONE] == ' ' || line[SKIP_ONE] == '\t')) { - p = skip_ws(line + IS_SOURCE_DOT); - } - if (!p) { - return; - } - - if (r->source_count >= IS_MAX_SOURCES) { - return; - } - - /* Strip surrounding quotes */ - char path[CBM_SZ_256]; - snprintf(path, sizeof(path), "%s", p); - rtrim(path); - size_t plen = strlen(path); - if (plen >= CBM_QUOTE_PAIR && ((path[0] == '"' && path[plen - SKIP_ONE] == '"') || - (path[0] == '\'' && path[plen - SKIP_ONE] == '\''))) { - memmove(path, path + CBM_QUOTE_OFFSET, plen - CBM_QUOTE_PAIR); - path[plen - IS_QUOTE_TRIM] = '\0'; - } - - /* Extract just the path token (no trailing args) */ - char *space = strchr(path, ' '); - if (space) { - *space = '\0'; - } - - snprintf(r->sources[r->source_count], sizeof(r->sources[0]), "%s", path); - r->source_count++; -} - -static void shell_parse_docker(const char *line, cbm_shell_result_t *r) { - /* docker or docker-compose */ - const char *p = NULL; - const char *tool = NULL; - - if (strncmp(line, "docker-compose", LEN_DOCKER_COMPOSE) == 0 && - (line[LEN_DOCKER_COMPOSE] == ' ' || line[LEN_DOCKER_COMPOSE] == '\t')) { - tool = "docker-compose"; - p = skip_ws(line + LEN_DOCKER_COMPOSE_SKIP); - } else if (strncmp(line, "docker", SLEN("docker")) == 0 && - (line[IS_EXPOSE_SKIP] == ' ' || line[IS_EXPOSE_SKIP] == '\t')) { - tool = "docker"; - p = skip_ws(line + IS_EXPORT_SKIP); - } - if (!tool || !p || !*p) { - return; - } - - if (r->docker_cmd_count >= IS_MAX_STAGES) { - return; - } - - char subcmd[CBM_SZ_64]; - extract_word(p, subcmd, sizeof(subcmd)); - if (subcmd[0]) { - snprintf(r->docker_cmds[r->docker_cmd_count], sizeof(r->docker_cmds[0]), "%s %s", tool, - subcmd); - r->docker_cmd_count++; - } -} - -/* Check for shebang on first non-empty line. Returns true if line was consumed. */ -static bool shell_try_shebang(const char *trimmed, cbm_shell_result_t *out, bool *shebang_checked) { - if (*shebang_checked) { - return false; - } - if (trimmed[0] == '#' && trimmed[SKIP_ONE] == '!') { - const char *sb = trimmed + PAIR_LEN; - while (*sb == ' ') { - sb++; - } - snprintf(out->shebang, sizeof(out->shebang), "%s", sb); - rtrim(out->shebang); - *shebang_checked = true; - return true; - } - *shebang_checked = true; - return false; -} - -/* Dispatch a single trimmed shell line to the appropriate parser. */ -static void shell_dispatch_line(const char *trimmed, cbm_shell_result_t *out) { - if (strncmp(trimmed, "export", SLEN("export")) == 0) { - shell_parse_export(trimmed, out); - return; - } - if (strncmp(trimmed, "source", SLEN("source")) == 0 || - (trimmed[0] == '.' && trimmed[SKIP_ONE] == ' ')) { - shell_parse_source(trimmed, out); - return; - } - if (strncmp(trimmed, "docker", SLEN("docker")) == 0) { - shell_parse_docker(trimmed, out); - } - shell_parse_plain_var(trimmed, out); -} - -int cbm_parse_shell_source(const char *source, cbm_shell_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - bool shebang_checked = false; - - while (*p) { - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - if (*trimmed == '\0') { - continue; - } - - if (shell_try_shebang(trimmed, out, &shebang_checked)) { - continue; - } - - if (*trimmed == '#') { - continue; - } - - shell_dispatch_line(trimmed, out); - } - - bool has_content = out->shebang[0] != '\0' || out->env_count > 0 || out->source_count > 0 || - out->docker_cmd_count > 0; - return has_content ? 0 : IS_NOT_FOUND; -} - -/* ── Terraform parser ───────────────────────────────────────────── */ - -typedef enum { - TF_BLOCK_NONE = 0, - TF_BLOCK_VARIABLE, - TF_BLOCK_MODULE, - TF_BLOCK_TERRAFORM -} tf_block_kind_t; - -/* Extract a double-quoted string value after = sign. - * Handles: key = "value" and key = value */ -static void tf_extract_quoted(const char *line, const char *prefix, char *out, size_t out_sz) { - const char *p = skip_ws(line); - size_t plen = strlen(prefix); - if (strncmp(p, prefix, plen) != 0) { - return; - } - p = skip_ws(p + plen); - if (*p != '=') { - return; - } - p = skip_ws(p + SKIP_ONE); - - /* Strip optional quotes */ - if (*p == '"') { - p++; - const char *end = strchr(p, '"'); - size_t vlen = end ? (size_t)(end - p) : strlen(p); - if (vlen >= out_sz) { - vlen = out_sz - SKIP_ONE; - } - memcpy(out, p, vlen); - out[vlen] = '\0'; - } else { - extract_token(p, out, out_sz); - } -} - -/* Count occurrences of a character in a string */ -static int count_char(const char *s, char c) { - int n = 0; - for (; *s; s++) { - if (*s == c) { - n++; - } - } - return n; -} - -/* Extract double-quoted identifier after keyword. - * Matches: keyword "identifier" → sets out to "identifier" - * Returns number of chars consumed from p, or 0 if no match. */ -static int tf_extract_ident(const char *p, const char *keyword, char *out, size_t out_sz) { - size_t klen = strlen(keyword); - if (strncmp(p, keyword, klen) != 0) { - return 0; - } - const char *after = skip_ws(p + klen); - if (*after != '"') { - return 0; - } - after++; - const char *end = strchr(after, '"'); - if (!end) { - return 0; - } - size_t ilen = (size_t)(end - after); - if (ilen >= out_sz) { - ilen = out_sz - SKIP_ONE; - } - memcpy(out, after, ilen); - out[ilen] = '\0'; - return (int)(end - p + SKIP_ONE); -} - -/* Parse a second quoted string after the first ident (for resource/data blocks). */ -static void tf_extract_second_quoted(const char *trimmed, int consumed, char *out, size_t outsz) { - const char *rest = skip_ws(trimmed + consumed); - out[0] = '\0'; - if (*rest == '"') { - rest++; - const char *end = strchr(rest, '"'); - if (end) { - size_t len = (size_t)(end - rest); - if (len >= outsz) { - len = outsz - SKIP_ONE; - } - memcpy(out, rest, len); - out[len] = '\0'; - } - } -} - -/* Try to parse a simple "keyword ident" block (variable, output, provider). */ -static bool tf_try_simple_block(const char *trimmed, const char *keyword, char *names, int name_sz, - int *count, int max_count) { - char ident[CBM_SZ_128]; - int consumed = tf_extract_ident(trimmed, keyword, ident, sizeof(ident)); - if (consumed <= 0 || *count >= max_count) { - return false; - } - snprintf(names + ((size_t)(*count) * (size_t)name_sz), name_sz, "%s", ident); - (*count)++; - return true; -} - -/* Try to match terraform/locals keyword blocks. */ -static bool tf_try_keyword_block(const char *trimmed, tf_block_kind_t *cur_block, - cbm_terraform_result_t *out) { - if (strncmp(trimmed, "terraform", SLEN("terraform")) == 0 && strstr(trimmed, "{")) { - *cur_block = TF_BLOCK_TERRAFORM; - return true; - } - if (strncmp(trimmed, "locals", SLEN("locals")) == 0 && strstr(trimmed, "{")) { - out->has_locals = true; - return true; - } - return false; -} - -/* Detect a top-level block header. Returns true if line was consumed as a header. */ -static bool tf_detect_block_header(const char *trimmed, cbm_terraform_result_t *out, - tf_block_kind_t *cur_block, int *cur_var_idx, int *cur_mod_idx) { - char ident1[CBM_SZ_128]; - - /* resource "type" "name" */ - { - int rc = tf_extract_ident(trimmed, "resource", ident1, sizeof(ident1)); - if (rc > 0 && out->resource_count < CBM_SZ_32) { - char ident2[CBM_SZ_128]; - tf_extract_second_quoted(trimmed, rc, ident2, sizeof(ident2)); - snprintf(out->resources[out->resource_count].type, sizeof(out->resources[0].type), "%s", - ident1); - snprintf(out->resources[out->resource_count].name, sizeof(out->resources[0].name), "%s", - ident2); - out->resource_count++; - return true; - } - } - - /* variable "name" */ - int consumed = tf_extract_ident(trimmed, "variable", ident1, sizeof(ident1)); - if (consumed > 0 && out->variable_count < CBM_SZ_32) { - *cur_block = TF_BLOCK_VARIABLE; - *cur_var_idx = out->variable_count; - snprintf(out->variables[*cur_var_idx].name, sizeof(out->variables[0].name), "%s", ident1); - out->variable_count++; - return true; - } - - /* output "name" */ - if (tf_try_simple_block(trimmed, "output", out->outputs[0], (int)sizeof(out->outputs[0]), - &out->output_count, CBM_SZ_32)) { - return true; - } - - /* provider "name" */ - if (tf_try_simple_block(trimmed, "provider", out->providers[0], (int)sizeof(out->providers[0]), - &out->provider_count, IS_MAX_STAGES)) { - return true; - } - - /* module "name" */ - consumed = tf_extract_ident(trimmed, "module", ident1, sizeof(ident1)); - if (consumed > 0 && out->module_count < IS_MAX_STAGES) { - *cur_block = TF_BLOCK_MODULE; - *cur_mod_idx = out->module_count; - snprintf(out->modules[*cur_mod_idx].tf_name, sizeof(out->modules[0].tf_name), "%s", ident1); - out->module_count++; - return true; - } - - /* data "type" "name" */ - { - int dc = tf_extract_ident(trimmed, "data", ident1, sizeof(ident1)); - if (dc > 0 && out->data_source_count < IS_MAX_STAGES) { - char ident2[CBM_SZ_128]; - tf_extract_second_quoted(trimmed, dc, ident2, sizeof(ident2)); - snprintf(out->data_sources[out->data_source_count].type, - sizeof(out->data_sources[0].type), "%s", ident1); - snprintf(out->data_sources[out->data_source_count].name, - sizeof(out->data_sources[0].name), "%s", ident2); - out->data_source_count++; - return true; - } - } - - return tf_try_keyword_block(trimmed, cur_block, out); -} - -/* Extract attributes from a line inside a variable block. */ -static void tf_parse_variable_attrs(const char *trimmed, cbm_tf_variable_t *v) { - char def_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "default", def_val, sizeof(def_val)); - if (def_val[0] && !cbm_is_secret_binding(v->name, def_val)) { - snprintf(v->default_val, sizeof(v->default_val), "%s", def_val); - } - char type_val[CBM_SZ_64] = {0}; - tf_extract_quoted(trimmed, "type", type_val, sizeof(type_val)); - if (type_val[0]) { - snprintf(v->type, sizeof(v->type), "%s", type_val); - } - char desc_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "description", desc_val, sizeof(desc_val)); - if (desc_val[0]) { - snprintf(v->description, sizeof(v->description), "%s", desc_val); - } -} - -/* Extract attributes inside a block (variable/module/terraform). */ -static void tf_parse_block_attrs(const char *trimmed, cbm_terraform_result_t *out, - tf_block_kind_t cur_block, int cur_var_idx, int cur_mod_idx) { - switch (cur_block) { - case TF_BLOCK_VARIABLE: - if (cur_var_idx >= 0) { - tf_parse_variable_attrs(trimmed, &out->variables[cur_var_idx]); - } - break; - case TF_BLOCK_MODULE: - if (cur_mod_idx >= 0) { - char src_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "source", src_val, sizeof(src_val)); - if (src_val[0]) { - snprintf(out->modules[cur_mod_idx].source, sizeof(out->modules[0].source), "%s", - src_val); - } - } - break; - case TF_BLOCK_TERRAFORM: { - const char *bp = skip_ws(trimmed); - char backend_name[CBM_SZ_128] = {0}; - tf_extract_ident(bp, "backend", backend_name, sizeof(backend_name)); - if (backend_name[0]) { - snprintf(out->backend, sizeof(out->backend), "%s", backend_name); - } - break; - } - default: - break; - } -} - -/* Check if any terraform content was parsed. */ -static bool tf_result_has_content(const cbm_terraform_result_t *out) { - return out->resource_count > 0 || out->variable_count > 0 || out->output_count > 0 || - out->provider_count > 0 || out->module_count > 0 || out->data_source_count > 0 || - out->backend[0] != '\0' || out->has_locals; -} - -/* Read next non-empty, non-comment line. Advances *pp. Returns trimmed or NULL. */ -static char *tf_next_line(const char **pp, char *line, size_t line_sz) { - const char *p = *pp; - const char *eol = strchr(p, '\n'); - size_t len = eol ? (size_t)(eol - p) : strlen(p); - if (len >= line_sz) { - len = line_sz - SKIP_ONE; - } - memcpy(line, p, len); - line[len] = '\0'; - *pp = eol ? eol + SKIP_ONE : p + len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - if (*trimmed == '\0' || *trimmed == '#' || (trimmed[0] == '/' && trimmed[SKIP_ONE] == '/')) { - return NULL; - } - return trimmed; -} - -int cbm_parse_terraform_source(const char *source, cbm_terraform_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - int brace_depth = 0; - tf_block_kind_t cur_block = TF_BLOCK_NONE; - int cur_var_idx = IS_NOT_FOUND; - int cur_mod_idx = IS_NOT_FOUND; - - while (*p) { - char *trimmed = tf_next_line(&p, line, sizeof(line)); - if (!trimmed) { - continue; - } - - brace_depth += count_char(trimmed, '{') - count_char(trimmed, '}'); - - if (brace_depth <= SKIP_ONE && cur_block == TF_BLOCK_NONE) { - if (tf_detect_block_header(trimmed, out, &cur_block, &cur_var_idx, &cur_mod_idx)) { - continue; - } - } - - if (cur_block != TF_BLOCK_NONE && brace_depth >= SKIP_ONE && brace_depth <= PAIR_LEN) { - tf_parse_block_attrs(trimmed, out, cur_block, cur_var_idx, cur_mod_idx); - } - - if (brace_depth == 0 && cur_block != TF_BLOCK_NONE) { - cur_block = TF_BLOCK_NONE; - cur_var_idx = IS_NOT_FOUND; - cur_mod_idx = IS_NOT_FOUND; - } - } - - return tf_result_has_content(out) ? 0 : IS_NOT_FOUND; -} - /* ── Infra QN helper ────────────────────────────────────────────── */ char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *infra_type, diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 544bc3412..4fe4b549d 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -886,11 +886,6 @@ int cbm_parse_compile_commands(const char *json_data, const char *repo_path, cha /* ── Infrascan helpers (pass_infrascan.c) ─────────────────────────── */ /* File identification helpers */ -bool cbm_is_dockerfile(const char *name); -bool cbm_is_compose_file(const char *name); -bool cbm_is_cloudbuild_file(const char *name); -bool cbm_is_env_file(const char *name); -bool cbm_is_shell_script(const char *name, const char *ext); bool cbm_is_kustomize_file(const char *name); bool cbm_is_k8s_manifest(const char *name, const char *content); @@ -898,103 +893,6 @@ bool cbm_is_k8s_manifest(const char *name, const char *content); bool cbm_is_secret_binding(const char *key, const char *value); bool cbm_is_secret_value(const char *value); -/* Clean JSON array brackets from CMD/ENTRYPOINT values. - * E.g. ["./app", "--flag"] → ./app --flag - * Writes result to out (up to out_sz). */ -void cbm_clean_json_brackets(const char *s, char *out, size_t out_sz); - -/* Key-value pair for environment variables / config entries */ -typedef struct { - char key[CBM_SZ_128]; - char value[CBM_SZ_512]; -} cbm_env_kv_t; - -/* Dockerfile parsing result */ -typedef struct { - char base_image[CBM_SZ_256]; - char stage_images[CBM_SZ_16][CBM_SZ_256]; - char stage_names[CBM_SZ_16][CBM_SZ_128]; - int stage_count; - char exposed_ports[CBM_SZ_16][CBM_SZ_32]; - int port_count; - cbm_env_kv_t env_vars[CBM_SZ_64]; - int env_count; - char build_args[CBM_SZ_32][CBM_SZ_128]; - int build_arg_count; - char workdir[CBM_SZ_256]; - char cmd[CBM_SZ_512]; - char entrypoint[CBM_SZ_512]; - char healthcheck[CBM_SZ_512]; - char user[CBM_SZ_64]; -} cbm_dockerfile_result_t; - -/* Dotenv parsing result */ -typedef struct { - cbm_env_kv_t env_vars[CBM_SZ_64]; - int env_count; -} cbm_dotenv_result_t; - -/* Shell script parsing result */ -typedef struct { - char shebang[CBM_SZ_256]; - cbm_env_kv_t env_vars[CBM_SZ_64]; - int env_count; - char sources[CBM_SZ_16][CBM_SZ_256]; - int source_count; - char docker_cmds[CBM_SZ_16][CBM_SZ_256]; - int docker_cmd_count; -} cbm_shell_result_t; - -/* Terraform variable */ -typedef struct { - char name[CBM_SZ_128]; - char type[CBM_SZ_64]; - char default_val[CBM_SZ_256]; - char description[CBM_SZ_256]; -} cbm_tf_variable_t; - -/* Terraform resource / data source */ -typedef struct { - char type[CBM_SZ_128]; - char name[CBM_SZ_128]; -} cbm_tf_resource_t; - -/* Terraform module */ -typedef struct { - char tf_name[CBM_SZ_128]; - char source[CBM_SZ_256]; -} cbm_tf_module_t; - -/* Terraform parsing result */ -typedef struct { - cbm_tf_resource_t resources[CBM_SZ_32]; - int resource_count; - cbm_tf_variable_t variables[CBM_SZ_32]; - int variable_count; - char outputs[CBM_SZ_32][CBM_SZ_128]; - int output_count; - char providers[CBM_SZ_16][CBM_SZ_128]; - int provider_count; - cbm_tf_module_t modules[CBM_SZ_16]; - int module_count; - cbm_tf_resource_t data_sources[CBM_SZ_16]; - int data_source_count; - char backend[CBM_SZ_128]; - bool has_locals; -} cbm_terraform_result_t; - -/* Parse a Dockerfile from source text. Returns 0 if parsed, -1 if empty/invalid. */ -int cbm_parse_dockerfile_source(const char *source, cbm_dockerfile_result_t *out); - -/* Parse a .env file from source text. Returns 0 if parsed, -1 if empty. */ -int cbm_parse_dotenv_source(const char *source, cbm_dotenv_result_t *out); - -/* Parse a shell script from source text. Returns 0 if parsed, -1 if empty. */ -int cbm_parse_shell_source(const char *source, cbm_shell_result_t *out); - -/* Parse a Terraform file from source text. Returns 0 if parsed, -1 if empty. */ -int cbm_parse_terraform_source(const char *source, cbm_terraform_result_t *out); - /* Helm Chart.yaml parse result: chart name + its dependencies (#338). */ enum { CBM_HELM_NAME_MAX = 128, CBM_HELM_REPO_MAX = 512 }; typedef struct { @@ -1215,26 +1113,6 @@ extern _Atomic uint64_t g_importance_name_visits; * closure-delta route actually took that path instead of passing vacuously. */ extern _Atomic uint64_t g_importance_store_rows; -/* ── Env URL scanner (pass_envscan.c) ────────────────────────────── */ - -typedef struct { - char key[CBM_SZ_128]; - char value[CBM_SZ_512]; - char file_path[CBM_SZ_256]; -} cbm_env_binding_t; - -/* Scan a project directory for environment variable assignments with URL values. - * Walks the filesystem, scans Dockerfiles, shell scripts, .env, YAML, TOML, - * Terraform, and .properties files. Filters out secrets. - * Returns number of bindings written to out (up to max_out). - * NOTE: this walker currently has no production callers — it is exercised - * only by tests. The _excluded variant honors discovery exclusions for - * consistency with the pkgmap/path-alias walks (#792); the plain variant - * scans unexcluded (NULL exclusion list). */ -int cbm_scan_project_env_urls(const char *root_path, cbm_env_binding_t *out, int max_out); -int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t *out, int max_out, - char **excluded_dirs, int excluded_count); - /* ── Incremental pipeline (pipeline_incremental.c) ───────────────── */ /* Run incremental re-index on an existing disk DB. diff --git a/tests/test_grammar_probe_g.c b/tests/test_grammar_probe_g.c index 215b4acee..8ac6f3ebe 100644 --- a/tests/test_grammar_probe_g.c +++ b/tests/test_grammar_probe_g.c @@ -688,7 +688,8 @@ TEST(probe_sosl_module_only) { * * DotEnv golden histogram: Module:1 (pure-data). * Environment variable files are pure-data; keys are NOT extracted as nodes. - * Extension: .env suffix detected by pass_envscan; filename ".env" also works. + * Extension: .env suffix and ".env." prefix route to DotEnv (language.c); + * filename ".env" also works. * ══════════════════════════════════════════════════════════════════ */ /* DotEnv: .env file → only Module node. */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 5ac761369..0ad711800 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -10002,8 +10002,8 @@ static int python_typed_field_scope_case(bool external) { "class TestCase:\n pass\n\n" "class Holder:\n field: %s\n", external ? "unittest.TestCase" : "TestCase"); - CBMFileResult *result = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, - "proj", "models.py", 0, NULL, NULL); + CBMFileResult *result = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "proj", + "models.py", 0, NULL, NULL); cbm_registry_t *reg = cbm_registry_new(); cbm_gbuf_t *gbuf = cbm_gbuf_new("proj", "/unused"); CBMArena arena; @@ -10019,11 +10019,11 @@ static int python_typed_field_scope_case(bool external) { const char **keys = NULL; const char **vals = NULL; int import_count = 0; - cbm_pxc_build_import_map(gbuf, "proj", "models.py", CBM_LANG_PYTHON, result, &keys, - &vals, &import_count); - cbm_resolution_t raw = cbm_registry_resolve( - reg, external ? "unittest.TestCase" : "TestCase", "proj.models", keys, vals, - import_count); + cbm_pxc_build_import_map(gbuf, "proj", "models.py", CBM_LANG_PYTHON, result, &keys, &vals, + &import_count); + cbm_resolution_t raw = + cbm_registry_resolve(reg, external ? "unittest.TestCase" : "TestCase", "proj.models", + keys, vals, import_count); bool candidate = raw.qualified_name && raw.strategy && strcmp(raw.qualified_name, "proj.models.TestCase") == 0 && strcmp(raw.strategy, "same_module") == 0; @@ -10032,18 +10032,18 @@ static int python_typed_field_scope_case(bool external) { cbm_file_info_t files[1] = {{.rel_path = "models.py", .language = CBM_LANG_PYTHON}}; CBMFileResult *cache[1] = {result}; int count = 0; - defs = cbm_pxc_collect_all_defs(&ctx, &arena, cache, files, 1, "proj", modules, &count, - NULL); + defs = + cbm_pxc_collect_all_defs(&ctx, &arena, cache, files, 1, "proj", modules, &count, NULL); int holders = 0; bool field_ok = false; for (int i = 0; defs && i < count; i++) { if (defs[i].qualified_name && strcmp(defs[i].qualified_name, "proj.models.Holder") == 0) { holders++; - field_ok = external ? !defs[i].field_defs - : defs[i].field_defs && - strcmp(defs[i].field_defs, - "field:proj.models.TestCase") == 0; + field_ok = external + ? !defs[i].field_defs + : defs[i].field_defs && + strcmp(defs[i].field_defs, "field:proj.models.TestCase") == 0; } } ok = candidate && holders == 1 && field_ok; @@ -10170,8 +10170,7 @@ TEST(override_explicit_walks_to_nearest_declaring_ancestor) { cbm_gbuf_insert_edge(gb, tied, tie_left_mid, "INHERITS", "{}"); /* The separate Go implicit-satisfaction pass owns .go classes. */ - int64_t go_type = - cbm_gbuf_upsert_node(gb, "Struct", "GoType", "m.GoType", "m.go", 1, 5, "{}"); + int64_t go_type = cbm_gbuf_upsert_node(gb, "Struct", "GoType", "m.GoType", "m.go", 1, 5, "{}"); int64_t go_process = ovr_method(gb, go_type, "GoType", "process"); cbm_gbuf_insert_edge(gb, go_type, processor, "IMPLEMENTS", "{}"); @@ -13637,49 +13636,6 @@ TEST(compile_commands_parse_invalid) { /* ── Infrascan: file identification ──────────────────────────────── */ -TEST(infra_is_compose_file) { - /* Port of TestIsComposeFile (8 cases) */ - ASSERT(cbm_is_compose_file("docker-compose.yml")); - ASSERT(cbm_is_compose_file("docker-compose.yaml")); - ASSERT(cbm_is_compose_file("docker-compose.prod.yml")); - ASSERT(cbm_is_compose_file("compose.yml")); - ASSERT(cbm_is_compose_file("compose.yaml")); - ASSERT(!cbm_is_compose_file("mycompose.yml")); - ASSERT(!cbm_is_compose_file("docker-compose.txt")); - ASSERT(!cbm_is_compose_file("Dockerfile")); - PASS(); -} - -TEST(infra_is_cloudbuild_file) { - /* Port of TestIsCloudbuildFile (5 cases) */ - ASSERT(cbm_is_cloudbuild_file("cloudbuild.yaml")); - ASSERT(cbm_is_cloudbuild_file("cloudbuild.yml")); - ASSERT(cbm_is_cloudbuild_file("cloudbuild-prod.yaml")); - ASSERT(cbm_is_cloudbuild_file("Cloudbuild.yml")); - ASSERT(!cbm_is_cloudbuild_file("build.yaml")); - PASS(); -} - -TEST(infra_is_shell_script) { - /* Port of TestIsShellScript (5 cases) */ - ASSERT(cbm_is_shell_script("run.sh", ".sh")); - ASSERT(cbm_is_shell_script("deploy.bash", ".bash")); - ASSERT(cbm_is_shell_script("init.zsh", ".zsh")); - ASSERT(!cbm_is_shell_script("main.py", ".py")); - ASSERT(!cbm_is_shell_script("Dockerfile", "")); - PASS(); -} - -TEST(infra_is_dockerfile) { - ASSERT(cbm_is_dockerfile("Dockerfile")); - ASSERT(cbm_is_dockerfile("dockerfile")); - ASSERT(cbm_is_dockerfile("Dockerfile.prod")); - ASSERT(cbm_is_dockerfile("app.dockerfile")); - ASSERT(!cbm_is_dockerfile("docker-compose.yml")); - ASSERT(!cbm_is_dockerfile("main.go")); - PASS(); -} - TEST(infra_is_kustomize_file) { ASSERT(cbm_is_kustomize_file("kustomization.yaml")); ASSERT(cbm_is_kustomize_file("kustomization.yml")); @@ -13704,36 +13660,6 @@ TEST(infra_is_k8s_manifest) { PASS(); } -TEST(infra_is_env_file) { - ASSERT(cbm_is_env_file(".env")); - ASSERT(cbm_is_env_file(".env.local")); - ASSERT(cbm_is_env_file("prod.env")); - ASSERT(!cbm_is_env_file("main.go")); - ASSERT(!cbm_is_env_file("env.txt")); - PASS(); -} - -/* ── Infrascan: cleanJSONBrackets ───────────────────────────────── */ - -TEST(infra_clean_json_brackets) { - /* Port of TestCleanJSONBrackets (4 cases) */ - char out[256]; - - cbm_clean_json_brackets("[\"./server\"]", out, sizeof(out)); - ASSERT_STR_EQ(out, "./server"); - - cbm_clean_json_brackets("[\"python\", \"main.py\"]", out, sizeof(out)); - ASSERT_STR_EQ(out, "python main.py"); - - cbm_clean_json_brackets("./server", out, sizeof(out)); - ASSERT_STR_EQ(out, "./server"); - - cbm_clean_json_brackets("[\"./app\", \"--flag\", \"value\"]", out, sizeof(out)); - ASSERT_STR_EQ(out, "./app --flag value"); - - PASS(); -} - /* ── Infrascan: secret detection ────────────────────────────────── */ TEST(infra_secret_detection) { @@ -13758,395 +13684,6 @@ TEST(infra_secret_detection) { PASS(); } -/* ── Infrascan: Dockerfile parser ───────────────────────────────── */ - -/* Helper: find env var by key in result */ -static const char *find_env_var(const cbm_env_kv_t *vars, int count, const char *key) { - for (int i = 0; i < count; i++) { - if (strcmp(vars[i].key, key) == 0) - return vars[i].value; - } - return NULL; -} - -/* Helper: check if string array contains value */ -static bool str_array_contains(const char (*arr)[32], int count, const char *val) { - for (int i = 0; i < count; i++) { - if (strcmp(arr[i], val) == 0) - return true; - } - return false; -} - -static bool str_array_128_contains(const char (*arr)[128], int count, const char *val) { - for (int i = 0; i < count; i++) { - if (strcmp(arr[i], val) == 0) - return true; - } - return false; -} - -static bool str_array_256_contains(const char (*arr)[256], int count, const char *val) { - for (int i = 0; i < count; i++) { - if (strcmp(arr[i], val) == 0) - return true; - } - return false; -} - -TEST(infra_parse_dockerfile_multistage) { - /* Port of TestParseDockerfile "multi-stage with all directives" */ - const char *src = "FROM golang:1.23-alpine AS builder\n" - "WORKDIR /app\n" - "ARG SSH_PRIVATE_KEY\n" - "RUN go build -o server .\n" - "\n" - "FROM alpine:3.19\n" - "WORKDIR /usr/app\n" - "ENV PORT=8080\n" - "ENV PYTHONUNBUFFERED=1\n" - "EXPOSE 8080 443\n" - "USER appuser\n" - "CMD [\"./server\"]\n" - "HEALTHCHECK CMD wget http://localhost:8080/health\n"; - - cbm_dockerfile_result_t r; - ASSERT_EQ(cbm_parse_dockerfile_source(src, &r), 0); - ASSERT_STR_EQ(r.base_image, "alpine:3.19"); - ASSERT_EQ(r.stage_count, 2); - ASSERT_STR_EQ(r.stage_images[0], "golang:1.23-alpine"); - ASSERT_STR_EQ(r.stage_images[1], "alpine:3.19"); - ASSERT(str_array_contains(r.exposed_ports, r.port_count, "8080")); - ASSERT(str_array_contains(r.exposed_ports, r.port_count, "443")); - ASSERT_STR_EQ(r.workdir, "/usr/app"); - ASSERT_STR_EQ(r.user, "appuser"); - ASSERT_STR_EQ(r.cmd, "./server"); - ASSERT_STR_EQ(r.healthcheck, "wget http://localhost:8080/health"); - - ASSERT_NOT_NULL(find_env_var(r.env_vars, r.env_count, "PORT")); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "PORT"), "8080"); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "PYTHONUNBUFFERED"), "1"); - - ASSERT(str_array_128_contains(r.build_args, r.build_arg_count, "SSH_PRIVATE_KEY")); - - PASS(); -} - -TEST(infra_parse_dockerfile_entrypoint) { - /* Port of TestParseDockerfile "single stage with entrypoint" */ - const char *src = "FROM python:3.9-slim\n" - "ENTRYPOINT [\"python\", \"main.py\"]\n"; - - cbm_dockerfile_result_t r; - ASSERT_EQ(cbm_parse_dockerfile_source(src, &r), 0); - ASSERT_STR_EQ(r.base_image, "python:3.9-slim"); - ASSERT_STR_EQ(r.entrypoint, "python main.py"); - ASSERT_EQ(r.stage_count, 1); - PASS(); -} - -TEST(infra_parse_dockerfile_secret_filtered) { - /* Port of TestParseDockerfile "secret env vars filtered" */ - const char *src = "FROM node:20\n" - "ENV API_KEY=sk-1234567890abcdef12345\n" - "ENV DATABASE_URL=https://db.example.com\n" - "ENV JWT_SECRET=supersecret\n"; - - cbm_dockerfile_result_t r; - ASSERT_EQ(cbm_parse_dockerfile_source(src, &r), 0); - - /* API_KEY and JWT_SECRET should be filtered */ - ASSERT(find_env_var(r.env_vars, r.env_count, "API_KEY") == NULL); - ASSERT(find_env_var(r.env_vars, r.env_count, "JWT_SECRET") == NULL); - - /* DATABASE_URL should remain */ - ASSERT_NOT_NULL(find_env_var(r.env_vars, r.env_count, "DATABASE_URL")); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "DATABASE_URL"), "https://db.example.com"); - PASS(); -} - -TEST(infra_parse_dockerfile_expose_protocol) { - /* Port of TestParseDockerfile "expose with protocol suffix" */ - const char *src = "FROM nginx:latest\n" - "EXPOSE 80/tcp 443/tcp\n"; - - cbm_dockerfile_result_t r; - ASSERT_EQ(cbm_parse_dockerfile_source(src, &r), 0); - ASSERT(str_array_contains(r.exposed_ports, r.port_count, "80")); - ASSERT(str_array_contains(r.exposed_ports, r.port_count, "443")); - PASS(); -} - -TEST(infra_parse_dockerfile_env_space) { - /* Port of TestParseDockerfile "ENV space-separated format" */ - const char *src = "FROM python:3.9\n" - "ENV PYTHONPATH /usr/app\n"; - - cbm_dockerfile_result_t r; - ASSERT_EQ(cbm_parse_dockerfile_source(src, &r), 0); - ASSERT_NOT_NULL(find_env_var(r.env_vars, r.env_count, "PYTHONPATH")); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "PYTHONPATH"), "/usr/app"); - PASS(); -} - -TEST(infra_parse_dockerfile_empty) { - /* Port of TestParseDockerfileEmpty */ - cbm_dockerfile_result_t r; - ASSERT_EQ(cbm_parse_dockerfile_source("# just a comment\n", &r), -1); - PASS(); -} - -/* ── Infrascan: Dotenv parser ───────────────────────────────────── */ - -TEST(infra_parse_dotenv) { - /* Port of TestParseDotenvFile */ - const char *src = "# Database config\n" - "DATABASE_HOST=localhost\n" - "DATABASE_PORT=5432\n" - "DATABASE_NAME=mydb\n" - "API_SECRET=should-not-appear\n" - "PLAIN_VALUE=hello world\n"; - - cbm_dotenv_result_t r; - ASSERT_EQ(cbm_parse_dotenv_source(src, &r), 0); - - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "DATABASE_HOST"), "localhost"); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "DATABASE_PORT"), "5432"); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "PLAIN_VALUE"), "hello world"); - - /* API_SECRET should be filtered */ - ASSERT(find_env_var(r.env_vars, r.env_count, "API_SECRET") == NULL); - PASS(); -} - -TEST(infra_parse_dotenv_quoted) { - /* Port of TestParseDotenvQuotedValues */ - const char *src = "KEY1=\"quoted value\"\n" - "KEY2='single quoted'\n"; - - cbm_dotenv_result_t r; - ASSERT_EQ(cbm_parse_dotenv_source(src, &r), 0); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "KEY1"), "quoted value"); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "KEY2"), "single quoted"); - PASS(); -} - -/* ── Infrascan: Shell script parser ─────────────────────────────── */ - -TEST(infra_parse_shell) { - /* Port of TestParseShellScript */ - const char *src = "#!/bin/bash\n" - "set -e\n" - "\n" - "# Configuration\n" - "YOUR_CONTAINER_NAME=\"order-email-extractor-endpoint\"\n" - "DOCKERFILE_PATH=\"/path/to/dockerfile\"\n" - "\n" - "export ENVIRONMENT=\"development\"\n" - "export USE_STACKDRIVER=\"false\"\n" - "\n" - "# Shut down existing containers\n" - "./shut-down-docker-container.sh\n" - "\n" - "docker build -t \"$YOUR_CONTAINER_NAME\" \"$DOCKERFILE_PATH\"\n" - "docker run -d --name \"$YOUR_CONTAINER_NAME\" \"$YOUR_CONTAINER_NAME\"\n" - "docker-compose up -d\n"; - - cbm_shell_result_t r; - ASSERT_EQ(cbm_parse_shell_source(src, &r), 0); - ASSERT_STR_EQ(r.shebang, "/bin/bash"); - - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "ENVIRONMENT"), "development"); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "YOUR_CONTAINER_NAME"), - "order-email-extractor-endpoint"); - - ASSERT(str_array_256_contains(r.docker_cmds, r.docker_cmd_count, "docker build")); - ASSERT(str_array_256_contains(r.docker_cmds, r.docker_cmd_count, "docker run")); - ASSERT(str_array_256_contains(r.docker_cmds, r.docker_cmd_count, "docker-compose up")); - - PASS(); -} - -TEST(infra_parse_shell_with_source) { - /* Port of TestParseShellScriptWithSource */ - const char *src = "#!/usr/bin/env bash\n" - "source ./config.sh\n" - ". /etc/profile.d/env.sh\n"; - - cbm_shell_result_t r; - ASSERT_EQ(cbm_parse_shell_source(src, &r), 0); - ASSERT_STR_EQ(r.shebang, "/usr/bin/env bash"); - ASSERT(str_array_256_contains(r.sources, r.source_count, "./config.sh")); - ASSERT(str_array_256_contains(r.sources, r.source_count, "/etc/profile.d/env.sh")); - PASS(); -} - -TEST(infra_parse_shell_secret_filtered) { - /* Port of TestParseShellScriptSecretFiltered */ - const char *src = "#!/bin/bash\n" - "export API_SECRET=\"should-not-appear\"\n" - "export DATABASE_URL=\"https://db.example.com\"\n"; - - cbm_shell_result_t r; - ASSERT_EQ(cbm_parse_shell_source(src, &r), 0); - ASSERT(find_env_var(r.env_vars, r.env_count, "API_SECRET") == NULL); - ASSERT_STR_EQ(find_env_var(r.env_vars, r.env_count, "DATABASE_URL"), "https://db.example.com"); - PASS(); -} - -TEST(infra_parse_shell_shebang_only) { - /* Port of TestParseShellScriptShebanOnly */ - const char *src = "#!/bin/bash\n# just comments\n"; - cbm_shell_result_t r; - ASSERT_EQ(cbm_parse_shell_source(src, &r), 0); - ASSERT_STR_EQ(r.shebang, "/bin/bash"); - PASS(); -} - -TEST(infra_parse_shell_truly_empty) { - /* Port of TestParseShellScriptTrulyEmpty */ - cbm_shell_result_t r; - ASSERT_EQ(cbm_parse_shell_source("# no shebang, just comments\n", &r), -1); - PASS(); -} - -/* ── Infrascan: Terraform parser ────────────────────────────────── */ - -TEST(infra_parse_terraform_full) { - /* Port of TestParseTerraformFile */ - const char *src = "\n" - "terraform {\n" - " required_providers {\n" - " google = {\n" - " source = \"hashicorp/google\"\n" - " version = \"~> 6.35.0\"\n" - " }\n" - " }\n" - " backend \"gcs\" {\n" - " bucket = \"example-tf\"\n" - " prefix = \"state\"\n" - " }\n" - "}\n" - "\n" - "variable \"project_id\" {\n" - " description = \"The GCP project ID\"\n" - " type = string\n" - " default = \"example-cloud\"\n" - "}\n" - "\n" - "variable \"region\" {\n" - " description = \"The region\"\n" - " type = string\n" - "}\n" - "\n" - "resource \"google_cloud_run_service\" \"main\" {\n" - " name = \"my-service\"\n" - " location = var.region\n" - "}\n" - "\n" - "resource \"google_compute_address\" \"nat_ip\" {\n" - " name = \"nat-ip\"\n" - " region = var.region\n" - "}\n" - "\n" - "output \"service_url\" {\n" - " value = google_cloud_run_service.main.status[0].url\n" - "}\n" - "\n" - "data \"google_project\" \"project\" {\n" - "}\n" - "\n" - "module \"vpc\" {\n" - " source = \"./modules/vpc\"\n" - "}\n" - "\n" - "locals {\n" - " env = \"prod\"\n" - "}\n"; - - cbm_terraform_result_t r; - ASSERT_EQ(cbm_parse_terraform_source(src, &r), 0); - ASSERT_STR_EQ(r.backend, "gcs"); - - /* Resources */ - ASSERT_EQ(r.resource_count, 2); - bool found_cloud_run = false; - for (int i = 0; i < r.resource_count; i++) { - if (strcmp(r.resources[i].type, "google_cloud_run_service") == 0 && - strcmp(r.resources[i].name, "main") == 0) { - found_cloud_run = true; - } - } - ASSERT(found_cloud_run); - - /* Variables */ - ASSERT_EQ(r.variable_count, 2); - bool found_project_id = false; - for (int i = 0; i < r.variable_count; i++) { - if (strcmp(r.variables[i].name, "project_id") == 0) { - ASSERT_STR_EQ(r.variables[i].default_val, "example-cloud"); - ASSERT_STR_EQ(r.variables[i].type, "string"); - ASSERT_STR_EQ(r.variables[i].description, "The GCP project ID"); - found_project_id = true; - } - } - ASSERT(found_project_id); - - /* Outputs */ - ASSERT_EQ(r.output_count, 1); - ASSERT_STR_EQ(r.outputs[0], "service_url"); - - /* Data sources */ - ASSERT_EQ(r.data_source_count, 1); - ASSERT_STR_EQ(r.data_sources[0].type, "google_project"); - ASSERT_STR_EQ(r.data_sources[0].name, "project"); - - /* Modules */ - ASSERT_EQ(r.module_count, 1); - ASSERT_STR_EQ(r.modules[0].tf_name, "vpc"); - ASSERT_STR_EQ(r.modules[0].source, "./modules/vpc"); - - /* Locals */ - ASSERT(r.has_locals); - - PASS(); -} - -TEST(infra_parse_terraform_variables_only) { - /* Port of TestParseTerraformVariablesOnly — secret default filtered */ - const char *src = "\n" - "variable \"project_id\" {\n" - " description = \"The GCP project ID\"\n" - " type = string\n" - " default = \"example-cloud\"\n" - "}\n" - "\n" - "variable \"secret_key\" {\n" - " description = \"A secret\"\n" - " type = string\n" - " default = \"sk-1234567890abcdef12345\"\n" - "}\n"; - - cbm_terraform_result_t r; - ASSERT_EQ(cbm_parse_terraform_source(src, &r), 0); - ASSERT_EQ(r.variable_count, 2); - - /* secret_key default should be filtered */ - for (int i = 0; i < r.variable_count; i++) { - if (strcmp(r.variables[i].name, "secret_key") == 0) { - ASSERT_STR_EQ(r.variables[i].default_val, ""); - } - } - PASS(); -} - -TEST(infra_parse_terraform_empty) { - /* Port of TestParseTerraformEmpty */ - cbm_terraform_result_t r; - ASSERT_EQ(cbm_parse_terraform_source("# just comments\n", &r), -1); - PASS(); -} - /* ── Helm Chart.yaml dependency parsing (#338) ──────────────────── */ TEST(helm_parse_chart_dependencies_issue338) { @@ -14673,50 +14210,6 @@ TEST(infra_qn_helper) { PASS(); } -/* ── Infrascan integration tests ────────────────────────────────── */ - -TEST(infra_pipeline_integration) { - /* Port of TestPassInfraFilesIntegration (Dockerfile + .env parts). - * Tests parse functions on source text (pipeline infrascan pass not - * wired yet — compose YAML also blocked on YAML parser). */ - - /* Parse Dockerfile */ - cbm_dockerfile_result_t dr; - ASSERT_EQ(cbm_parse_dockerfile_source("FROM alpine:3.19\nEXPOSE 8080\n", &dr), 0); - ASSERT_STR_EQ(dr.base_image, "alpine:3.19"); - ASSERT_GTE(dr.port_count, 1); - - /* Parse .env */ - cbm_dotenv_result_t er; - ASSERT_EQ(cbm_parse_dotenv_source("APP_PORT=8080\nDEBUG=true\n", &er), 0); - ASSERT_GTE(er.env_count, 1); - /* APP_PORT should be present */ - bool found_port = false; - for (int i = 0; i < er.env_count; i++) { - if (strcmp(er.env_vars[i].key, "APP_PORT") == 0 && - strcmp(er.env_vars[i].value, "8080") == 0) - found_port = true; - } - ASSERT_TRUE(found_port); - - PASS(); -} - -TEST(infra_pipeline_idempotent) { - /* Port of TestPassInfraFilesIdempotent: - * Parsing same source twice should produce identical results. */ - const char *src = "FROM alpine:3.19\nEXPOSE 8080\nENV PORT=8080\n"; - cbm_dockerfile_result_t r1, r2; - ASSERT_EQ(cbm_parse_dockerfile_source(src, &r1), 0); - ASSERT_EQ(cbm_parse_dockerfile_source(src, &r2), 0); - - ASSERT_STR_EQ(r1.base_image, r2.base_image); - ASSERT_EQ(r1.port_count, r2.port_count); - ASSERT_EQ(r1.env_count, r2.env_count); - - PASS(); -} - /* ── K8s / Kustomize extraction tests ──────────────────────────── */ TEST(k8s_extract_kustomize) { @@ -14818,7 +14311,7 @@ TEST(k8s_extract_manifest_multidoc) { PASS(); } -/* ── Envscan tests (port of envscan_test.go) ───────────────────── */ +/* ── Temp-file fixture helpers ───────────────────────────────────── */ /* Helper: write a file inside a temp dir */ static void write_temp_file(const char *dir, const char *name, const char *content) { @@ -15067,349 +14560,6 @@ TEST(pipeline_nested_fixture_graph_has_endpoints) { PASS(); } -/* Helper: find binding by key in results */ -static const cbm_env_binding_t *find_binding_by_key(const cbm_env_binding_t *bindings, int count, - const char *key) { - for (int i = 0; i < count; i++) { - if (strcmp(bindings[i].key, key) == 0) - return &bindings[i]; - } - return NULL; -} - -/* Helper: find binding by value in results */ -static int has_binding_value(const cbm_env_binding_t *bindings, int count, const char *value) { - for (int i = 0; i < count; i++) { - if (strcmp(bindings[i].value, value) == 0) - return 1; - } - return 0; -} - -TEST(envscan_dockerfile_env_urls) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_dock_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, "Dockerfile", - "FROM python:3.9-slim\n" - "ENV ORDER_URL=https://api.example.com/api/orders\n" - "ENV DB_HOST=localhost\n" - "ARG WEBHOOK_URL=https://hooks.example.com/webhook\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "ORDER_URL")); - ASSERT_STR_EQ(find_binding_by_key(bindings, count, "ORDER_URL")->value, - "https://api.example.com/api/orders"); - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "WEBHOOK_URL")); - ASSERT_STR_EQ(find_binding_by_key(bindings, count, "WEBHOOK_URL")->value, - "https://hooks.example.com/webhook"); - /* DB_HOST=localhost is NOT a URL → should be absent */ - ASSERT_TRUE(find_binding_by_key(bindings, count, "DB_HOST") == NULL); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_shell_env_urls) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_sh_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, "setup.sh", - "#!/bin/bash\n" - "export DB_URL=\"https://db.example.com/api/sync\"\n" - "APP_NAME=\"my-service\"\n" - "CALLBACK_URL=https://hooks.example.com/notify\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "DB_URL")); - ASSERT_STR_EQ(find_binding_by_key(bindings, count, "DB_URL")->value, - "https://db.example.com/api/sync"); - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "CALLBACK_URL")); - /* APP_NAME is NOT a URL → absent */ - ASSERT_TRUE(find_binding_by_key(bindings, count, "APP_NAME") == NULL); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_env_file_urls) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_env_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, ".env", - "\nAPI_URL=https://api.example.com/v1\n" - "DEBUG=true\n" - "SERVICE_URL=https://service.example.com/api\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "API_URL")); - ASSERT_STR_EQ(find_binding_by_key(bindings, count, "API_URL")->value, - "https://api.example.com/v1"); - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "SERVICE_URL")); - /* DEBUG=true is NOT a URL */ - ASSERT_TRUE(find_binding_by_key(bindings, count, "DEBUG") == NULL); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_toml_urls) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_toml_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, "config.toml", - "[service]\n" - "base_url = \"https://api.example.com\"\n" - "name = \"my-service\"\n" - "callback_url = \"https://hooks.example.com/notify\"\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "base_url")); - ASSERT_STR_EQ(find_binding_by_key(bindings, count, "base_url")->value, - "https://api.example.com"); - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "callback_url")); - /* name="my-service" is NOT a URL */ - ASSERT_TRUE(find_binding_by_key(bindings, count, "name") == NULL); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_yaml_urls) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_yaml_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, "config.yaml", - "service:\n" - " service_url: \"https://api.internal.com/api/process\"\n" - " timeout: 30\n" - " callback_url: \"https://hooks.internal.com/callback\"\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "service_url")); - ASSERT_STR_EQ(find_binding_by_key(bindings, count, "service_url")->value, - "https://api.internal.com/api/process"); - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "callback_url")); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_terraform_urls) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_tf_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, "variables.tf", - "variable \"webhook_url\" {\n" - " description = \"Webhook endpoint\"\n" - " default = \"https://api.example.com/webhook\"\n" - "}\n\n" - "variable \"region\" {\n" - " default = \"us-east-1\"\n" - "}\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - ASSERT_GTE(count, 1); - ASSERT_TRUE(has_binding_value(bindings, count, "https://api.example.com/webhook")); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_properties_urls) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_prop_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, "app.properties", - "api.url=https://api.example.com/health\n" - "app.name=myapp\n" - "service.endpoint=https://service.example.com/api\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - ASSERT_TRUE(has_binding_value(bindings, count, "https://api.example.com/health")); - ASSERT_TRUE(has_binding_value(bindings, count, "https://service.example.com/api")); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_secret_key_exclusion) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_skey_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, "Dockerfile", - "FROM node:18\n" - "ENV SECRET_TOKEN=https://api.example.com/api\n" - "ENV API_KEY=https://api.example.com/v1\n" - "ENV PASSWORD=https://auth.example.com/login\n" - "ENV NORMAL_URL=https://api.example.com/orders\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - /* Secret keys should be excluded */ - ASSERT_TRUE(find_binding_by_key(bindings, count, "SECRET_TOKEN") == NULL); - ASSERT_TRUE(find_binding_by_key(bindings, count, "API_KEY") == NULL); - ASSERT_TRUE(find_binding_by_key(bindings, count, "PASSWORD") == NULL); - /* Normal key should be present */ - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "NORMAL_URL")); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_secret_value_exclusion) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_sval_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file( - tmpdir, "deploy.sh", - "#!/bin/bash\n" - "export GH_URL=\"https://ghp_FAKEFAKEFAKEFAKEFAKEFAKEFAKEFAKEFAKE@github.com/repo\"\n" - "export NORMAL_ENDPOINT=\"https://api.example.com/orders\"\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - /* ghp_ token URL should be excluded */ - ASSERT_TRUE(find_binding_by_key(bindings, count, "GH_URL") == NULL); - /* Normal URL should be present */ - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "NORMAL_ENDPOINT")); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_secret_file_exclusion) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_sfile_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - /* Secret file should be skipped */ - write_temp_file(tmpdir, "credentials.sh", - "#!/bin/bash\nexport API_URL=\"https://api.example.com/v1\"\n"); - /* Normal file should be scanned */ - write_temp_file(tmpdir, "setup.sh", - "#!/bin/bash\nexport API_URL=\"https://api.example.com/v1\"\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - /* Should find binding from setup.sh but not credentials.sh */ - int from_credentials = 0; - int from_setup = 0; - for (int i = 0; i < count; i++) { - if (strcmp(bindings[i].file_path, "credentials.sh") == 0) - from_credentials = 1; - if (strcmp(bindings[i].file_path, "setup.sh") == 0) - from_setup = 1; - } - ASSERT_EQ(from_credentials, 0); - ASSERT_EQ(from_setup, 1); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_skips_ignored_dirs) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_ign_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - static const char ignored_content[] = - "#!/bin/bash\nexport API_URL=\"https://api.example.com/v1\"\n"; - write_temp_file(tmpdir, ".git/config.sh", ignored_content); - write_temp_file(tmpdir, "node_modules/pkg/config.sh", ignored_content); - bool git_written = fixture_file_matches(TH_PATH(tmpdir, ".git/config.sh"), ignored_content); - bool module_written = - fixture_file_matches(TH_PATH(tmpdir, "node_modules/pkg/config.sh"), ignored_content); - - /* File at root level should be scanned */ - write_temp_file(tmpdir, "deploy.sh", - "#!/bin/bash\nexport API_URL=\"https://api.example.com/v1\"\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - int from_git = 0, from_nm = 0, from_root = 0; - for (int i = 0; i < count; i++) { - if (strncmp(bindings[i].file_path, ".git/", 5) == 0) - from_git = 1; - if (strncmp(bindings[i].file_path, "node_modules/", 13) == 0) - from_nm = 1; - if (strcmp(bindings[i].file_path, "deploy.sh") == 0) - from_root = 1; - } - ASSERT_TRUE(git_written); - ASSERT_TRUE(module_written); - ASSERT_EQ(from_git, 0); - ASSERT_EQ(from_nm, 0); - ASSERT_EQ(from_root, 1); - - th_rmtree(tmpdir); - PASS(); -} - -TEST(envscan_non_url_values_skipped) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_nurl_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - write_temp_file(tmpdir, "Dockerfile", - "FROM python:3.9\n" - "ENV APP_NAME=my-service\n" - "ENV PORT=8080\n" - "ENV DEBUG=true\n" - "ENV LOG_LEVEL=info\n"); - write_temp_file(tmpdir, "config.sh", - "#!/bin/bash\n" - "export REGION=\"us-east-1\"\n" - "export COUNT=42\n"); - - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - - ASSERT_EQ(count, 0); - - th_rmtree(tmpdir); - PASS(); -} - /* ── Discovery-exclusion plumbing in auxiliary repo walks (#792) ── */ /* Boundary semantics of the shared exclusion predicate: anchored at the @@ -15755,40 +14905,6 @@ TEST(pkgmap_scan_repo_honors_discovery_exclusions) { PASS(); } -/* The env-URL walk must honor discovery exclusions the same way (#792). - * Control run first via the NULL-exclusion wrapper so the exclusion - * assertion cannot pass vacuously. */ -TEST(envscan_walk_honors_discovery_exclusions) { - char tmpdir[256]; - snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_envscan_excl_XXXXXX"); - if (!cbm_mkdtemp(tmpdir)) - FAIL("tmpdir"); - - char dir[512]; - snprintf(dir, sizeof(dir), "%s/big_generated", tmpdir); - cbm_mkdir(dir); - - write_temp_file(tmpdir, "deploy.sh", - "#!/bin/bash\nexport CONTROL_URL=\"https://api.example.com/v1\"\n"); - write_temp_file(tmpdir, "big_generated/env.sh", - "#!/bin/bash\nexport EXCLUDED_URL=\"https://excluded.example.com/v1\"\n"); - - /* Control: the NULL-exclusion wrapper sees both bindings. */ - cbm_env_binding_t bindings[32]; - int count = cbm_scan_project_env_urls(tmpdir, bindings, 32); - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "CONTROL_URL")); - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "EXCLUDED_URL")); - - /* With big_generated excluded, its binding must disappear. */ - char *excluded[] = {(char *)"big_generated"}; - count = cbm_scan_project_env_urls_excluded(tmpdir, bindings, 32, excluded, 1); - ASSERT_NOT_NULL(find_binding_by_key(bindings, count, "CONTROL_URL")); - ASSERT_TRUE(find_binding_by_key(bindings, count, "EXCLUDED_URL") == NULL); - - th_rmtree(tmpdir); - PASS(); -} - /* ── Git history tests (port of githistory_test.go) ────────────── */ /* Port of Go TestIsTrackableFile from githistory_test.go */ @@ -17544,31 +16660,6 @@ TEST(coupling_single_file_commit) { PASS(); } -TEST(clean_json_brackets_array) { - /* JSON array → space-joined */ - char out[256]; - cbm_clean_json_brackets("[\"./app\", \"--flag\"]", out, sizeof(out)); - ASSERT_TRUE(strstr(out, "./app") != NULL); - ASSERT_TRUE(strstr(out, "--flag") != NULL); - PASS(); -} - -TEST(clean_json_brackets_plain) { - /* Plain string → unchanged */ - char out[256]; - cbm_clean_json_brackets("./app --flag", out, sizeof(out)); - ASSERT_STR_EQ(out, "./app --flag"); - PASS(); -} - -TEST(clean_json_brackets_empty) { - /* Empty string → empty output */ - char out[256]; - cbm_clean_json_brackets("", out, sizeof(out)); - ASSERT_STR_EQ(out, ""); - PASS(); -} - TEST(fqn_compute_basic) { /* Basic FQN: project.dir.name */ char *fqn = cbm_pipeline_fqn_compute("proj", "pkg/handler.go", "Serve"); @@ -19718,62 +18809,20 @@ SUITE(pipeline) { RUN_TEST(compile_commands_parse_empty); RUN_TEST(compile_commands_parse_invalid); /* Infrascan helpers */ - RUN_TEST(infra_is_compose_file); - RUN_TEST(infra_is_cloudbuild_file); - RUN_TEST(infra_is_shell_script); - RUN_TEST(infra_is_dockerfile); RUN_TEST(infra_is_kustomize_file); RUN_TEST(infra_is_k8s_manifest); - RUN_TEST(infra_is_env_file); - RUN_TEST(infra_clean_json_brackets); RUN_TEST(infra_secret_detection); - /* Infrascan: Dockerfile parser */ - RUN_TEST(infra_parse_dockerfile_multistage); - RUN_TEST(infra_parse_dockerfile_entrypoint); - RUN_TEST(infra_parse_dockerfile_secret_filtered); - RUN_TEST(infra_parse_dockerfile_expose_protocol); - RUN_TEST(infra_parse_dockerfile_env_space); - RUN_TEST(infra_parse_dockerfile_empty); - /* Infrascan: Dotenv parser */ - RUN_TEST(infra_parse_dotenv); - RUN_TEST(infra_parse_dotenv_quoted); - /* Infrascan: Shell script parser */ - RUN_TEST(infra_parse_shell); - RUN_TEST(infra_parse_shell_with_source); - RUN_TEST(infra_parse_shell_secret_filtered); - RUN_TEST(infra_parse_shell_shebang_only); - RUN_TEST(infra_parse_shell_truly_empty); - /* Infrascan: Terraform parser */ - RUN_TEST(infra_parse_terraform_full); - RUN_TEST(infra_parse_terraform_variables_only); - RUN_TEST(infra_parse_terraform_empty); RUN_TEST(helm_parse_chart_dependencies_issue338); RUN_TEST(helm_parse_chart_no_deps_issue338); RUN_TEST(helm_parse_chart_dependency_entries); RUN_TEST(helm_parse_chart_many_dependencies); /* Infrascan: QN helper */ RUN_TEST(infra_qn_helper); - /* Infrascan: pipeline integration */ - RUN_TEST(infra_pipeline_integration); - RUN_TEST(infra_pipeline_idempotent); /* K8s / Kustomize extraction */ RUN_TEST(k8s_extract_kustomize); RUN_TEST(k8s_extract_manifest); RUN_TEST(k8s_extract_manifest_no_name); RUN_TEST(k8s_extract_manifest_multidoc); - /* Env URL scanning */ - RUN_TEST(envscan_dockerfile_env_urls); - RUN_TEST(envscan_shell_env_urls); - RUN_TEST(envscan_env_file_urls); - RUN_TEST(envscan_toml_urls); - RUN_TEST(envscan_yaml_urls); - RUN_TEST(envscan_terraform_urls); - RUN_TEST(envscan_properties_urls); - RUN_TEST(envscan_secret_key_exclusion); - RUN_TEST(envscan_secret_value_exclusion); - RUN_TEST(envscan_secret_file_exclusion); - RUN_TEST(envscan_skips_ignored_dirs); - RUN_TEST(envscan_non_url_values_skipped); /* SwiftPM Package.swift manifest resolution (issue #551 item 1) */ RUN_TEST(pkgmap_swift_targets_registers_module); RUN_TEST(pkgmap_swift_products_do_not_register_alias); @@ -19788,7 +18837,6 @@ SUITE(pipeline) { /* Discovery-exclusion plumbing in auxiliary repo walks (#792) */ RUN_TEST(pipeline_relpath_excluded_boundary); RUN_TEST(pkgmap_scan_repo_honors_discovery_exclusions); - RUN_TEST(envscan_walk_honors_discovery_exclusions); /* Function registry / resolver */ RUN_TEST(registry_resolve_single_candidate); RUN_TEST(registry_fuzzy_nonexistent); @@ -19889,9 +18937,6 @@ SUITE(pipeline) { RUN_TEST(coupling_empty_commits); RUN_TEST(coupling_single_file_commit); /* JSON bracket cleaning */ - RUN_TEST(clean_json_brackets_array); - RUN_TEST(clean_json_brackets_plain); - RUN_TEST(clean_json_brackets_empty); /* FQN computation */ RUN_TEST(fqn_compute_basic); RUN_TEST(fqn_compute_strips_ext); diff --git a/tests/test_platform.c b/tests/test_platform.c index 7545ea6c5..d1434c2ee 100644 --- a/tests/test_platform.c +++ b/tests/test_platform.c @@ -1272,7 +1272,7 @@ TEST(regex_compile_accepts_ordinary_intervals) { "(foo|bar|baz){2,5}", "^[A-Za-z_][A-Za-z0-9_]{0,255}$", "[0-9]{4}-[0-9]{2}-[0-9]{2}", - /* pass_calls.c and pass_envscan.c */ + /* pass_calls.c, and the retired env-URL scanner */ "Depends\\(([A-Za-z_][A-Za-z0-9_.]*)", "^(ENV|ARG)[[:space:]]+([A-Za-z0-9_]+)[= ](.*)", "(default|value)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", From 48278d5117ad4227434888ce7ecfae0fcaadff3b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 15:51:09 +0200 Subject: [PATCH 13/52] fix(infra): new Helm and kustomize code allocates through the memory core The Helm file:// dependency records and the exact kustomize entry resolver added raw strdup/free sites, so pass_k8s.c went from 6 to 9 raw allocator sites and pass_pkgmap.c from 82 to 85 -- the memory-core ratchet (make lint-memory-core, part of lint-ci) fails on both files. pass_k8s.c: a dependency record now holds core-owned copies (class other) of its chart path and target QN. The QN helper copies the libc string fqn.c hands out into the core and releases it in one place, and the record array grows before anything is allocated, so the failure path frees nothing. 9 -> 6, the baseline. pass_pkgmap.c: the resolver's three frees release fqn.c's libc-allocated QNs and paths and have to stay raw. The manifest read buffer, private to this file, now comes from the core (class extract: per-file source text) and its three callers release it there. 85 -> 81; the baseline drops from 82 to 81 in this change. No graph change: kubermatic and kubernetes/k8s.io indexes are identical to the previous build node for node and edge for edge, deckhouse in every edge and node identity. Its remaining difference, the order of the variants list on QN-colliding Go methods, also appears between two runs of the same binary and is not caused here. Signed-off-by: Martin Vogel --- scripts/memory-core-baseline.txt | 2 +- src/pipeline/pass_k8s.c | 36 +++++++++++++++++++------------- src/pipeline/pass_pkgmap.c | 11 +++++----- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 32c4a89db..b277da976 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -70,7 +70,7 @@ src/pipeline/pass_infrascan.c 2 src/pipeline/pass_k8s.c 6 src/pipeline/pass_lsp_cross.c 49 src/pipeline/pass_parallel.c 31 -src/pipeline/pass_pkgmap.c 82 +src/pipeline/pass_pkgmap.c 81 src/pipeline/pass_semantic.c 3 src/pipeline/pass_tests.c 3 src/pipeline/pass_usages.c 4 diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 34a6aeb76..040cb1688 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -646,39 +646,47 @@ static bool k8s_external_dep(cbm_pipeline_ctx_t *ctx, int64_t chart_id, const ch return true; } +/* QN of the chart a relative `file://` directory names, core-owned; NULL for + * an absolute path (it names the build host, not this repo). */ +static char *k8s_local_chart_qn(const char *project, const char *chart_rel, const char *dir) { + if (!dir[0] || dir[0] == '/') { + return NULL; + } + char rel[CBM_HELM_REPO_MAX + K8S_SLEN("./") + K8S_SLEN("/Chart.yaml")]; + snprintf(rel, sizeof(rel), "%s%s/Chart.yaml", strncmp(dir, "../", 3) == 0 ? "" : "./", dir); + /* fqn.c hands out libc-allocated strings: copy into the core, release them. */ + char *path = cbm_pipeline_resolve_relative_import(chart_rel, rel); + char *qn = path ? cbm_infra_qn(project, path, "helm-chart", NULL) : NULL; + free(path); + char *owned = qn ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, qn) : NULL; + free(qn); + return owned; +} + static void k8s_local_deps_push(k8s_local_deps_t *deps, int64_t chart_id, const char *chart_rel, const char *project, const cbm_helm_dep_t *dep) { - const char *dir = dep->repository + K8S_SLEN("file://"); - char *target_qn = NULL; - if (dir[0] && dir[0] != '/') { /* an absolute path names the build host, not this repo */ - char rel[CBM_HELM_REPO_MAX + K8S_SLEN("./") + K8S_SLEN("/Chart.yaml")]; - snprintf(rel, sizeof(rel), "%s%s/Chart.yaml", strncmp(dir, "../", 3) == 0 ? "" : "./", dir); - char *path = cbm_pipeline_resolve_relative_import(chart_rel, rel); - target_qn = path ? cbm_infra_qn(project, path, "helm-chart", NULL) : NULL; - free(path); - } if (deps->count == deps->cap) { int cap = deps->cap ? deps->cap * 2 : K8S_GROW_MIN; k8s_local_dep_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, deps->items, (size_t)cap * sizeof(*grown)); if (!grown) { - free(target_qn); return; } deps->items = grown; deps->cap = cap; } k8s_local_dep_t *d = &deps->items[deps->count++]; - *d = (k8s_local_dep_t){.chart_id = chart_id, .target_qn = target_qn}; + *d = (k8s_local_dep_t){.chart_id = chart_id}; + d->target_qn = k8s_local_chart_qn(project, chart_rel, dep->repository + K8S_SLEN("file://")); snprintf(d->name, sizeof(d->name), "%s", dep->name); - d->chart_rel = strdup(chart_rel); + d->chart_rel = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, chart_rel); d->props = k8s_dep_props(dep->repository); } static void k8s_local_deps_free(k8s_local_deps_t *deps) { for (int i = 0; i < deps->count; i++) { - free(deps->items[i].target_qn); - free(deps->items[i].chart_rel); + cbm_free(CBM_MEM_CLASS_OTHER, deps->items[i].target_qn); + cbm_free(CBM_MEM_CLASS_OTHER, deps->items[i].chart_rel); cbm_free(CBM_MEM_CLASS_OTHER, deps->items[i].props); } cbm_free(CBM_MEM_CLASS_OTHER, deps->items); diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index a2014c5d3..f2a451f89 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -31,7 +31,8 @@ #include #include -/* Read an entire file into a malloc'd buffer. Returns NULL on failure. */ +/* Read an entire file into a core-owned buffer (class extract; release with + * cbm_free(CBM_MEM_CLASS_EXTRACT, ...)). Returns NULL on failure. */ static char *pkgmap_read_file(const char *path, int *out_len) { FILE *f = cbm_fopen(path, "rb"); if (!f) { @@ -44,7 +45,7 @@ static char *pkgmap_read_file(const char *path, int *out_len) { (void)fclose(f); return NULL; } - char *buf = (char *)malloc((size_t)size + SKIP_ONE); + char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)size + SKIP_ONE); if (!buf) { (void)fclose(f); return NULL; @@ -1201,7 +1202,7 @@ static int pkgmap_walk_dir(const char *abs_dir, const char *rel_dir, cbm_pkg_ent if (cbm_pkgmap_try_parse(name, rel_path, source, source_len, entries)) { parsed++; } - free(source); + cbm_free(CBM_MEM_CLASS_EXTRACT, source); } cbm_closedir(dir); return parsed; @@ -1247,7 +1248,7 @@ CBMHashTable *cbm_pkgmap_build_from_files(const cbm_file_info_t *files, int file continue; } cbm_pkgmap_try_parse(basename, files[i].rel_path, source, source_len, &entries); - free(source); + cbm_free(CBM_MEM_CLASS_EXTRACT, source); } CBMHashTable *map = cbm_pkgmap_build(&entries, SKIP_ONE, project_name); @@ -1282,7 +1283,7 @@ CBMHashTable *cbm_pkgmap_build_from_repo(const char *repo_path, const cbm_file_i continue; } cbm_pkgmap_try_parse(basename, files[i].rel_path, source, source_len, &entries); - free(source); + cbm_free(CBM_MEM_CLASS_EXTRACT, source); } int from_walk = cbm_pkgmap_scan_repo(repo_path, &entries, excluded_dirs, excluded_count); From 610906d2f7b4563411a15bd51194f8bad40fb407 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 15:59:40 +0200 Subject: [PATCH 14/52] docs: list the node labels and edge types the index actually produces The README's Graph Data Model named USES_TYPE, MEMBER_OF and CONTAINS_PACKAGE, which nothing emits, and left out 14 labels and 18 edge types that real indexes hold -- among them the infrastructure ones (Resource is listed, but not Chart, Process, EnvVar, INFRA_MAPS or DEPENDS_ON) and Variable, Section, Field, Struct, OVERRIDE, INHERITS, TESTS_FILE and the CROSS_* family. The skill text `install` writes for agents carried the same CONTAINS_PACKAGE and no infra edges, so an agent following it queried an edge that never exists and did not learn the ones that do. Both lists now name exactly what is produced: checked against the labels and edge types of 17 infrastructure repositories and 86 local indexes, plus a quoted producer in the sources for the types those indexes do not exercise (Trait, Constant, DATA_FLOWS, CROSS_*). Against the same data the old README had 29 mismatches, the new one none. The edge summary also describes INFRA_MAPS (Service -> the workloads its whole selector selects, one namespace) and DEPENDS_ON (Helm chart -> declared charts with their repository; go.mod / requirements.txt -> external Package), and the infrastructure bullet names Helm charts and Terraform / OpenTofu files. Signed-off-by: Martin Vogel --- README.md | 8 +++++--- src/cli/cli.c | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index acf836f66..8b439385a 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,8 @@ The install script placed beside the binary is **reported, not deleted** — uni - `IMPORTS`, `DEFINES`, `IMPLEMENTS`, `INHERITS` - `HTTP_CALLS`, `ASYNC_CALLS` (cross-service) - `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, ...); the target is a `Process` node named after the program, or `` when the program is not a literal +- `INFRA_MAPS` — a Kubernetes Service to each workload whose pod labels carry every `key: value` of the Service's selector, within one namespace +- `DEPENDS_ON` — a Helm chart to the charts it declares (the local chart for a resolvable `file://` repository, otherwise an external `Chart`; the edge keeps `repository`), and a `go.mod` / `requirements.txt` file to an external `Package` - `EMITS`, `LISTENS_ON` (channels) - `DATA_FLOWS` with arg-to-param mapping + field access chains - `SIMILAR_TO` (MinHash + LSH near-clone detection, Jaccard scored) @@ -242,7 +244,7 @@ The install script placed beside the binary is **reported, not deleted** — uni ### Indexing pipeline - **158 vendored tree-sitter grammars** compiled into the binary - **Generic package / module resolution** — bare specifiers like `@myorg/pkg`, `github.com/foo/bar`, `use my_crate::foo` resolved via manifest scanning (`package.json`, `go.mod`, `Cargo.toml`, `pyproject.toml`, `composer.json`, `pubspec.yaml`, `pom.xml`, `build.gradle`, `mix.exs`, `*.gemspec`) -- **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests, Kustomize overlays as graph nodes +- **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests (`Resource`), Kustomize overlays (`IMPORTS` to their entries), Helm charts (`Chart`, `DEPENDS_ON`), Terraform / OpenTofu (`.tf`, `.tofu`, `.tfvars`) as graph nodes - **[Hybrid LSP semantic type resolution](#hybrid-lsp)** for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — a lightweight C implementation of language type-resolution algorithms, structurally inspired by and compatible with major language servers including tsserver / typescript-go, pyright, gopls, Roslyn, Eclipse JDT, and rust-analyzer (parameter binding, return-type inference, generic substitution, JSX component dispatch, JSDoc inference for plain JS files, namespace + trait + late-static-binding resolution for PHP, file-scoped namespaces + records + LINQ method syntax for C#, class-hierarchy + overload + lambda resolution for Java, extension-function + scope-function resolution for Kotlin, trait-method + UFCS resolution for Rust) - **RAM-first pipeline**: LZ4 compression, in-memory SQLite, single dump at end. Memory released after. @@ -741,11 +743,11 @@ mode. `index_status` keeps describing the published graph and its freshness. ### Node Labels -`Project`, `Package`, `Folder`, `File`, `Module`, `Class`, `Function`, `Method`, `Interface`, `Enum`, `Type`, `Route`, `Resource` +`Project`, `Package`, `Folder`, `File`, `Module`, `Section`, `Class`, `Struct`, `Interface`, `Trait`, `Enum`, `Type`, `Function`, `Method`, `Field`, `Variable`, `Constant`, `Macro`, `Decorator`, `Route`, `Channel`, `Table`, `Process`, `EnvVar`, `Resource`, `Chart`, `Branch` ### Edge Types -`CONTAINS_PACKAGE`, `CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `HTTP_CALLS`, `ASYNC_CALLS`, `SPAWNS`, `IMPLEMENTS`, `HANDLES`, `USAGE`, `CONFIGURES`, `REFERENCES_FILE`, `MENTIONS`, `WRITES`, `MEMBER_OF`, `TESTS`, `USES_TYPE`, `FILE_CHANGES_WITH` +`CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `USAGE`, `INHERITS`, `IMPLEMENTS`, `OVERRIDE`, `DECORATES`, `RAISES`, `THROWS`, `WRITES`, `DATA_FLOWS`, `HTTP_CALLS`, `GRPC_CALLS`, `ASYNC_CALLS`, `EMITS`, `LISTENS_ON`, `HANDLES`, `SPAWNS`, `CONFIGURES`, `INFRA_MAPS`, `DEPENDS_ON`, `REFERENCES_FILE`, `MENTIONS`, `TESTS`, `TESTS_FILE`, `SIMILAR_TO`, `SEMANTICALLY_RELATED`, `FILE_CHANGES_WITH`, `HAS_BRANCH`, and the cross-repo `CROSS_HTTP_CALLS`, `CROSS_GRPC_CALLS`, `CROSS_GRAPHQL_CALLS`, `CROSS_TRPC_CALLS`, `CROSS_ASYNC_CALLS`, `CROSS_CHANNEL` `MENTIONS` links a documented definition to the code its doc comment references (C# ``, ``, ``, ``), one edge per pair with `via`, `syntax`, `tier` (`exact` or `unique`), first `line` and `count`. A reference is bound only when it names exactly one definition under the language's own lookup rules; the rest are counted by reason in `index_status` under `doc_links` (samples with `diagnostics='full'`). diff --git a/src/cli/cli.c b/src/cli/cli.c index c99034847..b9cb25aff 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1567,7 +1567,8 @@ static const char skill_content[] = "CALLS, HTTP_CALLS, ASYNC_CALLS, SPAWNS, DATA_FLOWS, IMPORTS, DEFINES, DEFINES_METHOD,\n" "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, REFERENCES_FILE,\n" "MENTIONS (doc comment -> referenced code), FILE_CHANGES_WITH, SIMILAR_TO,\n" - "SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE\n" + "SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n" + "INFRA_MAPS (K8s Service -> selected workloads), DEPENDS_ON (chart / manifest -> dependency)\n" "\n" "## Cypher Examples (for query_graph)\n" "```\n" From f08641ee51695840351f1dd973c7d33031b74642 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 16:23:54 +0200 Subject: [PATCH 15/52] fix(k8s): a comment ahead of a YAML value no longer hides it The K8s/Kustomize extractor took a mapping pair's value as the pair's second named child, and a block node's content as its first. A comment can be either: one trailing the key line (`metadata: # owned by ...`) sits in the pair ahead of the value, and a commented-out first entry (`resources:` then ` # - argocd.yaml ...` then ` - atlantis.yaml`) is the block node's first named child, ahead of the sequence. The whole list, or the object's metadata and with it its name, was lost. unwrap_pair_value now skips comments on both levels, and every value pick in the extractor goes through it: the kustomize list keys, kind, metadata and metadata.name. kubernetes/k8s.io's kubernetes/apps/kustomization.yaml starts its resources with a commented-out entry; its 13 Argo CD applications had no IMPORTS edge. Measured on all 17 infrastructure corpora against the previous build: k8s.io gains exactly those 13 edges (kustomize entries linked 107 -> 120 of 123), every other graph is identical except the known run-to-run order of `variants` on deckhouse's QN-colliding Go methods. Of 866 linkable local kustomize entries in the corpora, 862 are now linked. Three probes fail without the fix: entries after a commented-out entry, entries after a key-line comment, and a Deployment whose `metadata:` line carries a comment. Signed-off-by: Martin Vogel --- internal/cbm/extract_k8s.c | 46 ++++++++++++------------- tests/test_grammar_probe_f.c | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 23 deletions(-) diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index bad926e8e..88f8a46f8 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -134,6 +134,27 @@ static TSNode unwrap_block_mapping(TSNode doc_child) { return mapping; } +// Unwrap a block_mapping_pair value through optional block_node. Comments are +// skipped on both levels: one trailing the key line (`metadata: # note`) is a +// named child of the pair ahead of its value, and a commented-out first entry +// (`resources:` / ` # - a.yaml` / ` - b.yaml`) is the block_node's first +// named child, ahead of the sequence. +static TSNode unwrap_pair_value(TSNode pair) { + TSNode val_node = {0}; + uint32_t n = ts_node_named_child_count(pair); + for (uint32_t i = SKIP_ONE; i < n; i++) { + TSNode child = ts_node_named_child(pair, i); + if (strcmp(ts_node_type(child), "comment") != 0) { + val_node = child; + break; + } + } + if (!ts_node_is_null(val_node) && strcmp(ts_node_type(val_node), "block_node") == 0) { + val_node = first_content_child(val_node); + } + return val_node; +} + // Process a single block_mapping_pair for kustomize list keys. static void process_kustomize_pair(CBMExtractCtx *ctx, TSNode pair) { if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { @@ -148,22 +169,13 @@ static void process_kustomize_pair(CBMExtractCtx *ctx, TSNode pair) { return; } - TSNode val_node = ts_node_named_child(pair, SKIP_ONE); - if (ts_node_is_null(val_node)) { - return; - } - if (strcmp(ts_node_type(val_node), "block_node") == 0) { - val_node = ts_node_named_child(val_node, 0); - } + TSNode val_node = unwrap_pair_value(pair); if (ts_node_is_null(val_node) || strcmp(ts_node_type(val_node), "block_sequence") != 0) { return; } emit_kustomize_sequence(ctx, val_node, key_text); } -// Forward declaration: defined with the K8s-manifest helpers below. -static TSNode unwrap_pair_value(TSNode pair); - // Emit a "Class" def named after the document's `kind` scalar. A kustomization // file has no metadata.name, so the def name is the bare kind ("Kustomization"). // Mirrors the K8s manifest kind-def so Kustomize resources are also discoverable. @@ -249,7 +261,7 @@ static void extract_metadata_name(CBMArena *a, TSNode meta_mapping, const char * if (!mkey_text || strcmp(mkey_text, "name") != 0) { continue; } - TSNode mval = ts_node_named_child(mpair, SKIP_ONE); + TSNode mval = unwrap_pair_value(mpair); if (ts_node_is_null(mval)) { continue; } @@ -260,18 +272,6 @@ static void extract_metadata_name(CBMArena *a, TSNode meta_mapping, const char * } } -// Unwrap a block_mapping_pair value through optional block_node. -static TSNode unwrap_pair_value(TSNode pair) { - TSNode val_node = ts_node_named_child(pair, SKIP_ONE); - if (ts_node_is_null(val_node)) { - return val_node; - } - if (strcmp(ts_node_type(val_node), "block_node") == 0) { - val_node = ts_node_named_child(val_node, 0); - } - return val_node; -} - // Descend into the first block_mapping of a document and extract // kind and metadata.name. Returns void; fills kind_buf and meta_name_buf. static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, char *kind_buf, size_t kind_sz, diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 5995b8f7e..37638e192 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -747,6 +747,21 @@ TEST(probe_k8s_manifest_after_leading_comment) { PASS(); } +/* A comment between a key and its block (`metadata: # owned by ...`) sits + * ahead of the value; the object still has its name. */ +TEST(probe_k8s_manifest_metadata_after_key_line_comment) { + GpfMetrics m = gpf_metrics("deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata: # owned by the platform team\n" + " name: web\n" + "spec:\n" + " replicas: 1\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 1); + PASS(); +} + /* The same comment in front of a kustomization hid its resource list: * home-ops resolved 1 of 348 entries. */ TEST(probe_kustomize_imports_after_leading_comment) { @@ -770,6 +785,55 @@ TEST(probe_kustomize_imports_after_leading_comment) { PASS(); } +/* A commented-out entry above the first item (kubernetes/k8s.io's + * kubernetes/apps: `# - argocd.yaml ...`) is a comment ahead of the list; + * the entries below it are still the list. */ +TEST(probe_kustomize_entries_after_commented_out_entry) { + static const GpfFile files[] = { + {"kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " # - argocd.yaml applied by hand\n" + " - atlantis.yaml\n" + " - datadog.yaml\n"}, + {"atlantis.yaml", + "apiVersion: v1\n" + "kind: Namespace\n" + "metadata:\n" + " name: atlantis\n"}, + {"datadog.yaml", + "apiVersion: v1\n" + "kind: Namespace\n" + "metadata:\n" + " name: datadog\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 3); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.imports, 2); + PASS(); +} + +/* Same with the comment trailing the key line. */ +TEST(probe_kustomize_entries_after_key_line_comment) { + static const GpfFile files[] = { + {"kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources: # applied by Argo CD\n" + " - atlantis.yaml\n"}, + {"atlantis.yaml", + "apiVersion: v1\n" + "kind: Namespace\n" + "metadata:\n" + " name: atlantis\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.imports, 1); + PASS(); +} + /* kustomize also reads an extensionless file named `Kustomization` * (kubermatic's addons use it); it was never discovered as a kustomization. */ TEST(probe_kustomize_extensionless_file) { @@ -2089,6 +2153,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_helm_unresolved_file_dependency_stays_external); RUN_TEST(probe_kustomize_nested_sibling_entries); RUN_TEST(probe_kustomize_single_module); + RUN_TEST(probe_kustomize_entries_after_commented_out_entry); + RUN_TEST(probe_kustomize_entries_after_key_line_comment); + RUN_TEST(probe_k8s_manifest_metadata_after_key_line_comment); /* Nix */ RUN_TEST(probe_nix_attrset_node); From 59ff283e5fcf77f6831d8d11181628a57a34ed97 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 17:19:41 +0200 Subject: [PATCH 16/52] fix(graph): a node's variant spans no longer depend on worker order Two runs of the same binary over deckhouse disagreed on the `variants` property of QN-colliding nodes. Two causes: - A cap: at most 64 spans per node. Generated code exceeds it (protobuf `String()`/`Reset()`, `DeepCopy` across a package, 7,586 junit `testcase` elements in one file), and which 64 survived followed the merge order of the parallel workers. - A label gate: spans were recorded only between definitions of the same label. A Function arriving between same-QN Methods (Go method QNs carry no receiver: pkg/log has func Error and three Error methods) replaced the node's list, so logger.go's span came and went with scheduling. The node now carries every definition span that collapsed into it, whatever the labels and however many: the list is the union of the arrivals, a function of the set, never of its order. That is also what the list is for -- a change in any of those bodies lands on this node. Lifting the cap made rewriting the JSON list on every collision quadratic in the collisions of one QN (deckhouse +60% wall time), so the spans are now gathered in a side table and written once, when the buffer is exported: dump, flush, store merge and the incremental delta patch. A worker buffer hands its spans to the main buffer on merge instead of writing them, so every list is written at the same point of the pipeline whichever worker held the colliding definitions, and the rewrite keeps the number text other passes printed. On the 17 infrastructure corpora: three runs per corpus are identical node for node and edge for edge (before: deckhouse differed on 5-8 nodes per pair of runs); against the previous build every edge and node identity is unchanged, 380 lists gained the spans the cap or the label gate dropped, none lost one, 6,391 only moved the key to the end. deckhouse wall time 10 s, as before. Tests: cross-label lists in eight arrival orders and 100 spans in two orders fail on the old code; the worker-merge hand-over and the write-once text are covered too. Two merge tests that pinned the cross-label exclusion now expect both definitions' spans. Signed-off-by: Martin Vogel --- src/graph_buffer/graph_buffer.c | 374 ++++++++++++++++++++++---------- src/graph_buffer/graph_buffer.h | 6 + src/pipeline/pipeline_delta.c | 3 + tests/test_graph_buffer.c | 157 +++++++++++++- 4 files changed, 421 insertions(+), 119 deletions(-) diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 7f44a2d19..fd81f2c40 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -105,6 +105,11 @@ struct cbm_gbuf { bool by_id_off; int64_t by_id_cap; + /* Same-QN collision spans, written on export (cbm_gbuf_settle_variants). + * key: the surviving node's own QN string, value: gb_span_vec_t*. NULL + * until the first collision. */ + CBMHashTable *variant_spans; + /* Secondary node indexes */ CBMHashTable *nodes_by_label; /* key: label, value: (node_ptr_array_t*) */ CBMHashTable *nodes_by_name; /* key: name, value: (node_ptr_array_t*) */ @@ -672,10 +677,16 @@ cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path, return gb; } +static void gb_free_span_vec(const char *qn, void *value, void *ud); + void cbm_gbuf_free(cbm_gbuf_t *gb) { if (!gb) { return; } + if (gb->variant_spans) { + cbm_ht_foreach(gb->variant_spans, gb_free_span_vec, NULL); + cbm_ht_free(gb->variant_spans); + } /* Free each individually-allocated node */ for (int i = 0; i < gb->nodes.count; i++) { @@ -864,14 +875,32 @@ static void gb_yy_free(void *ctx, void *block) { static const yyjson_alc gb_yy_alc = {gb_yy_malloc, gb_yy_realloc, gb_yy_free, NULL}; -enum { GB_VARIANTS_MAX = 64 }; - typedef struct { - const char *file; + const char *file; /* interned in the buffer's pool */ int start; int end; } gb_span_t; +/* One side of a same-QN collision. */ +typedef struct { + const char *label; + const char *props; + const char *file; + int start; + int end; +} gb_def_t; + +/* Every definition span that collapsed into one node, gathered while the + * buffer is built and written into the node's `variants` once, on export + * (cbm_gbuf_settle_variants). Rewriting a growing JSON list on every + * collision was quadratic in the collisions of one QN: 7,586 junit + * `testcase` elements in one file. */ +typedef struct { + gb_span_t *items; + size_t count; + size_t cap; +} gb_span_vec_t; + static bool gb_variant_label(const char *label) { static const char *const labels[] = {"Function", "Method", "Class", "Interface", "Enum", "Struct", "Trait", "Type", "Macro"}; @@ -896,99 +925,219 @@ static int gb_span_cmp(const void *a, const void *b) { return c; } -/* A definition's spans: its `variants` property, else its own span. */ -static int gb_spans_of(yyjson_doc *doc, const char *file, int start, int end, gb_span_t *out, - int n) { +/* The non-empty `variants` list of a properties document, or NULL. */ +static yyjson_val *gb_variants_list(yyjson_doc *doc) { yyjson_val *list = doc ? yyjson_obj_get(yyjson_doc_get_root(doc), "variants") : NULL; - if (yyjson_is_arr(list) && yyjson_arr_size(list) > 0) { - size_t i; - size_t max; - yyjson_val *v; - yyjson_arr_foreach(list, i, max, v) { - const char *f = yyjson_get_str(yyjson_obj_get(v, "file_path")); - int s = (int)yyjson_get_int(yyjson_obj_get(v, "start_line")); - int e = (int)yyjson_get_int(yyjson_obj_get(v, "end_line")); - if (f && s > 0 && n < GB_VARIANTS_MAX) { - out[n++] = (gb_span_t){f, s, e}; + return yyjson_is_arr(list) && yyjson_arr_size(list) > 0 ? list : NULL; +} + +static bool gb_props_have_variants(const char *props) { + return props && strstr(props, "\"variants\"") != NULL; +} + +/* Does this side carry a span: a variants list, or a definition's own span? */ +static bool gb_def_has_spans(const gb_def_t *d) { + return gb_props_have_variants(d->props) || (gb_variant_label(d->label) && d->start > 0); +} + +static void gb_span_push(gb_span_vec_t *vec, gb_span_t span) { + if (vec->count == vec->cap) { + enum { GB_SPANS_MIN = 4 }; + size_t cap = vec->cap ? vec->cap * PAIR_LEN : GB_SPANS_MIN; + gb_span_t *grown = cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, vec->items, cap * sizeof(*grown)); + if (!grown) { + return; /* a lost span costs precision, never data */ + } + vec->items = grown; + vec->cap = cap; + } + vec->items[vec->count++] = span; +} + +/* Append a side's spans: its `variants` list, else its own span when it is a + * definition. File paths are interned, so the spans outlive the props. */ +static void gb_collect_spans(cbm_gbuf_t *gb, gb_span_vec_t *vec, const gb_def_t *d) { + if (gb_props_have_variants(d->props)) { + yyjson_doc *doc = yyjson_read_opts((char *)d->props, strlen(d->props), 0, &gb_yy_alc, NULL); + yyjson_val *list = gb_variants_list(doc); + if (list) { + size_t i; + size_t max; + yyjson_val *v; + yyjson_arr_foreach(list, i, max, v) { + const char *f = yyjson_get_str(yyjson_obj_get(v, "file_path")); + int st = (int)yyjson_get_int(yyjson_obj_get(v, "start_line")); + int en = (int)yyjson_get_int(yyjson_obj_get(v, "end_line")); + if (f && st > 0) { + gb_span_push(vec, (gb_span_t){gb_intern(gb, f), st, en}); + } } + yyjson_doc_free(doc); + return; } - return n; + yyjson_doc_free(doc); } - if (start > 0 && n < GB_VARIANTS_MAX) { - out[n++] = (gb_span_t){file ? file : "", start, end}; + if (gb_variant_label(d->label) && d->start > 0) { + gb_span_push(vec, (gb_span_t){gb_intern(gb, d->file ? d->file : ""), d->start, d->end}); } - return n; } -/* The survivor's properties with the union of both definitions' variant - * spans. Returns a GBUF_STRING-owned JSON string, or NULL when there is - * nothing to record (fewer than two distinct spans) or on any failure — the - * survivor then keeps its own properties: a lost variants list costs - * precision, never data. */ -static char *gb_variants_props(const char *survivor_props, const char *a_props, const char *a_file, - int a_start, int a_end, const char *b_props, const char *b_file, - int b_start, int b_end) { - if (a_start <= 0 || b_start <= 0) { - return NULL; +/* Append spans gathered in another buffer (a worker's), re-interned here. */ +static void gb_append_spans(cbm_gbuf_t *gb, gb_span_vec_t *vec, const gb_span_vec_t *from) { + for (size_t i = 0; i < from->count; i++) { + gb_span_t span = from->items[i]; + span.file = gb_intern(gb, span.file); + gb_span_push(vec, span); } - yyjson_doc *a_doc = - a_props ? yyjson_read_opts((char *)a_props, strlen(a_props), 0, &gb_yy_alc, NULL) : NULL; - yyjson_doc *b_doc = - b_props ? yyjson_read_opts((char *)b_props, strlen(b_props), 0, &gb_yy_alc, NULL) : NULL; - gb_span_t spans[GB_VARIANTS_MAX * 2]; - int n = gb_spans_of(a_doc, a_file, a_start, a_end, spans, 0); - n = gb_spans_of(b_doc, b_file, b_start, b_end, spans, n); - qsort(spans, (size_t)n, sizeof(spans[0]), gb_span_cmp); - int unique = 0; - for (int i = 0; i < n; i++) { - if (unique == 0 || gb_span_cmp(&spans[unique - 1], &spans[i]) != 0) { - spans[unique++] = spans[i]; - } - } - char *out = NULL; - if (unique >= 2) { - yyjson_mut_doc *mut = yyjson_mut_doc_new(&gb_yy_alc); - yyjson_doc *s_doc = survivor_props - ? yyjson_read_opts((char *)survivor_props, strlen(survivor_props), - 0, &gb_yy_alc, NULL) - : NULL; - yyjson_mut_val *root = s_doc && yyjson_is_obj(yyjson_doc_get_root(s_doc)) - ? yyjson_val_mut_copy(mut, yyjson_doc_get_root(s_doc)) - : yyjson_mut_obj(mut); - yyjson_mut_val *list = yyjson_mut_arr(mut); - for (int i = 0; root && list && i < unique; i++) { - yyjson_mut_val *v = yyjson_mut_arr_add_obj(mut, list); - yyjson_mut_obj_add_strcpy(mut, v, "file_path", spans[i].file); - yyjson_mut_obj_add_int(mut, v, "start_line", spans[i].start); - yyjson_mut_obj_add_int(mut, v, "end_line", spans[i].end); - } - if (mut && root && list) { - yyjson_mut_obj_remove_key(root, "variants"); - yyjson_mut_obj_add_val(mut, root, "variants", list); - yyjson_mut_doc_set_root(mut, root); - out = yyjson_mut_write_opts(mut, 0, &gb_yy_alc, NULL, NULL); - } - yyjson_doc_free(s_doc); - yyjson_mut_doc_free(mut); - } - yyjson_doc_free(a_doc); - yyjson_doc_free(b_doc); - return out; } -/* Record the variant spans of a same-QN collision on the surviving node. */ -static void gb_record_variants(cbm_gbuf_node_t *survivor, const char *a_label, const char *a_props, - const char *a_file, int a_start, int a_end, const char *b_label, - const char *b_props, const char *b_file, int b_start, int b_end) { - if (!gb_variant_label(a_label) || !b_label || strcmp(a_label, b_label) != 0) { +/* A same-QN collision: gather both sides' spans for the surviving node, + * whichever label wins and however many there are. Call it before the + * winner is decided, while `existing` still holds its own fields. + * `arrival_spans` are the arrival's spans gathered in a worker buffer (they + * already include its own); NULL when it brings none. */ +static void gb_note_collision(cbm_gbuf_t *gb, const cbm_gbuf_node_t *existing, + const gb_def_t *arrival, const gb_span_vec_t *arrival_spans) { + gb_def_t prior = {existing->label, existing->properties_json, existing->file_path, + existing->start_line, existing->end_line}; + bool arrival_has = arrival_spans ? arrival_spans->count > 0 : gb_def_has_spans(arrival); + if (!existing->qualified_name || (!gb_def_has_spans(&prior) && !arrival_has)) { return; } - char *merged = gb_variants_props(survivor->properties_json, a_props, a_file, a_start, a_end, - b_props, b_file, b_start, b_end); - if (merged) { - cbm_free(CBM_MEM_CLASS_GBUF_STRING, survivor->properties_json); - survivor->properties_json = merged; + if (!gb->variant_spans) { + gb->variant_spans = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, 0); + if (!gb->variant_spans) { + return; + } } + gb_span_vec_t *vec = cbm_ht_get(gb->variant_spans, existing->qualified_name); + if (!vec) { + vec = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(*vec)); + if (!vec) { + return; + } + gb_collect_spans(gb, vec, &prior); + cbm_ht_set(gb->variant_spans, existing->qualified_name, vec); + } + if (arrival_spans) { + gb_append_spans(gb, vec, arrival_spans); + } else { + gb_collect_spans(gb, vec, arrival); + } +} + +/* A worker node copied into this buffer brings the spans it gathered there. */ +static void gb_adopt_spans(cbm_gbuf_t *gb, const cbm_gbuf_node_t *node, const gb_span_vec_t *from) { + if (!node || !node->qualified_name || !from || from->count == 0) { + return; + } + if (!gb->variant_spans) { + gb->variant_spans = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, 0); + if (!gb->variant_spans) { + return; + } + } + gb_span_vec_t *vec = cbm_ht_get(gb->variant_spans, node->qualified_name); + if (!vec) { + vec = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(*vec)); + if (!vec) { + return; + } + cbm_ht_set(gb->variant_spans, node->qualified_name, vec); + } + gb_append_spans(gb, vec, from); +} + +static void gb_span_vec_free(gb_span_vec_t *vec) { + if (vec) { + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, vec->items); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, vec); + } +} + +/* A node leaves the QN index: its gathered spans go with it, so a later node + * of the same QN starts from its own. */ +static void gb_drop_variant_spans(cbm_gbuf_t *gb, const char *qn) { + if (gb->variant_spans && qn) { + gb_span_vec_free(cbm_ht_delete(gb->variant_spans, qn)); + } +} + +/* Write a node's gathered spans into its properties: sorted, distinct, the + * list left out only when it would say no more than the node's own span. */ +static void gb_write_variants(cbm_gbuf_node_t *node, gb_span_vec_t *vec) { + qsort(vec->items, vec->count, sizeof(vec->items[0]), gb_span_cmp); + size_t unique = 0; + for (size_t i = 0; i < vec->count; i++) { + if (unique == 0 || gb_span_cmp(&vec->items[unique - SKIP_ONE], &vec->items[i]) != 0) { + vec->items[unique++] = vec->items[i]; + } + } + gb_span_t own = {node->file_path ? node->file_path : "", node->start_line, node->end_line}; + bool own_is_span = gb_variant_label(node->label) && node->start_line > 0; + bool only_own = + own_is_span ? unique == SKIP_ONE && gb_span_cmp(&vec->items[0], &own) == 0 : unique == 0; + if (only_own && !gb_props_have_variants(node->properties_json)) { + return; + } + /* Numbers stay as written: a pass printed them (importance 0.000000). */ + const char *props = node->properties_json; + yyjson_doc *doc = props ? yyjson_read_opts((char *)props, strlen(props), + YYJSON_READ_NUMBER_AS_RAW, &gb_yy_alc, NULL) + : NULL; + yyjson_mut_doc *mut = yyjson_mut_doc_new(&gb_yy_alc); + yyjson_mut_val *root = NULL; + if (mut) { + root = doc && yyjson_is_obj(yyjson_doc_get_root(doc)) + ? yyjson_val_mut_copy(mut, yyjson_doc_get_root(doc)) + : yyjson_mut_obj(mut); + } + yyjson_mut_val *list = root && !only_own ? yyjson_mut_arr(mut) : NULL; + for (size_t i = 0; list && i < unique; i++) { + yyjson_mut_val *v = yyjson_mut_arr_add_obj(mut, list); + yyjson_mut_obj_add_strcpy(mut, v, "file_path", vec->items[i].file); + yyjson_mut_obj_add_int(mut, v, "start_line", vec->items[i].start); + yyjson_mut_obj_add_int(mut, v, "end_line", vec->items[i].end); + } + if (root && (only_own || list)) { + yyjson_mut_obj_remove_key(root, "variants"); + if (list) { + yyjson_mut_obj_add_val(mut, root, "variants", list); + } + yyjson_mut_doc_set_root(mut, root); + char *out = yyjson_mut_write_opts(mut, 0, &gb_yy_alc, NULL, NULL); + if (out) { + cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->properties_json); + node->properties_json = out; + } + } + yyjson_mut_doc_free(mut); + yyjson_doc_free(doc); +} + +static void gb_settle_one(const char *qn, void *value, void *ud) { + cbm_gbuf_t *gb = ud; + gb_span_vec_t *vec = value; + cbm_gbuf_node_t *node = gb->node_by_qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL; + if (node && vec) { + gb_write_variants(node, vec); + } + gb_span_vec_free(vec); +} + +static void gb_free_span_vec(const char *qn, void *value, void *ud) { + (void)qn; + (void)ud; + gb_span_vec_free(value); +} + +void cbm_gbuf_settle_variants(cbm_gbuf_t *gb) { + if (!gb || !gb->variant_spans) { + return; + } + cbm_ht_foreach(gb->variant_spans, gb_settle_one, gb); + cbm_ht_free(gb->variant_spans); + gb->variant_spans = NULL; } /* ── Node operations ─────────────────────────────────────────────── */ @@ -1066,20 +1215,13 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name if (c == 0) { c = strcmp(existing->label ? existing->label : "", label ? label : ""); } + gb_def_t arrival = {label, properties_json, file_path, start_line, end_line}; + gb_note_collision(gb, existing, &arrival, NULL); if (c > 0) { - gb_record_variants(existing, existing->label, existing->properties_json, - existing->file_path, existing->start_line, existing->end_line, label, - properties_json, file_path, start_line, end_line); return existing->id; /* existing entity is the canonical winner */ } - /* The arrival wins: its properties, plus every variant span both carry. */ - char *variant_props = NULL; - if (gb_variant_label(label) && existing->label && strcmp(existing->label, label) == 0) { - variant_props = - gb_variants_props(properties_json, existing->properties_json, existing->file_path, - existing->start_line, existing->end_line, properties_json, - file_path, start_line, end_line); - } + /* The arrival wins: its properties; the variant spans both carry are + * written on export (cbm_gbuf_settle_variants). */ /* Update in-place. name/properties are strdup'd BEFORE freeing old ones * (callers may pass existing->name as an argument). label/file_path are * interned: gb_intern returns a stable pool pointer (idempotent even when @@ -1089,9 +1231,7 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name * label/name — cbm_gbuf_find_by_label/name then missed or mis-listed it, * which is how the flickering Function set reached the semantic pass). */ char *new_name = heap_strdup(name); - char *new_props = variant_props ? variant_props - : properties_json ? heap_strdup(properties_json) - : NULL; + char *new_props = properties_json ? heap_strdup(properties_json) : NULL; const char *new_label_interned = gb_intern(gb, label); bool label_changed = !existing->label || !new_label_interned || strcmp(existing->label, new_label_interned) != 0; @@ -1236,6 +1376,7 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label) { intptr_to_ptr(SKIP_ONE)); /* Remove from primary indexes */ + gb_drop_variant_spans(gb, n->qualified_name); cbm_ht_delete(gb->node_by_qn, n->qualified_name); if (n->id >= 0 && n->id < gb->by_id_cap) { gb->by_id[n->id] = NULL; @@ -1283,6 +1424,7 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) { remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_name, n->name), n->id); /* Remove from primary indexes */ + gb_drop_variant_spans(gb, n->qualified_name); cbm_ht_delete(gb->node_by_qn, n->qualified_name); if (n->id >= 0 && n->id < gb->by_id_cap) { gb->by_id[n->id] = NULL; @@ -1646,7 +1788,8 @@ static void free_remap_entry(const char *key, void *val, void *ud) { /* Handle QN collision: update dst node fields (src wins), record remap if IDs differ. * label/file_path are re-interned into dst's pool (sn's pointers belong to src). */ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, - const cbm_gbuf_node_t *sn, CBMHashTable **remap) { + const cbm_gbuf_node_t *sn, const gb_span_vec_t *sn_spans, + CBMHashTable **remap) { /* Same guard as cbm_gbuf_upsert_node: a per-file "Module" def coming from a * worker-local gbuf must not touch the structural directory node ("Project" * root or "Folder") that shares its QN in a directory-based-module language @@ -1685,22 +1828,12 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, c = strcmp(existing->label ? existing->label : "", sn->label ? sn->label : ""); } bool sn_wins = c <= 0; - if (!sn_wins) { - gb_record_variants(existing, existing->label, existing->properties_json, - existing->file_path, existing->start_line, existing->end_line, - sn->label, sn->properties_json, sn->file_path, sn->start_line, - sn->end_line); - } + gb_def_t src_def = {sn->label, sn->properties_json, sn->file_path, sn->start_line, + sn->end_line}; + gb_note_collision(dst, existing, &src_def, sn_spans); if (sn_wins) { - /* src's properties, plus every variant span both carry. */ - char *variant_props = NULL; - if (gb_variant_label(sn->label) && existing->label && - strcmp(existing->label, sn->label) == 0) { - variant_props = gb_variants_props(sn->properties_json, existing->properties_json, - existing->file_path, existing->start_line, - existing->end_line, sn->properties_json, - sn->file_path, sn->start_line, sn->end_line); - } + /* src's properties; the variant spans both carry are written on + * export (cbm_gbuf_settle_variants). */ /* Keep the secondary indexes consistent when the surviving * label/name changes (the old code left the node listed under its * original label/name, so find_by_label/name mis-listed it). */ @@ -1725,10 +1858,9 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, existing->file_path = (char *)gb_intern(dst, sn->file_path); existing->start_line = sn->start_line; existing->end_line = sn->end_line; - if (variant_props || sn->properties_json) { + if (sn->properties_json) { cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->properties_json); - existing->properties_json = - variant_props ? variant_props : heap_strdup(sn->properties_json); + existing->properties_json = heap_strdup(sn->properties_json); } if (label_changed) { node_ptr_array_t *by_label = get_or_create_node_array( @@ -1829,11 +1961,16 @@ int cbm_gbuf_merge(cbm_gbuf_t *dst, cbm_gbuf_t *src) { continue; } + /* The spans src gathered for this node travel with it; they are + * written once, when dst is exported. */ + const gb_span_vec_t *spans = + src->variant_spans ? cbm_ht_get(src->variant_spans, sn->qualified_name) : NULL; cbm_gbuf_node_t *existing = cbm_ht_get(dst->node_by_qn, sn->qualified_name); if (existing) { - merge_update_existing(dst, existing, sn, &remap); + merge_update_existing(dst, existing, sn, spans, &remap); } else { merge_copy_new_node(dst, sn); + gb_adopt_spans(dst, cbm_ht_get(dst->node_by_qn, sn->qualified_name), spans); } } @@ -2122,6 +2259,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { if (!gb || !path) { return CBM_NOT_FOUND; } + cbm_gbuf_settle_variants(gb); CBM_PROF_START(t_count); int live_count = count_live_nodes(gb); @@ -2239,6 +2377,7 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) { if (!gb || !store) { return CBM_NOT_FOUND; } + cbm_gbuf_settle_variants(gb); /* Upsert project */ cbm_store_upsert_project(store, gb->project, gb->root_path); @@ -2313,6 +2452,7 @@ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) { if (!gb || !store) { return CBM_NOT_FOUND; } + cbm_gbuf_settle_variants(gb); /* Begin bulk mode — no project wipe */ cbm_store_begin(store); diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index ffbff0469..a0783261d 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -188,4 +188,10 @@ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store); * (the old value is kept). */ int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json); +/* Write the definition spans gathered from same-QN collisions into each + * surviving node's `variants` property. The dump, flush and merge entry + * points do it themselves; call it directly only to read a node's + * properties_json from the buffer after collisions. */ +void cbm_gbuf_settle_variants(cbm_gbuf_t *gb); + #endif /* CBM_GRAPH_BUFFER_H */ diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index e7f5ee40a..e6dd7cc2e 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -596,6 +596,9 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i if (cbm_store_begin(store) != CBM_STORE_OK) { return CBM_NOT_FOUND; } + /* This writes the buffer's nodes itself: settle the collision spans first, + * as the buffer's own dump and flush do. */ + cbm_gbuf_settle_variants(gbuf); delta_patch_ctx_t ctx = {.db = db, .project = project, .max_db_id = max_db_id}; if (sqlite3_prepare_v2(db, "SELECT id FROM nodes WHERE project = ?1" diff --git a/tests/test_graph_buffer.c b/tests/test_graph_buffer.c index dd257f187..b57378691 100644 --- a/tests/test_graph_buffer.c +++ b/tests/test_graph_buffer.c @@ -569,6 +569,7 @@ TEST(gbuf_upsert_same_qn_updates_all_fields) { ASSERT_EQ(id1, id2); ASSERT_EQ(cbm_gbuf_node_count(gb), 1); + cbm_gbuf_settle_variants(gb); const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(gb, "pkg.fn"); ASSERT_NOT_NULL(n); ASSERT_STR_EQ(n->label, "Method"); @@ -576,7 +577,11 @@ TEST(gbuf_upsert_same_qn_updates_all_fields) { ASSERT_STR_EQ(n->file_path, "new.go"); ASSERT_EQ(n->start_line, 20); ASSERT_EQ(n->end_line, 30); - ASSERT_STR_EQ(n->properties_json, "{\"k\":\"v2\"}"); + /* the winner's own properties, plus both definitions' spans: whichever + * label survives, a change in either body lands on this node */ + ASSERT_STR_EQ(n->properties_json, + "{\"k\":\"v2\",\"variants\":[{\"file_path\":\"new.go\",\"start_line\":20," + "\"end_line\":30},{\"file_path\":\"old.go\",\"start_line\":1,\"end_line\":10}]}"); cbm_gbuf_free(gb); PASS(); @@ -647,6 +652,146 @@ TEST(gbuf_merge_definition_beats_same_qn_macro_c1) { PASS(); } +/* deckhouse pkg/log: a package function and three methods share the QN + * `pkg.log.Error` (Go method QNs carry no receiver). Every definition that + * collapsed into the node is a span a change can land in, so `variants` + * holds all four -- in every arrival order. A Function arriving between the + * methods used to replace the node's list, and logger.go's span came and went + * with worker scheduling. */ +TEST(gbuf_variants_cross_label_order_independent) { + static const struct { + const char *label; + const char *file; + int start; + int end; + } defs[] = {{"Method", "pkg/log/logger.go", 227, 232}, + {"Function", "pkg/log/global.go", 83, 85}, + {"Method", "pkg/log/fields_test.go", 76, 78}, + {"Method", "pkg/log/fields_test.go", 85, 87}}; + static const int perms[][4] = {{0, 1, 2, 3}, {1, 0, 2, 3}, {0, 2, 1, 3}, {3, 2, 1, 0}, + {1, 3, 0, 2}, {2, 0, 3, 1}, {3, 1, 2, 0}, {0, 3, 1, 2}}; + char first[1024] = ""; + for (size_t p = 0; p < sizeof(perms) / sizeof(perms[0]); p++) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp"); + for (int i = 0; i < 4; i++) { + int d = perms[p][i]; + cbm_gbuf_upsert_node(gb, defs[d].label, "Error", "p.pkg.log.Error", defs[d].file, + defs[d].start, defs[d].end, "{}"); + } + cbm_gbuf_settle_variants(gb); + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(gb, "p.pkg.log.Error"); + ASSERT_NOT_NULL(n); + ASSERT_NOT_NULL(n->properties_json); + ASSERT_NOT_NULL( + strstr(n->properties_json, + "{\"file_path\":\"pkg/log/logger.go\",\"start_line\":227,\"end_line\":232}")); + ASSERT_NOT_NULL( + strstr(n->properties_json, + "{\"file_path\":\"pkg/log/global.go\",\"start_line\":83,\"end_line\":85}")); + if (p == 0) { + snprintf(first, sizeof(first), "%s", n->properties_json); + } else { + ASSERT_STR_EQ(n->properties_json, first); + } + cbm_gbuf_free(gb); + } + PASS(); +} + +/* No count decides which spans a node keeps: 100 same-QN definitions (generated + * protobuf `String()` methods) keep all 100, whichever order they arrive in. */ +TEST(gbuf_variants_keep_every_span) { + enum { N_DEFS = 100 }; + char *props[2] = {NULL, NULL}; + for (int order = 0; order < 2; order++) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp"); + for (int i = 0; i < N_DEFS; i++) { + int k = order == 0 ? i : N_DEFS - 1 - i; + char file[32]; + snprintf(file, sizeof(file), "pb/m%03d.pb.go", k); + cbm_gbuf_upsert_node(gb, "Method", "String", "p.pb.String", file, 10, 12, "{}"); + } + cbm_gbuf_settle_variants(gb); + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(gb, "p.pb.String"); + ASSERT_NOT_NULL(n); + ASSERT_NOT_NULL(n->properties_json); + int spans = 0; + for (const char *s = n->properties_json; (s = strstr(s, "\"file_path\"")) != NULL; s++) { + spans++; + } + ASSERT_EQ(spans, N_DEFS); + props[order] = strdup(n->properties_json); + cbm_gbuf_free(gb); + } + ASSERT_STR_EQ(props[0], props[1]); + free(props[0]); + free(props[1]); + PASS(); +} + +/* Spans gathered in worker buffers travel with their nodes into the main + * buffer, whichever worker merges first. */ +TEST(gbuf_variants_travel_through_worker_merge) { + char first[512] = ""; + for (int order = 0; order < 2; order++) { + cbm_gbuf_t *main_gb = cbm_gbuf_new("test", "/tmp"); + cbm_gbuf_t *w1 = cbm_gbuf_new("test", "/tmp"); + cbm_gbuf_t *w2 = cbm_gbuf_new("test", "/tmp"); + cbm_gbuf_upsert_node(w1, "Method", "X", "p.X", "a.go", 10, 12, "{}"); + cbm_gbuf_upsert_node(w1, "Method", "X", "p.X", "b.go", 5, 6, "{}"); + cbm_gbuf_upsert_node(w2, "Method", "X", "p.X", "c.go", 1, 2, "{}"); + cbm_gbuf_upsert_node(w2, "Function", "X", "p.X", "d.go", 3, 4, "{}"); + ASSERT_EQ(cbm_gbuf_merge(main_gb, order == 0 ? w1 : w2), 0); + ASSERT_EQ(cbm_gbuf_merge(main_gb, order == 0 ? w2 : w1), 0); + cbm_gbuf_settle_variants(main_gb); + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(main_gb, "p.X"); + ASSERT_NOT_NULL(n); + ASSERT_STR_EQ(n->properties_json, + "{\"variants\":[{\"file_path\":\"a.go\",\"start_line\":10,\"end_line\":12}," + "{\"file_path\":\"b.go\",\"start_line\":5,\"end_line\":6}," + "{\"file_path\":\"c.go\",\"start_line\":1,\"end_line\":2}," + "{\"file_path\":\"d.go\",\"start_line\":3,\"end_line\":4}]}"); + if (order == 0) { + snprintf(first, sizeof(first), "%s", n->properties_json); + } else { + ASSERT_STR_EQ(n->properties_json, first); + } + cbm_gbuf_free(w1); + cbm_gbuf_free(w2); + cbm_gbuf_free(main_gb); + } + PASS(); +} + +/* The list is written once, on export: the properties come out the same + * whether the collision happened in the main buffer or in a worker's, after + * a later pass appended a key with its own number text. */ +TEST(gbuf_variants_written_once_at_export) { + char props[2][512]; + for (int via_worker = 0; via_worker < 2; via_worker++) { + cbm_gbuf_t *main_gb = cbm_gbuf_new("test", "/tmp"); + cbm_gbuf_t *w = via_worker ? cbm_gbuf_new("test", "/tmp") : main_gb; + cbm_gbuf_upsert_node(w, "Method", "X", "p.X", "a.go", 10, 12, "{\"k\":1}"); + cbm_gbuf_upsert_node(w, "Method", "X", "p.X", "b.go", 5, 6, "{\"k\":1}"); + if (via_worker) { + ASSERT_EQ(cbm_gbuf_merge(main_gb, w), 0); + cbm_gbuf_free(w); + } + cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)cbm_gbuf_find_by_qn(main_gb, "p.X"); + ASSERT_NOT_NULL(n); + char buf[512]; + snprintf(buf, sizeof(buf), "%.*s,\"importance\":0.000000}", + (int)strlen(n->properties_json) - 1, n->properties_json); + ASSERT_EQ(cbm_gbuf_node_set_properties_json(n, buf), 0); + cbm_gbuf_settle_variants(main_gb); + snprintf(props[via_worker], sizeof(props[0]), "%s", n->properties_json); + cbm_gbuf_free(main_gb); + } + ASSERT_STR_EQ(props[0], props[1]); + ASSERT_NOT_NULL(strstr(props[0], "\"importance\":0.000000,\"variants\":[")); + PASS(); +} + TEST(gbuf_upsert_long_qn) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp"); @@ -913,13 +1058,17 @@ TEST(gbuf_merge_overlapping_qns) { ASSERT_EQ(cbm_gbuf_node_count(dst), 3); /* Verify src fields won for the overlapping QN */ + cbm_gbuf_settle_variants(dst); const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(dst, "pkg.fn"); ASSERT_NOT_NULL(n); ASSERT_STR_EQ(n->label, "Method"); ASSERT_STR_EQ(n->name, "fn_new"); ASSERT_STR_EQ(n->file_path, "new.go"); ASSERT_EQ(n->start_line, 20); - ASSERT_STR_EQ(n->properties_json, "{\"from\":\"src\"}"); + /* src's properties, plus both definitions' spans (same rule as upsert) */ + ASSERT_STR_EQ(n->properties_json, + "{\"from\":\"src\",\"variants\":[{\"file_path\":\"new.go\",\"start_line\":20," + "\"end_line\":30},{\"file_path\":\"old.go\",\"start_line\":1,\"end_line\":10}]}"); /* Both unique nodes present */ ASSERT_NOT_NULL(cbm_gbuf_find_by_qn(dst, "pkg.unique_dst")); @@ -1317,6 +1466,10 @@ SUITE(graph_buffer) { RUN_TEST(gbuf_upsert_same_qn_updates_all_fields); RUN_TEST(gbuf_upsert_definition_beats_same_qn_macro_c1); RUN_TEST(gbuf_merge_definition_beats_same_qn_macro_c1); + RUN_TEST(gbuf_variants_cross_label_order_independent); + RUN_TEST(gbuf_variants_keep_every_span); + RUN_TEST(gbuf_variants_travel_through_worker_merge); + RUN_TEST(gbuf_variants_written_once_at_export); RUN_TEST(gbuf_upsert_long_qn); RUN_TEST(gbuf_find_by_qn_missing); RUN_TEST(gbuf_find_by_id_missing); From 69ac2846f2ade80b9cd82aa2cbb25a71b3875d11 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 18:15:06 +0200 Subject: [PATCH 17/52] feat(k8s): every manifest document is an object with a stable identity A Kubernetes manifest became at most one Resource: the extractor stopped after the first document. A Deployment and its Service in one file -- the usual way to ship them -- left the Service out of the graph, and with it every link from it; across 17 infrastructure repositories only 52% of the objects in plain manifests existed as nodes. Every document now yields its object, under an identity that holds when names repeat: - QN `::///`, where is the File node's QN without `.__file__` (the extension kept, so a.yaml/a.yml and dir/ + dir.yaml cannot collide), the apiVersion's group (`core` for v1). Same name in two namespaces = two objects; one object twice in a file = one node with both spans. - Properties `group`, `kind`, the declared `namespace` and `obj_key` `///`, the logical identity later joins use: `_` for a cluster-scoped kind, `?` when the deploy decides the namespace. The display name stays `Kind/name`. - Each object's selector or pod labels come from its own document, so a Service links to the Deployment beside it. - Values the YAML wraps are read through the wrapper: an anchored or tagged name (`name: &name app`, `!!str`) and a flow-style label map (`selector: { app: kdc }`, cut-short maps skipped whole so a selector never loses pairs). The Resource QNs change, so the index format moves to 2: every existing index is rebuilt once on upgrade, keeping its ADR and project metadata. On the 17 corpora (truth: every document of every plain manifest, parsed independently): objects present as Resource 2,634 -> 4,996 of 5,025 (99%; the rest: a vendored CRD directory, multi-megabyte CRDs the YAML grammar parses partially, one gitignored directory). Service -> workload pairs whose objects are both in the graph 36 -> 219 of 219, all 219 linked; the 17 links outside that set are in Go-templated files the truth parser skips and select what they should. Everything outside the K8s objects is identical node for node and edge for edge; two runs are identical. Tests: every document an object, a selector across documents, one name in two namespaces, the identity key and properties (incl. core group, declared namespace, cluster scope), anchored/tagged names and flow label maps fail on the old extractor; the extractor-level multi-document test now pins both documents and their QNs. Signed-off-by: Martin Vogel --- internal/cbm/extract_k8s.c | 155 ++++++++++++++++++-------- src/pipeline/pass_k8s.c | 179 +++++++++++++++++++++++++++--- src/store/store.h | 7 +- tests/test_grammar_probe_f.c | 205 +++++++++++++++++++++++++++++++++++ tests/test_pipeline.c | 34 +++--- 5 files changed, 502 insertions(+), 78 deletions(-) diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 88f8a46f8..46937668d 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -4,9 +4,11 @@ // matches "resources", "bases", "patches", "components", or // "patchesStrategicMerge", then emits one CBMImport per block_sequence item. // -// For CBM_LANG_K8S: finds apiVersion, kind, and metadata.name scalars in the -// first document's block_mapping and emits one CBMDefinition with label -// "Resource" and name "Kind/metadata-name". +// For CBM_LANG_K8S: reads apiVersion, kind, metadata.name and +// metadata.namespace of EVERY document and emits one CBMDefinition per object: +// label "Resource", name "Kind/metadata-name", QN +// "::///" (identity +// spec: the file base keeps the extension, so it is unique per file). #include "cbm.h" #include "arena.h" @@ -20,24 +22,39 @@ /* Local constants. */ enum { K8S_BUF_SIZE = 256, - RESULT_BUF_SIZE = 512, }; // --------------------------------------------------------------------------- // Internal helpers // --------------------------------------------------------------------------- +// The value inside a flow_node: its first named child that is not an anchor +// (`&name`), a tag (`!!str`) or a comment, all of which precede the value. +static TSNode flow_node_value(TSNode node) { + uint32_t n = ts_node_named_child_count(node); + for (uint32_t i = 0; i < n; i++) { + TSNode child = ts_node_named_child(node, i); + const char *type = ts_node_type(child); + if (strcmp(type, "anchor") != 0 && strcmp(type, "tag") != 0 && + strcmp(type, "comment") != 0) { + return child; + } + } + TSNode null_node = {0}; + return null_node; +} + // Return the raw source text for a scalar node (plain, single-quoted, or // double-quoted). Surrounding quote characters are stripped for quoted forms. -// Handles flow_node wrappers transparently by descending into the first named -// child (the tree-sitter YAML grammar often wraps scalars in flow_node). +// Handles flow_node wrappers transparently by descending into the value they +// wrap (the tree-sitter YAML grammar often wraps scalars in flow_node). // Returns NULL for non-scalar node types. static const char *get_scalar_text(CBMArena *a, TSNode node, const char *source) { enum { MAX_UNWRAP = 4 }; for (int depth = 0; depth < MAX_UNWRAP; depth++) { const char *type = ts_node_type(node); if (strcmp(type, "flow_node") == 0) { - TSNode inner = ts_node_named_child(node, 0); + TSNode inner = flow_node_value(node); if (ts_node_is_null(inner)) { return NULL; } @@ -241,9 +258,17 @@ static void extract_kustomize(CBMExtractCtx *ctx) { // K8s manifest extraction // --------------------------------------------------------------------------- -// Extract the "name" scalar from a metadata block_mapping. -static void extract_metadata_name(CBMArena *a, TSNode meta_mapping, const char *source, - char *meta_name_buf, size_t meta_sz) { +// The identity scalars of one manifest document. +typedef struct { + char api_version[K8S_BUF_SIZE]; + char kind[K8S_BUF_SIZE]; + char name[K8S_BUF_SIZE]; + char ns[K8S_BUF_SIZE]; +} k8s_doc_id_t; + +// Read metadata.name and metadata.namespace from a metadata block_mapping. +static void extract_metadata(CBMArena *a, TSNode meta_mapping, const char *source, + k8s_doc_id_t *id) { if (ts_node_is_null(meta_mapping) || strcmp(ts_node_type(meta_mapping), "block_mapping") != 0) { return; } @@ -258,28 +283,22 @@ static void extract_metadata_name(CBMArena *a, TSNode meta_mapping, const char * continue; } const char *mkey_text = get_scalar_text(a, mkey, source); - if (!mkey_text || strcmp(mkey_text, "name") != 0) { + bool is_name = mkey_text && strcmp(mkey_text, "name") == 0; + bool is_ns = mkey_text && strcmp(mkey_text, "namespace") == 0; + if (!is_name && !is_ns) { continue; } TSNode mval = unwrap_pair_value(mpair); - if (ts_node_is_null(mval)) { - continue; - } - const char *meta_name = get_scalar_text(a, mval, source); - if (meta_name) { - snprintf(meta_name_buf, meta_sz, "%s", meta_name); + const char *text = ts_node_is_null(mval) ? NULL : get_scalar_text(a, mval, source); + if (text) { + snprintf(is_name ? id->name : id->ns, K8S_BUF_SIZE, "%s", text); } } } -// Descend into the first block_mapping of a document and extract -// kind and metadata.name. Returns void; fills kind_buf and meta_name_buf. -static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, char *kind_buf, size_t kind_sz, - char *meta_name_buf, size_t meta_sz) { +// Read apiVersion, kind and metadata from a document's top-level mapping. +static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, k8s_doc_id_t *id) { CBMArena *a = ctx->arena; - kind_buf[0] = '\0'; - meta_name_buf[0] = '\0'; - uint32_t n = ts_node_child_count(mapping); for (uint32_t i = 0; i < n; i++) { TSNode pair = ts_node_child(mapping, i); @@ -294,25 +313,75 @@ static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, char *kind_b if (!key) { continue; } - TSNode val_node = unwrap_pair_value(pair); if (ts_node_is_null(val_node)) { continue; } - - if (strcmp(key, "kind") == 0) { - const char *v = get_scalar_text(a, val_node, ctx->source); - if (v) { - snprintf(kind_buf, kind_sz, "%s", v); - } - } else if (strcmp(key, "metadata") == 0) { - extract_metadata_name(a, val_node, ctx->source, meta_name_buf, meta_sz); + if (strcmp(key, "metadata") == 0) { + extract_metadata(a, val_node, ctx->source, id); + continue; + } + char *dst = strcmp(key, "kind") == 0 ? id->kind + : strcmp(key, "apiVersion") == 0 ? id->api_version + : NULL; + const char *v = dst ? get_scalar_text(a, val_node, ctx->source) : NULL; + if (v) { + snprintf(dst, K8S_BUF_SIZE, "%s", v); } } } +// "." with the extension kept: the File node's QN +// without ".__file__" (IDENTITY.md). Unique per file, unlike the module QN, +// which drops the extension (`a.yaml` / `a.yml`, `x/` / `x.yaml`). +static const char *k8s_file_base(CBMArena *a, const char *project, const char *rel_path) { + project = project ? project : ""; + rel_path = rel_path ? rel_path : ""; + size_t plen = strlen(project); + char *buf = cbm_arena_alloc(a, plen + strlen(rel_path) + PAIR_LEN); + if (!buf) { + return NULL; + } + char *o = buf; + memcpy(o, project, plen); + o += plen; + bool seg_start = true; + for (const char *p = rel_path; *p; p++) { + char c = *p == '\\' ? '/' : *p; + if (c == '/') { + seg_start = true; + continue; + } + if (seg_start) { + *o++ = '.'; + seg_start = false; + } + *o++ = c; + } + *o = '\0'; + return buf; +} + +// The API group of an apiVersion: the part before '/', "core" for the core +// group ("v1"), "?" when the document names none. The version is not part of +// an object's identity. +static void k8s_api_group(const char *api_version, char *out, size_t out_sz) { + const char *slash = strchr(api_version, '/'); + if (slash) { + snprintf(out, out_sz, "%.*s", (int)(slash - api_version), api_version); + } else { + snprintf(out, out_sz, "%s", api_version[0] ? "core" : "?"); + } +} + +// One Resource def per document: `Kind/name`, QN +// `::///` (IDENTITY.md). static void extract_k8s_manifest(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; + const char *base = k8s_file_base(a, ctx->project, ctx->rel_path); + if (!base) { + return; + } TSNode root = ctx->root; uint32_t root_n = ts_node_child_count(root); @@ -327,22 +396,22 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { continue; } - char kind_buf[K8S_BUF_SIZE] = {0}; - char meta_name_buf[K8S_BUF_SIZE] = {0}; - extract_k8s_scalars(ctx, mapping, kind_buf, sizeof(kind_buf), meta_name_buf, - sizeof(meta_name_buf)); + k8s_doc_id_t id; + memset(&id, 0, sizeof(id)); + extract_k8s_scalars(ctx, mapping, &id); // Skip malformed manifests (no kind or no metadata.name) - if (kind_buf[0] == '\0' || meta_name_buf[0] == '\0') { + if (id.kind[0] == '\0' || id.name[0] == '\0') { continue; } - char def_name[RESULT_BUF_SIZE]; - snprintf(def_name, sizeof(def_name), "%s/%s", kind_buf, meta_name_buf); + char group[K8S_BUF_SIZE]; + k8s_api_group(id.api_version, group, sizeof(group)); CBMDefinition def = {0}; - def.name = cbm_arena_strdup(a, def_name); - def.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->module_qn, def_name); + def.name = cbm_arena_sprintf(a, "%s/%s", id.kind, id.name); + def.qualified_name = cbm_arena_sprintf(a, "%s::%s/%s/%s/%s", base, group, id.kind, + id.ns[0] ? id.ns : "-", id.name); // "Resource" is the canonical def label for a K8s resource kind. It is a // valid graph label and is what the K8s pipeline pass (pass_k8s.c) filters // on to upsert Resource nodes and emit INFRA_MAPS edges. @@ -351,8 +420,6 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { def.start_line = ts_node_start_point(mapping).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(mapping).row + TS_LINE_OFFSET; cbm_defs_push(&ctx->result->defs, a, def); - - break; // Only the first document per file } } diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 040cb1688..ea7cdfa6a 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -284,7 +284,47 @@ static void k8s_parent_path(char *out, size_t out_sz, const char (*keys)[64], in } } -/* Scan a single-document k8s manifest's text for its name, namespace, kind and +/* `key: { a: b, c: d }` — a flow-style map is the same label map written on + * one line. One the line buffer cut short (no closing brace) is skipped + * whole: a selector missing pairs would select too much. */ +static void k8s_scan_flow_map(k8s_scanned_t *out, const char (*keys)[64], int depth, + const char *key, const char *val) { + size_t n = strlen(val); + if (n < PAIR_LEN || val[0] != '{' || val[n - SKIP_ONE] != '}') { + return; + } + char path[CBM_SZ_512]; + k8s_parent_path(path, sizeof(path), keys, depth); + size_t w = strlen(path); + snprintf(path + w, sizeof(path) - w, "%s%s", w ? "." : "", key); + k8s_pair_block_t block = k8s_pair_block(path); + if (block == K8S_PAIR_NONE) { + return; + } + char inner[CBM_SZ_512]; + snprintf(inner, sizeof(inner), "%.*s", (int)(n - PAIR_LEN), val + SKIP_ONE); + for (char *item = inner; item;) { + char *comma = strchr(item, ','); + if (comma) { + *comma = '\0'; + } + while (*item == ' ') { + item++; + } + char k[64]; + char v[K8S_LABEL_LEN]; + if (k8s_split_kv(item, k, sizeof(k), v, sizeof(v)) && v[0]) { + size_t kl = strlen(k); + while (kl > 0 && k[kl - SKIP_ONE] == ' ') { + k[--kl] = '\0'; + } + k8s_scanned_push(out, block, k, v); + } + item = comma ? comma + SKIP_ONE : NULL; + } +} + +/* Scan ONE manifest document's text for its name, namespace, kind and * label pairs into `rec` / `out`. A small indentation path-stack gives every * leaf its parent path, which tells spec.selector from the pod template's * labels and from the object's own metadata.labels. */ @@ -305,7 +345,7 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec, k8s_scanned_t memcpy(line, p, cp); line[cp] = '\0'; - /* End of first YAML document — stop (one Resource per file). */ + /* End of the document — stop (the caller hands over one document). */ const char *trimmed = line; while (*trimmed == ' ') { trimmed++; @@ -329,7 +369,7 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec, k8s_scanned_t depth--; } char key[64]; - char val[K8S_LABEL_LEN]; + char val[CBM_SZ_512]; /* a whole flow map fits: the line is the limit */ if (k8s_split_kv(trimmed, key, sizeof(key), val, sizeof(val))) { if (val[0] == '\0') { /* Block-opening key: push onto the path stack. */ @@ -340,6 +380,8 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec, k8s_scanned_t } } else if (ind == 0 && strcmp(key, "kind") == 0) { rec->kind = k8s_rec_kind(val); + } else if (val[0] == '{') { + k8s_scan_flow_map(out, (const char (*)[64])keys, depth, key, val); } else if (depth == 1 && strcmp(keys[0], "metadata") == 0) { if (!got_name && strcmp(key, "name") == 0) { snprintf(rec->name, sizeof(rec->name), "%s", val); @@ -419,6 +461,115 @@ static void k8s_record_manifest(k8s_record_array_t *recs, int64_t node_id, const cbm_free(CBM_MEM_CLASS_OTHER, scanned.items); } +/* Record the document at lines [start_line, end_line] (1-based, inclusive) of + * a multi-document manifest: every object's selector or pod labels come from + * its own document. */ +static void k8s_record_document(k8s_record_array_t *recs, int64_t node_id, const char *source, + int start_line, int end_line) { + const char *begin = source; + for (int line = 1; begin && line < start_line; line++) { + begin = strchr(begin, '\n'); + begin = begin ? begin + SKIP_ONE : NULL; + } + if (!begin) { + return; + } + const char *end = begin; + for (int line = start_line; *end && line <= end_line; line++) { + const char *eol = strchr(end, '\n'); + end = eol ? eol + SKIP_ONE : end + strlen(end); + } + size_t n = (size_t)(end - begin); + char *doc = cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE); + if (!doc) { + return; + } + memcpy(doc, begin, n); + doc[n] = '\0'; + k8s_record_manifest(recs, node_id, doc); + cbm_free(CBM_MEM_CLASS_OTHER, doc); +} + +/* Built-in kinds that live outside any namespace (IDENTITY.md). A CRD's own + * scope (spec.scope) is the curated / generated table's job. */ +static bool k8s_cluster_scoped(const char *kind) { + static const char *const kinds[] = {"Namespace", + "Node", + "PersistentVolume", + "ClusterRole", + "ClusterRoleBinding", + "CustomResourceDefinition", + "StorageClass", + "PriorityClass", + "ValidatingWebhookConfiguration", + "MutatingWebhookConfiguration", + "APIService", + "CSIDriver", + "CSINode", + "IngressClass", + "RuntimeClass", + "VolumeAttachment", + "ValidatingAdmissionPolicy", + "ValidatingAdmissionPolicyBinding", + "ClusterIssuer"}; + for (size_t i = 0; i < sizeof(kinds) / sizeof(kinds[0]); i++) { + if (strcmp(kind, kinds[i]) == 0) { + return true; + } + } + return false; +} + +/* A Resource's properties from its identity key, the part of its QN after + * "::": `///` (IDENTITY.md). + * `obj_key` is the logical identity: the namespace it lives in, `_` for a + * cluster-scoped kind, `?` when the deploy decides. NULL for a QN without a + * well-formed key (the caller keeps the plain source tag). */ +static char *k8s_resource_props(const char *qn) { + enum { K8S_KEY_PARTS = 4, K8S_ESC = 6 * K8S_LABEL_LEN }; + const char *key = qn ? strstr(qn, "::") : NULL; + if (!key) { + return NULL; + } + key += K8S_SLEN("::"); + char part[K8S_KEY_PARTS][K8S_LABEL_LEN]; + int parts = 0; + for (const char *p = key; parts < K8S_KEY_PARTS; parts++) { + const char *slash = parts < K8S_KEY_PARTS - SKIP_ONE ? strchr(p, '/') : NULL; + size_t len = slash ? (size_t)(slash - p) : strlen(p); + if (len == 0 || len >= K8S_LABEL_LEN || (!slash && parts < K8S_KEY_PARTS - SKIP_ONE)) { + return NULL; + } + memcpy(part[parts], p, len); + part[parts][len] = '\0'; + p = slash ? slash + SKIP_ONE : p + len; + } + const char *group = part[0]; + const char *kind = part[1]; + const char *ns = strcmp(part[2], "-") == 0 ? "" : part[2]; + const char *name = part[3]; + const char *effective = k8s_cluster_scoped(kind) ? "_" : ns[0] ? ns : "?"; + char esc[K8S_KEY_PARTS + SKIP_ONE][K8S_ESC]; + cbm_json_escape(esc[0], K8S_ESC, group); + cbm_json_escape(esc[1], K8S_ESC, kind); + cbm_json_escape(esc[2], K8S_ESC, ns); + cbm_json_escape(esc[3], K8S_ESC, name); + cbm_json_escape(esc[4], K8S_ESC, effective); + size_t n = K8S_KEY_PARTS * K8S_ESC + K8S_ESC + CBM_SZ_256; + char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, n); + if (!props) { + return NULL; + } + int w = + snprintf(props, n, "{\"source\":\"k8s\",\"group\":\"%s\",\"kind\":\"%s\"", esc[0], esc[1]); + if (ns[0]) { + w += snprintf(props + w, n - (size_t)w, ",\"namespace\":\"%s\"", esc[2]); + } + snprintf(props + w, n - (size_t)w, ",\"obj_key\":\"%s/%s/%s/%s\"}", esc[0], esc[1], esc[4], + esc[3]); + return props; +} + static void k8s_records_free(k8s_record_array_t *recs) { for (int i = 0; i < recs->pair_count; i++) { cbm_free(CBM_MEM_CLASS_OTHER, recs->pairs[i]); @@ -541,12 +692,11 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t /* ── K8s manifest handler ────────────────────────────────────────── */ /* source/src_len are the already-read file bytes (caller retains ownership and - * must free after this call returns). The first Resource's manifest is + * must free after this call returns). Every Resource's own document is * recorded in `recs` for cross-manifest selector matching. */ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, const CBMFileResult *res, k8s_record_array_t *recs) { int resource_count = 0; - int64_t first_node_id = 0; if (!res) { return; @@ -566,28 +716,25 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c continue; } - int64_t node_id = - cbm_gbuf_upsert_node(ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, - (int)def->start_line, (int)def->end_line, "{\"source\":\"k8s\"}"); + char *props = k8s_resource_props(def->qualified_name); + int64_t node_id = cbm_gbuf_upsert_node( + ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, (int)def->start_line, + (int)def->end_line, props ? props : "{\"source\":\"k8s\"}"); + cbm_free(CBM_MEM_CLASS_OTHER, props); /* DEFINES edge: File → Resource */ if (file_node && node_id > 0) { cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}"); } - /* Capture the first Resource for cross-manifest selector matching. */ - if (first_node_id <= 0 && node_id > 0) { - first_node_id = node_id; + /* Record its selector / pod-label pairs for Service → workload linking. */ + if (node_id > 0 && source) { + k8s_record_document(recs, node_id, source, (int)def->start_line, (int)def->end_line); } resource_count++; } - /* Record selector / pod-label pairs for later Service → workload linking. */ - if (first_node_id > 0 && source) { - k8s_record_manifest(recs, first_node_id, source); - } - cbm_log_info("pass.k8s.manifest", "file", rel_path, "resources", itoa_k8s(resource_count)); } diff --git a/src/store/store.h b/src/store/store.h index 8dc67ee48..d64041192 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -24,7 +24,12 @@ typedef struct cbm_store_read_scope cbm_store_read_scope_t; #define CBM_STORE_OK 0 #define CBM_STORE_ERR (-1) #define CBM_STORE_NOT_FOUND (-2) -#define CBM_INDEX_FORMAT_VERSION 1 +/* Persisted index format. A bump routes every existing index through one full + * rebuild (pipeline.c, format_change_reindex) that keeps ADR/project metadata. + * 1 #1108: File-node QNs keep the file extension. + * 2 Kubernetes objects: one Resource per YAML document, QN + * ::///. */ +#define CBM_INDEX_FORMAT_VERSION 2 #define CBM_STORE_CANCELLED (-3) #define CBM_STORE_SCAN_LIMIT (-4) #define CBM_STORE_CALLBACK_ERR (-5) diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 37638e192..c3e7e6afd 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -762,6 +762,204 @@ TEST(probe_k8s_manifest_metadata_after_key_line_comment) { PASS(); } +/* ── Identity: every document is an object (IDENTITY.md) ──────────── */ + +/* The QN and properties of the Resource named `name` after indexing `files`; + * false when there is no such node or it is not unique. */ +static bool gpf_resource(const GpfFile *files, int nfiles, const char *name, char *qn, + size_t qn_sz, char *props, size_t props_sz) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + bool found = false; + int hits = 0; + cbm_node_t *nodes = NULL; + int count = 0; + if (store && + cbm_store_find_nodes_by_label(store, lp.project, "Resource", &nodes, &count) == + CBM_STORE_OK) { + for (int i = 0; i < count; i++) { + if (nodes[i].name && strcmp(nodes[i].name, name) == 0) { + hits++; + const char *q = nodes[i].qualified_name ? nodes[i].qualified_name : ""; + const char *sep = strstr(q, "::"); + snprintf(qn, qn_sz, "%s", sep ? sep : q); /* the part after the file base */ + snprintf(props, props_sz, "%s", + nodes[i].properties_json ? nodes[i].properties_json : "{}"); + } + } + cbm_store_free_nodes(nodes, count); + found = hits == 1; + } + gpf_cleanup(&lp, store); + return found; +} + +/* A multi-document manifest is a list of objects: kubernetes-sigs and + * microservices-demo ship a Deployment and its Service in one file, and + * 3,139 objects of the 17 infra corpora sat behind a file's first `---`. */ +TEST(probe_k8s_every_document_is_an_object) { + GpfMetrics m = gpf_metrics("app.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "---\n" + "apiVersion: v1\n" + "kind: ConfigMap\n" + "metadata:\n" + " name: web-config\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 3); + PASS(); +} + +/* ...and the Service in the second document selects the Deployment in the + * first. */ +TEST(probe_k8s_selector_across_documents) { + GpfMetrics m = gpf_metrics("app.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n" + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " app: web\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 2); + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + +/* Flow-style label maps (`labels: { app: kdc }`, deckhouse's kerberos demo) + * are label maps; a selector given that way selects just the same. */ +TEST(probe_k8s_selector_flow_mapping) { + GpfMetrics m = gpf_metrics("kdc.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: kdc\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels: { app: kdc, tier: \"auth\" }\n" + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: kdc\n" + "spec:\n" + " selector: { app: kdc }\n" + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: other\n" + "spec:\n" + " selector: {app: kdc, tier: web}\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 3); + ASSERT_EQ(m.infra_maps, 1); /* `other` needs tier=web, which kdc lacks */ + PASS(); +} + +/* An anchored name (`name: &name app`, onedr0p/home-ops reuses it in the + * chart values) is still the object's name; so is a tagged one. */ +TEST(probe_k8s_anchored_and_tagged_names) { + GpfMetrics m = gpf_metrics("release.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\n" + "kind: HelmRelease\n" + "metadata:\n" + " name: &name actions-runner-controller\n" + "spec:\n" + " releaseName: *name\n" + "---\n" + "apiVersion: v1\n" + "kind: ConfigMap\n" + "metadata:\n" + " name: !!str settings\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 2); + PASS(); +} + +/* One name in two namespaces is two objects. */ +TEST(probe_k8s_same_name_two_namespaces) { + GpfMetrics m = gpf_metrics("app.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + " namespace: staging\n" + "---\n" + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + " namespace: prod\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 2); + PASS(); +} + +/* The node QN carries group, kind, declared namespace and name after the + * file; `obj_key` is the logical identity (effective namespace, `?` when the + * deploy decides it). */ +TEST(probe_k8s_identity_key) { + static const GpfFile files[] = {{"app.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n"}}; + char qn[256]; + char props[512]; + ASSERT_TRUE(gpf_resource(files, 1, "Deployment/web", qn, sizeof(qn), props, sizeof(props))); + ASSERT_STR_EQ(qn, "::apps/Deployment/-/web"); + ASSERT_NOT_NULL(strstr(props, "\"obj_key\":\"apps/Deployment/?/web\"")); + ASSERT_NOT_NULL(strstr(props, "\"kind\":\"Deployment\"")); + ASSERT_NOT_NULL(strstr(props, "\"group\":\"apps\"")); + PASS(); +} + +/* Core-group objects say `core`; a declared namespace is part of the QN; a + * cluster-scoped kind's logical key has no namespace (`_`). */ +TEST(probe_k8s_identity_core_namespaced_and_cluster_scoped) { + static const GpfFile svc[] = {{"svc.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + " namespace: prod\n"}}; + static const GpfFile ns[] = {{"ns.yaml", + "apiVersion: v1\n" + "kind: Namespace\n" + "metadata:\n" + " name: prod\n"}}; + char qn[256]; + char props[512]; + ASSERT_TRUE(gpf_resource(svc, 1, "Service/web", qn, sizeof(qn), props, sizeof(props))); + ASSERT_STR_EQ(qn, "::core/Service/prod/web"); + ASSERT_NOT_NULL(strstr(props, "\"obj_key\":\"core/Service/prod/web\"")); + ASSERT_NOT_NULL(strstr(props, "\"namespace\":\"prod\"")); + ASSERT_TRUE(gpf_resource(ns, 1, "Namespace/prod", qn, sizeof(qn), props, sizeof(props))); + ASSERT_NOT_NULL(strstr(props, "\"obj_key\":\"core/Namespace/_/prod\"")); + PASS(); +} + /* The same comment in front of a kustomization hid its resource list: * home-ops resolved 1 of 348 entries. */ TEST(probe_kustomize_imports_after_leading_comment) { @@ -2156,6 +2354,13 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_kustomize_entries_after_commented_out_entry); RUN_TEST(probe_kustomize_entries_after_key_line_comment); RUN_TEST(probe_k8s_manifest_metadata_after_key_line_comment); + RUN_TEST(probe_k8s_every_document_is_an_object); + RUN_TEST(probe_k8s_selector_across_documents); + RUN_TEST(probe_k8s_same_name_two_namespaces); + RUN_TEST(probe_k8s_identity_key); + RUN_TEST(probe_k8s_identity_core_namespaced_and_cluster_scoped); + RUN_TEST(probe_k8s_anchored_and_tagged_names); + RUN_TEST(probe_k8s_selector_flow_mapping); /* Nix */ RUN_TEST(probe_nix_attrset_node); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 0ad711800..f666f993c 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14274,15 +14274,8 @@ TEST(k8s_extract_manifest_no_name) { } TEST(k8s_extract_manifest_multidoc) { - /* Two-document YAML separated by "---". - * extract_k8s_manifest contains a "break" after the first successful push, - * so it processes only the first document that has both kind and - * metadata.name. This test pins that behaviour: the first document's - * resource must be present and no crash must occur. - * - * Note: with some tree-sitter YAML grammar versions the root stream may - * expose both documents as siblings; the break still fires after the first - * successful def push, so defs.count must be exactly 1. */ + /* Two-document YAML separated by "---": every document is an object, with + * the identity QN ::///. */ const char *src = "apiVersion: apps/v1\n" "kind: Deployment\n" "metadata:\n" @@ -14296,17 +14289,24 @@ TEST(k8s_extract_manifest_multidoc) { "k8s/multi.yaml", 0, NULL, NULL); ASSERT(r != NULL); ASSERT(!r->has_error); - /* First document's resource must be present */ - int found = 0; + int deploy = 0; + int svc = 0; for (int i = 0; i < r->defs.count; i++) { - if (r->defs.items[i].label && strcmp(r->defs.items[i].label, "Resource") == 0 && - r->defs.items[i].name && strcmp(r->defs.items[i].name, "Deployment/my-app") == 0) { - found = 1; + const CBMDefinition *d = &r->defs.items[i]; + if (!d->label || strcmp(d->label, "Resource") != 0 || !d->name || !d->qualified_name) { + continue; + } + if (strcmp(d->name, "Deployment/my-app") == 0 && + strcmp(d->qualified_name, "myproj.k8s.multi.yaml::apps/Deployment/-/my-app") == 0) { + deploy++; + } + if (strcmp(d->name, "Service/my-svc") == 0 && + strcmp(d->qualified_name, "myproj.k8s.multi.yaml::core/Service/-/my-svc") == 0) { + svc++; } } - ASSERT(found); - /* At least one def, no more than one (only first document processed) */ - ASSERT(r->defs.count >= 1); + ASSERT_EQ(deploy, 1); + ASSERT_EQ(svc, 1); cbm_free_result(r); PASS(); } From b95d0cad83b0a879ba5a8192452b08390b7ec507 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 18:32:15 +0200 Subject: [PATCH 18/52] feat(k8s): a Service selects inside what is deployed with it Selector matching ran repository-wide: a Service linked to every same-labelled workload whose namespace did not contradict it. Repositories ship one application several times -- microservices-demo as kubernetes-manifests/, release/ and kustomize/base/; kubernetes/k8s.io one prow per cluster; kubermatic connectivity checks per CNI -- so each Service also "selected" the Deployments of every other copy: 109 of 236 INFRA_MAPS edges across the 17 infrastructure corpora joined objects that are never deployed together. Links now stay inside a bundle, what one deploy applies together: - a kustomize build root (a kustomization no other one includes) with every manifest its entries reach, through directories and nested kustomizations; a base several overlays include belongs to each; - else the Helm chart the file sits in; else the file's directory. A Service and a workload link when they share a bundle and their namespaces there agree. A build root's `namespace:` is where its objects live (kustomize applies the outermost namespace transformer last): when every build of a file agrees on one, the objects' `obj_key` carries it; builds that disagree leave it to the deploy (`?`). On the 17 corpora: exactly the 109 cross-copy links are gone, all 127 links inside one file or directory remain, none is added; object coverage unchanged (4,996 / 5,025); two runs identical. One real link is lost with them: gardener's registry LoadBalancer selects a StatefulSet another directory deploys; a script applies both (deploy-registry.sh), which only a deploy-script reader can see. Tests: two copies in two directories, a Service and workload joined by a build root across directories while a copy outside stays apart, and the build root's namespace in obj_key fail on the repository-wide matcher. Signed-off-by: Martin Vogel --- src/pipeline/pass_k8s.c | 485 ++++++++++++++++++++++++++++++++++- tests/test_grammar_probe_f.c | 58 +++++ 2 files changed, 529 insertions(+), 14 deletions(-) diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index ea7cdfa6a..818a791d4 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -523,9 +523,10 @@ static bool k8s_cluster_scoped(const char *kind) { /* A Resource's properties from its identity key, the part of its QN after * "::": `///` (IDENTITY.md). * `obj_key` is the logical identity: the namespace it lives in, `_` for a - * cluster-scoped kind, `?` when the deploy decides. NULL for a QN without a - * well-formed key (the caller keeps the plain source tag). */ -static char *k8s_resource_props(const char *qn) { + * cluster-scoped kind, `?` when the deploy decides; `effective_ns` (when + * not NULL) is the namespace a kustomize build gives it. NULL for a QN + * without a well-formed key (the caller keeps the plain source tag). */ +static char *k8s_resource_props(const char *qn, const char *effective_ns) { enum { K8S_KEY_PARTS = 4, K8S_ESC = 6 * K8S_LABEL_LEN }; const char *key = qn ? strstr(qn, "::") : NULL; if (!key) { @@ -548,7 +549,10 @@ static char *k8s_resource_props(const char *qn) { const char *kind = part[1]; const char *ns = strcmp(part[2], "-") == 0 ? "" : part[2]; const char *name = part[3]; - const char *effective = k8s_cluster_scoped(kind) ? "_" : ns[0] ? ns : "?"; + const char *effective = k8s_cluster_scoped(kind) ? "_" + : effective_ns ? effective_ns + : ns[0] ? ns + : "?"; char esc[K8S_KEY_PARTS + SKIP_ONE][K8S_ESC]; cbm_json_escape(esc[0], K8S_ESC, group); cbm_json_escape(esc[1], K8S_ESC, kind); @@ -628,14 +632,447 @@ static bool k8s_has_pair(const k8s_record_array_t *recs, const k8s_record_t *rec return false; } -/* Two set namespaces must agree; an unset one is the deploy-time namespace. */ -static bool k8s_same_namespace(const k8s_record_t *a, const k8s_record_t *b) { - return !a->ns[0] || !b->ns[0] || strcmp(a->ns, b->ns) == 0; +/* ── Bundles: what is deployed together (IDENTITY.md) ────────────── */ + +/* A Service selects pods of its own deployment, never a same-labelled copy + * elsewhere in the repository. A bundle is a kustomize build root (a + * kustomization no other one includes) with every manifest its entries + * reach; outside kustomize, the Helm chart a file sits in, else its + * directory. A base several overlays include belongs to each of them. */ + +typedef struct { + char *path; /* the kustomization file */ + char *ns; /* its `namespace:`, NULL when unset */ + int *kids; /* kustomizations its entries name (indices) */ + int kid_n; + int kid_cap; + const char **files; /* manifests its entries name (graph-owned paths) */ + int file_n; + int file_cap; + bool referenced; /* included by another kustomization */ +} k8s_kust_t; + +typedef struct { + k8s_kust_t *items; + int count; + int cap; +} k8s_kusts_t; + +/* One bundle a manifest belongs to, with the namespace the bundle gives + * its objects (NULL: their own). `bundle` is an interned name, compared by + * pointer. */ +typedef struct { + const char *bundle; + const char *ns; +} k8s_member_t; + +typedef struct { + k8s_member_t *items; + int count; + int cap; +} k8s_members_t; + +typedef struct { + CBMHashTable *names; /* bundle name -> itself (owned key) */ + CBMHashTable *members; /* manifest path -> k8s_members_t* */ + CBMHashTable *charts; /* Helm chart directory -> itself (owned key) */ +} k8s_bundles_t; + +static bool k8s_grow(void **items, int *cap, int count, size_t elem) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : K8S_GROW_MIN; + void *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, *items, (size_t)ncap * elem); + if (!grown) { + return false; + } + *items = grown; + *cap = ncap; + return true; +} + +/* The top-level `namespace:` of a kustomization, NULL when it sets none. */ +static char *k8s_kustomize_namespace(const char *source) { + for (const char *p = source; p && *p;) { + const char *eol = strchr(p, '\n'); + size_t len = eol ? (size_t)(eol - p) : strlen(p); + if (strncmp(p, "namespace:", K8S_SLEN("namespace:")) == 0) { + char line[CBM_SZ_512]; + snprintf(line, sizeof(line), "%.*s", (int)len, p); + char key[64]; + char val[K8S_LABEL_LEN]; + if (k8s_split_kv(line, key, sizeof(key), val, sizeof(val)) && val[0]) { + return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, val); + } + return NULL; + } + p = eol ? eol + SKIP_ONE : NULL; + } + return NULL; +} + +static void k8s_kusts_push(k8s_kusts_t *ks, const char *path, char *ns) { + if (!k8s_grow((void **)&ks->items, &ks->cap, ks->count, sizeof(*ks->items))) { + cbm_free(CBM_MEM_CLASS_OTHER, ns); + return; + } + ks->items[ks->count++] = + (k8s_kust_t){.path = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, path), .ns = ns}; +} + +static void k8s_kusts_free(k8s_kusts_t *ks) { + for (int i = 0; i < ks->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].path); + cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].ns); + cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].kids); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)ks->items[i].files); + } + cbm_free(CBM_MEM_CLASS_OTHER, ks->items); + *ks = (k8s_kusts_t){0}; +} + +static int k8s_cmp_str(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +static int k8s_cmp_int(const void *a, const void *b) { + int x = *(const int *)a; + int y = *(const int *)b; + return (x > y) - (x < y); +} + +static int k8s_cmp_kust(const void *a, const void *b) { + return strcmp(((const k8s_kust_t *)a)->path, ((const k8s_kust_t *)b)->path); +} + +/* index + 1 of the kustomization at `path`, 0 when none. */ +static int k8s_kust_at(CBMHashTable *by_path, const char *path) { + return (int)(intptr_t)cbm_ht_get(by_path, path); +} + +/* The kustomization file inside directory `dir`, as kustomize looks for it. */ +static int k8s_kust_in_dir(CBMHashTable *by_path, const char *dir) { + static const char *const names[] = {"kustomization.yaml", "kustomization.yml", "Kustomization"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + int k = k8s_kust_at(by_path, path); + if (k) { + return k; + } + } + return 0; +} + +/* Wire each kustomization to what its entries name, from the IMPORTS edges + * the kustomize resolver drew: a directory means the kustomization in it, + * a file another kustomization or a manifest. */ +static void k8s_kusts_wire(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, CBMHashTable *by_path) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + if (cbm_gbuf_find_edges_by_type(ctx->gbuf, "IMPORTS", &edges, &count) != 0) { + return; + } + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, edges[i]->source_id); + int k = src && src->file_path ? k8s_kust_at(by_path, src->file_path) : 0; + const cbm_gbuf_node_t *tgt = k ? cbm_gbuf_find_by_id(ctx->gbuf, edges[i]->target_id) : NULL; + if (!tgt || !tgt->file_path) { + continue; + } + k8s_kust_t *kust = &ks->items[k - SKIP_ONE]; + int kid = tgt->label && strcmp(tgt->label, "Folder") == 0 + ? k8s_kust_in_dir(by_path, tgt->file_path) + : k8s_kust_at(by_path, tgt->file_path); + if (kid && kid != k) { + if (k8s_grow((void **)&kust->kids, &kust->kid_cap, kust->kid_n, sizeof(int))) { + kust->kids[kust->kid_n++] = kid - SKIP_ONE; + ks->items[kid - SKIP_ONE].referenced = true; + } + } else if (!kid && strcmp(tgt->label ? tgt->label : "", "Folder") != 0) { + if (k8s_grow((void **)&kust->files, &kust->file_cap, kust->file_n, + sizeof(*kust->files))) { + kust->files[kust->file_n++] = tgt->file_path; + } + } + } + for (int i = 0; i < ks->count; i++) { + qsort(ks->items[i].files, (size_t)ks->items[i].file_n, sizeof(*ks->items[i].files), + k8s_cmp_str); + qsort(ks->items[i].kids, (size_t)ks->items[i].kid_n, sizeof(int), k8s_cmp_int); + } +} + +static const char *k8s_bundle_name(k8s_bundles_t *b, const char *kind, const char *where) { + char name[CBM_SZ_1K]; + snprintf(name, sizeof(name), "%s:%s", kind, where); + const char *known = cbm_ht_get_key(b->names, name); + if (known) { + return known; + } + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, name); + if (owned) { + cbm_ht_set(b->names, owned, owned); + } + return owned; +} + +static void k8s_member_add(k8s_bundles_t *b, const char *file, const char *bundle, const char *ns) { + k8s_members_t *m = cbm_ht_get(b->members, file); + if (!m) { + m = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*m)); + if (!m) { + return; + } + cbm_ht_set(b->members, file, m); + } + for (int i = 0; i < m->count; i++) { + if (m->items[i].bundle == bundle) { + return; /* reached twice from one root: the first path's namespace */ + } + } + if (k8s_grow((void **)&m->items, &m->cap, m->count, sizeof(*m->items))) { + m->items[m->count++] = (k8s_member_t){bundle, ns}; + } +} + +/* Every build root's manifests, each with the namespace the outermost + * kustomization on its path sets (kustomize applies the root's namespace + * transformer last, so it wins). */ +static void k8s_bundles_from_kusts(k8s_bundles_t *b, k8s_kusts_t *ks) { + typedef struct { + int kust; + const char *ns; + } frame_t; + int *seen = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(ks->count ? ks->count : SKIP_ONE) * sizeof(int)); + frame_t *stack = NULL; + int depth = 0; + int cap = 0; + for (int r = 0; seen && r < ks->count; r++) { + if (ks->items[r].referenced) { + continue; + } + const char *bundle = k8s_bundle_name(b, "kustomize", ks->items[r].path); + if (!bundle || !k8s_grow((void **)&stack, &cap, depth, sizeof(*stack))) { + continue; + } + stack[depth++] = (frame_t){r, ks->items[r].ns}; + while (depth > 0) { + frame_t f = stack[--depth]; + if (seen[f.kust] == r + SKIP_ONE) { + continue; + } + seen[f.kust] = r + SKIP_ONE; + const k8s_kust_t *k = &ks->items[f.kust]; + for (int i = 0; i < k->file_n; i++) { + k8s_member_add(b, k->files[i], bundle, f.ns); + } + for (int i = k->kid_n - SKIP_ONE; i >= 0; i--) { + if (k8s_grow((void **)&stack, &cap, depth, sizeof(*stack))) { + int kid = k->kids[i]; + stack[depth++] = (frame_t){kid, f.ns ? f.ns : ks->items[kid].ns}; + } + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, stack); + cbm_free(CBM_MEM_CLASS_OTHER, seen); +} + +static void k8s_dirname(const char *path, char *out, size_t out_sz) { + const char *slash = strrchr(path, '/'); + snprintf(out, out_sz, "%.*s", slash ? (int)(slash - path) : 0, path); +} + +/* The bundles of the manifest at `file`: its kustomize build roots; else + * the Helm chart it sits in; else its directory. */ +static const k8s_members_t *k8s_bundles_of(k8s_bundles_t *b, const char *file, + k8s_members_t *single) { + const k8s_members_t *m = cbm_ht_get(b->members, file); + if (m && m->count > 0) { + return m; + } + char dir[CBM_SZ_1K]; + k8s_dirname(file, dir, sizeof(dir)); + char up[CBM_SZ_1K]; + snprintf(up, sizeof(up), "%s", dir); + const char *bundle = NULL; + while (!bundle) { + if (cbm_ht_get(b->charts, up)) { + bundle = k8s_bundle_name(b, "chart", up); + break; + } + if (!up[0]) { + break; + } + char parent[CBM_SZ_1K]; + k8s_dirname(up, parent, sizeof(parent)); + snprintf(up, sizeof(up), "%s", parent); + } + if (!bundle) { + bundle = k8s_bundle_name(b, "dir", dir); + } + single->items[0] = (k8s_member_t){bundle, NULL}; + single->count = bundle ? SKIP_ONE : 0; + return single; +} + +static void k8s_free_owned_key(const char *key, void *value, void *ud) { + (void)value; + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static void k8s_free_members(const char *key, void *value, void *ud) { + (void)key; + (void)ud; + k8s_members_t *m = value; + cbm_free(CBM_MEM_CLASS_OTHER, m->items); + cbm_free(CBM_MEM_CLASS_OTHER, m); +} + +static void k8s_bundles_free(k8s_bundles_t *b) { + if (b->members) { + cbm_ht_foreach(b->members, k8s_free_members, NULL); + cbm_ht_free(b->members); + } + if (b->names) { + cbm_ht_foreach(b->names, k8s_free_owned_key, NULL); + cbm_ht_free(b->names); + } + if (b->charts) { + cbm_ht_foreach(b->charts, k8s_free_owned_key, NULL); + cbm_ht_free(b->charts); + } + *b = (k8s_bundles_t){0}; +} + +/* A Helm chart's directory bundles the plain manifests below it. */ +static void k8s_chart_dir_add(k8s_bundles_t *b, const char *chart_yaml) { + char dir[CBM_SZ_1K]; + k8s_dirname(chart_yaml, dir, sizeof(dir)); + if (b->charts && !cbm_ht_get(b->charts, dir)) { + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir); + if (owned) { + cbm_ht_set(b->charts, owned, owned); + } + } +} + +/* Build the kustomize bundles (call once every manifest and kustomization is + * known and the IMPORTS edges exist). */ +static void k8s_bundles_build(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b) { + b->names = cbm_ht_create(0); + b->members = cbm_ht_create(0); + if (!b->charts) { + b->charts = cbm_ht_create(0); + } + if (!b->names || !b->members || !b->charts) { + return; + } + qsort(ks->items, (size_t)ks->count, sizeof(*ks->items), k8s_cmp_kust); + CBMHashTable *by_path = cbm_ht_create(0); + if (!by_path) { + return; + } + for (int i = 0; i < ks->count; i++) { + cbm_ht_set(by_path, ks->items[i].path, (void *)(intptr_t)(i + SKIP_ONE)); + } + k8s_kusts_wire(ctx, ks, by_path); + cbm_ht_free(by_path); + k8s_bundles_from_kusts(b, ks); +} + +/* A Service and a workload meet when they share a bundle whose namespaces + * agree: two set namespaces must be equal; an unset one is the deploy's. */ +static bool k8s_share_bundle(k8s_bundles_t *b, const cbm_pipeline_ctx_t *ctx, + const k8s_record_t *svc, const k8s_record_t *wl) { + const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_id(ctx->gbuf, svc->node_id); + const cbm_gbuf_node_t *wn = cbm_gbuf_find_by_id(ctx->gbuf, wl->node_id); + if (!sn || !wn || !sn->file_path || !wn->file_path) { + return false; + } + k8s_member_t one_s[SKIP_ONE]; + k8s_member_t one_w[SKIP_ONE]; + k8s_members_t single_s = {.items = one_s, .cap = SKIP_ONE}; + k8s_members_t single_w = {.items = one_w, .cap = SKIP_ONE}; + const k8s_members_t *ms = k8s_bundles_of(b, sn->file_path, &single_s); + const k8s_members_t *mw = k8s_bundles_of(b, wn->file_path, &single_w); + for (int i = 0; i < ms->count; i++) { + for (int j = 0; j < mw->count; j++) { + if (ms->items[i].bundle != mw->items[j].bundle) { + continue; + } + const char *ns_s = ms->items[i].ns ? ms->items[i].ns : svc->ns; + const char *ns_w = mw->items[j].ns ? mw->items[j].ns : wl->ns; + if (!ns_s[0] || !ns_w[0] || strcmp(ns_s, ns_w) == 0) { + return true; + } + } + } + return false; +} + +/* The declared namespace inside a Resource QN's key ("" when `-`). */ +static void k8s_declared_ns(const char *qn, char *out, size_t out_sz) { + out[0] = '\0'; + const char *key = qn ? strstr(qn, "::") : NULL; + const char *p = key ? key + K8S_SLEN("::") : NULL; + for (int part = 0; p && part < PAIR_LEN; part++) { + p = strchr(p, '/'); + p = p ? p + SKIP_ONE : NULL; + } + const char *end = p ? strchr(p, '/') : NULL; + if (end && !(end - p == SKIP_ONE && p[0] == '-')) { + snprintf(out, out_sz, "%.*s", (int)(end - p), p); + } +} + +/* Objects a kustomize build deploys live in the namespace it gives them: + * when every bundle of the file agrees on one, that is the namespace of + * the object's logical key; bundles that disagree leave it to the deploy. */ +static void k8s_apply_effective_namespaces(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!b->members || cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &count) != 0) { + return; + } + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const k8s_members_t *m = n->file_path ? cbm_ht_get(b->members, n->file_path) : NULL; + if (!m || m->count == 0) { + continue; + } + char declared[K8S_LABEL_LEN]; + k8s_declared_ns(n->qualified_name, declared, sizeof(declared)); + const char *eff = NULL; + bool agree = true; + bool any_set = false; + for (int j = 0; j < m->count; j++) { + const char *v = m->items[j].ns ? m->items[j].ns : declared; + any_set = any_set || m->items[j].ns; + if (!v[0] || (eff && strcmp(eff, v) != 0)) { + agree = false; + } + eff = eff ? eff : v; + } + if (!any_set) { + continue; /* no build sets a namespace: the declared one stands */ + } + char *props = k8s_resource_props(n->qualified_name, agree ? eff : "?"); + if (props) { + (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)n, props); + cbm_free(CBM_MEM_CLASS_OTHER, props); + } + } } /* Connect each Service to the workloads its selector targets via an INFRA_MAPS - * edge (Service Resource → workload Resource). */ -static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs) { + * edge (Service Resource → workload Resource), inside a shared bundle. */ +static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs, + k8s_bundles_t *bundles) { CBMHashTable *index = cbm_ht_create(0); if (!index) { return; @@ -668,7 +1105,7 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t } for (int c = 0; rarest && c < rarest->count; c++) { const k8s_record_t *wl = &recs->items[rarest->items[c]]; - bool all = k8s_same_namespace(svc, wl); + bool all = k8s_share_bundle(bundles, ctx, svc, wl); for (int p = 0; all && p < svc->pair_n; p++) { all = k8s_has_pair(recs, wl, recs->pairs[svc->pair_off + p]); } @@ -716,7 +1153,7 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c continue; } - char *props = k8s_resource_props(def->qualified_name); + char *props = k8s_resource_props(def->qualified_name, NULL); int64_t node_id = cbm_gbuf_upsert_node( ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, (int)def->start_line, (int)def->end_line, props ? props : "{\"source\":\"k8s\"}"); @@ -1056,7 +1493,7 @@ static void handle_dep_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c * the sequential loop did. Sequentially the pass re-read and re-parsed every * YAML manifest on one thread -- 1.1 s of the Go corpus (6,275 manifests; * profile + waste sanitizer, 2026-09-17). */ -typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_HELM, K8S_MANIFEST } k8s_kind_t; +typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_HELM, K8S_MANIFEST, K8S_KUSTOMIZE } k8s_kind_t; typedef struct { k8s_kind_t kind; @@ -1086,6 +1523,10 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { if (is_gomod_file(base) || lang == CBM_LANG_GOMOD || is_requirements_file(base)) { out->source = k8s_read_file(path, &out->src_len); out->kind = out->source ? K8S_DEP : K8S_SKIP; + } else if (lang == CBM_LANG_KUSTOMIZE) { + /* Its `namespace:` is the namespace of the objects it builds. */ + out->source = k8s_read_file(path, &out->src_len); + out->kind = out->source ? K8S_KUSTOMIZE : K8S_SKIP; } else if (lang == CBM_LANG_YAML || lang == CBM_LANG_K8S) { /* Read source once to classify (and reuse for uncached extraction). */ out->source = k8s_read_file(path, &out->src_len); @@ -1137,9 +1578,12 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, /* Collect per-manifest selector/label records for cross-manifest matching. */ k8s_record_array_t recs = {0}; k8s_local_deps_t local_deps = {0}; + k8s_kusts_t kusts = {0}; + k8s_bundles_t bundles = {.charts = cbm_ht_create(0)}; k8s_prep_t *prep = cbm_calloc(CBM_MEM_CLASS_OTHER, K8S_CHUNK * sizeof(*prep)); if (!prep) { + k8s_bundles_free(&bundles); return CBM_NOT_FOUND; } int workers = cbm_default_worker_count(false); @@ -1150,6 +1594,8 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_free(CBM_MEM_CLASS_OTHER, prep); k8s_records_free(&recs); k8s_local_deps_free(&local_deps); + k8s_kusts_free(&kusts); + k8s_bundles_free(&bundles); return CBM_NOT_FOUND; } memset(prep, 0, (size_t)K8S_CHUNK * sizeof(*prep)); @@ -1175,8 +1621,12 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, break; case K8S_HELM: handle_helm_chart(ctx, rel, p->source, &local_deps); + k8s_chart_dir_add(&bundles, rel); helm_count++; break; + case K8S_KUSTOMIZE: + k8s_kusts_push(&kusts, rel, k8s_kustomize_namespace(p->source)); + break; case K8S_MANIFEST: handle_k8s_manifest(ctx, rel, p->source, p->res, &recs); manifest_count++; @@ -1195,14 +1645,21 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_free(CBM_MEM_CLASS_OTHER, prep); k8s_records_free(&recs); k8s_local_deps_free(&local_deps); + k8s_kusts_free(&kusts); + k8s_bundles_free(&bundles); return CBM_NOT_FOUND; } } cbm_free(CBM_MEM_CLASS_OTHER, prep); - /* Connect Services to the workloads their selectors target (INFRA_MAPS). */ - k8s_link_selectors(ctx, &recs); + /* Connect Services to the workloads their selectors target (INFRA_MAPS), + * inside the bundles that deploy them together. */ + k8s_bundles_build(ctx, &kusts, &bundles); + k8s_apply_effective_namespaces(ctx, &bundles); + k8s_link_selectors(ctx, &recs, &bundles); k8s_records_free(&recs); + k8s_bundles_free(&bundles); + k8s_kusts_free(&kusts); /* Link `file://` chart dependencies now that every Chart.yaml is read. */ k8s_link_local_deps(ctx, &local_deps); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index c3e7e6afd..5785ba0ab 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -845,6 +845,61 @@ TEST(probe_k8s_selector_across_documents) { PASS(); } +/* ── Bundles: a Service selects inside what is deployed together ───── */ + +#define GPF_DEPLOY(name, label) \ + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: " name "\nspec:\n template:\n" \ + " metadata:\n labels:\n app: " label "\n" +#define GPF_SVC(name, label) \ + "apiVersion: v1\nkind: Service\nmetadata:\n name: " name "\nspec:\n selector:\n app: " \ + label "\n" + +/* Two copies of one app in two directories (microservices-demo ships + * kubernetes-manifests/, release/ and kustomize/base/; k8s.io one prow per + * cluster): each Service selects its own copy's Deployment, not the other. */ +TEST(probe_k8s_selector_stays_in_its_directory) { + static const GpfFile files[] = {{"gke/svc.yaml", GPF_SVC("web", "web")}, + {"gke/deploy.yaml", GPF_DEPLOY("web", "web")}, + {"eks/deploy.yaml", GPF_DEPLOY("web", "web")}}; + GpfMetrics m = gpf_metrics_files(files, 3); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 3); + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + +/* A kustomize build root deploys what its entries reach, across + * directories; a same-labelled Deployment outside it is another bundle. */ +TEST(probe_k8s_selector_within_kustomize_build) { + static const GpfFile files[] = { + {"overlay/kustomization.yaml", "resources:\n - ../svc\n - ../app\n"}, + {"svc/kustomization.yaml", "resources:\n - svc.yaml\n"}, + {"svc/svc.yaml", GPF_SVC("web", "web")}, + {"app/kustomization.yaml", "resources:\n - deploy.yaml\n"}, + {"app/deploy.yaml", GPF_DEPLOY("web", "web")}, + {"other/deploy.yaml", GPF_DEPLOY("web-copy", "web")}}; + GpfMetrics m = gpf_metrics_files(files, 6); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 3); + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + +/* The build root's `namespace:` is where its objects live: the logical + * key carries it (kustomize's namespace transformer, outermost wins). */ +TEST(probe_k8s_kustomize_namespace_is_effective) { + static const GpfFile files[] = { + {"prod/kustomization.yaml", "namespace: prod\nresources:\n - ../base\n"}, + {"base/kustomization.yaml", "namespace: base-ns\nresources:\n - deploy.yaml\n"}, + {"base/deploy.yaml", GPF_DEPLOY("web", "web")}}; + char qn[256]; + char props[512]; + ASSERT_TRUE(gpf_resource(files, 3, "Deployment/web", qn, sizeof(qn), props, sizeof(props))); + ASSERT_STR_EQ(qn, "::apps/Deployment/-/web"); + ASSERT_NOT_NULL(strstr(props, "\"obj_key\":\"apps/Deployment/prod/web\"")); + PASS(); +} + /* Flow-style label maps (`labels: { app: kdc }`, deckhouse's kerberos demo) * are label maps; a selector given that way selects just the same. */ TEST(probe_k8s_selector_flow_mapping) { @@ -2361,6 +2416,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_identity_core_namespaced_and_cluster_scoped); RUN_TEST(probe_k8s_anchored_and_tagged_names); RUN_TEST(probe_k8s_selector_flow_mapping); + RUN_TEST(probe_k8s_selector_stays_in_its_directory); + RUN_TEST(probe_k8s_selector_within_kustomize_build); + RUN_TEST(probe_k8s_kustomize_namespace_is_effective); /* Nix */ RUN_TEST(probe_nix_attrset_node); From 092c0d22c248895e7c3eab1d5d7e4909f5381d29 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 18:52:21 +0200 Subject: [PATCH 19/52] fix(incremental): a Kubernetes edit rebuilds what it reaches The closure repair re-parses the changed files plus their recorded dependents and re-runs the K8s pass over that plan only. Kubernetes inputs reach past their own file, so an incremental index drifted from a fresh one: - editing a Service's manifest dropped its INFRA_MAPS edge to the Deployment in the file beside it (the workload's file is the edge's target, not a dependent the plan knows); - a kustomization gaining or changing `namespace:` left the objects of every file it builds with their old obj_key; - a Chart.yaml's file:// dependencies resolve against other charts. The planner now declines -- the full rebuild every uncertain delta already falls back to -- when a changed or deleted file is a kustomization or a Chart.yaml, held Kubernetes objects, or holds them now. Measured with a new equivalence harness (clone the corpus, index, edit one file, re-index; index the edited tree fresh; compare node for node and edge for edge) on microservices-demo, onedr0p/home-ops, kubermatic and the voting app: 15 infrastructure edits -- kustomization namespace change and entry removal, Service / Deployment / HelmRelease edits across files, Chart.yaml and values changes, a deleted and a new manifest, a CI workflow -- all equivalent; before, the namespace change and the Service edit were not. incremental_k8s_edits_reach_other_files fails without the change (the Service edit loses its link). Signed-off-by: Martin Vogel --- src/pipeline/pipeline_incremental.c | 58 ++++++++++++++++++++++ tests/test_pipeline.c | 76 +++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index bc3beb7e6..a11f82356 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2343,6 +2343,59 @@ static const char *closure_add_variant_partners(cbm_store_t *store, const char * return decline; } +/* Kubernetes inputs reach past their own file (pass_k8s.c): a kustomization + * decides which manifests deploy together and in which namespace, a + * Chart.yaml links other charts, and a manifest's Services select workloads + * in other files of its bundle. The repair re-runs the K8s pass over its plan + * only, so a delta touching any of them -- a file that held objects, or holds + * them now -- declines to the full rebuild. Returns the reason, or NULL. */ +static const char *closure_k8s_decline(cbm_store_t *store, const char *project, + const char *repo_path, char **paths, int n_changed, + int n_deleted) { + for (int i = 0; i < n_changed + n_deleted; i++) { + const char *rel = paths[i]; + const char *slash = strrchr(rel, '/'); + const char *base = slash ? slash + SKIP_ONE : rel; + if (cbm_is_kustomize_file(base) || strcmp(base, "Kustomization") == 0) { + return "k8s_bundle_input_changed"; + } + if (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0) { + return "helm_chart_changed"; + } + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_file(store, project, rel, &nodes, &count) != CBM_STORE_OK) { + return "k8s_lookup_failed"; + } + bool held = false; + for (int j = 0; j < count && !held; j++) { + held = nodes[j].label && strcmp(nodes[j].label, "Resource") == 0; + } + cbm_store_free_nodes(nodes, count); + if (held) { + return "k8s_manifest_changed"; + } + const char *ext = strrchr(base, '.'); + if (i < n_changed && ext && (strcmp(ext, ".yaml") == 0 || strcmp(ext, ".yml") == 0)) { + enum { K8S_PEEK = 4096 }; + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", repo_path, rel); + char head[K8S_PEEK + SKIP_ONE]; + size_t n = 0; + FILE *f = cbm_fopen(path, "rb"); + if (f) { + n = fread(head, SKIP_ONE, K8S_PEEK, f); + (void)fclose(f); + } + head[n] = '\0'; + if (cbm_is_k8s_manifest(base, head)) { + return "k8s_manifest_changed"; + } + } + } + return NULL; +} + /* Decide whether this manifest delta is closure-repairable, and if so build * the plan. Returns 1 with *plan filled, or 0 (decline → FULL) with the * reason logged. Every early exit declines; nothing here mutates state. */ @@ -2465,6 +2518,11 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p goto done; } } + decline = closure_k8s_decline(store, project, cbm_pipeline_repo_path(p), changed_paths, + n_changed, n_deleted); + if (decline) { + goto done; + } /* Load the previous generation's surfaces and probe the changed files' * fresh ones. Missing rows fail closed. */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index f666f993c..f955f9822 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3347,6 +3347,81 @@ TEST(pipeline_staged_publish_commits_below_full_sync_issue1419) { PASS(); } +/* Kubernetes inputs reach past their own file: a Service selects a workload + * in another file of its bundle, and a kustomization decides the namespace + * of every object it builds. The closure repair re-runs the K8s pass over + * its plan only, so editing either must still leave the graph a fresh index + * builds. */ +static int k8s_inc_index(const char *repo, const char *db) { + cbm_pipeline_t *p = cbm_pipeline_new(repo, db, CBM_MODE_FULL); + if (!p) { + return -1; + } + (void)cbm_pipeline_set_project_name(p, "k8sinc"); + cbm_pipeline_set_persistence(p, false); + int rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + return rc; +} + +static int k8s_inc_count(const char *db_path, const char *sql) { + sqlite3 *db = NULL; + int n = -1; + if (sqlite3_open_v2(db_path, &db, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK) { + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK && + sqlite3_step(stmt) == SQLITE_ROW) { + n = sqlite3_column_int(stmt, 0); + } + sqlite3_finalize(stmt); + } + sqlite3_close(db); + return n; +} + +TEST(incremental_k8s_edits_reach_other_files) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_k8sinc_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "kustomization.yaml", "resources:\n - svc.yaml\n - deploy.yaml\n"); + write_temp_file(tmp, "svc.yaml", + "apiVersion: v1\nkind: Service\nmetadata:\n name: db\nspec:\n" + " selector:\n app: db\n"); + write_temp_file(tmp, "deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: db\nspec:\n" + " template:\n metadata:\n labels:\n app: db\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/k8sinc.db", tmp); + static const char links[] = "SELECT COUNT(*) FROM edges WHERE type='INFRA_MAPS'"; + static const char shop[] = "SELECT COUNT(*) FROM nodes WHERE label='Resource' AND " + "json_extract(properties,'$.obj_key')='apps/Deployment/shop/db'"; + + int rc_full = k8s_inc_index(tmp, db_path); + int links_full = k8s_inc_count(db_path, links); + /* the Service's file changes, the Deployment's does not */ + write_temp_file(tmp, "svc.yaml", + "apiVersion: v1\nkind: Service\nmetadata:\n name: db\nspec:\n" + " selector:\n app: db\n# edited\n"); + int rc_svc = k8s_inc_index(tmp, db_path); + int links_after_svc = k8s_inc_count(db_path, links); + /* the build gains a namespace: deploy.yaml's object lives in it now */ + write_temp_file(tmp, "kustomization.yaml", + "namespace: shop\nresources:\n - svc.yaml\n - deploy.yaml\n"); + int rc_kust = k8s_inc_index(tmp, db_path); + int shop_after_kust = k8s_inc_count(db_path, shop); + th_rmtree(tmp); + + ASSERT_EQ(rc_full, 0); + ASSERT_EQ(links_full, 1); + ASSERT_EQ(rc_svc, 0); + ASSERT_EQ(links_after_svc, 1); + ASSERT_EQ(rc_kust, 0); + ASSERT_EQ(shop_after_kust, 1); + PASS(); +} + /* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage, * while the shared name registry must NOT leak those relations into other * languages' textual resolution: a Python call or identifier sharing the @@ -18970,6 +19045,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delete); RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge); RUN_TEST(pipeline_staged_publish_commits_below_full_sync_issue1419); + RUN_TEST(incremental_k8s_edits_reach_other_files); RUN_TEST(pipeline_sql_lineage_and_relation_isolation); RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage); RUN_TEST(pipeline_dbt_jinja_lineage); From f7622fc1a2e0b99800e283d06189cfc8beb0f502 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 19:09:34 +0200 Subject: [PATCH 20/52] fix(infra): secret material in config values never reaches the graph Config-file URLs under an endpoint key became Route nodes verbatim -- name, QN and edge properties: credentials in the URL (`user:pass@`), a token in its query, and anything a Kubernetes Secret keeps under stringData were stored in the index as written. A Secret with `stringData: webhook_url: https://hooks.example.com/...` minted a Route carrying the webhook's secret path. The rule: an index holds no Secret data or stringData, no environment or .tfvars values, no credentials. - values under a manifest's data / stringData / binaryData are payload, never an endpoint the service exposes, and are denied by value (no sibling reference of the same URL mints it); - .tfvars, .tfvars.json and .env files assign values; they mint no Route and no infra binding; - a URL matching a known token shape (cbm_is_secret_value) mints nothing; - a kept URL is stored as scheme, host and path: userinfo, query and fragment are dropped. Infra bindings (topic -> endpoint) follow the same rule. On the 17 infrastructure corpora the graphs are identical to before -- none of them puts credentials in an endpoint URL -- so this is a guarantee, not a cleanup. (CALLS edges' captured argument literals can hold secrets from source code; that is outside the config scope here and is recorded separately.) Tests: a Secret's stringData URL and an endpoint with credentials and a token query fail on the old code; the .tfvars case already held and now guards the rule. Signed-off-by: Martin Vogel --- src/pipeline/pipeline.c | 76 ++++++++++++++++++++++++++++++++---- tests/test_grammar_probe_f.c | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 7 deletions(-) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 36846c819..9e8a66c8e 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1522,6 +1522,53 @@ static void *gh_compute_thread_fn(void *arg) { return NULL; } +/* ── Secret hygiene: config values that are secret material stay out ─── */ + +#define INFRA_SLEN(lit) (sizeof(lit) - SKIP_ONE) + +/* A file of values, not declarations: .tfvars assigns Terraform variables, + * .env the process environment -- where deployments keep their secrets. */ +static bool infra_values_file(const char *fp) { + const char *slash = strrchr(fp, '/'); + const char *base = slash ? slash + SKIP_ONE : fp; + size_t n = strlen(base); + static const char *const suffixes[] = {".tfvars", ".tfvars.json", ".env"}; + for (size_t i = 0; i < sizeof(suffixes) / sizeof(suffixes[0]); i++) { + size_t s = strlen(suffixes[i]); + if (n >= s && strcmp(base + n - s, suffixes[i]) == 0) { + return true; + } + } + return strcmp(base, ".env") == 0 || strncmp(base, ".env.", INFRA_SLEN(".env.")) == 0; +} + +/* A manifest's data / stringData / binaryData hold configuration payload + * (a Secret keeps its values there); a value under them never declares an + * endpoint the service exposes. */ +static bool infra_payload_key(const char *key_path) { + if (!key_path) { + return false; + } + size_t first = strcspn(key_path, "."); + return (first == INFRA_SLEN("data") && strncmp(key_path, "data", first) == 0) || + (first == INFRA_SLEN("stringData") && strncmp(key_path, "stringData", first) == 0) || + (first == INFRA_SLEN("binaryData") && strncmp(key_path, "binaryData", first) == 0); +} + +/* `url` as a route: scheme, host and path. Userinfo (`user:pass@`), the query + * and the fragment are where credentials and tokens ride; none is part of + * the endpoint. */ +static void infra_url_route_form(const char *url, char *out, size_t out_sz) { + const char *scheme_end = strstr(url, "://"); + const char *host = scheme_end ? scheme_end + INFRA_SLEN("://") : url; + const char *path = host + strcspn(host, "/?#"); + const char *at = memchr(host, '@', (size_t)(path - host)); + const char *host_start = at ? at + SKIP_ONE : host; + size_t path_len = strcspn(path, "?#"); + snprintf(out, out_sz, "%.*s%.*s%.*s", (int)(host - url), url, (int)(path - host_start), + host_start, (int)path_len, path); +} + /* Extract Route nodes from URL strings found in config files (YAML, HCL, TOML). * These are infrastructure-defined endpoints (Cloud Scheduler, Terraform). */ /* Process infra bindings: topic→URL pairs from IaC configs. @@ -1530,10 +1577,17 @@ static void *gh_compute_thread_fn(void *arg) { /* Process one infra binding: create Route node + INFRA_MAPS edge. */ static int process_one_infra_binding(cbm_gbuf_t *gbuf, const CBMInfraBinding *ib, const char *rel_path) { + /* Secret hygiene, as for config routes: a values file or a token-shaped + * URL mints nothing; a kept URL loses its credentials and query. */ + if (infra_values_file(rel_path) || cbm_is_secret_value(ib->target_url)) { + return 0; + } + char url[CBM_ROUTE_QN_SIZE]; + infra_url_route_form(ib->target_url, url, sizeof(url)); char url_route_qn[CBM_ROUTE_QN_SIZE]; - snprintf(url_route_qn, sizeof(url_route_qn), "__route__infra__%s", ib->target_url); - int64_t url_route_id = cbm_gbuf_upsert_node(gbuf, "Route", ib->target_url, url_route_qn, - rel_path, 0, 0, "{\"source\":\"infra\"}"); + snprintf(url_route_qn, sizeof(url_route_qn), "__route__infra__%s", url); + int64_t url_route_id = cbm_gbuf_upsert_node(gbuf, "Route", url, url_route_qn, rel_path, 0, 0, + "{\"source\":\"infra\"}"); char topic_route_qn[CBM_ROUTE_QN_SIZE]; snprintf(topic_route_qn, sizeof(topic_route_qn), "__route__%s__%s", ib->broker ? ib->broker : "async", ib->source_name); @@ -1554,7 +1608,7 @@ static int process_one_infra_binding(cbm_gbuf_t *gbuf, const CBMInfraBinding *ib } char props[CBM_SZ_512]; snprintf(props, sizeof(props), "{\"broker\":\"%s\",\"topic\":\"%s\",\"endpoint\":\"%s\"}", - ib->broker ? ib->broker : "async", ib->source_name, ib->target_url); + ib->broker ? ib->broker : "async", ib->source_name, url); cbm_gbuf_insert_edge(gbuf, topic_route_id, url_route_id, "INFRA_MAPS", props); return SKIP_ONE; } @@ -1681,8 +1735,10 @@ static void try_upsert_infra_route(cbm_gbuf_t *gbuf, const CBMStringRef *sr, con if (!is_exposed_endpoint_key(sr->key_path)) { return; } + char url[CBM_ROUTE_QN_SIZE]; + infra_url_route_form(sr->value, url, sizeof(url)); char route_qn[CBM_ROUTE_QN_SIZE]; - snprintf(route_qn, sizeof(route_qn), "__route__infra__%s", sr->value); + snprintf(route_qn, sizeof(route_qn), "__route__infra__%s", url); char route_props[CBM_SZ_512]; if (sr->key_path) { snprintf(route_props, sizeof(route_props), "{\"source\":\"infra\",\"key_path\":\"%s\"}", @@ -1690,7 +1746,7 @@ static void try_upsert_infra_route(cbm_gbuf_t *gbuf, const CBMStringRef *sr, con } else { snprintf(route_props, sizeof(route_props), "{\"source\":\"infra\"}"); } - cbm_gbuf_upsert_node(gbuf, "Route", sr->value, route_qn, fp, 0, 0, route_props); + cbm_gbuf_upsert_node(gbuf, "Route", url, route_qn, fp, 0, 0, route_props); } /* A URL string_ref that does NOT denote a route the service serves: a value @@ -1705,6 +1761,11 @@ static bool route_sr_denied(const CBMStringRef *sr) { if (!sr->key_path) { return true; } + /* Secret hygiene: payload values and token-shaped values are denied by + * value, so no sibling ref of the same URL mints it either. */ + if (infra_payload_key(sr->key_path) || cbm_is_secret_value(sr->value)) { + return true; + } return !is_exposed_endpoint_key(sr->key_path); } @@ -1726,7 +1787,8 @@ static void cbm_pipeline_extract_infra_routes(const cbm_pipeline_ctx_t *ctx, cbm CBMHashTable *denied = cbm_ht_create(16); for (int pass = 0; pass < 2; pass++) { for (int i = 0; i < file_count; i++) { - if (!is_infra_file(files[i].rel_path) || is_ci_tooling_config(files[i].rel_path)) { + if (!is_infra_file(files[i].rel_path) || is_ci_tooling_config(files[i].rel_path) || + infra_values_file(files[i].rel_path)) { continue; } bool loaded = false; diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 5785ba0ab..7fc7ee08d 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -143,6 +144,7 @@ typedef struct { int imports; /* IMPORTS edges */ int depends_on; /* DEPENDS_ON edges */ int infra_maps; /* INFRA_MAPS edges */ + int routes; /* Route label */ } GpfMetrics; static GpfMetrics gpf_metrics_files(const GpfFile *files, int nfiles) { @@ -161,6 +163,7 @@ static GpfMetrics gpf_metrics_files(const GpfFile *files, int nfiles) { m.imports = cbm_store_count_edges_by_type(store, lp.project, "IMPORTS"); m.depends_on = cbm_store_count_edges_by_type(store, lp.project, "DEPENDS_ON"); m.infra_maps = cbm_store_count_edges_by_type(store, lp.project, "INFRA_MAPS"); + m.routes = gpf_count_label(store, lp.project, "Route"); } gpf_cleanup(&lp, store); return m; @@ -845,6 +848,73 @@ TEST(probe_k8s_selector_across_documents) { PASS(); } +/* ── Secret hygiene: config values that are secret material stay out ── */ + +/* Does any node name / QN / properties or edge properties contain `needle`? */ +static int gpf_graph_mentions(const GpfFile *files, int nfiles, const char *needle) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int hits = -1; + if (store) { + hits = 0; + static const char *const sql[] = { + "SELECT COUNT(*) FROM nodes WHERE project=?1 AND (instr(name,?2) OR " + "instr(qualified_name,?2) OR instr(properties,?2))", + "SELECT COUNT(*) FROM edges WHERE project=?1 AND instr(properties,?2)"}; + for (size_t q = 0; q < sizeof(sql) / sizeof(sql[0]); q++) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql[q], -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 2, needle, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + hits += sqlite3_column_int(st, 0); + } + } + sqlite3_finalize(st); + } + } + gpf_cleanup(&lp, store); + return hits; +} + +/* A Secret's stringData is secret material, whatever its keys are called. */ +TEST(probe_secret_stringdata_mints_no_route) { + static const GpfFile files[] = {{"alerting-secret.yaml", + "apiVersion: v1\n" + "kind: Secret\n" + "metadata:\n" + " name: alerting\n" + "stringData:\n" + " webhook_url: https://hooks.example.com/T0/B0/s3cr3tpath\n"}}; + ASSERT_EQ(gpf_graph_mentions(files, 1, "s3cr3tpath"), 0); + PASS(); +} + +/* An endpoint keeps its path; the credentials and query riding on it stay + * out of the graph. */ +TEST(probe_endpoint_url_loses_credentials) { + static const GpfFile files[] = {{"subscription.yaml", + "subscription:\n" + " push_endpoint: https://svc:hunter2@svc.example.com/push" + "?token=abc123\n"}}; + GpfMetrics m = gpf_metrics_files(files, 1); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.routes, 1); + ASSERT_EQ(gpf_graph_mentions(files, 1, "hunter2"), 0); + ASSERT_EQ(gpf_graph_mentions(files, 1, "abc123"), 0); + ASSERT_TRUE(gpf_graph_mentions(files, 1, "https://svc.example.com/push") > 0); + PASS(); +} + +/* .tfvars assigns values -- a variable file is where secrets live; the + * declaration in the .tf is what the graph keeps. */ +TEST(probe_tfvars_values_mint_no_route) { + static const GpfFile files[] = { + {"prod.tfvars", "push_endpoint = \"https://svc.example.com/tfvars-only-path\"\n"}}; + ASSERT_EQ(gpf_graph_mentions(files, 1, "tfvars-only-path"), 0); + PASS(); +} + /* ── Bundles: a Service selects inside what is deployed together ───── */ #define GPF_DEPLOY(name, label) \ @@ -2416,6 +2486,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_identity_core_namespaced_and_cluster_scoped); RUN_TEST(probe_k8s_anchored_and_tagged_names); RUN_TEST(probe_k8s_selector_flow_mapping); + RUN_TEST(probe_secret_stringdata_mints_no_route); + RUN_TEST(probe_endpoint_url_loses_credentials); + RUN_TEST(probe_tfvars_values_mint_no_route); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); RUN_TEST(probe_k8s_kustomize_namespace_is_effective); From 3ef74c230770fac7cbd7a3a5e0de0f5b402b9ce8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 20:05:12 +0200 Subject: [PATCH 21/52] feat(k8s): objects link to the objects they name A Kubernetes object names the objects it runs with -- its service account, mounted ConfigMaps and claims, env sources, the role a binding grants, the Service an Ingress or webhook calls -- and none of that was in the graph: across 17 infrastructure repositories, 0 object-to-object references. Questions like "what breaks if this Secret changes" or "who uses this ServiceAccount" had no edges to walk. The K8s extractor now records, while it walks each document, the references at known paths (written from the Kubernetes API, one table): - pod spec (Pod, workloads, CronJob): serviceAccountName, priorityClassName, imagePullSecrets, volumes (configMap, secret, persistentVolumeClaim, projected sources), containers' env valueFrom and envFrom -> USAGE {ref}; - RoleBinding / ClusterRoleBinding roleRef and ServiceAccount subjects, HPA scaleTargetRef, StatefulSet serviceName, PVC storageClassName, Ingress tls / ingressClassName, Gateway-API parentRefs -> USAGE {ref}; - Ingress and Gateway-API route backends, webhook configurations' and APIServices' services -> INFRA_MAPS {kind}; - a custom resource -> the CRD declaring its group and kind -> USAGE {ref: crd}. The K8s pass resolves them after every object is registered, by name (an O(1) index), inside the source's bundle and namespace (explicit, or the source's own); an unresolved reference draws nothing. A name is one object per kind and namespace, so same-named declarations in one bundle are alternatives (versioned manifests side by side): the one in the reference's own file wins. A CRD vendored twice: the copy deployed with the resource, else all copies. Measured against an independent resolver (Ruby over the parse tree, the same table and bundle rules) on every plain manifest of the 17 corpora: 256 of 257 resolvable references are edges (the one left sits in a directory .gitignore hides), and all 256 graph edges there are right. In Go-templated files the truth parser cannot read, all 262 edges kubermatic gets were checked against the source documents. Nothing else in any graph changes; two runs are identical. Tests: pod-spec references, RBAC / Ingress / HPA / webhook references, CR -> CRD and an unresolved reference, bundle scope, alternatives in one directory and a CRD vendored twice -- each fails without the change. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 5 + internal/cbm/cbm.h | 25 +++ internal/cbm/extract_k8s.c | 307 +++++++++++++++++++++++++++++++++++ src/pipeline/pass_k8s.c | 280 +++++++++++++++++++++++++++++++- tests/test_grammar_probe_f.c | 193 ++++++++++++++++++++++ 5 files changed, 808 insertions(+), 2 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index b9e3237b2..2f0f9cc1c 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -262,6 +262,11 @@ void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBindi arr->items[arr->count++] = ib; } +void cbm_k8sfact_push(CBMK8sFactArray *arr, CBMArena *a, CBMK8sFact f) { + GROW_ARRAY(arr, a); + arr->items[arr->count++] = f; +} + void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it) { GROW_ARRAY(arr, a); arr->items[arr->count++] = it; diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 6cc5d4743..0ae684030 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -600,6 +600,29 @@ typedef struct { int cap; } CBMInfraBindingArray; +/* A fact a Kubernetes manifest states about other objects (extract_k8s.c), + * resolved by the K8s pass against the objects of the source's bundle. + * CBM_K8S_FACT_REF: object def_index names a kind/ns/name object at a + * reference path (via), drawn as edge (USAGE or INFRA_MAPS); ns NULL = + * the source's own namespace. + * CBM_K8S_FACT_CRD: the CRD def_index declares kind `kind` in group `name`. */ +typedef enum { CBM_K8S_FACT_REF = 0, CBM_K8S_FACT_CRD } CBMK8sFactKind; +typedef struct { + CBMK8sFactKind fact; + int def_index; + const char *kind; + const char *ns; + const char *name; + const char *via; + const char *edge; +} CBMK8sFact; + +typedef struct { + CBMK8sFact *items; + int count; + int cap; +} CBMK8sFactArray; + typedef struct { CBMImplTrait *items; int count; @@ -660,6 +683,7 @@ typedef struct CBMFileResult { CBMResolvedCallArray resolved_calls; // LSP-resolved invocations/references (high confidence) CBMStringRefArray string_refs; // URL/config string literals from AST CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs + CBMK8sFactArray k8s_facts; // Kubernetes references and CRD declarations CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation CBMFieldTypeArray field_types; // Python: annotated instance fields (#1277) @@ -1119,6 +1143,7 @@ void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta) void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft); void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); +void cbm_k8sfact_push(CBMK8sFactArray *arr, CBMArena *a, CBMK8sFact f); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc); void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch); diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 46937668d..29765d87c 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -331,6 +331,312 @@ static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, k8s_doc_id_t } } +// --------------------------------------------------------------------------- +// Object references (PHASE1.md): facts the K8s pass resolves per bundle +// --------------------------------------------------------------------------- + +// A reference held by a scalar at a path, relative to the pod spec, to a +// container, or (kinds) to the whole object. +typedef struct { + const char *path; + const char *kind; + const char *via; +} k8s_leaf_ref_t; + +typedef struct { + const char *kinds; // "|A|B|" of the source kinds + const char *path; + const char *kind; + const char *via; + const char *edge; +} k8s_object_ref_t; + +// A reference held by a mapping's kind/name/namespace fields. +typedef struct { + const char *kinds; // "|A|B|" of the source kinds + const char *path; // the mapping's path + const char *kind; // the target kind; NULL = the mapping's own `kind` field + const char *allowed; // with kind NULL: "|A|" of the kinds that are objects + const char *dflt; // with kind NULL: the kind when the field is absent + const char *via; + const char *edge; +} k8s_mapping_ref_t; + +static const k8s_leaf_ref_t k8s_pod_refs[] = { + {"serviceAccountName", "ServiceAccount", "serviceAccountName"}, + {"serviceAccount", "ServiceAccount", "serviceAccountName"}, + {"priorityClassName", "PriorityClass", "priorityClassName"}, + {"imagePullSecrets[].name", "Secret", "imagePullSecrets"}, + {"volumes[].configMap.name", "ConfigMap", "volume"}, + {"volumes[].secret.secretName", "Secret", "volume"}, + {"volumes[].persistentVolumeClaim.claimName", "PersistentVolumeClaim", "volume"}, + {"volumes[].projected.sources[].configMap.name", "ConfigMap", "volume"}, + {"volumes[].projected.sources[].secret.name", "Secret", "volume"}, +}; + +static const k8s_leaf_ref_t k8s_container_refs[] = { + {"env[].valueFrom.configMapKeyRef.name", "ConfigMap", "env"}, + {"env[].valueFrom.secretKeyRef.name", "Secret", "env"}, + {"envFrom[].configMapRef.name", "ConfigMap", "envFrom"}, + {"envFrom[].secretRef.name", "Secret", "envFrom"}, +}; + +static const k8s_object_ref_t k8s_object_refs[] = { + {"|Ingress|", "spec.rules[].http.paths[].backend.service.name", "Service", "ingress", + "INFRA_MAPS"}, + {"|Ingress|", "spec.defaultBackend.service.name", "Service", "ingress", "INFRA_MAPS"}, + {"|Ingress|", "spec.tls[].secretName", "Secret", "tls", "USAGE"}, + {"|Ingress|", "spec.ingressClassName", "IngressClass", "ingressClassName", "USAGE"}, + {"|StatefulSet|", "spec.serviceName", "Service", "serviceName", "USAGE"}, + {"|PersistentVolumeClaim|", "spec.storageClassName", "StorageClass", "storageClassName", + "USAGE"}, +}; + +static const k8s_mapping_ref_t k8s_mapping_refs[] = { + {"|RoleBinding|ClusterRoleBinding|", "roleRef", NULL, "|Role|ClusterRole|", NULL, "roleRef", + "USAGE"}, + {"|RoleBinding|ClusterRoleBinding|", "subjects[]", NULL, "|ServiceAccount|", NULL, "subject", + "USAGE"}, + {"|HorizontalPodAutoscaler|", "spec.scaleTargetRef", NULL, NULL, NULL, "scaleTarget", "USAGE"}, + {"|ValidatingWebhookConfiguration|MutatingWebhookConfiguration|", + "webhooks[].clientConfig.service", "Service", NULL, NULL, "webhook", "INFRA_MAPS"}, + {"|APIService|", "spec.service", "Service", NULL, NULL, "apiservice", "INFRA_MAPS"}, + {"|HTTPRoute|GRPCRoute|TCPRoute|TLSRoute|UDPRoute|", "spec.rules[].backendRefs[]", NULL, NULL, + "Service", "route", "INFRA_MAPS"}, + {"|HTTPRoute|GRPCRoute|TCPRoute|TLSRoute|UDPRoute|", "spec.parentRefs[]", NULL, NULL, "Gateway", + "parentRef", "USAGE"}, +}; + +// Is `kind` one of the "|A|B|" list? +static bool k8s_kind_in(const char *list, const char *kind) { + if (!list || !kind[0]) { + return false; + } + size_t n = strlen(kind); + for (const char *p = strstr(list, kind); p; p = strstr(p + SKIP_ONE, kind)) { + if (p[-SKIP_ONE] == '|' && p[n] == '|') { + return true; + } + } + return false; +} + +// Where a kind keeps its pod spec, "" for none. +static const char *k8s_pod_prefix(const char *kind) { + if (strcmp(kind, "Pod") == 0) { + return "spec"; + } + if (strcmp(kind, "CronJob") == 0) { + return "spec.jobTemplate.spec.template.spec"; + } + if (k8s_kind_in("|Deployment|StatefulSet|DaemonSet|ReplicaSet|ReplicationController|Job|", + kind)) { + return "spec.template.spec"; + } + return ""; +} + +typedef struct { + CBMExtractCtx *ctx; + const char *kind; // the document's kind + int def_index; + char crd_group[K8S_BUF_SIZE]; + char crd_kind[K8S_BUF_SIZE]; +} k8s_walk_t; + +static void k8s_emit_ref(k8s_walk_t *w, const char *kind, const char *ns, const char *name, + const char *via, const char *edge) { + if (!kind || !kind[0] || !name || !name[0] || strstr(name, "{{") || strstr(kind, "{{")) { + return; + } + CBMArena *a = w->ctx->arena; + CBMK8sFact f = {.fact = CBM_K8S_FACT_REF, + .def_index = w->def_index, + .kind = cbm_arena_strdup(a, kind), + .ns = ns && ns[0] && !strstr(ns, "{{") ? cbm_arena_strdup(a, ns) : NULL, + .name = cbm_arena_strdup(a, name), + .via = via, + .edge = edge}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, a, f); +} + +// A scalar at `path`: a leaf reference, or a CRD's group / kind. +static void k8s_leaf(k8s_walk_t *w, const char *path, const char *value) { + const char *pod = k8s_pod_prefix(w->kind); + size_t pl = strlen(pod); + if (pl && strncmp(path, pod, pl) == 0 && path[pl] == '.') { + const char *rel = path + pl + SKIP_ONE; + for (size_t i = 0; i < sizeof(k8s_pod_refs) / sizeof(k8s_pod_refs[0]); i++) { + if (strcmp(rel, k8s_pod_refs[i].path) == 0) { + k8s_emit_ref(w, k8s_pod_refs[i].kind, NULL, value, k8s_pod_refs[i].via, "USAGE"); + } + } + static const char *const holders[] = {"containers[].", "initContainers[].", + "ephemeralContainers[]."}; + for (size_t h = 0; h < sizeof(holders) / sizeof(holders[0]); h++) { + size_t hl = strlen(holders[h]); + if (strncmp(rel, holders[h], hl) != 0) { + continue; + } + for (size_t i = 0; i < sizeof(k8s_container_refs) / sizeof(k8s_container_refs[0]); + i++) { + if (strcmp(rel + hl, k8s_container_refs[i].path) == 0) { + k8s_emit_ref(w, k8s_container_refs[i].kind, NULL, value, + k8s_container_refs[i].via, "USAGE"); + } + } + } + } + for (size_t i = 0; i < sizeof(k8s_object_refs) / sizeof(k8s_object_refs[0]); i++) { + const k8s_object_ref_t *r = &k8s_object_refs[i]; + if (k8s_kind_in(r->kinds, w->kind) && strcmp(path, r->path) == 0) { + k8s_emit_ref(w, r->kind, NULL, value, r->via, r->edge); + } + } + if (strcmp(w->kind, "CustomResourceDefinition") == 0) { + if (strcmp(path, "spec.group") == 0) { + snprintf(w->crd_group, sizeof(w->crd_group), "%s", value); + } else if (strcmp(path, "spec.names.kind") == 0) { + snprintf(w->crd_kind, sizeof(w->crd_kind), "%s", value); + } + } +} + +// Through block_node / flow_node wrappers to the content node. +static TSNode k8s_unwrap(TSNode node) { + for (int i = 0; i < PAIR_LEN && !ts_node_is_null(node); i++) { + const char *t = ts_node_type(node); + if (strcmp(t, "block_node") == 0) { + node = first_content_child(node); + } else if (strcmp(t, "flow_node") == 0) { + node = flow_node_value(node); + } else { + break; + } + } + return node; +} + +static bool k8s_is_mapping(TSNode n) { + const char *t = ts_node_type(n); + return strcmp(t, "block_mapping") == 0 || strcmp(t, "flow_mapping") == 0; +} + +static bool k8s_is_pair(TSNode n) { + const char *t = ts_node_type(n); + return strcmp(t, "block_mapping_pair") == 0 || strcmp(t, "flow_pair") == 0; +} + +// The scalar value of `field` in a mapping, or NULL. +static const char *k8s_field(k8s_walk_t *w, TSNode mapping, const char *field) { + uint32_t n = ts_node_named_child_count(mapping); + for (uint32_t i = 0; i < n; i++) { + TSNode pair = ts_node_named_child(mapping, i); + if (!k8s_is_pair(pair)) { + continue; + } + TSNode key = ts_node_named_child(pair, 0); + const char *k = + ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); + if (k && strcmp(k, field) == 0) { + TSNode v = unwrap_pair_value(pair); + return ts_node_is_null(v) ? NULL : get_scalar_text(w->ctx->arena, v, w->ctx->source); + } + } + return NULL; +} + +// A mapping at `path` that holds a reference in its kind/name/namespace. +static void k8s_mapping(k8s_walk_t *w, const char *path, TSNode mapping) { + for (size_t i = 0; i < sizeof(k8s_mapping_refs) / sizeof(k8s_mapping_refs[0]); i++) { + const k8s_mapping_ref_t *r = &k8s_mapping_refs[i]; + if (!k8s_kind_in(r->kinds, w->kind) || strcmp(path, r->path) != 0) { + continue; + } + const char *kind = r->kind; + if (!kind) { + kind = k8s_field(w, mapping, "kind"); + kind = kind ? kind : r->dflt; + if (!kind || (r->allowed && !k8s_kind_in(r->allowed, kind))) { + continue; + } + } + k8s_emit_ref(w, kind, k8s_field(w, mapping, "namespace"), k8s_field(w, mapping, "name"), + r->via, r->edge); + } +} + +// Walk `node` (reached at `path`) for references. +static void k8s_walk(k8s_walk_t *w, TSNode node, char *path, size_t len, size_t cap) { + node = k8s_unwrap(node); + if (ts_node_is_null(node)) { + return; + } + const char *type = ts_node_type(node); + if (k8s_is_mapping(node)) { + k8s_mapping(w, path, node); + uint32_t n = ts_node_named_child_count(node); + for (uint32_t i = 0; i < n; i++) { + TSNode pair = ts_node_named_child(node, i); + if (!k8s_is_pair(pair)) { + continue; + } + TSNode key = ts_node_named_child(pair, 0); + const char *k = + ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); + int add = k ? snprintf(path + len, cap - len, "%s%s", len ? "." : "", k) : -1; + if (add <= 0 || len + (size_t)add >= cap) { + path[len] = '\0'; // a path too long for any reference path: not one + continue; + } + TSNode value = unwrap_pair_value(pair); + TSNode inner = k8s_unwrap(value); + const char *scalar = ts_node_is_null(inner) + ? NULL + : get_scalar_text(w->ctx->arena, inner, w->ctx->source); + if (scalar) { + k8s_leaf(w, path, scalar); + } else if (!ts_node_is_null(value)) { + k8s_walk(w, value, path, len + (size_t)add, cap); + } + path[len] = '\0'; + } + return; + } + if (strcmp(type, "block_sequence") == 0 || strcmp(type, "flow_sequence") == 0) { + int add = snprintf(path + len, cap - len, "[]"); + if (add <= 0 || len + (size_t)add >= cap) { + path[len] = '\0'; + return; + } + uint32_t n = ts_node_named_child_count(node); + for (uint32_t i = 0; i < n; i++) { + TSNode item = ts_node_named_child(node, i); + if (strcmp(ts_node_type(item), "block_sequence_item") == 0) { + item = first_content_child(item); + } + if (!ts_node_is_null(item) && strcmp(ts_node_type(item), "comment") != 0) { + k8s_walk(w, item, path, len + (size_t)add, cap); + } + } + path[len] = '\0'; + } +} + +// The references and CRD declaration of document def `def_index`. +static void k8s_emit_facts(CBMExtractCtx *ctx, TSNode mapping, const char *kind, int def_index) { + k8s_walk_t w = {.ctx = ctx, .kind = kind, .def_index = def_index}; + char path[CBM_SZ_512] = ""; + k8s_walk(&w, mapping, path, 0, sizeof(path)); + if (w.crd_group[0] && w.crd_kind[0]) { + CBMK8sFact f = {.fact = CBM_K8S_FACT_CRD, + .def_index = def_index, + .kind = cbm_arena_strdup(ctx->arena, w.crd_kind), + .name = cbm_arena_strdup(ctx->arena, w.crd_group)}; + cbm_k8sfact_push(&ctx->result->k8s_facts, ctx->arena, f); + } +} + // "." with the extension kept: the File node's QN // without ".__file__" (IDENTITY.md). Unique per file, unlike the module QN, // which drops the extension (`a.yaml` / `a.yml`, `x/` / `x.yaml`). @@ -420,6 +726,7 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { def.start_line = ts_node_start_point(mapping).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(mapping).row + TS_LINE_OFFSET; cbm_defs_push(&ctx->result->defs, a, def); + k8s_emit_facts(ctx, mapping, id.kind, ctx->result->defs.count - SKIP_ONE); } } diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 818a791d4..f09037b08 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -1069,6 +1069,265 @@ static void k8s_apply_effective_namespaces(cbm_pipeline_ctx_t *ctx, k8s_bundles_ } } +/* ── Object references (PHASE1.md) ───────────────────────────────── */ + +/* A reference fact of a manifest, kept past its file's extraction. */ +typedef struct { + int64_t src; /* the referencing Resource */ + char *kind; /* target kind */ + char *ns; /* explicit target namespace, NULL = the source's */ + char *name; /* target metadata.name */ + const char *via; /* static: the extractor's table */ + const char *edge; /* static: "USAGE" or "INFRA_MAPS" */ +} k8s_ref_t; + +/* A CRD and the group/kind it declares. */ +typedef struct { + int64_t node; + char *group; + char *kind; +} k8s_crd_t; + +typedef struct { + k8s_ref_t *refs; + int ref_n; + int ref_cap; + k8s_crd_t *crds; + int crd_n; + int crd_cap; +} k8s_facts_t; + +static void k8s_facts_free(k8s_facts_t *f) { + for (int i = 0; i < f->ref_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].kind); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].ns); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].name); + } + for (int i = 0; i < f->crd_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->crds[i].group); + cbm_free(CBM_MEM_CLASS_OTHER, f->crds[i].kind); + } + cbm_free(CBM_MEM_CLASS_OTHER, f->refs); + cbm_free(CBM_MEM_CLASS_OTHER, f->crds); + *f = (k8s_facts_t){0}; +} + +/* Keep a file's facts, its def indices mapped to the Resource nodes. */ +static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64_t *node_of) { + for (int i = 0; i < res->k8s_facts.count; i++) { + const CBMK8sFact *x = &res->k8s_facts.items[i]; + if (x->def_index < 0 || x->def_index >= res->defs.count || node_of[x->def_index] <= 0) { + continue; + } + int64_t node = node_of[x->def_index]; + if (x->fact == CBM_K8S_FACT_CRD) { + if (k8s_grow((void **)&f->crds, &f->crd_cap, f->crd_n, sizeof(*f->crds))) { + f->crds[f->crd_n++] = + (k8s_crd_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind)}; + } + } else if (k8s_grow((void **)&f->refs, &f->ref_cap, f->ref_n, sizeof(*f->refs))) { + f->refs[f->ref_n++] = + (k8s_ref_t){.src = node, + .kind = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind), + .ns = x->ns ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->ns) : NULL, + .name = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + .via = x->via, + .edge = x->edge}; + } + } +} + +/* Does the target at `tgt` answer `ref` from `src`: one bundle, and the + * namespace the reference asks for (explicit, else the source's own there) + * agrees with the target's there? A cluster-scoped kind has no namespace. */ +static bool k8s_ref_meets(k8s_bundles_t *b, const k8s_ref_t *ref, const cbm_gbuf_node_t *src, + const cbm_gbuf_node_t *tgt) { + k8s_member_t one_s[SKIP_ONE]; + k8s_member_t one_t[SKIP_ONE]; + k8s_members_t single_s = {.items = one_s, .cap = SKIP_ONE}; + k8s_members_t single_t = {.items = one_t, .cap = SKIP_ONE}; + const k8s_members_t *ms = k8s_bundles_of(b, src->file_path, &single_s); + const k8s_members_t *mt = k8s_bundles_of(b, tgt->file_path, &single_t); + char src_ns[K8S_LABEL_LEN]; + char tgt_ns[K8S_LABEL_LEN]; + k8s_declared_ns(src->qualified_name, src_ns, sizeof(src_ns)); + k8s_declared_ns(tgt->qualified_name, tgt_ns, sizeof(tgt_ns)); + bool cluster = k8s_cluster_scoped(ref->kind); + for (int i = 0; i < ms->count; i++) { + for (int j = 0; j < mt->count; j++) { + if (ms->items[i].bundle != mt->items[j].bundle) { + continue; + } + const char *want = ref->ns ? ref->ns : ms->items[i].ns ? ms->items[i].ns : src_ns; + const char *have = mt->items[j].ns ? mt->items[j].ns : tgt_ns; + if (cluster || !want[0] || !have[0] || strcmp(want, have) == 0) { + return true; + } + } + } + return false; +} + +/* Draw every reference to the object(s) it names inside its bundle; + * an unresolved reference draws nothing. */ +static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_facts_t *f) { + int edges = 0; + int unresolved = 0; + for (int i = 0; i < f->ref_n; i++) { + const k8s_ref_t *ref = &f->refs[i]; + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, ref->src); + char name[CBM_SZ_1K]; + snprintf(name, sizeof(name), "%s/%s", ref->kind, ref->name); + const cbm_gbuf_node_t **cands = NULL; + int n = 0; + if (!src || !src->file_path || cbm_gbuf_find_by_name(ctx->gbuf, name, &cands, &n) != 0) { + n = 0; + } + char props[CBM_SZ_256]; + snprintf(props, sizeof(props), "{\"%s\":\"%s\"}", + strcmp(ref->edge, "INFRA_MAPS") == 0 ? "kind" : "ref", ref->via); + /* A name is one object per kind and namespace, so two declarations + * of it in one bundle are alternatives (versioned manifests side by + * side): the declaration in the reference's own file, when there is + * one, is the one it means. */ + bool own_file = false; + for (int c = 0; c < n && !own_file; c++) { + const cbm_gbuf_node_t *t = cands[c]; + own_file = t != src && t->label && strcmp(t->label, "Resource") == 0 && t->file_path && + strcmp(t->file_path, src->file_path) == 0 && k8s_ref_meets(b, ref, src, t); + } + int hits = 0; + for (int c = 0; c < n; c++) { + const cbm_gbuf_node_t *t = cands[c]; + if (t == src || !t->label || strcmp(t->label, "Resource") != 0 || !t->file_path || + (own_file && strcmp(t->file_path, src->file_path) != 0) || + !k8s_ref_meets(b, ref, src, t)) { + continue; + } + cbm_gbuf_insert_edge(ctx->gbuf, src->id, t->id, ref->edge, props); + hits++; + } + edges += hits; + unresolved += hits == 0; + } + if (f->ref_n > 0) { + cbm_log_info("pass.k8s.refs", "refs", itoa_k8s(f->ref_n), "edges", itoa_k8s(edges), + "unresolved", itoa_k8s(unresolved)); + } +} + +/* A CRD's group/kind, for grouping the CRDs of one type. */ +typedef struct { + char key[CBM_SZ_512]; + int crd; +} k8s_crd_key_t; + +static int k8s_cmp_crd_key(const void *a, const void *b) { + const k8s_crd_key_t *x = a; + const k8s_crd_key_t *y = b; + int c = strcmp(x->key, y->key); + return c ? c : (x->crd > y->crd) - (x->crd < y->crd); +} + +/* Do the files at `a` and `b` share a bundle? */ +static bool k8s_same_bundle(k8s_bundles_t *b, const char *a, const char *c) { + k8s_member_t one_a[SKIP_ONE]; + k8s_member_t one_c[SKIP_ONE]; + k8s_members_t single_a = {.items = one_a, .cap = SKIP_ONE}; + k8s_members_t single_c = {.items = one_c, .cap = SKIP_ONE}; + const k8s_members_t *ma = k8s_bundles_of(b, a, &single_a); + const k8s_members_t *mc = k8s_bundles_of(b, c, &single_c); + for (int i = 0; i < ma->count; i++) { + for (int j = 0; j < mc->count; j++) { + if (ma->items[i].bundle == mc->items[j].bundle) { + return true; + } + } + } + return false; +} + +/* A custom resource is an instance of the CRD declaring its group and + * kind. A CRD vendored in several places: the copy deployed with the + * resource (one bundle) is the one; with no copy beside it, the resource + * stands for all of them. */ +static void k8s_link_crds(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_facts_t *f) { + if (f->crd_n == 0) { + return; + } + /* group/kind -> the run of its CRDs in `order` (sorted by group/kind), so + * each resource visits only the CRDs of its own type. */ + k8s_crd_key_t *order = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)f->crd_n * sizeof(*order)); + CBMHashTable *by_type = cbm_ht_create(0); + if (!order || !by_type) { + cbm_free(CBM_MEM_CLASS_OTHER, order); + if (by_type) { + cbm_ht_free(by_type); + } + return; + } + for (int i = 0; i < f->crd_n; i++) { + snprintf(order[i].key, sizeof(order[i].key), "%s/%s", f->crds[i].group, f->crds[i].kind); + order[i].crd = i; + } + qsort(order, (size_t)f->crd_n, sizeof(*order), k8s_cmp_crd_key); + for (int i = f->crd_n - SKIP_ONE; i >= 0; i--) { + cbm_ht_set(by_type, order[i].key, (void *)(intptr_t)(i + SKIP_ONE)); /* first of run */ + } + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + int edges = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const char *key = + nodes[i]->qualified_name ? strstr(nodes[i]->qualified_name, "::") : NULL; + if (!key || !nodes[i]->file_path) { + continue; + } + key += K8S_SLEN("::"); + const char *kind_end = strchr(key, '/'); + kind_end = kind_end ? strchr(kind_end + SKIP_ONE, '/') : NULL; + if (!kind_end) { + continue; + } + char type[CBM_SZ_512]; + snprintf(type, sizeof(type), "%.*s", (int)(kind_end - key), key); + int first = (int)(intptr_t)cbm_ht_get(by_type, type) - SKIP_ONE; + if (first < 0) { + continue; + } + int end = first; + while (end < f->crd_n && strcmp(order[end].key, type) == 0) { + end++; + } + bool beside = false; + for (int r = first; r < end && !beside; r++) { + const cbm_gbuf_node_t *crd = + cbm_gbuf_find_by_id(ctx->gbuf, f->crds[order[r].crd].node); + beside = crd && crd->file_path && + k8s_same_bundle(b, nodes[i]->file_path, crd->file_path); + } + for (int r = first; r < end; r++) { + int64_t crd_id = f->crds[order[r].crd].node; + const cbm_gbuf_node_t *crd = cbm_gbuf_find_by_id(ctx->gbuf, crd_id); + if (crd_id == nodes[i]->id || + (beside && (!crd || !crd->file_path || + !k8s_same_bundle(b, nodes[i]->file_path, crd->file_path)))) { + continue; + } + cbm_gbuf_insert_edge(ctx->gbuf, nodes[i]->id, crd_id, "USAGE", "{\"ref\":\"crd\"}"); + edges++; + } + } + } + cbm_ht_free(by_type); + cbm_free(CBM_MEM_CLASS_OTHER, order); + if (edges > 0) { + cbm_log_info("pass.k8s.crds", "instances", itoa_k8s(edges)); + } +} + /* Connect each Service to the workloads its selector targets via an INFRA_MAPS * edge (Service Resource → workload Resource), inside a shared bundle. */ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs, @@ -1132,7 +1391,8 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t * must free after this call returns). Every Resource's own document is * recorded in `recs` for cross-manifest selector matching. */ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, - const CBMFileResult *res, k8s_record_array_t *recs) { + const CBMFileResult *res, k8s_record_array_t *recs, + k8s_facts_t *facts) { int resource_count = 0; if (!res) { @@ -1144,6 +1404,9 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; free(file_qn); + int64_t *node_of = + cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)(res->defs.count ? res->defs.count : SKIP_ONE) * sizeof(int64_t)); for (int d = 0; d < res->defs.count; d++) { CBMDefinition *def = &res->defs.items[d]; if (!def->label || strcmp(def->label, "Resource") != 0) { @@ -1168,9 +1431,16 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c if (node_id > 0 && source) { k8s_record_document(recs, node_id, source, (int)def->start_line, (int)def->end_line); } + if (node_of) { + node_of[d] = node_id; + } resource_count++; } + if (node_of) { + k8s_facts_keep(facts, res, node_of); + cbm_free(CBM_MEM_CLASS_OTHER, node_of); + } cbm_log_info("pass.k8s.manifest", "file", rel_path, "resources", itoa_k8s(resource_count)); } @@ -1580,6 +1850,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_local_deps_t local_deps = {0}; k8s_kusts_t kusts = {0}; k8s_bundles_t bundles = {.charts = cbm_ht_create(0)}; + k8s_facts_t facts = {0}; k8s_prep_t *prep = cbm_calloc(CBM_MEM_CLASS_OTHER, K8S_CHUNK * sizeof(*prep)); if (!prep) { @@ -1596,6 +1867,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_local_deps_free(&local_deps); k8s_kusts_free(&kusts); k8s_bundles_free(&bundles); + k8s_facts_free(&facts); return CBM_NOT_FOUND; } memset(prep, 0, (size_t)K8S_CHUNK * sizeof(*prep)); @@ -1628,7 +1900,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_kusts_push(&kusts, rel, k8s_kustomize_namespace(p->source)); break; case K8S_MANIFEST: - handle_k8s_manifest(ctx, rel, p->source, p->res, &recs); + handle_k8s_manifest(ctx, rel, p->source, p->res, &recs, &facts); manifest_count++; break; case K8S_SKIP: @@ -1647,6 +1919,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_local_deps_free(&local_deps); k8s_kusts_free(&kusts); k8s_bundles_free(&bundles); + k8s_facts_free(&facts); return CBM_NOT_FOUND; } } @@ -1657,6 +1930,9 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_bundles_build(ctx, &kusts, &bundles); k8s_apply_effective_namespaces(ctx, &bundles); k8s_link_selectors(ctx, &recs, &bundles); + k8s_resolve_refs(ctx, &bundles, &facts); + k8s_link_crds(ctx, &bundles, &facts); + k8s_facts_free(&facts); k8s_records_free(&recs); k8s_bundles_free(&bundles); k8s_kusts_free(&kusts); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 7fc7ee08d..391687e23 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -915,6 +915,193 @@ TEST(probe_tfvars_values_mint_no_route) { PASS(); } +/* ── Object references (Phase 1, PHASE1.md) ────────────────────────── */ + +/* Edges of `type` between two Resources whose properties carry `prop` + * (e.g. "\"ref\":\"volume\""); -1 when indexing failed. */ +static int gpf_resource_edges(const GpfFile *files, int nfiles, const char *type, + const char *prop) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int n = -1; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id " + "JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 AND e.type=?2 " + "AND s.label='Resource' AND t.label='Resource' AND instr(e.properties,?3)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 2, type, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 3, prop, -1, SQLITE_STATIC); + n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return n; +} + +static const char k_pod_refs_app[] = + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " template:\n" + " spec:\n" + " serviceAccountName: web-sa\n" + " imagePullSecrets:\n" + " - name: registry-creds\n" + " volumes:\n" + " - name: cfg\n" + " configMap:\n" + " name: web-config\n" + " - name: data\n" + " persistentVolumeClaim:\n" + " claimName: web-data\n" + " containers:\n" + " - name: web\n" + " env:\n" + " - name: DB_PASSWORD\n" + " valueFrom:\n" + " secretKeyRef:\n" + " name: web-db\n" + " key: password\n" + " envFrom:\n" + " - configMapRef:\n" + " name: web-env\n" + "---\n" + "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: web-sa\n" + "---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: registry-creds\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: web-config\n" + "---\n" + "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: web-data\n" + "---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: web-db\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: web-env\n"; + +/* A workload's pod spec names the objects it runs with: its service + * account, pull secret, mounted ConfigMap and claim, env sources. */ +TEST(probe_k8s_pod_spec_references) { + static const GpfFile files[] = {{"app.yaml", k_pod_refs_app}}; + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"serviceAccountName\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"imagePullSecrets\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"volume\""), 2); + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"env\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"envFrom\""), 1); + PASS(); +} + +/* RBAC: a binding uses the role it names and the service accounts it + * grants; an Ingress routes to its backend Service; an HPA scales its + * target; a webhook configuration calls its Service. */ +TEST(probe_k8s_rbac_ingress_hpa_webhook_references) { + static const GpfFile files[] = { + {"rbac.yaml", + "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: reader\n" + "---\n" + "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: web-sa\n namespace: prod\n" + "---\n" + "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\n" + "metadata:\n name: web-reader\n" + "roleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: reader\n" + "subjects:\n - kind: ServiceAccount\n name: web-sa\n namespace: prod\n"}, + {"web.yaml", + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n namespace: prod\n" + "---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: prod\n" + "---\n" + "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: web\n" + " namespace: prod\nspec:\n rules:\n - host: web.example.com\n http:\n" + " paths:\n - path: /\n pathType: Prefix\n" + " backend:\n service:\n name: web\n" + " port:\n number: 80\n" + "---\n" + "apiVersion: autoscaling/v2\nkind: HorizontalPodAutoscaler\nmetadata:\n name: web\n" + " namespace: prod\nspec:\n scaleTargetRef:\n apiVersion: apps/v1\n" + " kind: Deployment\n name: web\n" + "---\n" + "apiVersion: admissionregistration.k8s.io/v1\nkind: ValidatingWebhookConfiguration\n" + "metadata:\n name: web-hook\nwebhooks:\n - name: validate.web.example.com\n" + " clientConfig:\n service:\n name: web\n namespace: prod\n"}}; + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"roleRef\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"subject\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "INFRA_MAPS", "\"kind\":\"ingress\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"scaleTarget\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "INFRA_MAPS", "\"kind\":\"webhook\""), 1); + PASS(); +} + +/* A custom resource is an instance of the CRD that declares its group and + * kind; a reference to an object that is not in the repo draws nothing. */ +TEST(probe_k8s_cr_uses_its_crd_and_unresolved_draws_nothing) { + static const GpfFile files[] = { + {"crd.yaml", + "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\n" + "metadata:\n name: widgets.example.com\nspec:\n group: example.com\n" + " names:\n kind: Widget\n plural: widgets\n scope: Namespaced\n"}, + {"widget.yaml", + "apiVersion: example.com/v1\nkind: Widget\nmetadata:\n name: w1\n" + "---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: lonely\nspec:\n" + " template:\n spec:\n serviceAccountName: not-in-repo\n"}}; + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"crd\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"serviceAccountName\""), 0); + PASS(); +} + +/* References resolve inside the bundle: a ConfigMap of the same name in + * another directory (another app's copy) is not the one mounted. */ +TEST(probe_k8s_reference_stays_in_bundle) { + static const GpfFile files[] = { + {"a/app.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\nspec:\n template:\n" + " spec:\n volumes:\n - name: cfg\n configMap:\n" + " name: web-config\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: web-config\n"}, + {"b/cm.yaml", "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: web-config\n"}}; + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"volume\""), 1); + PASS(); +} + +/* A name is one object per kind and namespace: two declarations of it in + * one bundle are alternatives (kubermatic keeps canal_v3.21 ... v3.31 side + * by side; one is applied). A reference means the declaration in its own + * file when there is one. */ +TEST(probe_k8s_reference_prefers_its_own_file) { + static const char v1[] = + "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: canal\nspec:\n template:\n" + " spec:\n volumes:\n - name: cfg\n configMap:\n" + " name: canal-config\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: canal-config\n"; + static const GpfFile files[] = {{"canal/canal_v1.yaml", v1}, {"canal/canal_v2.yaml", v1}}; + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"volume\""), 2); + PASS(); +} + +/* A CRD vendored twice (gardener: example/seed-crds and a chart): a custom + * resource is an instance of the copy deployed with it; only when no copy + * shares its bundle does it stand for all of them. */ +TEST(probe_k8s_cr_prefers_the_crd_in_its_bundle) { + static const char crd[] = + "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\n" + "metadata:\n name: widgets.example.com\nspec:\n group: example.com\n" + " names:\n kind: Widget\n plural: widgets\n scope: Namespaced\n"; + static const char cr[] = "apiVersion: example.com/v1\nkind: Widget\nmetadata:\n name: w1\n"; + static const GpfFile files[] = {{"app/crd.yaml", crd}, + {"app/widget.yaml", cr}, + {"vendor-chart/crds/crd.yaml", crd}, + {"other/widget.yaml", cr}}; + /* app/widget -> app/crd only; other/widget (no copy beside it) -> both */ + ASSERT_EQ(gpf_resource_edges(files, 4, "USAGE", "\"ref\":\"crd\""), 3); + PASS(); +} + /* ── Bundles: a Service selects inside what is deployed together ───── */ #define GPF_DEPLOY(name, label) \ @@ -2489,6 +2676,12 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_secret_stringdata_mints_no_route); RUN_TEST(probe_endpoint_url_loses_credentials); RUN_TEST(probe_tfvars_values_mint_no_route); + RUN_TEST(probe_k8s_pod_spec_references); + RUN_TEST(probe_k8s_rbac_ingress_hpa_webhook_references); + RUN_TEST(probe_k8s_cr_uses_its_crd_and_unresolved_draws_nothing); + RUN_TEST(probe_k8s_reference_stays_in_bundle); + RUN_TEST(probe_k8s_reference_prefers_its_own_file); + RUN_TEST(probe_k8s_cr_prefers_the_crd_in_its_bundle); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); RUN_TEST(probe_k8s_kustomize_namespace_is_effective); From dd3a294737df22c73c06a40a4262649e3690afae Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 21:07:27 +0200 Subject: [PATCH 22/52] feat(k8s): Go operators link to the kinds they reconcile A Kubernetes operator written with controller-runtime ties its Go code to cluster objects in ways the type graph does not carry: the controller builder names the kinds it reconciles, a CRD's group and kind are backed by a Go struct, and the manifests ship inside the binary through //go:embed. None of it was in the graph -- on the kubermatic, gardener, cluster-api and deckhouse operators, "which controller reconciles Seeds" or "which Go type is this CRD" had no edge to walk. A new operator pass (after USAGE resolution, reading each Go file once) adds: - HANDLES {via: controller, role: for|owns|watches} from the function that builds a controller -- `For(&v1.X{})`, `Owns(...)`, `Watches(...)`, `source.Kind(cache, &v1.X{})` -- to the Go type, resolved as Go resolves it: the alias is an import of the file, and the import path maps to a directory through the repository's go.mod module roots (kubermatic's `k8c.io/kubermatic/sdk/v2/apis/...` lives in `sdk/apis/...`); unqualified types are the file's own package. gofmt chains with the '.' on an earlier line, comment lines between links, and calls inside comments are handled; - USAGE {ref: crd-type} from a CRD Resource (now carrying crd_group and crd_kind) to `type struct` in the package marked `+groupName=` or declaring `GroupName = ""`; - REFERENCES_FILE {via: go:embed} from a Go file to every file its //go:embed patterns match (Go's path.Match per segment, hidden files only under all:). Go method QNs carry no receiver, so every reconciler's SetupWithManager in one package is a single node standing for several definitions. The graph buffer gains cbm_gbuf_foreach_span, which visits every definition span a node stands for (collision spans not yet settled, else its variants, else its own), so a call is placed in the right definition whichever file it is in. Measured on 17 infrastructure repositories against an independent reading of the Go sources: controller HANDLES recall 343/343, precision 335/335 (kubermatic 125, gardener 79, deckhouse 80, cluster-api 59 facts); kubermatic acceptance: reconciler -> kind 13/13, CRD -> Go type 32/32, go:embed files 115/115. Compared with the previous build only the new edges and the two CRD properties differ; three runs build identical graphs. Signed-off-by: Martin Vogel --- Makefile.cbm | 1 + src/graph_buffer/graph_buffer.c | 41 ++ src/graph_buffer/graph_buffer.h | 7 + src/pipeline/pass_k8s.c | 26 + src/pipeline/pass_operator.c | 944 ++++++++++++++++++++++++++++ src/pipeline/pipeline.c | 4 + src/pipeline/pipeline_incremental.c | 8 + src/pipeline/pipeline_internal.h | 5 + tests/test_grammar_probe_f.c | 197 ++++++ 9 files changed, 1233 insertions(+) create mode 100644 src/pipeline/pass_operator.c diff --git a/Makefile.cbm b/Makefile.cbm index 76ba5b467..1dfc9b8ec 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -518,6 +518,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_compile_commands.c \ src/pipeline/pass_infrascan.c \ src/pipeline/pass_k8s.c \ + src/pipeline/pass_operator.c \ src/pipeline/pass_similarity.c \ src/pipeline/pass_semantic_edges.c \ src/pipeline/pass_complexity.c \ diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index fd81f2c40..5b17ec273 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -1131,6 +1131,47 @@ static void gb_free_span_vec(const char *qn, void *value, void *ud) { gb_span_vec_free(value); } +void cbm_gbuf_foreach_span(const cbm_gbuf_t *gb, const cbm_gbuf_node_t *n, cbm_gbuf_span_fn fn, + void *ud) { + if (!gb || !n || !fn) { + return; + } + const gb_span_vec_t *vec = gb->variant_spans && n->qualified_name + ? cbm_ht_get(gb->variant_spans, n->qualified_name) + : NULL; + if (vec && vec->count > 0) { + for (size_t i = 0; i < vec->count; i++) { + fn(vec->items[i].file, vec->items[i].start, vec->items[i].end, ud); + } + return; + } + if (gb_props_have_variants(n->properties_json)) { + yyjson_doc *doc = yyjson_read_opts((char *)n->properties_json, strlen(n->properties_json), + 0, &gb_yy_alc, NULL); + yyjson_val *list = gb_variants_list(doc); + size_t i; + size_t max; + yyjson_val *v; + bool any = false; + yyjson_arr_foreach(list, i, max, v) { + const char *f = yyjson_get_str(yyjson_obj_get(v, "file_path")); + int s = (int)yyjson_get_int(yyjson_obj_get(v, "start_line")); + int e = (int)yyjson_get_int(yyjson_obj_get(v, "end_line")); + if (f && s > 0) { + fn(f, s, e, ud); + any = true; + } + } + yyjson_doc_free(doc); + if (any) { + return; + } + } + if (n->file_path && n->start_line > 0) { + fn(n->file_path, n->start_line, n->end_line, ud); + } +} + void cbm_gbuf_settle_variants(cbm_gbuf_t *gb) { if (!gb || !gb->variant_spans) { return; diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index a0783261d..b7a4e746a 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -194,4 +194,11 @@ int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json); * properties_json from the buffer after collisions. */ void cbm_gbuf_settle_variants(cbm_gbuf_t *gb); +/* Visit every definition span node `n` stands for: the spans gathered from + * same-QN collisions (not yet written, or already in its `variants`), else + * its own. For passes that place a source line inside a definition. */ +typedef void (*cbm_gbuf_span_fn)(const char *file, int start, int end, void *ud); +void cbm_gbuf_foreach_span(const cbm_gbuf_t *gb, const cbm_gbuf_node_t *n, cbm_gbuf_span_fn fn, + void *ud); + #endif /* CBM_GRAPH_BUFFER_H */ diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index f09037b08..bbc465be9 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -1252,10 +1252,36 @@ static bool k8s_same_bundle(k8s_bundles_t *b, const char *a, const char *c) { * kind. A CRD vendored in several places: the copy deployed with the * resource (one bundle) is the one; with no copy beside it, the resource * stands for all of them. */ +/* A CRD node says which group/kind it declares (search, and the Go operator + * pass's CRD -> type link). */ +static void k8s_mark_crd(cbm_pipeline_ctx_t *ctx, const k8s_crd_t *crd) { + cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)cbm_gbuf_find_by_id(ctx->gbuf, crd->node); + const char *props = n ? n->properties_json : NULL; + size_t pl = props ? strlen(props) : 0; + if (!props || pl < PAIR_LEN || props[pl - SKIP_ONE] != '}' || strstr(props, "\"crd_kind\"")) { + return; + } + char group[K8S_LABEL_LEN * PAIR_LEN]; + char kind[K8S_LABEL_LEN * PAIR_LEN]; + cbm_json_escape(group, (int)sizeof(group), crd->group); + cbm_json_escape(kind, (int)sizeof(kind), crd->kind); + size_t n_out = pl + strlen(group) + strlen(kind) + CBM_SZ_64; + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, n_out); + if (out) { + snprintf(out, n_out, "%.*s,\"crd_group\":\"%s\",\"crd_kind\":\"%s\"}", (int)(pl - SKIP_ONE), + props, group, kind); + (void)cbm_gbuf_node_set_properties_json(n, out); + cbm_free(CBM_MEM_CLASS_OTHER, out); + } +} + static void k8s_link_crds(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_facts_t *f) { if (f->crd_n == 0) { return; } + for (int i = 0; i < f->crd_n; i++) { + k8s_mark_crd(ctx, &f->crds[i]); + } /* group/kind -> the run of its CRDs in `order` (sorted by group/kind), so * each resource visits only the CRDs of its own type. */ k8s_crd_key_t *order = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)f->crd_n * sizeof(*order)); diff --git a/src/pipeline/pass_operator.c b/src/pipeline/pass_operator.c new file mode 100644 index 000000000..0c0bdc468 --- /dev/null +++ b/src/pipeline/pass_operator.c @@ -0,0 +1,944 @@ +/* + * pass_operator.c — Go Kubernetes operators (PHASE1 P1.2). + * + * A controller-runtime operator ties Go code to Kubernetes kinds in three + * ways the type graph alone does not carry: + * - a controller is built `For(&v1.Widget{})`, `Owns(&v1.Gadget{})`, + * `Watches(&v1.X{}, ...)` or watches `source.Kind(cache, &v1.X{})`: + * HANDLES {via: controller, role} from the function that builds it to + * the Go type, resolved as Go does: `v1` is an import of the file, and + * the type is the struct of that name in the imported package's + * directory (unqualified: the file's own package); + * - a CRD's group/kind is backed by `type struct` in the package + * marked `+groupName=` (or `GroupName = ""`): USAGE + * {ref: crd-type} from the CRD Resource to that Struct; + * - `//go:embed ` puts files into the binary: REFERENCES_FILE + * {via: go:embed} from the Go file to each embedded file. + * Runs after USAGE resolution; reads each Go file once. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/hash_table.h" +#include "foundation/limits.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" + +#include +#include +#include +#include + +#define OP_SLEN(lit) (sizeof(lit) - SKIP_ONE) + +enum { OP_GROW_MIN = 16, OP_IDENT_MAX = 128 }; + +/* Read a whole file into a core-owned NUL-terminated buffer, NULL on error + * or past the indexer's file-size policy. */ +static char *op_read_file(const char *path) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + SKIP_ONE); + if (buf) { + size_t n = fread(buf, SKIP_ONE, (size_t)size, f); + buf[n] = '\0'; + } + (void)fclose(f); + return buf; +} + +static void op_dirname(const char *path, char *out, size_t out_sz) { + const char *slash = strrchr(path, '/'); + snprintf(out, out_sz, "%.*s", slash ? (int)(slash - path) : 0, path); +} + +/* ── Functions per file ──────────────────────────────────────────────── */ + +/* One definition span of a function node: Go method QNs carry no receiver, + * so the methods of several types in one package (every reconciler's + * SetupWithManager) are one node standing for spans in several files. */ +typedef struct { + const cbm_gbuf_node_t *node; + int start; + int end; +} op_span_t; + +typedef struct { + op_span_t *items; + int count; + int cap; +} op_spans_t; + +static void op_spans_push(op_spans_t *v, op_span_t span) { + if (v->count == v->cap) { + int cap = v->cap ? v->cap * PAIR_LEN : OP_GROW_MIN; + op_span_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, v->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + v->items = grown; + v->cap = cap; + } + v->items[v->count++] = span; +} + +static void op_free_spans(const char *key, void *value, void *ud) { + (void)ud; + op_spans_t *v = value; + cbm_free(CBM_MEM_CLASS_OTHER, v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +typedef struct { + CBMHashTable *by_file; + const cbm_gbuf_node_t *node; +} op_index_ctx_t; + +static void op_index_span(const char *file, int start, int end, void *ud) { + op_index_ctx_t *c = ud; + op_spans_t *v = cbm_ht_get(c->by_file, file); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, file); + if (!v || !key) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, key); + return; + } + cbm_ht_set(c->by_file, key, v); + } + op_spans_push(v, (op_span_t){c->node, start, end}); +} + +/* file_path -> the spans of Function and Method nodes in it (owned keys). */ +static CBMHashTable *op_functions_by_file(const cbm_gbuf_t *gb) { + CBMHashTable *by_file = cbm_ht_create(0); + static const char *const labels[] = {"Function", "Method"}; + for (size_t l = 0; by_file && l < sizeof(labels) / sizeof(labels[0]); l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(gb, labels[l], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count; i++) { + op_index_ctx_t c = {by_file, nodes[i]}; + cbm_gbuf_foreach_span(gb, nodes[i], op_index_span, &c); + } + } + return by_file; +} + +/* Is span `a` a better enclosing function than `b`: shorter, then (equal + * spans of two nodes) the lower QN, so the choice never follows index order. */ +static bool op_span_better(const op_span_t *a, const op_span_t *b) { + int la = a->end - a->start; + int lb = b->end - b->start; + if (la != lb) { + return la < lb; + } + const char *qa = a->node->qualified_name ? a->node->qualified_name : ""; + const char *qb = b->node->qualified_name ? b->node->qualified_name : ""; + return strcmp(qa, qb) < 0; +} + +/* The innermost function of `file` spanning `line`. */ +static const cbm_gbuf_node_t *op_enclosing(CBMHashTable *by_file, const char *file, int line) { + const op_spans_t *v = cbm_ht_get(by_file, file); + const op_span_t *best = NULL; + for (int i = 0; v && i < v->count; i++) { + const op_span_t *sp = &v->items[i]; + if (sp->start <= line && line <= sp->end && (!best || op_span_better(sp, best))) { + best = sp; + } + } + return best ? best->node : NULL; +} + +/* ── Go modules: import path -> directory ─────────────────────────────── */ + +/* A go.mod of the repository: module path and the directory it roots. */ +typedef struct { + char *path; + char *dir; +} op_module_t; + +typedef struct { + op_module_t *items; + int count; + int cap; +} op_modules_t; + +static void op_modules_free(op_modules_t *m) { + for (int i = 0; i < m->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, m->items[i].path); + cbm_free(CBM_MEM_CLASS_OTHER, m->items[i].dir); + } + cbm_free(CBM_MEM_CLASS_OTHER, m->items); + *m = (op_modules_t){0}; +} + +/* Note the `module` line of a go.mod at `file`. */ +static void op_note_module(op_modules_t *m, const char *file, const char *text) { + for (const char *p = strstr(text, "module"); p; p = strstr(p + SKIP_ONE, "module")) { + if ((p > text && p[-SKIP_ONE] != '\n') || + (p[OP_SLEN("module")] != ' ' && p[OP_SLEN("module")] != '\t')) { + continue; + } + const char *q = p + OP_SLEN("module"); + while (*q == ' ' || *q == '\t' || *q == '"') { + q++; + } + size_t n = strcspn(q, " \t\r\n\"/"); + while (q[n] == '/') { /* module paths contain '/' */ + n += SKIP_ONE + strcspn(q + n + SKIP_ONE, " \t\r\n\"/"); + } + if (n == 0) { + return; + } + if (m->count == m->cap) { + int cap = m->cap ? m->cap * PAIR_LEN : OP_GROW_MIN; + op_module_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, m->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + m->items = grown; + m->cap = cap; + } + char path[CBM_SZ_1K]; + char dir[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%.*s", (int)n, q); + op_dirname(file, dir, sizeof(dir)); + m->items[m->count].path = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, path); + m->items[m->count].dir = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir); + m->count++; + return; + } +} + +/* The repository directory of import path `import`: below the go.mod whose + * module path is its longest prefix ("" when no module of the repository + * has it). */ +static bool op_import_dir(const op_modules_t *m, const char *import, char *out, size_t out_sz) { + const op_module_t *best = NULL; + for (int i = 0; i < m->count; i++) { + size_t pl = strlen(m->items[i].path); + if (strncmp(import, m->items[i].path, pl) == 0 && + (import[pl] == '\0' || import[pl] == '/') && (!best || pl > strlen(best->path))) { + best = &m->items[i]; + } + } + if (!best) { + return false; + } + const char *rest = import + strlen(best->path); + rest += *rest == '/'; + snprintf(out, out_sz, "%s%s%s", best->dir, best->dir[0] && rest[0] ? "/" : "", rest); + return true; +} + +/* ── Controllers: For / Owns / Watches / source.Kind ──────────────────── */ + +static int op_line_of(const char *text, const char *at) { + int line = SKIP_ONE; + for (const char *p = text; p < at; p++) { + line += *p == '\n'; + } + return line; +} + +static bool op_is_quote(char c) { + return c == '"' || c == '\'' || c == '`'; +} + +/* Where a `//` comment starts in the line text [ls, le), outside string and + * rune literals; le when there is none. */ +static const char *op_comment_start(const char *ls, const char *le) { + char quote = 0; + for (const char *p = ls; p < le; p++) { + if (quote) { + if (*p == '\\' && quote != '`' && p + SKIP_ONE < le) { + p++; + } else if (*p == quote) { + quote = 0; + } + } else if (op_is_quote(*p)) { + quote = *p; + } else if (*p == '/' && p + SKIP_ONE < le && p[SKIP_ONE] == '/') { + return p; + } + } + return le; +} + +static const char *op_line_start(const char *text, const char *p) { + while (p > text && p[-SKIP_ONE] != '\n') { + p--; + } + return p; +} + +/* The code character before `at`, past blanks and `//` comments: gofmt + * puts a chain's '.' at the end of an earlier line, and comment lines may + * sit between. NULL at the start of the text. */ +static const char *op_prev_code(const char *text, const char *at) { + const char *le = at; + for (;;) { + const char *ls = op_line_start(text, le); + const char *end = op_comment_start(ls, le); + while (end > ls && + (end[-SKIP_ONE] == ' ' || end[-SKIP_ONE] == '\t' || end[-SKIP_ONE] == '\r')) { + end--; + } + if (end > ls) { + return end - SKIP_ONE; + } + if (ls == text) { + return NULL; + } + le = ls - SKIP_ONE; /* the previous line, without its '\n' */ + } +} + +/* The end of the argument list opened at `open` ('('), balanced; parentheses + * in `//` comments and literals do not count. */ +static const char *op_args_end(const char *open) { + int depth = 0; + char quote = 0; + for (const char *p = open; *p; p++) { + if (quote) { + if (*p == '\\' && quote != '`' && p[SKIP_ONE]) { + p++; + } else if (*p == quote || (*p == '\n' && quote != '`')) { + quote = 0; + } + continue; + } + if (op_is_quote(*p)) { + quote = *p; + continue; + } + if (*p == '/' && p[SKIP_ONE] == '/') { + p = strchr(p, '\n'); + if (!p) { + break; + } + continue; + } + depth += *p == '('; + depth -= *p == ')'; + if (depth == 0) { + return p; + } + } + return open + strlen(open); +} + +static bool op_ident_char(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* The import path a file binds to `alias` (an explicit alias, else the + * path's last segment, Go's usual package name); "" when none. */ +static void op_import_path(const char *text, const char *alias, char *out, size_t out_sz) { + out[0] = '\0'; + size_t al = strlen(alias); + for (const char *p = strstr(text, "import"); p; p = strstr(p + SKIP_ONE, "import")) { + if ((p > text && p[-SKIP_ONE] != '\n') || op_ident_char(p[OP_SLEN("import")])) { + continue; + } + const char *q = p + OP_SLEN("import"); + const char *end = q; + while (*end == ' ' || *end == '\t') { + end++; + } + end = *end == '(' ? strchr(end, ')') : strchr(end, '\n'); + end = end ? end : q + strlen(q); + /* each import line: [alias] "path" */ + for (const char *l = q; l < end;) { + while (l < end && (*l == ' ' || *l == '\t' || *l == '(' || *l == '\n' || *l == '\r')) { + l++; + } + const char *name = l; + while (l < end && op_ident_char(*l)) { + l++; + } + size_t nl = (size_t)(l - name); + while (l < end && (*l == ' ' || *l == '\t')) { + l++; + } + if (l >= end || *l != '"') { + while (l < end && *l != '\n') { + l++; + } + continue; + } + const char *ps = l + SKIP_ONE; + const char *pe = strchr(ps, '"'); + if (!pe || pe > end) { + break; + } + const char *seg = pe; + while (seg > ps && seg[-SKIP_ONE] != '/') { + seg--; + } + bool hit = nl ? (nl == al && strncmp(name, alias, al) == 0) + : ((size_t)(pe - seg) == al && strncmp(seg, alias, al) == 0); + if (hit) { + snprintf(out, out_sz, "%.*s", (int)(pe - ps), ps); + return; + } + l = pe + SKIP_ONE; + } + } +} + +/* Is directory `dir` the package at import path `path`: the path ends with + * the directory, on a segment boundary (the module path is its prefix). */ +static bool op_dir_is_import(const char *dir, const char *path) { + size_t dl = strlen(dir); + size_t pl = strlen(path); + if (dl == 0 || dl > pl || strcmp(path + pl - dl, dir) != 0) { + return false; + } + return dl == pl || path[pl - dl - SKIP_ONE] == '/'; +} + +/* HANDLES from `fn` to the Go type `type` resolved as Go resolves it: the + * package the file imports as `qual`, or (unqualified) the file's own. */ +static int op_handle(cbm_gbuf_t *gb, const op_modules_t *mods, const cbm_gbuf_node_t *fn, + const char *file, const char *text, const char *qual, const char *type, + const char *role) { + char import[CBM_SZ_1K] = ""; + char own_dir[CBM_SZ_1K]; + char pkg_dir[CBM_SZ_1K] = ""; + op_dirname(file, own_dir, sizeof(own_dir)); + bool in_repo = false; + if (qual[0]) { + op_import_path(text, qual, import, sizeof(import)); + if (!import[0]) { + return 0; + } + in_repo = op_import_dir(mods, import, pkg_dir, sizeof(pkg_dir)); + } + const cbm_gbuf_node_t **types = NULL; + int tn = 0; + if (cbm_gbuf_find_by_name(gb, type, &types, &tn) != 0) { + return 0; + } + char props[CBM_SZ_128]; + snprintf(props, sizeof(props), "{\"via\":\"controller\",\"role\":\"%s\"}", role); + int linked = 0; + for (int i = 0; i < tn; i++) { + const cbm_gbuf_node_t *t = types[i]; + if (!t->label || !t->file_path || + (strcmp(t->label, "Struct") != 0 && strcmp(t->label, "Class") != 0 && + strcmp(t->label, "Type") != 0)) { + continue; + } + char dir[CBM_SZ_1K]; + op_dirname(t->file_path, dir, sizeof(dir)); + bool same = !qual[0] ? strcmp(dir, own_dir) == 0 + : in_repo ? strcmp(dir, pkg_dir) == 0 + : op_dir_is_import(dir, import); + if (same) { + cbm_gbuf_insert_edge(gb, fn->id, t->id, "HANDLES", props); + linked++; + } + } + return linked; +} + +/* Every `&[pkg.]Type{` in the arguments of a controller builder call. */ +static int op_controller_calls(cbm_gbuf_t *gb, CBMHashTable *by_file, const op_modules_t *mods, + const char *file, const char *text) { + /* method name + "(", called on a receiver outside a comment: the previous + * code char is '.', and Kind's receiver is the `source` package. */ + static const struct { + const char *name; + const char *recv; + const char *role; + } calls[] = {{"For(", NULL, "for"}, + {"Owns(", NULL, "owns"}, + {"Watches(", NULL, "watches"}, + {"Kind(", "source", "watches"}}; + int linked = 0; + for (size_t c = 0; c < sizeof(calls) / sizeof(calls[0]); c++) { + for (const char *p = strstr(text, calls[c].name); p; + p = strstr(p + SKIP_ONE, calls[c].name)) { + if (p > text && op_ident_char(p[-SKIP_ONE])) { + continue; /* part of a longer name */ + } + if (op_comment_start(op_line_start(text, p), p) < p) { + continue; + } + const char *dot = op_prev_code(text, p); + if (!dot || *dot != '.') { + continue; + } + if (calls[c].recv) { + size_t rl = strlen(calls[c].recv); + const char *r = dot - rl; + if (r < text || strncmp(r, calls[c].recv, rl) != 0 || + (r > text && op_ident_char(r[-SKIP_ONE]))) { + continue; + } + } + const char *open = p + strlen(calls[c].name) - SKIP_ONE; + const char *end = op_args_end(open); + const cbm_gbuf_node_t *fn = op_enclosing(by_file, file, op_line_of(text, p)); + if (!fn) { + continue; + } + for (const char *a = open; a < end; a++) { + if (*a == '/' && a[SKIP_ONE] == '/') { + const char *nl = strchr(a, '\n'); + a = nl && nl < end ? nl : end; + continue; + } + if (*a != '&') { + continue; + } + const char *q = a + SKIP_ONE; + const char *first = q; + const char *last = q; + while (q < end && (op_ident_char(*q) || *q == '.')) { + if (*q == '.') { + last = q + SKIP_ONE; + } + q++; + } + if (q < end && *q == '{' && q > last && (size_t)(q - last) < OP_IDENT_MAX && + (size_t)(last - first) < OP_IDENT_MAX) { + char type[OP_IDENT_MAX]; + char qual[OP_IDENT_MAX]; + snprintf(type, sizeof(type), "%.*s", (int)(q - last), last); + snprintf(qual, sizeof(qual), "%.*s", + last > first ? (int)(last - first - SKIP_ONE) : 0, first); + linked += op_handle(gb, mods, fn, file, text, qual, type, calls[c].role); + } + } + } + } + return linked; +} + +/* ── API groups: +groupName / GroupName ───────────────────────────────── */ + +/* A Go identifier-or-DNS token at p (the group name). */ +static void op_token(const char *p, char *out, size_t out_sz) { + size_t n = 0; + while (p[n] && (op_ident_char(p[n]) || p[n] == '.' || p[n] == '-') && n + SKIP_ONE < out_sz) { + n++; + } + snprintf(out, out_sz, "%.*s", (int)n, p); +} + +/* Owned strings. */ +typedef struct { + char **items; + int count; + int cap; +} op_strs_t; + +static void op_strs_push(op_strs_t *v, const char *s) { + if (v->count == v->cap) { + int cap = v->cap ? v->cap * PAIR_LEN : OP_GROW_MIN; + char **grown = cbm_realloc(CBM_MEM_CLASS_OTHER, v->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + v->items = grown; + v->cap = cap; + } + char *copy = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, s); + if (copy) { + v->items[v->count++] = copy; + } +} + +/* group -> the package directories that declare it (owned keys and lists). */ +static void op_note_group(CBMHashTable *groups, const char *group, const char *dir) { + if (!group[0]) { + return; + } + op_strs_t *dirs = cbm_ht_get(groups, group); + if (!dirs) { + dirs = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*dirs)); + char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, group); + if (!dirs || !key) { + cbm_free(CBM_MEM_CLASS_OTHER, dirs); + cbm_free(CBM_MEM_CLASS_OTHER, key); + return; + } + cbm_ht_set(groups, key, dirs); + } + for (int i = 0; i < dirs->count; i++) { + if (strcmp(dirs->items[i], dir) == 0) { + return; + } + } + op_strs_push(dirs, dir); +} + +static void op_scan_groups(CBMHashTable *groups, const char *file, const char *text) { + char dir[CBM_SZ_1K]; + op_dirname(file, dir, sizeof(dir)); + for (const char *p = strstr(text, "+groupName="); p; p = strstr(p + SKIP_ONE, "+groupName=")) { + char g[CBM_SZ_256]; + op_token(p + OP_SLEN("+groupName="), g, sizeof(g)); + op_note_group(groups, g, dir); + } + for (const char *p = strstr(text, "GroupName"); p; p = strstr(p + SKIP_ONE, "GroupName")) { + const char *q = p + OP_SLEN("GroupName"); + while (*q == ' ' || *q == '\t') { + q++; + } + if (*q != '=') { + continue; + } + q++; + while (*q == ' ' || *q == '\t') { + q++; + } + if (*q == '"') { + char g[CBM_SZ_256]; + op_token(q + SKIP_ONE, g, sizeof(g)); + op_note_group(groups, g, dir); + } + } +} + +static void op_free_groups(const char *key, void *value, void *ud) { + (void)ud; + op_strs_t *dirs = value; + for (int i = 0; i < dirs->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, dirs->items[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, dirs->items); + cbm_free(CBM_MEM_CLASS_OTHER, dirs); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +/* A JSON string property's value (simple, unescaped scalars). */ +static bool op_prop(const char *props, const char *key, char *out, size_t out_sz) { + char pat[CBM_SZ_64]; + snprintf(pat, sizeof(pat), "\"%s\":\"", key); + const char *p = props ? strstr(props, pat) : NULL; + if (!p) { + return false; + } + p += strlen(pat); + const char *e = strchr(p, '"'); + if (!e) { + return false; + } + snprintf(out, out_sz, "%.*s", (int)(e - p), p); + return true; +} + +/* CRD -> the Go type declared under its group, in a package marking it. */ +static int op_link_crd_types(cbm_gbuf_t *gb, CBMHashTable *groups) { + const cbm_gbuf_node_t **res = NULL; + int count = 0; + int linked = 0; + if (cbm_gbuf_find_by_label(gb, "Resource", &res, &count) != 0) { + return 0; + } + for (int i = 0; i < count; i++) { + char group[CBM_SZ_256]; + char kind[CBM_SZ_256]; + if (!op_prop(res[i]->properties_json, "crd_group", group, sizeof(group)) || + !op_prop(res[i]->properties_json, "crd_kind", kind, sizeof(kind))) { + continue; + } + const op_strs_t *dirs = cbm_ht_get(groups, group); + const cbm_gbuf_node_t **types = NULL; + int tn = 0; + if (!dirs || cbm_gbuf_find_by_name(gb, kind, &types, &tn) != 0) { + continue; + } + for (int t = 0; t < tn; t++) { + if (!types[t]->label || strcmp(types[t]->label, "Struct") != 0 || + !types[t]->file_path) { + continue; + } + char dir[CBM_SZ_1K]; + op_dirname(types[t]->file_path, dir, sizeof(dir)); + for (int d = 0; d < dirs->count; d++) { + if (strcmp(dirs->items[d], dir) == 0) { + cbm_gbuf_insert_edge(gb, res[i]->id, types[t]->id, "USAGE", + "{\"ref\":\"crd-type\"}"); + linked++; + break; + } + } + } + } + return linked; +} + +/* ── go:embed ─────────────────────────────────────────────────────────── */ + +/* Go's path.Match for one segment: '*', '?', '[...]', '\\' escapes. */ +static bool op_match(const char *pat, const char *s) { + while (*pat) { + if (*pat == '*') { + pat++; + for (const char *t = s;; t++) { + if (op_match(pat, t)) { + return true; + } + if (!*t) { + return false; + } + } + } + if (!*s) { + return false; + } + if (*pat == '?') { + pat++; + s++; + continue; + } + if (*pat == '[') { + const char *q = pat + SKIP_ONE; + bool neg = *q == '^'; + q += neg; + bool hit = false; + while (*q && *q != ']') { + char lo = *q; + char hi = lo; + if (q[SKIP_ONE] == '-' && q[PAIR_LEN] && q[PAIR_LEN] != ']') { + hi = q[PAIR_LEN]; + q += PAIR_LEN; + } + hit = hit || (*s >= lo && *s <= hi); + q++; + } + if (*q != ']' || hit == neg) { + return false; + } + pat = q + SKIP_ONE; + s++; + continue; + } + if (*pat == '\\' && pat[SKIP_ONE]) { + pat++; + } + if (*pat != *s) { + return false; + } + pat++; + s++; + } + return *s == '\0'; +} + +/* Does `rel` (a path below the Go file's directory) match the embed + * pattern: segment by segment, a matched directory taking its files except + * hidden ones ('.', '_') unless `all`. */ +static bool op_embeds(const char *pattern, const char *rel, bool all) { + char pat[CBM_SZ_1K]; + char path[CBM_SZ_1K]; + snprintf(pat, sizeof(pat), "%s", pattern); + snprintf(path, sizeof(path), "%s", rel); + char *ps = pat; + char *rs = path; + while (ps && rs) { + char *pn = strchr(ps, '/'); + char *rn = strchr(rs, '/'); + if (pn) { + *pn = '\0'; + } + if (rn) { + *rn = '\0'; + } + if (!op_match(ps, rs)) { + return false; + } + ps = pn ? pn + SKIP_ONE : NULL; + rs = rn ? rn + SKIP_ONE : NULL; + if (!ps) { + /* pattern consumed: the file itself, or a file under a matched dir */ + for (char *seg = rs; seg && !all;) { + if (seg[0] == '.' || seg[0] == '_') { + return false; + } + char *next = strchr(seg, '/'); + seg = next ? next + SKIP_ONE : NULL; + } + return true; + } + } + return false; +} + +static int op_cmp_path(const void *a, const void *b) { + const cbm_gbuf_node_t *x = *(const cbm_gbuf_node_t *const *)a; + const cbm_gbuf_node_t *y = *(const cbm_gbuf_node_t *const *)b; + return strcmp(x->file_path, y->file_path); +} + +/* REFERENCES_FILE from the Go file to every file its //go:embed lines name; + * `files` is every File node sorted by path, so a directory's files are one + * range. */ +static int op_embed(cbm_gbuf_t *gb, const cbm_gbuf_node_t *go_file, const char *text, + const cbm_gbuf_node_t **files, int nfiles) { + char dir[CBM_SZ_1K]; + op_dirname(go_file->file_path, dir, sizeof(dir)); + size_t dl = strlen(dir); + int linked = 0; + for (const char *p = strstr(text, "//go:embed "); p; p = strstr(p + SKIP_ONE, "//go:embed ")) { + if (p != text && p[-SKIP_ONE] != '\n') { + continue; /* a directive starts its line */ + } + const char *eol = strchr(p, '\n'); + char line[CBM_SZ_1K]; + const char *args = p + OP_SLEN("//go:embed "); + snprintf(line, sizeof(line), "%.*s", eol ? (int)(eol - args) : (int)strlen(args), args); + for (char *tok = line; *tok;) { + while (*tok == ' ' || *tok == '\t' || *tok == '\r') { + tok++; + } + char *e = tok; + while (*e && *e != ' ' && *e != '\t' && *e != '\r') { + e++; + } + char save = *e; + *e = '\0'; + bool all = strncmp(tok, "all:", OP_SLEN("all:")) == 0; + const char *pattern = all ? tok + OP_SLEN("all:") : tok; + if (pattern[0] && pattern[0] != '"' && pattern[0] != '`') { + /* files below dir: the sorted range starting at "dir/" */ + int lo = 0; + int hi = dl ? nfiles : 0; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (strncmp(files[mid]->file_path, dir, dl) < 0 || + (strncmp(files[mid]->file_path, dir, dl) == 0 && + files[mid]->file_path[dl] < '/')) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (int i = lo; i < nfiles; i++) { + const char *fp = files[i]->file_path; + if (dl && (strncmp(fp, dir, dl) != 0 || fp[dl] != '/')) { + break; + } + const char *rel = dl ? fp + dl + SKIP_ONE : fp; + if (files[i] != go_file && op_embeds(pattern, rel, all)) { + cbm_gbuf_insert_edge(gb, go_file->id, files[i]->id, "REFERENCES_FILE", + "{\"via\":\"go:embed\"}"); + linked++; + } + } + } + *e = save; + tok = e; + } + } + return linked; +} + +/* ── The pass ─────────────────────────────────────────────────────────── */ + +static bool op_is_go(const char *path) { + size_t n = strlen(path); + return n > OP_SLEN(".go") && strcmp(path + n - OP_SLEN(".go"), ".go") == 0; +} + +int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { + cbm_gbuf_t *gb = ctx->gbuf; + const cbm_gbuf_node_t **all = NULL; + int count = 0; + if (!gb || !ctx->repo_path || cbm_gbuf_find_by_label(gb, "File", &all, &count) != 0) { + return 0; + } + const cbm_gbuf_node_t **files = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(count ? count : SKIP_ONE) * sizeof(*files)); + if (!files) { + return CBM_NOT_FOUND; + } + int nfiles = 0; + for (int i = 0; i < count; i++) { + if (all[i]->file_path) { + files[nfiles++] = all[i]; + } + } + qsort((void *)files, (size_t)nfiles, sizeof(*files), op_cmp_path); + CBMHashTable *by_file = op_functions_by_file(gb); + CBMHashTable *groups = cbm_ht_create(0); + op_modules_t mods = {0}; + for (int i = 0; i < nfiles; i++) { + const char *slash = strrchr(files[i]->file_path, '/'); + if (strcmp(slash ? slash + SKIP_ONE : files[i]->file_path, "go.mod") == 0) { + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, files[i]->file_path); + char *text = op_read_file(path); + if (text) { + op_note_module(&mods, files[i]->file_path, text); + cbm_free(CBM_MEM_CLASS_OTHER, text); + } + } + } + int handles = 0; + int embeds = 0; + for (int i = 0; by_file && groups && i < nfiles; i++) { + if (!op_is_go(files[i]->file_path) || cbm_pipeline_check_cancel(ctx)) { + continue; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, files[i]->file_path); + char *text = op_read_file(path); + if (!text) { + continue; + } + if (strstr(text, "controller-runtime") || strstr(text, "source.Kind(")) { + handles += op_controller_calls(gb, by_file, &mods, files[i]->file_path, text); + } + op_scan_groups(groups, files[i]->file_path, text); + if (strstr(text, "//go:embed ")) { + embeds += op_embed(gb, files[i], text, files, nfiles); + } + cbm_free(CBM_MEM_CLASS_OTHER, text); + } + int crd_types = groups ? op_link_crd_types(gb, groups) : 0; + op_modules_free(&mods); + if (groups) { + cbm_ht_foreach(groups, op_free_groups, NULL); + cbm_ht_free(groups); + } + if (by_file) { + cbm_ht_foreach(by_file, op_free_spans, NULL); + cbm_ht_free(by_file); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)files); + if (handles + embeds + crd_types > 0) { + char h[CBM_SZ_32]; + char e[CBM_SZ_32]; + char c[CBM_SZ_32]; + snprintf(h, sizeof(h), "%d", handles); + snprintf(e, sizeof(e), "%d", embeds); + snprintf(c, sizeof(c), "%d", crd_types); + cbm_log_info("pass.operator", "handles", h, "embeds", e, "crd_types", c); + } + return 0; +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 9e8a66c8e..fa966538a 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1837,6 +1837,9 @@ static void predump_sem(cbm_pipeline_ctx_t *ctx) { static void predump_cfg(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_configlink(ctx); } +static void predump_operator(cbm_pipeline_ctx_t *ctx) { + (void)cbm_pipeline_pass_operator(ctx); +} static void predump_doclinks(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_doclinks(ctx); } @@ -2077,6 +2080,7 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { } passes[] = { {predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false}, + {predump_operator, "operator", false}, {predump_doclinks, "doclinks", false}, {predump_route, "route_match", false}, {predump_ensemble, "ensemble_routing", false}, diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index a11f82356..e9afe2811 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2020,6 +2020,14 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file return rc < 0 ? rc : CBM_NOT_FOUND; } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + rc = cbm_pipeline_pass_operator(ctx); + cbm_log_info("pass.timing", "pass", "incr_operator", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + if (rc < 0 || cbm_pipeline_check_cancel(ctx)) { + return rc < 0 ? rc : CBM_NOT_FOUND; + } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); rc = cbm_pipeline_pass_doclinks(ctx); cbm_log_info("pass.timing", "pass", "incr_doclinks", "elapsed_ms", diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 4fe4b549d..19a9d3a08 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1037,6 +1037,11 @@ int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project); /* Pre-dump pass: config ↔ code linking. */ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx); +/* Go Kubernetes operators (pass_operator.c): controller-runtime controllers + * HANDLES the kinds they reconcile, CRDs -> their Go types, //go:embed files. + * Runs after USAGE resolution. */ +int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx); + /* Pre-dump pass: markdown → file REFERENCES_FILE linking. */ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 391687e23..99b5b6ef5 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1084,6 +1084,197 @@ TEST(probe_k8s_reference_prefers_its_own_file) { PASS(); } +/* ── Go operator layer (P1.2) ───────────────────────────────────────── */ + +/* Edges of `type` whose properties carry `prop`, between nodes of + * `src_label` and `tgt_label`; -1 when indexing failed. */ +static int gpf_typed_edges(const GpfFile *files, int nfiles, const char *type, + const char *src_label, const char *tgt_label, const char *prop) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int n = -1; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id " + "JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 AND e.type=?2 " + "AND s.label=?3 AND t.label=?4 AND instr(e.properties,?5)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 2, type, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 3, src_label, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 4, tgt_label, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 5, prop, -1, SQLITE_STATIC); + n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return n; +} + +static const GpfFile k_operator_repo[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"api/v1/doc.go", "// +groupName=example.com\npackage v1\n"}, + {"api/v1/widget_types.go", "package v1\n\ntype Widget struct {\n\tName string\n}\n\n" + "type Gadget struct {\n\tName string\n}\n"}, + {"config/crd/widgets.yaml", + "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\n" + "metadata:\n name: widgets.example.com\nspec:\n group: example.com\n" + " names:\n kind: Widget\n plural: widgets\n scope: Namespaced\n"}, + {"internal/controller/widget.go", "package controller\n\n" + "import (\n\texamplev1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "func SetupWithManager(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).\n" + "\t\tFor(&examplev1.Widget{}).\n" + "\t\tOwns(&examplev1.Gadget{}).\n" + "\t\tComplete(nil)\n" + "}\n"}, + {"config/embed.go", + "package config\n\nimport \"embed\"\n\n//go:embed crd/*\nvar CRDs embed.FS\n"}}; + +/* A controller reconciles the kind it is built For, and the kinds it Owns. */ +TEST(probe_go_controller_handles_its_kinds) { + ASSERT_EQ( + gpf_typed_edges(k_operator_repo, 6, "HANDLES", "Function", "Struct", "\"role\":\"for\""), + 1); + ASSERT_EQ( + gpf_typed_edges(k_operator_repo, 6, "HANDLES", "Function", "Struct", "\"role\":\"owns\""), + 1); + PASS(); +} + +/* The CRD of group example.com / kind Widget is backed by `type Widget` + * in the package marked +groupName=example.com; a Go file embeds the CRD + * files its //go:embed names. */ +TEST(probe_go_crd_type_and_embed) { + ASSERT_EQ( + gpf_typed_edges(k_operator_repo, 6, "USAGE", "Resource", "Struct", "\"ref\":\"crd-type\""), + 1); + ASSERT_EQ(gpf_typed_edges(k_operator_repo, 6, "REFERENCES_FILE", "File", "File", + "\"via\":\"go:embed\""), + 1); + PASS(); +} + +/* The kind a controller reconciles is the type of the package it imports: + * kubermatic has `Cluster` structs and a method `Cluster` elsewhere, and a + * name-only lookup handed `For(&kubermaticv1.Cluster{})` to the method. */ +TEST(probe_go_controller_kind_is_the_imported_type) { + static const GpfFile files[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"api/v1/types.go", "package v1\n\ntype Cluster struct {\n\tName string\n}\n"}, + {"internal/legacy/types.go", "package legacy\n\ntype Cluster struct {\n\tID int\n}\n"}, + {"pkg/resources/creds.go", + "package resources\n\ntype Creds struct{}\n\nfunc (c Creds) Cluster() string {\n" + "\treturn \"\"\n}\n"}, + {"internal/controller/cluster.go", + "package controller\n\n" + "import (\n\tkubermaticv1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "func Add(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).For(&kubermaticv1.Cluster{}).Complete(nil)\n" + "}\n"}}; + ASSERT_EQ(gpf_typed_edges(files, 5, "HANDLES", "Function", "Struct", "\"role\":\"for\""), 1); + ASSERT_EQ(gpf_typed_edges(files, 5, "HANDLES", "Function", "Method", "\"role\":\"for\""), 0); + PASS(); +} + +/* Edges of `type` into nodes defined in `tgt_file`; -1 when indexing failed. */ +static int gpf_edges_into_file(const GpfFile *files, int nfiles, const char *type, + const char *tgt_file) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int n = -1; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type=?2 AND t.file_path=?3"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 2, type, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 3, tgt_file, -1, SQLITE_STATIC); + n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return n; +} + +/* An import path names the directory its go.mod roots: kubermatic's + * `k8c.io/kubermatic/sdk/v2/apis/...` lives in `sdk/apis/...`, which the + * path does not end with, while an unrelated `apis/v1` does. */ +TEST(probe_go_controller_import_follows_go_mod) { + static const GpfFile files[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"sdk/go.mod", "module example.com/proj/sdk/v2\n\ngo 1.22\n"}, + {"sdk/apis/v1/types.go", "package v1\n\ntype Cluster struct {\n\tName string\n}\n"}, + {"apis/v1/types.go", "package v1\n\ntype Cluster struct {\n\tID int\n}\n"}, + {"pkg/controller/cluster.go", + "package controller\n\n" + "import (\n\tkubermaticv1 \"example.com/proj/sdk/v2/apis/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "func Add(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).For(&kubermaticv1.Cluster{}).Complete(nil)\n" + "}\n"}}; + ASSERT_EQ(gpf_edges_into_file(files, 5, "HANDLES", "sdk/apis/v1/types.go"), 1); + ASSERT_EQ(gpf_edges_into_file(files, 5, "HANDLES", "apis/v1/types.go"), 0); + PASS(); +} + +/* A comment line may sit between a chain's '.' and the call (kubermatic's + * seed-sync: `}).` / `// watch all seeds` / `For(&kubermaticv1.Seed{}, ...`), + * and a call written inside a comment builds nothing. */ +TEST(probe_go_controller_chain_with_comment_lines) { + static const GpfFile files[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"api/v1/widget.go", "package v1\n\ntype Widget struct {\n\tName string\n}\n"}, + {"api/v1/gadget.go", "package v1\n\ntype Gadget struct {\n\tName string\n}\n"}, + {"internal/controller/widget.go", + "package controller\n\n" + "import (\n\tv1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "func Add(mgr ctrl.Manager) error {\n" + "\t// gadgets would be b.Owns(&v1.Gadget{}) (not yet)\n" + "\treturn ctrl.NewControllerManagedBy(mgr).\n" + "\t\tNamed(\"widget\").\n" + "\t\t// watch all widgets\n" + "\t\tFor(&v1.Widget{}).\n" + "\t\tComplete(nil)\n" + "}\n"}}; + ASSERT_EQ(gpf_edges_into_file(files, 4, "HANDLES", "api/v1/widget.go"), 1); + ASSERT_EQ(gpf_edges_into_file(files, 4, "HANDLES", "api/v1/gadget.go"), 0); + PASS(); +} + +/* Go method QNs carry no receiver, so every reconciler's SetupWithManager in + * one package is one node: each definition still handles its own kind. */ +TEST(probe_go_controller_methods_sharing_a_node) { + static const GpfFile files[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"api/v1/types.go", "package v1\n\ntype Widget struct {\n\tName string\n}\n\n" + "type Gadget struct {\n\tName string\n}\n"}, + {"internal/controller/widget.go", + "package controller\n\n" + "import (\n\tv1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "type WidgetReconciler struct{}\n\n" + "func (r *WidgetReconciler) SetupWithManager(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).For(&v1.Widget{}).Complete(r)\n" + "}\n"}, + {"internal/controller/gadget.go", + "package controller\n\n" + "import (\n\tv1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "type GadgetReconciler struct{}\n\n" + "func (r *GadgetReconciler) SetupWithManager(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).For(&v1.Gadget{}).Complete(r)\n" + "}\n"}}; + ASSERT_EQ(gpf_edges_into_file(files, 4, "HANDLES", "api/v1/types.go"), 2); + PASS(); +} + /* A CRD vendored twice (gardener: example/seed-crds and a chart): a custom * resource is an instance of the copy deployed with it; only when no copy * shares its bundle does it stand for all of them. */ @@ -2682,6 +2873,12 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_reference_stays_in_bundle); RUN_TEST(probe_k8s_reference_prefers_its_own_file); RUN_TEST(probe_k8s_cr_prefers_the_crd_in_its_bundle); + RUN_TEST(probe_go_controller_handles_its_kinds); + RUN_TEST(probe_go_crd_type_and_embed); + RUN_TEST(probe_go_controller_kind_is_the_imported_type); + RUN_TEST(probe_go_controller_import_follows_go_mod); + RUN_TEST(probe_go_controller_chain_with_comment_lines); + RUN_TEST(probe_go_controller_methods_sharing_a_node); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); RUN_TEST(probe_k8s_kustomize_namespace_is_effective); From 191b581de663425e9feee9672e7e3900c4bb45c8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 21:57:36 +0200 Subject: [PATCH 23/52] feat(k8s): Flux, Argo, Cluster API, cert-manager and KKP references An infrastructure repository wires its custom resources to each other: a Flux Kustomization names its source and the Kustomizations it waits for, a HelmRelease its chart source and values, a Cluster API Cluster its infrastructure, control plane and class, a Certificate its issuer, a Kubermatic Seed its kubeconfig. None of it was in the graph, and most of it crosses kustomize roots, which the K8s references could not. A second, curated reference table -- written from the API types of Flux, Argo CD/Rollouts/Workflows, Cluster API, cert-manager and Kubermatic -- uses the same mechanics as the built-in one. Source kinds are qualified by API group (a CloudNativePG `Cluster` is not a Cluster API one) and paths may use `*` within a segment (any DNS-solver `*SecretRef`, any cloud provider's credentialsReference). References become USAGE {ref}; Flux dependsOn becomes DEPENDS_ON {via: dependsOn}. An Argo Rollout's pod template carries the pod references like any workload. Deploy scopes: a Flux Kustomization's spec.path and an Argo Application's source path deploy a repository directory -- the build of a kustomization there, else every manifest below it, a nested kustomization by its build, as Flux generates one. What a root deploy reaches through such directories, transitively, is one scope (a cluster), and a reference also meets its target inside a shared scope, with the deploy's target namespace. A path naming no directory of the repository (another artifact's layout) deploys nothing. Measured on 17 infrastructure repositories against an independent reading of the documents (bundles and deploy scopes re-derived): curated references recall 168/168, precision 172/172 (4 targets sit in files the reading skips for `{{` and were checked by hand); the built-in table is unchanged at 257/258 and 257/257. Compared with the previous build only USAGE and DEPENDS_ON edges were added (home-ops +104, cluster-api +94, k8s.io +11, flux example +7, kubermatic +4); three runs build identical graphs. Signed-off-by: Martin Vogel --- internal/cbm/cbm.h | 11 +- internal/cbm/extract_k8s.c | 309 ++++++++++++++++++++++++-- src/pipeline/pass_k8s.c | 406 ++++++++++++++++++++++++++++++++++- tests/test_grammar_probe_f.c | 211 ++++++++++++++++++ 4 files changed, 908 insertions(+), 29 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 0ae684030..6be7102d5 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -603,10 +603,13 @@ typedef struct { /* A fact a Kubernetes manifest states about other objects (extract_k8s.c), * resolved by the K8s pass against the objects of the source's bundle. * CBM_K8S_FACT_REF: object def_index names a kind/ns/name object at a - * reference path (via), drawn as edge (USAGE or INFRA_MAPS); ns NULL = - * the source's own namespace. - * CBM_K8S_FACT_CRD: the CRD def_index declares kind `kind` in group `name`. */ -typedef enum { CBM_K8S_FACT_REF = 0, CBM_K8S_FACT_CRD } CBMK8sFactKind; + * reference path (via), drawn as edge (USAGE, INFRA_MAPS or DEPENDS_ON); + * ns NULL = the source's own namespace. + * CBM_K8S_FACT_CRD: the CRD def_index declares kind `kind` in group `name`. + * CBM_K8S_FACT_DEPLOY: object def_index (a Flux Kustomization, an Argo + * Application) deploys the repository directory `name` into namespace + * `ns` (NULL: the objects' own); via "flux" / "argo". */ +typedef enum { CBM_K8S_FACT_REF = 0, CBM_K8S_FACT_CRD, CBM_K8S_FACT_DEPLOY } CBMK8sFactKind; typedef struct { CBMK8sFactKind fact; int def_index; diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 29765d87c..6d5df5565 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -407,6 +407,220 @@ static const k8s_mapping_ref_t k8s_mapping_refs[] = { "parentRef", "USAGE"}, }; +// The curated second table: the CRDs of Flux, Argo, Cluster API, +// cert-manager and Kubermatic, written from their API types. Source kinds +// are qualified by API group ("|g|" list), and a path may use `*`, which +// matches within one segment (`spec.cloud.*.credentialsReference`). +typedef struct { + const char *groups; + const char *kinds; + const char *path; + const char *kind; + const char *via; + const char *edge; +} k8s_crd_leaf_ref_t; + +typedef struct { + const char *groups; + const char *kinds; + const char *path; + const char *kind; // the target kind; NULL = the mapping's own `kind` field + const char *allowed; // with kind NULL: "|A|" of the kinds that are objects (NULL: any) + const char *dflt; // with kind NULL: the kind when the field is absent + const char *via; + const char *edge; +} k8s_crd_mapping_ref_t; + +#define K8S_FLUX_KS "|kustomize.toolkit.fluxcd.io|" +#define K8S_FLUX_HELM "|helm.toolkit.fluxcd.io|" +#define K8S_FLUX_SRC "|source.toolkit.fluxcd.io|" +#define K8S_FLUX_NOTE "|notification.toolkit.fluxcd.io|" +#define K8S_FLUX_IMAGE "|image.toolkit.fluxcd.io|" +#define K8S_ARGO "|argoproj.io|" +#define K8S_CAPI "|cluster.x-k8s.io|" +#define K8S_CAPI_BOOT "|bootstrap.cluster.x-k8s.io|" +#define K8S_CAPI_CP "|controlplane.cluster.x-k8s.io|" +#define K8S_CAPI_ADDONS "|addons.cluster.x-k8s.io|" +#define K8S_CM "|cert-manager.io|" +#define K8S_KKP "|kubermatic.k8c.io|" +#define K8S_KKP_APPS "|apps.kubermatic.k8c.io|" +#define K8S_FLUX_SOURCES "|GitRepository|OCIRepository|HelmRepository|Bucket|" +#define K8S_ISSUERS "|Issuer|ClusterIssuer|" + +static const k8s_crd_leaf_ref_t k8s_crd_leaf_refs[] = { + {K8S_FLUX_KS, "|Kustomization|", "spec.decryption.secretRef.name", "Secret", "secretRef", + "USAGE"}, + {K8S_FLUX_KS "helm.toolkit.fluxcd.io|", "|Kustomization|HelmRelease|", + "spec.kubeConfig.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_KS "helm.toolkit.fluxcd.io|", "|Kustomization|HelmRelease|", + "spec.serviceAccountName", "ServiceAccount", "serviceAccountName", "USAGE"}, + {K8S_FLUX_SRC, K8S_FLUX_SOURCES, "spec.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_SRC, K8S_FLUX_SOURCES, "spec.certSecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_SRC, K8S_FLUX_SOURCES, "spec.proxySecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_SRC, "|GitRepository|OCIRepository|", "spec.verify.secretRef.name", "Secret", + "secretRef", "USAGE"}, + {K8S_FLUX_SRC, "|OCIRepository|HelmRepository|", "spec.serviceAccountName", "ServiceAccount", + "serviceAccountName", "USAGE"}, + {K8S_FLUX_NOTE, "|Alert|", "spec.providerRef.name", "Provider", "providerRef", "USAGE"}, + {K8S_FLUX_NOTE, "|Provider|Receiver|", "spec.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_NOTE, "|Provider|", "spec.certSecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_IMAGE, "|ImagePolicy|", "spec.imageRepositoryRef.name", "ImageRepository", + "imageRepositoryRef", "USAGE"}, + {K8S_FLUX_IMAGE, "|ImageRepository|", "spec.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_IMAGE, "|ImageRepository|", "spec.certSecretRef.name", "Secret", "secretRef", + "USAGE"}, + {K8S_FLUX_IMAGE, "|ImageRepository|", "spec.serviceAccountName", "ServiceAccount", + "serviceAccountName", "USAGE"}, + {K8S_ARGO, "|Application|", "spec.project", "AppProject", "project", "USAGE"}, + {K8S_ARGO, "|ApplicationSet|", "spec.template.spec.project", "AppProject", "project", "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.canary.canaryService", "Service", "rolloutService", + "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.canary.stableService", "Service", "rolloutService", + "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.blueGreen.activeService", "Service", "rolloutService", + "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.blueGreen.previewService", "Service", "rolloutService", + "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.canary.analysis.templates[].templateName", + "AnalysisTemplate", "analysis", "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.canary.steps[].analysis.templates[].templateName", + "AnalysisTemplate", "analysis", "USAGE"}, + {K8S_ARGO, "|Workflow|", "spec.workflowTemplateRef.name", "WorkflowTemplate", + "workflowTemplateRef", "USAGE"}, + {K8S_ARGO, "|CronWorkflow|", "spec.workflowSpec.workflowTemplateRef.name", "WorkflowTemplate", + "workflowTemplateRef", "USAGE"}, + {K8S_ARGO, "|Workflow|WorkflowTemplate|", "spec.templates[].steps[][].templateRef.name", + "WorkflowTemplate", "templateRef", "USAGE"}, + {K8S_ARGO, "|Workflow|WorkflowTemplate|", "spec.templates[].dag.tasks[].templateRef.name", + "WorkflowTemplate", "templateRef", "USAGE"}, + {K8S_CAPI, "|Cluster|", "spec.topology.class", "ClusterClass", "topologyClass", "USAGE"}, + {K8S_CAPI, "|Machine|MachineDeployment|MachineSet|MachinePool|MachineHealthCheck|", + "spec.clusterName", "Cluster", "clusterName", "USAGE"}, + {K8S_CAPI, "|Machine|", "spec.bootstrap.dataSecretName", "Secret", "dataSecretName", "USAGE"}, + {K8S_CAPI, "|MachineDeployment|MachineSet|MachinePool|", + "spec.template.spec.bootstrap.dataSecretName", "Secret", "dataSecretName", "USAGE"}, + {K8S_CAPI_BOOT, "|KubeadmConfig|", "spec.files[].contentFrom.secret.name", "Secret", "files", + "USAGE"}, + {K8S_CAPI_BOOT, "|KubeadmConfigTemplate|", "spec.template.spec.files[].contentFrom.secret.name", + "Secret", "files", "USAGE"}, + {K8S_CAPI_CP, "|KubeadmControlPlane|", "spec.kubeadmConfigSpec.files[].contentFrom.secret.name", + "Secret", "files", "USAGE"}, + {K8S_CAPI_CP, "|KubeadmControlPlaneTemplate|", + "spec.template.spec.kubeadmConfigSpec.files[].contentFrom.secret.name", "Secret", "files", + "USAGE"}, + {K8S_CM, "|Certificate|", "spec.secretName", "Secret", "secretName", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.ca.secretName", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.acme.privateKeySecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.acme.externalAccountBinding.keySecretRef.name", "Secret", + "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.acme.solvers[].dns01.*.*SecretRef.name", "Secret", "secretRef", + "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.vault.auth.tokenSecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.vault.auth.*.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.vault.caBundleSecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.venafi.*.*Ref.name", "Secret", "secretRef", "USAGE"}, + {"|networking.k8s.io|extensions|gateway.networking.k8s.io|", "|Ingress|Gateway|", + "metadata.annotations.cert-manager.io/cluster-issuer", "ClusterIssuer", "issuer", "USAGE"}, + {"|networking.k8s.io|extensions|gateway.networking.k8s.io|", "|Ingress|Gateway|", + "metadata.annotations.cert-manager.io/issuer", "Issuer", "issuer", "USAGE"}, + {K8S_KKP_APPS, "|ApplicationInstallation|", "spec.applicationRef.name", "ApplicationDefinition", + "applicationRef", "USAGE"}, + {K8S_KKP_APPS, "|ApplicationDefinition|", + "spec.versions[].template.source.*.credentials.*.name", "Secret", "credentials", "USAGE"}, +}; + +static const k8s_crd_mapping_ref_t k8s_crd_mapping_refs[] = { + {K8S_FLUX_KS, "|Kustomization|", "spec.sourceRef", NULL, NULL, NULL, "sourceRef", "USAGE"}, + {K8S_FLUX_KS, "|Kustomization|", "spec.dependsOn[]", "Kustomization", NULL, NULL, "dependsOn", + "DEPENDS_ON"}, + {K8S_FLUX_KS, "|Kustomization|", "spec.postBuild.substituteFrom[]", NULL, "|ConfigMap|Secret|", + NULL, "substituteFrom", "USAGE"}, + {K8S_FLUX_KS, "|Kustomization|", "spec.healthChecks[]", NULL, NULL, NULL, "healthCheck", + "USAGE"}, + {K8S_FLUX_HELM, "|HelmRelease|", "spec.chart.spec.sourceRef", NULL, NULL, NULL, "sourceRef", + "USAGE"}, + {K8S_FLUX_HELM, "|HelmRelease|", "spec.chartRef", NULL, NULL, NULL, "chartRef", "USAGE"}, + {K8S_FLUX_HELM, "|HelmRelease|", "spec.dependsOn[]", "HelmRelease", NULL, NULL, "dependsOn", + "DEPENDS_ON"}, + {K8S_FLUX_HELM, "|HelmRelease|", "spec.valuesFrom[]", NULL, "|ConfigMap|Secret|", NULL, + "valuesFrom", "USAGE"}, + {K8S_FLUX_SRC, "|HelmChart|", "spec.sourceRef", NULL, NULL, NULL, "sourceRef", "USAGE"}, + {K8S_FLUX_SRC, "|GitRepository|", "spec.include[].repository", "GitRepository", NULL, NULL, + "include", "USAGE"}, + {K8S_FLUX_NOTE, "|Alert|", "spec.eventSources[]", NULL, NULL, NULL, "eventSource", "USAGE"}, + {K8S_FLUX_NOTE, "|Receiver|", "spec.resources[]", NULL, NULL, NULL, "resource", "USAGE"}, + {K8S_FLUX_IMAGE, "|ImageUpdateAutomation|", "spec.sourceRef", NULL, NULL, "GitRepository", + "sourceRef", "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.workloadRef", NULL, NULL, NULL, "workloadRef", "USAGE"}, + {K8S_CAPI, "|Cluster|", "spec.infrastructureRef", NULL, NULL, NULL, "infrastructureRef", + "USAGE"}, + {K8S_CAPI, "|Cluster|", "spec.controlPlaneRef", NULL, NULL, NULL, "controlPlaneRef", "USAGE"}, + {K8S_CAPI, "|Cluster|", "spec.topology.classRef", "ClusterClass", NULL, NULL, "topologyClass", + "USAGE"}, + {K8S_CAPI, "|Machine|", "spec.bootstrap.configRef", NULL, NULL, NULL, "bootstrap", "USAGE"}, + {K8S_CAPI, "|Machine|", "spec.infrastructureRef", NULL, NULL, NULL, "infrastructureRef", + "USAGE"}, + {K8S_CAPI, "|MachineDeployment|MachineSet|MachinePool|", + "spec.template.spec.bootstrap.configRef", NULL, NULL, NULL, "bootstrap", "USAGE"}, + {K8S_CAPI, "|MachineDeployment|MachineSet|MachinePool|", "spec.template.spec.infrastructureRef", + NULL, NULL, NULL, "infrastructureRef", "USAGE"}, + {K8S_CAPI_CP, "|KubeadmControlPlane|", "spec.machineTemplate.infrastructureRef", NULL, NULL, + NULL, "infrastructureRef", "USAGE"}, + {K8S_CAPI_CP, "|KubeadmControlPlane|", "spec.machineTemplate.spec.infrastructureRef", NULL, + NULL, NULL, "infrastructureRef", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.infrastructure.ref", NULL, NULL, NULL, "template", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.infrastructure.templateRef", NULL, NULL, NULL, "template", + "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.controlPlane.ref", NULL, NULL, NULL, "template", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.controlPlane.templateRef", NULL, NULL, NULL, "template", + "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.controlPlane.machineInfrastructure.ref", NULL, NULL, NULL, + "template", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.controlPlane.machineInfrastructure.templateRef", NULL, NULL, + NULL, "template", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.workers.*[].template.*.ref", NULL, NULL, NULL, "template", + "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.workers.*[].*.templateRef", NULL, NULL, NULL, "template", + "USAGE"}, + {K8S_CAPI_ADDONS, "|ClusterResourceSet|", "spec.resources[]", NULL, "|ConfigMap|Secret|", NULL, + "resources", "USAGE"}, + {K8S_CM, "|Certificate|CertificateRequest|", "spec.issuerRef", NULL, NULL, "Issuer", + "issuerRef", "USAGE"}, + {K8S_KKP, "|KubermaticConfiguration|", "spec.ingress.certificateIssuer", NULL, NULL, NULL, + "certificateIssuer", "USAGE"}, + {K8S_KKP, "|Seed|", "spec.kubeconfig", "Secret", NULL, NULL, "kubeconfig", "USAGE"}, + {K8S_KKP, "|Seed|", "spec.etcdBackupRestore.destinations.*.credentials", "Secret", NULL, NULL, + "credentials", "USAGE"}, + {K8S_KKP, "|Cluster|", "spec.cloud.*.credentialsReference", "Secret", NULL, NULL, "credentials", + "USAGE"}, + {K8S_KKP, "|Addon|EtcdBackupConfig|EtcdRestore|", "spec.cluster", "Cluster", NULL, NULL, + "cluster", "USAGE"}, +}; + +// Does `path` match `pat`, where `*` matches any run within one segment? +static bool k8s_path_match(const char *pat, const char *path) { + const char *star = NULL; + const char *mark = NULL; + while (*path) { + if (*pat == '*') { + star = pat++; + mark = path; + } else if (*pat && *pat == *path) { + pat++; + path++; + } else if (star && *mark != '.') { + pat = star + SKIP_ONE; + path = ++mark; + } else { + return false; + } + } + while (*pat == '*') { + pat++; + } + return *pat == '\0'; +} + // Is `kind` one of the "|A|B|" list? static bool k8s_kind_in(const char *list, const char *kind) { if (!list || !kind[0]) { @@ -429,8 +643,9 @@ static const char *k8s_pod_prefix(const char *kind) { if (strcmp(kind, "CronJob") == 0) { return "spec.jobTemplate.spec.template.spec"; } - if (k8s_kind_in("|Deployment|StatefulSet|DaemonSet|ReplicaSet|ReplicationController|Job|", - kind)) { + if (k8s_kind_in( + "|Deployment|StatefulSet|DaemonSet|ReplicaSet|ReplicationController|Job|Rollout|", + kind)) { return "spec.template.spec"; } return ""; @@ -438,12 +653,42 @@ static const char *k8s_pod_prefix(const char *kind) { typedef struct { CBMExtractCtx *ctx; - const char *kind; // the document's kind + const char *group; // the document's API group + const char *kind; // the document's kind int def_index; char crd_group[K8S_BUF_SIZE]; char crd_kind[K8S_BUF_SIZE]; + const char *deploy_ns; // a deploying object's target namespace } k8s_walk_t; +// Is the document of a source kind in group list `groups`? +static bool k8s_source_is(const k8s_walk_t *w, const char *groups, const char *kinds) { + return k8s_kind_in(groups, w->group) && k8s_kind_in(kinds, w->kind); +} + +// A Flux Kustomization / Argo Application deploys a repository directory. +static void k8s_deploy_leaf(k8s_walk_t *w, const char *path, const char *value) { + bool flux = k8s_source_is(w, K8S_FLUX_KS, "|Kustomization|"); + bool argo = k8s_source_is(w, K8S_ARGO, "|Application|"); + if ((flux && strcmp(path, "spec.targetNamespace") == 0) || + (argo && strcmp(path, "spec.destination.namespace") == 0)) { + w->deploy_ns = strstr(value, "{{") ? NULL : value; + return; + } + if ((flux && strcmp(path, "spec.path") == 0) || + (argo && + (strcmp(path, "spec.source.path") == 0 || strcmp(path, "spec.sources[].path") == 0))) { + if (strstr(value, "{{")) { + return; + } + CBMK8sFact f = {.fact = CBM_K8S_FACT_DEPLOY, + .def_index = w->def_index, + .name = cbm_arena_strdup(w->ctx->arena, value), + .via = flux ? "flux" : "argo"}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); + } +} + static void k8s_emit_ref(k8s_walk_t *w, const char *kind, const char *ns, const char *name, const char *via, const char *edge) { if (!kind || !kind[0] || !name || !name[0] || strstr(name, "{{") || strstr(kind, "{{")) { @@ -493,6 +738,13 @@ static void k8s_leaf(k8s_walk_t *w, const char *path, const char *value) { k8s_emit_ref(w, r->kind, NULL, value, r->via, r->edge); } } + for (size_t i = 0; i < sizeof(k8s_crd_leaf_refs) / sizeof(k8s_crd_leaf_refs[0]); i++) { + const k8s_crd_leaf_ref_t *r = &k8s_crd_leaf_refs[i]; + if (k8s_source_is(w, r->groups, r->kinds) && k8s_path_match(r->path, path)) { + k8s_emit_ref(w, r->kind, NULL, value, r->via, r->edge); + } + } + k8s_deploy_leaf(w, path, value); if (strcmp(w->kind, "CustomResourceDefinition") == 0) { if (strcmp(path, "spec.group") == 0) { snprintf(w->crd_group, sizeof(w->crd_group), "%s", value); @@ -546,23 +798,35 @@ static const char *k8s_field(k8s_walk_t *w, TSNode mapping, const char *field) { return NULL; } +// The reference a mapping holds in its kind/name/namespace: to `kind`, or +// (NULL) to its own `kind` field, one of `allowed` (NULL: any), `dflt` +// when absent. +static void k8s_mapping_ref(k8s_walk_t *w, TSNode mapping, const char *kind, const char *allowed, + const char *dflt, const char *via, const char *edge) { + if (!kind) { + kind = k8s_field(w, mapping, "kind"); + kind = kind ? kind : dflt; + if (!kind || (allowed && !k8s_kind_in(allowed, kind))) { + return; + } + } + k8s_emit_ref(w, kind, k8s_field(w, mapping, "namespace"), k8s_field(w, mapping, "name"), via, + edge); +} + // A mapping at `path` that holds a reference in its kind/name/namespace. static void k8s_mapping(k8s_walk_t *w, const char *path, TSNode mapping) { for (size_t i = 0; i < sizeof(k8s_mapping_refs) / sizeof(k8s_mapping_refs[0]); i++) { const k8s_mapping_ref_t *r = &k8s_mapping_refs[i]; - if (!k8s_kind_in(r->kinds, w->kind) || strcmp(path, r->path) != 0) { - continue; + if (k8s_kind_in(r->kinds, w->kind) && strcmp(path, r->path) == 0) { + k8s_mapping_ref(w, mapping, r->kind, r->allowed, r->dflt, r->via, r->edge); } - const char *kind = r->kind; - if (!kind) { - kind = k8s_field(w, mapping, "kind"); - kind = kind ? kind : r->dflt; - if (!kind || (r->allowed && !k8s_kind_in(r->allowed, kind))) { - continue; - } + } + for (size_t i = 0; i < sizeof(k8s_crd_mapping_refs) / sizeof(k8s_crd_mapping_refs[0]); i++) { + const k8s_crd_mapping_ref_t *r = &k8s_crd_mapping_refs[i]; + if (k8s_source_is(w, r->groups, r->kinds) && k8s_path_match(r->path, path)) { + k8s_mapping_ref(w, mapping, r->kind, r->allowed, r->dflt, r->via, r->edge); } - k8s_emit_ref(w, kind, k8s_field(w, mapping, "namespace"), k8s_field(w, mapping, "name"), - r->via, r->edge); } } @@ -623,11 +887,20 @@ static void k8s_walk(k8s_walk_t *w, TSNode node, char *path, size_t len, size_t } } -// The references and CRD declaration of document def `def_index`. -static void k8s_emit_facts(CBMExtractCtx *ctx, TSNode mapping, const char *kind, int def_index) { - k8s_walk_t w = {.ctx = ctx, .kind = kind, .def_index = def_index}; +// The references, deploys and CRD declaration of document def `def_index`. +static void k8s_emit_facts(CBMExtractCtx *ctx, TSNode mapping, const char *group, const char *kind, + int def_index) { + k8s_walk_t w = {.ctx = ctx, .group = group, .kind = kind, .def_index = def_index}; char path[CBM_SZ_512] = ""; + int first = ctx->result->k8s_facts.count; k8s_walk(&w, mapping, path, 0, sizeof(path)); + /* the target namespace may follow the path in the document */ + for (int i = first; w.deploy_ns && i < ctx->result->k8s_facts.count; i++) { + CBMK8sFact *f = &ctx->result->k8s_facts.items[i]; + if (f->fact == CBM_K8S_FACT_DEPLOY) { + f->ns = cbm_arena_strdup(ctx->arena, w.deploy_ns); + } + } if (w.crd_group[0] && w.crd_kind[0]) { CBMK8sFact f = {.fact = CBM_K8S_FACT_CRD, .def_index = def_index, @@ -726,7 +999,7 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { def.start_line = ts_node_start_point(mapping).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(mapping).row + TS_LINE_OFFSET; cbm_defs_push(&ctx->result->defs, a, def); - k8s_emit_facts(ctx, mapping, id.kind, ctx->result->defs.count - SKIP_ONE); + k8s_emit_facts(ctx, mapping, group, id.kind, ctx->result->defs.count - SKIP_ONE); } } diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index bbc465be9..674eabdd4 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -676,6 +676,7 @@ typedef struct { CBMHashTable *names; /* bundle name -> itself (owned key) */ CBMHashTable *members; /* manifest path -> k8s_members_t* */ CBMHashTable *charts; /* Helm chart directory -> itself (owned key) */ + CBMHashTable *deploys; /* manifest path -> k8s_members_t* of its deploy scopes */ } k8s_bundles_t; static bool k8s_grow(void **items, int *cap, int count, size_t elem) { @@ -818,14 +819,15 @@ static const char *k8s_bundle_name(k8s_bundles_t *b, const char *kind, const cha return owned; } -static void k8s_member_add(k8s_bundles_t *b, const char *file, const char *bundle, const char *ns) { - k8s_members_t *m = cbm_ht_get(b->members, file); +static void k8s_member_add(CBMHashTable *table, const char *file, const char *bundle, + const char *ns) { + k8s_members_t *m = cbm_ht_get(table, file); if (!m) { m = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*m)); if (!m) { return; } - cbm_ht_set(b->members, file, m); + cbm_ht_set(table, file, m); } for (int i = 0; i < m->count; i++) { if (m->items[i].bundle == bundle) { @@ -867,7 +869,7 @@ static void k8s_bundles_from_kusts(k8s_bundles_t *b, k8s_kusts_t *ks) { seen[f.kust] = r + SKIP_ONE; const k8s_kust_t *k = &ks->items[f.kust]; for (int i = 0; i < k->file_n; i++) { - k8s_member_add(b, k->files[i], bundle, f.ns); + k8s_member_add(b->members, k->files[i], bundle, f.ns); } for (int i = k->kid_n - SKIP_ONE; i >= 0; i--) { if (k8s_grow((void **)&stack, &cap, depth, sizeof(*stack))) { @@ -938,6 +940,10 @@ static void k8s_bundles_free(k8s_bundles_t *b) { cbm_ht_foreach(b->members, k8s_free_members, NULL); cbm_ht_free(b->members); } + if (b->deploys) { + cbm_ht_foreach(b->deploys, k8s_free_members, NULL); + cbm_ht_free(b->deploys); + } if (b->names) { cbm_ht_foreach(b->names, k8s_free_owned_key, NULL); cbm_ht_free(b->names); @@ -1078,7 +1084,7 @@ typedef struct { char *ns; /* explicit target namespace, NULL = the source's */ char *name; /* target metadata.name */ const char *via; /* static: the extractor's table */ - const char *edge; /* static: "USAGE" or "INFRA_MAPS" */ + const char *edge; /* static: "USAGE", "INFRA_MAPS" or "DEPENDS_ON" */ } k8s_ref_t; /* A CRD and the group/kind it declares. */ @@ -1088,6 +1094,14 @@ typedef struct { char *kind; } k8s_crd_t; +/* An object that deploys a repository directory (Flux Kustomization, Argo + * Application). */ +typedef struct { + int64_t node; + char *dir; /* normalised, "" = the repository root */ + char *ns; /* the namespace it deploys into, NULL = the objects' own */ +} k8s_deploy_t; + typedef struct { k8s_ref_t *refs; int ref_n; @@ -1095,6 +1109,9 @@ typedef struct { k8s_crd_t *crds; int crd_n; int crd_cap; + k8s_deploy_t *deploys; + int dep_n; + int dep_cap; } k8s_facts_t; static void k8s_facts_free(k8s_facts_t *f) { @@ -1107,11 +1124,60 @@ static void k8s_facts_free(k8s_facts_t *f) { cbm_free(CBM_MEM_CLASS_OTHER, f->crds[i].group); cbm_free(CBM_MEM_CLASS_OTHER, f->crds[i].kind); } + for (int i = 0; i < f->dep_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->deploys[i].dir); + cbm_free(CBM_MEM_CLASS_OTHER, f->deploys[i].ns); + } cbm_free(CBM_MEM_CLASS_OTHER, f->refs); cbm_free(CBM_MEM_CLASS_OTHER, f->crds); + cbm_free(CBM_MEM_CLASS_OTHER, f->deploys); *f = (k8s_facts_t){0}; } +/* A deploy path as a repository directory: "./a//b/" -> "a/b", "." -> ""; + * false for URLs, templates, variables and paths leaving the repository. */ +static bool k8s_norm_dir(const char *in, char *out, size_t out_sz) { + if (!in || strstr(in, "://") || strstr(in, "{{") || strchr(in, '$') || out_sz == 0) { + return false; + } + size_t n = 0; + out[0] = '\0'; + for (const char *p = in; *p;) { + while (*p == '/') { + p++; + } + size_t sl = strcspn(p, "/"); + if (sl == 0) { + break; + } + if (sl == SKIP_ONE && p[0] == '.') { + p += sl; + continue; + } + if (sl == PAIR_LEN && p[0] == '.' && p[SKIP_ONE] == '.') { + if (n == 0) { + return false; + } + char *slash = strrchr(out, '/'); + n = slash ? (size_t)(slash - out) : 0; + out[n] = '\0'; + p += sl; + continue; + } + if (n + SKIP_ONE + sl >= out_sz) { + return false; + } + if (n) { + out[n++] = '/'; + } + memcpy(out + n, p, sl); + n += sl; + out[n] = '\0'; + p += sl; + } + return true; +} + /* Keep a file's facts, its def indices mapped to the Resource nodes. */ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64_t *node_of) { for (int i = 0; i < res->k8s_facts.count; i++) { @@ -1120,7 +1186,15 @@ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64 continue; } int64_t node = node_of[x->def_index]; - if (x->fact == CBM_K8S_FACT_CRD) { + if (x->fact == CBM_K8S_FACT_DEPLOY) { + char dir[CBM_SZ_1K]; + if (k8s_norm_dir(x->name, dir, sizeof(dir)) && + k8s_grow((void **)&f->deploys, &f->dep_cap, f->dep_n, sizeof(*f->deploys))) { + f->deploys[f->dep_n++] = + (k8s_deploy_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir), + x->ns ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->ns) : NULL}; + } + } else if (x->fact == CBM_K8S_FACT_CRD) { if (k8s_grow((void **)&f->crds, &f->crd_cap, f->crd_n, sizeof(*f->crds))) { f->crds[f->crd_n++] = (k8s_crd_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), @@ -1138,6 +1212,300 @@ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64 } } +/* ── Deploy scopes (PHASE1 P1.3) ───────────────────────────────────── */ + +/* A Flux Kustomization or Argo Application deploys a repository directory: + * the build of a kustomization there, else every manifest below it, a + * sub-directory's kustomization by its build (as Flux generates one). The + * objects a root deploy reaches through such directories -- and the deploys + * among them, transitively -- form one deploy scope, a cluster: references + * meet across kustomize roots inside it (a Kustomization's dependsOn, a + * Certificate's ClusterIssuer). A path that names no directory of the + * repository (another source's layout) deploys nothing. */ + +typedef struct { + const char **items; /* graph-owned paths */ + int count; + int cap; +} k8s_paths_t; + +static void k8s_paths_push(k8s_paths_t *v, const char *p) { + if (k8s_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = p; + } +} + +static void k8s_paths_sort_unique(k8s_paths_t *v) { + if (v->count == 0) { + return; + } + qsort((void *)v->items, (size_t)v->count, sizeof(*v->items), k8s_cmp_str); + int w = SKIP_ONE; + for (int i = SKIP_ONE; i < v->count; i++) { + if (strcmp(v->items[w - SKIP_ONE], v->items[i]) != 0) { + v->items[w++] = v->items[i]; + } + } + v->count = w; +} + +/* The manifests the build of kustomization `root` reaches; `seen` marks the + * kustomizations visited under `mark`. */ +static void k8s_kust_closure(const k8s_kusts_t *ks, int root, int *seen, int mark, + k8s_paths_t *out) { + int *stack = NULL; + int depth = 0; + int cap = 0; + if (!k8s_grow((void **)&stack, &cap, depth, sizeof(int))) { + return; + } + stack[depth++] = root; + while (depth > 0) { + int k = stack[--depth]; + if (seen[k] == mark) { + continue; + } + seen[k] = mark; + for (int i = 0; i < ks->items[k].file_n; i++) { + k8s_paths_push(out, ks->items[k].files[i]); + } + for (int i = 0; i < ks->items[k].kid_n; i++) { + if (k8s_grow((void **)&stack, &cap, depth, sizeof(int))) { + stack[depth++] = ks->items[k].kids[i]; + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, stack); +} + +/* The manifests a deploy of directory `dir` applies; `manifests` is every + * manifest path, sorted (a directory's are one range). */ +static void k8s_dir_members(const k8s_kusts_t *ks, CBMHashTable *by_path, + const k8s_paths_t *manifests, const char *dir, int *seen, int mark, + k8s_paths_t *out) { + size_t dl = strlen(dir); + int lo = 0; + int hi = dl ? manifests->count : 0; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + int c = strncmp(manifests->items[mid], dir, dl); + if (c < 0 || (c == 0 && manifests->items[mid][dl] < '/')) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (int i = lo; i < manifests->count; i++) { + const char *file = manifests->items[i]; + if (dl && (strncmp(file, dir, dl) != 0 || file[dl] != '/')) { + break; + } + /* the outermost kustomization between `dir` and the file applies it */ + char up[CBM_SZ_1K]; + k8s_dirname(file, up, sizeof(up)); + int top = 0; + for (;;) { + int k = k8s_kust_in_dir(by_path, up); + top = k ? k : top; + if (strlen(up) <= dl) { + break; + } + char parent[CBM_SZ_1K]; + k8s_dirname(up, parent, sizeof(parent)); + snprintf(up, sizeof(up), "%s", parent); + } + if (!top) { + k8s_paths_push(out, file); + } else if (seen[top - SKIP_ONE] != mark) { + k8s_kust_closure(ks, top - SKIP_ONE, seen, mark, out); + } + } + k8s_paths_sort_unique(out); +} + +/* A deploy's sort key: where it is declared, then which one. */ +typedef struct { + const char *file; + const char *qn; + const char *dir; + int deploy; +} k8s_deploy_key_t; + +static int k8s_cmp_deploy_key(const void *a, const void *b) { + const k8s_deploy_key_t *x = a; + const k8s_deploy_key_t *y = b; + int c = strcmp(x->file, y->file); + c = c ? c : strcmp(x->qn, y->qn); + c = c ? c : strcmp(x->dir, y->dir); + return c ? c : (x->deploy > y->deploy) - (x->deploy < y->deploy); +} + +/* The deploys declared in `file`: a range of `keys` (sorted by file). */ +static int k8s_deploys_in(const k8s_deploy_key_t *keys, int n, const char *file, int *first) { + int lo = 0; + int hi = n; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (strcmp(keys[mid].file, file) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + int end = lo; + while (end < n && strcmp(keys[end].file, file) == 0) { + end++; + } + *first = lo; + return end - lo; +} + +typedef struct { + int *items; + int count; + int cap; +} k8s_ints_t; + +/* Build the deploy scopes (after the kustomize bundles). */ +static void k8s_deploys_build(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b, + const k8s_facts_t *f) { + int n = f->dep_n; + const cbm_gbuf_node_t **res = NULL; + int rn = 0; + if (n == 0 || !b->names || cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &res, &rn) != 0) { + return; + } + b->deploys = cbm_ht_create(0); + k8s_paths_t manifests = {0}; + for (int i = 0; i < rn; i++) { + if (res[i]->file_path) { + k8s_paths_push(&manifests, res[i]->file_path); + } + } + k8s_paths_sort_unique(&manifests); + CBMHashTable *by_path = cbm_ht_create(0); + for (int i = 0; by_path && i < ks->count; i++) { + cbm_ht_set(by_path, ks->items[i].path, (void *)(intptr_t)(i + SKIP_ONE)); + } + size_t un = (size_t)n; + k8s_paths_t *members = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(*members)); + k8s_ints_t *adj = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(*adj)); + const cbm_gbuf_node_t **node = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(*node)); + k8s_deploy_key_t *keys = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(*keys)); + int *indeg = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(int)); + int *reached = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(int)); + int *visit = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(int)); + int *queue = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(int)); + int *seen = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(ks->count ? ks->count : SKIP_ONE) * sizeof(int)); + int roots = 0; + if (b->deploys && by_path && members && adj && node && keys && indeg && reached && visit && + queue && seen) { + for (int d = 0; d < n; d++) { + node[d] = cbm_gbuf_find_by_id(ctx->gbuf, f->deploys[d].node); + keys[d] = (k8s_deploy_key_t){ + node[d] && node[d]->file_path ? node[d]->file_path : "", + node[d] && node[d]->qualified_name ? node[d]->qualified_name : "", + f->deploys[d].dir, d}; + k8s_dir_members(ks, by_path, &manifests, f->deploys[d].dir, seen, d + SKIP_ONE, + &members[d]); + } + qsort(keys, un, sizeof(*keys), k8s_cmp_deploy_key); + /* deploy d reaches deploy e when e sits in a manifest d applies */ + for (int d = 0; d < n; d++) { + for (int i = 0; i < members[d].count; i++) { + int first = 0; + int cnt = k8s_deploys_in(keys, n, members[d].items[i], &first); + for (int j = first; j < first + cnt; j++) { + int e = keys[j].deploy; + if (e != d && + k8s_grow((void **)&adj[d].items, &adj[d].cap, adj[d].count, sizeof(int))) { + adj[d].items[adj[d].count++] = e; + indeg[e]++; + } + } + } + } + /* roots: deploys no other reaches; then, in order, any a cycle hides */ + for (int pass = 0; pass < PAIR_LEN; pass++) { + for (int oi = 0; oi < n; oi++) { + int r = keys[oi].deploy; + if (reached[r] || (pass == 0 && indeg[r] != 0)) { + continue; + } + const char *bundle = k8s_bundle_name( + b, "deploy", node[r] ? node[r]->qualified_name : f->deploys[r].dir); + if (!bundle) { + continue; + } + roots++; + int mark = pass * n + oi + SKIP_ONE; + int qh = 0; + int qt = 0; + queue[qt++] = r; + visit[r] = mark; + while (qh < qt) { + int d = queue[qh++]; + reached[d] = SKIP_ONE; + if (node[d] && node[d]->file_path) { + k8s_member_add(b->deploys, node[d]->file_path, bundle, NULL); + } + for (int i = 0; i < members[d].count; i++) { + k8s_member_add(b->deploys, members[d].items[i], bundle, f->deploys[d].ns); + } + for (int i = 0; i < adj[d].count; i++) { + int e = adj[d].items[i]; + if (visit[e] != mark) { + visit[e] = mark; + queue[qt++] = e; + } + } + } + } + } + } + for (int d = 0; members && d < n; d++) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)members[d].items); + } + for (int d = 0; adj && d < n; d++) { + cbm_free(CBM_MEM_CLASS_OTHER, adj[d].items); + } + cbm_free(CBM_MEM_CLASS_OTHER, members); + cbm_free(CBM_MEM_CLASS_OTHER, adj); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)node); + cbm_free(CBM_MEM_CLASS_OTHER, keys); + cbm_free(CBM_MEM_CLASS_OTHER, indeg); + cbm_free(CBM_MEM_CLASS_OTHER, reached); + cbm_free(CBM_MEM_CLASS_OTHER, visit); + cbm_free(CBM_MEM_CLASS_OTHER, queue); + cbm_free(CBM_MEM_CLASS_OTHER, seen); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)manifests.items); + if (by_path) { + cbm_ht_free(by_path); + } + cbm_log_info("pass.k8s.deploys", "deploys", itoa_k8s(n), "roots", itoa_k8s(roots)); +} + +/* The namespace of a Resource's logical key ("" when unknown `?` or + * cluster-scoped `_`), else its declared one. */ +static void k8s_effective_ns(const cbm_gbuf_node_t *n, char *out, size_t out_sz) { + out[0] = '\0'; + const char *k = n->properties_json ? strstr(n->properties_json, "\"obj_key\":\"") : NULL; + if (!k) { + k8s_declared_ns(n->qualified_name, out, out_sz); + return; + } + k += K8S_SLEN("\"obj_key\":\""); + for (int part = 0; k && part < PAIR_LEN; part++) { /* past group and kind */ + k = strchr(k, '/'); + k = k ? k + SKIP_ONE : NULL; + } + const char *end = k ? strchr(k, '/') : NULL; + if (end && !(end - k == SKIP_ONE && (k[0] == '?' || k[0] == '_'))) { + snprintf(out, out_sz, "%.*s", (int)(end - k), k); + } +} + /* Does the target at `tgt` answer `ref` from `src`: one bundle, and the * namespace the reference asks for (explicit, else the source's own there) * agrees with the target's there? A cluster-scoped kind has no namespace. */ @@ -1166,6 +1534,26 @@ static bool k8s_ref_meets(k8s_bundles_t *b, const k8s_ref_t *ref, const cbm_gbuf } } } + /* or one deploy scope: the cluster a Flux / Argo root deploys */ + const k8s_members_t *ds = b->deploys ? cbm_ht_get(b->deploys, src->file_path) : NULL; + const k8s_members_t *dt = ds ? cbm_ht_get(b->deploys, tgt->file_path) : NULL; + if (!dt) { + return false; + } + k8s_effective_ns(src, src_ns, sizeof(src_ns)); + k8s_effective_ns(tgt, tgt_ns, sizeof(tgt_ns)); + for (int i = 0; i < ds->count; i++) { + for (int j = 0; j < dt->count; j++) { + if (ds->items[i].bundle != dt->items[j].bundle) { + continue; + } + const char *want = ref->ns ? ref->ns : ds->items[i].ns ? ds->items[i].ns : src_ns; + const char *have = dt->items[j].ns ? dt->items[j].ns : tgt_ns; + if (cluster || !want[0] || !have[0] || strcmp(want, have) == 0) { + return true; + } + } + } return false; } @@ -1186,7 +1574,10 @@ static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8 } char props[CBM_SZ_256]; snprintf(props, sizeof(props), "{\"%s\":\"%s\"}", - strcmp(ref->edge, "INFRA_MAPS") == 0 ? "kind" : "ref", ref->via); + strcmp(ref->edge, "INFRA_MAPS") == 0 ? "kind" + : strcmp(ref->edge, "DEPENDS_ON") == 0 ? "via" + : "ref", + ref->via); /* A name is one object per kind and namespace, so two declarations * of it in one bundle are alternatives (versioned manifests side by * side): the declaration in the reference's own file, when there is @@ -1954,6 +2345,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, /* Connect Services to the workloads their selectors target (INFRA_MAPS), * inside the bundles that deploy them together. */ k8s_bundles_build(ctx, &kusts, &bundles); + k8s_deploys_build(ctx, &kusts, &bundles, &facts); k8s_apply_effective_namespaces(ctx, &bundles); k8s_link_selectors(ctx, &recs, &bundles); k8s_resolve_refs(ctx, &bundles, &facts); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 99b5b6ef5..812a6f16c 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1275,6 +1275,213 @@ TEST(probe_go_controller_methods_sharing_a_node) { PASS(); } +/* ── Curated CRD references + deploy scopes (P1.3) ──────────────────── */ + +/* A Flux bootstrap: the flux-system Kustomization deploys ./clusters/prod, + * whose Kustomizations deploy ./infra and ./apps/prod. Each is its own + * kustomize root, so the references between them -- a Kustomization's + * sourceRef to the flux-system GitRepository, a HelmRelease's dependsOn in + * another namespace, a Certificate's ClusterIssuer -- meet only in the + * deploy scope the root reaches. */ +TEST(probe_flux_refs_meet_in_the_deploy_scope) { + static const GpfFile files[] = { + {"clusters/prod/flux-system/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - gotk-sync.yaml\n"}, + {"clusters/prod/flux-system/gotk-sync.yaml", + "apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\n" + "metadata:\n name: flux-system\n namespace: flux-system\n" + "spec:\n url: ssh://git@example.com/fleet\n---\n" + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: flux-system\n namespace: flux-system\n" + "spec:\n path: ./clusters/prod\n sourceRef:\n kind: GitRepository\n" + " name: flux-system\n"}, + {"clusters/prod/apps.yaml", + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: infra\n namespace: flux-system\n" + "spec:\n path: ./infra\n sourceRef:\n kind: GitRepository\n" + " name: flux-system\n---\n" + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: apps\n namespace: flux-system\n" + "spec:\n dependsOn:\n - name: infra\n path: ./apps/prod\n" + " sourceRef:\n kind: GitRepository\n name: flux-system\n"}, + {"infra/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - cert-manager.yaml\n"}, + {"infra/cert-manager.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\n" + "metadata:\n name: cert-manager\n namespace: cert-manager\n" + "spec:\n interval: 1h\n---\n" + "apiVersion: cert-manager.io/v1\nkind: ClusterIssuer\n" + "metadata:\n name: letsencrypt\nspec:\n selfSigned: {}\n"}, + {"apps/prod/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "namespace: podinfo\nresources:\n - ../base/podinfo\n"}, + {"apps/base/podinfo/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - release.yaml\n - repo.yaml\n"}, + {"apps/base/podinfo/release.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\n" + "metadata:\n name: podinfo\nspec:\n chart:\n spec:\n chart: podinfo\n" + " sourceRef:\n kind: HelmRepository\n name: podinfo\n" + " dependsOn:\n - name: cert-manager\n namespace: cert-manager\n" + " valuesFrom:\n - kind: ConfigMap\n name: podinfo-values\n---\n" + "apiVersion: cert-manager.io/v1\nkind: Certificate\n" + "metadata:\n name: podinfo-tls\nspec:\n secretName: podinfo-tls\n" + " issuerRef:\n kind: ClusterIssuer\n name: letsencrypt\n"}, + {"apps/base/podinfo/repo.yaml", + "apiVersion: source.toolkit.fluxcd.io/v1\nkind: HelmRepository\n" + "metadata:\n name: podinfo\nspec:\n url: https://example.com/charts\n---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: podinfo-values\n" + "data:\n replicaCount: \"2\"\n"}}; + /* flux-system -> its GitRepository, infra and apps -> it, podinfo -> HelmRepository */ + ASSERT_EQ(gpf_typed_edges(files, 9, "USAGE", "Resource", "Resource", "\"ref\":\"sourceRef\""), + 4); + /* apps -> infra; podinfo -> cert-manager in another namespace and root */ + ASSERT_EQ(gpf_typed_edges(files, 9, "DEPENDS_ON", "Resource", "Resource", + "\"via\":\"dependsOn\""), + 2); + ASSERT_EQ(gpf_typed_edges(files, 9, "USAGE", "Resource", "Resource", "\"ref\":\"issuerRef\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 9, "USAGE", "Resource", "Resource", + "\"ref\":\"valuesFrom\""), + 1); + PASS(); +} + +/* Cluster API: a Cluster names its infrastructure, control plane and + * class; machines name their cluster, bootstrap config and templates. The + * source kind counts in its API group: a CloudNativePG `Cluster` with the + * same field is not a Cluster API one. */ +TEST(probe_capi_refs_follow_the_api_group) { + static const char tmpl[] = + "apiVersion: cluster.x-k8s.io/v1beta2\nkind: Cluster\nmetadata:\n name: c1\n" + "spec:\n infrastructureRef:\n kind: DockerCluster\n name: c1\n" + " controlPlaneRef:\n kind: KubeadmControlPlane\n name: c1-cp\n" + " topology:\n classRef:\n name: quick\n---\n" + "apiVersion: infrastructure.cluster.x-k8s.io/v1beta2\nkind: DockerCluster\n" + "metadata:\n name: c1\n---\n" + "apiVersion: controlplane.cluster.x-k8s.io/v1beta2\nkind: KubeadmControlPlane\n" + "metadata:\n name: c1-cp\nspec:\n machineTemplate:\n spec:\n" + " infrastructureRef:\n kind: DockerMachineTemplate\n name: cp-infra\n---\n" + "apiVersion: infrastructure.cluster.x-k8s.io/v1beta2\nkind: DockerMachineTemplate\n" + "metadata:\n name: cp-infra\n---\n" + "apiVersion: cluster.x-k8s.io/v1beta2\nkind: MachineDeployment\nmetadata:\n name: md\n" + "spec:\n clusterName: c1\n template:\n spec:\n bootstrap:\n" + " configRef:\n kind: KubeadmConfigTemplate\n name: md-boot\n" + " infrastructureRef:\n kind: DockerMachineTemplate\n name: md-infra\n---\n" + "apiVersion: bootstrap.cluster.x-k8s.io/v1beta2\nkind: KubeadmConfigTemplate\n" + "metadata:\n name: md-boot\n---\n" + "apiVersion: infrastructure.cluster.x-k8s.io/v1beta2\nkind: DockerMachineTemplate\n" + "metadata:\n name: md-infra\n---\n" + "apiVersion: cluster.x-k8s.io/v1beta2\nkind: ClusterClass\nmetadata:\n name: quick\n" + "spec:\n infrastructure:\n templateRef:\n kind: DockerClusterTemplate\n" + " name: quick-cluster\n workers:\n machineDeployments:\n - class: w\n" + " bootstrap:\n templateRef:\n kind: KubeadmConfigTemplate\n" + " name: md-boot\n---\n" + "apiVersion: infrastructure.cluster.x-k8s.io/v1beta2\nkind: DockerClusterTemplate\n" + "metadata:\n name: quick-cluster\n---\n" + "apiVersion: postgresql.cnpg.io/v1\nkind: Cluster\nmetadata:\n name: pg\n" + "spec:\n infrastructureRef:\n kind: DockerCluster\n name: c1\n"; + static const GpfFile files[] = {{"templates/cluster.yaml", tmpl}}; + /* Cluster, KubeadmControlPlane and MachineDeployment -- not the cnpg Cluster */ + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"infrastructureRef\""), + 3); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"controlPlaneRef\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"topologyClass\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"clusterName\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"bootstrap\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"template\""), + 2); + PASS(); +} + +/* cert-manager and Kubermatic: a Certificate's issuer (Issuer when the + * kind is omitted) and Secret, an Ingress's cluster-issuer annotation, an + * issuer's CA and DNS-solver Secrets (`*SecretRef` of any provider), a KKP + * Seed's kubeconfig and a user cluster's provider credentials. */ +TEST(probe_cert_manager_and_kkp_refs) { + static const char doc[] = + "apiVersion: cert-manager.io/v1\nkind: Issuer\nmetadata:\n name: ca\n" + "spec:\n ca:\n secretName: ca-key\n---\n" + "apiVersion: cert-manager.io/v1\nkind: ClusterIssuer\nmetadata:\n name: letsencrypt\n" + "spec:\n acme:\n solvers:\n - dns01:\n cloudflare:\n" + " apiTokenSecretRef:\n name: cf-token\n key: token\n" + "---\n" + "apiVersion: cert-manager.io/v1\nkind: Certificate\nmetadata:\n name: web\n" + "spec:\n secretName: web-tls\n issuerRef:\n name: ca\n---\n" + "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: web\n" + " annotations:\n cert-manager.io/cluster-issuer: letsencrypt\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: ca-key\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: cf-token\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: web-tls\n---\n" + "apiVersion: kubermatic.k8c.io/v1\nkind: Seed\nmetadata:\n name: europe\n" + " namespace: kubermatic\nspec:\n kubeconfig:\n name: seed-kubeconfig\n" + " namespace: kubermatic\n---\n" + "apiVersion: kubermatic.k8c.io/v1\nkind: Cluster\nmetadata:\n name: user1\n" + "spec:\n cloud:\n aws:\n credentialsReference:\n name: aws-creds\n" + " namespace: kubermatic\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: seed-kubeconfig\n" + " namespace: kubermatic\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: aws-creds\n namespace: kubermatic\n"; + static const GpfFile files[] = {{"pki/certs.yaml", doc}}; + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"issuerRef\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"secretName\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"issuer\""), 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"secretRef\""), + 2); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"kubeconfig\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"credentials\""), + 1); + PASS(); +} + +/* Argo: an Application's project, and a Rollout -- a workload with a pod + * template -- names its Services, analysis template and service account. */ +TEST(probe_argo_application_and_rollout_refs) { + static const char doc[] = + "apiVersion: argoproj.io/v1alpha1\nkind: AppProject\nmetadata:\n name: team-a\n" + " namespace: argocd\n---\n" + "apiVersion: argoproj.io/v1alpha1\nkind: Application\nmetadata:\n name: web\n" + " namespace: argocd\nspec:\n project: team-a\n source:\n path: apps/web\n" + " destination:\n namespace: web\n---\n" + "apiVersion: argoproj.io/v1alpha1\nkind: Rollout\nmetadata:\n name: web\n" + "spec:\n strategy:\n canary:\n canaryService: web-canary\n" + " stableService: web-stable\n analysis:\n templates:\n" + " - templateName: success-rate\n template:\n spec:\n" + " serviceAccountName: web\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: web-canary\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: web-stable\n---\n" + "apiVersion: argoproj.io/v1alpha1\nkind: AnalysisTemplate\nmetadata:\n" + " name: success-rate\n---\n" + "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: web\n"; + static const GpfFile files[] = {{"apps/web/rollout.yaml", doc}}; + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"project\""), 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"rolloutService\""), + 2); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"analysis\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"serviceAccountName\""), + 1); + PASS(); +} + /* A CRD vendored twice (gardener: example/seed-crds and a chart): a custom * resource is an instance of the copy deployed with it; only when no copy * shares its bundle does it stand for all of them. */ @@ -2879,6 +3086,10 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_go_controller_import_follows_go_mod); RUN_TEST(probe_go_controller_chain_with_comment_lines); RUN_TEST(probe_go_controller_methods_sharing_a_node); + RUN_TEST(probe_flux_refs_meet_in_the_deploy_scope); + RUN_TEST(probe_capi_refs_follow_the_api_group); + RUN_TEST(probe_cert_manager_and_kkp_refs); + RUN_TEST(probe_argo_application_and_rollout_refs); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); RUN_TEST(probe_k8s_kustomize_namespace_is_effective); From 0e75e31e9cb0ae77a0ea78f77e1400a0937c3153 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 22:41:13 +0200 Subject: [PATCH 24/52] feat(k8s): exposed routes and policy selectors What a cluster exposes and how its policies apply were not in the graph: an Ingress or HTTPRoute was linked to its Services, but the host and path it publishes had no node, and PodDisruptionBudgets and NetworkPolicies selected nothing. Exposed routes: each Ingress rule's host and http path, and each HTTPRoute hostname with each path match of a rule, is a Route node ("", QN __route__ingress__, props via/host/path). The declaring object DEFINES it, and the route maps (INFRA_MAPS {kind: ingress|httproute}) to the Service it is sent to, resolved like the object's own references. The route is not a code handler, so infra URL matching never takes it for one. Policy selectors: a PodDisruptionBudget's selector.matchLabels and a NetworkPolicy's podSelector.matchLabels pick workloads by their pod labels, like a Service selector: USAGE {ref: selector|podSelector}. An empty selector (every pod of the namespace) and one with matchExpressions draw nothing. An Argo Rollout is a workload for every selector. Versioned manifests side by side (kubermatic's canal_v3.21..v3.29) declare one Deployment in several files; as for references, a selector now picks the copy in its own file when there is one, instead of every version. Measured on 17 infrastructure repositories against an independent reading of the documents: routes defined 29/29, route -> Service precision 11/11 (5 more need the deploy script that applies a directory recursively); policy selectors precision 17/17 (the rest of the repository-wide pairs cross bundles: an optional kustomize component, per-CNI test data); references unchanged. Only the new nodes and edges were added, and 12 cross-version selector edges removed; three runs build identical graphs. Signed-off-by: Martin Vogel --- internal/cbm/cbm.h | 13 ++- internal/cbm/extract_k8s.c | 146 ++++++++++++++++++++++++++++ src/pipeline/pass_k8s.c | 167 +++++++++++++++++++++++++++++--- src/pipeline/pass_route_nodes.c | 2 +- tests/test_grammar_probe_f.c | 82 ++++++++++++++++ 5 files changed, 391 insertions(+), 19 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 6be7102d5..4cef7f0f5 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -608,8 +608,16 @@ typedef struct { * CBM_K8S_FACT_CRD: the CRD def_index declares kind `kind` in group `name`. * CBM_K8S_FACT_DEPLOY: object def_index (a Flux Kustomization, an Argo * Application) deploys the repository directory `name` into namespace - * `ns` (NULL: the objects' own); via "flux" / "argo". */ -typedef enum { CBM_K8S_FACT_REF = 0, CBM_K8S_FACT_CRD, CBM_K8S_FACT_DEPLOY } CBMK8sFactKind; + * `ns` (NULL: the objects' own); via "flux" / "argo". + * CBM_K8S_FACT_ROUTE: object def_index (an Ingress, an HTTPRoute) exposes + * `route` ("") and sends it to the kind/ns/name backend + * (name NULL: none); via "ingress" / "httproute". */ +typedef enum { + CBM_K8S_FACT_REF = 0, + CBM_K8S_FACT_CRD, + CBM_K8S_FACT_DEPLOY, + CBM_K8S_FACT_ROUTE +} CBMK8sFactKind; typedef struct { CBMK8sFactKind fact; int def_index; @@ -618,6 +626,7 @@ typedef struct { const char *name; const char *via; const char *edge; + const char *route; } CBMK8sFact; typedef struct { diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 6d5df5565..5c8394055 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -798,6 +798,144 @@ static const char *k8s_field(k8s_walk_t *w, TSNode mapping, const char *field) { return NULL; } +// The content node of `field` in a mapping (null when absent or not a mapping). +static TSNode k8s_child(k8s_walk_t *w, TSNode mapping, const char *field) { + TSNode none = {0}; + if (ts_node_is_null(mapping) || !k8s_is_mapping(mapping)) { + return none; + } + uint32_t n = ts_node_named_child_count(mapping); + for (uint32_t i = 0; i < n; i++) { + TSNode pair = ts_node_named_child(mapping, i); + if (!k8s_is_pair(pair)) { + continue; + } + TSNode key = ts_node_named_child(pair, 0); + const char *k = + ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); + if (k && strcmp(k, field) == 0) { + TSNode v = unwrap_pair_value(pair); + return ts_node_is_null(v) ? none : k8s_unwrap(v); + } + } + return none; +} + +// The items of a sequence node, unwrapped, comments skipped. +typedef struct { + TSNode seq; + uint32_t i; + uint32_t n; +} k8s_iter_t; + +static k8s_iter_t k8s_iter(TSNode seq) { + k8s_iter_t it = {.seq = seq}; + if (!ts_node_is_null(seq) && (strcmp(ts_node_type(seq), "block_sequence") == 0 || + strcmp(ts_node_type(seq), "flow_sequence") == 0)) { + it.n = ts_node_named_child_count(seq); + } + return it; +} + +static bool k8s_next(k8s_iter_t *it, TSNode *out) { + while (it->i < it->n) { + TSNode item = ts_node_named_child(it->seq, it->i++); + if (strcmp(ts_node_type(item), "block_sequence_item") == 0) { + item = first_content_child(item); + } + if (!ts_node_is_null(item) && strcmp(ts_node_type(item), "comment") != 0) { + *out = k8s_unwrap(item); + return true; + } + } + return false; +} + +// A scalar field of a node (NULL unless a mapping holding a scalar there). +static const char *k8s_text(k8s_walk_t *w, TSNode mapping, const char *field) { + TSNode v = k8s_child(w, mapping, field); + return ts_node_is_null(v) ? NULL : get_scalar_text(w->ctx->arena, v, w->ctx->source); +} + +// An exposed route "" to backend `name` (NULL: none). +static void k8s_emit_route(k8s_walk_t *w, const char *host, const char *path, const char *ns, + const char *name, const char *via) { + host = host && host[0] ? host : "*"; + path = path && path[0] ? path : "/"; + if (strstr(host, "{{") || strstr(path, "{{") || strchr(host, '/')) { + return; + } + CBMArena *a = w->ctx->arena; + CBMK8sFact f = {.fact = CBM_K8S_FACT_ROUTE, + .def_index = w->def_index, + .kind = name ? "Service" : NULL, + .ns = ns && ns[0] && !strstr(ns, "{{") ? cbm_arena_strdup(a, ns) : NULL, + .name = name && !strstr(name, "{{") ? cbm_arena_strdup(a, name) : NULL, + .via = via, + .edge = "INFRA_MAPS", + .route = cbm_arena_sprintf(a, "%s%s%s", host, path[0] == '/' ? "" : "/", path)}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, a, f); +} + +// The routes an Ingress rule (spec.rules[] item) exposes: its host and each +// http path, sent to the path's backend Service. +static void k8s_ingress_rule(k8s_walk_t *w, TSNode rule) { + const char *host = k8s_text(w, rule, "host"); + k8s_iter_t paths = k8s_iter(k8s_child(w, k8s_child(w, rule, "http"), "paths")); + TSNode p; + while (k8s_next(&paths, &p)) { + TSNode backend = k8s_child(w, p, "backend"); + const char *svc = k8s_text(w, k8s_child(w, backend, "service"), "name"); + svc = svc ? svc : k8s_text(w, backend, "serviceName"); + k8s_emit_route(w, host, k8s_text(w, p, "path"), NULL, svc, "ingress"); + } +} + +// One HTTPRoute host x path: to each Service backend of the rule, else none. +static void k8s_httproute_target(k8s_walk_t *w, TSNode rule, const char *host, const char *path) { + k8s_iter_t backs = k8s_iter(k8s_child(w, rule, "backendRefs")); + TSNode b; + bool any = false; + while (k8s_next(&backs, &b)) { + const char *kind = k8s_text(w, b, "kind"); + if (kind && strcmp(kind, "Service") != 0) { + continue; + } + k8s_emit_route(w, host, path, k8s_text(w, b, "namespace"), k8s_text(w, b, "name"), + "httproute"); + any = true; + } + if (!any) { + k8s_emit_route(w, host, path, NULL, NULL, "httproute"); + } +} + +// The routes an HTTPRoute spec exposes: every hostname (else any) with +// every path match of a rule (else `/`), sent to the rule's Service backends. +static void k8s_httproute_spec(k8s_walk_t *w, TSNode spec) { + TSNode hostnames = k8s_child(w, spec, "hostnames"); + k8s_iter_t rules = k8s_iter(k8s_child(w, spec, "rules")); + TSNode rule; + while (k8s_next(&rules, &rule)) { + k8s_iter_t hosts = k8s_iter(hostnames); + TSNode h; + bool any_host = k8s_next(&hosts, &h); + do { + const char *host = any_host ? get_scalar_text(w->ctx->arena, h, w->ctx->source) : NULL; + k8s_iter_t matches = k8s_iter(k8s_child(w, rule, "matches")); + TSNode m; + bool any_match = false; + while (k8s_next(&matches, &m)) { + k8s_httproute_target(w, rule, host, k8s_text(w, k8s_child(w, m, "path"), "value")); + any_match = true; + } + if (!any_match) { + k8s_httproute_target(w, rule, host, NULL); + } + } while (any_host && k8s_next(&hosts, &h)); + } +} + // The reference a mapping holds in its kind/name/namespace: to `kind`, or // (NULL) to its own `kind` field, one of `allowed` (NULL: any), `dflt` // when absent. @@ -828,6 +966,14 @@ static void k8s_mapping(k8s_walk_t *w, const char *path, TSNode mapping) { k8s_mapping_ref(w, mapping, r->kind, r->allowed, r->dflt, r->via, r->edge); } } + if (k8s_source_is(w, "|networking.k8s.io|extensions|", "|Ingress|") && + strcmp(path, "spec.rules[]") == 0) { + k8s_ingress_rule(w, mapping); + } + if (k8s_source_is(w, "|gateway.networking.k8s.io|", "|HTTPRoute|") && + strcmp(path, "spec") == 0) { + k8s_httproute_spec(w, mapping); + } } // Walk `node` (reached at `path`) for references. diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 674eabdd4..ca466b0ba 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -104,6 +104,8 @@ typedef enum { K8S_REC_SERVICE, /* pairs = spec.selector */ K8S_REC_WORKLOAD, /* pairs = pod-template labels */ K8S_REC_POD, /* pairs = metadata.labels */ + K8S_REC_PDB, /* pairs = spec.selector.matchLabels */ + K8S_REC_NETPOL, /* pairs = spec.podSelector.matchLabels */ } k8s_rec_kind_t; typedef struct { @@ -113,6 +115,7 @@ typedef struct { k8s_rec_kind_t kind; int pair_off; /* this record's pairs: pairs[pair_off .. pair_off + pair_n) */ int pair_n; + bool has_expr; /* its label selector also has matchExpressions */ } k8s_record_t; typedef struct { @@ -179,12 +182,20 @@ typedef enum { K8S_PAIR_SELECTOR, /* spec.selector (a Service's selector is a flat map) */ K8S_PAIR_TEMPLATE_LABEL, /* spec.template.metadata.labels, also under a CronJob's jobTemplate */ K8S_PAIR_META_LABEL, /* metadata.labels */ + K8S_PAIR_MATCH_LABELS, /* spec.selector.matchLabels (a PodDisruptionBudget's) */ + K8S_PAIR_POD_SELECTOR, /* spec.podSelector.matchLabels (a NetworkPolicy's) */ } k8s_pair_block_t; static k8s_pair_block_t k8s_pair_block(const char *parent_path) { if (strcmp(parent_path, "spec.selector") == 0) { return K8S_PAIR_SELECTOR; } + if (strcmp(parent_path, "spec.selector.matchLabels") == 0) { + return K8S_PAIR_MATCH_LABELS; + } + if (strcmp(parent_path, "spec.podSelector.matchLabels") == 0) { + return K8S_PAIR_POD_SELECTOR; + } if (strcmp(parent_path, "spec.template.metadata.labels") == 0 || strcmp(parent_path, "spec.jobTemplate.spec.template.metadata.labels") == 0) { return K8S_PAIR_TEMPLATE_LABEL; @@ -198,10 +209,16 @@ static k8s_pair_block_t k8s_pair_block(const char *parent_path) { static k8s_rec_kind_t k8s_rec_kind(const char *kind) { static const char *const workloads[] = { "Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "ReplicationController", - "Job", "CronJob", NULL}; + "Job", "CronJob", "Rollout", NULL}; if (strcmp(kind, "Service") == 0) { return K8S_REC_SERVICE; } + if (strcmp(kind, "PodDisruptionBudget") == 0) { + return K8S_REC_PDB; + } + if (strcmp(kind, "NetworkPolicy") == 0) { + return K8S_REC_NETPOL; + } if (strcmp(kind, "Pod") == 0) { return K8S_REC_POD; } @@ -222,6 +239,10 @@ static k8s_pair_block_t k8s_kind_block(k8s_rec_kind_t kind) { return K8S_PAIR_TEMPLATE_LABEL; case K8S_REC_POD: return K8S_PAIR_META_LABEL; + case K8S_REC_PDB: + return K8S_PAIR_MATCH_LABELS; + case K8S_REC_NETPOL: + return K8S_PAIR_POD_SELECTOR; case K8S_REC_OTHER: default: return K8S_PAIR_NONE; @@ -371,6 +392,12 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec, k8s_scanned_t char key[64]; char val[CBM_SZ_512]; /* a whole flow map fits: the line is the limit */ if (k8s_split_kv(trimmed, key, sizeof(key), val, sizeof(val))) { + if (strcmp(key, "matchExpressions") == 0) { + char parent[CBM_SZ_512]; + k8s_parent_path(parent, sizeof(parent), (const char (*)[64])keys, depth); + rec->has_expr = rec->has_expr || strcmp(parent, "spec.selector") == 0 || + strcmp(parent, "spec.podSelector") == 0; + } if (val[0] == '\0') { /* Block-opening key: push onto the path stack. */ if (depth < K8S_PATH_DEPTH) { @@ -1085,6 +1112,7 @@ typedef struct { char *name; /* target metadata.name */ const char *via; /* static: the extractor's table */ const char *edge; /* static: "USAGE", "INFRA_MAPS" or "DEPENDS_ON" */ + char *route; /* an exposed route "" the edge leaves from, NULL = src */ } k8s_ref_t; /* A CRD and the group/kind it declares. */ @@ -1119,6 +1147,7 @@ static void k8s_facts_free(k8s_facts_t *f) { cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].kind); cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].ns); cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].name); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].route); } for (int i = 0; i < f->crd_n; i++) { cbm_free(CBM_MEM_CLASS_OTHER, f->crds[i].group); @@ -1201,13 +1230,15 @@ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64 cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind)}; } } else if (k8s_grow((void **)&f->refs, &f->ref_cap, f->ref_n, sizeof(*f->refs))) { + bool route = x->fact == CBM_K8S_FACT_ROUTE && x->route; f->refs[f->ref_n++] = (k8s_ref_t){.src = node, - .kind = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind), + .kind = x->kind ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind) : NULL, .ns = x->ns ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->ns) : NULL, - .name = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + .name = x->name ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name) : NULL, .via = x->via, - .edge = x->edge}; + .edge = x->edge, + .route = route ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->route) : NULL}; } } } @@ -1557,14 +1588,52 @@ static bool k8s_ref_meets(k8s_bundles_t *b, const k8s_ref_t *ref, const cbm_gbuf return false; } +/* The Route node of a route `src` (an Ingress, an HTTPRoute) exposes, which + * `src` DEFINES; 0 on failure. Its QN is the route itself, so every object + * exposing one host and path shares it. */ +static int64_t k8s_route_node(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *src, + const k8s_ref_t *ref) { + enum { K8S_ROUTE_ESC = 6 * CBM_SZ_256 }; + const char *slash = strchr(ref->route, '/'); + if (!slash) { + return 0; + } + char qn[CBM_SZ_1K]; + char host[CBM_SZ_256]; + char esc_host[K8S_ROUTE_ESC]; + char esc_path[K8S_ROUTE_ESC]; + snprintf(qn, sizeof(qn), "__route__ingress__%s", ref->route); + snprintf(host, sizeof(host), "%.*s", (int)(slash - ref->route), ref->route); + cbm_json_escape(esc_host, sizeof(esc_host), host); + cbm_json_escape(esc_path, sizeof(esc_path), slash); + char props[2 * K8S_ROUTE_ESC + CBM_SZ_256]; + snprintf(props, sizeof(props), + "{\"source\":\"k8s\",\"via\":\"%s\",\"host\":\"%s\",\"path\":\"%s\"}", ref->via, + esc_host, esc_path); + int64_t id = cbm_gbuf_upsert_node(ctx->gbuf, "Route", ref->route, qn, src->file_path, + src->start_line, src->end_line, props); + if (id > 0) { + cbm_gbuf_insert_edge(ctx->gbuf, src->id, id, "DEFINES", "{}"); + } + return id; +} + /* Draw every reference to the object(s) it names inside its bundle; - * an unresolved reference draws nothing. */ + * an unresolved reference draws nothing. An exposed route's reference + * leaves from its Route node. */ static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_facts_t *f) { int edges = 0; int unresolved = 0; for (int i = 0; i < f->ref_n; i++) { const k8s_ref_t *ref = &f->refs[i]; const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, ref->src); + int64_t from = src ? src->id : 0; + if (ref->route) { + from = src && src->file_path ? k8s_route_node(ctx, src, ref) : 0; + if (from <= 0 || !ref->kind || !ref->name) { + continue; + } + } char name[CBM_SZ_1K]; snprintf(name, sizeof(name), "%s/%s", ref->kind, ref->name); const cbm_gbuf_node_t **cands = NULL; @@ -1596,7 +1665,7 @@ static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8 !k8s_ref_meets(b, ref, src, t)) { continue; } - cbm_gbuf_insert_edge(ctx->gbuf, src->id, t->id, ref->edge, props); + cbm_gbuf_insert_edge(ctx->gbuf, from, t->id, ref->edge, props); hits++; } edges += hits; @@ -1745,14 +1814,72 @@ static void k8s_link_crds(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_f } } +/* A workload a selector matched: its object name, declaring file and + * whether that is the selector's file. */ +typedef struct { + const char *name; /* "Kind/name" (graph-owned) */ + const char *ns; + bool own; + int rec; +} k8s_match_t; + +static int k8s_cmp_match(const void *a, const void *b) { + const k8s_match_t *x = a; + const k8s_match_t *y = b; + int c = strcmp(x->name, y->name); + c = c ? c : strcmp(x->ns, y->ns); + return c ? c : (x->rec > y->rec) - (x->rec < y->rec); +} + +/* Draw the selector's edge to each match. A name is one object per kind and + * namespace, so matches declaring one name in several files are alternatives + * (versioned manifests side by side): the one in the selector's file, when + * there is one, is the one it selects. */ +static int k8s_draw_matches(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs, + const k8s_record_t *svc, k8s_match_t *m, int n) { + qsort(m, (size_t)n, sizeof(*m), k8s_cmp_match); + int edges = 0; + for (int g = 0; g < n;) { + int end = g; + bool own = false; + while (end < n && strcmp(m[end].name, m[g].name) == 0 && strcmp(m[end].ns, m[g].ns) == 0) { + own = own || m[end].own; + end++; + } + for (int i = g; i < end; i++) { + if (own && !m[i].own) { + continue; + } + const k8s_record_t *wl = &recs->items[m[i].rec]; + if (svc->kind == K8S_REC_SERVICE) { + char props[CBM_SZ_256]; + snprintf(props, sizeof(props), + "{\"kind\":\"selector\",\"service\":\"%s\",\"workload\":\"%s\"}", + svc->name, wl->name); + cbm_gbuf_insert_edge(ctx->gbuf, svc->node_id, wl->node_id, "INFRA_MAPS", props); + } else { + cbm_gbuf_insert_edge(ctx->gbuf, svc->node_id, wl->node_id, "USAGE", + svc->kind == K8S_REC_PDB ? "{\"ref\":\"selector\"}" + : "{\"ref\":\"podSelector\"}"); + } + edges++; + } + g = end; + } + return edges; +} + /* Connect each Service to the workloads its selector targets via an INFRA_MAPS - * edge (Service Resource → workload Resource), inside a shared bundle. */ + * edge (Service Resource → workload Resource), inside a shared bundle; a + * PodDisruptionBudget or NetworkPolicy to the workloads it selects (USAGE). */ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs, k8s_bundles_t *bundles) { CBMHashTable *index = cbm_ht_create(0); if (!index) { return; } + k8s_match_t *matches = NULL; + int match_cap = 0; for (int i = 0; i < recs->count; i++) { const k8s_record_t *wl = &recs->items[i]; if (wl->kind == K8S_REC_WORKLOAD || wl->kind == K8S_REC_POD) { @@ -1763,8 +1890,13 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t } int edges = 0; for (int i = 0; i < recs->count; i++) { + /* a Service, PodDisruptionBudget or NetworkPolicy selector; one with + * matchExpressions selects fewer pods than its matchLabels, and an + * empty one every pod of the namespace: neither is drawn */ const k8s_record_t *svc = &recs->items[i]; - if (svc->kind != K8S_REC_SERVICE || svc->pair_n == 0) { + if ((svc->kind != K8S_REC_SERVICE && svc->kind != K8S_REC_PDB && + svc->kind != K8S_REC_NETPOL) || + svc->pair_n == 0 || svc->has_expr) { continue; } /* Candidates: the workloads carrying the Service's rarest pair. */ @@ -1779,22 +1911,25 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t rarest = posting; } } - for (int c = 0; rarest && c < rarest->count; c++) { + const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_id(ctx->gbuf, svc->node_id); + int n = 0; + for (int c = 0; sn && rarest && c < rarest->count; c++) { const k8s_record_t *wl = &recs->items[rarest->items[c]]; bool all = k8s_share_bundle(bundles, ctx, svc, wl); for (int p = 0; all && p < svc->pair_n; p++) { all = k8s_has_pair(recs, wl, recs->pairs[svc->pair_off + p]); } - if (all) { - char props[CBM_SZ_256]; - snprintf(props, sizeof(props), - "{\"kind\":\"selector\",\"service\":\"%s\",\"workload\":\"%s\"}", - svc->name, wl->name); - cbm_gbuf_insert_edge(ctx->gbuf, svc->node_id, wl->node_id, "INFRA_MAPS", props); - edges++; + const cbm_gbuf_node_t *wn = all ? cbm_gbuf_find_by_id(ctx->gbuf, wl->node_id) : NULL; + if (wn && wn->name && wn->file_path && + k8s_grow((void **)&matches, &match_cap, n, sizeof(*matches))) { + matches[n++] = (k8s_match_t){ + wn->name, wl->ns, sn->file_path && strcmp(wn->file_path, sn->file_path) == 0, + rarest->items[c]}; } } + edges += k8s_draw_matches(ctx, recs, svc, matches, n); } + cbm_free(CBM_MEM_CLASS_OTHER, matches); cbm_ht_foreach(index, k8s_posting_free, NULL); cbm_ht_free(index); if (edges > 0) { diff --git a/src/pipeline/pass_route_nodes.c b/src/pipeline/pass_route_nodes.c index 11fe8c7d7..78775b779 100644 --- a/src/pipeline/pass_route_nodes.c +++ b/src/pipeline/pass_route_nodes.c @@ -346,7 +346,7 @@ static bool is_broker_route(const char *qn) { static const char *const prefixes[] = { "__route__infra__", "__route__pubsub__", "__route__cloud_tasks__", "__route__async__", "__route__cloud_scheduler__", "__route__kafka__", - "__route__sqs__"}; + "__route__sqs__", "__route__ingress__"}; if (!qn) { return false; } diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 812a6f16c..ec87497fd 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1482,6 +1482,85 @@ TEST(probe_argo_application_and_rollout_refs) { PASS(); } +/* ── Exposed routes + policy selectors (P1.4) ───────────────────────── */ + +/* A PodDisruptionBudget's selector and a NetworkPolicy's podSelector pick + * workloads by their pod labels; an empty podSelector (every pod of the + * namespace) and a selector with matchExpressions draw nothing. */ +TEST(probe_k8s_pdb_and_networkpolicy_select_workloads) { + static const char doc[] = + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\nspec:\n template:\n" + " metadata:\n labels:\n app: web\n---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\nspec:\n template:\n" + " metadata:\n labels:\n app: api\n---\n" + "apiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: web\nspec:\n" + " minAvailable: 1\n selector:\n matchLabels:\n app: web\n---\n" + "apiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: narrowed\nspec:\n" + " selector:\n matchLabels:\n app: web\n matchExpressions:\n" + " - key: track\n operator: In\n values: [canary]\n---\n" + "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: api\nspec:\n" + " podSelector:\n matchLabels:\n app: api\n---\n" + "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: deny-all\n" + "spec:\n podSelector: {}\n"; + static const GpfFile files[] = {{"k8s/app.yaml", doc}}; + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"selector\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"podSelector\""), + 1); + PASS(); +} + +/* Versioned manifests side by side (kubermatic addons/canal/canal_v3.2x): + * each declares the same Deployment; a selector picks the copy in its own + * file, not every version's. */ +TEST(probe_k8s_selectors_pick_their_own_version) { + static const char v1[] = + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: kube-controllers\nspec:\n" + " template:\n metadata:\n labels:\n app: kube-controllers\n---\n" + "apiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n" + " name: kube-controllers\nspec:\n selector:\n matchLabels:\n" + " app: kube-controllers\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: metrics\nspec:\n selector:\n" + " app: kube-controllers\n"; + static const GpfFile files[] = {{"addons/canal/canal_v1.yaml", v1}, + {"addons/canal/canal_v2.yaml", v1}}; + ASSERT_EQ(gpf_typed_edges(files, 2, "USAGE", "Resource", "Resource", "\"ref\":\"selector\""), + 2); + ASSERT_EQ(gpf_typed_edges(files, 2, "INFRA_MAPS", "Resource", "Resource", + "\"kind\":\"selector\""), + 2); + PASS(); +} + +/* An Ingress rule's host and paths and an HTTPRoute's hostnames and path + * matches are routes the cluster exposes: the object DEFINES each, and the + * route maps to the Service it is sent to. */ +TEST(probe_k8s_ingress_and_httproute_expose_routes) { + static const char doc[] = + "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: shop\nspec:\n" + " rules:\n - host: shop.example.com\n http:\n paths:\n" + " - path: /api\n pathType: Prefix\n backend:\n" + " service:\n name: api\n port:\n" + " number: 80\n" + " - path: /\n pathType: Prefix\n backend:\n" + " service:\n name: web\n port:\n" + " number: 80\n---\n" + "apiVersion: gateway.networking.k8s.io/v1\nkind: HTTPRoute\nmetadata:\n name: api\n" + "spec:\n hostnames:\n - api.example.com\n rules:\n - matches:\n" + " - path:\n type: PathPrefix\n value: /v1\n" + " backendRefs:\n - name: api\n port: 80\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: api\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n"; + static const GpfFile files[] = {{"k8s/edge.yaml", doc}}; + ASSERT_EQ(gpf_typed_edges(files, 1, "DEFINES", "Resource", "Route", "{"), 3); + ASSERT_EQ(gpf_typed_edges(files, 1, "INFRA_MAPS", "Route", "Resource", "\"kind\":\"ingress\""), + 2); + ASSERT_EQ( + gpf_typed_edges(files, 1, "INFRA_MAPS", "Route", "Resource", "\"kind\":\"httproute\""), 1); + PASS(); +} + /* A CRD vendored twice (gardener: example/seed-crds and a chart): a custom * resource is an instance of the copy deployed with it; only when no copy * shares its bundle does it stand for all of them. */ @@ -3090,6 +3169,9 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_capi_refs_follow_the_api_group); RUN_TEST(probe_cert_manager_and_kkp_refs); RUN_TEST(probe_argo_application_and_rollout_refs); + RUN_TEST(probe_k8s_pdb_and_networkpolicy_select_workloads); + RUN_TEST(probe_k8s_selectors_pick_their_own_version); + RUN_TEST(probe_k8s_ingress_and_httproute_expose_routes); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); RUN_TEST(probe_k8s_kustomize_namespace_is_effective); From 722a7334ec54b6fd23e1ffa47663c1320e361c95 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 23:37:49 +0200 Subject: [PATCH 25/52] feat(mcp): infra filters for search_graph and a topology trace mode With objects, references, exposed routes and controllers in the graph, the tools still offered no way to ask for them as infrastructure: search_graph could not narrow to a kind or a namespace, and trace_path only followed call graph or service edges. search_graph takes `kind` (a Resource's kind property, e.g. Deployment, HelmRelease) and `namespace` (the namespace of its obj_key, i.e. the effective one a kustomize build gives it). Both apply in the structural and the BM25 query mode, in the ranked query and its count alike. The namespace pattern is built in SQL from the bound name with LIKE metacharacters escaped; properties are read under json_valid, since json_extract raises on a malformed value. trace_path mode=topology follows INFRA_MAPS, USAGE, DEPENDS_ON, HANDLES and DEFINES: from a Service out to its workloads and what they mount, in to the exposed routes and the Ingress declaring them; from a Flux Kustomization to what it waits for and health-checks. On the home-ops index: kind=HelmRelease returns 68 and namespace default / kube-system / flux-system return 25 / 12 / 5, each equal to a direct count over the graph; the topology trace from the kritika Kustomization reaches its dependsOn, its HelmRelease and, one hop on, their OCI source and database cluster. Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 118 +++++++++++++++++++++++++++++------ src/store/store.c | 28 +++++++++ src/store/store.h | 2 + tests/test_grammar_probe_f.c | 96 ++++++++++++++++++++++++++++ 4 files changed, 224 insertions(+), 20 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 5834ac89c..63b9e8a35 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -562,6 +562,9 @@ static const tool_def_t TOOLS[] = { "\"query\":{\"type\":\"string\"}," "\"label\":{\"type\":\"string\"},\"name_pattern\":{\"type\":\"string\"},\"qn_pattern\":{" "\"type\":\"string\"},\"file_pattern\":{\"type\":\"string\"}," + "\"kind\":{\"type\":\"string\",\"description\":\"Infra object kind of a Resource, e.g. " + "Deployment, HelmRelease.\"},\"namespace\":{\"type\":\"string\",\"description\":" + "\"Effective Kubernetes namespace of a Resource.\"}," "\"relationship\":{\"type\":\"string\"},\"min_degree\":{\"type\":\"integer\"}," "\"max_degree\":{\"type\":\"integer\"},\"exclude_entry_points\":{\"type\":\"boolean\"}," "\"include_connected\":{\"type\":\"boolean\"},\"semantic_query\":{" @@ -619,8 +622,9 @@ static const tool_def_t TOOLS[] = { "\"cursor\":{\"type\":\"string\",\"description\":\"Pass next/next_cursor with traversal " "args unchanged; budget may increase.\"}," "\"mode\":{" - "\"type\":\"string\",\"enum\":[\"calls\",\"data_flow\",\"cross_service\"],\"default\":" - "\"calls\",\"description\":\"calls, argument-aware data_flow, or service edges.\"}," + "\"type\":\"string\",\"enum\":[\"calls\",\"data_flow\",\"cross_service\",\"topology\"]," + "\"default\":\"calls\",\"description\":\"calls, argument-aware data_flow, service edges, " + "or topology: infra objects, routes and controllers (start at a Resource or Route).\"}," "\"parameter_name\":{\"type\":\"string\",\"description\":\"data_flow parameter filter.\"}," "\"edge_types\":{\"type\":\"array\",\"items\":{" "\"type\":\"string\"}},\"risk_labels\":{\"type\":\"boolean\",\"default\":false," @@ -1531,6 +1535,23 @@ char *cbm_mcp_get_string_arg(const char *args_json, const char *key) { return result; } +/* String argument `key` copied into `buf` (truncated to fit): buf, or NULL + * when absent or empty. Nothing to free. */ +static const char *mcp_string_arg_buf(const char *args_json, const char *key, char *buf, + size_t buf_sz) { + buf[0] = '\0'; + yyjson_doc *doc = yyjson_read(args_json, strlen(args_json), 0); + if (!doc) { + return NULL; + } + yyjson_val *val = yyjson_obj_get(yyjson_doc_get_root(doc), key); + if (val && yyjson_is_str(val)) { + snprintf(buf, buf_sz, "%s", yyjson_get_str(val)); + } + yyjson_doc_free(doc); + return buf[0] ? buf : NULL; +} + static char *canonicalize_repo_path_if_exists(char *repo_path) { if (!repo_path) { return NULL; @@ -3918,6 +3939,8 @@ enum { BM25_BIND_FILE = 6, BM25_BIND_LABEL = 7, BM25_BIND_EXACT = 8, + BM25_BIND_KIND = 9, + BM25_BIND_NS = 10, BM25_SQL_AUTO_LEN = -1, /* Inner FTS5 candidate cap. SQLite can early-terminate a plain FTS5 query * (no JOIN/WHERE on outer table) of the form: @@ -4175,8 +4198,9 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total, * Returns NULL if FTS5 is unavailable or the query produced no usable tokens, * in which case the caller falls back to the regex-based search path. */ static char *bm25_search(cbm_store_t *store, const char *project, const char *query, - const char *file_pattern, const char *label, int limit, int offset, - bool tree_format, size_t max_output_bytes) { + const char *file_pattern, const char *label, const char *kind, + const char *ns, int limit, int offset, bool tree_format, + size_t max_output_bytes) { sqlite3 *db = cbm_store_get_db(store); if (!db) { return NULL; @@ -4239,6 +4263,14 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu * in the structural mode (2026-09-16 probe: `label=Class` was ignored * here and five Methods came back). MIRRORED in the count query. */ " AND (?7 IS NULL OR n.label = ?7) " + /* Infra filters, guarded (json_extract raises on malformed JSON). + * MIRRORED in the count query. */ + " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) THEN " + " json_extract(n.properties,'$.kind') END) = ?9) " + " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) THEN " + " json_extract(n.properties,'$.obj_key') END) LIKE ('%/%/' || " + " replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),'_','\\_') || '/%') " + " ESCAPE '\\') " /* rank ties are common (boosted floats) — the id tie-break makes * offset pages contractually stable across calls. */ "ORDER BY rank, n.id " @@ -4265,25 +4297,42 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu sqlite3_bind_null(stmt, BM25_BIND_LABEL); } sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + if (kind && kind[0]) { + sqlite3_bind_text(stmt, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(stmt, BM25_BIND_KIND); + } + if (ns && ns[0]) { + sqlite3_bind_text(stmt, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(stmt, BM25_BIND_NS); + } /* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT. * Uses the identical subquery structure so the FTS5 early-exit applies here too. */ int total = 0; { - const char *count_sql = "SELECT COUNT(*) FROM (" - " SELECT fts.rowid FROM (" - " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1" - " ORDER BY " BM25_WEIGHTS " LIMIT ?3" - " ) fts " - " JOIN nodes n ON n.id = fts.rowid " - " WHERE n.project = ?2 " - /* MIRRORS the ranked query's filter verbatim — same weights, same - * label exclusions. Changing one alone reports a total that does - * not describe the rows returned. */ - " AND n.label NOT IN ('File','Folder','Variable','Project')" - " AND (?6 IS NULL OR n.file_path LIKE ?6)" - " AND (?7 IS NULL OR n.label = ?7)" - ")"; + const char *count_sql = + "SELECT COUNT(*) FROM (" + " SELECT fts.rowid FROM (" + " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1" + " ORDER BY " BM25_WEIGHTS " LIMIT ?3" + " ) fts " + " JOIN nodes n ON n.id = fts.rowid " + " WHERE n.project = ?2 " + /* MIRRORS the ranked query's filter verbatim — same weights, same + * label exclusions. Changing one alone reports a total that does + * not describe the rows returned. */ + " AND n.label NOT IN ('File','Folder','Variable','Project')" + " AND (?6 IS NULL OR n.file_path LIKE ?6)" + " AND (?7 IS NULL OR n.label = ?7)" + " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) " + " THEN json_extract(n.properties,'$.kind') END) = ?9)" + " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) " + " THEN json_extract(n.properties,'$.obj_key') END) " + " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%')," + " '_','\\_') || '/%') ESCAPE '\\')" + ")"; sqlite3_stmt *cs = NULL; if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) { sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, @@ -4303,6 +4352,17 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu } else { sqlite3_bind_null(cs, BM25_BIND_LABEL); } + if (kind && kind[0]) { + sqlite3_bind_text(cs, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, + MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(cs, BM25_BIND_KIND); + } + if (ns && ns[0]) { + sqlite3_bind_text(cs, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(cs, BM25_BIND_NS); + } if (sqlite3_step(cs) == SQLITE_ROW) { total = sqlite3_column_int(cs, 0); } @@ -5281,8 +5341,12 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { if (query && query[0]) { char *q_file_pattern = cbm_mcp_get_string_arg(args, "file_pattern"); char *q_label = cbm_mcp_get_string_arg(args, "label"); - char *bm25_json = bm25_search(store, project, query, q_file_pattern, q_label, limit, offset, - !json_format, max_output_bytes); + char q_kind_buf[CBM_SZ_256]; + char q_ns_buf[CBM_SZ_256]; + const char *q_kind = mcp_string_arg_buf(args, "kind", q_kind_buf, sizeof(q_kind_buf)); + const char *q_ns = mcp_string_arg_buf(args, "namespace", q_ns_buf, sizeof(q_ns_buf)); + char *bm25_json = bm25_search(store, project, query, q_file_pattern, q_label, q_kind, q_ns, + limit, offset, !json_format, max_output_bytes); free(q_file_pattern); free(q_label); if (bm25_json) { @@ -5296,6 +5360,10 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { free(query); char *label = cbm_mcp_get_string_arg(args, "label"); + char kind_buf[CBM_SZ_256]; + char ns_buf[CBM_SZ_256]; + const char *kind_arg = mcp_string_arg_buf(args, "kind", kind_buf, sizeof(kind_buf)); + const char *ns_arg = mcp_string_arg_buf(args, "namespace", ns_buf, sizeof(ns_buf)); char *name_pattern = cbm_mcp_get_string_arg(args, "name_pattern"); char *qn_pattern = cbm_mcp_get_string_arg(args, "qn_pattern"); char *file_pattern = cbm_mcp_get_string_arg(args, "file_pattern"); @@ -5328,6 +5396,8 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { .offset = offset, .min_degree = min_degree, .max_degree = max_degree, + .kind = kind_arg, + .namespace_name = ns_arg, }; const char *fields[SG_MAX_EXTRA_FIELDS]; yyjson_doc *fields_owner = NULL; @@ -8589,6 +8659,11 @@ static yyjson_doc *resolve_trace_edge_types(const char *args, const char *mode, "HTTP_CALLS", "ASYNC_CALLS", "DATA_FLOWS", "CALLS", "CROSS_HTTP_CALLS", "CROSS_ASYNC_CALLS", "CROSS_CHANNEL", "CROSS_GRPC_CALLS", "CROSS_GRAPHQL_CALLS", "CROSS_TRPC_CALLS"}; + /* topology: traffic (INFRA_MAPS), object references (USAGE), ordering + * (DEPENDS_ON), controllers (HANDLES) and what an object declares + * (DEFINES: an Ingress's routes) */ + static const char *mode_topology[] = {"INFRA_MAPS", "USAGE", "DEPENDS_ON", "HANDLES", + "DEFINES"}; *out_count = 0; @@ -8621,6 +8696,9 @@ static yyjson_doc *resolve_trace_edge_types(const char *args, const char *mode, } else if (mode && strcmp(mode, "cross_service") == 0) { defaults = mode_cross_svc; n_defaults = (int)(sizeof(mode_cross_svc) / sizeof(mode_cross_svc[0])); + } else if (mode && strcmp(mode, "topology") == 0) { + defaults = mode_topology; + n_defaults = (int)(sizeof(mode_topology) / sizeof(mode_topology[0])); } for (int i = 0; i < n_defaults; i++) { out_types[i] = defaults[i]; diff --git a/src/store/store.c b/src/store/store.c index 4f76915d5..c08e3988c 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -6542,6 +6542,33 @@ static void where_add_like_hints(const char *column, const char *pattern, char * } } +/* Infra filters: a Resource's kind property and the namespace of its + * obj_key "group/Kind/ns/name" (no part holds a '/', so `%/%//%` pins + * the third; the pattern is built in SQL from the bound name, LIKE + * metacharacters escaped). Properties are guarded: json_extract raises on + * malformed JSON. */ +static void search_where_infra(const cbm_search_params_t *params, char *where, int where_sz, + int *wlen, int *nparams, search_bind_t *binds, int *bind_idx) { + char clause[CBM_SZ_256]; + if (params->kind && params->kind[0]) { + snprintf(clause, sizeof(clause), + "(CASE WHEN json_valid(n.properties) THEN json_extract(n.properties,'$.kind') " + "END) = ?%d", + *bind_idx + SKIP_ONE); + *wlen = where_append(where, where_sz, *wlen, nparams, clause); + where_bind_text(binds, bind_idx, params->kind); + } + if (params->namespace_name && params->namespace_name[0]) { + snprintf(clause, sizeof(clause), + "(CASE WHEN json_valid(n.properties) THEN json_extract(n.properties,'$.obj_key') " + "END) LIKE ('%%/%%/' || replace(replace(replace(?%d,'\\','\\\\'),'%%','\\%%')," + "'_','\\_') || '/%%') ESCAPE '\\'", + *bind_idx + SKIP_ONE); + *wlen = where_append(where, where_sz, *wlen, nparams, clause); + where_bind_text(binds, bind_idx, params->namespace_name); + } +} + /* Build basic WHERE clauses: project, label, name, file, qn patterns. */ static int search_where_basic(const cbm_search_params_t *params, char *where, int where_sz, int *wlen, int *nparams, search_bind_t *binds, int *bind_idx, @@ -6601,6 +6628,7 @@ static int search_where_basic(const cbm_search_params_t *params, char *where, in where_bind_text(binds, bind_idx, lp); } } + search_where_infra(params, where, where_sz, wlen, nparams, binds, bind_idx); return *nparams; } diff --git a/src/store/store.h b/src/store/store.h index d64041192..15d02dc80 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -231,6 +231,8 @@ typedef struct { const char *sort_by; /* "relevance" / "name" / "degree", NULL = relevance */ bool case_sensitive; const char **exclude_labels; /* NULL-terminated array, or NULL */ + const char *kind; /* a Resource's kind property ("Deployment"), NULL = any */ + const char *namespace_name; /* the namespace of a Resource's obj_key, NULL = any */ } cbm_search_params_t; typedef struct { diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index ec87497fd..8dbce1871 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1561,6 +1561,100 @@ TEST(probe_k8s_ingress_and_httproute_expose_routes) { PASS(); } +/* ── MCP surface: infra filters + topology (P1.5) ───────────────────── */ + +/* Run MCP `tool` on the indexed fixture; `args_fmt` holds one %s, the + * project. The response is the caller's to free; NULL when indexing failed. */ +static char *gpf_tool_on(const GpfFile *files, int nfiles, const char *tool, + const char *args_fmt) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *resp = NULL; + if (store && lp.srv) { + char args[2048]; + snprintf(args, sizeof(args), args_fmt, lp.project); + resp = cbm_mcp_handle_tool(lp.srv, tool, args); + } + gpf_cleanup(&lp, store); + return resp; +} + +static const GpfFile k_two_teams[] = { + {"k8s/a.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: team-a\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n namespace: team-a\n"}, + {"k8s/b.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\n namespace: team-b\n"}}; + +/* search_graph narrows Resources by kind and by effective namespace, in the + * structural and the BM25 query mode alike. */ +TEST(probe_mcp_search_filters_kind_and_namespace) { + char *r = gpf_tool_on(k_two_teams, 2, "search_graph", + "{\"project\":\"%s\",\"label\":\"Resource\",\"kind\":\"Deployment\"," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "Deployment/team-a/web")); + ASSERT_NOT_NULL(strstr(r, "Deployment/team-b/api")); + ASSERT_NULL(strstr(r, "Service/team-a/web")); + free(r); + r = gpf_tool_on(k_two_teams, 2, "search_graph", + "{\"project\":\"%s\",\"label\":\"Resource\",\"namespace\":\"team-a\"," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "Deployment/team-a/web")); + ASSERT_NOT_NULL(strstr(r, "Service/team-a/web")); + ASSERT_NULL(strstr(r, "Deployment/team-b/api")); + free(r); + r = gpf_tool_on(k_two_teams, 2, "search_graph", + "{\"project\":\"%s\",\"query\":\"web\",\"kind\":\"Service\"," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "Service/team-a/web")); + ASSERT_NULL(strstr(r, "Deployment/team-a/web")); + free(r); + PASS(); +} + +/* trace_path mode=topology walks infra edges: from a Service out to its + * workload and the ConfigMap the workload mounts, and in to the exposed + * route and the Ingress that declares it. The call graph modes do not. */ +TEST(probe_mcp_trace_topology) { + static const char doc[] = + "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: shop\nspec:\n" + " rules:\n - host: shop.example.com\n http:\n paths:\n" + " - path: /api\n pathType: Prefix\n backend:\n" + " service:\n name: api\n port:\n" + " number: 80\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: api\nspec:\n selector:\n" + " app: api\n---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\nspec:\n template:\n" + " metadata:\n labels:\n app: api\n spec:\n volumes:\n" + " - name: cfg\n configMap:\n name: api-config\n---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: api-config\n"; + static const GpfFile files[] = {{"k8s/shop.yaml", doc}}; + char *r = gpf_tool_on(files, 1, "trace_path", + "{\"function_name\":\"Service/api\",\"project\":\"%s\"," + "\"direction\":\"outbound\",\"mode\":\"topology\",\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "Deployment/-/api")); + ASSERT_NOT_NULL(strstr(r, "ConfigMap/-/api-config")); + free(r); + r = gpf_tool_on(files, 1, "trace_path", + "{\"function_name\":\"Service/api\",\"project\":\"%s\"," + "\"direction\":\"inbound\",\"mode\":\"topology\",\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "__route__ingress__shop.example")); /* qn groups split at a dot */ + ASSERT_NOT_NULL(strstr(r, "Ingress/-/shop")); + free(r); + r = gpf_tool_on(files, 1, "trace_path", + "{\"function_name\":\"Service/api\",\"project\":\"%s\"," + "\"direction\":\"outbound\",\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(strstr(r, "Deployment/-/api")); + free(r); + PASS(); +} + /* A CRD vendored twice (gardener: example/seed-crds and a chart): a custom * resource is an instance of the copy deployed with it; only when no copy * shares its bundle does it stand for all of them. */ @@ -3172,6 +3266,8 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_pdb_and_networkpolicy_select_workloads); RUN_TEST(probe_k8s_selectors_pick_their_own_version); RUN_TEST(probe_k8s_ingress_and_httproute_expose_routes); + RUN_TEST(probe_mcp_search_filters_kind_and_namespace); + RUN_TEST(probe_mcp_trace_topology); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); RUN_TEST(probe_k8s_kustomize_namespace_is_effective); From 1eb55031e628a47d4cf8170d1895878e2d6fc57f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 00:24:41 +0200 Subject: [PATCH 26/52] perf(k8s): infra resolvers stay linear in the repository's copies The complexity guard (O(n^2) detection) had no infrastructure corpus, and the infra resolvers looked candidates up by name across the whole repository and then filtered them: an object reference by "Kind/name", a selector by its rarest label pair, a custom resource by its CRD's group/kind, a controller's kind by the Go type's name. Independent copies share every name, so with k copies each lookup visited all k: the work grew ~4x per doubling while the edges stayed ~2x. Candidates are now looked up where they can match: - K8s references, selectors and CRDs through indexes keyed by the bundle (kustomize build root / chart / dir, and the deploy scopes for references) and the name, type or label pair; a source visits only the bundles it is in. The repository-wide fallback for a custom resource without a CRD beside it stays (it stands for every copy by design); - the operator pass through a (package dir, type) index: a controller's kind is one lookup in the imported or own package; only an import no module of the repository has still checks the same-named types. Group declarations are deduplicated through a set, not a scan. Child access in the K8s extractor goes through a tree cursor: ts_node_named_child(n, i) walks from the first child on every call, so the index loops over mapping pairs, sequence items and a file's documents were quadratic in a wide node. The guard gains fixtures (tests/fixtures/complexity/Kubernetes: a kustomize bundle with workloads, Service, Ingress, PDB, NetworkPolicy, HPA, RBAC, CRD + CR, cert-manager; tests/fixtures/complexity/Go: a controller-runtime operator with +groupName types and an embedded CRD), work counters g_k8s_candidates / g_op_candidates, and a gate on them and on the fixtures' own node and edge growth; fixture dirs now also apply to templated languages. An API group is one identity across a repository, so the fixtures' CRDs use groups no copy's Go code declares -- the guard measures accidental growth, not that by-design join. The originality check covers the infra sources and scans them against yaml-language-server, helm-ls, kustomize, Flux's kustomize package and controller-tools. Before: k8s candidates 126 -> 504 (4.00x), operator 45 -> 288 (6.40x) per doubling; after: 42 -> 84 and 6 -> 12 (2.00x), fixture edges 2.01x. On 17 infrastructure repositories the graphs are identical to before; three runs build identical graphs. Index wall time against origin/main, median of three: deckhouse 9.58 -> 9.59 s, kubermatic 2.93 -> 2.93 s, cloud-foundation-fabric 1.90 -> 1.90 s. A manifest of thousands of documents indexes ~10 % faster (2.40 -> 2.13 s). Signed-off-by: Martin Vogel --- internal/cbm/extract_k8s.c | 146 +++++--- scripts/check-lsp-originality.sh | 16 + src/pipeline/pass_k8s.c | 330 ++++++++++++++---- src/pipeline/pass_operator.c | 213 ++++++++--- src/pipeline/pipeline_internal.h | 7 + tests/fixtures/complexity/Go/controller.go | 14 + tests/fixtures/complexity/Go/crd.yaml | 10 + tests/fixtures/complexity/Go/embed.go | 6 + tests/fixtures/complexity/Go/go.mod | 3 + tests/fixtures/complexity/Go/types.go | 10 + tests/fixtures/complexity/Kubernetes/app.yaml | 117 +++++++ tests/fixtures/complexity/Kubernetes/crd.yaml | 37 ++ .../complexity/Kubernetes/kustomization.yaml | 6 + tests/test_complexity.c | 72 +++- 14 files changed, 817 insertions(+), 170 deletions(-) create mode 100644 tests/fixtures/complexity/Go/controller.go create mode 100644 tests/fixtures/complexity/Go/crd.yaml create mode 100644 tests/fixtures/complexity/Go/embed.go create mode 100644 tests/fixtures/complexity/Go/go.mod create mode 100644 tests/fixtures/complexity/Go/types.go create mode 100644 tests/fixtures/complexity/Kubernetes/app.yaml create mode 100644 tests/fixtures/complexity/Kubernetes/crd.yaml create mode 100644 tests/fixtures/complexity/Kubernetes/kustomization.yaml diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 5c8394055..636db83b9 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -28,6 +28,51 @@ enum { // Internal helpers // --------------------------------------------------------------------------- +// The children of `node` (named only when `named`), in order, through a +// tree cursor: ts_node_child(n, i) / ts_node_named_child(n, i) walk from the +// first child on every call, so an index loop over a wide node (a list of +// 100k items, a file of thousands of documents) is quadratic. The cursor is +// released when the children are exhausted, or by k8s_kids_done. +typedef struct { + TSTreeCursor cur; + bool live; +} k8s_kids_t; + +static k8s_kids_t k8s_kids(TSNode node) { + k8s_kids_t k = {.live = false}; + if (ts_node_is_null(node)) { + return k; + } + k.cur = ts_tree_cursor_new(node); + k.live = ts_tree_cursor_goto_first_child(&k.cur); + if (!k.live) { + ts_tree_cursor_delete(&k.cur); + } + return k; +} + +static bool k8s_kid(k8s_kids_t *k, bool named, TSNode *out) { + while (k->live) { + TSNode n = ts_tree_cursor_current_node(&k->cur); + if (!ts_tree_cursor_goto_next_sibling(&k->cur)) { + ts_tree_cursor_delete(&k->cur); + k->live = false; + } + if (!named || ts_node_is_named(n)) { + *out = n; + return true; + } + } + return false; +} + +static void k8s_kids_done(k8s_kids_t *k) { + if (k->live) { + ts_tree_cursor_delete(&k->cur); + k->live = false; + } +} + // The value inside a flow_node: its first named child that is not an anchor // (`&name`), a tag (`!!str`) or a comment, all of which precede the value. static TSNode flow_node_value(TSNode node) { @@ -96,16 +141,16 @@ static int is_kustomize_list_key(const char *key) { // scalar child, using key_name as the local_name. static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, const char *key_name) { CBMArena *a = ctx->arena; - uint32_t n = ts_node_child_count(seq_node); - for (uint32_t i = 0; i < n; i++) { - TSNode item = ts_node_child(seq_node, i); + k8s_kids_t items = k8s_kids(seq_node); + TSNode item; + while (k8s_kid(&items, false, &item)) { if (strcmp(ts_node_type(item), "block_sequence_item") != 0) { continue; } // block_sequence_item has one named child: the value - uint32_t ic = ts_node_child_count(item); - for (uint32_t j = 0; j < ic; j++) { - TSNode val = ts_node_child(item, j); + k8s_kids_t vals = k8s_kids(item); + TSNode val; + while (k8s_kid(&vals, false, &val)) { const char *scalar = get_scalar_text(a, val, ctx->source); if (!scalar) { continue; @@ -198,9 +243,9 @@ static void process_kustomize_pair(CBMExtractCtx *ctx, TSNode pair) { // Mirrors the K8s manifest kind-def so Kustomize resources are also discoverable. static void emit_kustomize_kind_def(CBMExtractCtx *ctx, TSNode mapping) { CBMArena *a = ctx->arena; - uint32_t pair_n = ts_node_child_count(mapping); - for (uint32_t pi = 0; pi < pair_n; pi++) { - TSNode pair = ts_node_child(mapping, pi); + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { continue; } @@ -228,15 +273,15 @@ static void emit_kustomize_kind_def(CBMExtractCtx *ctx, TSNode mapping) { def.start_line = ts_node_start_point(mapping).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(mapping).row + TS_LINE_OFFSET; cbm_defs_push(&ctx->result->defs, a, def); + k8s_kids_done(&pairs); return; } } static void extract_kustomize(CBMExtractCtx *ctx) { - TSNode root = ctx->root; - uint32_t root_n = ts_node_child_count(root); - for (uint32_t si = 0; si < root_n; si++) { - TSNode stream_child = ts_node_child(root, si); + k8s_kids_t docs = k8s_kids(ctx->root); + TSNode stream_child; + while (k8s_kid(&docs, false, &stream_child)) { if (strcmp(ts_node_type(stream_child), "document") != 0) { continue; } @@ -247,9 +292,10 @@ static void extract_kustomize(CBMExtractCtx *ctx) { emit_kustomize_kind_def(ctx, mapping); - uint32_t pair_n = ts_node_child_count(mapping); - for (uint32_t pi = 0; pi < pair_n; pi++) { - process_kustomize_pair(ctx, ts_node_child(mapping, pi)); + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { + process_kustomize_pair(ctx, pair); } } } @@ -272,9 +318,9 @@ static void extract_metadata(CBMArena *a, TSNode meta_mapping, const char *sourc if (ts_node_is_null(meta_mapping) || strcmp(ts_node_type(meta_mapping), "block_mapping") != 0) { return; } - uint32_t mn = ts_node_child_count(meta_mapping); - for (uint32_t mi = 0; mi < mn; mi++) { - TSNode mpair = ts_node_child(meta_mapping, mi); + k8s_kids_t pairs = k8s_kids(meta_mapping); + TSNode mpair; + while (k8s_kid(&pairs, false, &mpair)) { if (strcmp(ts_node_type(mpair), "block_mapping_pair") != 0) { continue; } @@ -299,9 +345,9 @@ static void extract_metadata(CBMArena *a, TSNode meta_mapping, const char *sourc // Read apiVersion, kind and metadata from a document's top-level mapping. static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, k8s_doc_id_t *id) { CBMArena *a = ctx->arena; - uint32_t n = ts_node_child_count(mapping); - for (uint32_t i = 0; i < n; i++) { - TSNode pair = ts_node_child(mapping, i); + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { continue; } @@ -781,9 +827,9 @@ static bool k8s_is_pair(TSNode n) { // The scalar value of `field` in a mapping, or NULL. static const char *k8s_field(k8s_walk_t *w, TSNode mapping, const char *field) { - uint32_t n = ts_node_named_child_count(mapping); - for (uint32_t i = 0; i < n; i++) { - TSNode pair = ts_node_named_child(mapping, i); + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, true, &pair)) { if (!k8s_is_pair(pair)) { continue; } @@ -791,6 +837,7 @@ static const char *k8s_field(k8s_walk_t *w, TSNode mapping, const char *field) { const char *k = ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); if (k && strcmp(k, field) == 0) { + k8s_kids_done(&pairs); TSNode v = unwrap_pair_value(pair); return ts_node_is_null(v) ? NULL : get_scalar_text(w->ctx->arena, v, w->ctx->source); } @@ -804,9 +851,9 @@ static TSNode k8s_child(k8s_walk_t *w, TSNode mapping, const char *field) { if (ts_node_is_null(mapping) || !k8s_is_mapping(mapping)) { return none; } - uint32_t n = ts_node_named_child_count(mapping); - for (uint32_t i = 0; i < n; i++) { - TSNode pair = ts_node_named_child(mapping, i); + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, true, &pair)) { if (!k8s_is_pair(pair)) { continue; } @@ -814,6 +861,7 @@ static TSNode k8s_child(k8s_walk_t *w, TSNode mapping, const char *field) { const char *k = ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); if (k && strcmp(k, field) == 0) { + k8s_kids_done(&pairs); TSNode v = unwrap_pair_value(pair); return ts_node_is_null(v) ? none : k8s_unwrap(v); } @@ -821,25 +869,20 @@ static TSNode k8s_child(k8s_walk_t *w, TSNode mapping, const char *field) { return none; } -// The items of a sequence node, unwrapped, comments skipped. -typedef struct { - TSNode seq; - uint32_t i; - uint32_t n; -} k8s_iter_t; +// The items of a sequence node, unwrapped, comments skipped (iterate to the +// end: the cursor is released when the items are exhausted). +typedef k8s_kids_t k8s_iter_t; static k8s_iter_t k8s_iter(TSNode seq) { - k8s_iter_t it = {.seq = seq}; - if (!ts_node_is_null(seq) && (strcmp(ts_node_type(seq), "block_sequence") == 0 || - strcmp(ts_node_type(seq), "flow_sequence") == 0)) { - it.n = ts_node_named_child_count(seq); - } - return it; + TSNode none = {0}; + bool is_seq = !ts_node_is_null(seq) && (strcmp(ts_node_type(seq), "block_sequence") == 0 || + strcmp(ts_node_type(seq), "flow_sequence") == 0); + return k8s_kids(is_seq ? seq : none); } static bool k8s_next(k8s_iter_t *it, TSNode *out) { - while (it->i < it->n) { - TSNode item = ts_node_named_child(it->seq, it->i++); + TSNode item; + while (k8s_kid(it, true, &item)) { if (strcmp(ts_node_type(item), "block_sequence_item") == 0) { item = first_content_child(item); } @@ -985,9 +1028,9 @@ static void k8s_walk(k8s_walk_t *w, TSNode node, char *path, size_t len, size_t const char *type = ts_node_type(node); if (k8s_is_mapping(node)) { k8s_mapping(w, path, node); - uint32_t n = ts_node_named_child_count(node); - for (uint32_t i = 0; i < n; i++) { - TSNode pair = ts_node_named_child(node, i); + k8s_kids_t pairs = k8s_kids(node); + TSNode pair; + while (k8s_kid(&pairs, true, &pair)) { if (!k8s_is_pair(pair)) { continue; } @@ -1019,9 +1062,9 @@ static void k8s_walk(k8s_walk_t *w, TSNode node, char *path, size_t len, size_t path[len] = '\0'; return; } - uint32_t n = ts_node_named_child_count(node); - for (uint32_t i = 0; i < n; i++) { - TSNode item = ts_node_named_child(node, i); + k8s_kids_t items = k8s_kids(node); + TSNode item; + while (k8s_kid(&items, true, &item)) { if (strcmp(ts_node_type(item), "block_sequence_item") == 0) { item = first_content_child(item); } @@ -1108,10 +1151,9 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { return; } - TSNode root = ctx->root; - uint32_t root_n = ts_node_child_count(root); - for (uint32_t si = 0; si < root_n; si++) { - TSNode stream_child = ts_node_child(root, si); + k8s_kids_t docs = k8s_kids(ctx->root); + TSNode stream_child; + while (k8s_kid(&docs, false, &stream_child)) { if (strcmp(ts_node_type(stream_child), "document") != 0) { continue; } diff --git a/scripts/check-lsp-originality.sh b/scripts/check-lsp-originality.sh index a50af9de4..a107d60f0 100755 --- a/scripts/check-lsp-originality.sh +++ b/scripts/check-lsp-originality.sh @@ -64,8 +64,21 @@ REFS=( # Perl (PR #461) was authored clean-room. We scan against PerlNavigator (MIT) # — the leading OSS Perl language server — for defensive copy-detection. "perl|https://github.com/bscan/PerlNavigator|server/src" + # Infrastructure hybrid LSPs (K8s YAML, Kustomize bundles, Flux/Argo deploy + # scopes, the Go operator extension) were written from the Kubernetes, Flux, + # Argo, Cluster API, cert-manager and Kubermatic API types. Scanned against + # the YAML/K8s language server, helm-ls, kustomize, Flux's kustomization + # generator and controller-tools (the +groupName markers). + "yaml|https://github.com/redhat-developer/yaml-language-server|src" + "helm|https://github.com/mrjosh/helm-ls|internal" + "kustomize|https://github.com/kubernetes-sigs/kustomize|api/krusty,api/internal/target" + "flux|https://github.com/fluxcd/pkg|kustomize" + "kube|https://github.com/kubernetes-sigs/controller-tools|pkg" ) +# Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too). +INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c) + ONLY_LANG=""; REFRESH=0; LIST_ONLY=0 while [ $# -gt 0 ]; do case "$1" in @@ -87,6 +100,9 @@ fi LSP_SRC=() while IFS= read -r f; do LSP_SRC+=("$f"); done < <( find "$LSP_DIR" -type f \( -name '*.c' -o -name '*.h' \) -not -path '*/generated/*' | sort) +for f in "${INFRA_SRC[@]}"; do + [ -f "$ROOT/$f" ] && LSP_SRC+=("$ROOT/$f") +done if [ "${#LSP_SRC[@]}" -eq 0 ]; then echo "$TAG ERROR: no LSP sources under $LSP_DIR" >&2; exit 2 fi diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index ca466b0ba..3dea7f979 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -1102,6 +1102,9 @@ static void k8s_apply_effective_namespaces(cbm_pipeline_ctx_t *ctx, k8s_bundles_ } } +/* Candidates examined by the resolvers below (pipeline_internal.h). */ +_Atomic uint64_t g_k8s_candidates = 0; + /* ── Object references (PHASE1.md) ───────────────────────────────── */ /* A reference fact of a manifest, kept past its file's extraction. */ @@ -1517,6 +1520,144 @@ static void k8s_deploys_build(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bund cbm_log_info("pass.k8s.deploys", "deploys", itoa_k8s(n), "roots", itoa_k8s(roots)); } +/* ── Bundle-keyed candidate indexes ───────────────────────────────── */ + +/* Candidates are looked up per bundle the source shares, never by name + * across the repository: copies of one name in independent bundles would + * otherwise make every lookup visit every copy (quadratic in copies). */ + +typedef struct { + int64_t *items; + int count; + int cap; +} k8s_ids_t; + +static void k8s_ids_push(k8s_ids_t *v, int64_t id) { + if (k8s_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = id; + } +} + +static int k8s_cmp_id(const void *a, const void *b) { + int64_t x = *(const int64_t *)a; + int64_t y = *(const int64_t *)b; + return (x > y) - (x < y); +} + +static void k8s_ids_sort_unique(k8s_ids_t *v) { + if (v->count < PAIR_LEN) { + return; + } + qsort(v->items, (size_t)v->count, sizeof(*v->items), k8s_cmp_id); + int w = SKIP_ONE; + for (int i = SKIP_ONE; i < v->count; i++) { + if (v->items[i] != v->items[w - SKIP_ONE]) { + v->items[w++] = v->items[i]; + } + } + v->count = w; +} + +static void k8s_free_ids_entry(const char *key, void *value, void *ud) { + (void)ud; + k8s_ids_t *v = value; + cbm_free(CBM_MEM_CLASS_OTHER, v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +/* "\x1f": an interned bundle (by address) with an object + * name, a type or a label pair. */ +static void k8s_bkey(char *out, size_t out_sz, const char *bundle, const char *name) { + snprintf(out, out_sz, "%p\x1f%s", (const void *)bundle, name); +} + +static void k8s_index_add(CBMHashTable *ix, const char *key, int64_t id) { + k8s_ids_t *v = cbm_ht_get(ix, key); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!v || !owned) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, owned); + return; + } + cbm_ht_set(ix, owned, v); + } + if (v->count == 0 || v->items[v->count - SKIP_ONE] != id) { + k8s_ids_push(v, id); + } +} + +/* Calls `fn(bundle, ud)` for every bundle of `file`: its build roots / + * chart / dir and, when `deploys`, the deploy scopes reaching it. */ +typedef void (*k8s_scope_fn)(const char *bundle, void *ud); + +static void k8s_each_scope(k8s_bundles_t *b, const char *file, bool deploys, k8s_scope_fn fn, + void *ud) { + k8s_member_t one[SKIP_ONE]; + k8s_members_t single = {.items = one, .cap = SKIP_ONE}; + const k8s_members_t *m = k8s_bundles_of(b, file, &single); + for (int i = 0; i < m->count; i++) { + fn(m->items[i].bundle, ud); + } + const k8s_members_t *d = deploys && b->deploys ? cbm_ht_get(b->deploys, file) : NULL; + for (int i = 0; d && i < d->count; i++) { + fn(d->items[i].bundle, ud); + } +} + +typedef struct { + CBMHashTable *ix; + const char *name; + int64_t id; + k8s_ids_t *found; +} k8s_scope_job_t; + +static void k8s_scope_add(const char *bundle, void *ud) { + k8s_scope_job_t *j = ud; + char key[CBM_SZ_1K]; + k8s_bkey(key, sizeof(key), bundle, j->name); + k8s_index_add(j->ix, key, j->id); +} + +static void k8s_scope_find(const char *bundle, void *ud) { + k8s_scope_job_t *j = ud; + char key[CBM_SZ_1K]; + k8s_bkey(key, sizeof(key), bundle, j->name); + const k8s_ids_t *v = cbm_ht_get(j->ix, key); + for (int i = 0; v && i < v->count; i++) { + k8s_ids_push(j->found, v->items[i]); + } +} + +/* Index every Resource by "\x1f" over its bundles (and + * deploy scopes when `deploys`). */ +static CBMHashTable *k8s_index_objects(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, bool deploys) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!ix || cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &count) != 0) { + return ix; + } + for (int i = 0; i < count; i++) { + if (nodes[i]->file_path && nodes[i]->name) { + k8s_scope_job_t j = {ix, nodes[i]->name, nodes[i]->id, NULL}; + k8s_each_scope(b, nodes[i]->file_path, deploys, k8s_scope_add, &j); + } + } + return ix; +} + +/* The ids `name` has in any scope of `file`, sorted, unique. */ +static void k8s_find_in_scopes(k8s_bundles_t *b, CBMHashTable *ix, const char *file, + const char *name, bool deploys, k8s_ids_t *found) { + found->count = 0; + k8s_scope_job_t j = {ix, name, 0, found}; + k8s_each_scope(b, file, deploys, k8s_scope_find, &j); + k8s_ids_sort_unique(found); +} + /* The namespace of a Resource's logical key ("" when unknown `?` or * cluster-scoped `_`), else its declared one. */ static void k8s_effective_ns(const cbm_gbuf_node_t *n, char *out, size_t out_sz) { @@ -1624,7 +1765,11 @@ static int64_t k8s_route_node(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *sr static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_facts_t *f) { int edges = 0; int unresolved = 0; - for (int i = 0; i < f->ref_n; i++) { + CBMHashTable *ix = f->ref_n > 0 ? k8s_index_objects(ctx, b, true) : NULL; + k8s_ids_t found = {0}; + const cbm_gbuf_node_t **cands = NULL; + int cand_cap = 0; + for (int i = 0; ix && i < f->ref_n; i++) { const k8s_ref_t *ref = &f->refs[i]; const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, ref->src); int64_t from = src ? src->id : 0; @@ -1636,11 +1781,17 @@ static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8 } char name[CBM_SZ_1K]; snprintf(name, sizeof(name), "%s/%s", ref->kind, ref->name); - const cbm_gbuf_node_t **cands = NULL; int n = 0; - if (!src || !src->file_path || cbm_gbuf_find_by_name(ctx->gbuf, name, &cands, &n) != 0) { - n = 0; + if (src && src->file_path) { + k8s_find_in_scopes(b, ix, src->file_path, name, true, &found); + for (int c = 0; c < found.count; c++) { + const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(ctx->gbuf, found.items[c]); + if (t && k8s_grow((void **)&cands, &cand_cap, n, sizeof(*cands))) { + cands[n++] = t; + } + } } + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)(n), memory_order_relaxed); char props[CBM_SZ_256]; snprintf(props, sizeof(props), "{\"%s\":\"%s\"}", strcmp(ref->edge, "INFRA_MAPS") == 0 ? "kind" @@ -1671,6 +1822,12 @@ static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8 edges += hits; unresolved += hits == 0; } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)cands); + cbm_free(CBM_MEM_CLASS_OTHER, found.items); + if (ix) { + cbm_ht_foreach(ix, k8s_free_ids_entry, NULL); + cbm_ht_free(ix); + } if (f->ref_n > 0) { cbm_log_info("pass.k8s.refs", "refs", itoa_k8s(f->ref_n), "edges", itoa_k8s(edges), "unresolved", itoa_k8s(unresolved)); @@ -1690,24 +1847,6 @@ static int k8s_cmp_crd_key(const void *a, const void *b) { return c ? c : (x->crd > y->crd) - (x->crd < y->crd); } -/* Do the files at `a` and `b` share a bundle? */ -static bool k8s_same_bundle(k8s_bundles_t *b, const char *a, const char *c) { - k8s_member_t one_a[SKIP_ONE]; - k8s_member_t one_c[SKIP_ONE]; - k8s_members_t single_a = {.items = one_a, .cap = SKIP_ONE}; - k8s_members_t single_c = {.items = one_c, .cap = SKIP_ONE}; - const k8s_members_t *ma = k8s_bundles_of(b, a, &single_a); - const k8s_members_t *mc = k8s_bundles_of(b, c, &single_c); - for (int i = 0; i < ma->count; i++) { - for (int j = 0; j < mc->count; j++) { - if (ma->items[i].bundle == mc->items[j].bundle) { - return true; - } - } - } - return false; -} - /* A custom resource is an instance of the CRD declaring its group and * kind. A CRD vendored in several places: the copy deployed with the * resource (one bundle) is the one; with no copy beside it, the resource @@ -1761,10 +1900,20 @@ static void k8s_link_crds(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_f for (int i = f->crd_n - SKIP_ONE; i >= 0; i--) { cbm_ht_set(by_type, order[i].key, (void *)(intptr_t)(i + SKIP_ONE)); /* first of run */ } + /* "\x1f" -> the CRDs of that type in that bundle */ + CBMHashTable *in_bundle = cbm_ht_create(0); + for (int i = 0; in_bundle && i < f->crd_n; i++) { + const cbm_gbuf_node_t *crd = cbm_gbuf_find_by_id(ctx->gbuf, f->crds[order[i].crd].node); + if (crd && crd->file_path) { + k8s_scope_job_t j = {in_bundle, order[i].key, crd->id, NULL}; + k8s_each_scope(b, crd->file_path, false, k8s_scope_add, &j); + } + } + k8s_ids_t beside = {0}; const cbm_gbuf_node_t **nodes = NULL; int count = 0; int edges = 0; - if (cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &count) == 0) { + if (in_bundle && cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &count) == 0) { for (int i = 0; i < count; i++) { const char *key = nodes[i]->qualified_name ? strstr(nodes[i]->qualified_name, "::") : NULL; @@ -1783,30 +1932,40 @@ static void k8s_link_crds(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_f if (first < 0) { continue; } + k8s_find_in_scopes(b, in_bundle, nodes[i]->file_path, type, false, &beside); + if (beside.count > 0) { + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)beside.count, + memory_order_relaxed); + for (int r = 0; r < beside.count; r++) { + if (beside.items[r] != nodes[i]->id) { + cbm_gbuf_insert_edge(ctx->gbuf, nodes[i]->id, beside.items[r], "USAGE", + "{\"ref\":\"crd\"}"); + edges++; + } + } + continue; + } int end = first; while (end < f->crd_n && strcmp(order[end].key, type) == 0) { end++; } - bool beside = false; - for (int r = first; r < end && !beside; r++) { - const cbm_gbuf_node_t *crd = - cbm_gbuf_find_by_id(ctx->gbuf, f->crds[order[r].crd].node); - beside = crd && crd->file_path && - k8s_same_bundle(b, nodes[i]->file_path, crd->file_path); - } + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)(end - first), + memory_order_relaxed); for (int r = first; r < end; r++) { int64_t crd_id = f->crds[order[r].crd].node; - const cbm_gbuf_node_t *crd = cbm_gbuf_find_by_id(ctx->gbuf, crd_id); - if (crd_id == nodes[i]->id || - (beside && (!crd || !crd->file_path || - !k8s_same_bundle(b, nodes[i]->file_path, crd->file_path)))) { - continue; + if (crd_id != nodes[i]->id) { + cbm_gbuf_insert_edge(ctx->gbuf, nodes[i]->id, crd_id, "USAGE", + "{\"ref\":\"crd\"}"); + edges++; } - cbm_gbuf_insert_edge(ctx->gbuf, nodes[i]->id, crd_id, "USAGE", "{\"ref\":\"crd\"}"); - edges++; } } } + cbm_free(CBM_MEM_CLASS_OTHER, beside.items); + if (in_bundle) { + cbm_ht_foreach(in_bundle, k8s_free_ids_entry, NULL); + cbm_ht_free(in_bundle); + } cbm_ht_free(by_type); cbm_free(CBM_MEM_CLASS_OTHER, order); if (edges > 0) { @@ -1814,6 +1973,60 @@ static void k8s_link_crds(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_f } } +/* Selector postings keyed by "\x1f" (owned keys): the + * workloads (record indices) of one bundle carrying one label pair. */ +static void k8s_posting_free_owned(const char *key, void *value, void *userdata) { + k8s_posting_free(key, value, userdata); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +typedef struct { + CBMHashTable *index; + const k8s_record_array_t *recs; + const k8s_record_t *rec; + int rec_idx; + k8s_ids_t *cand; /* record indices */ +} k8s_sel_job_t; + +static void k8s_sel_post(const char *bundle, void *ud) { + k8s_sel_job_t *j = ud; + for (int p = 0; p < j->rec->pair_n; p++) { + char key[CBM_SZ_1K]; + k8s_bkey(key, sizeof(key), bundle, j->recs->pairs[j->rec->pair_off + p]); + if (!cbm_ht_get(j->index, key)) { + k8s_posting_t *posting = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*posting)); + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!posting || !owned) { + cbm_free(CBM_MEM_CLASS_OTHER, posting); + cbm_free(CBM_MEM_CLASS_OTHER, owned); + continue; + } + cbm_ht_set(j->index, owned, posting); + } + (void)k8s_posting_add(j->index, key, j->rec_idx); + } +} + +/* The workloads of `bundle` carrying the selector's rarest pair there. */ +static void k8s_sel_rarest(const char *bundle, void *ud) { + k8s_sel_job_t *j = ud; + const k8s_posting_t *rarest = NULL; + for (int p = 0; p < j->rec->pair_n; p++) { + char key[CBM_SZ_1K]; + k8s_bkey(key, sizeof(key), bundle, j->recs->pairs[j->rec->pair_off + p]); + const k8s_posting_t *posting = cbm_ht_get(j->index, key); + if (!posting) { + return; + } + if (!rarest || posting->count < rarest->count) { + rarest = posting; + } + } + for (int c = 0; rarest && c < rarest->count; c++) { + k8s_ids_push(j->cand, rarest->items[c]); + } +} + /* A workload a selector matched: its object name, declaring file and * whether that is the selector's file. */ typedef struct { @@ -1882,12 +2095,13 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t int match_cap = 0; for (int i = 0; i < recs->count; i++) { const k8s_record_t *wl = &recs->items[i]; - if (wl->kind == K8S_REC_WORKLOAD || wl->kind == K8S_REC_POD) { - for (int p = 0; p < wl->pair_n; p++) { - (void)k8s_posting_add(index, recs->pairs[wl->pair_off + p], i); - } + const cbm_gbuf_node_t *wn = cbm_gbuf_find_by_id(ctx->gbuf, wl->node_id); + if ((wl->kind == K8S_REC_WORKLOAD || wl->kind == K8S_REC_POD) && wn && wn->file_path) { + k8s_sel_job_t j = {index, recs, wl, i, NULL}; + k8s_each_scope(bundles, wn->file_path, false, k8s_sel_post, &j); } } + k8s_ids_t cand = {0}; int edges = 0; for (int i = 0; i < recs->count; i++) { /* a Service, PodDisruptionBudget or NetworkPolicy selector; one with @@ -1899,22 +2113,20 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t svc->pair_n == 0 || svc->has_expr) { continue; } - /* Candidates: the workloads carrying the Service's rarest pair. */ - const k8s_posting_t *rarest = NULL; - for (int p = 0; p < svc->pair_n; p++) { - const k8s_posting_t *posting = cbm_ht_get(index, recs->pairs[svc->pair_off + p]); - if (!posting) { - rarest = NULL; - break; - } - if (!rarest || posting->count < rarest->count) { - rarest = posting; - } - } + /* Candidates: per bundle of the selector, the workloads there + * carrying its rarest pair. */ const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_id(ctx->gbuf, svc->node_id); + cand.count = 0; + if (sn && sn->file_path) { + k8s_sel_job_t j = {index, recs, svc, i, &cand}; + k8s_each_scope(bundles, sn->file_path, false, k8s_sel_rarest, &j); + } + k8s_ids_sort_unique(&cand); + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)cand.count, memory_order_relaxed); int n = 0; - for (int c = 0; sn && rarest && c < rarest->count; c++) { - const k8s_record_t *wl = &recs->items[rarest->items[c]]; + for (int c = 0; c < cand.count; c++) { + int rec = (int)cand.items[c]; + const k8s_record_t *wl = &recs->items[rec]; bool all = k8s_share_bundle(bundles, ctx, svc, wl); for (int p = 0; all && p < svc->pair_n; p++) { all = k8s_has_pair(recs, wl, recs->pairs[svc->pair_off + p]); @@ -1922,15 +2134,15 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t const cbm_gbuf_node_t *wn = all ? cbm_gbuf_find_by_id(ctx->gbuf, wl->node_id) : NULL; if (wn && wn->name && wn->file_path && k8s_grow((void **)&matches, &match_cap, n, sizeof(*matches))) { - matches[n++] = (k8s_match_t){ - wn->name, wl->ns, sn->file_path && strcmp(wn->file_path, sn->file_path) == 0, - rarest->items[c]}; + matches[n++] = + (k8s_match_t){wn->name, wl->ns, strcmp(wn->file_path, sn->file_path) == 0, rec}; } } edges += k8s_draw_matches(ctx, recs, svc, matches, n); } cbm_free(CBM_MEM_CLASS_OTHER, matches); - cbm_ht_foreach(index, k8s_posting_free, NULL); + cbm_free(CBM_MEM_CLASS_OTHER, cand.items); + cbm_ht_foreach(index, k8s_posting_free_owned, NULL); cbm_ht_free(index); if (edges > 0) { cbm_log_info("pass.k8s.selectors", "linked", itoa_k8s(edges)); diff --git a/src/pipeline/pass_operator.c b/src/pipeline/pass_operator.c index 0c0bdc468..e4b96e900 100644 --- a/src/pipeline/pass_operator.c +++ b/src/pipeline/pass_operator.c @@ -28,6 +28,7 @@ #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" +#include #include #include #include @@ -37,6 +38,9 @@ enum { OP_GROW_MIN = 16, OP_IDENT_MAX = 128 }; +/* Candidates examined by the resolvers below (pipeline_internal.h). */ +_Atomic uint64_t g_op_candidates = 0; + /* Read a whole file into a core-owned NUL-terminated buffer, NULL on error * or past the indexer's file-size policy. */ static char *op_read_file(const char *path) { @@ -251,6 +255,82 @@ static bool op_import_dir(const op_modules_t *m, const char *import, char *out, return true; } +/* ── Indexes: Go types by package dir, module roots ───────────────────── */ + +/* Graph-owned nodes. */ +typedef struct { + const cbm_gbuf_node_t **items; + int count; + int cap; +} op_nodes_t; + +static void op_nodes_push(op_nodes_t *v, const cbm_gbuf_node_t *n) { + if (v->count == v->cap) { + int cap = v->cap ? v->cap * PAIR_LEN : OP_GROW_MIN; + const cbm_gbuf_node_t **grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, (void *)v->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + v->items = grown; + v->cap = cap; + } + v->items[v->count++] = n; +} + +static void op_free_nodes_entry(const char *key, void *value, void *ud) { + (void)ud; + op_nodes_t *v = value; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +/* "\x1f": a directory (or module root) with a name. */ +static void op_key(char *out, size_t out_sz, const char *a, const char *b) { + snprintf(out, out_sz, "%s\x1f%s", a, b); +} + +/* "\x1f" -> the Struct / Class / Type nodes Name names in dir. */ +static CBMHashTable *op_types_by_dir(const cbm_gbuf_t *gb) { + CBMHashTable *ix = cbm_ht_create(0); + static const char *const labels[] = {"Struct", "Class", "Type"}; + for (size_t l = 0; ix && l < sizeof(labels) / sizeof(labels[0]); l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(gb, labels[l], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count; i++) { + if (!nodes[i]->file_path || !nodes[i]->name) { + continue; + } + char dir[CBM_SZ_1K]; + char key[PAIR_LEN * CBM_SZ_1K]; + op_dirname(nodes[i]->file_path, dir, sizeof(dir)); + op_key(key, sizeof(key), dir, nodes[i]->name); + op_nodes_t *v = cbm_ht_get(ix, key); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!v || !owned) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, owned); + continue; + } + cbm_ht_set(ix, owned, v); + } + op_nodes_push(v, nodes[i]); + } + } + return ix; +} + +typedef struct { + const op_modules_t *mods; + CBMHashTable *dir_types; /* op_types_by_dir */ +} op_index_t; + /* ── Controllers: For / Owns / Watches / source.Kind ──────────────────── */ static int op_line_of(const char *text, const char *at) { @@ -419,30 +499,46 @@ static bool op_dir_is_import(const char *dir, const char *path) { } /* HANDLES from `fn` to the Go type `type` resolved as Go resolves it: the - * package the file imports as `qual`, or (unqualified) the file's own. */ -static int op_handle(cbm_gbuf_t *gb, const op_modules_t *mods, const cbm_gbuf_node_t *fn, + * package the file imports as `qual`, or (unqualified) the file's own -- one + * directory, looked up in the type index. Only an import no module of the + * repository has falls back to the same-named types whose directory ends + * the import path. */ +static int op_handle(cbm_gbuf_t *gb, const op_index_t *ix, const cbm_gbuf_node_t *fn, const char *file, const char *text, const char *qual, const char *type, const char *role) { char import[CBM_SZ_1K] = ""; - char own_dir[CBM_SZ_1K]; char pkg_dir[CBM_SZ_1K] = ""; - op_dirname(file, own_dir, sizeof(own_dir)); bool in_repo = false; if (qual[0]) { op_import_path(text, qual, import, sizeof(import)); if (!import[0]) { return 0; } - in_repo = op_import_dir(mods, import, pkg_dir, sizeof(pkg_dir)); + in_repo = op_import_dir(ix->mods, import, pkg_dir, sizeof(pkg_dir)); + } else { + op_dirname(file, pkg_dir, sizeof(pkg_dir)); + } + char props[CBM_SZ_128]; + snprintf(props, sizeof(props), "{\"via\":\"controller\",\"role\":\"%s\"}", role); + int linked = 0; + if (!qual[0] || in_repo) { + char key[PAIR_LEN * CBM_SZ_1K]; + op_key(key, sizeof(key), pkg_dir, type); + const op_nodes_t *v = cbm_ht_get(ix->dir_types, key); + atomic_fetch_add_explicit(&g_op_candidates, (uint64_t)(v ? v->count : 0), + memory_order_relaxed); + for (int i = 0; v && i < v->count; i++) { + cbm_gbuf_insert_edge(gb, fn->id, v->items[i]->id, "HANDLES", props); + linked++; + } + return linked; } const cbm_gbuf_node_t **types = NULL; int tn = 0; if (cbm_gbuf_find_by_name(gb, type, &types, &tn) != 0) { return 0; } - char props[CBM_SZ_128]; - snprintf(props, sizeof(props), "{\"via\":\"controller\",\"role\":\"%s\"}", role); - int linked = 0; + atomic_fetch_add_explicit(&g_op_candidates, (uint64_t)(tn), memory_order_relaxed); for (int i = 0; i < tn; i++) { const cbm_gbuf_node_t *t = types[i]; if (!t->label || !t->file_path || @@ -452,10 +548,7 @@ static int op_handle(cbm_gbuf_t *gb, const op_modules_t *mods, const cbm_gbuf_no } char dir[CBM_SZ_1K]; op_dirname(t->file_path, dir, sizeof(dir)); - bool same = !qual[0] ? strcmp(dir, own_dir) == 0 - : in_repo ? strcmp(dir, pkg_dir) == 0 - : op_dir_is_import(dir, import); - if (same) { + if (op_dir_is_import(dir, import)) { cbm_gbuf_insert_edge(gb, fn->id, t->id, "HANDLES", props); linked++; } @@ -464,7 +557,7 @@ static int op_handle(cbm_gbuf_t *gb, const op_modules_t *mods, const cbm_gbuf_no } /* Every `&[pkg.]Type{` in the arguments of a controller builder call. */ -static int op_controller_calls(cbm_gbuf_t *gb, CBMHashTable *by_file, const op_modules_t *mods, +static int op_controller_calls(cbm_gbuf_t *gb, CBMHashTable *by_file, const op_index_t *ix, const char *file, const char *text) { /* method name + "(", called on a receiver outside a comment: the previous * code char is '.', and Kind's receiver is the `source` package. */ @@ -529,7 +622,7 @@ static int op_controller_calls(cbm_gbuf_t *gb, CBMHashTable *by_file, const op_m snprintf(type, sizeof(type), "%.*s", (int)(q - last), last); snprintf(qual, sizeof(qual), "%.*s", last > first ? (int)(last - first - SKIP_ONE) : 0, first); - linked += op_handle(gb, mods, fn, file, text, qual, type, calls[c].role); + linked += op_handle(gb, ix, fn, file, text, qual, type, calls[c].role); } } } @@ -571,37 +664,42 @@ static void op_strs_push(op_strs_t *v, const char *s) { } } -/* group -> the package directories that declare it (owned keys and lists). */ -static void op_note_group(CBMHashTable *groups, const char *group, const char *dir) { - if (!group[0]) { +/* group -> the package directories that declare it (owned keys and lists); + * `seen` holds "group\x1fdir" (owned keys) so a repeat is one lookup. */ +static void op_note_group(CBMHashTable *groups, CBMHashTable *seen, const char *group, + const char *dir) { + char key[PAIR_LEN * CBM_SZ_1K]; + op_key(key, sizeof(key), group, dir); + if (!group[0] || cbm_ht_get(seen, key)) { return; } + char *seen_key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!seen_key) { + return; + } + cbm_ht_set(seen, seen_key, seen_key); op_strs_t *dirs = cbm_ht_get(groups, group); if (!dirs) { dirs = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*dirs)); - char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, group); - if (!dirs || !key) { + char *gkey = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, group); + if (!dirs || !gkey) { cbm_free(CBM_MEM_CLASS_OTHER, dirs); - cbm_free(CBM_MEM_CLASS_OTHER, key); - return; - } - cbm_ht_set(groups, key, dirs); - } - for (int i = 0; i < dirs->count; i++) { - if (strcmp(dirs->items[i], dir) == 0) { + cbm_free(CBM_MEM_CLASS_OTHER, gkey); return; } + cbm_ht_set(groups, gkey, dirs); } op_strs_push(dirs, dir); } -static void op_scan_groups(CBMHashTable *groups, const char *file, const char *text) { +static void op_scan_groups(CBMHashTable *groups, CBMHashTable *seen, const char *file, + const char *text) { char dir[CBM_SZ_1K]; op_dirname(file, dir, sizeof(dir)); for (const char *p = strstr(text, "+groupName="); p; p = strstr(p + SKIP_ONE, "+groupName=")) { char g[CBM_SZ_256]; op_token(p + OP_SLEN("+groupName="), g, sizeof(g)); - op_note_group(groups, g, dir); + op_note_group(groups, seen, g, dir); } for (const char *p = strstr(text, "GroupName"); p; p = strstr(p + SKIP_ONE, "GroupName")) { const char *q = p + OP_SLEN("GroupName"); @@ -618,11 +716,17 @@ static void op_scan_groups(CBMHashTable *groups, const char *file, const char *t if (*q == '"') { char g[CBM_SZ_256]; op_token(q + SKIP_ONE, g, sizeof(g)); - op_note_group(groups, g, dir); + op_note_group(groups, seen, g, dir); } } } +static void op_free_seen(const char *key, void *value, void *ud) { + (void)value; + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + static void op_free_groups(const char *key, void *value, void *ud) { (void)ud; op_strs_t *dirs = value; @@ -651,8 +755,10 @@ static bool op_prop(const char *props, const char *key, char *out, size_t out_sz return true; } -/* CRD -> the Go type declared under its group, in a package marking it. */ -static int op_link_crd_types(cbm_gbuf_t *gb, CBMHashTable *groups) { +/* CRD -> the Go type declared under its group, in every package marking it: + * an API group is one identity, and its CRDs and Go types routinely sit in + * different modules (cluster-api's api/, kubermatic's sdk/). */ +static int op_link_crd_types(cbm_gbuf_t *gb, const op_index_t *ix, CBMHashTable *groups) { const cbm_gbuf_node_t **res = NULL; int count = 0; int linked = 0; @@ -662,29 +768,25 @@ static int op_link_crd_types(cbm_gbuf_t *gb, CBMHashTable *groups) { for (int i = 0; i < count; i++) { char group[CBM_SZ_256]; char kind[CBM_SZ_256]; - if (!op_prop(res[i]->properties_json, "crd_group", group, sizeof(group)) || + if (!res[i]->file_path || + !op_prop(res[i]->properties_json, "crd_group", group, sizeof(group)) || !op_prop(res[i]->properties_json, "crd_kind", kind, sizeof(kind))) { continue; } + char key[PAIR_LEN * CBM_SZ_1K]; const op_strs_t *dirs = cbm_ht_get(groups, group); - const cbm_gbuf_node_t **types = NULL; - int tn = 0; - if (!dirs || cbm_gbuf_find_by_name(gb, kind, &types, &tn) != 0) { + if (!dirs) { continue; } - for (int t = 0; t < tn; t++) { - if (!types[t]->label || strcmp(types[t]->label, "Struct") != 0 || - !types[t]->file_path) { - continue; - } - char dir[CBM_SZ_1K]; - op_dirname(types[t]->file_path, dir, sizeof(dir)); - for (int d = 0; d < dirs->count; d++) { - if (strcmp(dirs->items[d], dir) == 0) { - cbm_gbuf_insert_edge(gb, res[i]->id, types[t]->id, "USAGE", + atomic_fetch_add_explicit(&g_op_candidates, (uint64_t)dirs->count, memory_order_relaxed); + for (int d = 0; d < dirs->count; d++) { + op_key(key, sizeof(key), dirs->items[d], kind); + const op_nodes_t *v = cbm_ht_get(ix->dir_types, key); + for (int t = 0; v && t < v->count; t++) { + if (v->items[t]->label && strcmp(v->items[t]->label, "Struct") == 0) { + cbm_gbuf_insert_edge(gb, res[i]->id, v->items[t]->id, "USAGE", "{\"ref\":\"crd-type\"}"); linked++; - break; } } } @@ -886,6 +988,7 @@ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { qsort((void *)files, (size_t)nfiles, sizeof(*files), op_cmp_path); CBMHashTable *by_file = op_functions_by_file(gb); CBMHashTable *groups = cbm_ht_create(0); + CBMHashTable *seen = cbm_ht_create(0); op_modules_t mods = {0}; for (int i = 0; i < nfiles; i++) { const char *slash = strrchr(files[i]->file_path, '/'); @@ -899,9 +1002,11 @@ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { } } } + op_index_t ix = {.mods = &mods, .dir_types = op_types_by_dir(gb)}; int handles = 0; int embeds = 0; - for (int i = 0; by_file && groups && i < nfiles; i++) { + bool ready = by_file && groups && seen && ix.dir_types; + for (int i = 0; ready && i < nfiles; i++) { if (!op_is_go(files[i]->file_path) || cbm_pipeline_check_cancel(ctx)) { continue; } @@ -912,20 +1017,28 @@ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { continue; } if (strstr(text, "controller-runtime") || strstr(text, "source.Kind(")) { - handles += op_controller_calls(gb, by_file, &mods, files[i]->file_path, text); + handles += op_controller_calls(gb, by_file, &ix, files[i]->file_path, text); } - op_scan_groups(groups, files[i]->file_path, text); + op_scan_groups(groups, seen, files[i]->file_path, text); if (strstr(text, "//go:embed ")) { embeds += op_embed(gb, files[i], text, files, nfiles); } cbm_free(CBM_MEM_CLASS_OTHER, text); } - int crd_types = groups ? op_link_crd_types(gb, groups) : 0; + int crd_types = ready ? op_link_crd_types(gb, &ix, groups) : 0; + if (ix.dir_types) { + cbm_ht_foreach(ix.dir_types, op_free_nodes_entry, NULL); + cbm_ht_free(ix.dir_types); + } op_modules_free(&mods); if (groups) { cbm_ht_foreach(groups, op_free_groups, NULL); cbm_ht_free(groups); } + if (seen) { + cbm_ht_foreach(seen, op_free_seen, NULL); + cbm_ht_free(seen); + } if (by_file) { cbm_ht_foreach(by_file, op_free_spans, NULL); cbm_ht_free(by_file); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 19a9d3a08..7b20b2787 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1118,6 +1118,13 @@ extern _Atomic uint64_t g_importance_name_visits; * closure-delta route actually took that path instead of passing vacuously. */ extern _Atomic uint64_t g_importance_store_rows; +/* Work counters of the infra resolvers, gated by the complexity suite: + * candidate objects the K8s pass examines for references, selectors and + * CRDs (pass_k8s.c), and candidate Go types / package dirs the operator pass + * examines (pass_operator.c). Independent copies must grow them linearly. */ +extern _Atomic uint64_t g_k8s_candidates; +extern _Atomic uint64_t g_op_candidates; + /* ── Incremental pipeline (pipeline_incremental.c) ───────────────── */ /* Run incremental re-index on an existing disk DB. diff --git a/tests/fixtures/complexity/Go/controller.go b/tests/fixtures/complexity/Go/controller.go new file mode 100644 index 000000000..bc88e26b7 --- /dev/null +++ b/tests/fixtures/complexity/Go/controller.go @@ -0,0 +1,14 @@ +package op + +import ( + ctrl "sigs.k8s.io/controller-runtime" +) + +type WidgetReconciler struct{} + +func (r *WidgetReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&Widget{}). + Owns(&Gadget{}). + Complete(r) +} diff --git a/tests/fixtures/complexity/Go/crd.yaml b/tests/fixtures/complexity/Go/crd.yaml new file mode 100644 index 000000000..9e1d2174f --- /dev/null +++ b/tests/fixtures/complexity/Go/crd.yaml @@ -0,0 +1,10 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.embedded.example.org +spec: + group: embedded.example.org + names: + kind: Widget + plural: widgets + scope: Namespaced diff --git a/tests/fixtures/complexity/Go/embed.go b/tests/fixtures/complexity/Go/embed.go new file mode 100644 index 000000000..99d9f2142 --- /dev/null +++ b/tests/fixtures/complexity/Go/embed.go @@ -0,0 +1,6 @@ +package op + +import "embed" + +//go:embed crd.yaml +var CRDs embed.FS diff --git a/tests/fixtures/complexity/Go/go.mod b/tests/fixtures/complexity/Go/go.mod new file mode 100644 index 000000000..2e5105206 --- /dev/null +++ b/tests/fixtures/complexity/Go/go.mod @@ -0,0 +1,3 @@ +module example.com/op + +go 1.22 diff --git a/tests/fixtures/complexity/Go/types.go b/tests/fixtures/complexity/Go/types.go new file mode 100644 index 000000000..625acb8e5 --- /dev/null +++ b/tests/fixtures/complexity/Go/types.go @@ -0,0 +1,10 @@ +// +groupName=example.com +package op + +type Widget struct { + Name string +} + +type Gadget struct { + Name string +} diff --git a/tests/fixtures/complexity/Kubernetes/app.yaml b/tests/fixtures/complexity/Kubernetes/app.yaml new file mode 100644 index 000000000..c4d0a839d --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/app.yaml @@ -0,0 +1,117 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web +spec: + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + serviceAccountName: web + volumes: + - name: config + configMap: + name: web-config + containers: + - name: web + image: example.com/web:1.0 + envFrom: + - secretRef: + name: web-env +--- +apiVersion: v1 +kind: Service +metadata: + name: web +spec: + selector: + app: web + ports: + - port: 80 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: web-config +data: + mode: production +--- +apiVersion: v1 +kind: Secret +metadata: + name: web-env +type: Opaque +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: web +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: web +spec: + rules: + - host: web.example.com + http: + paths: + - path: /api + pathType: Prefix + backend: + service: + name: web + port: + number: 80 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: web +spec: + minAvailable: 1 + selector: + matchLabels: + app: web +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: web +spec: + podSelector: + matchLabels: + app: web +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: web +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: web + minReplicas: 1 + maxReplicas: 3 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: web +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: web +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: web +subjects: + - kind: ServiceAccount + name: web diff --git a/tests/fixtures/complexity/Kubernetes/crd.yaml b/tests/fixtures/complexity/Kubernetes/crd.yaml new file mode 100644 index 000000000..e2617023a --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/crd.yaml @@ -0,0 +1,37 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.k8s.example.org +spec: + group: k8s.example.org + names: + kind: Widget + plural: widgets + scope: Namespaced +--- +apiVersion: k8s.example.org/v1 +kind: Widget +metadata: + name: sample +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: web +spec: + ca: + secretName: web-ca +--- +apiVersion: v1 +kind: Secret +metadata: + name: web-ca +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: web-tls +spec: + secretName: web-tls + issuerRef: + name: web diff --git a/tests/fixtures/complexity/Kubernetes/kustomization.yaml b/tests/fixtures/complexity/Kubernetes/kustomization.yaml new file mode 100644 index 000000000..b8ebf3724 --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: shop +resources: + - app.yaml + - crd.yaml diff --git a/tests/test_complexity.c b/tests/test_complexity.c index f20eac16e..16c707f56 100644 --- a/tests/test_complexity.c +++ b/tests/test_complexity.c @@ -67,6 +67,9 @@ * lsp_resolve.h — re-declared here to avoid pulling that header's statics). */ extern _Atomic uint64_t g_lsp_tail_lookups; extern _Atomic uint64_t g_lsp_tail_candidates; +/* Infra resolver candidate counters (pass_k8s.c, pass_operator.c). */ +extern _Atomic uint64_t g_k8s_candidates; +extern _Atomic uint64_t g_op_candidates; /* ── Corpus scale ────────────────────────────────────────────────────── * Small on purpose: the gate reads exponents from ratios, not magnitudes. @@ -230,15 +233,8 @@ static int cx_build_corpus(const char *root, int k_modules) { * k module copies. Distinct paths make distinct modules/QNs, which is all * the independence argument needs. */ for (int lang = 0; lang < CBM_LANG_COUNT; lang++) { - bool templated = false; - for (int t = 0; t < CX_TEMPLATE_COUNT; t++) { - if (CX_TEMPLATES[t].lang == (CBMLanguage)lang) { - templated = true; - } - } - if (templated) { - continue; - } + /* a templated language may add fixtures too (Go: the operator + * corpus); they land in their own fx_ root */ char fixdir[512]; if (!cx_fixture_dir_for((CBMLanguage)lang, fixdir, sizeof(fixdir))) { continue; @@ -342,6 +338,16 @@ static void cx_pass_sink(const char *line) { g_cx_passes[n].ms = atol(ee + 11); } +/* Fixture-provided infra corpora (tests/fixtures/complexity//), scoped + * by their copy root: Kubernetes manifests with a kustomization, and a Go + * operator (controller, API types with +groupName, an embedded CRD). An API + * group/kind is one identity across a repository: copies declaring the same + * one ARE joined (a CRD links to every package marking its group), so the + * fixtures' CRDs use groups no copy's Go code declares -- the guard measures + * accidental growth, not that by-design join. */ +static const char *const CX_INFRA_DIRS[] = {"fx_Kubernetes", "fx_Go"}; +enum { CX_INFRA_COUNT = sizeof(CX_INFRA_DIRS) / sizeof(CX_INFRA_DIRS[0]) }; + typedef struct { int nodes; int edges; @@ -357,6 +363,10 @@ typedef struct { uint64_t fallback_rows; uint64_t imp_nodes; /* symbols scored by the importance pass */ uint64_t imp_name_visits; /* same-name-group members visited by that pass */ + uint64_t k8s_candidates; /* objects the K8s resolvers examined */ + uint64_t op_candidates; /* Go types / dirs the operator pass examined */ + int infra_nodes[CX_INFRA_COUNT]; + int infra_edges[CX_INFRA_COUNT]; double wall_s; } CxMetrics; @@ -379,6 +389,8 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { uint64_t fb0 = cbm_pp_lsp_linear_fallback_rows(); uint64_t in0 = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed); uint64_t iv0 = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed); + uint64_t kc0 = atomic_load_explicit(&g_k8s_candidates, memory_order_relaxed); + uint64_t oc0 = atomic_load_explicit(&g_op_candidates, memory_order_relaxed); atomic_store_explicit(&g_cx_pass_count, 0, memory_order_relaxed); cbm_log_set_sink_ex(cx_pass_sink, CBM_LOG_SINK_TEE); @@ -416,6 +428,8 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->imp_nodes = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed) - in0; out->imp_name_visits = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed) - iv0; + out->k8s_candidates = atomic_load_explicit(&g_k8s_candidates, memory_order_relaxed) - kc0; + out->op_candidates = atomic_load_explicit(&g_op_candidates, memory_order_relaxed) - oc0; cbm_store_t *s = cbm_store_open_path(db_path); if (!s) { @@ -427,6 +441,10 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->lang_nodes[t] = cbm_store_count_nodes_scoped(s, project, CX_TEMPLATES[t].dirname); out->lang_edges[t] = cbm_store_count_edges_scoped(s, project, CX_TEMPLATES[t].dirname); } + for (int i = 0; i < CX_INFRA_COUNT; i++) { + out->infra_nodes[i] = cbm_store_count_nodes_scoped(s, project, CX_INFRA_DIRS[i]); + out->infra_edges[i] = cbm_store_count_edges_scoped(s, project, CX_INFRA_DIRS[i]); + } cbm_store_close(s); return 0; } @@ -529,6 +547,41 @@ TEST(complexity_replicated_modules_scale_linearly) { PASS(); } +/* The infra resolvers (K8s references / selectors / CRDs, the Go operator's + * controller kinds and CRD types) examine candidates by name. Independent + * copies share every name, so a candidate set that is not scoped to the + * bundle / package / module grows with the copy count: the examined count + * goes ~4x per doubling while the edges stay ~2x. Both are gated, on the + * fixture corpora's own subtrees, and neither may be vacuous. */ +TEST(complexity_infra_resolution_is_linear) { + if (cx_measure_pair() != 0) { + FAIL("failed to build/run the complexity corpus pair"); + } + const CxMetrics *a = &g_cx_base; + const CxMetrics *b = &g_cx_doubled; + printf(" k8s_candidates %llu -> %llu op_candidates %llu -> %llu\n", + (unsigned long long)a->k8s_candidates, (unsigned long long)b->k8s_candidates, + (unsigned long long)a->op_candidates, (unsigned long long)b->op_candidates); + ASSERT_GT((long long)a->k8s_candidates, 0); + ASSERT_GT((long long)a->op_candidates, 0); + double kr = cx_ratio((double)b->k8s_candidates, (double)a->k8s_candidates); + double orr = cx_ratio((double)b->op_candidates, (double)a->op_candidates); + printf(" k8s_candidates ratio %.2f op_candidates ratio %.2f\n", kr, orr); + ASSERT_TRUE(kr >= CX_RATIO_LO && kr <= CX_RATIO_HI); + ASSERT_TRUE(orr >= CX_RATIO_LO && orr <= CX_RATIO_HI); + for (int i = 0; i < CX_INFRA_COUNT; i++) { + ASSERT_GT(a->infra_nodes[i], 0); + ASSERT_GT(a->infra_edges[i], 0); + double nr = cx_ratio(b->infra_nodes[i], a->infra_nodes[i]); + double er = cx_ratio(b->infra_edges[i], a->infra_edges[i]); + printf(" %-14s nodes %d -> %d (%.2f) edges %d -> %d (%.2f)\n", CX_INFRA_DIRS[i], + a->infra_nodes[i], b->infra_nodes[i], nr, a->infra_edges[i], b->infra_edges[i], er); + ASSERT_TRUE(nr >= CX_RATIO_LO && nr <= CX_RATIO_HI); + ASSERT_TRUE(er >= CX_RATIO_LO && er <= CX_RATIO_HI); + } + PASS(); +} + /* Σ per-file registry work must be linear in independent copies. This is the * counter that measured ~4x/doubling on the v0.10.x Java path (defs_per_file * tracked defs_total — #1669). Guarded against vacuous passes: the base run @@ -845,6 +898,7 @@ SUITE(complexity) { RUN_TEST(complexity_rescript_error_recovery_lexing_is_linear); RUN_TEST(complexity_replicated_modules_scale_linearly); RUN_TEST(complexity_perfile_registry_work_is_linear); + RUN_TEST(complexity_infra_resolution_is_linear); RUN_TEST(complexity_importance_scoring_is_linear); RUN_TEST(complexity_shared_package_growth_stays_linear); RUN_TEST(complexity_throughput_report_written); From 7210f067d80e527f0ec85c5dd01d5580485f7ab6 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 00:53:05 +0200 Subject: [PATCH 27/52] feat(helm): chart templates call the named templates they include A Helm chart's templates build every object from named templates -- `{{ include "app.labels" . }}`, `{{ template "app.fullname" . }}` -- and none of those calls was in the graph: across 17 infrastructure repositories, 6,918 include/template calls, 0 linked to their `define`. The Go template grammar reads .tpl files (their defines are Functions), but the YAML templates that call them were never scanned for calls. The K8s pass now reads each chart template (.yaml / .yml / .json under a chart's templates/) for literal `include "name"` / `template "name"` calls inside `{{ }}` actions and draws CALLS {strategy: helm_include, via} to the define of that name, resolved as Helm loads templates: in the chart family -- the outermost enclosing chart and every chart below it, whose templates share one namespace -- and in the families its charts depend on locally (file://). An unrelated chart's same-named define is never taken, and a dynamic name draws nothing. The call leaves from the document (Resource) it sits in, else from the template file. A define written in a template itself (deckhouse, gardener) becomes a Function like a .tpl define (same QN scheme, DEFINES from its file), spanning to its matching `end` through nested block actions; calls inside it leave from it. Measured against an independent reading of the charts: chart-template calls 2,816 of 2,816 resolvable (file x name) linked, precision 2,816 of 2,816 (prometheus-community 2,112, deckhouse 516, gardener 148, kubermatic 40); the rest name defines no chart of the repository has (deckhouse's remote helm_lib). Three runs build identical graphs; editing a _helpers.tpl or a template's include keeps incremental equal to a full rebuild. Signed-off-by: Martin Vogel --- src/pipeline/pass_k8s.c | 395 ++++++++++++++++++++++++++++++++++- tests/test_grammar_probe_f.c | 34 +++ 2 files changed, 427 insertions(+), 2 deletions(-) diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 3dea7f979..4005d9079 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -2601,6 +2601,395 @@ static void k8s_prep_worker(int worker_id, void *arg) { cbm_kind_in_set_free_cache(); } +/* ── Helm includes (PHASE 2 P2.1) ──────────────────────────────────── */ + +/* A chart template's `include "name"` / `template "name"` calls a named + * template (`define "name"`, a Function of a .tpl file). Helm loads a chart's + * templates with those of every chart below it into one namespace, so the + * name resolves among the defines of the chart family -- the outermost + * enclosing chart and the charts below it -- and of the families its members + * depend on locally (file://). A dynamic name draws nothing. */ + +/* The nearest chart dir enclosing file `path` ("" for a root chart). */ +static bool k8s_chart_of(k8s_bundles_t *b, const char *path, char *out, size_t out_sz) { + char dir[CBM_SZ_1K]; + k8s_dirname(path, dir, sizeof(dir)); + for (;;) { + if (cbm_ht_get(b->charts, dir)) { + snprintf(out, out_sz, "%s", dir); + return true; + } + if (!dir[0]) { + return false; + } + char parent[CBM_SZ_1K]; + k8s_dirname(dir, parent, sizeof(parent)); + snprintf(dir, sizeof(dir), "%s", parent); + } +} + +/* The outermost chart dir enclosing chart dir `chart` (itself when none). */ +static void k8s_chart_family(k8s_bundles_t *b, const char *chart, char *out, size_t out_sz) { + snprintf(out, out_sz, "%s", chart); + char dir[CBM_SZ_1K]; + snprintf(dir, sizeof(dir), "%s", chart); + while (dir[0]) { + char parent[CBM_SZ_1K]; + k8s_dirname(dir, parent, sizeof(parent)); + snprintf(dir, sizeof(dir), "%s", parent); + if (cbm_ht_get(b->charts, dir)) { + snprintf(out, out_sz, "%s", dir); + } + } +} + +static bool k8s_has_suffix(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t m = strlen(suffix); + return n >= m && strcmp(s + n - m, suffix) == 0; +} + +static bool k8s_ident_char(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +typedef struct { + cbm_pipeline_ctx_t *ctx; + k8s_bundles_t *b; + CBMHashTable *defines; /* "\x1f" -> define Functions */ + CBMHashTable *deps; /* "" -> Chart nodes its members depend on */ + CBMHashTable *resources; /* file -> its Resource nodes */ + CBMHashTable *file_nodes; /* file -> its File node */ + CBMHashTable *yaml_defs; /* template file -> the defines written in it */ + CBMArena arena; /* define QNs */ + const char *file; + const char *family; + k8s_ids_t found; + int edges; + int unresolved; +} k8s_inc_t; + +/* The node a call on `line` of the file leaves from: the innermost document + * (Resource) spanning it, else the File. */ +static int64_t k8s_inc_source(k8s_inc_t *s, int line) { + const cbm_gbuf_node_t *best = NULL; + for (int pass = 0; pass < PAIR_LEN && !best; pass++) { + const k8s_ids_t *res = cbm_ht_get(pass == 0 ? s->yaml_defs : s->resources, s->file); + for (int i = 0; res && i < res->count; i++) { + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_id(s->ctx->gbuf, res->items[i]); + if (n && n->start_line <= line && line <= n->end_line && + (!best || n->end_line - n->start_line < best->end_line - best->start_line)) { + best = n; + } + } + } + if (best) { + return best->id; + } + const k8s_ids_t *f = cbm_ht_get(s->file_nodes, s->file); + return f && f->count > 0 ? f->items[0] : 0; +} + +static void k8s_inc_lookup(k8s_inc_t *s, const char *family, const char *name) { + char key[CBM_SZ_1K]; + snprintf(key, sizeof(key), "%s\x1f%s", family, name); + const k8s_ids_t *v = cbm_ht_get(s->defines, key); + for (int i = 0; v && i < v->count; i++) { + k8s_ids_push(&s->found, v->items[i]); + } +} + +/* One `include "name"` / `template "name"` call on `line`. */ +static void k8s_inc_call(k8s_inc_t *s, const char *name, int line, const char *via) { + s->found.count = 0; + k8s_inc_lookup(s, s->family, name); + const k8s_ids_t *deps = cbm_ht_get(s->deps, s->family); + for (int d = 0; deps && d < deps->count; d++) { + const cbm_gbuf_node_t *dep = cbm_gbuf_find_by_id(s->ctx->gbuf, deps->items[d]); + char chart[CBM_SZ_1K]; + char family[CBM_SZ_1K]; + if (dep && dep->file_path) { + k8s_dirname(dep->file_path, chart, sizeof(chart)); + k8s_chart_family(s->b, chart, family, sizeof(family)); + k8s_inc_lookup(s, family, name); + } + } + k8s_ids_sort_unique(&s->found); + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)s->found.count, memory_order_relaxed); + int64_t src = s->found.count > 0 ? k8s_inc_source(s, line) : 0; + if (src <= 0) { + s->unresolved++; + return; + } + char esc[CBM_SZ_512]; + cbm_json_escape(esc, (int)sizeof(esc), name); + char props[CBM_SZ_1K]; + snprintf(props, sizeof(props), + "{\"callee\":\"%s\",\"strategy\":\"helm_include\",\"via\":\"%s\"}", esc, via); + for (int i = 0; i < s->found.count; i++) { + if (s->found.items[i] != src) { + cbm_gbuf_insert_edge(s->ctx->gbuf, src, s->found.items[i], "CALLS", props); + s->edges++; + } + } +} + +/* Every literal `include "name"` / `template "name"` inside the `{{ }}` + * actions of a template's text, with its line. */ +static void k8s_scan_includes(k8s_inc_t *s, const char *src) { + int line = SKIP_ONE; + const char *at = src; + for (const char *p = strstr(src, "{{"); p;) { + const char *e = strstr(p + PAIR_LEN, "}}"); + if (!e) { + break; + } + for (const char *q = p + PAIR_LEN; q < e; q++) { + const char *via = strncmp(q, "include", K8S_SLEN("include")) == 0 ? "include" + : strncmp(q, "template", K8S_SLEN("template")) == 0 ? "template" + : NULL; + if (!via || k8s_ident_char(q[-SKIP_ONE]) || q[-SKIP_ONE] == '.' || + q[-SKIP_ONE] == '$') { + continue; + } + const char *r = q + strlen(via); + if (r >= e || (*r != ' ' && *r != '\t' && *r != '\n' && *r != '\r')) { + continue; + } + while (r < e && (*r == ' ' || *r == '\t' || *r == '\n' || *r == '\r')) { + r++; + } + const char *ne = + r < e && *r == '"' ? memchr(r + SKIP_ONE, '"', (size_t)(e - r - SKIP_ONE)) : NULL; + if (!ne || ne == r + SKIP_ONE) { + continue; + } + for (; at < q; at++) { + line += *at == '\n'; + } + char name[CBM_SZ_256]; + snprintf(name, sizeof(name), "%.*s", (int)(ne - r - SKIP_ONE), r + SKIP_ONE); + k8s_inc_call(s, name, line, via); + q = ne; + } + p = strstr(e + PAIR_LEN, "{{"); + } +} + +static CBMHashTable *k8s_index_by_file(cbm_pipeline_ctx_t *ctx, const char *label) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!ix || cbm_gbuf_find_by_label(ctx->gbuf, label, &nodes, &count) != 0) { + return ix; + } + for (int i = 0; i < count; i++) { + if (nodes[i]->file_path) { + k8s_index_add(ix, nodes[i]->file_path, nodes[i]->id); + } + } + return ix; +} + +static void k8s_free_index(CBMHashTable *ix) { + if (ix) { + cbm_ht_foreach(ix, k8s_free_ids_entry, NULL); + cbm_ht_free(ix); + } +} + +/* Does the action text at `q` start with keyword `kw`? */ +static bool k8s_action_is(const char *q, const char *kw) { + size_t n = strlen(kw); + return strncmp(q, kw, n) == 0 && !k8s_ident_char(q[n]); +} + +/* A `define "name"` written in a template (outside a .tpl file the grammar + * reads): a Function like a .tpl define, spanning to its matching `end`. */ +static void k8s_yaml_define(k8s_inc_t *s, const char *name, int start, int end) { + char *qn = cbm_fqn_compute(&s->arena, s->ctx->project_name, s->file, name); + if (!qn) { + return; + } + int64_t id = cbm_gbuf_upsert_node(s->ctx->gbuf, "Function", name, qn, s->file, start, end, + "{\"source\":\"helm_template\"}"); + if (id <= 0) { + return; + } + const k8s_ids_t *f = cbm_ht_get(s->file_nodes, s->file); + if (f && f->count > 0) { + cbm_gbuf_insert_edge(s->ctx->gbuf, f->items[0], id, "DEFINES", "{}"); + } + char key[CBM_SZ_1K]; + snprintf(key, sizeof(key), "%s\x1f%s", s->family, name); + k8s_index_add(s->defines, key, id); + k8s_index_add(s->yaml_defs, s->file, id); +} + +/* Every `define "name"` ... `end` of a template's text: block actions (if, + * range, with, block, define) nest until their `end`. */ +static void k8s_scan_defines(k8s_inc_t *s, const char *src) { + int line = SKIP_ONE; + const char *at = src; + int depth = 0; + int start = 0; + char name[CBM_SZ_256] = ""; + for (const char *p = strstr(src, "{{"); p;) { + const char *e = strstr(p + PAIR_LEN, "}}"); + if (!e) { + break; + } + for (; at < p; at++) { + line += *at == '\n'; + } + const char *q = p + PAIR_LEN; + q += *q == '-'; + while (q < e && (*q == ' ' || *q == '\t' || *q == '\n' || *q == '\r')) { + q++; + } + if (depth == 0 && k8s_action_is(q, "define")) { + const char *r = q + K8S_SLEN("define"); + while (r < e && (*r == ' ' || *r == '\t')) { + r++; + } + const char *ne = + r < e && *r == '"' ? memchr(r + SKIP_ONE, '"', (size_t)(e - r - SKIP_ONE)) : NULL; + if (ne && ne > r + SKIP_ONE) { + snprintf(name, sizeof(name), "%.*s", (int)(ne - r - SKIP_ONE), r + SKIP_ONE); + depth = SKIP_ONE; + start = line; + } + } else if (depth > 0 && (k8s_action_is(q, "if") || k8s_action_is(q, "range") || + k8s_action_is(q, "with") || k8s_action_is(q, "block") || + k8s_action_is(q, "define"))) { + depth++; + } else if (depth > 0 && k8s_action_is(q, "end")) { + depth--; + if (depth == 0) { + for (; at < e; at++) { + line += *at == '\n'; + } + k8s_yaml_define(s, name, start, line); + } + } + p = strstr(e + PAIR_LEN, "{{"); + } +} + +/* A chart template file: .yaml / .yml / .json under a chart's templates/; + * fills its chart's family. */ +static bool k8s_template_family(k8s_bundles_t *b, const char *rel, char *family, size_t sz) { + char chart[CBM_SZ_1K]; + if (!rel || + !(k8s_has_suffix(rel, ".yaml") || k8s_has_suffix(rel, ".yml") || + k8s_has_suffix(rel, ".json")) || + !k8s_chart_of(b, rel, chart, sizeof(chart))) { + return false; + } + size_t cl = strlen(chart); + const char *sub = cl ? rel + cl + SKIP_ONE : rel; + if (strncmp(sub, "templates/", K8S_SLEN("templates/")) != 0) { + return false; + } + k8s_chart_family(b, chart, family, sz); + return true; +} + +/* Index the defines and dependencies, then scan every template of a chart. */ +static void k8s_includes_run(k8s_inc_t *s, const cbm_file_info_t *files, int file_count) { + /* defines by chart family: Functions of the charts' Go template files */ + const cbm_gbuf_node_t **fns = NULL; + int fn_n = 0; + if (cbm_gbuf_find_by_label(s->ctx->gbuf, "Function", &fns, &fn_n) == 0) { + for (int i = 0; i < fn_n; i++) { + char chart[CBM_SZ_1K]; + char family[CBM_SZ_1K]; + char key[CBM_SZ_1K]; + const char *fp = fns[i]->file_path; + if (!fp || !fns[i]->name || + !(k8s_has_suffix(fp, ".tpl") || k8s_has_suffix(fp, ".gotmpl")) || + !k8s_chart_of(s->b, fp, chart, sizeof(chart))) { + continue; + } + k8s_chart_family(s->b, chart, family, sizeof(family)); + snprintf(key, sizeof(key), "%s\x1f%s", family, fns[i]->name); + k8s_index_add(s->defines, key, fns[i]->id); + } + } + /* local dependencies: Chart -DEPENDS_ON-> Chart, by the source's family */ + const cbm_gbuf_edge_t **deps = NULL; + int dep_n = 0; + if (cbm_gbuf_find_edges_by_type(s->ctx->gbuf, "DEPENDS_ON", &deps, &dep_n) == 0) { + for (int i = 0; i < dep_n; i++) { + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(s->ctx->gbuf, deps[i]->source_id); + const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_id(s->ctx->gbuf, deps[i]->target_id); + if (!src || !tgt || !src->label || !tgt->label || strcmp(src->label, "Chart") != 0 || + strcmp(tgt->label, "Chart") != 0 || !src->file_path || !tgt->file_path) { + continue; + } + char chart[CBM_SZ_1K]; + char family[CBM_SZ_1K]; + k8s_dirname(src->file_path, chart, sizeof(chart)); + k8s_chart_family(s->b, chart, family, sizeof(family)); + k8s_index_add(s->deps, family, tgt->id); + } + } + /* every template of a chart: its own defines, then its includes */ + for (int pass = 0; pass < PAIR_LEN; pass++) { + for (int i = 0; i < file_count; i++) { + const char *rel = files[i].rel_path; + char family[CBM_SZ_1K]; + if (!k8s_template_family(s->b, rel, family, sizeof(family))) { + continue; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", s->ctx->repo_path, rel); + int len = 0; + char *src = k8s_read_file(path, &len); + if (!src) { + continue; + } + s->file = rel; + s->family = family; + if (pass == 0) { + k8s_scan_defines(s, src); + } else { + k8s_scan_includes(s, src); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + } +} + +static void k8s_link_includes(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, + const cbm_file_info_t *files, int file_count) { + if (!b->charts || !ctx->repo_path) { + return; + } + k8s_inc_t s = {.ctx = ctx, + .b = b, + .defines = cbm_ht_create(0), + .deps = cbm_ht_create(0), + .resources = k8s_index_by_file(ctx, "Resource"), + .file_nodes = k8s_index_by_file(ctx, "File"), + .yaml_defs = cbm_ht_create(0)}; + cbm_arena_init(&s.arena); + if (s.defines && s.deps && s.resources && s.file_nodes && s.yaml_defs) { + k8s_includes_run(&s, files, file_count); + } + cbm_arena_destroy(&s.arena); + cbm_free(CBM_MEM_CLASS_OTHER, s.found.items); + k8s_free_index(s.defines); + k8s_free_index(s.deps); + k8s_free_index(s.resources); + k8s_free_index(s.file_nodes); + k8s_free_index(s.yaml_defs); + if (s.edges + s.unresolved > 0) { + cbm_log_info("pass.k8s.includes", "edges", itoa_k8s(s.edges), "unresolved", + itoa_k8s(s.unresolved)); + } +} + int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { cbm_log_info("pass.start", "pass", "k8s", "files", itoa_k8s(file_count)); @@ -2699,10 +3088,12 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_crds(ctx, &bundles, &facts); k8s_facts_free(&facts); k8s_records_free(&recs); + /* Link `file://` chart dependencies now that every Chart.yaml is read, + * then the templates' includes, which see those dependencies' defines. */ + k8s_link_local_deps(ctx, &local_deps); + k8s_link_includes(ctx, &bundles, files, file_count); k8s_bundles_free(&bundles); k8s_kusts_free(&kusts); - /* Link `file://` chart dependencies now that every Chart.yaml is read. */ - k8s_link_local_deps(ctx, &local_deps); cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count)); (void)helm_count; diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 8dbce1871..400293992 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1655,6 +1655,39 @@ TEST(probe_mcp_trace_topology) { PASS(); } +/* ── Helm symbolic (Phase 2) ────────────────────────────────────────── */ + +/* A template's `include "name"` calls the define of that name in its chart + * family -- the chart and its subcharts share one template namespace -- never + * a same-named define of an unrelated chart; a define may also be written in + * the template itself (deckhouse, gardener). */ +TEST(probe_helm_include_resolves_in_the_chart_family) { + static const GpfFile files[] = { + {"app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 0.1.0\n"}, + {"app/templates/_helpers.tpl", + "{{- define \"app.fullname\" -}}app{{- end }}\n" + "{{- define \"app.labels\" -}}tier: web{{- end }}\n"}, + {"app/templates/deployment.yaml", + "{{- define \"web.resources\" }}\ncpu: 1\n{{- if .Values.big }}\nmemory: 2Gi\n{{- end }}\n" + "{{- end }}\n---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n labels:\n" + " {{- include \"app.labels\" . | nindent 4 }}\n" + " owner: {{ include \"app.fullname\" . }}\n" + " sub: {{ template \"sub.name\" . }}\n" + " dyn: {{ include (printf \"%s.x\" .Chart.Name) . }}\n" + "spec:\n resources: {{ include \"web.resources\" . | nindent 4 }}\n"}, + {"app/charts/sub/Chart.yaml", "apiVersion: v2\nname: sub\nversion: 0.1.0\n"}, + {"app/charts/sub/templates/_helpers.tpl", "{{- define \"sub.name\" -}}sub{{- end }}\n"}, + {"other/Chart.yaml", "apiVersion: v2\nname: other\nversion: 0.1.0\n"}, + {"other/templates/_helpers.tpl", "{{- define \"app.fullname\" -}}other{{- end }}\n"}}; + ASSERT_EQ(gpf_edges_into_file(files, 7, "CALLS", "app/templates/_helpers.tpl"), 2); + ASSERT_EQ(gpf_edges_into_file(files, 7, "CALLS", "app/charts/sub/templates/_helpers.tpl"), 1); + ASSERT_EQ(gpf_edges_into_file(files, 7, "CALLS", "other/templates/_helpers.tpl"), 0); + /* a define written in the template itself */ + ASSERT_EQ(gpf_edges_into_file(files, 7, "CALLS", "app/templates/deployment.yaml"), 1); + PASS(); +} + /* A CRD vendored twice (gardener: example/seed-crds and a chart): a custom * resource is an instance of the copy deployed with it; only when no copy * shares its bundle does it stand for all of them. */ @@ -3268,6 +3301,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_ingress_and_httproute_expose_routes); RUN_TEST(probe_mcp_search_filters_kind_and_namespace); RUN_TEST(probe_mcp_trace_topology); + RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); RUN_TEST(probe_k8s_kustomize_namespace_is_effective); From 73cd4b88ab1ece1b34f05ff84ea3aaad7f1f44c3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 01:17:39 +0200 Subject: [PATCH 28/52] feat(helm): templated manifests become objects with symbolic names Most of an infrastructure repository's Kubernetes objects are written as templates -- Helm charts, Go-template addons -- and those are YAML only once rendered: `{{- if .Values.x }}` opens the document and `name: {{ include "app.fullname" . }}` is a flow mapping to a YAML parser. Across 17 repositories 92 % of the objects written in chart templates (1,127 of 1,233) never became Resources. A templated manifest is now extracted from its skeleton: the text with every action rewritten line for line, so document lines stay exact. A line holding nothing but actions is blanked; an action inside text becomes a symbolic scalar -- `` for an include/template call, the first `.`/`$` path otherwise (`<.Values.image.tag>`), else ``. Objects get the chart-scoped symbolic names IDENTITY.md describes (`-config`), and a reference or selector written with the same expression meets them inside the chart. A templated namespace is the deploy's; a templated apiVersion leaves the group unknown; a templated kind is unidentifiable and skipped. Measured on 17 repositories: objects written in chart templates as Resources 8 % -> 94 % (1,166 of 1,233: prometheus-community 0 -> 343 of 357, microservices-demo 0 -> 84 of 84, deckhouse 431 of 479, gardener 149 of 151, kubermatic 133 of 136); kubermatic's templated addon objects 88 % -> 97 % (708 of 727). References and selectors touching templated objects, 201 sampled across 6 repositories and judged against the original template text: 201 correct (4 by hand, quoting the sampler did not normalise). The plain-manifest gates, include resolution and selectors are unchanged; three runs build identical graphs; editing a template keeps incremental equal to a full rebuild. Signed-off-by: Martin Vogel --- internal/cbm/extract_k8s.c | 19 ++++- src/pipeline/pass_k8s.c | 148 ++++++++++++++++++++++++++++++++++- tests/test_grammar_probe_f.c | 25 ++++++ 3 files changed, 187 insertions(+), 5 deletions(-) diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 636db83b9..d734ed95c 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -744,7 +744,8 @@ static void k8s_emit_ref(k8s_walk_t *w, const char *kind, const char *ns, const CBMK8sFact f = {.fact = CBM_K8S_FACT_REF, .def_index = w->def_index, .kind = cbm_arena_strdup(a, kind), - .ns = ns && ns[0] && !strstr(ns, "{{") ? cbm_arena_strdup(a, ns) : NULL, + .ns = ns && ns[0] && ns[0] != '<' && !strstr(ns, "{{") ? cbm_arena_strdup(a, ns) + : NULL, .name = cbm_arena_strdup(a, name), .via = via, .edge = edge}; @@ -912,7 +913,8 @@ static void k8s_emit_route(k8s_walk_t *w, const char *host, const char *path, co CBMK8sFact f = {.fact = CBM_K8S_FACT_ROUTE, .def_index = w->def_index, .kind = name ? "Service" : NULL, - .ns = ns && ns[0] && !strstr(ns, "{{") ? cbm_arena_strdup(a, ns) : NULL, + .ns = ns && ns[0] && ns[0] != '<' && !strstr(ns, "{{") ? cbm_arena_strdup(a, ns) + : NULL, .name = name && !strstr(name, "{{") ? cbm_arena_strdup(a, name) : NULL, .via = via, .edge = "INFRA_MAPS", @@ -1134,6 +1136,10 @@ static const char *k8s_file_base(CBMArena *a, const char *project, const char *r // group ("v1"), "?" when the document names none. The version is not part of // an object's identity. static void k8s_api_group(const char *api_version, char *out, size_t out_sz) { + if (api_version[0] == '<') { // templated (a skeleton's symbolic `<...>`) + snprintf(out, out_sz, "?"); + return; + } const char *slash = strchr(api_version, '/'); if (slash) { snprintf(out, out_sz, "%.*s", (int)(slash - api_version), api_version); @@ -1167,10 +1173,15 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { memset(&id, 0, sizeof(id)); extract_k8s_scalars(ctx, mapping, &id); - // Skip malformed manifests (no kind or no metadata.name) - if (id.kind[0] == '\0' || id.name[0] == '\0') { + // Skip malformed manifests (no kind or no metadata.name), and a + // templated kind (a skeleton's symbolic `<...>`): unidentifiable. + if (id.kind[0] == '\0' || id.name[0] == '\0' || id.kind[0] == '<') { continue; } + // A templated namespace (`<.Release.Namespace>`) is the deploy's. + if (id.ns[0] == '<') { + id.ns[0] = '\0'; + } char group[K8S_BUF_SIZE]; k8s_api_group(id.api_version, group, sizeof(group)); diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 4005d9079..83ac8185f 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -2545,7 +2545,144 @@ typedef struct { _Atomic int next; } k8s_prep_job_t; -enum { K8S_CHUNK = 1024 }; +enum { K8S_CHUNK = 1024, K8S_TS_PAD = 16 }; + +/* ── Templated manifests (PHASE 2 P2.3) ────────────────────────────── */ + +/* A Helm chart template or Go-template addon is YAML only once rendered: + * `{{- if .Values.x }}` opens the document, `name: {{ include "app.fullname" + * . }}` is a flow mapping to a YAML parser. The skeleton is the text with + * every action rewritten, line for line (document lines stay exact): a line + * holding nothing but actions is blanked; an action inside text becomes a + * symbolic scalar -- `` for an include/template call, the + * first `.`/`$` path otherwise (`<.Values.image.tag>`), else ``. Objects + * then get the chart-scoped symbolic names IDENTITY.md describes, and a + * reference written with the same expression meets them in the chart. */ + +static bool k8s_tpl_ident(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* Does the action text at q (before e) call a named template: `include` / + * `template` then a quoted name? Fills `out` with ``. */ +static bool k8s_action_call(const char *a, const char *q, const char *e, char *out, size_t out_sz) { + size_t kl = strncmp(q, "include", K8S_SLEN("include")) == 0 ? K8S_SLEN("include") + : strncmp(q, "template", K8S_SLEN("template")) == 0 ? K8S_SLEN("template") + : 0; + if (!kl || q + kl >= e || q[kl] != ' ' || (q > a && k8s_tpl_ident(q[-SKIP_ONE]))) { + return false; + } + const char *r = q + kl; + while (r < e && *r == ' ') { + r++; + } + const char *ne = + r < e && *r == '"' ? memchr(r + SKIP_ONE, '"', (size_t)(e - r - SKIP_ONE)) : NULL; + if (!ne || ne == r + SKIP_ONE) { + return false; + } + snprintf(out, out_sz, "<%.*s>", (int)(ne - r - SKIP_ONE), r + SKIP_ONE); + return true; +} + +/* The symbolic token of the action text [a, e) into out (NUL-terminated). */ +static void k8s_action_token(const char *a, const char *e, char *out, size_t out_sz) { + for (const char *q = a; q < e; q++) { + if (k8s_action_call(a, q, e, out, out_sz)) { + return; + } + } + for (const char *q = a; q < e; q++) { + if ((*q == '.' || *q == '$') && (q == a || q[-SKIP_ONE] == ' ' || q[-SKIP_ONE] == '(')) { + const char *r = q + SKIP_ONE; + while (r < e && (k8s_tpl_ident(*r) || *r == '.' || *r == '$')) { + r++; + } + if (r > q + SKIP_ONE || *q == '$') { + snprintf(out, out_sz, "<%.*s>", (int)(r - q), q); + return; + } + } + } + snprintf(out, out_sz, ""); +} + +/* The end of the logical line starting at ls: its newline, or -- when an + * action starting on it spans lines -- the newline after that action. Sets + * *only_actions when the line holds actions and nothing else but blanks. */ +static const char *k8s_logical_line(const char *ls, const char *end, bool *only_actions) { + const char *le = memchr(ls, '\n', (size_t)(end - ls)); + le = le ? le : end; + bool blank = true; + bool any = false; + for (const char *p = ls; p < le;) { + if (p + SKIP_ONE < le && p[0] == '{' && p[SKIP_ONE] == '{') { + const char *ae = strstr(p + PAIR_LEN, "}}"); + if (!ae || ae >= end) { + blank = false; + break; + } + any = true; + p = ae + PAIR_LEN; + if (p > le) { + const char *nl = memchr(p, '\n', (size_t)(end - p)); + le = nl ? nl : end; + } + continue; + } + blank = blank && (*p == ' ' || *p == '\t' || *p == '\r'); + p++; + } + *only_actions = any && blank; + return le; +} + +/* The skeleton of templated text (cbm_alloc'd, *out_len set); NULL on OOM. */ +static char *k8s_skeleton(const char *src, int len, int *out_len) { + size_t cap = (size_t)len * PAIR_LEN + CBM_SZ_64 + K8S_TS_PAD; + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, cap); + if (!out) { + return NULL; + } + size_t w = 0; + const char *end = src + len; + for (const char *ls = src; ls < end;) { + bool only_actions = false; + const char *le = k8s_logical_line(ls, end, &only_actions); + for (const char *p = ls; p < le;) { + const char *ae = p + SKIP_ONE < le && p[0] == '{' && p[SKIP_ONE] == '{' + ? strstr(p + PAIR_LEN, "}}") + : NULL; + if (!ae || ae >= end) { + if (!only_actions) { + out[w++] = *p; + } + p++; + continue; + } + if (!only_actions) { + char tok[CBM_SZ_256]; + k8s_action_token(p + PAIR_LEN, ae, tok, sizeof(tok)); + size_t tl = strlen(tok); + memcpy(out + w, tok, tl); + w += tl; + } + for (const char *q = p; q < ae; q++) { /* the action's own newlines stay */ + if (*q == '\n') { + out[w++] = '\n'; + } + } + p = ae + PAIR_LEN; + } + if (le < end) { + out[w++] = '\n'; + } + ls = le < end ? le + SKIP_ONE : end; + } + memset(out + w, 0, K8S_TS_PAD); /* lexer lookahead, as k8s_read_file pads */ + *out_len = (int)w; + return out; +} static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { cbm_pipeline_ctx_t *ctx = job->ctx; @@ -2574,6 +2711,15 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { * result: cached results were produced during the parallel YAML * pass and contain no "Resource" definitions. */ out->kind = K8S_MANIFEST; + if (strstr(out->source, "{{")) { + int sk_len = 0; + char *sk = k8s_skeleton(out->source, out->src_len, &sk_len); + if (sk) { + cbm_free(CBM_MEM_CLASS_OTHER, out->source); + out->source = sk; + out->src_len = sk_len; + } + } out->res = cbm_extract_file(out->source, out->src_len, CBM_LANG_K8S, ctx->project_name, rel, CBM_EXTRACT_BUDGET, NULL, NULL); } else { diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 400293992..3f33f2861 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1657,6 +1657,30 @@ TEST(probe_mcp_trace_topology) { /* ── Helm symbolic (Phase 2) ────────────────────────────────────────── */ +/* A chart template is YAML only once rendered: a document opened by + * `{{- if }}`, with a name built by `include`. Its objects are Resources with + * the chart-scoped symbolic name (`-config`), and a reference + * written with the same expression meets them; a templated namespace is the + * deploy's. */ +TEST(probe_helm_templated_objects_are_resources) { + static const GpfFile files[] = { + {"chart/Chart.yaml", "apiVersion: v2\nname: app\nversion: 0.1.0\n"}, + {"chart/templates/_helpers.tpl", "{{- define \"app.fullname\" -}}app{{- end }}\n"}, + {"chart/templates/configmap.yaml", + "{{- if .Values.enabled }}\napiVersion: v1\nkind: ConfigMap\nmetadata:\n" + " name: {{ include \"app.fullname\" . }}-config\n" + " namespace: {{ .Release.Namespace }}\n labels:\n" + " {{- include \"app.labels\" . | nindent 4 }}\n" + "data:\n key: {{ .Values.value | quote }}\n{{- end }}\n"}, + {"chart/templates/deployment.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n" + " name: {{ include \"app.fullname\" . }}\nspec:\n template:\n spec:\n" + " volumes:\n - name: cfg\n configMap:\n" + " name: {{ include \"app.fullname\" . }}-config\n"}}; + ASSERT_EQ(gpf_typed_edges(files, 4, "USAGE", "Resource", "Resource", "\"ref\":\"volume\""), 1); + PASS(); +} + /* A template's `include "name"` calls the define of that name in its chart * family -- the chart and its subcharts share one template namespace -- never * a same-named define of an unrelated chart; a define may also be written in @@ -3301,6 +3325,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_ingress_and_httproute_expose_routes); RUN_TEST(probe_mcp_search_filters_kind_and_namespace); RUN_TEST(probe_mcp_trace_topology); + RUN_TEST(probe_helm_templated_objects_are_resources); RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); RUN_TEST(probe_k8s_selector_within_kustomize_build); From 523e68b13e805a3ba87e991e3130cdb6945490a4 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 01:35:11 +0200 Subject: [PATCH 29/52] feat(k8s): GitOps objects DEPLOYS what they deploy A Flux Kustomization, an Argo Application and a HelmRelease each name what they put on a cluster -- a directory of the repository or a chart -- and none of it was an edge: "what deploys this kustomization" or "which release installs this chart" had no answer in the graph. DEPLOYS (a new edge type, approved for this program) links them to existing nodes only: - a Flux Kustomization / Argo Application -> the units its path applies: the outermost kustomizations below it (their Kustomization node) and the manifests outside any (their File), as the deploy scopes already read them; a path that is a chart dir -> that Chart; - a HelmRelease -> its chart: the repository chart dir of a Git or Bucket source, else the shared external Chart of that name (the node chart dependencies already use) for a Helm repository. Properties carry via (flux / argo / helmrelease) and the path or chart. Measured on 17 repositories against an independent reading of the manifests (units derived from kustomize closures, chart dirs and external chart names): 129 of 129 deploys linked, 129 of 129 correct (home-ops 92, k8s.io 36, flux example 1). Only DEPLOYS edges (and one external Chart) were added; three runs build identical graphs. Signed-off-by: Martin Vogel --- internal/cbm/extract_k8s.c | 20 ++++ src/pipeline/pass_k8s.c | 189 +++++++++++++++++++++++++++++++++-- tests/test_grammar_probe_f.c | 42 ++++++++ 3 files changed, 243 insertions(+), 8 deletions(-) diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index d734ed95c..ddd114f0e 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -705,6 +705,8 @@ typedef struct { char crd_group[K8S_BUF_SIZE]; char crd_kind[K8S_BUF_SIZE]; const char *deploy_ns; // a deploying object's target namespace + const char *hr_chart; // a HelmRelease's spec.chart.spec.chart + const char *hr_source; // ... and its sourceRef kind } k8s_walk_t; // Is the document of a source kind in group list `groups`? @@ -721,6 +723,14 @@ static void k8s_deploy_leaf(k8s_walk_t *w, const char *path, const char *value) w->deploy_ns = strstr(value, "{{") ? NULL : value; return; } + if (k8s_source_is(w, K8S_FLUX_HELM, "|HelmRelease|")) { + if (strcmp(path, "spec.chart.spec.chart") == 0 && !strstr(value, "{{")) { + w->hr_chart = value; + } else if (strcmp(path, "spec.chart.spec.sourceRef.kind") == 0) { + w->hr_source = value; + } + return; + } if ((flux && strcmp(path, "spec.path") == 0) || (argo && (strcmp(path, "spec.source.path") == 0 || strcmp(path, "spec.sources[].path") == 0))) { @@ -1085,6 +1095,16 @@ static void k8s_emit_facts(CBMExtractCtx *ctx, TSNode mapping, const char *group char path[CBM_SZ_512] = ""; int first = ctx->result->k8s_facts.count; k8s_walk(&w, mapping, path, 0, sizeof(path)); + /* a HelmRelease deploys its chart: a path of a Git/Bucket source, else + * a chart name of a Helm repository (kind: the source kind) */ + if (w.hr_chart) { + CBMK8sFact f = {.fact = CBM_K8S_FACT_DEPLOY, + .def_index = def_index, + .kind = w.hr_source ? cbm_arena_strdup(ctx->arena, w.hr_source) : NULL, + .name = cbm_arena_strdup(ctx->arena, w.hr_chart), + .via = "helmrelease"}; + cbm_k8sfact_push(&ctx->result->k8s_facts, ctx->arena, f); + } /* the target namespace may follow the path in the document */ for (int i = first; w.deploy_ns && i < ctx->result->k8s_facts.count; i++) { CBMK8sFact *f = &ctx->result->k8s_facts.items[i]; diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 83ac8185f..4d8b5e224 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -1129,10 +1129,19 @@ typedef struct { * Application). */ typedef struct { int64_t node; - char *dir; /* normalised, "" = the repository root */ - char *ns; /* the namespace it deploys into, NULL = the objects' own */ + char *dir; /* normalised, "" = the repository root */ + char *ns; /* the namespace it deploys into, NULL = the objects' own */ + const char *via; /* static: "flux" / "argo" */ } k8s_deploy_t; +/* A HelmRelease and the chart it deploys: a repository path (Git / Bucket + * source), else a chart name of a Helm repository. */ +typedef struct { + int64_t node; + char *chart; + char *source; /* the sourceRef kind, NULL when unset */ +} k8s_hr_t; + typedef struct { k8s_ref_t *refs; int ref_n; @@ -1143,6 +1152,9 @@ typedef struct { k8s_deploy_t *deploys; int dep_n; int dep_cap; + k8s_hr_t *hrs; + int hr_n; + int hr_cap; } k8s_facts_t; static void k8s_facts_free(k8s_facts_t *f) { @@ -1160,6 +1172,11 @@ static void k8s_facts_free(k8s_facts_t *f) { cbm_free(CBM_MEM_CLASS_OTHER, f->deploys[i].dir); cbm_free(CBM_MEM_CLASS_OTHER, f->deploys[i].ns); } + for (int i = 0; i < f->hr_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->hrs[i].chart); + cbm_free(CBM_MEM_CLASS_OTHER, f->hrs[i].source); + } + cbm_free(CBM_MEM_CLASS_OTHER, f->hrs); cbm_free(CBM_MEM_CLASS_OTHER, f->refs); cbm_free(CBM_MEM_CLASS_OTHER, f->crds); cbm_free(CBM_MEM_CLASS_OTHER, f->deploys); @@ -1218,13 +1235,19 @@ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64 continue; } int64_t node = node_of[x->def_index]; - if (x->fact == CBM_K8S_FACT_DEPLOY) { + if (x->fact == CBM_K8S_FACT_DEPLOY && x->via && strcmp(x->via, "helmrelease") == 0) { + if (x->name && k8s_grow((void **)&f->hrs, &f->hr_cap, f->hr_n, sizeof(*f->hrs))) { + f->hrs[f->hr_n++] = + (k8s_hr_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + x->kind ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind) : NULL}; + } + } else if (x->fact == CBM_K8S_FACT_DEPLOY) { char dir[CBM_SZ_1K]; if (k8s_norm_dir(x->name, dir, sizeof(dir)) && k8s_grow((void **)&f->deploys, &f->dep_cap, f->dep_n, sizeof(*f->deploys))) { - f->deploys[f->dep_n++] = - (k8s_deploy_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir), - x->ns ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->ns) : NULL}; + f->deploys[f->dep_n++] = (k8s_deploy_t){ + node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir), + x->ns ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->ns) : NULL, x->via}; } } else if (x->fact == CBM_K8S_FACT_CRD) { if (k8s_grow((void **)&f->crds, &f->crd_cap, f->crd_n, sizeof(*f->crds))) { @@ -1316,7 +1339,7 @@ static void k8s_kust_closure(const k8s_kusts_t *ks, int root, int *seen, int mar * manifest path, sorted (a directory's are one range). */ static void k8s_dir_members(const k8s_kusts_t *ks, CBMHashTable *by_path, const k8s_paths_t *manifests, const char *dir, int *seen, int mark, - k8s_paths_t *out) { + k8s_paths_t *out, k8s_paths_t *units) { size_t dl = strlen(dir); int lo = 0; int hi = dl ? manifests->count : 0; @@ -1350,7 +1373,13 @@ static void k8s_dir_members(const k8s_kusts_t *ks, CBMHashTable *by_path, } if (!top) { k8s_paths_push(out, file); + if (units) { + k8s_paths_push(units, file); + } } else if (seen[top - SKIP_ONE] != mark) { + if (units) { + k8s_paths_push(units, ks->items[top - SKIP_ONE].path); + } k8s_kust_closure(ks, top - SKIP_ONE, seen, mark, out); } } @@ -1442,7 +1471,7 @@ static void k8s_deploys_build(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bund node[d] && node[d]->qualified_name ? node[d]->qualified_name : "", f->deploys[d].dir, d}; k8s_dir_members(ks, by_path, &manifests, f->deploys[d].dir, seen, d + SKIP_ONE, - &members[d]); + &members[d], NULL); } qsort(keys, un, sizeof(*keys), k8s_cmp_deploy_key); /* deploy d reaches deploy e when e sits in a manifest d applies */ @@ -3136,6 +3165,149 @@ static void k8s_link_includes(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, } } +/* ── DEPLOYS (PHASE 3 P3.1) ────────────────────────────────────────── */ + +/* What a GitOps object deploys, as DEPLOYS edges to existing nodes only: + * a Flux Kustomization / Argo Application -> the units its path applies (the + * outermost kustomizations below it -- their Kustomization node -- and the + * manifests outside any, their File node; a chart dir: its Chart); a + * HelmRelease -> its chart (a repository chart dir of a Git/Bucket source, + * else the shared external Chart of that name). */ + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMHashTable *files; /* file -> File node */ + CBMHashTable *charts; /* Chart.yaml path -> Chart node */ + CBMHashTable *kusts; /* kustomization path -> its Kustomization Resource */ + int edges; +} k8s_dep_link_t; + +static int64_t k8s_first_id(CBMHashTable *ix, const char *key) { + const k8s_ids_t *v = cbm_ht_get(ix, key); + return v && v->count > 0 ? v->items[0] : 0; +} + +static void k8s_deploys_edge(k8s_dep_link_t *l, int64_t src, int64_t tgt, const char *props) { + if (src > 0 && tgt > 0 && src != tgt) { + cbm_gbuf_insert_edge(l->ctx->gbuf, src, tgt, "DEPLOYS", props); + l->edges++; + } +} + +/* The Chart node of chart dir `dir`, 0 when it is none. */ +static int64_t k8s_chart_node(k8s_dep_link_t *l, const char *dir) { + static const char *const names[] = {"Chart.yaml", "Chart.yml"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + int64_t id = k8s_first_id(l->charts, path); + if (id > 0) { + return id; + } + } + return 0; +} + +/* The shared external Chart of `name` (the one chart dependencies use). */ +static int64_t k8s_external_chart(k8s_dep_link_t *l, const char *name, const char *file) { + char qn[CBM_SZ_512]; + snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", l->ctx->project_name, name); + const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(l->ctx->gbuf, qn); + if (known) { + return known->id; + } + return cbm_gbuf_upsert_node(l->ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, + "{\"source\":\"helm\",\"external\":true}"); +} + +static void k8s_link_deploys(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b, + const k8s_facts_t *f) { + if (f->dep_n == 0 && f->hr_n == 0) { + return; + } + k8s_dep_link_t l = {.ctx = ctx, + .files = k8s_index_by_file(ctx, "File"), + .charts = k8s_index_by_file(ctx, "Chart"), + .kusts = cbm_ht_create(0)}; + const cbm_gbuf_node_t **res = NULL; + int rn = 0; + k8s_paths_t manifests = {0}; + if (l.kusts && cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &res, &rn) == 0) { + for (int i = 0; i < rn; i++) { + if (!res[i]->file_path) { + continue; + } + k8s_paths_push(&manifests, res[i]->file_path); + if (res[i]->name && strcmp(res[i]->name, "Kustomization") == 0) { + k8s_index_add(l.kusts, res[i]->file_path, res[i]->id); + } + } + } + k8s_paths_sort_unique(&manifests); + CBMHashTable *by_path = cbm_ht_create(0); + for (int i = 0; by_path && i < ks->count; i++) { + cbm_ht_set(by_path, ks->items[i].path, (void *)(intptr_t)(i + SKIP_ONE)); + } + int *seen = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(ks->count ? ks->count : SKIP_ONE) * sizeof(int)); + k8s_paths_t members = {0}; + k8s_paths_t units = {0}; + for (int d = 0; l.files && l.charts && l.kusts && by_path && seen && d < f->dep_n; d++) { + const k8s_deploy_t *dp = &f->deploys[d]; + char esc[CBM_SZ_1K]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), dp->dir); + snprintf(props, sizeof(props), "{\"via\":\"%s\",\"path\":\"%s\"}", dp->via, esc); + int64_t chart = cbm_ht_get(b->charts, dp->dir) ? k8s_chart_node(&l, dp->dir) : 0; + if (chart > 0) { + k8s_deploys_edge(&l, dp->node, chart, props); + continue; + } + members.count = 0; + units.count = 0; + k8s_dir_members(ks, by_path, &manifests, dp->dir, seen, d + SKIP_ONE, &members, &units); + k8s_paths_sort_unique(&units); + for (int u = 0; u < units.count; u++) { + int64_t tgt = k8s_first_id(l.kusts, units.items[u]); + k8s_deploys_edge(&l, dp->node, tgt > 0 ? tgt : k8s_first_id(l.files, units.items[u]), + props); + } + } + for (int h = 0; l.files && l.charts && h < f->hr_n; h++) { + const k8s_hr_t *hr = &f->hrs[h]; + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, hr->node); + bool repo_path = hr->source && (strcmp(hr->source, "GitRepository") == 0 || + strcmp(hr->source, "Bucket") == 0); + char esc[CBM_SZ_1K]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), hr->chart); + snprintf(props, sizeof(props), "{\"via\":\"helmrelease\",\"chart\":\"%s\"%s}", esc, + repo_path ? "" : ",\"external\":true"); + char dir[CBM_SZ_1K]; + if (repo_path) { + if (k8s_norm_dir(hr->chart, dir, sizeof(dir)) && cbm_ht_get(b->charts, dir)) { + k8s_deploys_edge(&l, hr->node, k8s_chart_node(&l, dir), props); + } + } else if (src && src->file_path && !strchr(hr->chart, '/') && !strchr(hr->chart, '$')) { + k8s_deploys_edge(&l, hr->node, k8s_external_chart(&l, hr->chart, src->file_path), + props); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)members.items); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)units.items); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)manifests.items); + cbm_free(CBM_MEM_CLASS_OTHER, seen); + if (by_path) { + cbm_ht_free(by_path); + } + k8s_free_index(l.files); + k8s_free_index(l.charts); + k8s_free_index(l.kusts); + if (l.edges > 0) { + cbm_log_info("pass.k8s.deploys_edges", "edges", itoa_k8s(l.edges)); + } +} + int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { cbm_log_info("pass.start", "pass", "k8s", "files", itoa_k8s(file_count)); @@ -3232,6 +3404,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_selectors(ctx, &recs, &bundles); k8s_resolve_refs(ctx, &bundles, &facts); k8s_link_crds(ctx, &bundles, &facts); + k8s_link_deploys(ctx, &kusts, &bundles, &facts); k8s_facts_free(&facts); k8s_records_free(&recs); /* Link `file://` chart dependencies now that every Chart.yaml is read, diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 3f33f2861..cca16bcd3 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1655,6 +1655,47 @@ TEST(probe_mcp_trace_topology) { PASS(); } +/* ── DEPLOYS (Phase 3) ──────────────────────────────────────────────── */ + +/* A Flux Kustomization deploys what its path applies: the outermost + * kustomization below it (its Kustomization node) and a manifest outside any + * (its File); a HelmRelease its chart -- the repository chart of a Git + * source, else the external chart of that name; an Argo Application whose + * path is a chart dir, that Chart. */ +TEST(probe_gitops_objects_deploy_their_units) { + static const GpfFile files[] = { + {"clusters/prod/flux-system/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - gotk-sync.yaml\n"}, + {"clusters/prod/flux-system/gotk-sync.yaml", + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: flux-system\n namespace: flux-system\n" + "spec:\n path: ./clusters/prod\n"}, + {"clusters/prod/apps.yaml", + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: apps\n namespace: flux-system\nspec:\n path: ./apps\n"}, + {"apps/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - release.yaml\n"}, + {"apps/release.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: web\n" + "spec:\n chart:\n spec:\n chart: ./charts/web\n sourceRef:\n" + " kind: GitRepository\n name: flux-system\n---\n" + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: redis\n" + "spec:\n chart:\n spec:\n chart: redis\n sourceRef:\n" + " kind: HelmRepository\n name: bitnami\n"}, + {"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 0.1.0\n"}, + {"argo/web.yaml", + "apiVersion: argoproj.io/v1alpha1\nkind: Application\nmetadata:\n name: web\n" + "spec:\n source:\n path: charts/web\n"}}; + ASSERT_EQ(gpf_typed_edges(files, 7, "DEPLOYS", "Resource", "Resource", "\"via\":\"flux\""), 2); + ASSERT_EQ(gpf_typed_edges(files, 7, "DEPLOYS", "Resource", "File", "\"via\":\"flux\""), 1); + ASSERT_EQ( + gpf_typed_edges(files, 7, "DEPLOYS", "Resource", "Chart", "\"via\":\"helmrelease\""), 2); + ASSERT_EQ(gpf_typed_edges(files, 7, "DEPLOYS", "Resource", "Chart", "\"via\":\"argo\""), 1); + PASS(); +} + /* ── Helm symbolic (Phase 2) ────────────────────────────────────────── */ /* A chart template is YAML only once rendered: a document opened by @@ -3325,6 +3366,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_k8s_ingress_and_httproute_expose_routes); RUN_TEST(probe_mcp_search_filters_kind_and_namespace); RUN_TEST(probe_mcp_trace_topology); + RUN_TEST(probe_gitops_objects_deploy_their_units); RUN_TEST(probe_helm_templated_objects_are_resources); RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); From b3a5b788f77ed3a893d28a263071e9c5ae51e553 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 03:20:49 +0200 Subject: [PATCH 30/52] fix(k8s): cppcheck 2.20 findings in the K8s pass cppcheck 2.20 (the CI version) reports three findings in pass_k8s.c: - autoVariables (error): the include scanner kept a pointer to a loop-local buffer holding the template's chart family; the scanner state now owns that buffer. - CastAddressToIntegerAtReturn: the kustomization index lookup returned a pointer-sized value cast to int directly; it goes through intptr_t. - variableScope: two per-dependency buffers move into the block using them. No behaviour change: the 17-corpus graphs are identical before and after. Signed-off-by: Martin Vogel --- src/pipeline/pass_k8s.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 4d8b5e224..810f3610b 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -776,7 +776,8 @@ static int k8s_cmp_kust(const void *a, const void *b) { /* index + 1 of the kustomization at `path`, 0 when none. */ static int k8s_kust_at(CBMHashTable *by_path, const char *path) { - return (int)(intptr_t)cbm_ht_get(by_path, path); + intptr_t at = (intptr_t)cbm_ht_get(by_path, path); + return (int)at; } /* The kustomization file inside directory `dir`, as kustomize looks for it. */ @@ -2838,7 +2839,7 @@ typedef struct { CBMHashTable *yaml_defs; /* template file -> the defines written in it */ CBMArena arena; /* define QNs */ const char *file; - const char *family; + char family[CBM_SZ_1K]; /* the chart family of `file` */ k8s_ids_t found; int edges; int unresolved; @@ -2881,9 +2882,9 @@ static void k8s_inc_call(k8s_inc_t *s, const char *name, int line, const char *v const k8s_ids_t *deps = cbm_ht_get(s->deps, s->family); for (int d = 0; deps && d < deps->count; d++) { const cbm_gbuf_node_t *dep = cbm_gbuf_find_by_id(s->ctx->gbuf, deps->items[d]); - char chart[CBM_SZ_1K]; - char family[CBM_SZ_1K]; if (dep && dep->file_path) { + char chart[CBM_SZ_1K]; + char family[CBM_SZ_1K]; k8s_dirname(dep->file_path, chart, sizeof(chart)); k8s_chart_family(s->b, chart, family, sizeof(family)); k8s_inc_lookup(s, family, name); @@ -3113,8 +3114,7 @@ static void k8s_includes_run(k8s_inc_t *s, const cbm_file_info_t *files, int fil for (int pass = 0; pass < PAIR_LEN; pass++) { for (int i = 0; i < file_count; i++) { const char *rel = files[i].rel_path; - char family[CBM_SZ_1K]; - if (!k8s_template_family(s->b, rel, family, sizeof(family))) { + if (!k8s_template_family(s->b, rel, s->family, sizeof(s->family))) { continue; } char path[CBM_SZ_4K]; @@ -3125,7 +3125,6 @@ static void k8s_includes_run(k8s_inc_t *s, const cbm_file_info_t *files, int fil continue; } s->file = rel; - s->family = family; if (pass == 0) { k8s_scan_defines(s, src); } else { From 3323cae0f8fcbf01de6e0a8f78fd26ccaf2e256f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 03:21:09 +0200 Subject: [PATCH 31/52] feat(shell): shell and Make deploy commands DEPLOYS what they apply A script or Make recipe running kubectl apply, helm upgrade or a kustomize build piped into kubectl puts manifests, a kustomization or a chart on a cluster, and none of it was an edge: "which script deploys this manifest" had no answer in the graph. A new pass (the Bash + Make layer, run at the end of the K8s pass) reads scripts and Makefiles that name a deploy tool and links, to existing nodes only: - kubectl apply|create|replace -f: the File of a manifest, a directory's manifests (-R: below it too), a glob's; -k: the Kustomization; - kustomize build D (or kubectl kustomize D) | ... | kubectl apply -f -: the Kustomization of D; - helm install|upgrade [NAME] CHART, helm template piped into kubectl: the Chart of a chart dir, else the shared external Chart of repo/name or oci://.../name; - find D -name G -exec kubectl apply -f {} \;: D's matching manifests. The edge starts at the bash function or Make rule running the command, top-level code at its file, with via (kubectl / helm / kustomize), the deployed path or chart, and the line. Words are evaluated by constant propagation, nothing is executed: assignments (local / export / readonly / declare, arrays), ${X:-default}, $0 / BASH_SOURCE, $(dirname ..), $(cd .. && pwd), $(realpath ..), $(git rev-parse --show-toplevel); cd / pushd / popd track the directory. Anything else is unknown, and an unknown target is no edge: a value set in a branch or loop is unknown after it, a subshell's changes do not leak, an unquoted unknown word that may shift helm's positionals blocks the chart. With no directory tracked, a relative path resolves against the root and the script's directory, linked only when exactly one of them has a target. Make: the Makefile's variables expand into each recipe line ($(shell pwd) / $(CURDIR) = the Makefile's directory; a conditional's value is unknown), which is then read as a shell line. Measured on 17 repositories against an independent reading of the scripts (its own tokenizer, variable and directory tracking, the file system): 86 of 86 deploys linked, 86 of 86 correct (k8s.io 66, kubermatic 12, gardener 8). Only DEPLOYS edges and 3 external Charts were added; three runs build identical graphs; index time unchanged within 1 s (deckhouse 10 s, gardener 5-6 s, k8s.io 3-4 s, median of 3). Nine incremental edits (deploy lines edited, removed and added, a Makefile, a deployed manifest, a new manifest and a new non-manifest in an applied directory) leave the graph a fresh index builds. tree-sitter-bash folds `a | b` followed by a line `cat < --- Makefile.cbm | 1 + scripts/check-lsp-originality.sh | 7 +- src/pipeline/pass_k8s.c | 2 + src/pipeline/pass_shell.c | 2628 ++++++++++++++++++++++++++++++ src/pipeline/pipeline_internal.h | 5 + tests/test_grammar_probe_f.c | 208 +++ tests/test_pipeline.c | 36 + 7 files changed, 2886 insertions(+), 1 deletion(-) create mode 100644 src/pipeline/pass_shell.c diff --git a/Makefile.cbm b/Makefile.cbm index 1dfc9b8ec..35fa3296a 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -519,6 +519,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_infrascan.c \ src/pipeline/pass_k8s.c \ src/pipeline/pass_operator.c \ + src/pipeline/pass_shell.c \ src/pipeline/pass_similarity.c \ src/pipeline/pass_semantic_edges.c \ src/pipeline/pass_complexity.c \ diff --git a/scripts/check-lsp-originality.sh b/scripts/check-lsp-originality.sh index a107d60f0..22c33e153 100755 --- a/scripts/check-lsp-originality.sh +++ b/scripts/check-lsp-originality.sh @@ -74,10 +74,15 @@ REFS=( "kustomize|https://github.com/kubernetes-sigs/kustomize|api/krusty,api/internal/target" "flux|https://github.com/fluxcd/pkg|kustomize" "kube|https://github.com/kubernetes-sigs/controller-tools|pkg" + # The Bash (+ Make) layer (deploy commands, constant propagation) was written + # from the bash, GNU make, kubectl and helm documentation. Scanned against + # bash-language-server (MIT), the leading OSS bash language server. + "bash|https://github.com/bash-lsp/bash-language-server|server/src" ) # Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too). -INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c) +INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c + src/pipeline/pass_shell.c) ONLY_LANG=""; REFRESH=0; LIST_ONLY=0 while [ $# -gt 0 ]; do diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 810f3610b..e4d26bcf3 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -3412,6 +3412,8 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_includes(ctx, &bundles, files, file_count); k8s_bundles_free(&bundles); k8s_kusts_free(&kusts); + /* Shell and Make deploy commands, against the nodes linked above. */ + cbm_pipeline_pass_shell(ctx, files, file_count); cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count)); (void)helm_count; diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c new file mode 100644 index 000000000..9a64214a2 --- /dev/null +++ b/src/pipeline/pass_shell.c @@ -0,0 +1,2628 @@ +/* + * pass_shell.c — the Bash (+ Make) infrastructure layer (PHASE 3 P3.2). + * + * A shell script or a Make recipe that puts manifests, a kustomization or a + * chart on a cluster DEPLOYS them: an edge from the function running the + * command (a bash function, a Make rule; top-level code: the Module) to the + * node of what it deploys -- existing nodes only: + * kubectl apply|create|replace -f F a File holding Kubernetes objects: + * F itself, F's *.yaml|*.yml|*.json + * (-R: below it), or a glob's matches + * kubectl ... -k D the Kustomization of D + * kustomize build D | ... | kubectl apply -f - (also kubectl kustomize D) + * helm install|upgrade [NAME] CHART the Chart of a chart dir, else the + * shared external Chart (repo/name, + * oci://.../name) + * helm template [NAME] CHART | ... | kubectl apply -f - + * find D -name G -exec kubectl apply -f {} \; D's manifests matching G + * + * Words are evaluated by constant propagation over the script, nothing is + * executed: assignments (local / export / readonly / declare, arrays and + * "${a[@]}"), ${X:-default}, $0 / BASH_SOURCE, $(dirname ..), $(basename ..), + * $(cd .. && pwd), $(realpath ..), $(pwd), $(git rev-parse --show-toplevel). + * A word with any other part is unknown, and an unknown target is no edge. + * A value set in a branch, a loop or the right side of && / || is unknown + * after it; a subshell's or pipeline element's changes do not leak; a + * function body is read where it is defined, and the globals it sets are + * unknown after it. cd / pushd / popd track the working directory. A + * relative path resolves against it; with none tracked (a script runs from + * wherever it is started), against the repository root and the script's + * directory, linked only when exactly one of the two names a target. + * + * Make: the Makefile's variables expand into each recipe line (recursively, + * with the last assignment; $(CURDIR) / $(shell pwd) are the Makefile's + * directory, where its recipes run), which then evaluates as a shell line of + * its own. A variable assigned under a conditional is unknown; an included + * .mk file runs in its includer's directory, so it tracks none. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/hash_table.h" +#include "foundation/limits.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "cbm.h" +#include "lang_specs.h" + +#include +#include +#include +#include +#include + +/* Candidate paths a target lookup scanned (complexity guard counter). */ +_Atomic uint64_t g_sh_candidates = 0; + +enum { + SH_GROW_MIN = 16, + SH_DEC = 10, + SH_NO_DEPTH = -1, +}; + +/* A repository-anchored absolute path value: "\x01hack/ci" is /hack/ci. */ +#define SH_ANCHOR '\x01' +static const char SH_ROOT[] = "\x01"; + +static bool sh_grow(void **items, int *cap, int count, size_t elem) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : SH_GROW_MIN; + void *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, *items, (size_t)ncap * elem); + if (!grown) { + return false; + } + *items = grown; + *cap = ncap; + return true; +} + +static const char *sh_basename(const char *p) { + const char *s = strrchr(p, '/'); + return s ? s + SKIP_ONE : p; +} + +static bool sh_eq(const char *a, const char *b) { + return a && b && strcmp(a, b) == 0; +} + +static bool sh_starts(const char *s, const char *prefix) { + return strncmp(s, prefix, strlen(prefix)) == 0; +} + +/* ── The graph's deploy targets ────────────────────────────────────── */ + +typedef struct { + const char **items; /* graph-owned paths */ + int count; + int cap; +} sh_paths_t; + +static void sh_paths_push(sh_paths_t *v, const char *p) { + if (sh_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = p; + } +} + +static int sh_cmp_str(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +static void sh_paths_sort_unique(sh_paths_t *v) { + if (v->count == 0) { + return; + } + qsort((void *)v->items, (size_t)v->count, sizeof(*v->items), sh_cmp_str); + int w = SKIP_ONE; + for (int i = SKIP_ONE; i < v->count; i++) { + if (strcmp(v->items[i], v->items[w - SKIP_ONE]) != 0) { + v->items[w++] = v->items[i]; + } + } + v->count = w; +} + +/* First index whose path is >= key. */ +static int sh_lower_bound(const sh_paths_t *v, const char *key) { + int lo = 0; + int hi = v->count; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (strcmp(v->items[mid], key) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +static bool sh_paths_has(const sh_paths_t *v, const char *p) { + int i = sh_lower_bound(v, p); + return i < v->count && strcmp(v->items[i], p) == 0; +} + +typedef struct { + cbm_pipeline_ctx_t *ctx; + sh_paths_t files; /* every File node's path */ + sh_paths_t manifests; /* files holding Kubernetes objects (.yaml/.yml/.json) */ + CBMHashTable *file_ids; /* path -> File node id */ + CBMHashTable *kusts; /* kustomization path -> its Kustomization Resource id */ + CBMHashTable *charts; /* Chart.yaml path -> Chart node id */ +} sh_index_t; + +static bool sh_manifest_ext(const char *path) { + const char *dot = strrchr(sh_basename(path), '.'); + return dot && + (strcmp(dot, ".yaml") == 0 || strcmp(dot, ".yml") == 0 || strcmp(dot, ".json") == 0); +} + +static bool sh_kustomize_base(const char *base) { + return cbm_is_kustomize_file(base) || strcmp(base, "Kustomization") == 0; +} + +static bool sh_index_build(sh_index_t *ix, cbm_pipeline_ctx_t *ctx) { + ix->ctx = ctx; + ix->file_ids = cbm_ht_create(0); + ix->kusts = cbm_ht_create(0); + ix->charts = cbm_ht_create(0); + if (!ix->file_ids || !ix->kusts || !ix->charts) { + return false; + } + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "File", &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + if (nodes[i]->file_path) { + sh_paths_push(&ix->files, nodes[i]->file_path); + cbm_ht_set(ix->file_ids, nodes[i]->file_path, (void *)(intptr_t)nodes[i]->id); + } + } + } + if (cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + const char *fp = nodes[i]->file_path; + if (!fp) { + continue; + } + bool kust_file = sh_kustomize_base(sh_basename(fp)); + if (kust_file && sh_eq(nodes[i]->name, "Kustomization")) { + cbm_ht_set(ix->kusts, fp, (void *)(intptr_t)nodes[i]->id); + } else if (!kust_file && sh_manifest_ext(fp)) { + sh_paths_push(&ix->manifests, fp); + } + } + } + if (cbm_gbuf_find_by_label(ctx->gbuf, "Chart", &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + const char *fp = nodes[i]->file_path; + const char *base = fp ? sh_basename(fp) : ""; + const char *qn = nodes[i]->qualified_name; + if ((strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0) && qn && + !strstr(qn, ".__helm_dep__.")) { + cbm_ht_set(ix->charts, fp, (void *)(intptr_t)nodes[i]->id); + } + } + } + sh_paths_sort_unique(&ix->files); + sh_paths_sort_unique(&ix->manifests); + return true; +} + +static void sh_index_free(sh_index_t *ix) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)ix->files.items); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)ix->manifests.items); + if (ix->file_ids) { + cbm_ht_free(ix->file_ids); + } + if (ix->kusts) { + cbm_ht_free(ix->kusts); + } + if (ix->charts) { + cbm_ht_free(ix->charts); + } +} + +/* The paths of `v` below directory `dir` ("" = the root): [*lo, *hi). */ +static void sh_range(const sh_paths_t *v, const char *dir, char *key, size_t key_sz, int *lo, + int *hi) { + if (!dir[0]) { + key[0] = '\0'; + *lo = 0; + *hi = v->count; + return; + } + snprintf(key, key_sz, "%s/", dir); + size_t kl = strlen(key); + *lo = sh_lower_bound(v, key); + int h = *lo; + while (h < v->count && strncmp(v->items[h], key, kl) == 0) { + h++; + } + *hi = h; +} + +static bool sh_dir_exists(const sh_index_t *ix, const char *dir) { + if (!dir[0]) { + return true; + } + char key[CBM_SZ_4K]; + int lo = 0; + int hi = 0; + sh_range(&ix->files, dir, key, sizeof(key), &lo, &hi); + return hi > lo; +} + +/* ── Path values ───────────────────────────────────────────────────── */ + +/* `rel` (a relative path: ./ and ../ allowed) inside repository directory + * `base` ("" = the root), normalized; NULL when it leaves the repository. */ +static char *sh_join(CBMArena *a, const char *base, const char *rel) { + size_t bl = strlen(base); + char *out = cbm_arena_alloc(a, bl + strlen(rel) + PAIR_LEN); + if (!out) { + return NULL; + } + memcpy(out, base, bl); + size_t n = bl; + out[n] = '\0'; + for (const char *p = rel; *p;) { + while (*p == '/') { + p++; + } + size_t sl = strcspn(p, "/"); + if (sl == 0) { + break; + } + if (sl == SKIP_ONE && p[0] == '.') { + p += sl; + continue; + } + if (sl == PAIR_LEN && p[0] == '.' && p[SKIP_ONE] == '.') { + if (n == 0) { + return NULL; + } + while (n > 0 && out[n - SKIP_ONE] != '/') { + n--; + } + if (n > 0) { + n--; + } + out[n] = '\0'; + p += sl; + continue; + } + if (n) { + out[n++] = '/'; + } + memcpy(out + n, p, sl); + n += sl; + out[n] = '\0'; + p += sl; + } + return out; +} + +static const char *sh_anchor(CBMArena *a, const char *repo_path) { + return repo_path ? cbm_arena_sprintf(a, "%c%s", SH_ANCHOR, repo_path) : NULL; +} + +/* dirname(1), on a value: textual, keeps an anchored path anchored. */ +static const char *sh_dirname(CBMArena *a, const char *v) { + bool anch = v[0] == SH_ANCHOR; + const char *p = anch ? v + SKIP_ONE : v; + size_t len = strlen(p); + while (len > SKIP_ONE && p[len - SKIP_ONE] == '/') { + len--; + } + size_t k = len; + while (k > 0 && p[k - SKIP_ONE] != '/') { + k--; + } + if (k == 0) { + return anch ? SH_ROOT : "."; + } + size_t dl = k - SKIP_ONE; + while (dl > 0 && p[dl - SKIP_ONE] == '/') { + dl--; + } + if (dl == 0) { + return anch ? SH_ROOT : "/"; + } + return anch ? cbm_arena_sprintf(a, "%c%.*s", SH_ANCHOR, (int)dl, p) + : cbm_arena_strndup(a, p, dl); +} + +/* basename(1), on a value. */ +static const char *sh_value_basename(CBMArena *a, const char *v) { + const char *p = v[0] == SH_ANCHOR ? v + SKIP_ONE : v; + size_t len = strlen(p); + while (len > SKIP_ONE && p[len - SKIP_ONE] == '/') { + len--; + } + size_t k = len; + while (k > 0 && p[k - SKIP_ONE] != '/') { + k--; + } + return cbm_arena_strndup(a, p + k, len - k); +} + +/* ── Hits: what a command deploys ──────────────────────────────────── */ + +typedef struct { + const char *file; /* the script (caller-owned path) */ + int line; + const char *via; /* static */ + int64_t tgt; /* the target node; 0: the external chart `ext` */ + const char *ext; /* pass-arena */ + const char *path; /* pass-arena: the deployed path or chart */ + bool chart; +} sh_hit_t; + +typedef struct { + sh_hit_t *items; + int count; + int cap; + CBMArena arena; +} sh_hits_t; + +typedef struct { + int64_t id; + const char *path; /* graph-owned or file-arena */ +} sh_target_t; + +typedef struct { + sh_target_t *items; + int count; + int cap; +} sh_targets_t; + +static void sh_targets_push(sh_targets_t *v, int64_t id, const char *path) { + if (id > 0 && sh_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = (sh_target_t){id, path}; + } +} + +/* ── Shell state ───────────────────────────────────────────────────── */ + +typedef struct { + const char *s; /* scalar value; NULL: unknown */ + const char **items; /* array items (an item NULL: unknown); NULL: no array */ + int n; +} sh_val_t; + +static const sh_val_t SH_UNKNOWN_VAL = {NULL, NULL, 0}; + +typedef enum { SH_FREE, SH_AT, SH_LOST } sh_cwd_mode_t; + +typedef struct { + sh_cwd_mode_t mode; + const char *dir; /* SH_AT: repository path ("" = the root) */ +} sh_cwd_t; + +typedef struct { + const char *name; + const sh_val_t *prev; /* NULL: was unset */ + bool local; +} sh_undo_t; + +typedef struct { + int undo; + sh_cwd_t cwd; + int ndirs; +} sh_mark_t; + +typedef struct { + const char **names; + int n; + int cap; + bool cwd; +} sh_touch_t; + +typedef struct { + sh_index_t *ix; + sh_hits_t *hits; + CBMArena *a; /* this file's values */ + const char *src; /* the parsed text */ + const char *file; /* the script / Makefile path */ + const char *script; /* $0: anchored script path; NULL: none (Make) */ + const char *sdir; /* the script's directory */ + int line_base; /* Make: the recipe line's row */ + int in_func; + CBMHashTable *env; /* name -> const sh_val_t * */ + sh_undo_t *undo; + int nundo; + int cap_undo; + sh_cwd_t cwd; + sh_cwd_t *dirs; /* pushd stack */ + int ndirs; + int cap_dirs; +} sh_t; + +static void sh_set(sh_t *s, const char *name, const sh_val_t *v, bool local) { + if (!sh_grow((void **)&s->undo, &s->cap_undo, s->nundo, sizeof(*s->undo))) { + return; + } + s->undo[s->nundo++] = (sh_undo_t){name, cbm_ht_get(s->env, name), local}; + cbm_ht_set(s->env, name, (void *)v); +} + +static sh_mark_t sh_mark(const sh_t *s) { + return (sh_mark_t){s->nundo, s->cwd, s->ndirs}; +} + +static bool sh_cwd_same(sh_cwd_t a, sh_cwd_t b) { + return a.mode == b.mode && (a.mode != SH_AT || strcmp(a.dir, b.dir) == 0); +} + +static void sh_rewind(sh_t *s, sh_mark_t m) { + while (s->nundo > m.undo) { + const sh_undo_t *u = &s->undo[--s->nundo]; + if (u->prev) { + cbm_ht_set(s->env, u->name, (void *)u->prev); + } else { + (void)cbm_ht_delete(s->env, u->name); + } + } + s->cwd = m.cwd; + s->ndirs = m.ndirs; +} + +static void sh_touch_add(sh_touch_t *t, const char *name) { + if (sh_grow((void **)&t->names, &t->cap, t->n, sizeof(*t->names))) { + t->names[t->n++] = name; + } +} + +/* End one alternative of a conditional: what it set is noted, then undone. */ +static void sh_clause_end(sh_t *s, sh_mark_t m, sh_touch_t *t) { + for (int i = m.undo; i < s->nundo; i++) { + sh_touch_add(t, s->undo[i].name); + } + if (!sh_cwd_same(s->cwd, m.cwd) || s->ndirs != m.ndirs) { + t->cwd = true; + } + sh_rewind(s, m); +} + +/* After the conditional: whatever any alternative set is unknown. */ +static void sh_group_end(sh_t *s, sh_touch_t *t) { + for (int i = 0; i < t->n; i++) { + sh_set(s, t->names[i], &SH_UNKNOWN_VAL, false); + } + if (t->cwd) { + s->cwd = (sh_cwd_t){SH_LOST, NULL}; + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)t->names); + *t = (sh_touch_t){0}; +} + +/* ── Word evaluation ───────────────────────────────────────────────── */ + +static const char *sh_text(sh_t *s, TSNode n) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + return cbm_arena_strndup(s->a, s->src + b, e - b); +} + +static bool sh_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +/* A growable string in the file arena. */ +typedef struct { + char *buf; + size_t len; + size_t cap; + bool bad; /* an unknown part */ +} sh_str_t; + +static void sh_str_add(CBMArena *a, sh_str_t *b, const char *p, size_t n) { + if (b->bad || !p) { + b->bad = true; + return; + } + if (b->len + n + SKIP_ONE > b->cap) { + size_t ncap = (b->cap ? b->cap : SH_GROW_MIN) * PAIR_LEN; + while (ncap < b->len + n + SKIP_ONE) { + ncap *= PAIR_LEN; + } + char *nb = cbm_arena_alloc(a, ncap); + if (!nb) { + b->bad = true; + return; + } + if (b->len) { + memcpy(nb, b->buf, b->len); + } + b->buf = nb; + b->cap = ncap; + } + memcpy(b->buf + b->len, p, n); + b->len += n; + b->buf[b->len] = '\0'; +} + +static const char *sh_str_done(sh_str_t *b) { + if (b->bad) { + return NULL; + } + return b->buf ? b->buf : ""; +} + +/* Text with backslash escapes removed; in_dq: only the double-quote ones. */ +static void sh_add_unescaped(CBMArena *a, sh_str_t *b, const char *p, size_t n, bool in_dq) { + size_t i = 0; + while (i < n) { + size_t j = i; + while (j < n && p[j] != '\\') { + j++; + } + sh_str_add(a, b, p + i, j - i); + if (j + SKIP_ONE >= n) { + if (j < n) { + sh_str_add(a, b, p + j, SKIP_ONE); + } + return; + } + char c = p[j + SKIP_ONE]; + if (c == '\n') { + i = j + PAIR_LEN; + continue; + } + if (!in_dq || c == '"' || c == '\\' || c == '$' || c == '`') { + sh_str_add(a, b, p + j + SKIP_ONE, SKIP_ONE); + } else { + sh_str_add(a, b, p + j, PAIR_LEN); + } + i = j + PAIR_LEN; + } +} + +static const char *sh_eval(sh_t *s, TSNode n); + +/* The cwd as a value: the anchored directory, or NULL. */ +static const char *sh_cwd_value(sh_t *s) { + return s->cwd.mode == SH_AT ? sh_anchor(s->a, s->cwd.dir) : NULL; +} + +/* A variable's value; *found: it is set (or a special that exists). */ +static const sh_val_t *sh_lookup(sh_t *s, const char *name, bool *found) { + *found = true; + if (strcmp(name, "0") == 0 || strcmp(name, "BASH_SOURCE") == 0) { + if (!s->script) { + return &SH_UNKNOWN_VAL; + } + sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); + if (v) { + *v = (sh_val_t){s->script, NULL, 0}; + } + return v ? v : &SH_UNKNOWN_VAL; + } + if (strcmp(name, "PWD") == 0) { + sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); + if (v) { + *v = (sh_val_t){sh_cwd_value(s), NULL, 0}; + } + return v ? v : &SH_UNKNOWN_VAL; + } + if (!name[0] || (strchr("123456789@*#?$!-", name[0]) && + (name[SKIP_ONE] == '\0' || (name[0] >= '1' && name[0] <= '9')))) { + return &SH_UNKNOWN_VAL; /* positional and special parameters */ + } + const sh_val_t *v = cbm_ht_get(s->env, name); + if (!v) { + *found = false; + return &SH_UNKNOWN_VAL; + } + return v; +} + +/* Concatenate the children of `n` in [from, to), source gaps included. */ +static void sh_concat(sh_t *s, sh_str_t *b, TSNode n, uint32_t from, uint32_t to, bool in_dq) { + uint32_t pos = from; + for (uint32_t i = 0; i < ts_node_child_count(n) && !b->bad; i++) { + TSNode c = ts_node_child(n, i); + uint32_t cb = ts_node_start_byte(c); + uint32_t ce = ts_node_end_byte(c); + if (cb < from || ce > to) { + continue; + } + if (cb > pos) { + sh_add_unescaped(s->a, b, s->src + pos, cb - pos, in_dq); + } + if (!ts_node_is_named(c)) { + if (!(in_dq && strcmp(ts_node_type(c), "\"") == 0)) { + sh_str_add(s->a, b, s->src + cb, ce - cb); + } + } else if (sh_type(c, "string_content")) { + sh_add_unescaped(s->a, b, s->src + cb, ce - cb, in_dq); + } else { + const char *v = sh_eval(s, c); + sh_str_add(s->a, b, v, v ? strlen(v) : 0); + } + pos = ce; + } + if (pos < to && !b->bad) { + sh_add_unescaped(s->a, b, s->src + pos, to - pos, in_dq); + } +} + +/* ${...}: a plain variable, an element 0, or a default (:- - := =). */ +static const char *sh_eval_expansion(sh_t *s, TSNode n) { + TSNode name = {0}; + const char *op = NULL; + uint32_t op_end = 0; + uint32_t close = ts_node_end_byte(n); + uint32_t cc = ts_node_child_count(n); + for (uint32_t i = 0; i < cc; i++) { + TSNode c = ts_node_child(n, i); + const char *t = ts_node_type(c); + if (ts_node_is_named(c)) { + if (ts_node_is_null(name)) { + name = c; + } else if (!op) { + return NULL; /* ${X%pattern} and the like: no plain value */ + } + continue; + } + if (strcmp(t, "${") == 0) { + continue; + } + if (strcmp(t, "}") == 0) { + close = ts_node_start_byte(c); + continue; + } + if (op) { + continue; /* inside the default */ + } + if (ts_node_is_null(name)) { + return NULL; /* ${#X}, ${!X}: not the value */ + } + if (strcmp(t, ":-") != 0 && strcmp(t, "-") != 0 && strcmp(t, ":=") != 0 && + strcmp(t, "=") != 0) { + return NULL; /* ${X%..}, ${X/..}, ${X:+..}, ... */ + } + op = t; + op_end = ts_node_end_byte(c); + } + if (ts_node_is_null(name)) { + return NULL; + } + const char *nm = NULL; + int index = SH_NO_DEPTH; + if (sh_type(name, "subscript")) { + TSNode base = ts_node_child_by_field_name(name, TS_FIELD("name")); + TSNode idx = ts_node_child_by_field_name(name, TS_FIELD("index")); + if (ts_node_is_null(base) || ts_node_is_null(idx)) { + return NULL; + } + const char *it = sh_text(s, idx); + if (!it || strcmp(it, "0") != 0) { + return NULL; + } + nm = sh_text(s, base); + index = 0; + } else if (sh_type(name, "variable_name") || sh_type(name, "special_variable_name")) { + nm = sh_text(s, name); + } else { + return NULL; + } + if (!nm) { + return NULL; + } + bool found = false; + const sh_val_t *v = sh_lookup(s, nm, &found); + const char *val = v->s; + if (index == 0 && v->items) { + val = v->n > 0 ? v->items[0] : ""; + } + if (!op) { + return found ? val : NULL; + } + bool colon = op[0] == ':'; + bool use_default = !found || (colon && val && !val[0]); + if (!use_default) { + return val; + } + sh_str_t b = {0}; + sh_concat(s, &b, n, op_end, close, true); + const char *d = sh_str_done(&b); + if (d && strchr(op, '=')) { + sh_val_t *nv = cbm_arena_alloc(s->a, sizeof(*nv)); + if (nv) { + *nv = (sh_val_t){d, NULL, 0}; + sh_set(s, nm, nv, false); + } + } + return d; +} + +/* A value as a repository path: anchored, or relative to a tracked + * directory. NULL when it is neither. */ +static const char *sh_cd_target(sh_t *s, const char *v, sh_cwd_t *out); + +/* realpath / readlink -f: the anchored path, when it is decidable. */ +static const char *sh_absolute(sh_t *s, const char *v) { + if (!v) { + return NULL; + } + if (v[0] == SH_ANCHOR) { + return sh_anchor(s->a, sh_join(s->a, "", v + SKIP_ONE)); + } + if (v[0] == '/' || s->cwd.mode != SH_AT) { + return NULL; + } + return sh_anchor(s->a, sh_join(s->a, s->cwd.dir, v)); +} + +typedef struct { + const char **v; + bool *split; + int n; + int cap; +} sh_argv_t; + +static void sh_argv_build(sh_t *s, TSNode cmd, sh_argv_t *out); + +static void sh_argv_free(sh_argv_t *a) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)a->v); + cbm_free(CBM_MEM_CLASS_OTHER, a->split); + *a = (sh_argv_t){0}; +} + +/* The command a statement runs: the body of a redirection. */ +static TSNode sh_unwrap(TSNode n) { + while (sh_type(n, "redirected_statement")) { + TSNode body = ts_node_child_by_field_name(n, TS_FIELD("body")); + if (ts_node_is_null(body)) { + break; + } + n = body; + } + return n; +} + +/* The first non-option argument after argv[0]. */ +static const char *sh_first_operand(const sh_argv_t *a, int from) { + for (int i = from; i < a->n; i++) { + if (a->v[i] && a->v[i][0] == '-' && a->v[i][SKIP_ONE]) { + continue; + } + return a->v[i]; + } + return NULL; +} + +/* $(dirname ..), $(basename ..), $(pwd), $(realpath ..), $(readlink -f ..), + * $(git rev-parse --show-toplevel). */ +static const char *sh_eval_subst_command(sh_t *s, TSNode cmd) { + sh_argv_t a = {0}; + sh_argv_build(s, cmd, &a); + const char *out = NULL; + const char *name = a.n > 0 ? a.v[0] : NULL; + if (sh_eq(name, "dirname") && a.n == PAIR_LEN && a.v[SKIP_ONE]) { + out = sh_dirname(s->a, a.v[SKIP_ONE]); + } else if (sh_eq(name, "basename") && a.n == PAIR_LEN && a.v[SKIP_ONE]) { + out = sh_value_basename(s->a, a.v[SKIP_ONE]); + } else if (sh_eq(name, "pwd")) { + out = sh_cwd_value(s); + } else if (sh_eq(name, "realpath") || + (sh_eq(name, "readlink") && a.n > PAIR_LEN && a.v[SKIP_ONE] && + (sh_eq(a.v[SKIP_ONE], "-f") || sh_eq(a.v[SKIP_ONE], "-e") || + sh_eq(a.v[SKIP_ONE], "-m")))) { + out = sh_absolute(s, sh_first_operand(&a, SKIP_ONE)); + } else if (sh_eq(name, "git") && a.n == 3 && sh_eq(a.v[SKIP_ONE], "rev-parse") && + sh_eq(a.v[PAIR_LEN], "--show-toplevel")) { + out = SH_ROOT; + } + sh_argv_free(&a); + return out; +} + +/* `cd DIR` as the left half of $(cd DIR && pwd): the directory, or NULL. */ +static const char *sh_subst_cd(sh_t *s, TSNode cd) { + cd = sh_unwrap(cd); + if (!sh_type(cd, "command")) { + return NULL; + } + sh_argv_t a = {0}; + sh_argv_build(s, cd, &a); + const char *out = NULL; + if (a.n > 0 && sh_eq(a.v[0], "cd")) { + sh_cwd_t to = {SH_LOST, NULL}; + (void)sh_cd_target(s, sh_first_operand(&a, SKIP_ONE), &to); + out = to.mode == SH_AT ? sh_anchor(s->a, to.dir) : NULL; + } + sh_argv_free(&a); + return out; +} + +static bool sh_is_pwd(sh_t *s, TSNode n) { + n = sh_unwrap(n); + if (!sh_type(n, "command")) { + return false; + } + sh_argv_t a = {0}; + sh_argv_build(s, n, &a); + bool pwd = a.n > 0 && sh_eq(a.v[0], "pwd"); + sh_argv_free(&a); + return pwd; +} + +static const char *sh_eval_subst(sh_t *s, TSNode n) { + TSNode st[PAIR_LEN]; + int k = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (sh_type(c, "comment")) { + continue; + } + if (k == PAIR_LEN) { + return NULL; + } + st[k++] = c; + } + if (k == SKIP_ONE && sh_type(st[0], "list") && ts_node_named_child_count(st[0]) == PAIR_LEN) { + TSNode l = st[0]; + st[0] = ts_node_named_child(l, 0); + st[SKIP_ONE] = ts_node_named_child(l, SKIP_ONE); + k = PAIR_LEN; + } + if (k == PAIR_LEN) { + return sh_is_pwd(s, st[SKIP_ONE]) ? sh_subst_cd(s, st[0]) : NULL; + } + if (k == SKIP_ONE) { + TSNode c = sh_unwrap(st[0]); + return sh_type(c, "command") ? sh_eval_subst_command(s, c) : NULL; + } + return NULL; +} + +static const char *sh_eval(sh_t *s, TSNode n) { + if (ts_node_is_null(n)) { + return NULL; + } + const char *t = ts_node_type(n); + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + if (strcmp(t, "word") == 0 || strcmp(t, "number") == 0 || strcmp(t, "extglob_pattern") == 0) { + if (s->src[b] == '~') { + return NULL; + } + sh_str_t out = {0}; + sh_add_unescaped(s->a, &out, s->src + b, e - b, false); + return sh_str_done(&out); + } + if (strcmp(t, "raw_string") == 0) { + return e - b >= PAIR_LEN ? cbm_arena_strndup(s->a, s->src + b + SKIP_ONE, e - b - PAIR_LEN) + : NULL; + } + if (strcmp(t, "string") == 0 || strcmp(t, "concatenation") == 0) { + sh_str_t out = {0}; + bool dq = strcmp(t, "string") == 0; + uint32_t from = dq ? b + SKIP_ONE : b; + uint32_t to = dq && e > b + SKIP_ONE ? e - SKIP_ONE : e; + sh_concat(s, &out, n, from, to, dq); + return sh_str_done(&out); + } + if (strcmp(t, "simple_expansion") == 0) { + TSNode v = ts_node_named_child(n, 0); + const char *nm = ts_node_is_null(v) ? NULL : sh_text(s, v); + if (!nm) { + return NULL; + } + bool found = false; + const sh_val_t *val = sh_lookup(s, nm, &found); + return found ? val->s : NULL; + } + if (strcmp(t, "expansion") == 0) { + return sh_eval_expansion(s, n); + } + if (strcmp(t, "command_substitution") == 0) { + return sh_eval_subst(s, n); + } + return NULL; +} + +/* A command word: its value; an option whose value is unknown is still that + * option (--context="${ctx}" -> "--context="). */ +static const char *sh_eval_word(sh_t *s, TSNode w) { + const char *v = sh_eval(s, w); + if (v || s->src[ts_node_start_byte(w)] != '-') { + return v; + } + uint32_t b = ts_node_start_byte(w); + uint32_t e = b; + while (e < ts_node_end_byte(w) && !strchr("$\"'`\\", s->src[e])) { + e++; + } + return cbm_arena_strndup(s->a, s->src + b, e - b); +} + +/* "${name[@]}" / ${name[*]} as a whole word: the array, NULL when not. */ +static const sh_val_t *sh_array_word(sh_t *s, TSNode w) { + if (sh_type(w, "string") && ts_node_named_child_count(w) == SKIP_ONE) { + w = ts_node_named_child(w, 0); + } + if (!sh_type(w, "expansion") || ts_node_named_child_count(w) != SKIP_ONE) { + return NULL; + } + TSNode sub = ts_node_named_child(w, 0); + if (!sh_type(sub, "subscript")) { + return NULL; + } + TSNode base = ts_node_child_by_field_name(sub, TS_FIELD("name")); + TSNode idx = ts_node_child_by_field_name(sub, TS_FIELD("index")); + const char *it = ts_node_is_null(idx) ? NULL : sh_text(s, idx); + if (ts_node_is_null(base) || !it || (strcmp(it, "@") != 0 && strcmp(it, "*") != 0)) { + return NULL; + } + bool found = false; + const sh_val_t *v = sh_lookup(s, sh_text(s, base), &found); + return v; +} + +/* An unquoted expansion may split into several words. */ +static bool sh_may_split(TSNode w) { + const char *t = ts_node_type(w); + if (strcmp(t, "simple_expansion") == 0 || strcmp(t, "expansion") == 0 || + strcmp(t, "command_substitution") == 0) { + return true; + } + if (strcmp(t, "concatenation") == 0) { + for (uint32_t i = 0; i < ts_node_named_child_count(w); i++) { + if (sh_may_split(ts_node_named_child(w, i))) { + return true; + } + } + } + return false; +} + +static void sh_argv_push(sh_argv_t *a, const char *v, bool split) { + if (a->n >= a->cap) { + int ncap = a->cap ? a->cap * PAIR_LEN : SH_GROW_MIN; + const char **nv = + cbm_realloc(CBM_MEM_CLASS_OTHER, (void *)a->v, (size_t)ncap * sizeof(*nv)); + if (!nv) { + return; + } + a->v = nv; + bool *ns = cbm_realloc(CBM_MEM_CLASS_OTHER, a->split, (size_t)ncap * sizeof(*ns)); + if (!ns) { + return; + } + a->split = ns; + a->cap = ncap; + } + a->v[a->n] = v; + a->split[a->n] = split; + a->n++; +} + +static void sh_argv_word(sh_t *s, TSNode w, sh_argv_t *out) { + const sh_val_t *arr = sh_array_word(s, w); + if (arr) { + if (!arr->items) { + sh_argv_push(out, NULL, true); + return; + } + for (int i = 0; i < arr->n; i++) { + sh_argv_push(out, arr->items[i], false); + } + return; + } + const char *v = sh_eval_word(s, w); + sh_argv_push(out, v, v ? false : sh_may_split(w)); +} + +/* argv of a command: its name and arguments, evaluated (NULL: unknown). */ +static void sh_argv_build(sh_t *s, TSNode cmd, sh_argv_t *out) { + uint32_t cc = ts_node_child_count(cmd); + for (uint32_t i = 0; i < cc; i++) { + const char *field = ts_node_field_name_for_child(cmd, i); + TSNode c = ts_node_child(cmd, i); + if (sh_eq(field, "name")) { + TSNode w = ts_node_named_child(c, 0); + if (!ts_node_is_null(w)) { + sh_argv_word(s, w, out); + } + } else if (sh_eq(field, "argument")) { + sh_argv_word(s, c, out); + } + } +} + +/* ── Paths to targets ──────────────────────────────────────────────── */ + +typedef struct { + bool recursive; + const char *name; /* find -name glob; NULL: any */ + int maxdepth; /* find -maxdepth; SH_NO_DEPTH: none */ +} sh_find_t; + +typedef int (*sh_target_fn)(sh_t *s, const char *path, const sh_find_t *f, sh_targets_t *out); + +/* Candidate repository paths of a value: the anchored path; a relative one + * in the tracked directory, or -- none tracked -- in the root and in the + * script's directory. */ +static int sh_candidates(sh_t *s, const char *v, const char *out[PAIR_LEN]) { + if (!v || !v[0] || strstr(v, "://")) { + return 0; + } + if (v[0] == SH_ANCHOR) { + out[0] = sh_join(s->a, "", v + SKIP_ONE); + return out[0] ? SKIP_ONE : 0; + } + if (v[0] == '/' || v[0] == '~') { + return 0; + } + if (s->cwd.mode == SH_LOST) { + return 0; + } + if (s->cwd.mode == SH_AT) { + out[0] = sh_join(s->a, s->cwd.dir, v); + return out[0] ? SKIP_ONE : 0; + } + int n = 0; + const char *r = sh_join(s->a, "", v); + const char *d = sh_join(s->a, s->sdir, v); + if (r) { + out[n++] = r; + } + if (d && (!r || strcmp(d, r) != 0)) { + out[n++] = d; + } + return n; +} + +/* The targets of value `v`: of its one candidate, or of the only one of two + * candidates that has any. */ +static void sh_targets_of(sh_t *s, const char *v, sh_target_fn fn, const sh_find_t *f, + sh_targets_t *out) { + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, v, c); + int mark = out->count; + int found = 0; + for (int i = 0; i < n; i++) { + if (fn(s, c[i], f, out) > 0) { + found++; + } + } + if (found > SKIP_ONE) { + out->count = mark; + } +} + +static bool sh_glob_seg(const char *p, size_t pn, const char *t, size_t tn) { + size_t pi = 0; + size_t ti = 0; + size_t star = (size_t)-1; + size_t mark = 0; + while (ti < tn) { + if (pi < pn && (p[pi] == '?' || p[pi] == t[ti])) { + pi++; + ti++; + } else if (pi < pn && p[pi] == '*') { + star = pi++; + mark = ti; + } else if (star != (size_t)-1) { + pi = star + SKIP_ONE; + ti = ++mark; + } else { + return false; + } + } + while (pi < pn && p[pi] == '*') { + pi++; + } + return pi == pn; +} + +/* A path glob: * and ? within a segment, segment for segment. */ +static bool sh_glob_path(const char *pat, const char *path) { + for (;;) { + size_t pl = strcspn(pat, "/"); + size_t tl = strcspn(path, "/"); + if (!sh_glob_seg(pat, pl, path, tl)) { + return false; + } + if (!pat[pl] || !path[tl]) { + return !pat[pl] && !path[tl]; + } + pat += pl + SKIP_ONE; + path += tl + SKIP_ONE; + } +} + +static int sh_add_manifest(sh_t *s, const char *p, sh_targets_t *out) { + int64_t id = (int64_t)(intptr_t)cbm_ht_get(s->ix->file_ids, p); + if (id <= 0) { + return 0; + } + sh_targets_push(out, id, p); + return SKIP_ONE; +} + +/* kubectl -f PATH: the manifest, a directory's manifests, a glob's. */ +static int sh_t_files(sh_t *s, const char *path, const sh_find_t *f, sh_targets_t *out) { + const sh_paths_t *m = &s->ix->manifests; + char key[CBM_SZ_4K]; + int lo = 0; + int hi = 0; + int added = 0; + if (strpbrk(path, "*?[")) { + if (strchr(path, '[')) { + return 0; + } + const char *g = strpbrk(path, "*?"); + size_t dl = (size_t)(g - path); + while (dl > 0 && path[dl - SKIP_ONE] != '/') { + dl--; + } + char dir[CBM_SZ_4K]; + snprintf(dir, sizeof(dir), "%.*s", (int)(dl ? dl - SKIP_ONE : 0), path); + sh_range(m, dir, key, sizeof(key), &lo, &hi); + for (int i = lo; i < hi; i++) { + atomic_fetch_add_explicit(&g_sh_candidates, SKIP_ONE, memory_order_relaxed); + if (sh_glob_path(path, m->items[i])) { + added += sh_add_manifest(s, m->items[i], out); + } + } + return added; + } + if (sh_paths_has(m, path)) { + return sh_add_manifest(s, path, out); + } + sh_range(m, path, key, sizeof(key), &lo, &hi); + size_t kl = strlen(key); + for (int i = lo; i < hi; i++) { + atomic_fetch_add_explicit(&g_sh_candidates, SKIP_ONE, memory_order_relaxed); + if (f->recursive || !strchr(m->items[i] + kl, '/')) { + added += sh_add_manifest(s, m->items[i], out); + } + } + return added; +} + +/* find DIR -name G: DIR's manifests (to -maxdepth) whose name matches. */ +static int sh_t_found(sh_t *s, const char *dir, const sh_find_t *f, sh_targets_t *out) { + const sh_paths_t *m = &s->ix->manifests; + char key[CBM_SZ_4K]; + int lo = 0; + int hi = 0; + int added = 0; + if (!sh_dir_exists(s->ix, dir)) { + return 0; + } + sh_range(m, dir, key, sizeof(key), &lo, &hi); + size_t kl = strlen(key); + for (int i = lo; i < hi; i++) { + atomic_fetch_add_explicit(&g_sh_candidates, SKIP_ONE, memory_order_relaxed); + const char *rest = m->items[i] + kl; + int depth = SKIP_ONE; + for (const char *p = rest; *p; p++) { + depth += *p == '/'; + } + const char *base = sh_basename(rest); + if ((f->maxdepth == SH_NO_DEPTH || depth <= f->maxdepth) && + (!f->name || sh_glob_seg(f->name, strlen(f->name), base, strlen(base)))) { + added += sh_add_manifest(s, m->items[i], out); + } + } + return added; +} + +/* -k DIR / kustomize build DIR: the Kustomization of DIR. */ +static int sh_t_kust(sh_t *s, const char *dir, const sh_find_t *f, sh_targets_t *out) { + (void)f; + static const char *const names[] = {"kustomization.yaml", "kustomization.yml", "Kustomization"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + const char *p = cbm_arena_sprintf(s->a, "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + const char *key = p ? cbm_ht_get_key(s->ix->kusts, p) : NULL; + if (key) { + sh_targets_push(out, (int64_t)(intptr_t)cbm_ht_get(s->ix->kusts, key), dir); + return SKIP_ONE; + } + } + return 0; +} + +/* A chart directory: its Chart. */ +static int sh_t_chart(sh_t *s, const char *dir, const sh_find_t *f, sh_targets_t *out) { + (void)f; + static const char *const names[] = {"Chart.yaml", "Chart.yml"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + const char *p = cbm_arena_sprintf(s->a, "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + const char *key = p ? cbm_ht_get_key(s->ix->charts, p) : NULL; + if (key) { + sh_targets_push(out, (int64_t)(intptr_t)cbm_ht_get(s->ix->charts, key), dir); + return SKIP_ONE; + } + } + return 0; +} + +/* Where `cd v` lands: *out = AT an existing directory, else LOST. */ +static const char *sh_cd_target(sh_t *s, const char *v, sh_cwd_t *out) { + *out = (sh_cwd_t){SH_LOST, NULL}; + if (!v) { + return NULL; + } + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, v, c); + const char *hit = NULL; + for (int i = 0; i < n; i++) { + if (sh_dir_exists(s->ix, c[i])) { + if (hit) { + return NULL; + } + hit = c[i]; + } + } + if (hit) { + *out = (sh_cwd_t){SH_AT, hit}; + } + return hit; +} + +/* ── Deploy commands ───────────────────────────────────────────────── */ + +static void sh_hit(sh_t *s, TSNode cmd, const char *via, const sh_target_t *t, const char *ext, + bool chart) { + sh_hits_t *h = s->hits; + if (!sh_grow((void **)&h->items, &h->cap, h->count, sizeof(*h->items))) { + return; + } + const char *path = t ? t->path : ext; + h->items[h->count++] = + (sh_hit_t){.file = s->file, + .line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base, + .via = via, + .tgt = t ? t->id : 0, + .ext = ext ? cbm_arena_strdup(&h->arena, ext) : NULL, + .path = path ? cbm_arena_strdup(&h->arena, path) : NULL, + .chart = chart}; +} + +static void sh_hit_all(sh_t *s, TSNode cmd, const char *via, const sh_targets_t *ts, bool chart) { + for (int i = 0; i < ts->count; i++) { + sh_hit(s, cmd, via, &ts->items[i], NULL, chart); + } +} + +static bool sh_in(const char *v, const char *const *set) { + for (int i = 0; v && set[i]; i++) { + if (strcmp(v, set[i]) == 0) { + return true; + } + } + return false; +} + +static bool sh_numberish(const char *v) { + if (!v[0] || v[0] < '0' || v[0] > '9') { + return false; + } + for (const char *p = v; *p; p++) { + if (!((*p >= '0' && *p <= '9') || *p == '.' || strchr("smhd", *p))) { + return false; + } + } + return true; +} + +/* The index of the program argv runs, past wrappers (time, sudo, retry N, + * timeout D, env A=B, ...); -1 when it is unknown. */ +static int sh_program(const sh_argv_t *a) { + static const char *const wrappers[] = {"time", "sudo", "exec", "nice", "nohup", + "command", "env", "timeout", "retry", NULL}; + int i = 0; + while (i < a->n) { + if (!a->v[i]) { + return -1; + } + if (!sh_in(sh_basename(a->v[i]), wrappers)) { + return i; + } + i++; + while (i < a->n && a->v[i] && + (a->v[i][0] == '-' || strchr(a->v[i], '=') || sh_numberish(a->v[i]))) { + i++; + } + } + return -1; +} + +/* kubectl's options that take a separate value (before the verb). */ +static const char *const SH_KUBECTL_VALUED[] = {"-n", + "--namespace", + "--context", + "--kubeconfig", + "--cluster", + "--user", + "-l", + "--selector", + "-o", + "--output", + "--server", + "-s", + "--token", + "--as", + "--as-group", + "--request-timeout", + "--cache-dir", + "--certificate-authority", + "--client-certificate", + "--client-key", + NULL}; + +/* helm's options that take a separate value. */ +static const char *const SH_HELM_VALUED[] = {"-n", + "--namespace", + "-f", + "--values", + "--set", + "--set-string", + "--set-file", + "--set-json", + "--set-literal", + "--version", + "--timeout", + "--kubeconfig", + "--kube-context", + "--repo", + "--post-renderer", + "--post-renderer-args", + "--description", + "--username", + "--password", + "--ca-file", + "--cert-file", + "--key-file", + "--keyring", + "--history-max", + "--name-template", + "--registry-config", + "--repository-config", + "--repository-cache", + "-o", + "--output", + "--api-versions", + "-a", + "--kube-version", + "--labels", + "-l", + "--show-only", + "-s", + "--output-dir", + "--release-name", + "--kube-apiserver", + "--kube-as-user", + "--kube-as-group", + "--kube-ca-file", + "--kube-token", + "--burst-limit", + "--qps", + NULL}; + +typedef enum { SH_SRC_NONE, SH_SRC_KUST, SH_SRC_CHART } sh_src_kind_t; + +typedef struct { + sh_src_kind_t kind; + const char *v; + const char *via; +} sh_src_t; + +typedef struct { + int verb; /* index of the verb in argv; -1 none */ + const char **files; + int nfiles; + const char **kusts; + int nkusts; + int cap_files; + int cap_kusts; + bool recursive; +} sh_kubectl_t; + +static void sh_kubectl_free(sh_kubectl_t *k) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)k->files); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)k->kusts); +} + +static void sh_kubectl_add(const char ***v, int *n, int *cap, const char *p) { + if (sh_grow((void **)v, cap, *n, sizeof(**v))) { + (*v)[(*n)++] = p; + } +} + +/* kubectl [options] VERB ...: the verb and its -f / -k / -R. */ +static void sh_kubectl_parse(const sh_argv_t *a, int from, sh_kubectl_t *k) { + k->verb = -1; + int i = from; + while (i < a->n && k->verb < 0) { + const char *w = a->v[i]; + if (!w) { + return; + } + if (w[0] != '-') { + k->verb = i++; + break; + } + i += (sh_in(w, SH_KUBECTL_VALUED) && !strchr(w, '=')) ? PAIR_LEN : SKIP_ONE; + } + for (; i < a->n; i++) { + const char *w = a->v[i]; + if (!w || w[0] != '-') { + continue; + } + const char *next = i + SKIP_ONE < a->n ? a->v[i + SKIP_ONE] : NULL; + if (strcmp(w, "--recursive") == 0) { + k->recursive = true; + } else if (strcmp(w, "--filename") == 0 || strcmp(w, "--kustomize") == 0) { + bool fl = w[PAIR_LEN] == 'f'; + sh_kubectl_add(fl ? &k->files : &k->kusts, fl ? &k->nfiles : &k->nkusts, + fl ? &k->cap_files : &k->cap_kusts, next); + i++; + } else if (sh_starts(w, "--filename=")) { + sh_kubectl_add(&k->files, &k->nfiles, &k->cap_files, strchr(w, '=') + SKIP_ONE); + } else if (sh_starts(w, "--kustomize=")) { + sh_kubectl_add(&k->kusts, &k->nkusts, &k->cap_kusts, strchr(w, '=') + SKIP_ONE); + } else if (w[SKIP_ONE] != '-') { + /* a cluster of short options: -f, -k, -R, -Rf */ + size_t len = strlen(w); + bool ok = len > SKIP_ONE; + for (size_t c = SKIP_ONE; c < len; c++) { + ok = ok && ((w[c] >= 'a' && w[c] <= 'z') || (w[c] >= 'A' && w[c] <= 'Z')); + } + if (!ok) { + continue; + } + if (strchr(w, 'R')) { + k->recursive = true; + } + char last = w[len - SKIP_ONE]; + if (last == 'f' || last == 'k') { + sh_kubectl_add(last == 'f' ? &k->files : &k->kusts, + last == 'f' ? &k->nfiles : &k->nkusts, + last == 'f' ? &k->cap_files : &k->cap_kusts, next); + i++; + } + } + } +} + +static void sh_deploy_src(sh_t *s, TSNode cmd, const sh_src_t *src); + +/* kubectl: apply / create / replace -f / -k, or `kubectl kustomize D` as a + * pipe's source. */ +static void sh_kubectl(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog, const sh_src_t *in, + sh_src_t *out) { + sh_kubectl_t k = {0}; + sh_kubectl_parse(a, prog + SKIP_ONE, &k); + const char *verb = k.verb >= 0 ? a->v[k.verb] : NULL; + if (sh_eq(verb, "kustomize")) { + const char *d = sh_first_operand(a, k.verb + SKIP_ONE); + *out = (sh_src_t){SH_SRC_KUST, d ? d : ".", "kustomize"}; + } else if (sh_eq(verb, "apply") || sh_eq(verb, "create") || sh_eq(verb, "replace")) { + sh_find_t f = {k.recursive, NULL, SH_NO_DEPTH}; + for (int i = 0; i < k.nfiles; i++) { + if (sh_eq(k.files[i], "-")) { + sh_deploy_src(s, cmd, in); + continue; + } + sh_targets_t ts = {0}; + sh_targets_of(s, k.files[i], sh_t_files, &f, &ts); + sh_hit_all(s, cmd, "kubectl", &ts, false); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + } + for (int i = 0; i < k.nkusts; i++) { + sh_targets_t ts = {0}; + sh_targets_of(s, k.kusts[i], sh_t_kust, &f, &ts); + sh_hit_all(s, cmd, "kubectl", &ts, false); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + } + } + sh_kubectl_free(&k); +} + +/* The chart a helm word names: a chart dir, else an external chart. */ +static void sh_chart(sh_t *s, TSNode cmd, const char *v) { + if (!v) { + return; + } + if (sh_starts(v, "oci://")) { + const char *ext = sh_value_basename(s->a, v + strlen("oci://")); + if (ext && ext[0] && !strchr(ext, ':')) { + sh_hit(s, cmd, "helm", NULL, ext, true); + } + return; + } + sh_targets_t ts = {0}; + sh_targets_of(s, v, sh_t_chart, NULL, &ts); + if (ts.count > 0) { + sh_hit_all(s, cmd, "helm", &ts, true); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + /* repo/name: one slash, a plain name on each side */ + const char *slash = strchr(v, '/'); + if (!slash || strchr(slash + SKIP_ONE, '/') || v[0] == '.' || slash == v || !slash[SKIP_ONE]) { + return; + } + for (const char *p = v; *p; p++) { + if (!(*p == '/' || *p == '.' || *p == '_' || *p == '-' || (*p >= '0' && *p <= '9') || + (*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z'))) { + return; + } + } + sh_hit(s, cmd, "helm", NULL, slash + SKIP_ONE, true); +} + +/* helm install|upgrade [NAME] CHART; helm template [NAME] CHART as a source. */ +static void sh_helm(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog, sh_src_t *out) { + const char *verb = NULL; + int pos[PAIR_LEN] = {0, 0}; + int npos = 0; + int first_split = SH_NO_DEPTH; /* an unknown unquoted word: maybe several */ + bool gen = false; + for (int i = prog + SKIP_ONE; i < a->n; i++) { + const char *w = a->v[i]; + if (w && w[0] == '-') { + gen = gen || strcmp(w, "--generate-name") == 0 || strcmp(w, "-g") == 0; + if (sh_in(w, SH_HELM_VALUED) && !strchr(w, '=')) { + i++; + } + continue; + } + if (!verb) { + if (!w) { + return; + } + verb = w; + continue; + } + if (!w && a->split[i] && first_split == SH_NO_DEPTH) { + first_split = npos; + } + if (npos < PAIR_LEN) { + pos[npos] = i; + } + npos++; + } + if (!verb) { + return; + } + if (strcmp(verb, "install") == 0 || strcmp(verb, "upgrade") == 0) { + int at = gen ? 0 : SKIP_ONE; + if (npos > at && (first_split == SH_NO_DEPTH || first_split > at)) { + sh_chart(s, cmd, a->v[pos[at]]); + } + } else if (strcmp(verb, "template") == 0 && npos > 0 && + (first_split == SH_NO_DEPTH || first_split > SKIP_ONE)) { + int at = npos >= PAIR_LEN ? SKIP_ONE : 0; + *out = (sh_src_t){SH_SRC_CHART, a->v[pos[at]], "helm"}; + } +} + +/* What a `kubectl apply -f -` reading the pipe deploys. */ +static void sh_deploy_src(sh_t *s, TSNode cmd, const sh_src_t *src) { + if (!src || src->kind == SH_SRC_NONE || !src->v) { + return; + } + if (src->kind == SH_SRC_CHART) { + sh_chart(s, cmd, src->v); + return; + } + sh_targets_t ts = {0}; + sh_targets_of(s, src->v, sh_t_kust, NULL, &ts); + sh_hit_all(s, cmd, src->via, &ts, false); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); +} + +/* find DIR [-type f] [-name G] [-maxdepth N] -exec kubectl apply -f {} \; */ +static void sh_find(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog) { + int i = prog + SKIP_ONE; + if (i >= a->n || !a->v[i] || a->v[i][0] == '-') { + return; + } + const char *root = a->v[i++]; + sh_find_t f = {false, NULL, SH_NO_DEPTH}; + while (i < a->n) { + const char *w = a->v[i]; + const char *next = i + SKIP_ONE < a->n ? a->v[i + SKIP_ONE] : NULL; + if (!w) { + return; + } + if (strcmp(w, "-exec") == 0) { + break; + } + if (strcmp(w, "-name") == 0 && next) { + f.name = next; + } else if (strcmp(w, "-maxdepth") == 0 && next && sh_numberish(next)) { + f.maxdepth = (int)strtol(next, NULL, SH_DEC); + } else if (!(strcmp(w, "-type") == 0 && sh_eq(next, "f"))) { + return; + } + i += PAIR_LEN; /* each test takes one value */ + } + /* -exec PROGRAM ... {} ; */ + sh_argv_t e = {0}; + for (i++; i < a->n && !sh_eq(a->v[i], ";") && !sh_eq(a->v[i], "+"); i++) { + sh_argv_push(&e, a->v[i], a->split[i]); + } + int p = sh_program(&e); + if (p >= 0 && strcmp(sh_basename(e.v[p]), "kubectl") == 0) { + sh_kubectl_t k = {0}; + sh_kubectl_parse(&e, p + SKIP_ONE, &k); + const char *verb = k.verb >= 0 ? e.v[k.verb] : NULL; + if ((sh_eq(verb, "apply") || sh_eq(verb, "create") || sh_eq(verb, "replace")) && + k.nfiles == SKIP_ONE && k.nkusts == 0 && sh_eq(k.files[0], "{}")) { + sh_targets_t ts = {0}; + sh_targets_of(s, root, sh_t_found, &f, &ts); + sh_hit_all(s, cmd, "kubectl", &ts, false); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + } + sh_kubectl_free(&k); + } + sh_argv_free(&e); +} + +/* One command of a pipeline (or a lone one): its deploys; *out: what it + * writes to the pipe, when that is a kustomize build or a helm template. */ +static void sh_deploy_command(sh_t *s, TSNode cmd, const sh_argv_t *a, const sh_src_t *in, + sh_src_t *out) { + *out = (sh_src_t){SH_SRC_NONE, NULL, NULL}; + int prog = sh_program(a); + if (prog < 0) { + return; + } + const char *base = sh_basename(a->v[prog]); + if (strcmp(base, "kubectl") == 0) { + sh_kubectl(s, cmd, a, prog, in, out); + } else if (strcmp(base, "helm") == 0) { + sh_helm(s, cmd, a, prog, out); + } else if (strcmp(base, "kustomize") == 0) { + const char *verb = sh_first_operand(a, prog + SKIP_ONE); + if (sh_eq(verb, "build")) { + int at = SH_NO_DEPTH; + for (int i = prog + SKIP_ONE; i < a->n; i++) { + if (a->v[i] == verb) { + at = i; + break; + } + } + const char *d = at >= 0 ? sh_first_operand(a, at + SKIP_ONE) : NULL; + *out = (sh_src_t){SH_SRC_KUST, d ? d : ".", "kustomize"}; + } + } else if (strcmp(base, "find") == 0) { + sh_find(s, cmd, a, prog); + } +} + +/* ── Statements ────────────────────────────────────────────────────── */ + +static void sh_walk(sh_t *s, TSNode n); + +static void sh_walk_children(sh_t *s, TSNode n) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + sh_walk(s, ts_node_named_child(n, i)); + } +} + +static void sh_assign(sh_t *s, TSNode n, bool local) { + TSNode name = ts_node_child_by_field_name(n, TS_FIELD("name")); + TSNode value = ts_node_child_by_field_name(n, TS_FIELD("value")); + if (ts_node_is_null(name)) { + return; + } + if (sh_type(name, "subscript")) { + TSNode base = ts_node_child_by_field_name(name, TS_FIELD("name")); + if (!ts_node_is_null(base)) { + sh_set(s, sh_text(s, base), &SH_UNKNOWN_VAL, local); + } + return; + } + const char *nm = sh_text(s, name); + sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); + if (!nm || !v) { + return; + } + *v = (sh_val_t){"", NULL, 0}; + if (sh_type(value, "array")) { + uint32_t cnt = ts_node_named_child_count(value); + const char **items = cbm_arena_alloc(s->a, (cnt ? cnt : SKIP_ONE) * sizeof(*items)); + if (!items) { + return; + } + int k = 0; + for (uint32_t i = 0; i < cnt; i++) { + TSNode c = ts_node_named_child(value, i); + if (!sh_type(c, "comment")) { + items[k++] = sh_eval_word(s, c); + } + } + *v = (sh_val_t){k > 0 ? items[0] : "", items, k}; + } else if (!ts_node_is_null(value)) { + v->s = sh_eval(s, value); + } + bool append = false; + for (uint32_t i = 0; i < ts_node_child_count(n); i++) { + TSNode c = ts_node_child(n, i); + append = append || (!ts_node_is_named(c) && strcmp(ts_node_type(c), "+=") == 0); + } + if (append) { + bool found = false; + const sh_val_t *prev = sh_lookup(s, nm, &found); + if (found) { + v->s = (prev->s && v->s) ? cbm_arena_sprintf(s->a, "%s%s", prev->s, v->s) : NULL; + v->items = NULL; + } + } + sh_set(s, nm, v, local); +} + +/* local / declare / typeset / export / readonly NAME[=VALUE] ... */ +static void sh_declare(sh_t *s, TSNode n) { + TSNode kw = ts_node_child(n, 0); + const char *k = ts_node_is_null(kw) ? "" : ts_node_type(kw); + bool local = strcmp(k, "local") == 0 || + (s->in_func && (strcmp(k, "declare") == 0 || strcmp(k, "typeset") == 0)); + bool declares = local || strcmp(k, "declare") == 0 || strcmp(k, "typeset") == 0; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (sh_type(c, "variable_assignment")) { + sh_assign(s, c, local); + } else if (declares && sh_type(c, "variable_name")) { + sh_set(s, sh_text(s, c), &SH_UNKNOWN_VAL, local); + } + } +} + +/* cd / pushd / popd; true when the command is one. */ +static bool sh_chdir(sh_t *s, const sh_argv_t *a) { + const char *name = a->n > 0 ? a->v[0] : NULL; + if (sh_eq(name, "popd")) { + s->cwd = s->ndirs > 0 ? s->dirs[--s->ndirs] : (sh_cwd_t){SH_LOST, NULL}; + return true; + } + if (!sh_eq(name, "cd") && !sh_eq(name, "pushd")) { + return false; + } + if (sh_eq(name, "pushd")) { + if (sh_grow((void **)&s->dirs, &s->cap_dirs, s->ndirs, sizeof(*s->dirs))) { + s->dirs[s->ndirs++] = s->cwd; + } + } + const char *to = sh_first_operand(a, SKIP_ONE); + sh_cwd_t next = {SH_LOST, NULL}; + if (to && strcmp(to, "-") != 0) { + (void)sh_cd_target(s, to, &next); + } + s->cwd = next; + return true; +} + +static void sh_command(sh_t *s, TSNode cmd) { + sh_argv_t a = {0}; + sh_argv_build(s, cmd, &a); + if (!sh_chdir(s, &a)) { + sh_src_t out; + sh_deploy_command(s, cmd, &a, NULL, &out); + } + sh_argv_free(&a); +} + +/* `cat <in_func++; + if (sh_type(body, "compound_statement")) { + sh_walk_children(s, body); + } else { + sh_walk(s, body); + } + s->in_func--; + sh_touch_t t = {0}; + CBMHashTable *locals = cbm_ht_create(0); + for (int i = m.undo; locals && i < s->nundo; i++) { + if (s->undo[i].local) { + cbm_ht_set(locals, s->undo[i].name, (void *)s->undo[i].name); + } + } + for (int i = m.undo; i < s->nundo; i++) { + if (!locals || !cbm_ht_get(locals, s->undo[i].name)) { + sh_touch_add(&t, s->undo[i].name); + } + } + if (locals) { + cbm_ht_free(locals); + } + t.cwd = !sh_cwd_same(s->cwd, m.cwd) || s->ndirs != m.ndirs; + sh_rewind(s, m); + sh_group_end(s, &t); +} + +static void sh_walk(sh_t *s, TSNode n) { + const char *t = ts_node_type(n); + if (strcmp(t, "program") == 0 || strcmp(t, "compound_statement") == 0 || + strcmp(t, "negated_command") == 0) { + sh_walk_children(s, n); + } else if (strcmp(t, "variable_assignment") == 0) { + sh_assign(s, n, false); + } else if (strcmp(t, "variable_assignments") == 0) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + sh_assign(s, ts_node_named_child(n, i), false); + } + } else if (strcmp(t, "declaration_command") == 0) { + sh_declare(s, n); + } else if (strcmp(t, "unset_command") == 0) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (sh_type(c, "variable_name") || sh_type(c, "word")) { + sh_set(s, sh_text(s, c), &SH_UNKNOWN_VAL, false); + } + } + } else if (strcmp(t, "command") == 0) { + sh_command(s, n); + } else if (strcmp(t, "redirected_statement") == 0) { + TSNode body = ts_node_child_by_field_name(n, TS_FIELD("body")); + if (!ts_node_is_null(body)) { + sh_walk(s, body); + } + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode red = ts_node_named_child(n, i); + if (sh_type(red, "heredoc_redirect")) { + sh_mark_t m = sh_mark(s); + sh_walk_children(s, red); + sh_rewind(s, m); + } + } + } else if (strcmp(t, "pipeline") == 0) { + sh_pipeline(s, n); + } else if (strcmp(t, "list") == 0) { + sh_list(s, n); + } else if (strcmp(t, "subshell") == 0) { + sh_mark_t m = sh_mark(s); + sh_walk_children(s, n); + sh_rewind(s, m); + } else if (strcmp(t, "if_statement") == 0) { + sh_if(s, n); + } else if (strcmp(t, "case_statement") == 0) { + sh_case(s, n); + } else if (strcmp(t, "for_statement") == 0 || strcmp(t, "c_style_for_statement") == 0 || + strcmp(t, "while_statement") == 0) { + sh_loop(s, n); + } else if (strcmp(t, "function_definition") == 0) { + sh_function(s, n); + } +} + +/* ── Files ─────────────────────────────────────────────────────────── */ + +static bool sh_mentions_deploy_tool(const char *src, size_t n) { + static const char *const tools[] = {"kubectl", "helm", "kustomize"}; + for (size_t i = 0; i < n; i++) { + char c = src[i] | ' '; + if (c != 'k' && c != 'h') { + continue; + } + for (size_t t = 0; t < sizeof(tools) / sizeof(tools[0]); t++) { + size_t tl = strlen(tools[t]); + size_t j = 0; + while (j < tl && i + j < n && (src[i + j] | ' ') == tools[t][j]) { + j++; + } + if (j == tl) { + return true; + } + } + } + return false; +} + +static char *sh_read(const char *path, size_t *len) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + enum { SH_TS_PAD = 16 }; /* tree-sitter's lexer reads past the end */ + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + SH_TS_PAD); + if (!buf) { + (void)fclose(f); + return NULL; + } + size_t got = fread(buf, SKIP_ONE, (size_t)size, f); + (void)fclose(f); + memset(buf + got, 0, SH_TS_PAD); + *len = got; + return buf; +} + +typedef struct { + sh_index_t *ix; + sh_hits_t *hits; + TSParser *parser; + bool ix_ready; + bool ix_ok; +} sh_pass_t; + +static void sh_state_init(sh_t *s, sh_pass_t *p, CBMArena *a, const char *rel) { + *s = (sh_t){.ix = p->ix, .hits = p->hits, .a = a, .file = rel}; + s->env = cbm_ht_create(0); + const char *dir = sh_dirname(a, rel); + s->sdir = (dir && strcmp(dir, ".") != 0) ? dir : ""; +} + +static void sh_state_free(sh_t *s) { + if (s->env) { + cbm_ht_free(s->env); + } + cbm_free(CBM_MEM_CLASS_OTHER, s->undo); + cbm_free(CBM_MEM_CLASS_OTHER, s->dirs); +} + +/* Parse `text` as bash and walk it. */ +static void sh_run(sh_t *s, TSParser *parser, const char *text, size_t len) { + if (!ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_BASH))) { + return; + } + TSTree *tree = ts_parser_parse_string(parser, NULL, text, (uint32_t)len); + if (!tree) { + return; + } + s->src = text; + sh_walk(s, ts_tree_root_node(tree)); + ts_tree_delete(tree); +} + +static void sh_script(sh_pass_t *p, const char *rel, const char *src, size_t len) { + CBMArena a; + cbm_arena_init(&a); + sh_t s; + sh_state_init(&s, p, &a, rel); + if (s.env) { + s.script = sh_anchor(&a, rel); + s.cwd = (sh_cwd_t){SH_FREE, NULL}; + sh_run(&s, p->parser, src, len); + } + sh_state_free(&s); + cbm_arena_destroy(&a); +} + +/* ── Make ──────────────────────────────────────────────────────────── */ + +typedef struct { + const char *raw; /* the assigned text */ + const char *expanded; /* memo; NULL: not yet */ + bool known; + bool busy; +} mk_var_t; + +typedef struct { + CBMArena *a; + const char *src; + CBMHashTable *vars; /* name -> mk_var_t */ + bool makefile; /* a Makefile (its recipes run in its dir), not an included .mk */ + TSNode *recipes; + int nrecipes; + int cap_recipes; +} mk_t; + +static const char MK_UNKNOWN[] = "${__cbm_unknown}"; + +static const char *mk_expand(mk_t *m, const char *text); + +static const char *mk_lookup(mk_t *m, const char *name) { + if (strcmp(name, "CURDIR") == 0) { + return m->makefile ? "${__cbm_curdir}" : MK_UNKNOWN; + } + mk_var_t *v = cbm_ht_get(m->vars, name); + if (!v || !v->known || v->busy) { + return MK_UNKNOWN; + } + if (!v->expanded) { + v->busy = true; + v->expanded = mk_expand(m, v->raw); + v->busy = false; + } + return v->expanded ? v->expanded : MK_UNKNOWN; +} + +static const char *mk_trim(CBMArena *a, const char *p, size_t n) { + while (n && (*p == ' ' || *p == '\t')) { + p++; + n--; + } + while (n && (p[n - SKIP_ONE] == ' ' || p[n - SKIP_ONE] == '\t')) { + n--; + } + return cbm_arena_strndup(a, p, n); +} + +/* `cmd` is exactly the words `words` (NULL-terminated), however spaced. */ +static bool mk_words(const char *cmd, const char *const *words) { + const char *p = cmd; + for (int i = 0; words[i]; i++) { + while (*p == ' ' || *p == '\t') { + p++; + } + size_t wl = strlen(words[i]); + if (strncmp(p, words[i], wl) != 0 || (p[wl] && p[wl] != ' ' && p[wl] != '\t')) { + return false; + } + p += wl; + } + while (*p == ' ' || *p == '\t') { + p++; + } + return *p == '\0'; +} + +/* $(NAME) / ${NAME} / $(shell pwd) / $(abspath X): its expansion. */ +static const char *mk_reference(mk_t *m, const char *ref) { + static const char *const pwd[] = {"pwd", NULL}; + static const char *const pwd_p[] = {"pwd", "-P", NULL}; + static const char *const toplevel[] = {"git", "rev-parse", "--show-toplevel", NULL}; + size_t rl = strcspn(ref, " \t"); + if (!ref[rl]) { + return mk_lookup(m, ref); + } + const char *fn = cbm_arena_strndup(m->a, ref, rl); + const char *arg = mk_trim(m->a, ref + rl, strlen(ref + rl)); + if (!fn || !arg) { + return MK_UNKNOWN; + } + if (strcmp(fn, "shell") == 0) { + const char *cmd = mk_expand(m, arg); + if (cmd && (mk_words(cmd, pwd) || mk_words(cmd, pwd_p))) { + return m->makefile ? "${__cbm_curdir}" : MK_UNKNOWN; + } + if (cmd && mk_words(cmd, toplevel)) { + return "${__cbm_root}"; + } + return MK_UNKNOWN; + } + if (strcmp(fn, "abspath") == 0 || strcmp(fn, "realpath") == 0) { + const char *v = mk_expand(m, arg); + return v && !strchr(v, ' ') ? v : MK_UNKNOWN; + } + return MK_UNKNOWN; +} + +/* Make's expansion of `text`: references replaced, $$ -> $, what is not + * known -> a shell variable no line sets. */ +static const char *mk_expand(mk_t *m, const char *text) { + sh_str_t b = {0}; + size_t n = strlen(text); + size_t i = 0; + while (i < n && !b.bad) { + if (text[i] != '$') { + size_t j = i; + while (j < n && text[j] != '$') { + j++; + } + sh_str_add(m->a, &b, text + i, j - i); + i = j; + continue; + } + if (i + SKIP_ONE >= n) { + sh_str_add(m->a, &b, "$", SKIP_ONE); + i++; + continue; + } + char c = text[i + SKIP_ONE]; + if (c == '$') { + sh_str_add(m->a, &b, "$", SKIP_ONE); + i += PAIR_LEN; + continue; + } + if (c != '(' && c != '{') { + sh_str_add(m->a, &b, MK_UNKNOWN, strlen(MK_UNKNOWN)); + i += PAIR_LEN; + continue; + } + char close = c == '(' ? ')' : '}'; + int depth = 0; + size_t j = i + SKIP_ONE; + for (; j < n; j++) { + if (text[j] == c) { + depth++; + } else if (text[j] == close && --depth == 0) { + break; + } + } + if (j >= n) { + sh_str_add(m->a, &b, MK_UNKNOWN, strlen(MK_UNKNOWN)); + break; + } + const char *ref = cbm_arena_strndup(m->a, text + i + PAIR_LEN, j - i - PAIR_LEN); + const char *v = ref ? mk_reference(m, ref) : MK_UNKNOWN; + sh_str_add(m->a, &b, v, strlen(v)); + i = j + SKIP_ONE; + } + return sh_str_done(&b); +} + +static void mk_assign(mk_t *m, TSNode n, bool cond) { + TSNode name = ts_node_child_by_field_name(n, TS_FIELD("name")); + TSNode value = ts_node_child_by_field_name(n, TS_FIELD("value")); + if (ts_node_is_null(name)) { + return; + } + uint32_t nb = ts_node_start_byte(name); + const char *nm = mk_trim(m->a, m->src + nb, ts_node_end_byte(name) - nb); + const char *raw = ""; + if (!ts_node_is_null(value)) { + uint32_t vb = ts_node_start_byte(value); + raw = mk_trim(m->a, m->src + vb, ts_node_end_byte(value) - vb); + } + const char *op = "="; + for (uint32_t i = 0; i < ts_node_child_count(n); i++) { + TSNode c = ts_node_child(n, i); + if (!ts_node_is_named(c) && strchr(ts_node_type(c), '=')) { + op = ts_node_type(c); + } + } + if (!nm || !raw) { + return; + } + mk_var_t *old = cbm_ht_get(m->vars, nm); + if (strcmp(op, "?=") == 0 && old) { + return; + } + mk_var_t *v = cbm_arena_alloc(m->a, sizeof(*v)); + if (!v) { + return; + } + *v = (mk_var_t){raw, NULL, !cond && strcmp(op, "!=") != 0, false}; + if (strcmp(op, "+=") == 0 && old) { + v->known = v->known && old->known; + v->raw = cbm_arena_sprintf(m->a, "%s %s", old->raw, raw); + } + cbm_ht_set(m->vars, nm, v); +} + +/* Variables and recipe lines of a makefile; `cond`: under a conditional. */ +static void mk_collect(mk_t *m, TSNode n, bool cond) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + const char *t = ts_node_type(c); + if (strcmp(t, "variable_assignment") == 0) { + mk_assign(m, c, cond); + } else if (strcmp(t, "rule") == 0) { + TSNode recipe = {0}; + for (uint32_t j = 0; j < ts_node_named_child_count(c); j++) { + if (sh_type(ts_node_named_child(c, j), "recipe")) { + recipe = ts_node_named_child(c, j); + } + } + for (uint32_t j = 0; !ts_node_is_null(recipe) && j < ts_node_named_child_count(recipe); + j++) { + TSNode line = ts_node_named_child(recipe, j); + if (sh_type(line, "recipe_line") && sh_grow((void **)&m->recipes, &m->cap_recipes, + m->nrecipes, sizeof(*m->recipes))) { + m->recipes[m->nrecipes++] = line; + } + } + } else if (strcmp(t, "define_directive") == 0) { + TSNode name = ts_node_child_by_field_name(c, TS_FIELD("name")); + if (!ts_node_is_null(name)) { + uint32_t nb = ts_node_start_byte(name); + const char *nm = mk_trim(m->a, m->src + nb, ts_node_end_byte(name) - nb); + mk_var_t *v = cbm_arena_alloc(m->a, sizeof(*v)); + if (nm && v) { + *v = (mk_var_t){"", NULL, false, false}; + cbm_ht_set(m->vars, nm, v); + } + } + } else if (strstr(t, "conditional") || strstr(t, "if") == t || + strcmp(t, "else_directive") == 0 || strcmp(t, "elsif_directive") == 0) { + mk_collect(m, c, true); + } else if (strstr(t, "directive")) { + mk_collect(m, c, cond); + } + } +} + +static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t len) { + if (!ts_parser_set_language(p->parser, cbm_ts_language(CBM_LANG_MAKEFILE))) { + return; + } + TSTree *tree = ts_parser_parse_string(p->parser, NULL, src, (uint32_t)len); + if (!tree) { + return; + } + CBMArena a; + cbm_arena_init(&a); + const char *base = sh_basename(rel); + mk_t m = {.a = &a, + .src = src, + .vars = cbm_ht_create(0), + .makefile = strcmp(base, "Makefile") == 0 || strcmp(base, "GNUmakefile") == 0 || + strcmp(base, "makefile") == 0}; + if (m.vars) { + mk_collect(&m, ts_tree_root_node(tree), false); + } + for (int r = 0; m.vars && r < m.nrecipes; r++) { + TSNode line = m.recipes[r]; + uint32_t b = ts_node_start_byte(line); + uint32_t e = ts_node_end_byte(line); + while (b < e && (src[b] == ' ' || src[b] == '\t' || src[b] == '@' || src[b] == '-' || + src[b] == '+')) { + b++; + } + const char *raw = cbm_arena_strndup(&a, src + b, e - b); + const char *text = raw ? mk_expand(&m, raw) : NULL; + if (!text || !sh_mentions_deploy_tool(text, strlen(text))) { + continue; + } + size_t tl = strlen(text); + char *buf = cbm_arena_alloc(&a, tl + CBM_SZ_16); + if (!buf) { + continue; + } + memcpy(buf, text, tl); + memset(buf + tl, 0, CBM_SZ_16); + sh_t s; + sh_state_init(&s, p, &a, rel); + if (s.env) { + s.line_base = (int)ts_node_start_point(line).row; + sh_val_t *root = cbm_arena_alloc(&a, sizeof(*root)); + sh_val_t *cur = cbm_arena_alloc(&a, sizeof(*cur)); + if (root && cur) { + *root = (sh_val_t){SH_ROOT, NULL, 0}; + *cur = (sh_val_t){sh_anchor(&a, s.sdir), NULL, 0}; + cbm_ht_set(s.env, "__cbm_root", root); + if (m.makefile) { + cbm_ht_set(s.env, "__cbm_curdir", cur); + } + } + s.cwd = m.makefile ? (sh_cwd_t){SH_AT, s.sdir} : (sh_cwd_t){SH_FREE, NULL}; + sh_run(&s, p->parser, buf, tl); + } + sh_state_free(&s); + } + if (m.vars) { + cbm_ht_free(m.vars); + } + cbm_free(CBM_MEM_CLASS_OTHER, m.recipes); + cbm_arena_destroy(&a); + ts_tree_delete(tree); +} + +/* ── Edges ─────────────────────────────────────────────────────────── */ + +typedef struct { + const char *file; + const cbm_gbuf_node_t *node; +} sh_span_t; + +static int sh_cmp_span(const void *a, const void *b) { + const sh_span_t *x = a; + const sh_span_t *y = b; + return strcmp(x->file, y->file); +} + +/* The innermost Function around `line` in `file`, else its Module, else its + * File node. */ +static int64_t sh_source(cbm_pipeline_ctx_t *ctx, const sh_span_t *spans, int n, + CBMHashTable *modules, const char *file, int line) { + int lo = 0; + int hi = n; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (strcmp(spans[mid].file, file) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + const cbm_gbuf_node_t *best = NULL; + for (int i = lo; i < n && strcmp(spans[i].file, file) == 0; i++) { + const cbm_gbuf_node_t *f = spans[i].node; + if (f->start_line > line || f->end_line < line) { + continue; + } + int w = f->end_line - f->start_line; + int bw = best ? best->end_line - best->start_line : 0; + if (!best || w < bw || + (w == bw && strcmp(f->qualified_name ? f->qualified_name : "", + best->qualified_name ? best->qualified_name : "") < 0)) { + best = f; + } + } + if (best) { + return best->id; + } + int64_t mod = (int64_t)(intptr_t)cbm_ht_get(modules, file); + if (mod > 0) { + return mod; + } + const cbm_gbuf_node_t *fn = cbm_pipeline_file_node(ctx->gbuf, ctx->project_name, file); + return fn ? fn->id : 0; +} + +/* The shared external Chart of `name` (the one chart dependencies use). */ +static int64_t sh_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, const char *file) { + char qn[CBM_SZ_512]; + snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", ctx->project_name, name); + const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(ctx->gbuf, qn); + if (known) { + return known->id; + } + return cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, + "{\"source\":\"helm\",\"external\":true}"); +} + +static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { + CBMHashTable *files = cbm_ht_create(0); + CBMHashTable *modules = cbm_ht_create(0); + if (!files || !modules) { + if (files) { + cbm_ht_free(files); + } + if (modules) { + cbm_ht_free(modules); + } + return 0; + } + for (int i = 0; i < h->count; i++) { + cbm_ht_set(files, h->items[i].file, (void *)h->items[i].file); + } + sh_span_t *spans = NULL; + int n = 0; + int cap = 0; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "Function", &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + const char *key = fp ? cbm_ht_get_key(files, fp) : NULL; + if (key && sh_grow((void **)&spans, &cap, n, sizeof(*spans))) { + spans[n++] = (sh_span_t){key, nodes[i]}; + } + } + } + if (n > SKIP_ONE) { + qsort(spans, (size_t)n, sizeof(*spans), sh_cmp_span); + } + if (cbm_gbuf_find_by_label(ctx->gbuf, "Module", &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + const char *key = fp ? cbm_ht_get_key(files, fp) : NULL; + if (key && !cbm_ht_get(modules, key)) { + cbm_ht_set(modules, key, (void *)(intptr_t)nodes[i]->id); + } + } + } + int edges = 0; + for (int i = 0; i < h->count; i++) { + const sh_hit_t *x = &h->items[i]; + int64_t src = sh_source(ctx, spans, n, modules, x->file, x->line); + int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file); + if (src <= 0 || tgt <= 0 || src == tgt) { + continue; + } + char esc[CBM_SZ_1K]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), x->path ? x->path : ""); + snprintf(props, sizeof(props), "{\"via\":\"%s\",\"%s\":\"%s\"%s,\"line\":%d}", x->via, + x->chart ? "chart" : "path", esc, x->tgt > 0 ? "" : ",\"external\":true", x->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "DEPLOYS", props); + edges++; + } + cbm_free(CBM_MEM_CLASS_OTHER, spans); + cbm_ht_free(files); + cbm_ht_free(modules); + return edges; +} + +void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count) { + sh_index_t ix = {0}; + sh_hits_t hits = {0}; + cbm_arena_init(&hits.arena); + sh_pass_t p = {.ix = &ix, .hits = &hits, .parser = NULL}; + for (int i = 0; i < file_count; i++) { + CBMLanguage lang = files[i].language; + if (lang != CBM_LANG_BASH && lang != CBM_LANG_MAKEFILE) { + continue; + } + size_t len = 0; + char *src = sh_read(files[i].path, &len); + if (!src) { + continue; + } + if (sh_mentions_deploy_tool(src, len)) { + if (!p.ix_ready) { + p.ix_ready = true; + p.ix_ok = sh_index_build(&ix, ctx); + p.parser = ts_parser_new(); + } + if (p.ix_ok && p.parser) { + if (lang == CBM_LANG_BASH) { + sh_script(&p, files[i].rel_path, src, len); + } else { + sh_makefile(&p, files[i].rel_path, src, len); + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + int edges = hits.count > 0 ? sh_link(ctx, &hits) : 0; + if (p.parser) { + ts_parser_delete(p.parser); + } + cbm_free(CBM_MEM_CLASS_OTHER, hits.items); + cbm_arena_destroy(&hits.arena); + if (p.ix_ready) { + sh_index_free(&ix); + } + if (edges > 0) { + char buf[CBM_SZ_32]; + snprintf(buf, sizeof(buf), "%d", edges); + cbm_log_info("pass.shell.deploys", "edges", buf); + } +} diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7b20b2787..5658e17ea 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1042,6 +1042,10 @@ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx); * Runs after USAGE resolution. */ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx); +/* Shell scripts and Make recipes (pass_shell.c): the commands that put + * manifests, kustomizations or charts on a cluster DEPLOYS them. Run at the + * end of the K8s pass, over the same files. */ +void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); /* Pre-dump pass: markdown → file REFERENCES_FILE linking. */ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx); @@ -1124,6 +1128,7 @@ extern _Atomic uint64_t g_importance_store_rows; * examines (pass_operator.c). Independent copies must grow them linearly. */ extern _Atomic uint64_t g_k8s_candidates; extern _Atomic uint64_t g_op_candidates; +extern _Atomic uint64_t g_sh_candidates; /* ── Incremental pipeline (pipeline_incremental.c) ───────────────── */ diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index cca16bcd3..28b3ecb29 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1696,6 +1696,210 @@ TEST(probe_gitops_objects_deploy_their_units) { PASS(); } +/* Every DEPLOYS edge as "source -> target (via)", sorted, one per line: a + * function by name, top-level code as "@"; a File or Kustomization by + * its path, a Chart by name ("(ext)": the external one). NULL on failure; + * the caller frees. */ +static char *gpf_deploys(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT " + "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END " + "|| ' -> ' || CASE WHEN t.label='Chart' THEN 'chart:' || t.name || " + "CASE WHEN instr(t.qualified_name,'__helm_dep__') THEN '(ext)' ELSE '' END " + "ELSE t.label || ':' || t.file_path END " + "|| ' (' || json_extract(e.properties,'$.via') || ')' AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type='DEPLOYS' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +#define GPF_ASSERT_DEPLOYS(files, n, want) \ + do { \ + char *got_ = gpf_deploys(files, n); \ + if (!got_ || strcmp(got_, want) != 0) { \ + fprintf(stderr, "DEPLOYS got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \ + free(got_); \ + FAIL("DEPLOYS edge set"); \ + } \ + free(got_); \ + } while (0) + +#define GPF_CM(name) "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " name "\n" +#define GPF_KUST "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + +/* kubectl apply -f deploys a manifest, a directory's manifests (-R: below + * it too; not a file holding no objects), a glob's; -k and a kustomize build + * piped into kubectl a Kustomization. The source is the function running the + * command, top-level code its file; paths resolve against the cd'd root. */ +TEST(probe_shell_kubectl_deploys_what_it_applies) { + static const GpfFile files[] = { + {"manifests/app.yaml", GPF_CM("app")}, + {"manifests/svc.yaml", GPF_CM("svc")}, + {"manifests/notes.yaml", "notes:\n - not an object\n"}, + {"manifests/nested/cm.yaml", GPF_CM("cm")}, + {"crds/a.yaml", GPF_CM("a")}, + {"crds/b.yaml", GPF_CM("b")}, + {"overlays/prod/kustomization.yaml", GPF_KUST}, + {"overlays/dev/kustomization.yaml", GPF_KUST}, + {"hack/deploy.sh", "#!/usr/bin/env bash\n" + "cd \"$(dirname \"$0\")/..\"\n\n" + "apply_all() {\n" + " kubectl apply -f manifests/app.yaml\n" + " kubectl --namespace \"$NS\" apply -f manifests\n" + " kubectl apply -Rf manifests/nested\n" + "}\n\n" + "kubectl apply -f 'crds/*.yaml'\n" + "kubectl apply -k overlays/prod\n" + "kustomize build overlays/dev | sed s/a/b/ | kubectl apply -f -\n" + "cat < File:crds/a.yaml (kubectl)\n" + "@hack/deploy.sh -> File:crds/b.yaml (kubectl)\n" + "@hack/deploy.sh -> Resource:overlays/dev/kustomization.yaml (kustomize)\n" + "@hack/deploy.sh -> Resource:overlays/prod/kustomization.yaml (kubectl)\n" + "apply_all -> File:manifests/app.yaml (kubectl)\n" + "apply_all -> File:manifests/nested/cm.yaml (kubectl)\n" + "apply_all -> File:manifests/svc.yaml (kubectl)"); + PASS(); +} + +/* helm install / upgrade deploys its chart: a chart dir (through options, + * a wrapper, $(git rev-parse --show-toplevel)), else the external chart of + * repo/name or oci://.../name; a helm template piped into kubectl too. No + * edge from an unknown word that may shift the chart's position, nor from + * a variable an if/else leaves undecided. */ +TEST(probe_shell_helm_deploys_its_chart) { + static const GpfFile files[] = { + {"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 0.1.0\n"}, + {"charts/web/values-prod.yaml", "replicas: 2\n"}, + {"charts/api/Chart.yaml", "apiVersion: v2\nname: api\nversion: 0.1.0\n"}, + {"scripts/install.sh", + "#!/bin/sh\n" + "ROOT=\"$(git rev-parse --show-toplevel)\"\n" + "NS=${NAMESPACE:-web}\n" + "install_web() {\n" + " retry 3 helm upgrade --install --namespace \"$NS\" --create-namespace --atomic " + "web \"$ROOT/charts/web/\"\n" + "}\n" + "install_cache() {\n" + " helm install cache bitnami/redis --version 1.2.3\n" + " helm install queue oci://registry.example.com/charts/nats\n" + "}\n" + "render_web() {\n" + " helm template web charts/web -f charts/web/values-prod.yaml | kubectl apply -f -\n" + "}\n" + "split_args() {\n" + " helm upgrade $ARGS charts/api\n" + "}\n" + "branchy() {\n" + " CHART=charts/web\n" + " if [ -n \"$PROD\" ]; then CHART=charts/api; fi\n" + " helm upgrade --install x \"$CHART\"\n" + "}\n"}}; + GPF_ASSERT_DEPLOYS(files, 4, + "install_cache -> chart:nats(ext) (helm)\n" + "install_cache -> chart:redis(ext) (helm)\n" + "install_web -> chart:web (helm)\n" + "render_web -> chart:web (helm)"); + PASS(); +} + +/* Constant propagation: SCRIPT_ROOT from $(cd "$(dirname ...)" && pwd -P), + * pushd / popd, an option whose value is unknown, an array holding the + * command; a subshell's cd does not leak; with no directory tracked a path + * resolves against the script's dir or the root -- only when one of the two + * has it. */ +TEST(probe_shell_paths_follow_the_script) { + static const GpfFile files[] = { + {"apps/web/deployment.yaml", GPF_CM("deployment")}, + {"apps/web/service.yaml", GPF_CM("service")}, + {"apps/web/sub/cm.yaml", GPF_CM("cm")}, + {"apps/web/sub/extra.yaml", GPF_CM("extra")}, + {"apps/web/config.yaml", GPF_CM("config-here")}, + {"config.yaml", GPF_CM("config-root")}, + {"apps/web/deploy.sh", + "#!/usr/bin/env bash\n" + "SCRIPT_ROOT=$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd -P)\n" + "context=\"${KUBECTL_CONTEXT:-gke}\"\n" + "if ! kubectl config get-contexts \"${context}\"; then\n" + " context=\"other\"\n" + "fi\n" + "pushd \"${SCRIPT_ROOT}\" >/dev/null\n" + "kubectl --context=\"${context}\" apply -f .\n" + "popd >/dev/null\n" + "( cd sub && kubectl apply -f cm.yaml )\n" + "kubectl apply -f extra.yaml\n" + "ambiguous() { kubectl apply -f config.yaml; }\n" + "unique() { kubectl apply -f deployment.yaml; }\n" + "deploy() {\n" + " local kubectl=(\n" + " kubectl\n" + " --context=\"${context}\"\n" + " )\n" + " \"${kubectl[@]}\" apply -f service.yaml\n" + "}\n"}}; + GPF_ASSERT_DEPLOYS(files, 7, + "@apps/web/deploy.sh -> File:apps/web/config.yaml (kubectl)\n" + "@apps/web/deploy.sh -> File:apps/web/deployment.yaml (kubectl)\n" + "@apps/web/deploy.sh -> File:apps/web/service.yaml (kubectl)\n" + "@apps/web/deploy.sh -> File:apps/web/sub/cm.yaml (kubectl)\n" + "deploy -> File:apps/web/service.yaml (kubectl)\n" + "unique -> File:apps/web/deployment.yaml (kubectl)"); + PASS(); +} + +/* A Make recipe runs in its Makefile's directory with the Makefile's + * variables expanded ($(shell pwd), ?=, a conditional's: unknown): a + * kustomize build into kubectl, kubectl -f a directory, find -name -exec + * kubectl apply -f {}. */ +TEST(probe_make_recipes_deploy) { + static const GpfFile files[] = { + {"operator/config/default/kustomization.yaml", GPF_KUST}, + {"operator/config/prod/kustomization.yaml", GPF_KUST}, + {"operator/config/dev/kustomization.yaml", GPF_KUST}, + {"operator/config/crd/widgets.yaml", GPF_CM("widgets")}, + {"operator/config/flux/flux-a.yaml", GPF_CM("flux-a")}, + {"operator/config/flux/other.yaml", GPF_CM("other")}, + {"operator/config/flux/sub/flux-b.yaml", GPF_CM("flux-b")}, + {"operator/Makefile", "LOCALBIN ?= $(shell pwd)/bin\n" + "KUBECTL ?= kubectl\n" + "KUSTOMIZE ?= $(LOCALBIN)/kustomize\n" + "ifeq ($(ENV),prod)\n" + "OVERLAY = config/prod\n" + "else\n" + "OVERLAY = config/dev\n" + "endif\n\n" + "deploy:\n" + "\t$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -\n\n" + "install:\n" + "\t@$(KUBECTL) apply -f config/crd\n\n" + "flux:\n" + "\tfind ./config/flux -type f -name \"flux-*\" -exec $(KUBECTL) " + "apply -f {} \\;\n\n" + "overlay:\n" + "\t$(KUBECTL) apply -k $(OVERLAY)\n"}}; + GPF_ASSERT_DEPLOYS(files, 8, + "deploy -> Resource:operator/config/default/kustomization.yaml (kustomize)\n" + "flux -> File:operator/config/flux/flux-a.yaml (kubectl)\n" + "flux -> File:operator/config/flux/sub/flux-b.yaml (kubectl)\n" + "install -> File:operator/config/crd/widgets.yaml (kubectl)"); + PASS(); +} + /* ── Helm symbolic (Phase 2) ────────────────────────────────────────── */ /* A chart template is YAML only once rendered: a document opened by @@ -3367,6 +3571,10 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_mcp_search_filters_kind_and_namespace); RUN_TEST(probe_mcp_trace_topology); RUN_TEST(probe_gitops_objects_deploy_their_units); + RUN_TEST(probe_shell_kubectl_deploys_what_it_applies); + RUN_TEST(probe_shell_helm_deploys_its_chart); + RUN_TEST(probe_shell_paths_follow_the_script); + RUN_TEST(probe_make_recipes_deploy); RUN_TEST(probe_helm_templated_objects_are_resources); RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index f955f9822..c3d668180 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3422,6 +3422,41 @@ TEST(incremental_k8s_edits_reach_other_files) { PASS(); } +/* A script's deploy commands link nodes of other files (pass_shell.c): the + * closure repair re-reads the changed script alone, so editing it must still + * leave the edges a fresh index builds. */ +TEST(incremental_shell_deploy_edit_keeps_its_edges) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_shinc_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "manifests/app.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: app\n"); + write_temp_file(tmp, "hack/deploy.sh", + "#!/bin/sh\ncd \"$(dirname \"$0\")/..\"\nkubectl apply -f manifests/app.yaml\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/shinc.db", tmp); + static const char deploys[] = "SELECT COUNT(*) FROM edges WHERE type='DEPLOYS'"; + + int rc_full = k8s_inc_index(tmp, db_path); + int deploys_full = k8s_inc_count(db_path, deploys); + write_temp_file(tmp, "hack/deploy.sh", + "#!/bin/sh\ncd \"$(dirname \"$0\")/..\"\nkubectl apply -f manifests/app.yaml\n" + "# edited\n"); + int rc_edit = k8s_inc_index(tmp, db_path); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + int deploys_after = k8s_inc_count(db_path, deploys); + th_rmtree(tmp); + + ASSERT_EQ(rc_full, 0); + ASSERT_EQ(deploys_full, 1); + ASSERT_EQ(rc_edit, 0); + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_EQ(deploys_after, 1); + PASS(); +} + /* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage, * while the shared name registry must NOT leak those relations into other * languages' textual resolution: a Python call or identifier sharing the @@ -19046,6 +19081,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge); RUN_TEST(pipeline_staged_publish_commits_below_full_sync_issue1419); RUN_TEST(incremental_k8s_edits_reach_other_files); + RUN_TEST(incremental_shell_deploy_edit_keeps_its_edges); RUN_TEST(pipeline_sql_lineage_and_relation_isolation); RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage); RUN_TEST(pipeline_dbt_jinja_lineage); From 352d047782a46b1725eedbbc0c5e6406f37bd0f2 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 04:18:10 +0200 Subject: [PATCH 32/52] feat(shell): programs a script starts by path SPAWNS their Process "Which scripts drive kubermatic-installer?" had no answer in the graph: a script running ./_build/kubermatic-installer deploy ... or a Make recipe running $(CONTROLLER_GEN) starts another program, and the call resolvers draw SPAWNS for that only in code with a spawn API. The shell pass now draws it for scripts and recipes too: a program started by a path (./_build/x, ${DIR}/x, $(LOCALBIN)/x, /usr/sbin/x) SPAWNS the Process of its name -- the node and edge shape pass_spawns.c uses, with api "shell". The name is the last segment of the path's value, or, the value unknown, the word's literal last segment. Not drawn: a path naming a file of the repository (a script of its own), `command -v X` (a lookup, not a run), a name that a branch leaves undecided. The commands inside $( ) and <( ) are read too, each in a subshell of its own, for deploys as well. Make: recipes are now read line by line as make runs them (a tab-led line and its continuations): tree-sitter-make can run one recipe_line over several lines when a quote opens in "Don't" and closes lines later, and an inner line kept its @ prefix. Rules tree-sitter-make leaves inside an ERROR node (an indented, nested ifeq before them) are read, and their variables count when no conditional encloses their line. Measured on 17 repositories against an independent reading of the scripts and Makefiles (its own tokenizer, variable expansion and directory tracking): 229 matched; of the 231 edges drawn, 229 agree and 2 are scripts in .txt golden files cbm reads as bash; the 29 rows only the reading has are its own line-based misreadings (case patterns, multi-line arrays and strings), names a branch leaves undecided, a Go-templated script and a directory discovery excludes. kubermatic: every hack script running kubermatic-installer deploy SPAWNS it. Only SPAWNS edges and Process nodes were added (231, 115); deploys unchanged (86 of 86); three runs build identical graphs; index time unchanged within 1 s (deckhouse 9 s, gardener 5 s, kubermatic 2 s). Signed-off-by: Martin Vogel --- src/pipeline/pass_shell.c | 418 +++++++++++++++++++++++++++-------- tests/test_grammar_probe_f.c | 114 ++++++++++ 2 files changed, 443 insertions(+), 89 deletions(-) diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index 9a64214a2..02fc6fe2e 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -16,6 +16,11 @@ * helm template [NAME] CHART | ... | kubectl apply -f - * find D -name G -exec kubectl apply -f {} \; D's manifests matching G * + * A program started by a path (./_build/kubermatic-installer, ${DIR}/x, + * $(LOCALBIN)/controller-gen) SPAWNS the Process of its name, as + * pass_spawns.c draws a call that starts a program; a path naming a file of + * the repository is a script of its own and not linked here. + * * Words are evaluated by constant propagation over the script, nothing is * executed: assignments (local / export / readonly / declare, arrays and * "${a[@]}"), ${X:-default}, $0 / BASH_SOURCE, $(dirname ..), $(basename ..), @@ -362,6 +367,7 @@ typedef struct { const char *ext; /* pass-arena */ const char *path; /* pass-arena: the deployed path or chart */ bool chart; + bool spawn; /* SPAWNS the Process `ext`, not DEPLOYS */ } sh_hit_t; typedef struct { @@ -764,6 +770,7 @@ static const char *sh_absolute(sh_t *s, const char *v) { typedef struct { const char **v; bool *split; + uint32_t *span; /* source bytes of the word: [2i] start, [2i+1] end */ int n; int cap; } sh_argv_t; @@ -773,6 +780,7 @@ static void sh_argv_build(sh_t *s, TSNode cmd, sh_argv_t *out); static void sh_argv_free(sh_argv_t *a) { cbm_free(CBM_MEM_CLASS_OTHER, (void *)a->v); cbm_free(CBM_MEM_CLASS_OTHER, a->split); + cbm_free(CBM_MEM_CLASS_OTHER, a->span); *a = (sh_argv_t){0}; } @@ -985,7 +993,7 @@ static bool sh_may_split(TSNode w) { return false; } -static void sh_argv_push(sh_argv_t *a, const char *v, bool split) { +static void sh_argv_push(sh_argv_t *a, const char *v, bool split, uint32_t b, uint32_t e) { if (a->n >= a->cap) { int ncap = a->cap ? a->cap * PAIR_LEN : SH_GROW_MIN; const char **nv = @@ -999,27 +1007,37 @@ static void sh_argv_push(sh_argv_t *a, const char *v, bool split) { return; } a->split = ns; + uint32_t *nsp = + cbm_realloc(CBM_MEM_CLASS_OTHER, a->span, (size_t)ncap * PAIR_LEN * sizeof(*nsp)); + if (!nsp) { + return; + } + a->span = nsp; a->cap = ncap; } a->v[a->n] = v; a->split[a->n] = split; + a->span[PAIR_LEN * a->n] = b; + a->span[PAIR_LEN * a->n + SKIP_ONE] = e; a->n++; } static void sh_argv_word(sh_t *s, TSNode w, sh_argv_t *out) { + uint32_t b = ts_node_start_byte(w); + uint32_t e = ts_node_end_byte(w); const sh_val_t *arr = sh_array_word(s, w); if (arr) { if (!arr->items) { - sh_argv_push(out, NULL, true); + sh_argv_push(out, NULL, true, b, e); return; } for (int i = 0; i < arr->n; i++) { - sh_argv_push(out, arr->items[i], false); + sh_argv_push(out, arr->items[i], false, b, e); } return; } const char *v = sh_eval_word(s, w); - sh_argv_push(out, v, v ? false : sh_may_split(w)); + sh_argv_push(out, v, v ? false : sh_may_split(w), b, e); } /* argv of a command: its name and arguments, evaluated (NULL: unknown). */ @@ -1319,18 +1337,20 @@ static bool sh_numberish(const char *v) { } /* The index of the program argv runs, past wrappers (time, sudo, retry N, - * timeout D, env A=B, ...); -1 when it is unknown. */ + * timeout D, env A=B, ...); -1 when argv has none. Its value may be unknown + * (NULL). */ static int sh_program(const sh_argv_t *a) { static const char *const wrappers[] = {"time", "sudo", "exec", "nice", "nohup", "command", "env", "timeout", "retry", NULL}; int i = 0; while (i < a->n) { - if (!a->v[i]) { - return -1; - } - if (!sh_in(sh_basename(a->v[i]), wrappers)) { + if (!a->v[i] || !sh_in(sh_basename(a->v[i]), wrappers)) { return i; } + if (strcmp(a->v[i], "command") == 0 && i + SKIP_ONE < a->n && + (sh_eq(a->v[i + SKIP_ONE], "-v") || sh_eq(a->v[i + SKIP_ONE], "-V"))) { + return -1; /* command -v X: is there an X, not a run of it */ + } i++; while (i < a->n && a->v[i] && (a->v[i][0] == '-' || strchr(a->v[i], '=') || sh_numberish(a->v[i]))) { @@ -1656,10 +1676,11 @@ static void sh_find(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog) { /* -exec PROGRAM ... {} ; */ sh_argv_t e = {0}; for (i++; i < a->n && !sh_eq(a->v[i], ";") && !sh_eq(a->v[i], "+"); i++) { - sh_argv_push(&e, a->v[i], a->split[i]); + sh_argv_push(&e, a->v[i], a->split[i], a->span[PAIR_LEN * i], + a->span[PAIR_LEN * i + SKIP_ONE]); } int p = sh_program(&e); - if (p >= 0 && strcmp(sh_basename(e.v[p]), "kubectl") == 0) { + if (p >= 0 && e.v[p] && strcmp(sh_basename(e.v[p]), "kubectl") == 0) { sh_kubectl_t k = {0}; sh_kubectl_parse(&e, p + SKIP_ONE, &k); const char *verb = k.verb >= 0 ? e.v[k.verb] : NULL; @@ -1681,7 +1702,7 @@ static void sh_deploy_command(sh_t *s, TSNode cmd, const sh_argv_t *a, const sh_ sh_src_t *out) { *out = (sh_src_t){SH_SRC_NONE, NULL, NULL}; int prog = sh_program(a); - if (prog < 0) { + if (prog < 0 || !a->v[prog]) { return; } const char *base = sh_basename(a->v[prog]); @@ -1707,6 +1728,75 @@ static void sh_deploy_command(sh_t *s, TSNode cmd, const sh_argv_t *a, const sh_ } } +/* A repository file: the path names a File node. */ +static int sh_t_repo_file(sh_t *s, const char *path, const sh_find_t *f, sh_targets_t *out) { + (void)f; + int64_t id = (int64_t)(intptr_t)cbm_ht_get(s->ix->file_ids, path); + if (id <= 0) { + return 0; + } + sh_targets_push(out, id, path); + return SKIP_ONE; +} + +/* A program started by a path (./_build/x, ${DIR}/x, $(LOCALBIN)/x) SPAWNS + * the Process of its name -- the value's last segment, or, the value + * unknown, the word's literal last segment. A path naming a file of the + * repository is a script of its own, not linked here. */ +static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) { + int p = sh_program(a); + if (p < 0) { + return; + } + const char *v = a->v[p]; + const char *name = NULL; + if (v) { + if (!strchr(v, '/') || strpbrk(v, "*?[")) { + return; + } + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, v, c); + for (int i = 0; i < n; i++) { + sh_targets_t ts = {0}; + int hit = sh_t_repo_file(s, c[i], NULL, &ts); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + if (hit) { + return; + } + } + name = sh_value_basename(s->a, v); + } else { + uint32_t b = a->span[PAIR_LEN * p]; + uint32_t e = a->span[PAIR_LEN * p + SKIP_ONE]; + while (e > b && (s->src[e - SKIP_ONE] == '"' || s->src[e - SKIP_ONE] == '\'')) { + e--; + } + uint32_t k = e; + while (k > b && s->src[k - SKIP_ONE] != '/') { + k--; + } + if (k == b) { + return; + } + name = cbm_arena_strndup(s->a, s->src + k, e - k); + if (name && strpbrk(name, "$`*?[]{}()\"' \t\\")) { + return; + } + } + if (!name || !name[0] || strcmp(name, ".") == 0 || strcmp(name, "..") == 0) { + return; + } + sh_hits_t *h = s->hits; + if (sh_grow((void **)&h->items, &h->cap, h->count, sizeof(*h->items))) { + h->items[h->count++] = + (sh_hit_t){.file = s->file, + .line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base, + .via = "shell", + .ext = cbm_arena_strdup(&h->arena, name), + .spawn = true}; + } +} + /* ── Statements ────────────────────────────────────────────────────── */ static void sh_walk(sh_t *s, TSNode n); @@ -1717,9 +1807,26 @@ static void sh_walk_children(sh_t *s, TSNode n) { } } +/* The commands inside $( ) and <( ) of a word, a value or a redirect run + * too, each in a subshell of its own. */ +static void sh_walk_substs(sh_t *s, TSNode n) { + if (sh_type(n, "command_substitution") || sh_type(n, "process_substitution")) { + sh_mark_t m = sh_mark(s); + sh_walk_children(s, n); + sh_rewind(s, m); + return; + } + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + sh_walk_substs(s, ts_node_named_child(n, i)); + } +} + static void sh_assign(sh_t *s, TSNode n, bool local) { TSNode name = ts_node_child_by_field_name(n, TS_FIELD("name")); TSNode value = ts_node_child_by_field_name(n, TS_FIELD("value")); + if (!ts_node_is_null(value)) { + sh_walk_substs(s, value); + } if (ts_node_is_null(name)) { return; } @@ -1811,23 +1918,30 @@ static bool sh_chdir(sh_t *s, const sh_argv_t *a) { } static void sh_command(sh_t *s, TSNode cmd) { + sh_walk_substs(s, cmd); sh_argv_t a = {0}; sh_argv_build(s, cmd, &a); if (!sh_chdir(s, &a)) { sh_src_t out; sh_deploy_command(s, cmd, &a, NULL, &out); + sh_spawn(s, cmd, &a); } sh_argv_free(&a); } -/* `cat < mk_var_t */ bool makefile; /* a Makefile (its recipes run in its dir), not an included .mk */ - TSNode *recipes; - int nrecipes; - int cap_recipes; + mk_line_t *lines; + int nlines; + int cap_lines; } mk_t; static const char MK_UNKNOWN[] = "${__cbm_unknown}"; @@ -2345,45 +2475,132 @@ static void mk_assign(mk_t *m, TSNode n, bool cond) { } /* Variables and recipe lines of a makefile; `cond`: under a conditional. */ +/* The recipe lines in the bytes of a recipe_line node, split as make splits + * them: tree-sitter-make can let one node run over several lines (a quote in + * `"Don't ..."` does it). */ +static void mk_split_lines(mk_t *m, TSNode node) { + const char *src = m->src; + uint32_t p = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + uint32_t row = ts_node_start_point(node).row; + while (p < end) { + uint32_t b = p; + uint32_t brow = row; + uint32_t q = p; + for (;;) { + while (q < end && src[q] != '\n') { + q++; + } + uint32_t k = (q > p && src[q - SKIP_ONE] == '\r') ? q - SKIP_ONE : q; + bool cont = k > p && src[k - SKIP_ONE] == '\\'; + if (!cont || q >= end) { + break; + } + q++; + row++; + p = q; + } + uint32_t e = q; + p = q < end ? q + SKIP_ONE : q; + row++; + while (e > b && src[e - SKIP_ONE] == '\r') { + e--; + } + while (b < e && src[b] && strchr(" \t@-+", src[b])) { + b++; + } + if (b < e && src[b] != '#' && + sh_grow((void **)&m->lines, &m->cap_lines, m->nlines, sizeof(*m->lines))) { + m->lines[m->nlines++] = (mk_line_t){b, e, brow}; + } + } +} + +static int mk_cmp_line(const void *a, const void *b) { + uint32_t x = ((const mk_line_t *)a)->b; + uint32_t y = ((const mk_line_t *)b)->b; + return (x > y) - (x < y); +} + +static void mk_collect(mk_t *m, TSNode n, bool cond); +static void mk_collect_error(mk_t *m, TSNode err, bool cond); + +static void mk_collect_one(mk_t *m, TSNode c, bool cond) { + const char *t = ts_node_type(c); + if (strcmp(t, "variable_assignment") == 0) { + mk_assign(m, c, cond); + } else if (strcmp(t, "recipe_line") == 0) { + mk_split_lines(m, c); + } else if (strcmp(t, "rule") == 0 || strcmp(t, "recipe") == 0) { + mk_collect(m, c, cond); + } else if (strcmp(t, "ERROR") == 0) { + mk_collect_error(m, c, cond); + } else if (strcmp(t, "define_directive") == 0) { + TSNode name = ts_node_child_by_field_name(c, TS_FIELD("name")); + if (!ts_node_is_null(name)) { + uint32_t nb = ts_node_start_byte(name); + const char *nm = mk_trim(m->a, m->src + nb, ts_node_end_byte(name) - nb); + mk_var_t *v = cbm_arena_alloc(m->a, sizeof(*v)); + if (nm && v) { + *v = (mk_var_t){"", NULL, false, false}; + cbm_ht_set(m->vars, nm, v); + } + } + } else if (strstr(t, "conditional") || strstr(t, "if") == t || + strcmp(t, "else_directive") == 0 || strcmp(t, "elsif_directive") == 0) { + mk_collect(m, c, true); + } else if (strstr(t, "directive")) { + mk_collect(m, c, cond); + } +} + static void mk_collect(mk_t *m, TSNode n, bool cond) { for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { - TSNode c = ts_node_named_child(n, i); - const char *t = ts_node_type(c); - if (strcmp(t, "variable_assignment") == 0) { - mk_assign(m, c, cond); - } else if (strcmp(t, "rule") == 0) { - TSNode recipe = {0}; - for (uint32_t j = 0; j < ts_node_named_child_count(c); j++) { - if (sh_type(ts_node_named_child(c, j), "recipe")) { - recipe = ts_node_named_child(c, j); - } - } - for (uint32_t j = 0; !ts_node_is_null(recipe) && j < ts_node_named_child_count(recipe); - j++) { - TSNode line = ts_node_named_child(recipe, j); - if (sh_type(line, "recipe_line") && sh_grow((void **)&m->recipes, &m->cap_recipes, - m->nrecipes, sizeof(*m->recipes))) { - m->recipes[m->nrecipes++] = line; - } - } - } else if (strcmp(t, "define_directive") == 0) { - TSNode name = ts_node_child_by_field_name(c, TS_FIELD("name")); - if (!ts_node_is_null(name)) { - uint32_t nb = ts_node_start_byte(name); - const char *nm = mk_trim(m->a, m->src + nb, ts_node_end_byte(name) - nb); - mk_var_t *v = cbm_arena_alloc(m->a, sizeof(*v)); - if (nm && v) { - *v = (mk_var_t){"", NULL, false, false}; - cbm_ht_set(m->vars, nm, v); - } - } - } else if (strstr(t, "conditional") || strstr(t, "if") == t || - strcmp(t, "else_directive") == 0 || strcmp(t, "elsif_directive") == 0) { - mk_collect(m, c, true); - } else if (strstr(t, "directive")) { - mk_collect(m, c, cond); + mk_collect_one(m, ts_node_named_child(n, i), cond); + } +} + +/* +1 for a line opening a conditional or a define, -1 for its end, else 0. */ +static int mk_directive_delta(const char *p, const char *end) { + static const struct { + const char *kw; + int delta; + } kws[] = {{"ifeq", SKIP_ONE}, {"ifneq", SKIP_ONE}, {"ifdef", SKIP_ONE}, {"ifndef", SKIP_ONE}, + {"define", SKIP_ONE}, {"endif", -1}, {"endef", -1}}; + while (p < end && (*p == ' ' || *p == '\t')) { + p++; + } + for (size_t i = 0; i < sizeof(kws) / sizeof(kws[0]); i++) { + size_t kl = strlen(kws[i].kw); + if ((size_t)(end - p) >= kl && strncmp(p, kws[i].kw, kl) == 0 && + (p + kl == end || strchr(" \t(\r", p[kl]))) { + return kws[i].delta; } } + return 0; +} + +/* A region tree-sitter-make could not parse (an indented, nested ifeq does + * it) still holds rules and assignments; an assignment counts as certain + * only outside the conditionals its lines open. */ +static void mk_collect_error(mk_t *m, TSNode err, bool cond) { + uint32_t scan = ts_node_start_byte(err); + uint32_t end = ts_node_end_byte(err); + int depth = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(err); i++) { + TSNode c = ts_node_named_child(err, i); + uint32_t cb = ts_node_start_byte(c); + while (scan < cb && scan < end) { + uint32_t le = scan; + while (le < end && m->src[le] != '\n') { + le++; + } + depth += mk_directive_delta(m->src + scan, m->src + le); + depth = depth < 0 ? 0 : depth; + scan = le + SKIP_ONE; + } + mk_collect_one(m, c, cond || depth > 0); + } } static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t len) { @@ -2405,18 +2622,19 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l if (m.vars) { mk_collect(&m, ts_tree_root_node(tree), false); } - for (int r = 0; m.vars && r < m.nrecipes; r++) { - TSNode line = m.recipes[r]; - uint32_t b = ts_node_start_byte(line); - uint32_t e = ts_node_end_byte(line); - while (b < e && (src[b] == ' ' || src[b] == '\t' || src[b] == '@' || src[b] == '-' || - src[b] == '+')) { - b++; + if (m.nlines > SKIP_ONE) { + qsort(m.lines, (size_t)m.nlines, sizeof(*m.lines), mk_cmp_line); + } + for (int r = 0; m.vars && r < m.nlines; r++) { + if (r > 0 && m.lines[r].b == m.lines[r - SKIP_ONE].b) { + continue; /* nested nodes covered it already */ } + uint32_t b = m.lines[r].b; + uint32_t e = m.lines[r].e; const char *raw = cbm_arena_strndup(&a, src + b, e - b); const char *text = raw ? mk_expand(&m, raw) : NULL; - if (!text || !sh_mentions_deploy_tool(text, strlen(text))) { - continue; + if (!text || (!strchr(text, '/') && !sh_mentions_deploy_tool(text, strlen(text)))) { + continue; /* neither a deploy nor a program started by a path */ } size_t tl = strlen(text); char *buf = cbm_arena_alloc(&a, tl + CBM_SZ_16); @@ -2428,7 +2646,7 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l sh_t s; sh_state_init(&s, p, &a, rel); if (s.env) { - s.line_base = (int)ts_node_start_point(line).row; + s.line_base = (int)m.lines[r].row; sh_val_t *root = cbm_arena_alloc(&a, sizeof(*root)); sh_val_t *cur = cbm_arena_alloc(&a, sizeof(*cur)); if (root && cur) { @@ -2447,7 +2665,7 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l if (m.vars) { cbm_ht_free(m.vars); } - cbm_free(CBM_MEM_CLASS_OTHER, m.recipes); + cbm_free(CBM_MEM_CLASS_OTHER, m.lines); cbm_arena_destroy(&a); ts_tree_delete(tree); } @@ -2516,6 +2734,26 @@ static int64_t sh_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, cons "{\"source\":\"helm\",\"external\":true}"); } +/* SPAWNS to the Process of `program`, as pass_spawns.c draws it. */ +static int sh_link_spawn(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x) { + char program[CBM_SZ_128]; + cbm_json_escape(program, (int)sizeof(program), x->ext); + char qn[CBM_SZ_256]; + snprintf(qn, sizeof(qn), "__process__%s", x->ext); + char node_props[CBM_SZ_256]; + snprintf(node_props, sizeof(node_props), "{\"program\":\"%s\"}", program); + int64_t process = cbm_gbuf_upsert_node(ctx->gbuf, "Process", x->ext, qn, "", 0, 0, node_props); + if (process <= 0) { + return 0; + } + char props[CBM_SZ_512]; + snprintf(props, sizeof(props), + "{\"api\":\"%s\",\"program\":\"%s\",\"line\":%d,\"confidence\":1.00}", x->via, program, + x->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, process, "SPAWNS", props); + return SKIP_ONE; +} + static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { CBMHashTable *files = cbm_ht_create(0); CBMHashTable *modules = cbm_ht_create(0); @@ -2561,6 +2799,10 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { for (int i = 0; i < h->count; i++) { const sh_hit_t *x = &h->items[i]; int64_t src = sh_source(ctx, spans, n, modules, x->file, x->line); + if (x->spawn) { + edges += src > 0 ? sh_link_spawn(ctx, src, x) : 0; + continue; + } int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file); if (src <= 0 || tgt <= 0 || src == tgt) { continue; @@ -2595,18 +2837,16 @@ void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil if (!src) { continue; } - if (sh_mentions_deploy_tool(src, len)) { - if (!p.ix_ready) { - p.ix_ready = true; - p.ix_ok = sh_index_build(&ix, ctx); - p.parser = ts_parser_new(); - } - if (p.ix_ok && p.parser) { - if (lang == CBM_LANG_BASH) { - sh_script(&p, files[i].rel_path, src, len); - } else { - sh_makefile(&p, files[i].rel_path, src, len); - } + if (!p.ix_ready) { + p.ix_ready = true; + p.ix_ok = sh_index_build(&ix, ctx); + p.parser = ts_parser_new(); + } + if (p.ix_ok && p.parser) { + if (lang == CBM_LANG_BASH) { + sh_script(&p, files[i].rel_path, src, len); + } else { + sh_makefile(&p, files[i].rel_path, src, len); } } cbm_free(CBM_MEM_CLASS_OTHER, src); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 28b3ecb29..3ee57159a 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1738,6 +1738,44 @@ static char *gpf_deploys(const GpfFile *files, int nfiles) { free(got_); \ } while (0) +/* Every SPAWNS edge a script or recipe draws (api=shell) as "source -> + * program", sorted, one per line; top-level code as "@". */ +static char *gpf_spawns(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT DISTINCT " + "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END " + "|| ' -> ' || t.name AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type='SPAWNS' AND t.label='Process' " + "AND json_extract(e.properties,'$.api')='shell' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +#define GPF_ASSERT_SPAWNS(files, n, want) \ + do { \ + char *got_ = gpf_spawns(files, n); \ + if (!got_ || strcmp(got_, want) != 0) { \ + fprintf(stderr, "SPAWNS got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \ + free(got_); \ + FAIL("SPAWNS edge set"); \ + } \ + free(got_); \ + } while (0) + #define GPF_CM(name) "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " name "\n" #define GPF_KUST "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" @@ -1862,6 +1900,80 @@ TEST(probe_shell_paths_follow_the_script) { PASS(); } +/* A program a script starts by a path SPAWNS the Process of its name: from + * a value, or a literal last segment after an unknown prefix; inside $( ) + * and < <( ) too; past a wrapper. Not a script of the repository, not + * `command -v X` (a lookup, not a run). */ +TEST(probe_shell_programs_by_path_spawn) { + static const GpfFile files[] = { + {"hack/lib.sh", "#!/bin/sh\necho lib\n"}, + {"hack/run.sh", + "#!/usr/bin/env bash\n" + "cd \"$(dirname \"$0\")/..\"\n" + "./_build/kubermatic-installer deploy kubermatic-master --config x.yaml\n" + "\"${INSTALL_DIR}/kubermatic-installer\" deploy seed-mla\n" + "./hack/lib.sh\n" + "if command -v bin/checker >/dev/null; then echo ok; fi\n" + "version=$(bin/werf version)\n" + "retry 3 /usr/local/bin/helm version\n" + "while read -r l; do echo \"$l\"; done < <(./bin/lister --all)\n" + "run() { bin/tool --flag; }\n"}}; + GPF_ASSERT_SPAWNS(files, 2, + "@hack/run.sh -> helm\n" + "@hack/run.sh -> kubermatic-installer\n" + "@hack/run.sh -> lister\n" + "@hack/run.sh -> werf\n" + "run -> tool"); + PASS(); +} + +/* Make recipes start programs by path as well: through a variable whose + * prefix is unknown ($(GOBIN)/controller-gen); in a rule tree-sitter-make + * leaves inside an ERROR (an indented, nested ifeq before it), with a + * variable assigned after the conditionals; and line by line where the + * grammar runs one recipe_line over several lines (a quote opened in + * "Don't" and closed lines later) -- there a repository script stays out. */ +TEST(probe_make_programs_by_path_spawn) { + static const GpfFile files[] = { + {"a/tools/validate.sh", "#!/bin/sh\necho ok\n"}, + {"a/Makefile", "check:\n" + "\t@docker run --rm img -f $(file)\n" + "\t@./tools/validate.sh\n" + "\t@./_build/tool run\n" + "\t@echo \"Don't forget\"\n" + "\tdocker run --rm img -c \"/app/check.sh 2>/dev/null | sed '/Checking all " + "the pages/ d; /^Maybe wrong/d' | sort -u\"\n"}, + {"b/Makefile", "LOCALBIN ?= $(shell pwd)/bin\n\n" + "envs:\n" + " ifeq ($(A),)\n" + " \t\texport A=1\n" + " endif\n" + " ifeq ($(B),)\n" + "\t\tifeq ($(C),)\n" + "\t\t\texport B := $(C)\n" + "\t\telse\n" + "\t\t\texport B := x\n" + "\t\tendif\n" + " \tendif\n\n" + "\texport R := $(B)\n\n" + "TOOL ?= $(LOCALBIN)/lint-tool\n\n" + "build: envs\n" + "\tbin/werf build\n" + "\t$(TOOL) run\n"}, + {"c/Makefile", "GOBIN ?= $(shell go env GOPATH)/bin\n" + "CONTROLLER_GEN = $(GOBIN)/controller-gen\n\n" + "generate:\n" + "\t$(CONTROLLER_GEN) object paths=./...\n\n" + "probe:\n" + "\t@command -v bin/checker >/dev/null || echo missing\n"}}; + GPF_ASSERT_SPAWNS(files, 4, + "build -> lint-tool\n" + "build -> werf\n" + "check -> tool\n" + "generate -> controller-gen"); + PASS(); +} + /* A Make recipe runs in its Makefile's directory with the Makefile's * variables expanded ($(shell pwd), ?=, a conditional's: unknown): a * kustomize build into kubectl, kubectl -f a directory, find -name -exec @@ -3575,6 +3687,8 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_shell_helm_deploys_its_chart); RUN_TEST(probe_shell_paths_follow_the_script); RUN_TEST(probe_make_recipes_deploy); + RUN_TEST(probe_shell_programs_by_path_spawn); + RUN_TEST(probe_make_programs_by_path_spawn); RUN_TEST(probe_helm_templated_objects_are_resources); RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); From ae93174b70b76e007dd7ecd692f1bd3e5b17d300 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 05:03:19 +0200 Subject: [PATCH 33/52] feat(go): an installer's chart loads DEPLOYS the charts it installs "Which charts does the kubermatic installer deploy?" had no answer in the graph: pkg/install loads each chart with helm.LoadChart(filepath.Join(opt.ChartsDirectory, MonitoringChartsPrefix, NodeExporterChartName)) and installs it, and nothing linked the function to the chart directory. The Go operator pass now reads chart loads: a call of a chart loader -- the Helm SDK's loader.Load / loader.LoadDir, or a top-level function of the repository whose body reads /Chart.yaml or hands its parameter to the SDK loader -- DEPLOYS {via: helm-sdk, chart, line} from the calling function to the Chart its argument names. The argument is evaluated over string literals, the package's constants (an imported package's through the file's imports), filepath.Join / path.Join, + and a local assigned once in the function; a parameter stays unknown. A leading unknown part (a charts root chosen at run time) leaves a known tail: linked to the one chart directory ending with it, or, when two do, to / where the other loads under the same root expression put that root (property "root"). Measured on 17 repositories against an independent reading of the Go sources (its own constant evaluation, imports and suffix matching): 33 of 33 chart loads linked, 33 of 33 correct, all in kubermatic (27 installer functions, every chart-name constant of pkg/install that the installer loads; 6 integration tests loading testdata charts). Only these DEPLOYS edges were added; three runs build identical graphs; index time unchanged (kubermatic 2.94 s, gardener 5.22 -> 5.24 s, deckhouse 9.62 -> 9.59 s, medians). Files mentioning a chart are summarized as they are read and read again only when a loader exists. Signed-off-by: Martin Vogel --- src/pipeline/pass_operator.c | 1055 +++++++++++++++++++++++++++++++++- tests/test_grammar_probe_f.c | 57 ++ 2 files changed, 1108 insertions(+), 4 deletions(-) diff --git a/src/pipeline/pass_operator.c b/src/pipeline/pass_operator.c index e4b96e900..85fd106d8 100644 --- a/src/pipeline/pass_operator.c +++ b/src/pipeline/pass_operator.c @@ -13,8 +13,12 @@ * marked `+groupName=` (or `GroupName = ""`): USAGE * {ref: crd-type} from the CRD Resource to that Struct; * - `//go:embed ` puts files into the binary: REFERENCES_FILE - * {via: go:embed} from the Go file to each embedded file. - * Runs after USAGE resolution; reads each Go file once. + * {via: go:embed} from the Go file to each embedded file; + * - an installer loads a chart of the repository (helm.LoadChart( + * filepath.Join(opt.ChartsDirectory, ...)), the Helm SDK's loader): + * DEPLOYS {via: helm-sdk} from the function to the Chart (PHASE 3 P3.3). + * Runs after USAGE resolution; reads each Go file once, and the files that + * mention a chart again. */ #include "foundation/constants.h" #include "foundation/compat.h" @@ -967,6 +971,1037 @@ static bool op_is_go(const char *path) { return n > OP_SLEN(".go") && strcmp(path + n - OP_SLEN(".go"), ".go") == 0; } +/* ── Helm charts the Go code loads (PHASE 3 P3.3) ─────────────────────── */ + +/* An installer written in Go (kubermatic's) deploys charts it loads from a + * charts directory: helm.LoadChart(filepath.Join(opt.ChartsDirectory, + * MonitoringChartsPrefix, NodeExporterChartName)). A chart loader is the + * Helm SDK's loader.Load / loader.LoadDir, or a function of the repository + * whose body reads `/Chart.yaml` or hands its parameter to the SDK + * loader. Its call DEPLOYS {via: helm-sdk} the chart directory its argument + * names, evaluated over string literals, the package's constants (and an + * imported package's), filepath.Join / path.Join, + and a local assigned + * once: a leading unknown part (a charts root chosen at run time) leaves a + * known tail, linked when exactly one chart directory of the repository ends + * with it. */ + +typedef struct { + char *dir; /* package directory */ + char *name; /* function */ + int param; /* index of the chart-directory parameter */ +} gc_loader_t; + +typedef struct { + const char *dir; /* graph-owned chart dir path */ + int64_t id; +} gc_chart_t; + +typedef struct { + gc_chart_t *items; + int count; + int cap; +} gc_charts_t; + +/* One chart load whose argument evaluated: linked once all are read. */ +typedef struct { + int64_t fn; + int line; + char *tail; + char *root; /* NULL: no unknown root */ +} gc_site_t; + +typedef struct { + cbm_pipeline_ctx_t *ctx; + const op_modules_t *mods; + CBMHashTable *by_file; + const cbm_gbuf_node_t **files; /* sorted File nodes */ + int nfiles; + gc_loader_t *loaders; + int nloaders; + int cap_loaders; + char **noted; /* Go files that mention a chart */ + int nnoted; + int cap_noted; + CBMHashTable *consts; /* dir -> CBMHashTable name -> expression (owned) */ + CBMHashTable *charts; /* chart dir basename -> gc_charts_t */ + gc_site_t *sites; + int nsites; + int cap_sites; + int edges; +} gc_t; + +typedef struct { + char s[CBM_SZ_1K]; + bool known; + bool open_root; /* an unknown leading part: s is a tail */ + char root[CBM_SZ_256]; /* that part's source text (opt.ChartsDirectory) */ +} gc_val_t; + +enum { GC_ARGS_MAX = 8, GC_EVAL_DEPTH = 16 }; + +static bool gc_grow(void **items, int *cap, int count, size_t elem) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : OP_GROW_MIN; + void *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, *items, (size_t)ncap * elem); + if (!grown) { + return false; + } + *items = grown; + *cap = ncap; + return true; +} + +/* Past a string / rune literal or a comment starting at p; p when none. */ +static const char *gc_skip(const char *p) { + if (*p == '"' || *p == '\'') { + char q = *p++; + while (*p && *p != q && *p != '\n') { + p += (*p == '\\' && p[SKIP_ONE]) ? PAIR_LEN : SKIP_ONE; + } + return *p == q ? p + SKIP_ONE : p; + } + if (*p == '`') { + const char *e = strchr(p + SKIP_ONE, '`'); + return e ? e + SKIP_ONE : p + strlen(p); + } + if (p[0] == '/' && p[SKIP_ONE] == '/') { + const char *e = strchr(p, '\n'); + return e ? e : p + strlen(p); + } + if (p[0] == '/' && p[SKIP_ONE] == '*') { + const char *e = strstr(p + PAIR_LEN, "*/"); + return e ? e + PAIR_LEN : p + strlen(p); + } + return p; +} + +/* The '}' closing the block opened at `open`. */ +static const char *gc_block_end(const char *open) { + int depth = 0; + for (const char *p = open; *p;) { + const char *q = gc_skip(p); + if (q != p) { + p = q; + continue; + } + depth += *p == '{'; + depth -= *p == '}'; + if (depth == 0) { + return p; + } + p++; + } + return open + strlen(open); +} + +/* The top-level comma-separated parts of the list opened at `open` ('('). + * Returns their count (at most `max`); *end: the closing ')'. */ +static int gc_args(const char *open, const char *starts[], size_t lens[], int max, + const char **end) { + int n = 0; + int depth = 0; + const char *start = open + SKIP_ONE; + for (const char *p = open; *p;) { + const char *q = gc_skip(p); + if (q != p) { + p = q; + continue; + } + if (strchr("([{", *p)) { + depth++; + } else if (strchr(")]}", *p)) { + depth--; + if (depth == 0) { + if (n < max && p > start) { + starts[n] = start; + lens[n++] = (size_t)(p - start); + } + *end = p; + return n; + } + } else if (*p == ',' && depth == SKIP_ONE && n < max) { + starts[n] = start; + lens[n++] = (size_t)(p - start); + start = p + SKIP_ONE; + } + p++; + } + *end = open + strlen(open); + return n; +} + +static void gc_trim(const char **s, size_t *n) { + while (*n && strchr(" \t\r\n", **s)) { + (*s)++; + (*n)--; + } + while (*n && strchr(" \t\r\n", (*s)[*n - SKIP_ONE])) { + (*n)--; + } +} + +static bool gc_is_ident(const char *s, size_t n) { + if (n == 0 || (s[0] >= '0' && s[0] <= '9')) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!op_ident_char(s[i])) { + return false; + } + } + return true; +} + +static bool gc_ident_eq(const char *s, size_t n, const char *name) { + return strlen(name) == n && strncmp(s, name, n) == 0; +} + +/* The repository directory a file binds package qualifier `qual` to. */ +static bool gc_qual_dir(const gc_t *g, const char *text, const char *qual, size_t ql, char *out, + size_t out_sz) { + char alias[OP_IDENT_MAX]; + char import[CBM_SZ_1K]; + if (ql == 0 || ql >= sizeof(alias)) { + return false; + } + snprintf(alias, sizeof(alias), "%.*s", (int)ql, qual); + op_import_path(text, alias, import, sizeof(import)); + return import[0] && op_import_dir(g->mods, import, out, out_sz); +} + +static bool gc_qual_is(const char *text, const char *qual, size_t ql, const char *path) { + char alias[OP_IDENT_MAX]; + char import[CBM_SZ_1K]; + if (ql == 0 || ql >= sizeof(alias)) { + return false; + } + snprintf(alias, sizeof(alias), "%.*s", (int)ql, qual); + op_import_path(text, alias, import, sizeof(import)); + return strcmp(import, path) == 0; +} + +static const char GC_SDK_LOADER[] = "helm.sh/helm/v3/pkg/chart/loader"; + +/* The flag before a noted file's path: it imports the SDK loader, or not. */ +enum { GC_NOTED_SDK = 'S', GC_NOTED_PLAIN = '-' }; + +/* ── Loaders: functions whose parameter is a chart directory ── */ + +static void gc_note_loader(gc_t *g, const char *dir, const char *name, size_t nl, int param) { + for (int i = 0; i < g->nloaders; i++) { + if (strcmp(g->loaders[i].dir, dir) == 0 && gc_ident_eq(name, nl, g->loaders[i].name)) { + return; + } + } + if (!gc_grow((void **)&g->loaders, &g->cap_loaders, g->nloaders, sizeof(*g->loaders))) { + return; + } + char nm[OP_IDENT_MAX]; + snprintf(nm, sizeof(nm), "%.*s", (int)nl, name); + g->loaders[g->nloaders++] = (gc_loader_t){cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir), + cbm_mem_strdup(CBM_MEM_CLASS_OTHER, nm), param}; +} + +/* Does body [b, e) load a chart from the parameter named p: Join(p, + * "Chart.yaml") or the SDK loader on p. */ +static bool gc_body_loads(const char *text, const char *b, const char *e, const char *p, + size_t pl) { + for (const char *s = b; s < e;) { + const char *q = gc_skip(s); + if (q != s) { + s = q; + continue; + } + const char *open = NULL; + bool join = false; + if (strncmp(s, "filepath.Join(", OP_SLEN("filepath.Join(")) == 0) { + open = s + OP_SLEN("filepath.Join"); + join = true; + } else if (strncmp(s, "path.Join(", OP_SLEN("path.Join(")) == 0 && + (s == b || !op_ident_char(s[-SKIP_ONE]))) { + open = s + OP_SLEN("path.Join"); + join = true; + } else if ((strncmp(s, ".Load(", OP_SLEN(".Load(")) == 0 || + strncmp(s, ".LoadDir(", OP_SLEN(".LoadDir(")) == 0)) { + const char *qe = s; + const char *qs = qe; + while (qs > text && op_ident_char(qs[-SKIP_ONE])) { + qs--; + } + if (gc_qual_is(text, qs, (size_t)(qe - qs), GC_SDK_LOADER)) { + open = strchr(s, '('); + } + } + if (!open) { + s++; + continue; + } + const char *starts[GC_ARGS_MAX]; + size_t lens[GC_ARGS_MAX]; + const char *end = NULL; + int n = gc_args(open, starts, lens, GC_ARGS_MAX, &end); + for (int i = 0; i < n; i++) { + gc_trim(&starts[i], &lens[i]); + } + if (n > 0 && lens[0] == pl && strncmp(starts[0], p, pl) == 0) { + if (!join && n == SKIP_ONE) { + return true; + } + if (join && n == PAIR_LEN && + (gc_ident_eq(starts[SKIP_ONE], lens[SKIP_ONE], "\"Chart.yaml\"") || + gc_ident_eq(starts[SKIP_ONE], lens[SKIP_ONE], "\"Chart.yml\""))) { + return true; + } + } + s = end > s ? end : s + SKIP_ONE; + } + return false; +} + +/* Note the file's top-level functions (not methods) that load a chart from + * a parameter. */ +static void gc_summarize(gc_t *g, const char *file, const char *text) { + char dir[CBM_SZ_1K]; + op_dirname(file, dir, sizeof(dir)); + for (const char *p = strstr(text, "func "); p; p = strstr(p + SKIP_ONE, "func ")) { + if (p > text && p[-SKIP_ONE] != '\n') { + continue; + } + const char *name = p + OP_SLEN("func "); + const char *ne = name; + while (op_ident_char(*ne)) { + ne++; + } + if (ne == name || *ne != '(') { + continue; /* a method, or generic */ + } + const char *starts[GC_ARGS_MAX]; + size_t lens[GC_ARGS_MAX]; + const char *pend = NULL; + int np = gc_args(ne, starts, lens, GC_ARGS_MAX, &pend); + const char *open = pend; + while (*open && *open != '{' && *open != '\n') { + open++; + } + if (*open != '{') { + continue; + } + const char *close = gc_block_end(open); + for (int i = 0; i < np; i++) { + const char *ps = starts[i]; + size_t pl = lens[i]; + gc_trim(&ps, &pl); + size_t idl = 0; + while (idl < pl && op_ident_char(ps[idl])) { + idl++; + } + if (idl && gc_body_loads(text, open, close, ps, idl)) { + gc_note_loader(g, dir, name, (size_t)(ne - name), i); + } + } + } +} + +/* ── Constants of a package ── */ + +static void gc_free_expr(const char *key, void *value, void *ud) { + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); + cbm_free(CBM_MEM_CLASS_OTHER, value); +} + +static void gc_free_consts(const char *key, void *value, void *ud) { + (void)ud; + cbm_ht_foreach(value, gc_free_expr, NULL); + cbm_ht_free(value); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +/* One `NAME [TYPE] = expr` of a const declaration, line [ls, le). */ +static void gc_const_line(CBMHashTable *t, const char *ls, const char *le) { + while (ls < le && (*ls == ' ' || *ls == '\t')) { + ls++; + } + const char *ne = ls; + while (ne < le && op_ident_char(*ne)) { + ne++; + } + const char *eq = memchr(ne, '=', (size_t)(le - ne)); + if (ne == ls || !eq || (eq + SKIP_ONE < le && eq[SKIP_ONE] == '=')) { + return; + } + const char *ve = op_comment_start(eq + SKIP_ONE, le); + const char *vs = eq + SKIP_ONE; + size_t vl = (size_t)(ve - vs); + gc_trim(&vs, &vl); + char name[OP_IDENT_MAX]; + snprintf(name, sizeof(name), "%.*s", (int)(ne - ls), ls); + if (vl == 0 || cbm_ht_get(t, name)) { + return; + } + char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, name); + char *val = cbm_alloc(CBM_MEM_CLASS_OTHER, vl + SKIP_ONE); + if (!key || !val) { + cbm_free(CBM_MEM_CLASS_OTHER, key); + cbm_free(CBM_MEM_CLASS_OTHER, val); + return; + } + memcpy(val, vs, vl); + val[vl] = '\0'; + cbm_ht_set(t, key, val); +} + +static void gc_scan_consts(CBMHashTable *t, const char *text) { + for (const char *p = strstr(text, "const"); p; p = strstr(p + SKIP_ONE, "const")) { + if ((p > text && p[-SKIP_ONE] != '\n') || op_ident_char(p[OP_SLEN("const")])) { + continue; + } + const char *q = p + OP_SLEN("const"); + while (*q == ' ' || *q == '\t') { + q++; + } + if (*q == '(') { + const char *starts[SKIP_ONE]; + size_t lens[SKIP_ONE]; + const char *end = NULL; + (void)gc_args(q, starts, lens, 0, &end); + for (const char *l = q + SKIP_ONE; l < end;) { + const char *le = memchr(l, '\n', (size_t)(end - l)); + le = le ? le : end; + gc_const_line(t, l, le); + l = le + SKIP_ONE; + } + } else { + const char *le = strchr(q, '\n'); + gc_const_line(t, q, le ? le : q + strlen(q)); + } + } +} + +/* The constants of package directory `dir` (its non-test Go files). */ +static CBMHashTable *gc_consts(gc_t *g, const char *dir) { + CBMHashTable *t = cbm_ht_get(g->consts, dir); + if (t) { + return t; + } + t = cbm_ht_create(0); + char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir); + if (!t || !key) { + if (t) { + cbm_ht_free(t); + } + cbm_free(CBM_MEM_CLASS_OTHER, key); + return NULL; + } + cbm_ht_set(g->consts, key, t); + size_t dl = strlen(dir); + for (int i = 0; i < g->nfiles; i++) { + const char *fp = g->files[i]->file_path; + const char *slash = strrchr(fp, '/'); + size_t fdl = slash ? (size_t)(slash - fp) : 0; + size_t fl = strlen(fp); + if (fdl != dl || strncmp(fp, dir, dl) != 0 || !op_is_go(fp) || + (fl > OP_SLEN("_test.go") && strcmp(fp + fl - OP_SLEN("_test.go"), "_test.go") == 0)) { + continue; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", g->ctx->repo_path, fp); + char *text = op_read_file(path); + if (text) { + gc_scan_consts(t, text); + cbm_free(CBM_MEM_CLASS_OTHER, text); + } + } + return t; +} + +/* ── Values ── */ + +typedef struct { + gc_t *g; + const char *text; /* the call's file */ + const char *dir; /* its package */ + const char *sig; /* the enclosing function's `func` .. its body, NULL at top level */ + const char *body; + const char *body_end; + const char *at; /* the call */ +} gc_scope_t; + +/* Is `name` a word of the enclosing function's signature (a parameter). */ +static bool gc_is_param(const gc_scope_t *sc, const char *name, size_t nl) { + for (const char *p = sc->sig; sc->sig && p + nl <= sc->body; p++) { + if (strncmp(p, name, nl) == 0 && !op_ident_char(p[-SKIP_ONE]) && !op_ident_char(p[nl])) { + return true; + } + } + return false; +} + +static void gc_eval(const gc_scope_t *sc, const char *s, size_t n, int depth, gc_val_t *out); + +static void gc_join(gc_val_t *acc, const gc_val_t *part, bool path) { + if (!acc->known) { + return; + } + bool empty = !acc->s[0]; + if (!part->known || part->open_root) { + if (!path || !empty) { + acc->known = false; /* an unknown part after a known one */ + return; + } + acc->open_root = true; /* still the leading, unknown root */ + if (!part->known) { + return; + } + } + size_t al = strlen(acc->s); + bool sep = path && al && part->s[0] && acc->s[al - SKIP_ONE] != '/' && part->s[0] != '/'; + snprintf(acc->s + al, sizeof(acc->s) - al, "%s%s", sep ? "/" : "", part->s); +} + +/* A local assigned exactly once in the enclosing function, before the call. */ +static bool gc_local(const gc_scope_t *sc, const char *name, size_t nl, const char **vs, + size_t *vlen) { + if (!sc->body) { + return false; + } + int seen = 0; + for (const char *p = sc->body; p < sc->body_end;) { + const char *q = gc_skip(p); + if (q != p) { + p = q; + continue; + } + if (strncmp(p, name, nl) == 0 && (p == sc->body || !op_ident_char(p[-SKIP_ONE])) && + !op_ident_char(p[nl]) && p[-SKIP_ONE] != '.') { + const char *a = p + nl; + while (*a == ' ' || *a == '\t') { + a++; + } + if (a[0] && strchr("+-*/%|&^", a[0]) && a[SKIP_ONE] == '=') { + return false; /* modified in place */ + } + bool assign = + (a[0] == ':' && a[SKIP_ONE] == '=') || (a[0] == '=' && a[SKIP_ONE] != '='); + if (assign) { + if (++seen > SKIP_ONE || p > sc->at) { + return false; + } + const char *v = a + (a[0] == ':' ? PAIR_LEN : SKIP_ONE); + const char *le = strchr(v, '\n'); + le = le ? le : v + strlen(v); + *vs = v; + *vlen = (size_t)(op_comment_start(v, le) - v); + } + } + p++; + } + return seen == SKIP_ONE; +} + +static void gc_const_value(const gc_scope_t *sc, const char *dir, const char *name, int depth, + gc_val_t *out) { + CBMHashTable *t = gc_consts(sc->g, dir); + const char *expr = t ? cbm_ht_get(t, name) : NULL; + if (!expr) { + out->known = false; + return; + } + gc_scope_t inner = {sc->g, sc->text, dir, NULL, NULL, NULL, NULL}; + gc_eval(&inner, expr, strlen(expr), depth + SKIP_ONE, out); +} + +static void gc_eval(const gc_scope_t *sc, const char *s, size_t n, int depth, gc_val_t *out) { + *out = (gc_val_t){.known = false}; + gc_trim(&s, &n); + if (n == 0 || depth > GC_EVAL_DEPTH) { + return; + } + /* a + b */ + int pd = 0; + for (size_t i = 0; i < n; i++) { + const char *q = gc_skip(s + i); + if (q != s + i) { + i = (size_t)(q - s) - SKIP_ONE; + continue; + } + pd += strchr("([{", s[i]) != NULL; + pd -= strchr(")]}", s[i]) != NULL; + if (s[i] == '+' && pd == 0) { + gc_val_t a; + gc_val_t b; + gc_eval(sc, s, i, depth + SKIP_ONE, &a); + gc_eval(sc, s + i + SKIP_ONE, n - i - SKIP_ONE, depth + SKIP_ONE, &b); + *out = a; + if (!b.known || b.open_root) { + out->known = false; + } else if (out->known) { + gc_join(out, &b, false); + } + return; + } + } + if ((s[0] == '"' || s[0] == '`') && gc_skip(s) == s + n) { + size_t w = 0; + for (size_t i = SKIP_ONE; i + SKIP_ONE < n && w + SKIP_ONE < sizeof(out->s); i++) { + if (s[0] == '"' && s[i] == '\\' && i + PAIR_LEN < n) { + i++; + } + out->s[w++] = s[i]; + } + out->s[w] = '\0'; + out->known = true; + return; + } + bool fp = n > OP_SLEN("filepath.Join(") && + strncmp(s, "filepath.Join(", OP_SLEN("filepath.Join(")) == 0; + bool pp = n > OP_SLEN("path.Join(") && strncmp(s, "path.Join(", OP_SLEN("path.Join(")) == 0; + if ((fp || pp) && s[n - SKIP_ONE] == ')') { + const char *open = s + (fp ? OP_SLEN("filepath.Join") : OP_SLEN("path.Join")); + const char *starts[GC_ARGS_MAX]; + size_t lens[GC_ARGS_MAX]; + const char *end = NULL; + int na = gc_args(open, starts, lens, GC_ARGS_MAX, &end); + if (end != s + n - SKIP_ONE || na == 0 || na == GC_ARGS_MAX) { + return; + } + *out = (gc_val_t){.known = true}; + for (int i = 0; i < na && out->known; i++) { + gc_val_t part; + gc_eval(sc, starts[i], lens[i], depth + SKIP_ONE, &part); + bool was_open = out->open_root; + gc_join(out, &part, true); + if (out->known && out->open_root && !was_open) { + if (part.known && part.root[0]) { + snprintf(out->root, sizeof(out->root), "%s", part.root); + } else { + const char *rs = starts[i]; + size_t rl = lens[i]; + gc_trim(&rs, &rl); + snprintf(out->root, sizeof(out->root), "%.*s", (int)rl, rs); + } + } else if (out->known && was_open && !part.known) { + out->root[0] = '\0'; /* several unknown parts: no one root */ + } + } + return; + } + if (gc_is_ident(s, n)) { + const char *vs = NULL; + size_t vl = 0; + char name[OP_IDENT_MAX]; + snprintf(name, sizeof(name), "%.*s", (int)n, s); + if (gc_local(sc, s, n, &vs, &vl)) { + gc_scope_t outer = *sc; + outer.at = vs; /* what the local reads, read before it */ + gc_eval(&outer, vs, vl, depth + SKIP_ONE, out); + return; + } + if (gc_is_param(sc, s, n)) { + return; /* an argument: whatever the caller passes */ + } + gc_const_value(sc, sc->dir, name, depth, out); + return; + } + const char *dot = memchr(s, '.', n); + if (dot && gc_is_ident(s, (size_t)(dot - s)) && + gc_is_ident(dot + SKIP_ONE, n - (size_t)(dot - s) - SKIP_ONE)) { + char dir[CBM_SZ_1K]; + char name[OP_IDENT_MAX]; + snprintf(name, sizeof(name), "%.*s", (int)(n - (size_t)(dot - s) - SKIP_ONE), + dot + SKIP_ONE); + if (gc_qual_dir(sc->g, sc->text, s, (size_t)(dot - s), dir, sizeof(dir))) { + gc_const_value(sc, dir, name, depth, out); + } + } +} + +/* ── Charts and edges ── */ + +static void gc_free_chart_list(const char *key, void *value, void *ud) { + (void)ud; + gc_charts_t *v = value; + cbm_free(CBM_MEM_CLASS_OTHER, v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static CBMHashTable *gc_chart_index(const cbm_gbuf_t *gb) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!ix || cbm_gbuf_find_by_label(gb, "Chart", &nodes, &count) != 0) { + return ix; + } + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + const char *qn = nodes[i]->qualified_name; + const char *slash = fp ? strrchr(fp, '/') : NULL; + const char *base = slash ? slash + SKIP_ONE : fp; + if (!fp || !qn || strstr(qn, ".__helm_dep__.") || + (strcmp(base, "Chart.yaml") != 0 && strcmp(base, "Chart.yml") != 0) || !slash) { + continue; + } + const char *db = slash; + while (db > fp && db[-SKIP_ONE] != '/') { + db--; + } + char key[CBM_SZ_256]; + snprintf(key, sizeof(key), "%.*s", (int)(slash - db), db); + gc_charts_t *v = cbm_ht_get(ix, key); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *k = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!v || !k) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, k); + continue; + } + cbm_ht_set(ix, k, v); + } + if (gc_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = (gc_chart_t){fp, nodes[i]->id}; + } + } + return ix; +} + +/* The one chart directory a value names: the tail itself, or the only chart + * directory ending with it. */ +static const gc_chart_t *gc_match(gc_t *g, const gc_val_t *v, char *dir, size_t dir_sz) { + const char *t = v->s; + while (t[0] == '.' && t[SKIP_ONE] == '/') { + t += PAIR_LEN; + } + while (*t == '/') { + t++; + } + snprintf(dir, dir_sz, "%s", t); + size_t dl = strlen(dir); + while (dl && dir[dl - SKIP_ONE] == '/') { + dir[--dl] = '\0'; + } + if (!dl) { + return NULL; + } + const char *base = strrchr(dir, '/'); + base = base ? base + SKIP_ONE : dir; + const gc_charts_t *list = cbm_ht_get(g->charts, base); + const gc_chart_t *hit = NULL; + for (int i = 0; list && i < list->count; i++) { + atomic_fetch_add_explicit(&g_op_candidates, SKIP_ONE, memory_order_relaxed); + const char *cd = list->items[i].dir; /* "/Chart.yaml" */ + size_t cl = (size_t)(strrchr(cd, '/') - cd); + bool same = cl == dl && strncmp(cd, dir, dl) == 0; + bool tail = cl > dl && cd[cl - dl - SKIP_ONE] == '/' && strncmp(cd + cl - dl, dir, dl) == 0; + if (same || tail) { + if (hit) { + return NULL; + } + hit = &list->items[i]; + } + } + if (hit) { + const char *cd = hit->dir; + snprintf(dir, dir_sz, "%.*s", (int)(strrchr(cd, '/') - cd), cd); + } + return hit; +} + +/* The body of the top-level function around `at`, else NULL; *sig: its + * `func` keyword. */ +static const char *gc_body_of(const char *text, const char *at, const char **end, + const char **sig) { + const char *best = NULL; + for (const char *p = strstr(text, "func "); p && p < at; p = strstr(p + SKIP_ONE, "func ")) { + if (p == text || p[-SKIP_ONE] == '\n') { + best = p; + } + } + if (!best) { + return NULL; + } + const char *open = best; + int pd = 0; + while (*open && !(*open == '{' && pd == 0)) { + const char *q = gc_skip(open); + if (q != open) { + open = q; + continue; + } + pd += *open == '('; + pd -= *open == ')'; + open++; + } + if (*open != '{') { + return NULL; + } + const char *close = gc_block_end(open); + if (close < at) { + return NULL; + } + *end = close; + *sig = best; + return open; +} + +static void gc_link_call(gc_t *g, const char *file, const char *text, const char *name_at, + const char *open, int param) { + const char *starts[GC_ARGS_MAX]; + size_t lens[GC_ARGS_MAX]; + const char *end = NULL; + int n = gc_args(open, starts, lens, GC_ARGS_MAX, &end); + if (param >= n) { + return; + } + char dir[CBM_SZ_1K]; + op_dirname(file, dir, sizeof(dir)); + const char *body_end = NULL; + const char *sig = NULL; + const char *body = gc_body_of(text, name_at, &body_end, &sig); + gc_scope_t sc = {g, text, dir, body ? sig : NULL, body, body_end, name_at}; + gc_val_t v; + gc_eval(&sc, starts[param], lens[param], 0, &v); + int line = op_line_of(text, name_at); + const cbm_gbuf_node_t *fn = v.known ? op_enclosing(g->by_file, file, line) : NULL; + if (!fn || !gc_grow((void **)&g->sites, &g->cap_sites, g->nsites, sizeof(*g->sites))) { + return; + } + g->sites[g->nsites++] = + (gc_site_t){fn->id, line, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, v.s), + v.open_root && v.root[0] ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, v.root) : NULL}; +} + +static void gc_edge(gc_t *g, const gc_site_t *s, const gc_chart_t *hit, const char *chart, + const char *root) { + char esc[CBM_SZ_1K]; + char resc[CBM_SZ_512]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), chart); + cbm_json_escape(resc, (int)sizeof(resc), root ? root : ""); + snprintf(props, sizeof(props), "{\"via\":\"helm-sdk\",\"chart\":\"%s\"%s%s%s,\"line\":%d}", esc, + root ? ",\"root\":\"" : "", root ? resc : "", root ? "\"" : "", s->line); + cbm_gbuf_insert_edge(g->ctx->gbuf, s->fn, hit->id, "DEPLOYS", props); + g->edges++; +} + +/* The repository dir a chart load's unknown root stood for: the matched + * chart directory without the tail. */ +static void gc_root_dir(const char *chart, const char *tail, char *out, size_t out_sz) { + while (tail[0] == '.' && tail[SKIP_ONE] == '/') { + tail += PAIR_LEN; + } + while (*tail == '/') { + tail++; + } + size_t cl = strlen(chart); + size_t tl = strlen(tail); + while (tl && tail[tl - SKIP_ONE] == '/') { + tl--; + } + size_t rl = cl > tl ? cl - tl - SKIP_ONE : 0; + snprintf(out, out_sz, "%.*s", (int)rl, chart); +} + +static const char GC_CONFLICT[] = "\x01"; + +/* Remember that root text `root` stood for directory `rdir`; two different + * directories for one root text: it stands for none. */ +static void gc_note_root(CBMHashTable *roots, const char *root, const char *rdir) { + const char *known = cbm_ht_get(roots, root); + if (!known) { + char *k = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, root); + char *d = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, rdir); + if (k && d) { + cbm_ht_set(roots, k, d); + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, k); + cbm_free(CBM_MEM_CLASS_OTHER, d); + return; + } + if (strcmp(known, GC_CONFLICT) != 0 && strcmp(known, rdir) != 0) { + char *c = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, GC_CONFLICT); + if (c) { + cbm_free(CBM_MEM_CLASS_OTHER, cbm_ht_set(roots, cbm_ht_get_key(roots, root), c)); + } + } +} + +/* Link the chart loads: a tail exactly one chart directory ends with; else, + * when the loads under the same unknown root (opt.ChartsDirectory) whose + * tail matched one directory all put that root at one place, /. */ +static void gc_link_sites(gc_t *g) { + CBMHashTable *roots = cbm_ht_create(0); /* root text -> its dir (owned) */ + for (int i = 0; roots && i < g->nsites; i++) { + const gc_site_t *s = &g->sites[i]; + gc_val_t v = {.known = true, .open_root = s->root != NULL}; + snprintf(v.s, sizeof(v.s), "%s", s->tail); + char chart[CBM_SZ_1K]; + if (s->root && gc_match(g, &v, chart, sizeof(chart))) { + char rdir[CBM_SZ_1K]; + gc_root_dir(chart, s->tail, rdir, sizeof(rdir)); + gc_note_root(roots, s->root, rdir); + } + } + for (int i = 0; roots && i < g->nsites; i++) { + const gc_site_t *s = &g->sites[i]; + gc_val_t v = {.known = true, .open_root = s->root != NULL}; + snprintf(v.s, sizeof(v.s), "%s", s->tail); + char chart[CBM_SZ_1K]; + const gc_chart_t *hit = gc_match(g, &v, chart, sizeof(chart)); + if (hit) { + gc_edge(g, s, hit, chart, NULL); + continue; + } + const char *rdir = s->root ? cbm_ht_get(roots, s->root) : NULL; + if (!rdir || strcmp(rdir, GC_CONFLICT) == 0) { + continue; + } + gc_val_t full = {.known = true}; + snprintf(full.s, sizeof(full.s), "%s%s%s", rdir, rdir[0] ? "/" : "", s->tail); + char want[CBM_SZ_1K]; + snprintf(want, sizeof(want), "%s", full.s); + hit = gc_match(g, &full, chart, sizeof(chart)); + if (hit && strcmp(chart, want) == 0) { + gc_edge(g, s, hit, chart, rdir); + } + } + if (roots) { + cbm_ht_foreach(roots, gc_free_expr, NULL); + cbm_ht_free(roots); + } +} + +/* The chart loads in one file: calls of a noted loader (qualified by an + * import of its package, or unqualified in it) and of the SDK loader. */ +static void gc_calls(gc_t *g, const char *file, const char *text) { + char dir[CBM_SZ_1K]; + op_dirname(file, dir, sizeof(dir)); + for (const char *p = text; *p;) { + const char *q = gc_skip(p); + if (q != p) { + p = q; + continue; + } + if (!op_ident_char(*p) || (p > text && op_ident_char(p[-SKIP_ONE]))) { + p++; + continue; + } + const char *ne = p; + while (op_ident_char(*ne)) { + ne++; + } + if (*ne != '(') { + p = ne; + continue; + } + size_t nl = (size_t)(ne - p); + const char *qual = NULL; + size_t ql = 0; + if (p > text && p[-SKIP_ONE] == '.') { + qual = p - SKIP_ONE; + while (qual > text && op_ident_char(qual[-SKIP_ONE])) { + qual--; + } + ql = (size_t)(p - SKIP_ONE - qual); + } + bool is_def = p >= text + OP_SLEN("func ") && + strncmp(p - OP_SLEN("func "), "func ", OP_SLEN("func ")) == 0; + int param = -1; + if (!is_def && qual && (gc_ident_eq(p, nl, "Load") || gc_ident_eq(p, nl, "LoadDir")) && + gc_qual_is(text, qual, ql, GC_SDK_LOADER)) { + param = 0; + } + char qdir[CBM_SZ_1K]; + bool resolved = !qual || gc_qual_dir(g, text, qual, ql, qdir, sizeof(qdir)); + for (int i = 0; !is_def && param < 0 && resolved && i < g->nloaders; i++) { + if (gc_ident_eq(p, nl, g->loaders[i].name) && + strcmp(g->loaders[i].dir, qual ? qdir : dir) == 0) { + param = g->loaders[i].param; + } + } + if (param >= 0) { + gc_link_call(g, file, text, p, ne, param); + } + p = ne; + } +} + +/* A Go file that mentions a chart: its loaders noted now; the file is read + * again for its loader calls once every loader is known. A file importing + * the SDK loader is marked: it is read again even when no loader of the + * repository exists. */ +static void gc_note(gc_t *g, const char *file, const char *text) { + if (!strstr(text, "Chart") && !strstr(text, "chart/loader")) { + return; + } + gc_summarize(g, file, text); + bool sdk = strstr(text, GC_SDK_LOADER) != NULL; + size_t fl = strlen(file); + char *entry = cbm_alloc(CBM_MEM_CLASS_OTHER, fl + PAIR_LEN); /* flag + path */ + if (!entry || !gc_grow((void **)&g->noted, &g->cap_noted, g->nnoted, sizeof(*g->noted))) { + cbm_free(CBM_MEM_CLASS_OTHER, entry); + return; + } + entry[0] = sdk ? GC_NOTED_SDK : GC_NOTED_PLAIN; + memcpy(entry + SKIP_ONE, file, fl + SKIP_ONE); + g->noted[g->nnoted++] = entry; +} + +static int gc_run(gc_t *g) { + if (g->nnoted == 0) { + return 0; + } + g->consts = cbm_ht_create(0); + g->charts = gc_chart_index(g->ctx->gbuf); + for (int i = 0; g->consts && g->charts && i < g->nnoted; i++) { + const char *noted = g->noted[i]; + if (!noted || (g->nloaders == 0 && noted[0] != GC_NOTED_SDK)) { + continue; /* no loader it could call */ + } + const char *file = noted + SKIP_ONE; + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", g->ctx->repo_path, file); + char *text = op_read_file(path); + if (text) { + gc_calls(g, file, text); + cbm_free(CBM_MEM_CLASS_OTHER, text); + } + } + if (g->charts) { + gc_link_sites(g); + } + if (g->consts) { + cbm_ht_foreach(g->consts, gc_free_consts, NULL); + cbm_ht_free(g->consts); + } + if (g->charts) { + cbm_ht_foreach(g->charts, gc_free_chart_list, NULL); + cbm_ht_free(g->charts); + } + return g->edges; +} + +static void gc_free(gc_t *g) { + for (int i = 0; i < g->nloaders; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, g->loaders[i].dir); + cbm_free(CBM_MEM_CLASS_OTHER, g->loaders[i].name); + } + cbm_free(CBM_MEM_CLASS_OTHER, g->loaders); + for (int i = 0; i < g->nnoted; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, g->noted[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)g->noted); + for (int i = 0; i < g->nsites; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, g->sites[i].tail); + cbm_free(CBM_MEM_CLASS_OTHER, g->sites[i].root); + } + cbm_free(CBM_MEM_CLASS_OTHER, g->sites); +} + int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { cbm_gbuf_t *gb = ctx->gbuf; const cbm_gbuf_node_t **all = NULL; @@ -1003,6 +2038,7 @@ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { } } op_index_t ix = {.mods = &mods, .dir_types = op_types_by_dir(gb)}; + gc_t gc = {0}; int handles = 0; int embeds = 0; bool ready = by_file && groups && seen && ix.dir_types; @@ -1023,9 +2059,17 @@ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { if (strstr(text, "//go:embed ")) { embeds += op_embed(gb, files[i], text, files, nfiles); } + gc_note(&gc, files[i]->file_path, text); cbm_free(CBM_MEM_CLASS_OTHER, text); } int crd_types = ready ? op_link_crd_types(gb, &ix, groups) : 0; + gc.ctx = ctx; + gc.mods = &mods; + gc.by_file = by_file; + gc.files = files; + gc.nfiles = nfiles; + int deploys = ready ? gc_run(&gc) : 0; + gc_free(&gc); if (ix.dir_types) { cbm_ht_foreach(ix.dir_types, op_free_nodes_entry, NULL); cbm_ht_free(ix.dir_types); @@ -1044,14 +2088,17 @@ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { cbm_ht_free(by_file); } cbm_free(CBM_MEM_CLASS_OTHER, (void *)files); - if (handles + embeds + crd_types > 0) { + if (handles + embeds + crd_types + deploys > 0) { char h[CBM_SZ_32]; char e[CBM_SZ_32]; char c[CBM_SZ_32]; + char d[CBM_SZ_32]; snprintf(h, sizeof(h), "%d", handles); snprintf(e, sizeof(e), "%d", embeds); snprintf(c, sizeof(c), "%d", crd_types); - cbm_log_info("pass.operator", "handles", h, "embeds", e, "crd_types", c); + snprintf(d, sizeof(d), "%d", deploys); + cbm_log_info("pass.operator", "handles", h, "embeds", e, "crd_types", c, "chart_deploys", + d); } return 0; } diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 3ee57159a..4e23dfe2d 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1974,6 +1974,62 @@ TEST(probe_make_programs_by_path_spawn) { PASS(); } +/* Go code that loads a chart DEPLOYS it: a call of a loader (a function + * reading /Chart.yaml, or the Helm SDK's loader.Load) whose argument + * evaluates over the package's constants, filepath.Join and a local assigned + * once. Under an unknown root (opt.ChartsDirectory) the tail names the one + * chart dir ending with it; a tail two dirs end with takes the root the + * other loads under that root showed. A parameter is not a constant of the + * same name. */ +TEST(probe_go_installer_deploys_charts) { + static const GpfFile files[] = { + {"go.mod", "module example.com/inst\n\ngo 1.22\n"}, + {"charts/monitoring/node-exporter/Chart.yaml", + "apiVersion: v2\nname: node-exporter\nversion: 0.1.0\n"}, + {"charts/minio/Chart.yaml", "apiVersion: v2\nname: minio\nversion: 0.1.0\n"}, + {"charts/mla/minio/Chart.yaml", "apiVersion: v2\nname: mla-minio\nversion: 0.1.0\n"}, + {"charts/telemetry/Chart.yaml", "apiVersion: v2\nname: telemetry\nversion: 0.1.0\n"}, + {"pkg/sdk/testdata/examplechart/Chart.yaml", + "apiVersion: v2\nname: examplechart\nversion: 0.1.0\n"}, + {"pkg/helm/types.go", "package helm\n\n" + "import (\n\t\"os\"\n\t\"path/filepath\"\n)\n\n" + "type Chart struct{ Name string }\n\n" + "func LoadChart(directory string) (*Chart, error) {\n" + "\tf, err := os.Open(filepath.Join(directory, \"Chart.yaml\"))\n" + "\tif err != nil {\n\t\treturn nil, err\n\t}\n" + "\tdefer f.Close()\n\treturn &Chart{}, nil\n}\n"}, + {"pkg/stack/stack.go", "package stack\n\n" + "import (\n\t\"path/filepath\"\n\n" + "\t\"example.com/inst/pkg/helm\"\n)\n\n" + "const (\n" + "\tMonitoringPrefix = \"monitoring\"\n" + "\tNodeExporterChartName = \"node-exporter\"\n" + "\tMinioChartName = \"minio\"\n" + "\tchartName = \"telemetry\"\n)\n\n" + "type Options struct{ ChartsDirectory string }\n\n" + "func deployNodeExporter(opt Options) {\n" + "\thelm.LoadChart(filepath.Join(opt.ChartsDirectory, " + "MonitoringPrefix, NodeExporterChartName))\n}\n\n" + "func deployMinio(opt Options) {\n" + "\tname := MinioChartName\n" + "\thelm.LoadChart(filepath.Join(opt.ChartsDirectory, name))\n}\n\n" + "func deployTelemetry(opt Options) {\n" + "\thelm.LoadChart(filepath.Join(opt.ChartsDirectory, " + "\"telemetry\"))\n}\n\n" + "func deployFromArg(opt Options, chartName string) {\n" + "\thelm.LoadChart(filepath.Join(opt.ChartsDirectory, chartName))\n}\n"}, + {"pkg/sdk/load.go", "package sdk\n\n" + "import \"helm.sh/helm/v3/pkg/chart/loader\"\n\n" + "func loadExample() {\n" + "\t_, _ = loader.Load(\"testdata/examplechart\")\n}\n"}}; + GPF_ASSERT_DEPLOYS(files, 9, + "deployMinio -> chart:minio (helm-sdk)\n" + "deployNodeExporter -> chart:node-exporter (helm-sdk)\n" + "deployTelemetry -> chart:telemetry (helm-sdk)\n" + "loadExample -> chart:examplechart (helm-sdk)"); + PASS(); +} + /* A Make recipe runs in its Makefile's directory with the Makefile's * variables expanded ($(shell pwd), ?=, a conditional's: unknown): a * kustomize build into kubectl, kubectl -f a directory, find -name -exec @@ -3689,6 +3745,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_make_recipes_deploy); RUN_TEST(probe_shell_programs_by_path_spawn); RUN_TEST(probe_make_programs_by_path_spawn); + RUN_TEST(probe_go_installer_deploys_charts); RUN_TEST(probe_helm_templated_objects_are_resources); RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); From a197d0c6843306ecf79aa3bcf4ffebb80b186d9c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 05:27:23 +0200 Subject: [PATCH 34/52] feat(terraform): a helm_release DEPLOYS its chart A Terraform module installing a chart with `resource "helm_release"` had no edge to the chart: "which modules install this chart" had no answer. A new Terraform / OpenTofu pass (run at the end of the K8s pass, like the shell pass) reads the helm_release blocks of .tf / .tofu files and links, to existing nodes only, from the block's node (resource.helm_release.): - chart = "./charts/x" or "${path.module}/charts/x": the Chart of that directory of the module; - a repository and a chart name, or chart = "oci://.../": the shared external Chart of that name (the node chart dependencies use). Values are read from string literals and ${path.module} only; a variable chart, or a directory without a Chart.yaml, is no edge. Properties: via terraform, the chart, external, line. Measured on 17 repositories against an independent reading of the Terraform files: 21 of 21 releases linked, 21 of 21 correct (EKS blueprints 17, k8s.io 3, kube-hetzner 1). Only these DEPLOYS edges and 8 external Charts were added; three runs build identical graphs; index time unchanged (cloud-foundation-fabric 1.91 -> 1.89 s, EKS blueprints 0.87 -> 0.88 s, k8s.io 3.44 -> 3.45 s, medians); editing or deleting the files leaves the graph a fresh index builds. Signed-off-by: Martin Vogel --- Makefile.cbm | 1 + scripts/check-lsp-originality.sh | 2 +- src/pipeline/pass_k8s.c | 1 + src/pipeline/pass_terraform.c | 441 +++++++++++++++++++++++++++++++ src/pipeline/pipeline_internal.h | 6 + tests/test_grammar_probe_f.c | 35 +++ 6 files changed, 485 insertions(+), 1 deletion(-) create mode 100644 src/pipeline/pass_terraform.c diff --git a/Makefile.cbm b/Makefile.cbm index 35fa3296a..fa664bd18 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -520,6 +520,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_k8s.c \ src/pipeline/pass_operator.c \ src/pipeline/pass_shell.c \ + src/pipeline/pass_terraform.c \ src/pipeline/pass_similarity.c \ src/pipeline/pass_semantic_edges.c \ src/pipeline/pass_complexity.c \ diff --git a/scripts/check-lsp-originality.sh b/scripts/check-lsp-originality.sh index 22c33e153..3647ff50e 100755 --- a/scripts/check-lsp-originality.sh +++ b/scripts/check-lsp-originality.sh @@ -82,7 +82,7 @@ REFS=( # Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too). INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c - src/pipeline/pass_shell.c) + src/pipeline/pass_shell.c src/pipeline/pass_terraform.c) ONLY_LANG=""; REFRESH=0; LIST_ONLY=0 while [ $# -gt 0 ]; do diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index e4d26bcf3..1e6ba15f8 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -3414,6 +3414,7 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_kusts_free(&kusts); /* Shell and Make deploy commands, against the nodes linked above. */ cbm_pipeline_pass_shell(ctx, files, file_count); + cbm_pipeline_pass_terraform(ctx, files, file_count); cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count)); (void)helm_count; diff --git a/src/pipeline/pass_terraform.c b/src/pipeline/pass_terraform.c new file mode 100644 index 000000000..b0132f7a9 --- /dev/null +++ b/src/pipeline/pass_terraform.c @@ -0,0 +1,441 @@ +/* + * pass_terraform.c — the Terraform / OpenTofu infrastructure layer. + * + * PHASE 3 P3.4: a `resource "helm_release"` DEPLOYS its chart -- an edge from + * the resource block (its node `resource.helm_release.`) to existing + * nodes only: + * chart = "./charts/x" / "${path.module}/charts/x" the Chart of that + * directory, relative to + * the module's directory + * repository = "", chart = "" the shared external + * chart = "oci://registry/.../" Chart of that name + * Values are evaluated from string literals and ${path.module}; any other + * expression is unknown, and an unknown chart is no edge. Run at the end of + * the K8s pass, over the same files. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/hash_table.h" +#include "foundation/limits.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "cbm.h" +#include "lang_specs.h" + +#include +#include +#include +#include + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMHashTable *charts; /* Chart.yaml path -> local Chart node id */ + const char *src; + const char *file; + char mdir[CBM_SZ_1K]; /* the module's directory */ + int edges; +} tf_t; + +static bool tf_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +static const char *tf_basename(const char *p) { + const char *s = strrchr(p, '/'); + return s ? s + SKIP_ONE : p; +} + +/* Append [p, p+n) to out (size sz); false when it does not fit. */ +static bool tf_add(char *out, size_t sz, size_t *len, const char *p, size_t n) { + if (*len + n + SKIP_ONE > sz) { + return false; + } + memcpy(out + *len, p, n); + *len += n; + out[*len] = '\0'; + return true; +} + +/* A string value: a literal, or a template of literals and ${path.module}. + * false when any part is something else. */ +static bool tf_string(const tf_t *t, TSNode n, char *out, size_t sz) { + size_t len = 0; + out[0] = '\0'; + while (tf_type(n, "expression") || tf_type(n, "literal_value") || tf_type(n, "template_expr") || + tf_type(n, "string_lit") || tf_type(n, "quoted_template")) { + if (tf_type(n, "string_lit") || tf_type(n, "quoted_template")) { + break; + } + if (ts_node_named_child_count(n) != SKIP_ONE) { + return false; + } + n = ts_node_named_child(n, 0); + } + if (!tf_type(n, "string_lit") && !tf_type(n, "quoted_template")) { + return false; + } + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + uint32_t b = ts_node_start_byte(c); + uint32_t e = ts_node_end_byte(c); + if (tf_type(c, "quoted_template_start") || tf_type(c, "quoted_template_end")) { + continue; + } + if (tf_type(c, "template_literal")) { + if (!tf_add(out, sz, &len, t->src + b, e - b)) { + return false; + } + continue; + } + if (!tf_type(c, "template_interpolation")) { + return false; + } + /* ${path.module} only */ + TSNode expr = {0}; + for (uint32_t j = 0; j < ts_node_named_child_count(c); j++) { + if (tf_type(ts_node_named_child(c, j), "expression")) { + expr = ts_node_named_child(c, j); + } + } + if (ts_node_is_null(expr)) { + return false; + } + uint32_t xb = ts_node_start_byte(expr); + uint32_t xe = ts_node_end_byte(expr); + static const char PATH_MODULE[] = "path.module"; + if (xe - xb != sizeof(PATH_MODULE) - SKIP_ONE || + strncmp(t->src + xb, PATH_MODULE, xe - xb) != 0) { + return false; + } + /* an absolute marker: the module directory, from the repo root */ + if (!tf_add(out, sz, &len, "\x01", SKIP_ONE) || + !tf_add(out, sz, &len, t->mdir, strlen(t->mdir))) { + return false; + } + } + return true; +} + +/* A path inside the module directory: "./a/../b" -> "/b"; NULL past + * the repository root. A value starting with the \x01 marker is already + * from the root. */ +static bool tf_repo_path(const char *mdir, const char *v, char *out, size_t sz) { + bool anchored = v[0] == '\x01'; + const char *rel = anchored ? v + SKIP_ONE : v; + char buf[CBM_SZ_2K]; + if (anchored) { + snprintf(buf, sizeof(buf), "%s", rel); + } else { + snprintf(buf, sizeof(buf), "%s%s%s", mdir, mdir[0] ? "/" : "", rel); + } + size_t n = 0; + out[0] = '\0'; + for (const char *p = buf; *p;) { + while (*p == '/') { + p++; + } + size_t sl = strcspn(p, "/"); + if (sl == 0) { + break; + } + if (sl == SKIP_ONE && p[0] == '.') { + p += sl; + continue; + } + if (sl == PAIR_LEN && p[0] == '.' && p[SKIP_ONE] == '.') { + if (n == 0) { + return false; + } + while (n > 0 && out[n - SKIP_ONE] != '/') { + n--; + } + if (n > 0) { + n--; + } + out[n] = '\0'; + p += sl; + continue; + } + if (n + sl + PAIR_LEN > sz) { + return false; + } + if (n) { + out[n++] = '/'; + } + memcpy(out + n, p, sl); + n += sl; + out[n] = '\0'; + p += sl; + } + return true; +} + +/* Chart.yaml path -> its (local) Chart node. */ +static CBMHashTable *tf_chart_index(cbm_pipeline_ctx_t *ctx) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!ix || cbm_gbuf_find_by_label(ctx->gbuf, "Chart", &nodes, &count) != 0) { + return ix; + } + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + const char *qn = nodes[i]->qualified_name; + const char *base = fp ? tf_basename(fp) : ""; + if (fp && qn && !strstr(qn, ".__helm_dep__.") && + (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0)) { + cbm_ht_set(ix, fp, (void *)(intptr_t)nodes[i]->id); + } + } + return ix; +} + +/* The Chart node of repository chart directory `dir`, 0 when none. */ +static int64_t tf_local_chart(const tf_t *t, const char *dir) { + static const char *const names[] = {"Chart.yaml", "Chart.yml"}; + for (size_t k = 0; t->charts && k < sizeof(names) / sizeof(names[0]); k++) { + char want[CBM_SZ_2K]; + snprintf(want, sizeof(want), "%s%s%s", dir, dir[0] ? "/" : "", names[k]); + int64_t id = (int64_t)(intptr_t)cbm_ht_get(t->charts, want); + if (id > 0) { + return id; + } + } + return 0; +} + +/* The shared external Chart of `name` (the one chart dependencies use). */ +static int64_t tf_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, const char *file) { + char qn[CBM_SZ_512]; + snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", ctx->project_name, name); + const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(ctx->gbuf, qn); + if (known) { + return known->id; + } + return cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, + "{\"source\":\"helm\",\"external\":true}"); +} + +/* The node of block `resource.helm_release.` in the file. */ +static int64_t tf_block_node(const tf_t *t, const char *name) { + char full[CBM_SZ_512]; + snprintf(full, sizeof(full), "resource.helm_release.%s", name); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(t->ctx->gbuf, full, &nodes, &count) != 0) { + return 0; + } + for (int i = 0; i < count; i++) { + if (nodes[i]->file_path && strcmp(nodes[i]->file_path, t->file) == 0) { + return nodes[i]->id; + } + } + return 0; +} + +static bool tf_plain_name(const char *s) { + if (!s[0]) { + return false; + } + for (const char *p = s; *p; p++) { + if (!((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') || + *p == '-' || *p == '_' || *p == '.')) { + return false; + } + } + return true; +} + +/* One `resource "helm_release" "" { ... }` block. */ +static void tf_helm_release(tf_t *t, TSNode block, const char *name) { + TSNode body = {0}; + for (uint32_t i = 0; i < ts_node_named_child_count(block); i++) { + if (tf_type(ts_node_named_child(block, i), "body")) { + body = ts_node_named_child(block, i); + } + } + if (ts_node_is_null(body)) { + return; + } + char chart[CBM_SZ_1K] = ""; + bool has_chart = false; + bool has_repo = false; /* a repository, even of unknown value: not a local chart */ + for (uint32_t i = 0; i < ts_node_named_child_count(body); i++) { + TSNode a = ts_node_named_child(body, i); + if (!tf_type(a, "attribute") || ts_node_named_child_count(a) < PAIR_LEN) { + continue; + } + TSNode key = ts_node_named_child(a, 0); + TSNode val = ts_node_named_child(a, SKIP_ONE); + uint32_t kb = ts_node_start_byte(key); + uint32_t ke = ts_node_end_byte(key); + size_t kl = ke - kb; + if (kl == strlen("chart") && strncmp(t->src + kb, "chart", kl) == 0) { + has_chart = tf_string(t, val, chart, sizeof(chart)); + } else if (kl == strlen("repository") && strncmp(t->src + kb, "repository", kl) == 0) { + has_repo = true; + } + } + if (!has_chart) { + return; + } + int64_t src = tf_block_node(t, name); + int64_t tgt = 0; + bool external = false; + char path[CBM_SZ_1K] = ""; + static const char OCI[] = "oci://"; + if (strncmp(chart, OCI, sizeof(OCI) - SKIP_ONE) == 0) { + const char *base = tf_basename(chart); + if (tf_plain_name(base)) { + snprintf(path, sizeof(path), "%s", base); + external = true; + } + } else if (has_repo) { + if (tf_plain_name(chart)) { + snprintf(path, sizeof(path), "%s", chart); + external = true; + } + } else if (tf_repo_path(t->mdir, chart, path, sizeof(path))) { + tgt = tf_local_chart(t, path); + } + if (external) { + tgt = tf_external_chart(t->ctx, path, t->file); + } + if (src <= 0 || tgt <= 0) { + return; + } + char esc[CBM_SZ_1K]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), path); + snprintf(props, sizeof(props), "{\"via\":\"terraform\",\"chart\":\"%s\"%s,\"line\":%d}", esc, + external ? ",\"external\":true" : "", (int)ts_node_start_point(block).row + SKIP_ONE); + cbm_gbuf_insert_edge(t->ctx->gbuf, src, tgt, "DEPLOYS", props); + t->edges++; +} + +/* The text of a string_lit label: "helm_release" -> helm_release. */ +static bool tf_label(const tf_t *t, TSNode n, char *out, size_t sz) { + if (!tf_type(n, "string_lit")) { + return false; + } + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "template_literal")) { + uint32_t b = ts_node_start_byte(c); + snprintf(out, sz, "%.*s", (int)(ts_node_end_byte(c) - b), t->src + b); + return true; + } + } + return false; +} + +static void tf_file(tf_t *t, TSNode root) { + TSNode body = {0}; /* comments are children of the file too */ + for (uint32_t i = 0; i < ts_node_named_child_count(root) && ts_node_is_null(body); i++) { + if (tf_type(ts_node_named_child(root, i), "body")) { + body = ts_node_named_child(root, i); + } + } + for (uint32_t i = 0; !ts_node_is_null(body) && i < ts_node_named_child_count(body); i++) { + TSNode b = ts_node_named_child(body, i); + if (!tf_type(b, "block") || ts_node_named_child_count(b) < 3) { + continue; + } + TSNode kw = ts_node_named_child(b, 0); + uint32_t kb = ts_node_start_byte(kw); + uint32_t ke = ts_node_end_byte(kw); + char type[CBM_SZ_128]; + char name[CBM_SZ_256]; + if (ke - kb != strlen("resource") || strncmp(t->src + kb, "resource", ke - kb) != 0 || + !tf_label(t, ts_node_named_child(b, SKIP_ONE), type, sizeof(type)) || + !tf_label(t, ts_node_named_child(b, PAIR_LEN), name, sizeof(name))) { + continue; + } + if (strcmp(type, "helm_release") == 0) { + tf_helm_release(t, b, name); + } + } +} + +static char *tf_read(const char *path, size_t *len) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + enum { TF_TS_PAD = 16 }; /* tree-sitter's lexer reads past the end */ + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + TF_TS_PAD); + if (!buf) { + (void)fclose(f); + return NULL; + } + size_t got = fread(buf, SKIP_ONE, (size_t)size, f); + (void)fclose(f); + memset(buf + got, 0, TF_TS_PAD); + *len = got; + return buf; +} + +static bool tf_is_module_file(const char *rel) { + const char *dot = strrchr(tf_basename(rel), '.'); + return dot && (strcmp(dot, ".tf") == 0 || strcmp(dot, ".tofu") == 0); +} + +void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count) { + TSParser *parser = NULL; + CBMHashTable *charts = NULL; + int edges = 0; + for (int i = 0; i < file_count; i++) { + if (files[i].language != CBM_LANG_HCL || !tf_is_module_file(files[i].rel_path)) { + continue; + } + size_t len = 0; + char *src = tf_read(files[i].path, &len); + if (!src) { + continue; + } + if (strstr(src, "helm_release")) { + if (!parser) { + parser = ts_parser_new(); + charts = tf_chart_index(ctx); + } + TSTree *tree = NULL; + if (parser && ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_HCL))) { + tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + } + if (tree) { + tf_t t = {.ctx = ctx, .charts = charts, .src = src, .file = files[i].rel_path}; + const char *slash = strrchr(files[i].rel_path, '/'); + snprintf(t.mdir, sizeof(t.mdir), "%.*s", + slash ? (int)(slash - files[i].rel_path) : 0, files[i].rel_path); + tf_file(&t, ts_tree_root_node(tree)); + edges += t.edges; + ts_tree_delete(tree); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + if (parser) { + ts_parser_delete(parser); + } + if (charts) { + cbm_ht_free(charts); + } + if (edges > 0) { + char buf[CBM_SZ_32]; + snprintf(buf, sizeof(buf), "%d", edges); + cbm_log_info("pass.terraform.deploys", "edges", buf); + } +} diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 5658e17ea..311f92d2a 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1046,6 +1046,12 @@ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx); * manifests, kustomizations or charts on a cluster DEPLOYS them. Run at the * end of the K8s pass, over the same files. */ void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); + +/* Terraform / OpenTofu modules (pass_terraform.c): a helm_release DEPLOYS its + * chart. Run at the end of the K8s pass, over the same files. */ +void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count); + /* Pre-dump pass: markdown → file REFERENCES_FILE linking. */ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 4e23dfe2d..c6ac29041 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1974,6 +1974,40 @@ TEST(probe_make_programs_by_path_spawn) { PASS(); } +/* A Terraform helm_release DEPLOYS its chart: a chart dir of the module + * (./charts/x, ${path.module}/charts/x), else -- with a repository, or as + * oci://.../ -- the external chart of that name. A variable chart or a + * directory without a Chart.yaml is no edge; a file opening with comments is + * read all the same. */ +TEST(probe_terraform_helm_release_deploys_its_chart) { + static const GpfFile files[] = { + {"infra/charts/app/Chart.yaml", "apiVersion: v2\nname: app\nversion: 0.1.0\n"}, + {"infra/charts/api/Chart.yaml", "apiVersion: v2\nname: api\nversion: 0.1.0\n"}, + {"infra/main.tf", "################\n# Helm releases\n################\n\n" + "resource \"helm_release\" \"app\" {\n" + " name = \"app\"\n chart = \"./charts/app\"\n}\n\n" + "resource \"helm_release\" \"api\" {\n" + " name = \"api\"\n chart = \"${path.module}/charts/api\"\n" + " set {\n name = \"replicas\"\n value = \"2\"\n }\n}\n\n" + "resource \"helm_release\" \"cache\" {\n" + " name = \"cache\"\n" + " repository = \"https://charts.example.com\"\n" + " chart = \"redis\"\n}\n\n" + "resource \"helm_release\" \"queue\" {\n" + " name = \"queue\"\n" + " chart = \"oci://registry.example.com/charts/nats\"\n}\n\n" + "resource \"helm_release\" \"dynamic\" {\n" + " name = \"dynamic\"\n chart = var.chart\n}\n\n" + "resource \"helm_release\" \"missing\" {\n" + " name = \"missing\"\n chart = \"./charts/missing\"\n}\n"}}; + GPF_ASSERT_DEPLOYS(files, 3, + "resource.helm_release.api -> chart:api (terraform)\n" + "resource.helm_release.app -> chart:app (terraform)\n" + "resource.helm_release.cache -> chart:redis(ext) (terraform)\n" + "resource.helm_release.queue -> chart:nats(ext) (terraform)"); + PASS(); +} + /* Go code that loads a chart DEPLOYS it: a call of a loader (a function * reading /Chart.yaml, or the Helm SDK's loader.Load) whose argument * evaluates over the package's constants, filepath.Join and a local assigned @@ -3746,6 +3780,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_shell_programs_by_path_spawn); RUN_TEST(probe_make_programs_by_path_spawn); RUN_TEST(probe_go_installer_deploys_charts); + RUN_TEST(probe_terraform_helm_release_deploys_its_chart); RUN_TEST(probe_helm_templated_objects_are_resources); RUN_TEST(probe_helm_include_resolves_in_the_chart_family); RUN_TEST(probe_k8s_selector_stays_in_its_directory); From b8a5050813dfc1ae178135942a1031230edbca7e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 06:37:17 +0200 Subject: [PATCH 35/52] feat(shell,ci): scripts and CI jobs SPAWNS the scripts they run "Which CI jobs run hack/ci/run-e2e-tests.sh, and what does that script start?" had no answer: a command running a script of the repository drew no edge (SPAWNS reached Process nodes only, and repository scripts were left out), and CI configurations were not read at all. - A shell script, Make recipe or CI command that runs a script of the repository -- by its path (./hack/x.sh, "$ROOT/hack/x.sh") or through a shell (bash -e hack/x.sh) -- SPAWNS that script: its Module (the source of the script's own edges, so CI -> script -> deploy chains connect), else its File. The path must resolve to exactly one file; a program that is no repository file keeps its Process edge. - A new pass_ci.c reads GitHub Actions workflows and composite actions (run: steps with their working directory, shell, env literals, ${{ github.workspace }} and ${{ github.action_path }}; no pwsh step, no step of a Windows job) and Prow job configs (container command + args, exec'd; bash -c TEXT as shell; a decorated job starts in the checkout). The shell layer evaluates them like a script; their DEPLOYS and SPAWNS edges come from the workflow / job file and carry "ci" and "job". - env A="$X/y" PROGRAM: an assignment with an unknown value is no longer taken for the program (it made a Process named after the value's last segment and hid the program). Measured on 17 repositories against an independent reading (libyaml for the CI files, a separate shell reader for the commands): 292 of 292 script runs linked, 292 of 292 correct -- from Prow jobs (115 containers), GitHub Actions steps, shell scripts and Make recipes, across 10 repositories (kubermatic, gardener, kube-hetzner, cluster-api, deckhouse, k8s.io, ...); CI Process and deploy edges 4 of 4. Only SPAWNS (+295), DEPLOYS (+1), Process (+3) and Chart (+1) changed; three runs build identical graphs; index time unchanged (kubermatic 2.95 -> 2.93 s, gardener 5.25 -> 5.25 s, kube-hetzner 1.13 -> 1.14 s, medians); editing or deleting scripts, workflows, Prow jobs and Makefiles leaves the graph a fresh index builds (two scenarios differ in SEMANTICALLY_RELATED edges and Make pattern-rule nodes exactly as the previous build does). Signed-off-by: Martin Vogel --- Makefile.cbm | 1 + README.md | 2 +- scripts/check-lsp-originality.sh | 3 +- src/mcp/test_impact_engine_reach.c | 5 +- src/pipeline/pass_ci.c | 727 +++++++++++++++++++++++++++++ src/pipeline/pass_shell.c | 312 ++++++++++--- src/pipeline/pipeline_internal.h | 33 +- tests/test_grammar_probe_f.c | 144 ++++++ 8 files changed, 1154 insertions(+), 73 deletions(-) create mode 100644 src/pipeline/pass_ci.c diff --git a/Makefile.cbm b/Makefile.cbm index fa664bd18..da481b1fa 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -521,6 +521,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_operator.c \ src/pipeline/pass_shell.c \ src/pipeline/pass_terraform.c \ + src/pipeline/pass_ci.c \ src/pipeline/pass_similarity.c \ src/pipeline/pass_semantic_edges.c \ src/pipeline/pass_complexity.c \ diff --git a/README.md b/README.md index 8b439385a..c98c179e4 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ The install script placed beside the binary is **reported, not deleted** — uni - `USAGE` — an identifier is used, but a unique callable target is not proven (including ambiguous or complex expressions) - `IMPORTS`, `DEFINES`, `IMPLEMENTS`, `INHERITS` - `HTTP_CALLS`, `ASYNC_CALLS` (cross-service) -- `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, ...); the target is a `Process` node named after the program, or `` when the program is not a literal +- `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, a shell command or Make recipe running a program by path, ...); the target is a `Process` node named after the program, or `` when the program is not a literal. When a shell script, Make recipe, GitHub Actions step or Prow job runs a script of the repository (`./hack/x.sh`, `bash hack/x.sh`), the target is that script's `Module` (else its `File`), and edges from CI carry `ci` and `job` - `INFRA_MAPS` — a Kubernetes Service to each workload whose pod labels carry every `key: value` of the Service's selector, within one namespace - `DEPENDS_ON` — a Helm chart to the charts it declares (the local chart for a resolvable `file://` repository, otherwise an external `Chart`; the edge keeps `repository`), and a `go.mod` / `requirements.txt` file to an external `Package` - `EMITS`, `LISTENS_ON` (channels) diff --git a/scripts/check-lsp-originality.sh b/scripts/check-lsp-originality.sh index 3647ff50e..9ddb3944c 100755 --- a/scripts/check-lsp-originality.sh +++ b/scripts/check-lsp-originality.sh @@ -82,7 +82,8 @@ REFS=( # Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too). INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c - src/pipeline/pass_shell.c src/pipeline/pass_terraform.c) + src/pipeline/pass_shell.c src/pipeline/pass_terraform.c + src/pipeline/pass_ci.c) ONLY_LANG=""; REFRESH=0; LIST_ONLY=0 while [ $# -gt 0 ]; do diff --git a/src/mcp/test_impact_engine_reach.c b/src/mcp/test_impact_engine_reach.c index dd581a4f5..49a1142cc 100644 --- a/src/mcp/test_impact_engine_reach.c +++ b/src/mcp/test_impact_engine_reach.c @@ -22,8 +22,9 @@ * reached, never expanded (their caller is the runner's main). */ #define TE_RUNNER_ROLE_PREFIX "tf_maybe_run_" -/* SPAWNS only carries reach from a Process node, which enters the walk - * through te_spawn_lift alone. */ +/* SPAWNS carries reach from a Process node, which enters the walk through + * te_spawn_lift alone, and from a script of the repository (its Module / + * File, pass_shell.c) to whoever runs it. */ static const char *const te_walk_types[] = {"CALLS", "ASYNC_CALLS", "USAGE", "CALL_REFERENCE", "READS", "SPAWNS"}; diff --git a/src/pipeline/pass_ci.c b/src/pipeline/pass_ci.c new file mode 100644 index 000000000..2804e78d1 --- /dev/null +++ b/src/pipeline/pass_ci.c @@ -0,0 +1,727 @@ +/* + * pass_ci.c — the command lines CI configurations run, handed to the shell + * layer (pass_shell.c), which links what they deploy and start. + * + * PHASE 3 P3.5. Read with tree-sitter-yaml: + * - GitHub Actions: a workflow (.github/workflows/.yml|yaml at the + * root -- GitHub reads no other) and a composite action (any + * action.yml|yaml with runs.using: composite). A step's run: is a shell + * text unless a shell other than bash / sh is named (step > job + * defaults.run > workflow defaults.run), or none is and the job's runs-on + * names a Windows runner (whose default is pwsh). It starts in the + * workspace -- the root -- or the working-directory (same precedence); env + * literals (workflow < job < step) are variables, a value holding `${{` an + * unknown one. In the text, `${{ github.workspace }}` is + * $GITHUB_WORKSPACE, `${{ github.action_path }}` $GITHUB_ACTION_PATH, any + * other `${{ }}` an unknown variable (its line breaks kept). + * - Prow: a file with a top-level presubmits / postsubmits / periodics key + * (in-repo job config). A job container's command + args is exec'd, no + * shell: the words are literal, a kubelet `$(VAR)` unknown; `bash -c TEXT` + * runs TEXT as shell. A decorated job starts in the repository's checkout + * (the root); undecorated, with an extra_ref taking the working directory, + * or with a container workingDir, the directory is unknown. + * Scalars: plain, quoted and literal block (|) values; a folded (>) block or + * one with an indentation indicator is skipped. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "pipeline/pipeline_internal.h" +#include "cbm.h" +#include "lang_specs.h" + +#include +#include +#include +#include + +static const char CI_GHA[] = "github-actions"; +static const char CI_PROW[] = "prow"; + +static const char CI_EXPR_OPEN[] = "${{"; +#define CI_EXPR_OPEN_LEN (sizeof(CI_EXPR_OPEN) - SKIP_ONE) +static const char CI_QUOTED_QUOTE[] = "'\\''"; +#define CI_QUOTED_QUOTE_LEN (sizeof(CI_QUOTED_QUOTE) - SKIP_ONE) +/* A rewritten run: text is at most this many times the original: the + * shortest expression `${{}}` (5 bytes) becomes `${__ci_expr}` (12). */ +enum { CI_TEXT_GROWTH = 3 }; + +typedef struct { + CBMArena *a; + const char *src; + const char *rel; + cbm_ci_cmd_fn fn; + void *ud; +} ci_t; + +/* name, value pairs; a value NULL: unknown. */ +typedef struct { + const char **kv; + int n; +} ci_env_t; + +/* What a step inherits from its job and workflow. */ +typedef struct { + ci_env_t env; + const char *wd; /* working-directory; NULL: none named */ + const char *shell; /* NULL: none named */ + bool windows; + const char *action_dir; /* a composite action's directory */ +} ci_scope_t; + +typedef struct { + const char *key; + TSNode value; /* may be null: `key:` without a value */ +} ci_pair_t; + +static bool ci_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +/* The content of a node, through block_node / flow_node wrappers and past + * comments, anchors and tags; null for nothing (an alias stays an alias). */ +static TSNode ci_content(TSNode n) { + while (ci_type(n, "block_node") || ci_type(n, "flow_node")) { + TSNode next = {0}; + uint32_t k = ts_node_named_child_count(n); + for (uint32_t i = 0; i < k; i++) { + TSNode c = ts_node_named_child(n, i); + if (!ci_type(c, "comment") && !ci_type(c, "anchor") && !ci_type(c, "tag")) { + next = c; + break; + } + } + n = next; + } + return n; +} + +/* A flow scalar's value: line breaks folded (a single one to a space, n + * empty lines to n newlines), each line trimmed; `q` the quote (0: plain) + * whose escapes apply. */ +static const char *ci_fold(CBMArena *a, const char *s, size_t n, char q) { + char *out = cbm_arena_alloc(a, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t o = 0; + size_t i = 0; + while (i < n) { + char ch = s[i]; + if (ch == '\n' || ch == '\r') { + while (o > 0 && (out[o - SKIP_ONE] == ' ' || out[o - SKIP_ONE] == '\t')) { + o--; + } + size_t j = i; + int breaks = 0; + while (j < n && (s[j] == ' ' || s[j] == '\t' || s[j] == '\r' || s[j] == '\n')) { + breaks += s[j] == '\n'; + j++; + } + if (breaks > SKIP_ONE) { + for (int k = SKIP_ONE; k < breaks; k++) { + out[o++] = '\n'; + } + } else { + out[o++] = ' '; + } + i = j; + continue; + } + if (q == '"' && ch == '\\' && i + SKIP_ONE < n) { + char x = s[i + SKIP_ONE]; + i += PAIR_LEN; + if (x == '\n' || x == '\r') { + while (i < n && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) { + i++; + } + continue; + } + out[o++] = x == 'n' ? '\n' : x == 't' ? '\t' : x; + continue; + } + if (q == '\'' && ch == '\'' && i + SKIP_ONE < n && s[i + SKIP_ONE] == '\'') { + out[o++] = '\''; + i += PAIR_LEN; + continue; + } + out[o++] = ch; + i++; + } + out[o] = '\0'; + return out; +} + +/* A literal block scalar's value (s at its `|`); NULL for a folded block or + * an indentation indicator. */ +static const char *ci_literal(CBMArena *a, const char *s, size_t n) { + if (n == 0 || s[0] != '|') { + return NULL; + } + size_t i = SKIP_ONE; + while (i < n && s[i] != '\n' && s[i] != '#') { + if (s[i] >= '1' && s[i] <= '9') { + return NULL; + } + i++; + } + while (i < n && s[i] != '\n') { + i++; + } + i += i < n ? SKIP_ONE : 0; + size_t indent = 0; + for (size_t j = i; j < n;) { + size_t k = j; + while (k < n && s[k] == ' ') { + k++; + } + if (k < n && s[k] != '\n' && s[k] != '\r') { + indent = k - j; + break; + } + while (k < n && s[k] != '\n') { + k++; + } + j = k + SKIP_ONE; + } + char *out = cbm_arena_alloc(a, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t o = 0; + while (i < n) { + size_t e = i; + while (e < n && s[e] != '\n') { + e++; + } + size_t k = i; + while (k < e && k - i < indent && s[k] == ' ') { + k++; + } + if (k - i == indent) { + memcpy(out + o, s + k, e - k); + o += e - k; + } + if (o > 0 && out[o - SKIP_ONE] == '\r') { + o--; + } + if (e < n) { + out[o++] = '\n'; + } + i = e + SKIP_ONE; + } + out[o] = '\0'; + return out; +} + +/* A scalar's value and the 0-based row of its first character; NULL: no + * scalar this reader decodes. */ +static const char *ci_scalar(const ci_t *c, TSNode n, int *row) { + n = ci_content(n); + if (ts_node_is_null(n)) { + return NULL; + } + uint32_t b = ts_node_start_byte(n); + size_t len = ts_node_end_byte(n) - b; + const char *s = c->src + b; + if (row) { + *row = (int)ts_node_start_point(n).row; + } + if (ci_type(n, "plain_scalar")) { + return ci_fold(c->a, s, len, 0); + } + if (len >= PAIR_LEN && ci_type(n, "double_quote_scalar")) { + return ci_fold(c->a, s + SKIP_ONE, len - PAIR_LEN, '"'); + } + if (len >= PAIR_LEN && ci_type(n, "single_quote_scalar")) { + return ci_fold(c->a, s + SKIP_ONE, len - PAIR_LEN, '\''); + } + if (ci_type(n, "block_scalar")) { + if (row) { + (*row)++; + } + return ci_literal(c->a, s, len); + } + return NULL; +} + +/* The pairs of mapping `m` with a scalar key; *n their count. */ +static ci_pair_t *ci_pairs(const ci_t *c, TSNode m, int *n) { + *n = 0; + m = ci_content(m); + if (!ci_type(m, "block_mapping") && !ci_type(m, "flow_mapping")) { + return NULL; + } + uint32_t k = ts_node_named_child_count(m); + ci_pair_t *out = cbm_arena_alloc(c->a, (k + SKIP_ONE) * sizeof(*out)); + if (!out) { + return NULL; + } + for (uint32_t i = 0; i < k; i++) { + TSNode p = ts_node_named_child(m, i); + if (!ci_type(p, "block_mapping_pair") && !ci_type(p, "flow_pair")) { + continue; + } + const char *key = ci_scalar(c, ts_node_child_by_field_name(p, TS_FIELD("key")), NULL); + if (key) { + out[(*n)++] = (ci_pair_t){key, ts_node_child_by_field_name(p, TS_FIELD("value"))}; + } + } + return out; +} + +/* The value of `key` in mapping `m` (the first such pair); null: none. */ +static TSNode ci_get(const ci_t *c, TSNode m, const char *key) { + int n = 0; + ci_pair_t *pairs = ci_pairs(c, m, &n); + for (int i = 0; i < n; i++) { + if (strcmp(pairs[i].key, key) == 0) { + return pairs[i].value; + } + } + TSNode none = {0}; + return none; +} + +static const char *ci_get_str(const ci_t *c, TSNode m, const char *key) { + return ci_scalar(c, ci_get(c, m, key), NULL); +} + +/* The item contents of a sequence; *n their count. */ +static TSNode *ci_items(const ci_t *c, TSNode seq, int *n) { + *n = 0; + seq = ci_content(seq); + bool block = ci_type(seq, "block_sequence"); + if (!block && !ci_type(seq, "flow_sequence")) { + return NULL; + } + uint32_t k = ts_node_named_child_count(seq); + TSNode *out = cbm_arena_alloc(c->a, (k + SKIP_ONE) * sizeof(*out)); + if (!out) { + return NULL; + } + for (uint32_t i = 0; i < k; i++) { + TSNode it = ts_node_named_child(seq, i); + if (block && ci_type(it, "block_sequence_item")) { + TSNode v = {0}; + for (uint32_t j = 0; j < ts_node_named_child_count(it); j++) { + TSNode x = ts_node_named_child(it, j); + if (!ci_type(x, "comment")) { + v = x; + break; + } + } + out[(*n)++] = ci_content(v); + } else if (!block && ci_type(it, "flow_node")) { + out[(*n)++] = ci_content(it); + } + } + return out; +} + +/* `base` and then the pairs of env mapping `m`: a non-scalar or `${{` value + * is unknown, an absent one empty. */ +static ci_env_t ci_env_add(const ci_t *c, const ci_env_t *base, TSNode m) { + int n = 0; + ci_pair_t *pairs = ci_pairs(c, m, &n); + int bn = base ? base->n : 0; + ci_env_t out = {cbm_arena_alloc(c->a, (size_t)(bn + n + SKIP_ONE) * PAIR_LEN * sizeof(char *)), + 0}; + if (!out.kv) { + return base ? *base : out; + } + for (int i = 0; i < bn; i++) { + out.kv[PAIR_LEN * out.n] = base->kv[PAIR_LEN * i]; + out.kv[PAIR_LEN * out.n + SKIP_ONE] = base->kv[PAIR_LEN * i + SKIP_ONE]; + out.n++; + } + for (int i = 0; i < n; i++) { + const char *v = ts_node_is_null(pairs[i].value) ? "" : ci_scalar(c, pairs[i].value, NULL); + out.kv[PAIR_LEN * out.n] = pairs[i].key; + out.kv[PAIR_LEN * out.n + SKIP_ONE] = v && !strstr(v, "${{") ? v : NULL; + out.n++; + } + return out; +} + +/* ── GitHub Actions ────────────────────────────────────────────────── */ + +/* The n bytes at b are exactly `word`. */ +static bool ci_span_is(const char *b, size_t n, const char *word) { + return strlen(word) == n && strncmp(b, word, n) == 0; +} + +/* The run: text with its `${{ }}` expressions replaced (see the header), + * CBM_SZ_16 zero bytes past its end. */ +static const char *ci_gha_text(CBMArena *a, const char *t, size_t *len) { + size_t n = strlen(t); + char *out = cbm_arena_alloc(a, n * CI_TEXT_GROWTH + CBM_SZ_16 + CBM_SZ_16); + if (!out) { + return NULL; + } + size_t o = 0; + size_t i = 0; + while (i < n) { + const char *open = t + i + CI_EXPR_OPEN_LEN; + const char *end = + strncmp(t + i, CI_EXPR_OPEN, CI_EXPR_OPEN_LEN) == 0 ? strstr(open, "}}") : NULL; + if (!end) { + out[o++] = t[i++]; + continue; + } + const char *b = open; + const char *e = end; + while (b < e && (*b == ' ' || *b == '\t')) { + b++; + } + while (e > b && (e[-SKIP_ONE] == ' ' || e[-SKIP_ONE] == '\t')) { + e--; + } + size_t il = (size_t)(e - b); + const char *rep = "${__ci_expr}"; + if (ci_span_is(b, il, "github.workspace")) { + rep = "${GITHUB_WORKSPACE}"; + } else if (ci_span_is(b, il, "github.action_path")) { + rep = "${GITHUB_ACTION_PATH}"; + } + size_t rl = strlen(rep); + memcpy(out + o, rep, rl); + o += rl; + for (const char *p = open; p < end; p++) { + if (*p == '\n') { + out[o++] = '\n'; + } + } + i = (size_t)(end - t) + PAIR_LEN; + } + memset(out + o, 0, CBM_SZ_16); + *len = o; + return out; +} + +/* A shell GitHub runs a step with as bash: `bash ...` / `sh ...`. */ +static bool ci_bash_shell(const char *sh) { + size_t k = strncmp(sh, "bash", strlen("bash")) == 0 ? strlen("bash") + : strncmp(sh, "sh", strlen("sh")) == 0 ? strlen("sh") + : 0; + if (!k) { + return false; + } + char ch = sh[k]; + return !(ch == '_' || (ch >= '0' && ch <= '9') || ((ch | ' ') >= 'a' && (ch | ' ') <= 'z')); +} + +/* `s` contains `word` (lowercase ASCII), ignoring case. */ +static bool ci_has_word(const char *s, const char *word) { + size_t wl = strlen(word); + for (const char *p = s; p && *p; p++) { + size_t k = 0; + while (k < wl && p[k] && (char)(p[k] | ' ') == word[k]) { + k++; + } + if (k == wl) { + return true; + } + } + return false; +} + +/* runs-on names a Windows runner: a label, or any label of a list. */ +static bool ci_windows(const ci_t *c, TSNode runs_on) { + int n = 0; + TSNode *items = ci_items(c, runs_on, &n); + if (!items) { + return ci_has_word(ci_scalar(c, runs_on, NULL), "windows"); + } + for (int i = 0; i < n; i++) { + if (ci_has_word(ci_scalar(c, items[i], NULL), "windows")) { + return true; + } + } + return false; +} + +static const char *ci_run_default(const ci_t *c, TSNode m, const char *key) { + return ci_get_str(c, ci_get(c, ci_get(c, m, "defaults"), "run"), key); +} + +static void ci_gha_steps(const ci_t *c, TSNode steps, const char *job, const ci_scope_t *sc) { + int n = 0; + TSNode *items = ci_items(c, steps, &n); + for (int i = 0; i < n; i++) { + int row = 0; + const char *run = ci_scalar(c, ci_get(c, items[i], "run"), &row); + if (!run) { + continue; + } + const char *shell = ci_get_str(c, items[i], "shell"); + shell = shell ? shell : sc->shell; + if (shell && shell[0] ? !ci_bash_shell(shell) : sc->windows) { + continue; + } + const char *wd = ci_get_str(c, items[i], "working-directory"); + wd = wd ? wd : sc->wd; + if (!wd) { + wd = ""; + } else if (strstr(wd, "${{") || wd[0] == '/' || wd[0] == '~') { + wd = NULL; + } + ci_env_t env = ci_env_add(c, &sc->env, ci_get(c, items[i], "env")); + size_t len = 0; + const char *text = ci_gha_text(c->a, run, &len); + if (!text) { + continue; + } + cbm_ci_cmd_t cmd = {.ci = CI_GHA, + .job = job, + .text = text, + .len = len, + .row = row, + .wd = wd, + .action_dir = sc->action_dir, + .env = env.kv, + .nenv = env.n, + .exec = false}; + c->fn(&cmd, c->ud); + } +} + +static void ci_workflow(const ci_t *c, TSNode root) { + ci_env_t wenv = ci_env_add(c, NULL, ci_get(c, root, "env")); + const char *wwd = ci_run_default(c, root, "working-directory"); + const char *wsh = ci_run_default(c, root, "shell"); + int n = 0; + ci_pair_t *jobs = ci_pairs(c, ci_get(c, root, "jobs"), &n); + for (int i = 0; i < n; i++) { + TSNode j = jobs[i].value; + const char *wd = ci_run_default(c, j, "working-directory"); + const char *sh = ci_run_default(c, j, "shell"); + ci_scope_t sc = {.env = ci_env_add(c, &wenv, ci_get(c, j, "env")), + .wd = wd ? wd : wwd, + .shell = sh ? sh : wsh, + .windows = ci_windows(c, ci_get(c, j, "runs-on")), + .action_dir = NULL}; + ci_gha_steps(c, ci_get(c, j, "steps"), jobs[i].key, &sc); + } +} + +static void ci_action(const ci_t *c, TSNode root) { + TSNode runs = ci_get(c, root, "runs"); + const char *using = ci_get_str(c, runs, "using"); + if (!using || strcmp(using, "composite") != 0) { + return; + } + const char *slash = strrchr(c->rel, '/'); + ci_scope_t sc = {.action_dir = + slash ? cbm_arena_strndup(c->a, c->rel, (size_t)(slash - c->rel)) : ""}; + ci_gha_steps(c, ci_get(c, runs, "steps"), NULL, &sc); +} + +/* ── Prow ──────────────────────────────────────────────────────────── */ + +static bool ci_is_shell(const char *w) { + static const char *const shells[] = {"bash", "sh", "zsh", "ksh", "dash", NULL}; + const char *slash = strrchr(w, '/'); + const char *base = slash ? slash + SKIP_ONE : w; + for (int i = 0; shells[i]; i++) { + if (strcmp(base, shells[i]) == 0) { + return true; + } + } + return false; +} + +/* argv as one shell line of quoted words (a `$(VAR)` word unknown), with + * CBM_SZ_16 zero bytes past its end. */ +static const char *ci_exec_line(CBMArena *a, const char **w, int n, size_t *len) { + size_t cap = CBM_SZ_16 + CBM_SZ_16; + for (int i = 0; i < n; i++) { + cap += strlen(w[i]) * CI_QUOTED_QUOTE_LEN + CBM_SZ_16; + } + char *out = cbm_arena_alloc(a, cap); + if (!out) { + return NULL; + } + size_t o = 0; + for (int i = 0; i < n; i++) { + if (i) { + out[o++] = ' '; + } + if (strstr(w[i], "$(")) { + static const char unknown[] = "\"$__k8s_var\""; + memcpy(out + o, unknown, sizeof(unknown) - SKIP_ONE); + o += sizeof(unknown) - SKIP_ONE; + continue; + } + out[o++] = '\''; + for (const char *p = w[i]; *p; p++) { + if (*p == '\'') { + memcpy(out + o, CI_QUOTED_QUOTE, CI_QUOTED_QUOTE_LEN); + o += CI_QUOTED_QUOTE_LEN; + } else { + out[o++] = *p; + } + } + out[o++] = '\''; + } + memset(out + o, 0, CBM_SZ_16); + *len = o; + return out; +} + +static void ci_prow_container(const ci_t *c, TSNode ct, const char *job, bool root_wd) { + int nc = 0; + int na = 0; + TSNode *cmd = ci_items(c, ci_get(c, ct, "command"), &nc); + TSNode *args = ci_items(c, ci_get(c, ct, "args"), &na); + int n = nc + na; + if (n == 0) { + return; + } + const char **w = cbm_arena_alloc(c->a, (size_t)n * sizeof(*w)); + int *rows = cbm_arena_alloc(c->a, (size_t)n * sizeof(*rows)); + if (!w || !rows) { + return; + } + for (int i = 0; i < n; i++) { + w[i] = ci_scalar(c, i < nc ? cmd[i] : args[i - nc], &rows[i]); + if (!w[i]) { + return; + } + } + const char *wdir = ci_get_str(c, ct, "workingDir"); + cbm_ci_cmd_t out = {.ci = CI_PROW, .job = job, .wd = root_wd && !(wdir && wdir[0]) ? "" : NULL}; + if (ci_is_shell(w[0])) { + int k = SKIP_ONE; + while (k < n && strcmp(w[k], "-c") != 0) { + k++; + } + if (k < n) { + if (k + SKIP_ONE >= n) { + return; + } + size_t tl = strlen(w[k + SKIP_ONE]); + char *text = cbm_arena_alloc(c->a, tl + CBM_SZ_16); + if (!text) { + return; + } + memcpy(text, w[k + SKIP_ONE], tl); + memset(text + tl, 0, CBM_SZ_16); + out.text = text; + out.len = tl; + out.row = rows[k + SKIP_ONE]; + c->fn(&out, c->ud); + return; + } + } + out.text = ci_exec_line(c->a, w, n, &out.len); + out.row = rows[0]; + out.exec = true; + if (out.text) { + c->fn(&out, c->ud); + } +} + +static void ci_prow_job(const ci_t *c, TSNode j) { + const char *name = ci_get_str(c, j, "name"); + const char *decorate = ci_get_str(c, j, "decorate"); + bool root_wd = decorate && strcmp(decorate, "true") == 0; + int n = 0; + TSNode *refs = ci_items(c, ci_get(c, j, "extra_refs"), &n); + for (int i = 0; i < n; i++) { + const char *wd = ci_get_str(c, refs[i], "workdir"); + root_wd = root_wd && !(wd && strcmp(wd, "true") == 0); + } + TSNode *cts = ci_items(c, ci_get(c, ci_get(c, j, "spec"), "containers"), &n); + for (int i = 0; i < n; i++) { + ci_prow_container(c, cts[i], name, root_wd); + } +} + +static void ci_prow_jobs(const ci_t *c, TSNode seq) { + int n = 0; + TSNode *jobs = ci_items(c, seq, &n); + for (int i = 0; i < n; i++) { + ci_prow_job(c, jobs[i]); + } +} + +static void ci_prow(const ci_t *c, TSNode root) { + static const char *const keys[] = {"presubmits", "postsubmits", "periodics", NULL}; + for (int k = 0; keys[k]; k++) { + TSNode v = ci_content(ci_get(c, root, keys[k])); + int n = 0; + ci_pair_t *repos = ci_pairs(c, v, &n); + if (!repos) { + ci_prow_jobs(c, v); + continue; + } + for (int i = 0; i < n; i++) { + ci_prow_jobs(c, repos[i].value); + } + } +} + +/* ── Entry ─────────────────────────────────────────────────────────── */ + +static bool ci_yaml_ext(const char *p) { + const char *dot = strrchr(p, '.'); + return dot && (strcmp(dot, ".yml") == 0 || strcmp(dot, ".yaml") == 0); +} + +static bool ci_workflow_path(const char *rel) { + static const char dir[] = ".github/workflows/"; + return strncmp(rel, dir, sizeof(dir) - SKIP_ONE) == 0 && + !strchr(rel + sizeof(dir) - SKIP_ONE, '/') && ci_yaml_ext(rel); +} + +static bool ci_action_path(const char *rel) { + const char *slash = strrchr(rel, '/'); + const char *base = slash ? slash + SKIP_ONE : rel; + return strcmp(base, "action.yml") == 0 || strcmp(base, "action.yaml") == 0; +} + +bool cbm_ci_path(const char *rel) { + return rel && (ci_workflow_path(rel) || ci_action_path(rel)); +} + +void cbm_ci_commands(CBMArena *a, const char *rel, const char *src, size_t len, cbm_ci_cmd_fn fn, + void *ud) { + if (!ci_yaml_ext(rel)) { + return; + } + TSParser *parser = ts_parser_new(); + if (!parser) { + return; + } + TSTree *tree = NULL; + if (ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_YAML))) { + tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + } + if (tree) { + ci_t c = {.a = a, .src = src, .rel = rel, .fn = fn, .ud = ud}; + TSNode stream = ts_tree_root_node(tree); + for (uint32_t i = 0; i < ts_node_named_child_count(stream); i++) { + TSNode doc = ts_node_named_child(stream, i); + if (!ci_type(doc, "document")) { + continue; + } + TSNode root = {0}; + for (uint32_t k = 0; k < ts_node_named_child_count(doc); k++) { + TSNode x = ts_node_named_child(doc, k); + if (ci_type(x, "block_node") || ci_type(x, "flow_node")) { + root = ci_content(x); + break; + } + } + if (ci_workflow_path(rel)) { + ci_workflow(&c, root); + } else if (ci_action_path(rel)) { + ci_action(&c, root); + } else { + ci_prow(&c, root); + } + } + ts_tree_delete(tree); + } + ts_parser_delete(parser); +} diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index 02fc6fe2e..fca3b49fb 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -1,5 +1,5 @@ /* - * pass_shell.c — the Bash (+ Make) infrastructure layer (PHASE 3 P3.2). + * pass_shell.c — the Bash (+ Make, CI) infrastructure layer (PHASE 3 P3.2, P3.5). * * A shell script or a Make recipe that puts manifests, a kustomization or a * chart on a cluster DEPLOYS them: an edge from the function running the @@ -18,8 +18,14 @@ * * A program started by a path (./_build/kubermatic-installer, ${DIR}/x, * $(LOCALBIN)/controller-gen) SPAWNS the Process of its name, as - * pass_spawns.c draws a call that starts a program; a path naming a file of - * the repository is a script of its own and not linked here. + * pass_spawns.c draws a call that starts a program. A program that is a file + * of the repository -- the path names one, or a shell runs one + * (bash|sh|zsh|ksh|dash [-opts] FILE) -- SPAWNS that script instead: its + * Module (the source of its own top-level edges), else its File (P3.5). + * + * CI (P3.5): the command lines GitHub Actions steps and Prow job containers + * run (pass_ci.c reads them) evaluate the same way, from the step's + * directory with its env literals; their edges carry the CI and the job. * * Words are evaluated by constant propagation over the script, nothing is * executed: assignments (local / export / readonly / declare, arrays and @@ -365,9 +371,11 @@ typedef struct { const char *via; /* static */ int64_t tgt; /* the target node; 0: the external chart `ext` */ const char *ext; /* pass-arena */ - const char *path; /* pass-arena: the deployed path or chart */ + const char *path; /* pass-arena: the deployed path or chart; a spawned script */ + const char *ci; /* static: the CI running the command; NULL: none */ + const char *job; /* pass-arena: its job; NULL: none */ bool chart; - bool spawn; /* SPAWNS the Process `ext`, not DEPLOYS */ + bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ } sh_hit_t; typedef struct { @@ -438,7 +446,10 @@ typedef struct { const char *file; /* the script / Makefile path */ const char *script; /* $0: anchored script path; NULL: none (Make) */ const char *sdir; /* the script's directory */ - int line_base; /* Make: the recipe line's row */ + int line_base; /* Make / CI: the text's first row */ + const char *ci; /* the CI running the text (static); NULL: none */ + const char *job; /* its job; NULL: none */ + bool exec; /* an argv exec'd without a shell (Prow) */ int in_func; CBMHashTable *env; /* name -> const sh_val_t * */ sh_undo_t *undo; @@ -1292,21 +1303,30 @@ static const char *sh_cd_target(sh_t *s, const char *v, sh_cwd_t *out) { /* ── Deploy commands ───────────────────────────────────────────────── */ -static void sh_hit(sh_t *s, TSNode cmd, const char *via, const sh_target_t *t, const char *ext, - bool chart) { +/* Record hit `x` of command `cmd`: its file, line and CI context are the + * state's; ext / path are copied. */ +static void sh_hit_push(sh_t *s, TSNode cmd, sh_hit_t x) { sh_hits_t *h = s->hits; if (!sh_grow((void **)&h->items, &h->cap, h->count, sizeof(*h->items))) { return; } - const char *path = t ? t->path : ext; - h->items[h->count++] = - (sh_hit_t){.file = s->file, - .line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base, - .via = via, - .tgt = t ? t->id : 0, - .ext = ext ? cbm_arena_strdup(&h->arena, ext) : NULL, - .path = path ? cbm_arena_strdup(&h->arena, path) : NULL, - .chart = chart}; + x.file = s->file; + x.line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base; + x.ext = x.ext ? cbm_arena_strdup(&h->arena, x.ext) : NULL; + x.path = x.path ? cbm_arena_strdup(&h->arena, x.path) : NULL; + x.ci = s->ci; + x.job = s->job ? cbm_arena_strdup(&h->arena, s->job) : NULL; + h->items[h->count++] = x; +} + +static void sh_hit(sh_t *s, TSNode cmd, const char *via, const sh_target_t *t, const char *ext, + bool chart) { + sh_hit_push(s, cmd, + (sh_hit_t){.via = via, + .tgt = t ? t->id : 0, + .ext = ext, + .path = t ? t->path : ext, + .chart = chart}); } static void sh_hit_all(sh_t *s, TSNode cmd, const char *via, const sh_targets_t *ts, bool chart) { @@ -1336,10 +1356,23 @@ static bool sh_numberish(const char *v) { return true; } +/* Word i of argv is an option, assignment or count a wrapper takes before + * the program: by its value, or -- the value unknown -- by its text + * (env OUT="$TMP/x" prog: the assignment is no program). */ +static bool sh_wrapper_arg(const sh_t *s, const sh_argv_t *a, int i) { + const char *v = a->v[i]; + if (v) { + return v[0] == '-' || strchr(v, '=') || sh_numberish(v); + } + uint32_t b = a->span[PAIR_LEN * i]; + uint32_t e = a->span[PAIR_LEN * i + SKIP_ONE]; + return e > b && (s->src[b] == '-' || memchr(s->src + b, '=', e - b)); +} + /* The index of the program argv runs, past wrappers (time, sudo, retry N, * timeout D, env A=B, ...); -1 when argv has none. Its value may be unknown * (NULL). */ -static int sh_program(const sh_argv_t *a) { +static int sh_program(const sh_t *s, const sh_argv_t *a) { static const char *const wrappers[] = {"time", "sudo", "exec", "nice", "nohup", "command", "env", "timeout", "retry", NULL}; int i = 0; @@ -1352,8 +1385,7 @@ static int sh_program(const sh_argv_t *a) { return -1; /* command -v X: is there an X, not a run of it */ } i++; - while (i < a->n && a->v[i] && - (a->v[i][0] == '-' || strchr(a->v[i], '=') || sh_numberish(a->v[i]))) { + while (i < a->n && sh_wrapper_arg(s, a, i)) { i++; } } @@ -1679,7 +1711,7 @@ static void sh_find(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog) { sh_argv_push(&e, a->v[i], a->split[i], a->span[PAIR_LEN * i], a->span[PAIR_LEN * i + SKIP_ONE]); } - int p = sh_program(&e); + int p = sh_program(s, &e); if (p >= 0 && e.v[p] && strcmp(sh_basename(e.v[p]), "kubectl") == 0) { sh_kubectl_t k = {0}; sh_kubectl_parse(&e, p + SKIP_ONE, &k); @@ -1701,7 +1733,7 @@ static void sh_find(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog) { static void sh_deploy_command(sh_t *s, TSNode cmd, const sh_argv_t *a, const sh_src_t *in, sh_src_t *out) { *out = (sh_src_t){SH_SRC_NONE, NULL, NULL}; - int prog = sh_program(a); + int prog = sh_program(s, a); if (prog < 0 || !a->v[prog]) { return; } @@ -1728,39 +1760,74 @@ static void sh_deploy_command(sh_t *s, TSNode cmd, const sh_argv_t *a, const sh_ } } -/* A repository file: the path names a File node. */ -static int sh_t_repo_file(sh_t *s, const char *path, const sh_find_t *f, sh_targets_t *out) { - (void)f; - int64_t id = (int64_t)(intptr_t)cbm_ht_get(s->ix->file_ids, path); - if (id <= 0) { - return 0; +/* The File node a repository path names; 0: none. */ +static int64_t sh_repo_file_id(const sh_t *s, const char *path) { + return (int64_t)(intptr_t)cbm_ht_get(s->ix->file_ids, path); +} + +static const char *const SH_SHELLS[] = {"bash", "sh", "zsh", "ksh", "dash", NULL}; + +/* The program is a script of the repository -- its path names a file of it, + * or a shell runs one (bash [-opts] FILE; -c TEXT is no file) -- resolved to + * exactly one file: that script is spawned. */ +static void sh_spawn_script(sh_t *s, TSNode cmd, const sh_argv_t *a, int p) { + const char *v = a->v[p]; + const char *path = NULL; + if (sh_in(sh_basename(v), SH_SHELLS)) { + int j = p + SKIP_ONE; + while (j < a->n && a->v[j] && a->v[j][0] == '-' && !sh_eq(a->v[j], "-c")) { + j++; + } + path = j < a->n && !sh_eq(a->v[j], "-c") ? a->v[j] : NULL; + } else if (strchr(v, '/') && !strpbrk(v, "*?[")) { + path = v; + } + if (!path) { + return; + } + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, path, c); + const char *hit = NULL; + int64_t id = 0; + int found = 0; + for (int i = 0; i < n; i++) { + int64_t f = sh_repo_file_id(s, c[i]); + if (f > 0) { + found++; + id = f; + hit = c[i]; + } + } + if (found == SKIP_ONE) { + sh_hit_push(s, cmd, + (sh_hit_t){.via = s->exec ? "exec" : "shell", + .tgt = id, + .ext = sh_basename(hit), + .path = hit, + .spawn = true}); } - sh_targets_push(out, id, path); - return SKIP_ONE; } /* A program started by a path (./_build/x, ${DIR}/x, $(LOCALBIN)/x) SPAWNS * the Process of its name -- the value's last segment, or, the value * unknown, the word's literal last segment. A path naming a file of the - * repository is a script of its own, not linked here. */ + * repository is a script (sh_spawn_script), not a Process. */ static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) { - int p = sh_program(a); + int p = sh_program(s, a); if (p < 0) { return; } const char *v = a->v[p]; const char *name = NULL; if (v) { + sh_spawn_script(s, cmd, a, p); if (!strchr(v, '/') || strpbrk(v, "*?[")) { return; } const char *c[PAIR_LEN] = {NULL, NULL}; int n = sh_candidates(s, v, c); for (int i = 0; i < n; i++) { - sh_targets_t ts = {0}; - int hit = sh_t_repo_file(s, c[i], NULL, &ts); - cbm_free(CBM_MEM_CLASS_OTHER, ts.items); - if (hit) { + if (sh_repo_file_id(s, c[i]) > 0) { return; } } @@ -1786,15 +1853,7 @@ static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) { if (!name || !name[0] || strcmp(name, ".") == 0 || strcmp(name, "..") == 0) { return; } - sh_hits_t *h = s->hits; - if (sh_grow((void **)&h->items, &h->cap, h->count, sizeof(*h->items))) { - h->items[h->count++] = - (sh_hit_t){.file = s->file, - .line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base, - .via = "shell", - .ext = cbm_arena_strdup(&h->arena, name), - .spawn = true}; - } + sh_hit_push(s, cmd, (sh_hit_t){.via = s->exec ? "exec" : "shell", .ext = name, .spawn = true}); } /* ── Statements ────────────────────────────────────────────────────── */ @@ -2633,8 +2692,9 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l uint32_t e = m.lines[r].e; const char *raw = cbm_arena_strndup(&a, src + b, e - b); const char *text = raw ? mk_expand(&m, raw) : NULL; - if (!text || (!strchr(text, '/') && !sh_mentions_deploy_tool(text, strlen(text)))) { - continue; /* neither a deploy nor a program started by a path */ + if (!text || (!strchr(text, '/') && !strstr(text, "sh") && + !sh_mentions_deploy_tool(text, strlen(text)))) { + continue; /* neither a deploy, a program started by a path, nor a shell */ } size_t tl = strlen(text); char *buf = cbm_arena_alloc(&a, tl + CBM_SZ_16); @@ -2734,23 +2794,53 @@ static int64_t sh_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, cons "{\"source\":\"helm\",\"external\":true}"); } -/* SPAWNS to the Process of `program`, as pass_spawns.c draws it. */ -static int sh_link_spawn(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x) { +/* A hit's CI context as JSON members (`,"ci":..,"job":..`); "" for none. */ +static void sh_ci_props(const sh_hit_t *x, char *out, size_t sz) { + out[0] = '\0'; + if (!x->ci) { + return; + } + char job[CBM_SZ_512]; + cbm_json_escape(job, (int)sizeof(job), x->job ? x->job : ""); + if (x->job) { + snprintf(out, sz, ",\"ci\":\"%s\",\"job\":\"%s\"", x->ci, job); + } else { + snprintf(out, sz, ",\"ci\":\"%s\"", x->ci); + } +} + +/* SPAWNS to the script `x->tgt` (its Module, else its File) or to the + * Process of `x->ext`, as pass_spawns.c draws it. */ +static int sh_link_spawn(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x, + CBMHashTable *modules) { char program[CBM_SZ_128]; cbm_json_escape(program, (int)sizeof(program), x->ext); - char qn[CBM_SZ_256]; - snprintf(qn, sizeof(qn), "__process__%s", x->ext); - char node_props[CBM_SZ_256]; - snprintf(node_props, sizeof(node_props), "{\"program\":\"%s\"}", program); - int64_t process = cbm_gbuf_upsert_node(ctx->gbuf, "Process", x->ext, qn, "", 0, 0, node_props); - if (process <= 0) { + char path[CBM_SZ_1K + CBM_SZ_16]; + path[0] = '\0'; + int64_t tgt = 0; + if (x->tgt > 0) { + int64_t mod = (int64_t)(intptr_t)cbm_ht_get(modules, x->path); + tgt = mod > 0 ? mod : x->tgt; + char esc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), x->path); + snprintf(path, sizeof(path), ",\"path\":\"%s\"", esc); + } else { + char qn[CBM_SZ_256]; + snprintf(qn, sizeof(qn), "__process__%s", x->ext); + char node_props[CBM_SZ_256]; + snprintf(node_props, sizeof(node_props), "{\"program\":\"%s\"}", program); + tgt = cbm_gbuf_upsert_node(ctx->gbuf, "Process", x->ext, qn, "", 0, 0, node_props); + } + if (tgt <= 0 || tgt == src) { return 0; } - char props[CBM_SZ_512]; + char ci[CBM_SZ_1K]; + sh_ci_props(x, ci, sizeof(ci)); + char props[CBM_SZ_4K]; snprintf(props, sizeof(props), - "{\"api\":\"%s\",\"program\":\"%s\",\"line\":%d,\"confidence\":1.00}", x->via, program, - x->line); - cbm_gbuf_insert_edge(ctx->gbuf, src, process, "SPAWNS", props); + "{\"api\":\"%s\",\"program\":\"%s\"%s%s,\"line\":%d,\"confidence\":1.00}", x->via, + program, path, ci, x->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "SPAWNS", props); return SKIP_ONE; } @@ -2767,7 +2857,11 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { return 0; } for (int i = 0; i < h->count; i++) { - cbm_ht_set(files, h->items[i].file, (void *)h->items[i].file); + const sh_hit_t *x = &h->items[i]; + cbm_ht_set(files, x->file, (void *)x->file); + if (x->spawn && x->tgt > 0) { + cbm_ht_set(files, x->path, (void *)x->path); /* a spawned script's Module */ + } } sh_span_t *spans = NULL; int n = 0; @@ -2800,7 +2894,7 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { const sh_hit_t *x = &h->items[i]; int64_t src = sh_source(ctx, spans, n, modules, x->file, x->line); if (x->spawn) { - edges += src > 0 ? sh_link_spawn(ctx, src, x) : 0; + edges += src > 0 ? sh_link_spawn(ctx, src, x, modules) : 0; continue; } int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file); @@ -2808,10 +2902,13 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { continue; } char esc[CBM_SZ_1K]; - char props[CBM_SZ_2K]; + char ci[CBM_SZ_1K]; + char props[CBM_SZ_4K]; cbm_json_escape(esc, (int)sizeof(esc), x->path ? x->path : ""); - snprintf(props, sizeof(props), "{\"via\":\"%s\",\"%s\":\"%s\"%s,\"line\":%d}", x->via, - x->chart ? "chart" : "path", esc, x->tgt > 0 ? "" : ",\"external\":true", x->line); + sh_ci_props(x, ci, sizeof(ci)); + snprintf(props, sizeof(props), "{\"via\":\"%s\",\"%s\":\"%s\"%s%s,\"line\":%d}", x->via, + x->chart ? "chart" : "path", esc, x->tgt > 0 ? "" : ",\"external\":true", ci, + x->line); cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "DEPLOYS", props); edges++; } @@ -2821,15 +2918,93 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { return edges; } +/* ── CI configurations (pass_ci.c) ─────────────────────────────────── */ + +typedef struct { + sh_pass_t *p; + const char *rel; + CBMArena *a; +} sh_ci_t; + +static void sh_ci_var(sh_t *s, const char *name, const char *value) { + sh_val_t *v = value ? cbm_arena_alloc(s->a, sizeof(*v)) : NULL; + if (v) { + *v = (sh_val_t){value, NULL, 0}; + } + sh_set(s, name, v ? v : &SH_UNKNOWN_VAL, false); +} + +/* A command line CI runs: evaluated from its directory with its variables; + * $0 is unknown. */ +static void sh_ci_cmd(const cbm_ci_cmd_t *cmd, void *ud) { + sh_ci_t *x = ud; + sh_t s; + sh_state_init(&s, x->p, x->a, x->rel); + if (s.env) { + s.line_base = cmd->row; + s.ci = cmd->ci; + s.job = cmd->job; + s.exec = cmd->exec; + const char *wd = cmd->wd ? sh_join(x->a, "", cmd->wd) : NULL; + s.cwd = wd ? (sh_cwd_t){SH_AT, wd} : (sh_cwd_t){SH_LOST, NULL}; + if (strcmp(cmd->ci, "github-actions") == 0) { + sh_ci_var(&s, "GITHUB_WORKSPACE", SH_ROOT); + if (cmd->action_dir) { + sh_ci_var(&s, "GITHUB_ACTION_PATH", sh_anchor(x->a, cmd->action_dir)); + } + } + for (int i = 0; i < cmd->nenv; i++) { + sh_ci_var(&s, cmd->env[PAIR_LEN * i], cmd->env[PAIR_LEN * i + SKIP_ONE]); + } + sh_run(&s, x->p->parser, cmd->text, cmd->len); + } + sh_state_free(&s); +} + +static void sh_ci_file(sh_pass_t *p, const char *rel, const char *src, size_t len) { + CBMArena a; + cbm_arena_init(&a); + sh_ci_t x = {.p = p, .rel = rel, .a = &a}; + cbm_ci_commands(&a, rel, src, len, sh_ci_cmd, &x); + cbm_arena_destroy(&a); +} + +/* Prow job configs: the files whose top-level keys (YAML Variables) include + * presubmits / postsubmits / periodics. */ +static CBMHashTable *sh_prow_files(cbm_pipeline_ctx_t *ctx) { + static const char *const keys[] = {"presubmits", "postsubmits", "periodics", NULL}; + CBMHashTable *out = NULL; + for (int k = 0; keys[k]; k++) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_name(ctx->gbuf, keys[k], &nodes, &n) != 0) { + continue; + } + for (int i = 0; i < n; i++) { + const char *fp = nodes[i]->file_path; + if (!fp || !sh_eq(nodes[i]->label, "Variable")) { + continue; + } + out = out ? out : cbm_ht_create(0); + if (out) { + cbm_ht_set(out, fp, (void *)fp); + } + } + } + return out; +} + void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { sh_index_t ix = {0}; sh_hits_t hits = {0}; cbm_arena_init(&hits.arena); sh_pass_t p = {.ix = &ix, .hits = &hits, .parser = NULL}; + CBMHashTable *prow = sh_prow_files(ctx); for (int i = 0; i < file_count; i++) { CBMLanguage lang = files[i].language; - if (lang != CBM_LANG_BASH && lang != CBM_LANG_MAKEFILE) { + bool ci = cbm_ci_path(files[i].rel_path) || (prow && cbm_ht_get(prow, files[i].rel_path)); + if (lang != CBM_LANG_BASH && lang != CBM_LANG_MAKEFILE && !ci) { continue; } size_t len = 0; @@ -2843,7 +3018,9 @@ void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil p.parser = ts_parser_new(); } if (p.ix_ok && p.parser) { - if (lang == CBM_LANG_BASH) { + if (ci) { + sh_ci_file(&p, files[i].rel_path, src, len); + } else if (lang == CBM_LANG_BASH) { sh_script(&p, files[i].rel_path, src, len); } else { sh_makefile(&p, files[i].rel_path, src, len); @@ -2851,6 +3028,9 @@ void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil } cbm_free(CBM_MEM_CLASS_OTHER, src); } + if (prow) { + cbm_ht_free(prow); + } int edges = hits.count > 0 ? sh_link(ctx, &hits) : 0; if (p.parser) { ts_parser_delete(p.parser); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 311f92d2a..9ab34181c 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1042,11 +1042,38 @@ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx); * Runs after USAGE resolution. */ int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx); -/* Shell scripts and Make recipes (pass_shell.c): the commands that put - * manifests, kustomizations or charts on a cluster DEPLOYS them. Run at the - * end of the K8s pass, over the same files. */ +/* Shell scripts, Make recipes and the commands CI runs (pass_shell.c): a + * command that puts manifests, kustomizations or charts on a cluster DEPLOYS + * them; one that starts a program by a path, or a script of the repository, + * SPAWNS it. Run at the end of the K8s pass, over the same files. */ void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); +/* One command line a CI configuration runs (pass_ci.c). */ +typedef struct { + const char *ci; /* "github-actions" | "prow" */ + const char *job; /* the job; NULL: none (a composite action's step) */ + const char *text; /* shell text; CBM_SZ_16 zero bytes past len */ + size_t len; /* bytes of text */ + int row; /* 0-based source row of the text's first line */ + const char *wd; /* start directory, repo-relative ("" = root); NULL: unknown */ + const char *action_dir; /* a composite action's directory; NULL: none */ + const char **env; /* name, value pairs (a value NULL: unknown) */ + int nenv; /* pairs in env */ + bool exec; /* an argv exec'd without a shell, quoted into text */ +} cbm_ci_cmd_t; + +typedef void (*cbm_ci_cmd_fn)(const cbm_ci_cmd_t *cmd, void *ud); + +/* `rel` is a CI configuration by its path: a GitHub workflow + * (.github/workflows/.yml|yaml) or a composite action (action.yml|yaml). */ +bool cbm_ci_path(const char *rel); + +/* Hand each command line the CI configuration `rel` (YAML text src) runs to + * fn: GitHub Actions run: steps for a workflow or composite action, else + * Prow job containers. Strings live in arena `a`. */ +void cbm_ci_commands(CBMArena *a, const char *rel, const char *src, size_t len, cbm_ci_cmd_fn fn, + void *ud); + /* Terraform / OpenTofu modules (pass_terraform.c): a helm_release DEPLOYS its * chart. Run at the end of the K8s pass, over the same files. */ void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index c6ac29041..83df57008 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1776,6 +1776,50 @@ static char *gpf_spawns(const GpfFile *files, int nfiles) { free(got_); \ } while (0) +/* Every SPAWNS and DEPLOYS edge as "source -> target (TYPE api|via[ + * ci[/job]])", sorted, one per line: top-level code as "@"; a script + * as "script:", a Process as "proc:", a Chart as "chart:". */ +static char *gpf_runs(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT DISTINCT " + "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END " + "|| ' -> ' || CASE WHEN t.label IN ('Module','File') THEN 'script:' || t.file_path " + "WHEN t.label='Process' THEN 'proc:' || t.name WHEN t.label='Chart' THEN " + "'chart:' || t.name ELSE t.label || ':' || t.file_path END " + "|| ' (' || e.type || ' ' || coalesce(json_extract(e.properties,'$.api'), " + "json_extract(e.properties,'$.via')) || coalesce(' ' || " + "json_extract(e.properties,'$.ci') || coalesce('/' || " + "json_extract(e.properties,'$.job'), ''), '') || ')' AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type IN ('SPAWNS','DEPLOYS') ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +#define GPF_ASSERT_RUNS(files, n, want) \ + do { \ + char *got_ = gpf_runs(files, n); \ + if (!got_ || strcmp(got_, want) != 0) { \ + fprintf(stderr, "runs got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \ + free(got_); \ + FAIL("SPAWNS / DEPLOYS edge set"); \ + } \ + free(got_); \ + } while (0) + #define GPF_CM(name) "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " name "\n" #define GPF_KUST "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" @@ -1974,6 +2018,105 @@ TEST(probe_make_programs_by_path_spawn) { PASS(); } +/* A script of the repository a command runs SPAWNS that script (its Module): + * by its path, through a shell (bash -e FILE), past `env A="$X/y"`; not + * itself. CI runs commands too: a GitHub workflow's run: steps from the + * workspace (`${{ github.workspace }}`, env literals, working-directory; + * an unknown `${{ }}` prefix leaves a Process; no pwsh step, no step of a + * Windows job), a composite action's from its directory + * (`${{ github.action_path }}`), a decorated Prow job's container command + * exec'd from the checkout (`bash -c TEXT` as shell; undecorated: the + * directory unknown, so a Process). Their edges carry the CI and the job. */ +TEST(probe_ci_and_scripts_run_scripts) { + static const GpfFile files[] = { + {"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 0.1.0\n"}, + {"hack/lib/helper.sh", "#!/bin/sh\necho helper\n"}, + {"hack/lint.sh", "#!/bin/sh\necho lint\n"}, + {"hack/ci/run-e2e.sh", "#!/bin/sh\necho e2e\n"}, + {"hack/ci/lint.sh", "#!/bin/sh\necho lint\n"}, + {"ops/apply.sh", "#!/bin/sh\necho apply\n"}, + {"hack/test.sh", "#!/usr/bin/env bash\n" + "bash -e \"$(dirname \"$0\")/lib/helper.sh\"\n" + "env OUT=\"$TMPDIR/x.rpm\" ./hack/lint.sh\n"}, + {"hack/e2e.sh", "#!/bin/sh\n" + "./hack/test.sh\n" + "[ -n \"$AGAIN\" ] || AGAIN=1 \"$0\"\n"}, + {"Makefile", "test:\n" + "\t./hack/test.sh\n" + "\t/bin/bash hack/lint.sh\n"}, + {".github/workflows/ci.yml", + "name: ci\n" + "on: [push]\n" + "env:\n" + " CHART_DIR: charts/web\n" + "jobs:\n" + " test:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - uses: actions/checkout@v4\n" + " - run: ./hack/test.sh\n" + " - name: multi\n" + " run: |\n" + " sh hack/lint.sh --fix\n" + " ${{ github.workspace }}/hack/e2e.sh\n" + " helm upgrade --install web \"$CHART_DIR\"\n" + " ./bin/tool --x\n" + " - working-directory: ops\n" + " run: ./apply.sh\n" + " - run: \"${{ matrix.dir }}/hack/other.sh\"\n" + " - shell: pwsh\n" + " run: ./hack/ci/lint.sh\n" + " win:\n" + " runs-on: windows-latest\n" + " steps:\n" + " - run: ./hack/ci/run-e2e.sh\n"}, + {".github/actions/setup/setup.sh", "#!/bin/sh\necho setup\n"}, + {".github/actions/setup/action.yml", "name: setup\n" + "runs:\n" + " using: composite\n" + " steps:\n" + " - shell: bash\n" + " run: ${{ github.action_path }}/setup.sh\n"}, + {".prow/jobs.yaml", "presubmits:\n" + " - name: pre-e2e\n" + " decorate: true\n" + " spec:\n" + " containers:\n" + " - image: x\n" + " command:\n" + " - \"./hack/ci/run-e2e.sh\"\n" + " - name: pre-lint\n" + " decorate: true\n" + " spec:\n" + " containers:\n" + " - command: [\"bash\", \"-c\", \"./hack/ci/lint.sh\"]\n" + " - name: pre-raw\n" + " spec:\n" + " containers:\n" + " - command: [\"./hack/ci/run-e2e.sh\"]\n"}}; + GPF_ASSERT_RUNS( + files, 13, + "@.github/actions/setup/action.yml -> script:.github/actions/setup/setup.sh " + "(SPAWNS shell github-actions)\n" + "@.github/workflows/ci.yml -> chart:web (DEPLOYS helm github-actions/test)\n" + "@.github/workflows/ci.yml -> proc:other.sh (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> proc:tool (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> script:hack/e2e.sh (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> script:hack/lint.sh (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> script:hack/test.sh (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> script:ops/apply.sh (SPAWNS shell github-actions/test)\n" + "@.prow/jobs.yaml -> proc:run-e2e.sh (SPAWNS exec prow/pre-raw)\n" + "@.prow/jobs.yaml -> script:hack/ci/lint.sh (SPAWNS shell prow/pre-lint)\n" + "@.prow/jobs.yaml -> script:hack/ci/run-e2e.sh (SPAWNS exec prow/pre-e2e)\n" + "@hack/e2e.sh -> script:hack/test.sh (SPAWNS shell)\n" + "@hack/test.sh -> script:hack/lib/helper.sh (SPAWNS shell)\n" + "@hack/test.sh -> script:hack/lint.sh (SPAWNS shell)\n" + "test -> proc:bash (SPAWNS shell)\n" + "test -> script:hack/lint.sh (SPAWNS shell)\n" + "test -> script:hack/test.sh (SPAWNS shell)"); + PASS(); +} + /* A Terraform helm_release DEPLOYS its chart: a chart dir of the module * (./charts/x, ${path.module}/charts/x), else -- with a repository, or as * oci://.../ -- the external chart of that name. A variable chart or a @@ -3779,6 +3922,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_make_recipes_deploy); RUN_TEST(probe_shell_programs_by_path_spawn); RUN_TEST(probe_make_programs_by_path_spawn); + RUN_TEST(probe_ci_and_scripts_run_scripts); RUN_TEST(probe_go_installer_deploys_charts); RUN_TEST(probe_terraform_helm_release_deploys_its_chart); RUN_TEST(probe_helm_templated_objects_are_resources); From b080e982deffacb8e5ee34b9474d67d73801c31a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 07:39:09 +0200 Subject: [PATCH 36/52] feat(shell): make TARGET SPAWNS the rule of TARGET "Which CI jobs and scripts run `make verify`, and through which makefile?" had no answer: a `make` command drew no edge. - make [-C DIR] [-f FILE] [VAR=V] TARGET... -- from a shell script, a Make recipe ($(MAKE), which is make unless the makefile sets it) or a CI step -- SPAWNS the rule node of each TARGET: in the makefile make reads (the first -f, else GNUmakefile / makefile / Makefile of the directory, -C cumulative, resolved like any path to exactly one), else in the files it includes, in reading order. Includes are read when linking, with the makefile's variables, so an unchanged makefile is followed too. An unknown -C / -f value, an unknown or ruleless target, and a bare `make` draw no edge. Properties: program make, path (the makefile), target. - A Make variable whose value spans backslash-continued lines is folded to one line, as make reads it: expanded into a recipe line it added lines, so every later command of that line was attributed to a later rule. Measured on 17 repositories against an independent reading (its own make argument parser and rule / include reader): 282 of 300 make runs linked, 282 of 282 correct (cluster-api 115, kubermatic 72, gardener 56, otel-demo 21, deckhouse 17, k8s.io 1). The 18 not linked are targets that are not the first of a multi-target rule (`seed-up seed-dev seed-down:`), which have no node of their own yet. Only SPAWNS (+282) changed; three runs build identical graphs; index time unchanged (cluster-api 2.67 -> 2.67 s, gardener 5.24 -> 5.24 s, kubermatic 2.94 -> 2.95 s, medians); renaming or adding a rule, editing the command or the workflow leaves the graph a fresh index builds (deleting a makefile differs only in SEMANTICALLY_RELATED edges, as the previous build does). Signed-off-by: Martin Vogel --- src/pipeline/pass_shell.c | 452 +++++++++++++++++++++++++++++++++-- tests/test_grammar_probe_f.c | 65 ++++- 2 files changed, 492 insertions(+), 25 deletions(-) diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index fca3b49fb..b74ab402c 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -376,6 +376,7 @@ typedef struct { const char *job; /* pass-arena: its job; NULL: none */ bool chart; bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ + bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */ } sh_hit_t; typedef struct { @@ -1767,6 +1768,24 @@ static int64_t sh_repo_file_id(const sh_t *s, const char *path) { static const char *const SH_SHELLS[] = {"bash", "sh", "zsh", "ksh", "dash", NULL}; +/* The one candidate of `path` that names a File of the repository (its + * File node in *id); NULL: none, or two. */ +static const char *sh_unique_file(sh_t *s, const char *path, int64_t *id) { + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, path, c); + const char *hit = NULL; + int found = 0; + for (int i = 0; i < n; i++) { + int64_t f = sh_repo_file_id(s, c[i]); + if (f > 0) { + found++; + *id = f; + hit = c[i]; + } + } + return found == SKIP_ONE ? hit : NULL; +} + /* The program is a script of the repository -- its path names a file of it, * or a shell runs one (bash [-opts] FILE; -c TEXT is no file) -- resolved to * exactly one file: that script is spawned. */ @@ -1785,20 +1804,9 @@ static void sh_spawn_script(sh_t *s, TSNode cmd, const sh_argv_t *a, int p) { if (!path) { return; } - const char *c[PAIR_LEN] = {NULL, NULL}; - int n = sh_candidates(s, path, c); - const char *hit = NULL; int64_t id = 0; - int found = 0; - for (int i = 0; i < n; i++) { - int64_t f = sh_repo_file_id(s, c[i]); - if (f > 0) { - found++; - id = f; - hit = c[i]; - } - } - if (found == SKIP_ONE) { + const char *hit = sh_unique_file(s, path, &id); + if (hit) { sh_hit_push(s, cmd, (sh_hit_t){.via = s->exec ? "exec" : "shell", .tgt = id, @@ -1808,6 +1816,177 @@ static void sh_spawn_script(sh_t *s, TSNode cmd, const sh_argv_t *a, int p) { } } +/* ── make TARGET (P3.5b) ───────────────────────────────────────────── */ + +static const char *const SH_MAKES[] = {"make", "gmake", NULL}; +static const char *const SH_MAKEFILES[] = {"GNUmakefile", "makefile", "Makefile", NULL}; + +typedef enum { MK_OPT_NONE, MK_OPT_DIR, MK_OPT_FILE, MK_OPT_ARG } mk_opt_t; + +/* A long option of make that takes a value (v: n bytes, no `=...`). */ +static mk_opt_t sh_make_long(const char *v, size_t n) { + static const struct { + const char *name; + mk_opt_t kind; + } opts[] = { + {"--directory", MK_OPT_DIR}, {"--file", MK_OPT_FILE}, {"--makefile", MK_OPT_FILE}, + {"--include-dir", MK_OPT_ARG}, {"--old-file", MK_OPT_ARG}, {"--assume-old", MK_OPT_ARG}, + {"--what-if", MK_OPT_ARG}, {"--new-file", MK_OPT_ARG}, {"--assume-new", MK_OPT_ARG}}; + for (size_t i = 0; i < sizeof(opts) / sizeof(opts[0]); i++) { + if (strlen(opts[i].name) == n && strncmp(v, opts[i].name, n) == 0) { + return opts[i].kind; + } + } + return MK_OPT_NONE; +} + +/* A command-line variable: NAME=, NAME:=, NAME::=, NAME+=, NAME?=, NAME!=. */ +static bool sh_make_assignment(const char *v) { + if (!(v[0] == '_' || ((v[0] | ' ') >= 'a' && (v[0] | ' ') <= 'z'))) { + return false; + } + size_t i = SKIP_ONE; + while (v[i] && (v[i] == '_' || v[i] == '.' || v[i] == '-' || (v[i] >= '0' && v[i] <= '9') || + ((v[i] | ' ') >= 'a' && (v[i] | ' ') <= 'z'))) { + i++; + } + while (v[i] == ' ' || v[i] == '\t') { + i++; + } + if (v[i] == ':') { + while (v[i] == ':') { + i++; + } + return v[i] == '='; + } + return v[i] == '=' || (v[i] && strchr("+?!", v[i]) && v[i + SKIP_ONE] == '='); +} + +/* -j N / -l N: N is a count. */ +static bool sh_make_count(const char *v) { + if (!v || !v[0]) { + return false; + } + for (const char *p = v; *p; p++) { + if (!(*p == '.' || (*p >= '0' && *p <= '9'))) { + return false; + } + } + return true; +} + +/* -C DIR after `base`: DIR from base, an absolute (anchored) DIR alone. */ +static const char *sh_make_join(CBMArena *a, const char *base, const char *dir) { + if (dir[0] == SH_ANCHOR || dir[0] == '/' || sh_eq(base, ".")) { + return dir; + } + return cbm_arena_sprintf(a, "%s/%s", base, dir); +} + +/* The makefile make reads in directory `dir` (GNUmakefile, makefile, + * Makefile, the first there is), for the one candidate of `dir` that has + * one; NULL otherwise. */ +static const char *sh_make_default(sh_t *s, const char *dir) { + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, dir, c); + const char *hit = NULL; + int found = 0; + for (int i = 0; i < n; i++) { + for (int k = 0; SH_MAKEFILES[k]; k++) { + const char *p = sh_join(s->a, c[i], SH_MAKEFILES[k]); + if (p && sh_repo_file_id(s, p) > 0) { + found++; + hit = p; + break; + } + } + } + return found == SKIP_ONE ? hit : NULL; +} + +/* make [-C DIR] [-f FILE] [VAR=V] TARGET...: each TARGET SPAWNS the rule of + * that name the makefile reads -- the first -f, else the directory's (-C is + * cumulative) -- resolved when linking, through its includes. An unknown -C + * or -f value is no edge, an unknown word no target; no target (the default + * goal) is no edge. */ +static void sh_make(sh_t *s, TSNode cmd, const sh_argv_t *a, int p) { + const char *dir = "."; + const char *file = NULL; + const char **targets = cbm_arena_alloc(s->a, (size_t)(a->n + SKIP_ONE) * sizeof(*targets)); + if (!targets) { + return; + } + int nt = 0; + bool rest = false; + for (int j = p + SKIP_ONE; j < a->n; j++) { + const char *v = a->v[j]; + if (!v) { + continue; + } + if (rest || v[0] != '-' || sh_eq(v, "-")) { + if (rest || !sh_make_assignment(v)) { + targets[nt++] = v; + } + continue; + } + if (sh_eq(v, "--")) { + rest = true; + continue; + } + const char *arg = NULL; + mk_opt_t kind = MK_OPT_NONE; + if (v[SKIP_ONE] == '-') { + const char *eq = strchr(v, '='); + kind = sh_make_long(v, eq ? (size_t)(eq - v) : strlen(v)); + arg = eq ? eq + SKIP_ONE : NULL; + if (kind != MK_OPT_NONE && !eq) { + j++; + arg = j < a->n ? a->v[j] : NULL; + } + } else { + for (size_t k = SKIP_ONE; v[k]; k++) { + if (strchr("CfIoW", v[k])) { + kind = v[k] == 'C' ? MK_OPT_DIR : v[k] == 'f' ? MK_OPT_FILE : MK_OPT_ARG; + arg = v[k + SKIP_ONE] ? v + k + SKIP_ONE : NULL; + if (!arg) { + j++; + arg = j < a->n ? a->v[j] : NULL; + } + break; + } + if (v[k] == 'j' || v[k] == 'l') { + if (!v[k + SKIP_ONE] && j + SKIP_ONE < a->n && + sh_make_count(a->v[j + SKIP_ONE])) { + j++; + } + break; + } + } + } + if (kind == MK_OPT_DIR || kind == MK_OPT_FILE) { + if (!arg) { + return; + } + if (kind == MK_OPT_DIR) { + dir = sh_make_join(s->a, dir, arg); + } else if (!file) { + file = arg; + } + } + } + int64_t id = 0; + const char *mkfile = + file ? sh_unique_file(s, sh_make_join(s->a, dir, file), &id) : sh_make_default(s, dir); + for (int i = 0; mkfile && i < nt; i++) { + sh_hit_push(s, cmd, + (sh_hit_t){.via = s->exec ? "exec" : "shell", + .ext = targets[i], + .path = mkfile, + .spawn = true, + .make = true}); + } +} + /* A program started by a path (./_build/x, ${DIR}/x, $(LOCALBIN)/x) SPAWNS * the Process of its name -- the value's last segment, or, the value * unknown, the word's literal last segment. A path naming a file of the @@ -1820,6 +1999,9 @@ static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) { const char *v = a->v[p]; const char *name = NULL; if (v) { + if (sh_in(sh_basename(v), SH_MAKES)) { + sh_make(s, cmd, a, p); + } sh_spawn_script(s, cmd, a, p); if (!strchr(v, '/') || strpbrk(v, "*?[")) { return; @@ -2349,7 +2531,12 @@ typedef struct { uint32_t row; /* 0-based row of b */ } mk_line_t; -typedef struct { +typedef struct mk mk_t; + +/* An include directive's expanded file names (mk_closure). */ +typedef void (*mk_include_fn)(mk_t *m, const char *names); + +struct mk { CBMArena *a; const char *src; CBMHashTable *vars; /* name -> mk_var_t */ @@ -2357,7 +2544,9 @@ typedef struct { mk_line_t *lines; int nlines; int cap_lines; -} mk_t; + mk_include_fn on_include; /* NULL: includes are not followed */ + void *ud; +}; static const char MK_UNKNOWN[] = "${__cbm_unknown}"; @@ -2368,6 +2557,9 @@ static const char *mk_lookup(mk_t *m, const char *name) { return m->makefile ? "${__cbm_curdir}" : MK_UNKNOWN; } mk_var_t *v = cbm_ht_get(m->vars, name); + if (!v && strcmp(name, "MAKE") == 0) { + return "make"; /* make's own command, unless the makefile sets it */ + } if (!v || !v->known || v->busy) { return MK_UNKNOWN; } @@ -2494,6 +2686,34 @@ static const char *mk_expand(mk_t *m, const char *text) { return sh_str_done(&b); } +/* A variable's value as make reads it: each backslash-newline, with the + * blanks around it, is one space (a value expanded into a recipe line adds + * no lines). */ +static const char *mk_fold(CBMArena *a, const char *p, size_t n) { + char *out = cbm_arena_alloc(a, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t o = 0; + for (size_t i = 0; i < n; i++) { + size_t k = i + SKIP_ONE; + k += k < n && p[k] == '\r'; + if (p[i] != '\\' || k >= n || p[k] != '\n') { + out[o++] = p[i]; + continue; + } + while (o > 0 && (out[o - SKIP_ONE] == ' ' || out[o - SKIP_ONE] == '\t')) { + o--; + } + out[o++] = ' '; + i = k; + while (i + SKIP_ONE < n && (p[i + SKIP_ONE] == ' ' || p[i + SKIP_ONE] == '\t')) { + i++; + } + } + return mk_trim(a, out, o); +} + static void mk_assign(mk_t *m, TSNode n, bool cond) { TSNode name = ts_node_child_by_field_name(n, TS_FIELD("name")); TSNode value = ts_node_child_by_field_name(n, TS_FIELD("value")); @@ -2505,7 +2725,7 @@ static void mk_assign(mk_t *m, TSNode n, bool cond) { const char *raw = ""; if (!ts_node_is_null(value)) { uint32_t vb = ts_node_start_byte(value); - raw = mk_trim(m->a, m->src + vb, ts_node_end_byte(value) - vb); + raw = mk_fold(m->a, m->src + vb, ts_node_end_byte(value) - vb); } const char *op = "="; for (uint32_t i = 0; i < ts_node_child_count(n); i++) { @@ -2608,6 +2828,16 @@ static void mk_collect_one(mk_t *m, TSNode c, bool cond) { } else if (strstr(t, "conditional") || strstr(t, "if") == t || strcmp(t, "else_directive") == 0 || strcmp(t, "elsif_directive") == 0) { mk_collect(m, c, true); + } else if (strcmp(t, "include_directive") == 0) { + TSNode names = ts_node_child_by_field_name(c, TS_FIELD("filenames")); + if (m->on_include && !ts_node_is_null(names)) { + uint32_t nb = ts_node_start_byte(names); + const char *raw = mk_trim(m->a, m->src + nb, ts_node_end_byte(names) - nb); + const char *v = raw ? mk_expand(m, raw) : NULL; + if (v) { + m->on_include(m, v); + } + } } else if (strstr(t, "directive")) { mk_collect(m, c, cond); } @@ -2692,9 +2922,9 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l uint32_t e = m.lines[r].e; const char *raw = cbm_arena_strndup(&a, src + b, e - b); const char *text = raw ? mk_expand(&m, raw) : NULL; - if (!text || (!strchr(text, '/') && !strstr(text, "sh") && + if (!text || (!strchr(text, '/') && !strstr(text, "sh") && !strstr(text, "make") && !sh_mentions_deploy_tool(text, strlen(text)))) { - continue; /* neither a deploy, a program started by a path, nor a shell */ + continue; /* no deploy, program by path, shell or make */ } size_t tl = strlen(text); char *buf = cbm_arena_alloc(&a, tl + CBM_SZ_16); @@ -2730,6 +2960,171 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l ts_tree_delete(tree); } +/* ── make targets: the rules a makefile reads (P3.5b) ──────────────── */ + +typedef struct { + sh_pass_t *p; + cbm_pipeline_ctx_t *ctx; + CBMArena *a; + CBMHashTable *seen; /* the files read */ + CBMHashTable *vars; /* shared by them, as make shares them */ + const char **order; /* the files, in reading order */ + int n; + int cap; + const char *mkdir; /* make's directory ("" = the root) */ +} mk_closure_t; + +static void mk_closure_file(mk_closure_t *cl, const char *rel); + +/* An include directive: each name -- from make's directory ($(CURDIR) is + * it), or the root ($(shell git rev-parse --show-toplevel)) -- is read in + * turn, recursively, where the directive stands. A name not fully known is + * skipped. */ +static void mk_closure_include(mk_t *m, const char *names) { + static const char curdir[] = "${__cbm_curdir}"; + static const char root[] = "${__cbm_root}"; + mk_closure_t *cl = m->ud; + const char *p = names; + for (;;) { + while (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') { + p++; + } + size_t wl = strcspn(p, " \t\r\n"); + if (!wl) { + return; + } + const char *w = cbm_arena_strndup(cl->a, p, wl); + p += wl; + const char *base = cl->mkdir; + if (w && strncmp(w, curdir, sizeof(curdir) - SKIP_ONE) == 0) { + w += sizeof(curdir) - SKIP_ONE; + } else if (w && strncmp(w, root, sizeof(root) - SKIP_ONE) == 0) { + w += sizeof(root) - SKIP_ONE; + base = ""; + } else if (w && w[0] == '/') { + continue; + } + const char *path = w && !strchr(w, '$') ? sh_join(cl->a, base, w) : NULL; + if (path) { + mk_closure_file(cl, path); + } + } +} + +static void mk_closure_file(mk_closure_t *cl, const char *rel) { + if (cbm_ht_get(cl->seen, rel) || + !sh_grow((void **)&cl->order, &cl->cap, cl->n, sizeof(*cl->order))) { + return; + } + cbm_ht_set(cl->seen, rel, (void *)rel); + cl->order[cl->n++] = rel; + const char *abs = cbm_arena_sprintf(cl->a, "%s/%s", cl->ctx->repo_path, rel); + size_t len = 0; + char *src = abs ? sh_read(abs, &len) : NULL; + if (!src) { + return; + } + TSTree *tree = NULL; + if (ts_parser_set_language(cl->p->parser, cbm_ts_language(CBM_LANG_MAKEFILE))) { + tree = ts_parser_parse_string(cl->p->parser, NULL, src, (uint32_t)len); + } + if (tree) { + mk_t m = {.a = cl->a, + .src = src, + .vars = cl->vars, + .makefile = true, + .on_include = mk_closure_include, + .ud = cl}; + mk_collect(&m, ts_tree_root_node(tree), false); + cbm_free(CBM_MEM_CLASS_OTHER, m.lines); + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); +} + +static const char *mk_rule_key(CBMArena *a, const char *file, const char *name) { + return cbm_arena_sprintf(a, "%s\x1f%s", file, name); +} + +/* Each make hit's target: the rule node of that name in the makefile, else + * in the files it includes, in reading order (the first rule of a file). + * Read here, when linking, so an unchanged makefile is read too. */ +static void sh_make_resolve(sh_pass_t *p, cbm_pipeline_ctx_t *ctx, sh_hits_t *h) { + CBMArena a; + cbm_arena_init(&a); + CBMHashTable *closures = cbm_ht_create(0); /* makefile -> mk_closure_t */ + CBMHashTable *want = cbm_ht_create(0); /* every file a closure reads */ + CBMHashTable *rules = cbm_ht_create(0); /* file \x1f name -> rule node */ + for (int i = 0; closures && want && i < h->count; i++) { + const sh_hit_t *x = &h->items[i]; + if (!x->make || cbm_ht_get(closures, x->path)) { + continue; + } + mk_closure_t *cl = cbm_arena_alloc(&a, sizeof(*cl)); + if (!cl) { + continue; + } + const char *dir = sh_dirname(&a, x->path); + *cl = (mk_closure_t){.p = p, + .ctx = ctx, + .a = &a, + .seen = cbm_ht_create(0), + .vars = cbm_ht_create(0), + .mkdir = dir && strcmp(dir, ".") != 0 ? dir : ""}; + if (cl->seen && cl->vars) { + mk_closure_file(cl, x->path); + } + for (int k = 0; k < cl->n; k++) { + cbm_ht_set(want, cl->order[k], (void *)cl->order[k]); + } + cbm_ht_set(closures, x->path, cl); + } + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (rules && want && cbm_ht_count(want) > 0 && + cbm_gbuf_find_by_label(ctx->gbuf, "Function", &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + if (!fp || !nodes[i]->name || !cbm_ht_get(want, fp)) { + continue; + } + const char *key = mk_rule_key(&a, fp, nodes[i]->name); + const cbm_gbuf_node_t *old = key ? cbm_ht_get(rules, key) : NULL; + if (key && (!old || nodes[i]->start_line < old->start_line)) { + cbm_ht_set(rules, key, (void *)nodes[i]); + } + } + } + for (int i = 0; rules && closures && i < h->count; i++) { + sh_hit_t *x = &h->items[i]; + const mk_closure_t *cl = x->make ? cbm_ht_get(closures, x->path) : NULL; + for (int k = 0; cl && k < cl->n && x->tgt <= 0; k++) { + const char *key = mk_rule_key(&a, cl->order[k], x->ext); + const cbm_gbuf_node_t *rule = key ? cbm_ht_get(rules, key) : NULL; + x->tgt = rule ? rule->id : 0; + } + } + for (int i = 0; closures && i < h->count; i++) { + mk_closure_t *cl = h->items[i].make ? cbm_ht_get(closures, h->items[i].path) : NULL; + if (cl) { + cbm_ht_delete(closures, h->items[i].path); + cbm_ht_free(cl->seen); + cbm_ht_free(cl->vars); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)cl->order); + } + } + if (closures) { + cbm_ht_free(closures); + } + if (want) { + cbm_ht_free(want); + } + if (rules) { + cbm_ht_free(rules); + } + cbm_arena_destroy(&a); +} + /* ── Edges ─────────────────────────────────────────────────────────── */ typedef struct { @@ -2809,16 +3204,24 @@ static void sh_ci_props(const sh_hit_t *x, char *out, size_t sz) { } } -/* SPAWNS to the script `x->tgt` (its Module, else its File) or to the - * Process of `x->ext`, as pass_spawns.c draws it. */ +/* SPAWNS to the rule `x->tgt` of a make target, the script `x->tgt` (its + * Module, else its File) or the Process of `x->ext`, as pass_spawns.c draws + * it. */ static int sh_link_spawn(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x, CBMHashTable *modules) { char program[CBM_SZ_128]; - cbm_json_escape(program, (int)sizeof(program), x->ext); - char path[CBM_SZ_1K + CBM_SZ_16]; + cbm_json_escape(program, (int)sizeof(program), x->make ? "make" : x->ext); + char path[CBM_SZ_2K]; path[0] = '\0'; int64_t tgt = 0; - if (x->tgt > 0) { + if (x->make) { + char esc[CBM_SZ_1K]; + char target[CBM_SZ_512]; + cbm_json_escape(esc, (int)sizeof(esc), x->path); + cbm_json_escape(target, (int)sizeof(target), x->ext); + snprintf(path, sizeof(path), ",\"path\":\"%s\",\"target\":\"%s\"", esc, target); + tgt = x->tgt; + } else if (x->tgt > 0) { int64_t mod = (int64_t)(intptr_t)cbm_ht_get(modules, x->path); tgt = mod > 0 ? mod : x->tgt; char esc[CBM_SZ_1K]; @@ -3031,6 +3434,9 @@ void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil if (prow) { cbm_ht_free(prow); } + if (hits.count > 0 && p.ix_ok && p.parser) { + sh_make_resolve(&p, ctx, &hits); + } int edges = hits.count > 0 ? sh_link(ctx, &hits) : 0; if (p.parser) { ts_parser_delete(p.parser); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 83df57008..a18b23011 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1778,7 +1778,8 @@ static char *gpf_spawns(const GpfFile *files, int nfiles) { /* Every SPAWNS and DEPLOYS edge as "source -> target (TYPE api|via[ * ci[/job]])", sorted, one per line: top-level code as "@"; a script - * as "script:", a Process as "proc:", a Chart as "chart:". */ + * as "script:", a Process as "proc:", a Chart as "chart:", + * a Make rule as "rule::". */ static char *gpf_runs(const GpfFile *files, int nfiles) { GpfProj lp; cbm_store_t *store = gpf_index_files(&lp, files, nfiles); @@ -1790,7 +1791,8 @@ static char *gpf_runs(const GpfFile *files, int nfiles) { "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END " "|| ' -> ' || CASE WHEN t.label IN ('Module','File') THEN 'script:' || t.file_path " "WHEN t.label='Process' THEN 'proc:' || t.name WHEN t.label='Chart' THEN " - "'chart:' || t.name ELSE t.label || ':' || t.file_path END " + "'chart:' || t.name WHEN t.label='Function' THEN 'rule:' || t.file_path || ':' || " + "t.name ELSE t.label || ':' || t.file_path END " "|| ' (' || e.type || ' ' || coalesce(json_extract(e.properties,'$.api'), " "json_extract(e.properties,'$.via')) || coalesce(' ' || " "json_extract(e.properties,'$.ci') || coalesce('/' || " @@ -2117,6 +2119,64 @@ TEST(probe_ci_and_scripts_run_scripts) { PASS(); } +/* make TARGET SPAWNS the rule of TARGET the makefile reads: the + * directory's Makefile (-C, --directory=), a -f file, a rule of a file it + * includes; from scripts, CI steps and recipes ($(MAKE)), on the line of + * the command (a multi-line variable expanded before it adds no lines). No + * edge for an unknown target or -C value, a target no rule has, a rule only + * inside a conditional (no node), or no target. */ +TEST(probe_make_targets_spawn_their_rules) { + static const GpfFile files[] = { + {"Makefile", "include mk/common.mk\n" + "VERSION := 1\n" + "KEYS = a \\\n" + " b \\\n" + " c\n\n" + "build:\n" + "\tgo build ./...\n\n" + "test: build\n" + "\t$(MAKE) lint\n" + "\t$(MAKE) -C sub check\n\n" + "ifeq ($(CI),true)\n" + "cond:\n" + "\techo ci\n" + "endif\n\n" + "keys:\n" + "\t@for k in $(KEYS); do \\\n" + "\t echo $$k; \\\n" + "\tdone; \\\n" + "\t$(MAKE) build\n\n" + "after:\n" + "\techo one\n"}, + {"mk/common.mk", "lint:\n\techo lint\n"}, + {"sub/Makefile", "check:\n\techo check\n"}, + {"hack/ci.sh", "#!/bin/sh\n" + "make test VERBOSE=1\n" + "make -C sub -j 4 check\n" + "make -f Makefile build unknown-target\n" + "make \"$TARGET\"\n" + "make -C \"$DIR\" build\n" + "make cond\n" + "make\n" + "gmake --directory=sub check\n"}, + {".github/workflows/ci.yml", "on: [push]\n" + "jobs:\n" + " ci:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: make -j8 build\n"}}; + GPF_ASSERT_RUNS( + files, 5, + "@.github/workflows/ci.yml -> rule:Makefile:build (SPAWNS shell github-actions/ci)\n" + "@hack/ci.sh -> rule:Makefile:build (SPAWNS shell)\n" + "@hack/ci.sh -> rule:Makefile:test (SPAWNS shell)\n" + "@hack/ci.sh -> rule:sub/Makefile:check (SPAWNS shell)\n" + "keys -> rule:Makefile:build (SPAWNS shell)\n" + "test -> rule:mk/common.mk:lint (SPAWNS shell)\n" + "test -> rule:sub/Makefile:check (SPAWNS shell)"); + PASS(); +} + /* A Terraform helm_release DEPLOYS its chart: a chart dir of the module * (./charts/x, ${path.module}/charts/x), else -- with a repository, or as * oci://.../ -- the external chart of that name. A variable chart or a @@ -3923,6 +3983,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_shell_programs_by_path_spawn); RUN_TEST(probe_make_programs_by_path_spawn); RUN_TEST(probe_ci_and_scripts_run_scripts); + RUN_TEST(probe_make_targets_spawn_their_rules); RUN_TEST(probe_go_installer_deploys_charts); RUN_TEST(probe_terraform_helm_release_deploys_its_chart); RUN_TEST(probe_helm_templated_objects_are_resources); From 7842cd8a69178efbf8d3c34dcc32eb30e2159dcc Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 11 Oct 2026 08:30:28 +0200 Subject: [PATCH 37/52] fix(make): a node per rule target; target-variable lines are no rules Make rules were read with three gaps: - A rule naming several targets (`seed-up seed-dev seed-down: ...`, `kind-% seed-%:`) became one Function named after its first target: `seed-down` had no node, so nothing could link `make seed-down` and a search for it found nothing. Every target now gets a Function over the rule's span (without fingerprint, structural profile or body tokens, so the copies of one recipe do not read as similar code), and the recipe's DEPLOYS / SPAWNS edges leave from each of them. - A target-specific variable line (`kind-%: export X := y`) was taken for a rule: a phantom Function for its first target, or a variant span merged into the real rule of that name. Such a line is no rule now. Its tree-sitter ERROR region is no longer masked, so the file reports it as partially parsed, as for any other unparsed region. - `include hack/tools.mk` imported a Folder `hack/tools` (local name "mk\n\n": the generic import reader kept the directive's newlines and split at the extension). Make includes are read on their own now: every name of include / -include / sinclude, outside recipes, imports that file; a name with a variable reference is left out. Measured on 17 repositories against an independent reading: make runs linked 298 of 300 (was 282), all 298 correct; the 2 left pick their target in an if/else, which stays unknown by design. Graph changes are confined to Makefiles (gardener +26 targets, 13 phantoms removed; deckhouse +4), their edges, and two side effects of new nodes: SEMANTICALLY_RELATED edges elsewhere re-rank, and one Go member reference in deckhouse lost its unique-name binding to a new Make rule of the same name. Three runs build identical graphs; index time unchanged (deckhouse 9.61 -> 9.60 s, gardener 5.24 -> 5.27 s, cluster-api 2.68 -> 2.68 s, medians); renaming a target, editing a Makefile or an included file leaves the graph a fresh index builds (gardener's Makefile edit differed before, through the phantom nodes, and no longer does). Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 110 +++++++++++++++++++++++++++++++++ internal/cbm/extract_imports.c | 50 +++++++++++++++ src/pipeline/pass_shell.c | 75 ++++++++++++++++------ tests/test_grammar_probe_f.c | 79 +++++++++++++++++++++++ 4 files changed, 294 insertions(+), 20 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index feb64086c..f3e651c74 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -6275,6 +6275,107 @@ void cbm_test_declarations_finish(CBMExtractCtx *ctx) { } } +static bool make_name_char(char c) { + return c == '_' || c == '.' || c == '-' || (c >= '0' && c <= '9') || + ((c | ' ') >= 'a' && (c | ' ') <= 'z'); +} + +/* A `rule` node that is a target-specific variable: after the targets' + * colon(s) come [export|override|private|unexport]... NAME and an + * assignment operator (= := ::= :::= += ?= !=). */ +static bool make_rule_sets_variable(TSNode rule, const char *src) { + static const char *const kws[] = {"export", "override", "private", "unexport", NULL}; + TSNode targets = cbm_find_child_by_kind(rule, "targets"); + if (ts_node_is_null(targets)) { + return false; + } + uint32_t p = ts_node_end_byte(targets); + uint32_t end = ts_node_end_byte(rule); + while (p < end && (src[p] == ' ' || src[p] == '\t')) { + p++; + } + if (p >= end || src[p] != ':') { + return false; + } + while (p < end && src[p] == ':') { + p++; + } + for (bool kw = true; kw;) { + while (p < end && (src[p] == ' ' || src[p] == '\t')) { + p++; + } + kw = false; + for (int i = 0; kws[i]; i++) { + size_t n = strlen(kws[i]); + if (p + n < end && strncmp(src + p, kws[i], n) == 0 && + (src[p + n] == ' ' || src[p + n] == '\t')) { + p += (uint32_t)n; + kw = true; + break; + } + } + } + uint32_t name = p; + while (p < end && make_name_char(src[p])) { + p++; + } + if (p == name) { + return false; + } + while (p < end && (src[p] == ' ' || src[p] == '\t')) { + p++; + } + if (p < end && src[p] == ':') { + while (p < end && src[p] == ':') { + p++; + } + return p < end && src[p] == '='; + } + return p < end && (src[p] == '=' || + (strchr("+?!", src[p]) && p + SKIP_ONE < end && src[p + SKIP_ONE] == '=')); +} + +/* Make: a rule names several targets (`up down: deps`), each a goal + * `make ` runs. The rule's definition carries the first target; + * every further one gets a definition of its own over the same rule span -- + * without fingerprint, structural profile or body tokens, so the copies of + * one recipe do not read as similar code. Special (`.X`) targets and + * repeats stay out. */ +static void push_make_rule_targets(CBMExtractCtx *ctx, TSNode node, const CBMDefinition *first) { + CBMArena *a = ctx->arena; + TSNode targets = cbm_find_child_by_kind(node, "targets"); + uint32_t n = ts_node_is_null(targets) ? 0 : ts_node_named_child_count(targets); + for (uint32_t i = SKIP_ONE; i < n; i++) { + char *name = + cbm_func_name_node_text(a, ts_node_named_child(targets, i), ctx->source, ctx->language); + if (!name || !name[0] || name[0] == '.' || strcmp(name, first->name) == 0) { + continue; + } + bool repeat = false; + for (uint32_t j = SKIP_ONE; j < i && !repeat; j++) { + char *prev = cbm_func_name_node_text(a, ts_node_named_child(targets, j), ctx->source, + ctx->language); + repeat = prev && strcmp(prev, name) == 0; + } + if (repeat) { + continue; + } + CBMDefinition d = *first; + d.name = name; + d.qualified_name = + cbm_fqn_compute_source_lang(a, ctx->project, ctx->rel_path, name, ctx->language); + d.is_exported = cbm_is_exported(name, ctx->language); + d.is_entry_point = strcmp(name, "main") == 0; + d.fingerprint = NULL; + d.fingerprint_k = 0; + d.structural_profile = NULL; + d.body_tokens = NULL; + if (d.qualified_name) { + cbm_defs_push(&ctx->result->defs, a, d); + } + } +} + static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { CBMArena *a = ctx->arena; @@ -6308,6 +6409,12 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec if (ctx->language == CBM_LANG_MAKEFILE && name[0] == '.') { return; } + // `targets: [export] VAR := value` sets a variable for those targets: + // no rule, so no def (else a target only ever configured there is a + // phantom, and a real rule of it gains the line as a variant). + if (ctx->language == CBM_LANG_MAKEFILE && make_rule_sets_variable(node, ctx->source)) { + return; + } /* C++/CUDA: GoogleTest macros (TEST, TEST_F, TEST_P, ...) parse as * function definitions whose name resolves to the bare macro identifier. @@ -6557,6 +6664,9 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec return; } cbm_defs_push(&ctx->result->defs, a, def); + if (ctx->language == CBM_LANG_MAKEFILE && strcmp(ts_node_type(node), "rule") == 0) { + push_make_rule_targets(ctx, node, &def); + } } // --- Class definition extraction --- diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 0875c8912..f3d627cd0 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -48,6 +48,7 @@ static void parse_starlark_imports(CBMExtractCtx *ctx); static void parse_tcl_imports(CBMExtractCtx *ctx); static void parse_teal_imports(CBMExtractCtx *ctx); static void parse_shell_source_imports(CBMExtractCtx *ctx); +static void parse_make_imports(CBMExtractCtx *ctx); static void parse_css_imports(CBMExtractCtx *ctx); static void parse_html_imports(CBMExtractCtx *ctx); static void parse_cmake_imports(CBMExtractCtx *ctx); @@ -2221,6 +2222,52 @@ static void parse_shell_source_imports(CBMExtractCtx *ctx) { } } +// --- Make imports: include / -include / sinclude FILE... --- +// Every name of an include directive -- outside recipes, conditionals +// included -- imports that file; its local name is the file's base name. A +// name holding a variable reference is known only to make and stays out. +// (The generic text fallback kept the directive's trailing newlines in the +// path and split it at the extension's dot.) +static void push_make_include(CBMExtractCtx *ctx, TSNode word) { + CBMArena *a = ctx->arena; + char *path = cbm_node_text(a, word, ctx->source); + if (!path || !path[0] || strchr(path, '$')) { + return; + } + const char *slash = strrchr(path, '/'); + CBMImport imp = {.local_name = slash ? slash + SKIP_ONE : path, .module_path = path}; + cbm_imports_push(&ctx->result->imports, a, imp); +} + +static void parse_make_imports(CBMExtractCtx *ctx) { + TSNodeStack stack; + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); + while (stack.count > 0) { + TSNode node = ts_nstack_pop(&stack); + const char *k = ts_node_type(node); + if (strcmp(k, "recipe") == 0) { + continue; + } + if (strcmp(k, "include_directive") != 0) { + ts_nstack_push_children(&stack, node); + continue; + } + TSNode names = ts_node_child_by_field_name(node, TS_FIELD("filenames")); + if (ts_node_is_null(names)) { + continue; + } + uint32_t n = ts_node_named_child_count(names); + if (strcmp(ts_node_type(names), "list") != 0 || n == 0) { + push_make_include(ctx, names); + continue; + } + for (uint32_t i = 0; i < n; i++) { + push_make_include(ctx, ts_node_named_child(names, i)); + } + } +} + // --- CSS / SCSS imports --- // CSS @import "x.css" and SCSS @use/@forward 'x' are top-level statements that // carry a `string_value` (whose `string_content` is the unquoted path). The @@ -3054,6 +3101,9 @@ void cbm_extract_imports(CBMExtractCtx *ctx) { case CBM_LANG_BASH: parse_shell_source_imports(ctx); break; + case CBM_LANG_MAKEFILE: + parse_make_imports(ctx); + break; case CBM_LANG_ZIG: parse_zig_imports(ctx); break; diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index b74ab402c..1bbc5ea3f 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -3138,10 +3138,25 @@ static int sh_cmp_span(const void *a, const void *b) { return strcmp(x->file, y->file); } +typedef struct { + const cbm_gbuf_node_t **items; + int count; + int cap; +} sh_srcs_t; + +static int sh_cmp_node_qn(const void *a, const void *b) { + const cbm_gbuf_node_t *x = *(const cbm_gbuf_node_t *const *)a; + const cbm_gbuf_node_t *y = *(const cbm_gbuf_node_t *const *)b; + return strcmp(x->qualified_name ? x->qualified_name : "", + y->qualified_name ? y->qualified_name : ""); +} + /* The innermost Function around `line` in `file`, else its Module, else its - * File node. */ + * File node. Into `twins` (when given): every Function of exactly that + * span, in QN order -- the targets of one Make rule share it, and the + * rule's recipe runs for each of them. */ static int64_t sh_source(cbm_pipeline_ctx_t *ctx, const sh_span_t *spans, int n, - CBMHashTable *modules, const char *file, int line) { + CBMHashTable *modules, const char *file, int line, sh_srcs_t *twins) { int lo = 0; int hi = n; while (lo < hi) { @@ -3167,6 +3182,17 @@ static int64_t sh_source(cbm_pipeline_ctx_t *ctx, const sh_span_t *spans, int n, } } if (best) { + for (int i = lo; twins && i < n && strcmp(spans[i].file, file) == 0; i++) { + const cbm_gbuf_node_t *f = spans[i].node; + if (f->start_line == best->start_line && f->end_line == best->end_line && + sh_grow((void **)&twins->items, &twins->cap, twins->count, sizeof(*twins->items))) { + twins->items[twins->count++] = f; + } + } + if (twins && twins->count > SKIP_ONE) { + qsort((void *)twins->items, (size_t)twins->count, sizeof(*twins->items), + sh_cmp_node_qn); + } return best->id; } int64_t mod = (int64_t)(intptr_t)cbm_ht_get(modules, file); @@ -3247,6 +3273,27 @@ static int sh_link_spawn(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x return SKIP_ONE; } +/* The edge hit `x` draws from source `src`; 1 when drawn. */ +static int sh_link_hit(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, int64_t src, + CBMHashTable *modules) { + if (x->spawn) { + return src > 0 ? sh_link_spawn(ctx, src, x, modules) : 0; + } + int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file); + if (src <= 0 || tgt <= 0 || src == tgt) { + return 0; + } + char esc[CBM_SZ_1K]; + char ci[CBM_SZ_1K]; + char props[CBM_SZ_4K]; + cbm_json_escape(esc, (int)sizeof(esc), x->path ? x->path : ""); + sh_ci_props(x, ci, sizeof(ci)); + snprintf(props, sizeof(props), "{\"via\":\"%s\",\"%s\":\"%s\"%s%s,\"line\":%d}", x->via, + x->chart ? "chart" : "path", esc, x->tgt > 0 ? "" : ",\"external\":true", ci, x->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "DEPLOYS", props); + return SKIP_ONE; +} + static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { CBMHashTable *files = cbm_ht_create(0); CBMHashTable *modules = cbm_ht_create(0); @@ -3293,28 +3340,16 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { } } int edges = 0; + sh_srcs_t twins = {0}; for (int i = 0; i < h->count; i++) { const sh_hit_t *x = &h->items[i]; - int64_t src = sh_source(ctx, spans, n, modules, x->file, x->line); - if (x->spawn) { - edges += src > 0 ? sh_link_spawn(ctx, src, x, modules) : 0; - continue; + twins.count = 0; + int64_t one = sh_source(ctx, spans, n, modules, x->file, x->line, &twins); + for (int k = 0; k < (twins.count ? twins.count : SKIP_ONE); k++) { + edges += sh_link_hit(ctx, x, twins.count ? twins.items[k]->id : one, modules); } - int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file); - if (src <= 0 || tgt <= 0 || src == tgt) { - continue; - } - char esc[CBM_SZ_1K]; - char ci[CBM_SZ_1K]; - char props[CBM_SZ_4K]; - cbm_json_escape(esc, (int)sizeof(esc), x->path ? x->path : ""); - sh_ci_props(x, ci, sizeof(ci)); - snprintf(props, sizeof(props), "{\"via\":\"%s\",\"%s\":\"%s\"%s%s,\"line\":%d}", x->via, - x->chart ? "chart" : "path", esc, x->tgt > 0 ? "" : ",\"external\":true", ci, - x->line); - cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "DEPLOYS", props); - edges++; } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)twins.items); cbm_free(CBM_MEM_CLASS_OTHER, spans); cbm_ht_free(files); cbm_ht_free(modules); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index a18b23011..818301477 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -2119,6 +2119,84 @@ TEST(probe_ci_and_scripts_run_scripts) { PASS(); } +/* The Make model of a project: every Function of a Makefile / .mk as + * "fn : -[ +variants]" and every IMPORTS edge of one as + * "import ->