diff --git a/Makefile.cbm b/Makefile.cbm index 0e505cf20..c0e0aa4a0 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -533,10 +533,14 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ - src/pipeline/pass_envscan.c \ src/pipeline/pass_compile_commands.c \ src/pipeline/pass_infrascan.c \ src/pipeline/pass_k8s.c \ + src/pipeline/pass_operator.c \ + src/pipeline/pass_shell.c \ + src/pipeline/pass_terraform.c \ + src/pipeline/pass_docker.c \ + src/pipeline/pass_ci.c \ src/pipeline/pass_similarity.c \ src/pipeline/pass_semantic_edges.c \ src/pipeline/pass_complexity.c \ diff --git a/README.md b/README.md index 795849a20..b87f0244e 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-si - **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search. One graph query replaces dozens of grep/read cycles. - **45 supported automatic/conditional client surfaces** — `install` configures detected clients and safely activates conditional clients only when their documented platform, marker, or explicit existing config path is present. See [Multi-Agent Support](#multi-agent-support) for the complete matrix and manual/UI-only boundaries. - **CBM Atlas** — the built-in human interface at `localhost:9749`: a reading IDE over the index (explorer and read-only Monaco reader, a semantic twin that follows the caret, flow, bug-hunt and change-scope reading modes, the 3D galaxy, source-cited answers, project management), served from the binary itself. -- **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests, and Kustomize overlays indexed as graph nodes with cross-references. `Resource` nodes for K8s kinds, `Module` nodes for Kustomize overlays with `IMPORTS` edges to referenced resources. +- **Infrastructure-as-code indexing** — Dockerfiles, container images, docker compose, Kubernetes manifests, and Kustomize overlays indexed as graph nodes with cross-references. `Resource` nodes for K8s kinds and compose services, `Image` nodes joining the images that are built and run, `Module` nodes for Kustomize overlays with `IMPORTS` edges to referenced resources — a Service walks to the program its container runs. - **17 MCP tools** — search, trace, architecture, impact analysis, targeted index-coverage checks, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. ## Quick Start @@ -248,7 +248,12 @@ The install script placed beside the binary is **reported, not deleted** — uni - `USAGE` — an identifier is used, but a unique callable target is not proven (including ambiguous or complex expressions) - `IMPORTS`, `DEFINES`, `IMPLEMENTS`, `INHERITS` - `HTTP_CALLS`, `ASYNC_CALLS` (cross-service) -- `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, ...); the target is a `Process` node named after the program, or `` when the program is not a literal +- `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, a shell command or Make recipe running a program by path, ...); the target is a `Process` node named after the program, or `` when the program is not a literal. When a shell script, Make recipe, GitHub Actions step or Prow job runs a script of the repository (`./hack/x.sh`, `bash hack/x.sh`), the target is that script's `Module` (else its `File`); `make TARGET` (also `$(MAKE)`, `-C`, `-f`) targets the rule of that target, followed through included makefiles; edges from CI carry `ci` and `job`. A Dockerfile's last stage `ENTRYPOINT` (else `CMD`) spawns what the container runs (`api: dockerfile`): the repository script or source copied into the image, the Go `main` of a binary a stage builds with `go build -o`, else a `Process` +- `INFRA_MAPS` — a Kubernetes Service to each workload whose pod labels carry every `key: value` of the Service's selector, within one namespace +- `USAGE` with `ref: image` — a Kubernetes workload (any pod template: containers, initContainers, CronJobs, custom resources) or a docker compose service to the `Image` it runs +- `DEPLOYS` — what puts manifests, kustomizations or charts on a cluster, to what it puts there: a Flux Kustomization or Argo Application to the kustomizations / manifests / chart of its path, a Flux HelmRelease to its chart (`spec.chart` or `chartRef` → OCIRepository), a shell command or Make recipe (`kubectl apply -f/-k`, `kustomize build | kubectl apply`, `helm install/upgrade/template`), a Go installer loading a chart with the Helm SDK, a Terraform `helm_release`. A chart outside the repository is a shared external `Chart`; the edge says `via` (`flux`, `argo`, `helmrelease`, `kubectl`, `kustomize`, `helm`, `helm-sdk`, `terraform`, ...) +- `DEPENDS_ON` — a Helm chart to the charts it declares (the local chart for a resolvable `file://` repository, otherwise an external `Chart`; the edge keeps `repository`), and a `go.mod` / `requirements.txt` file to an external `Package`; a Dockerfile to the `Image`s it builds on (`via: from`, `copy-from`); an `Image` to the Dockerfile it is built from (`via: build`; `tool`: `skaffold`, `compose`, `docker-build` for `docker` / `podman` / `nerdctl` / `buildah` build commands in scripts, Make and CI); a compose service to the services its `depends_on` / `links` name +- `CONFIGURES` — code reading an environment variable to the shared `EnvVar` of that name (`strategy: env_access`); a Kubernetes container, compose service or Dockerfile (`ENV` of the image) setting one to that container's own `EnvVar` (`strategy: env_set`, with `source`: `value`, `configMapKeyRef`, `secretKeyRef`, `fieldRef`, ..., `compose`, `dockerfile` — values are never stored) - `EMITS`, `LISTENS_ON` (channels) - `DATA_FLOWS` with arg-to-param mapping + field access chains - `SIMILAR_TO` (MinHash + LSH near-clone detection, Jaccard scored) @@ -257,7 +262,7 @@ The install script placed beside the binary is **reported, not deleted** — uni ### Indexing pipeline - **158 vendored tree-sitter grammars** compiled into the binary - **Generic package / module resolution** — bare specifiers like `@myorg/pkg`, `github.com/foo/bar`, `use my_crate::foo` resolved via manifest scanning (`package.json`, `go.mod`, `Cargo.toml`, `pyproject.toml`, `composer.json`, `pubspec.yaml`, `pom.xml`, `build.gradle`, `mix.exs`, `*.gemspec`) -- **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests, Kustomize overlays as graph nodes +- **Infrastructure-as-code indexing** — Dockerfiles (`Dockerfile.*`, `*.Dockerfile`, `Containerfile`: bases, what the container runs, `ENV`), container images (`Image`: the normalised repository, tag and digest on the edges; a repository built from several Dockerfiles, one per component tag, gets an `Image` per component), docker compose services (`Resource`), Kubernetes manifests (`Resource`), Kustomize overlays (`IMPORTS` to their entries), Helm charts (`Chart`, `DEPENDS_ON`), Terraform / OpenTofu (`.tf`, `.tofu`, `.tfvars`) as graph nodes. A Service walks to the code it runs: Service → workload → `Image` → Dockerfile → `main` - **[Hybrid LSP semantic type resolution](#hybrid-lsp)** for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — a lightweight C implementation of language type-resolution algorithms, structurally inspired by and compatible with major language servers including tsserver / typescript-go, pyright, gopls, Roslyn, Eclipse JDT, and rust-analyzer (parameter binding, return-type inference, generic substitution, JSX component dispatch, JSDoc inference for plain JS files, namespace + trait + late-static-binding resolution for PHP, file-scoped namespaces + records + LINQ method syntax for C#, class-hierarchy + overload + lambda resolution for Java, extension-function + scope-function resolution for Kotlin, trait-method + UFCS resolution for Rust) - **RAM-first pipeline**: LZ4 compression, in-memory SQLite, single dump at end. Memory released after. @@ -736,7 +741,7 @@ mode. `index_status` keeps describing the published graph and its freshness. | Tool | Description | |------|-------------| | `search_graph` | Structural, BM25, and semantic search. Page structural rows with `offset`/`limit` and ranked semantic rows independently with `semantic_offset`/`semantic_limit`. | -| `trace_path` | BFS traversal — who calls a function and what it calls (alias: `trace_call_path`). Depth 1-5. | +| `trace_path` | BFS traversal — who calls a function and what it calls (alias: `trace_call_path`). Depth 1-5. `mode`: `calls` (default), `data_flow`, `cross_service`, `topology` (Kubernetes objects, routes, controllers and what they deploy — start at a Resource or Route), `deploy` (what a CI job, script or Make rule runs and deploys — outbound from the job, inbound from a chart), `run` (what a Service or workload runs: workload → `Image` → Dockerfile → the program its container starts, with the environment each container is given — outbound from a Service, workload or compose service). | | `detect_changes` | Map git diff to affected symbols + blast radius with risk classification. | | `query_graph` | Execute Cypher-like graph queries (read-only). | | `get_graph_schema` | Node/edge counts, relationship patterns, property definitions per label. Run this first. | @@ -756,11 +761,11 @@ mode. `index_status` keeps describing the published graph and its freshness. ### Node Labels -`Project`, `Package`, `Folder`, `File`, `Module`, `Class`, `Function`, `Method`, `Interface`, `Enum`, `Type`, `Route`, `Resource` +`Project`, `Package`, `Folder`, `File`, `Module`, `Section`, `Class`, `Struct`, `Interface`, `Trait`, `Enum`, `Type`, `Function`, `Method`, `Field`, `Variable`, `Constant`, `Macro`, `Decorator`, `Route`, `Channel`, `Table`, `Process`, `EnvVar`, `Resource`, `Chart`, `Image`, `Branch` ### Edge Types -`CONTAINS_PACKAGE`, `CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `HTTP_CALLS`, `ASYNC_CALLS`, `SPAWNS`, `IMPLEMENTS`, `HANDLES`, `USAGE`, `CONFIGURES`, `REFERENCES_FILE`, `MENTIONS`, `WRITES`, `MEMBER_OF`, `TESTS`, `USES_TYPE`, `FILE_CHANGES_WITH` +`CONTAINS_FOLDER`, `CONTAINS_FILE`, `DEFINES`, `DEFINES_METHOD`, `IMPORTS`, `CALLS`, `CALL_REFERENCE`, `USAGE`, `INHERITS`, `IMPLEMENTS`, `OVERRIDE`, `DECORATES`, `RAISES`, `THROWS`, `WRITES`, `DATA_FLOWS`, `HTTP_CALLS`, `GRPC_CALLS`, `ASYNC_CALLS`, `EMITS`, `LISTENS_ON`, `HANDLES`, `SPAWNS`, `CONFIGURES`, `INFRA_MAPS`, `DEPENDS_ON`, `REFERENCES_FILE`, `MENTIONS`, `TESTS`, `TESTS_FILE`, `SIMILAR_TO`, `SEMANTICALLY_RELATED`, `FILE_CHANGES_WITH`, `HAS_BRANCH`, and the cross-repo `CROSS_HTTP_CALLS`, `CROSS_GRPC_CALLS`, `CROSS_GRAPHQL_CALLS`, `CROSS_TRPC_CALLS`, `CROSS_ASYNC_CALLS`, `CROSS_CHANNEL` `MENTIONS` links a documented definition to the code its doc comment references (C# ``, ``, ``, ``), one edge per pair with `via`, `syntax`, `tier` (`exact` or `unique`), first `line` and `count`. A reference is bound only when it names exactly one definition under the language's own lookup rules; the rest are counted by reason in `index_status` under `doc_links` (samples with `diagnostics='full'`). diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index d60c9a9bd..c3b1f050e 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -262,6 +262,11 @@ void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBindi arr->items[arr->count++] = ib; } +void cbm_k8sfact_push(CBMK8sFactArray *arr, CBMArena *a, CBMK8sFact f) { + GROW_ARRAY(arr, a); + arr->items[arr->count++] = f; +} + void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it) { GROW_ARRAY(arr, a); arr->items[arr->count++] = it; diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index f5b90a1f5..d1f71f948 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -604,6 +604,54 @@ typedef struct { int cap; } CBMInfraBindingArray; +/* A fact a Kubernetes manifest states about other objects (extract_k8s.c), + * resolved by the K8s pass against the objects of the source's bundle. + * CBM_K8S_FACT_REF: object def_index names a kind/ns/name object at a + * reference path (via), drawn as edge (USAGE, INFRA_MAPS or DEPENDS_ON); + * ns NULL = the source's own namespace. + * CBM_K8S_FACT_CRD: the CRD def_index declares kind `kind` in group `name`. + * CBM_K8S_FACT_DEPLOY: object def_index (a Flux Kustomization, an Argo + * Application) deploys the repository directory `name` into namespace + * `ns` (NULL: the objects' own); via "flux" / "argo". + * CBM_K8S_FACT_ROUTE: object def_index (an Ingress, an HTTPRoute) exposes + * `route` ("") and sends it to the kind/ns/name backend + * (name NULL: none); via "ingress" / "httproute". + * CBM_K8S_FACT_SOURCE: object def_index (a Flux OCIRepository) fetches + * from the URL `name`; a HelmRelease whose chartRef names it deploys the + * chart there. + * CBM_K8S_FACT_IMAGE: object def_index runs a container of image `name` + * (a literal reference); via "container". + * CBM_K8S_FACT_ENV: object def_index sets environment variable `name` in + * its container `container` (NULL: unnamed) at `line`; via = where the + * value comes from ("value", or the valueFrom kind) -- never the value. */ +typedef enum { + CBM_K8S_FACT_REF = 0, + CBM_K8S_FACT_CRD, + CBM_K8S_FACT_DEPLOY, + CBM_K8S_FACT_ROUTE, + CBM_K8S_FACT_SOURCE, + CBM_K8S_FACT_IMAGE, + CBM_K8S_FACT_ENV +} CBMK8sFactKind; +typedef struct { + CBMK8sFactKind fact; + int def_index; + const char *kind; + const char *ns; + const char *name; + const char *via; + const char *edge; + const char *route; + const char *container; + int line; +} CBMK8sFact; + +typedef struct { + CBMK8sFact *items; + int count; + int cap; +} CBMK8sFactArray; + typedef struct { CBMImplTrait *items; int count; @@ -681,6 +729,7 @@ typedef struct CBMFileResult { CBMResolvedCallArray resolved_calls; // LSP-resolved invocations/references (high confidence) CBMStringRefArray string_refs; // URL/config string literals from AST CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs + CBMK8sFactArray k8s_facts; // Kubernetes references and CRD declarations CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation CBMDiagramArrowArray diagram_arrows; // arrows between the file's DiagramElement defs CBMFieldTypeArray field_types; // Python: annotated instance fields (#1277) @@ -1141,6 +1190,7 @@ void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta) void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft); void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); +void cbm_k8sfact_push(CBMK8sFactArray *arr, CBMArena *a, CBMK8sFact f); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc); void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch); diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index e046bca83..e78b2cafe 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -6409,6 +6409,107 @@ void cbm_test_declarations_finish(CBMExtractCtx *ctx) { } } +static bool make_name_char(char c) { + return c == '_' || c == '.' || c == '-' || (c >= '0' && c <= '9') || + ((c | ' ') >= 'a' && (c | ' ') <= 'z'); +} + +/* A `rule` node that is a target-specific variable: after the targets' + * colon(s) come [export|override|private|unexport]... NAME and an + * assignment operator (= := ::= :::= += ?= !=). */ +static bool make_rule_sets_variable(TSNode rule, const char *src) { + static const char *const kws[] = {"export", "override", "private", "unexport", NULL}; + TSNode targets = cbm_find_child_by_kind(rule, "targets"); + if (ts_node_is_null(targets)) { + return false; + } + uint32_t p = ts_node_end_byte(targets); + uint32_t end = ts_node_end_byte(rule); + while (p < end && (src[p] == ' ' || src[p] == '\t')) { + p++; + } + if (p >= end || src[p] != ':') { + return false; + } + while (p < end && src[p] == ':') { + p++; + } + for (bool kw = true; kw;) { + while (p < end && (src[p] == ' ' || src[p] == '\t')) { + p++; + } + kw = false; + for (int i = 0; kws[i]; i++) { + size_t n = strlen(kws[i]); + if (p + n < end && strncmp(src + p, kws[i], n) == 0 && + (src[p + n] == ' ' || src[p + n] == '\t')) { + p += (uint32_t)n; + kw = true; + break; + } + } + } + uint32_t name = p; + while (p < end && make_name_char(src[p])) { + p++; + } + if (p == name) { + return false; + } + while (p < end && (src[p] == ' ' || src[p] == '\t')) { + p++; + } + if (p < end && src[p] == ':') { + while (p < end && src[p] == ':') { + p++; + } + return p < end && src[p] == '='; + } + return p < end && (src[p] == '=' || + (strchr("+?!", src[p]) && p + SKIP_ONE < end && src[p + SKIP_ONE] == '=')); +} + +/* Make: a rule names several targets (`up down: deps`), each a goal + * `make ` runs. The rule's definition carries the first target; + * every further one gets a definition of its own over the same rule span -- + * without fingerprint, structural profile or body tokens, so the copies of + * one recipe do not read as similar code. Special (`.X`) targets and + * repeats stay out. */ +static void push_make_rule_targets(CBMExtractCtx *ctx, TSNode node, const CBMDefinition *first) { + CBMArena *a = ctx->arena; + TSNode targets = cbm_find_child_by_kind(node, "targets"); + uint32_t n = ts_node_is_null(targets) ? 0 : ts_node_named_child_count(targets); + for (uint32_t i = SKIP_ONE; i < n; i++) { + char *name = + cbm_func_name_node_text(a, ts_node_named_child(targets, i), ctx->source, ctx->language); + if (!name || !name[0] || name[0] == '.' || strcmp(name, first->name) == 0) { + continue; + } + bool repeat = false; + for (uint32_t j = SKIP_ONE; j < i && !repeat; j++) { + char *prev = cbm_func_name_node_text(a, ts_node_named_child(targets, j), ctx->source, + ctx->language); + repeat = prev && strcmp(prev, name) == 0; + } + if (repeat) { + continue; + } + CBMDefinition d = *first; + d.name = name; + d.qualified_name = + cbm_fqn_compute_source_lang(a, ctx->project, ctx->rel_path, name, ctx->language); + d.is_exported = cbm_is_exported(name, ctx->language); + d.is_entry_point = strcmp(name, "main") == 0; + d.fingerprint = NULL; + d.fingerprint_k = 0; + d.structural_profile = NULL; + d.body_tokens = NULL; + if (d.qualified_name) { + cbm_defs_push(&ctx->result->defs, a, d); + } + } +} + static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { CBMArena *a = ctx->arena; @@ -6442,6 +6543,12 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec if (ctx->language == CBM_LANG_MAKEFILE && name[0] == '.') { return; } + // `targets: [export] VAR := value` sets a variable for those targets: + // no rule, so no def (else a target only ever configured there is a + // phantom, and a real rule of it gains the line as a variant). + if (ctx->language == CBM_LANG_MAKEFILE && make_rule_sets_variable(node, ctx->source)) { + return; + } /* C++/CUDA: GoogleTest macros (TEST, TEST_F, TEST_P, ...) parse as * function definitions whose name resolves to the bare macro identifier. @@ -6691,6 +6798,9 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec return; } cbm_defs_push(&ctx->result->defs, a, def); + if (ctx->language == CBM_LANG_MAKEFILE && strcmp(ts_node_type(node), "rule") == 0) { + push_make_rule_targets(ctx, node, &def); + } } // --- Class definition extraction --- @@ -7054,6 +7164,19 @@ static char *find_ini_section_name(CBMArena *a, TSNode node, const char *source) return NULL; } +// HCL: a top-level block (resource, data, module, variable, output, locals, +// provider, terraform; Terragrunt's include, dependency, ...) sits directly in +// the file's body. Blocks nested in one (lifecycle, dynamic, content, a +// resource's own argument blocks) are part of it, not entities of their own. +static bool hcl_block_is_top_level(TSNode node) { + TSNode body = ts_node_parent(node); + if (ts_node_is_null(body) || strcmp(ts_node_type(body), "body") != 0) { + return false; + } + TSNode file = ts_node_parent(body); + return !ts_node_is_null(file) && strcmp(ts_node_type(file), "config_file") == 0; +} + // HCL: extract block name from identifier child. static char *find_hcl_block_name(CBMArena *a, TSNode node, const char *source) { TSNode id = cbm_find_child_by_kind(node, "identifier"); @@ -7115,7 +7238,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char // #518: index what the section SAYS, not just its title. docstring = extract_markdown_section_body(ctx, node); } else if (ctx->language == CBM_LANG_HCL && strcmp(kind, "block") == 0) { - name = find_hcl_block_name(a, node, ctx->source); + name = hcl_block_is_top_level(node) ? find_hcl_block_name(a, node, ctx->source) : NULL; } else { return false; } @@ -10153,6 +10276,31 @@ static void extract_yaml_toplevel_keys(CBMExtractCtx *ctx, TSNode root) { } } +// True when the YAML document's top level holds both `apiVersion` and `kind`: +// a Kubernetes object, whose top-level keys are its envelope. +static bool yaml_is_k8s_object(CBMExtractCtx *ctx, TSNode root) { + TSNode bm = find_yaml_toplevel_mapping(root); + if (ts_node_is_null(bm)) { + return false; + } + bool api_version = false; + bool kind = false; + uint32_t n = ts_node_named_child_count(bm); + for (uint32_t i = 0; i < n && !(api_version && kind); i++) { + TSNode pair = ts_node_named_child(bm, i); + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + TSNode key = ts_node_child_by_field_name(pair, TS_FIELD("key")); + const char *k = ts_node_is_null(key) ? NULL : cbm_node_text(ctx->arena, key, ctx->source); + if (k) { + api_version = api_version || strcmp(k, "apiVersion") == 0; + kind = kind || strcmp(k, "kind") == 0; + } + } + return api_version && kind; +} + static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) { if (!spec->variable_node_types || !spec->variable_node_types[0]) { return; @@ -10164,6 +10312,13 @@ static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec return; } + // A Kubernetes object's top-level keys (apiVersion, kind, metadata, spec, + // status) are its envelope, which its Resource node carries; as Variables + // they repeated in every manifest and fed name-similarity config links. + if (ctx->language == CBM_LANG_YAML && yaml_is_k8s_object(ctx, root)) { + return; + } + // Config languages with nested structure: use recursive walk if (ctx->language == CBM_LANG_YAML || ctx->language == CBM_LANG_TOML || ctx->language == CBM_LANG_INI || ctx->language == CBM_LANG_JSON) { @@ -10171,6 +10326,26 @@ static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec return; } + /* HCL: a file's own assignments -- a .tfvars file, Terragrunt's `inputs` + * -- sit in the top-level body, one level below the root the generic loop + * scans, so none was extracted. Attributes inside blocks belong to their + * block. */ + if (ctx->language == CBM_LANG_HCL) { + TSNode body = cbm_find_child_by_kind(root, "body"); + uint32_t n = ts_node_is_null(body) ? 0 : ts_node_named_child_count(body); + for (uint32_t i = 0; i < n; i++) { + TSNode attr = ts_node_named_child(body, i); + if (strcmp(ts_node_type(attr), "attribute") != 0) { + continue; + } + TSNode id = cbm_find_child_by_kind(attr, "identifier"); + if (!ts_node_is_null(id)) { + push_var_def(ctx, cbm_node_text(ctx->arena, id, ctx->source), attr); + } + } + return; + } + /* Nix: the file's top level sits behind its header lambda(s), so the root's * only child is a function_expression and the generic loop below would see * nothing. Resolve past the header to the binding container(s) that actually diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 0875c8912..f3d627cd0 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -48,6 +48,7 @@ static void parse_starlark_imports(CBMExtractCtx *ctx); static void parse_tcl_imports(CBMExtractCtx *ctx); static void parse_teal_imports(CBMExtractCtx *ctx); static void parse_shell_source_imports(CBMExtractCtx *ctx); +static void parse_make_imports(CBMExtractCtx *ctx); static void parse_css_imports(CBMExtractCtx *ctx); static void parse_html_imports(CBMExtractCtx *ctx); static void parse_cmake_imports(CBMExtractCtx *ctx); @@ -2221,6 +2222,52 @@ static void parse_shell_source_imports(CBMExtractCtx *ctx) { } } +// --- Make imports: include / -include / sinclude FILE... --- +// Every name of an include directive -- outside recipes, conditionals +// included -- imports that file; its local name is the file's base name. A +// name holding a variable reference is known only to make and stays out. +// (The generic text fallback kept the directive's trailing newlines in the +// path and split it at the extension's dot.) +static void push_make_include(CBMExtractCtx *ctx, TSNode word) { + CBMArena *a = ctx->arena; + char *path = cbm_node_text(a, word, ctx->source); + if (!path || !path[0] || strchr(path, '$')) { + return; + } + const char *slash = strrchr(path, '/'); + CBMImport imp = {.local_name = slash ? slash + SKIP_ONE : path, .module_path = path}; + cbm_imports_push(&ctx->result->imports, a, imp); +} + +static void parse_make_imports(CBMExtractCtx *ctx) { + TSNodeStack stack; + ts_nstack_init(&stack, ctx, CBM_SZ_512); + ts_nstack_push(&stack, ctx->root); + while (stack.count > 0) { + TSNode node = ts_nstack_pop(&stack); + const char *k = ts_node_type(node); + if (strcmp(k, "recipe") == 0) { + continue; + } + if (strcmp(k, "include_directive") != 0) { + ts_nstack_push_children(&stack, node); + continue; + } + TSNode names = ts_node_child_by_field_name(node, TS_FIELD("filenames")); + if (ts_node_is_null(names)) { + continue; + } + uint32_t n = ts_node_named_child_count(names); + if (strcmp(ts_node_type(names), "list") != 0 || n == 0) { + push_make_include(ctx, names); + continue; + } + for (uint32_t i = 0; i < n; i++) { + push_make_include(ctx, ts_node_named_child(names, i)); + } + } +} + // --- CSS / SCSS imports --- // CSS @import "x.css" and SCSS @use/@forward 'x' are top-level statements that // carry a `string_value` (whose `string_content` is the unquoted path). The @@ -3054,6 +3101,9 @@ void cbm_extract_imports(CBMExtractCtx *ctx) { case CBM_LANG_BASH: parse_shell_source_imports(ctx); break; + case CBM_LANG_MAKEFILE: + parse_make_imports(ctx); + break; case CBM_LANG_ZIG: parse_zig_imports(ctx); break; diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 0396ee0e4..e6602c3f2 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -4,9 +4,11 @@ // matches "resources", "bases", "patches", "components", or // "patchesStrategicMerge", then emits one CBMImport per block_sequence item. // -// For CBM_LANG_K8S: finds apiVersion, kind, and metadata.name scalars in the -// first document's block_mapping and emits one CBMDefinition with label -// "Resource" and name "Kind/metadata-name". +// For CBM_LANG_K8S: reads apiVersion, kind, metadata.name and +// metadata.namespace of EVERY document and emits one CBMDefinition per object: +// label "Resource", name "Kind/metadata-name", QN +// "::///" (identity +// spec: the file base keeps the extension, so it is unique per file). #include "cbm.h" #include "arena.h" @@ -20,24 +22,84 @@ /* Local constants. */ enum { K8S_BUF_SIZE = 256, - RESULT_BUF_SIZE = 512, }; // --------------------------------------------------------------------------- // Internal helpers // --------------------------------------------------------------------------- +// The children of `node` (named only when `named`), in order, through a +// tree cursor: ts_node_child(n, i) / ts_node_named_child(n, i) walk from the +// first child on every call, so an index loop over a wide node (a list of +// 100k items, a file of thousands of documents) is quadratic. The cursor is +// released when the children are exhausted, or by k8s_kids_done. +typedef struct { + TSTreeCursor cur; + bool live; +} k8s_kids_t; + +static k8s_kids_t k8s_kids(TSNode node) { + k8s_kids_t k = {.live = false}; + if (ts_node_is_null(node)) { + return k; + } + k.cur = ts_tree_cursor_new(node); + k.live = ts_tree_cursor_goto_first_child(&k.cur); + if (!k.live) { + ts_tree_cursor_delete(&k.cur); + } + return k; +} + +static bool k8s_kid(k8s_kids_t *k, bool named, TSNode *out) { + while (k->live) { + TSNode n = ts_tree_cursor_current_node(&k->cur); + if (!ts_tree_cursor_goto_next_sibling(&k->cur)) { + ts_tree_cursor_delete(&k->cur); + k->live = false; + } + if (!named || ts_node_is_named(n)) { + *out = n; + return true; + } + } + return false; +} + +static void k8s_kids_done(k8s_kids_t *k) { + if (k->live) { + ts_tree_cursor_delete(&k->cur); + k->live = false; + } +} + +// The value inside a flow_node: its first named child that is not an anchor +// (`&name`), a tag (`!!str`) or a comment, all of which precede the value. +static TSNode flow_node_value(TSNode node) { + uint32_t n = ts_node_named_child_count(node); + for (uint32_t i = 0; i < n; i++) { + TSNode child = ts_node_named_child(node, i); + const char *type = ts_node_type(child); + if (strcmp(type, "anchor") != 0 && strcmp(type, "tag") != 0 && + strcmp(type, "comment") != 0) { + return child; + } + } + TSNode null_node = {0}; + return null_node; +} + // Return the raw source text for a scalar node (plain, single-quoted, or // double-quoted). Surrounding quote characters are stripped for quoted forms. -// Handles flow_node wrappers transparently by descending into the first named -// child (the tree-sitter YAML grammar often wraps scalars in flow_node). +// Handles flow_node wrappers transparently by descending into the value they +// wrap (the tree-sitter YAML grammar often wraps scalars in flow_node). // Returns NULL for non-scalar node types. static const char *get_scalar_text(CBMArena *a, TSNode node, const char *source) { enum { MAX_UNWRAP = 4 }; for (int depth = 0; depth < MAX_UNWRAP; depth++) { const char *type = ts_node_type(node); if (strcmp(type, "flow_node") == 0) { - TSNode inner = ts_node_named_child(node, 0); + TSNode inner = flow_node_value(node); if (ts_node_is_null(inner)) { return NULL; } @@ -79,16 +141,16 @@ static int is_kustomize_list_key(const char *key) { // scalar child, using key_name as the local_name. static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, const char *key_name) { CBMArena *a = ctx->arena; - uint32_t n = ts_node_child_count(seq_node); - for (uint32_t i = 0; i < n; i++) { - TSNode item = ts_node_child(seq_node, i); + k8s_kids_t items = k8s_kids(seq_node); + TSNode item; + while (k8s_kid(&items, false, &item)) { if (strcmp(ts_node_type(item), "block_sequence_item") != 0) { continue; } // block_sequence_item has one named child: the value - uint32_t ic = ts_node_child_count(item); - for (uint32_t j = 0; j < ic; j++) { - TSNode val = ts_node_child(item, j); + k8s_kids_t vals = k8s_kids(item); + TSNode val; + while (k8s_kid(&vals, false, &val)) { const char *scalar = get_scalar_text(a, val, ctx->source); if (!scalar) { continue; @@ -102,15 +164,30 @@ static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, const c } } +// First named child that is not a comment. A comment between `---` and the +// mapping (the `# yaml-language-server: $schema=...` line editors add) is the +// document's first named child; the content follows it. +static TSNode first_content_child(TSNode node) { + uint32_t n = ts_node_named_child_count(node); + for (uint32_t i = 0; i < n; i++) { + TSNode child = ts_node_named_child(node, i); + if (strcmp(ts_node_type(child), "comment") != 0) { + return child; + } + } + TSNode null_node = {0}; + return null_node; +} + // Unwrap a YAML node through optional block_node wrapper to get block_mapping. // Returns null node if not a block_mapping. static TSNode unwrap_block_mapping(TSNode doc_child) { - TSNode mapping = ts_node_named_child(doc_child, 0); + TSNode mapping = first_content_child(doc_child); if (ts_node_is_null(mapping)) { return mapping; } if (strcmp(ts_node_type(mapping), "block_node") == 0) { - mapping = ts_node_named_child(mapping, 0); + mapping = first_content_child(mapping); } if (ts_node_is_null(mapping) || strcmp(ts_node_type(mapping), "block_mapping") != 0) { TSNode null_node = {0}; @@ -119,6 +196,27 @@ static TSNode unwrap_block_mapping(TSNode doc_child) { return mapping; } +// Unwrap a block_mapping_pair value through optional block_node. Comments are +// skipped on both levels: one trailing the key line (`metadata: # note`) is a +// named child of the pair ahead of its value, and a commented-out first entry +// (`resources:` / ` # - a.yaml` / ` - b.yaml`) is the block_node's first +// named child, ahead of the sequence. +static TSNode unwrap_pair_value(TSNode pair) { + TSNode val_node = {0}; + uint32_t n = ts_node_named_child_count(pair); + for (uint32_t i = SKIP_ONE; i < n; i++) { + TSNode child = ts_node_named_child(pair, i); + if (strcmp(ts_node_type(child), "comment") != 0) { + val_node = child; + break; + } + } + if (!ts_node_is_null(val_node) && strcmp(ts_node_type(val_node), "block_node") == 0) { + val_node = first_content_child(val_node); + } + return val_node; +} + // Process a single block_mapping_pair for kustomize list keys. static void process_kustomize_pair(CBMExtractCtx *ctx, TSNode pair) { if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { @@ -133,30 +231,21 @@ static void process_kustomize_pair(CBMExtractCtx *ctx, TSNode pair) { return; } - TSNode val_node = ts_node_named_child(pair, SKIP_ONE); - if (ts_node_is_null(val_node)) { - return; - } - if (strcmp(ts_node_type(val_node), "block_node") == 0) { - val_node = ts_node_named_child(val_node, 0); - } + TSNode val_node = unwrap_pair_value(pair); if (ts_node_is_null(val_node) || strcmp(ts_node_type(val_node), "block_sequence") != 0) { return; } emit_kustomize_sequence(ctx, val_node, key_text); } -// Forward declaration: defined with the K8s-manifest helpers below. -static TSNode unwrap_pair_value(TSNode pair); - // Emit a "Class" def named after the document's `kind` scalar. A kustomization // file has no metadata.name, so the def name is the bare kind ("Kustomization"). // Mirrors the K8s manifest kind-def so Kustomize resources are also discoverable. static void emit_kustomize_kind_def(CBMExtractCtx *ctx, TSNode mapping) { CBMArena *a = ctx->arena; - uint32_t pair_n = ts_node_child_count(mapping); - for (uint32_t pi = 0; pi < pair_n; pi++) { - TSNode pair = ts_node_child(mapping, pi); + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { continue; } @@ -184,15 +273,15 @@ static void emit_kustomize_kind_def(CBMExtractCtx *ctx, TSNode mapping) { def.start_line = ts_node_start_point(mapping).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(mapping).row + TS_LINE_OFFSET; cbm_defs_push(&ctx->result->defs, a, def); + k8s_kids_done(&pairs); return; } } static void extract_kustomize(CBMExtractCtx *ctx) { - TSNode root = ctx->root; - uint32_t root_n = ts_node_child_count(root); - for (uint32_t si = 0; si < root_n; si++) { - TSNode stream_child = ts_node_child(root, si); + k8s_kids_t docs = k8s_kids(ctx->root); + TSNode stream_child; + while (k8s_kid(&docs, false, &stream_child)) { if (strcmp(ts_node_type(stream_child), "document") != 0) { continue; } @@ -203,9 +292,10 @@ static void extract_kustomize(CBMExtractCtx *ctx) { emit_kustomize_kind_def(ctx, mapping); - uint32_t pair_n = ts_node_child_count(mapping); - for (uint32_t pi = 0; pi < pair_n; pi++) { - process_kustomize_pair(ctx, ts_node_child(mapping, pi)); + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { + process_kustomize_pair(ctx, pair); } } } @@ -214,15 +304,23 @@ static void extract_kustomize(CBMExtractCtx *ctx) { // K8s manifest extraction // --------------------------------------------------------------------------- -// Extract the "name" scalar from a metadata block_mapping. -static void extract_metadata_name(CBMArena *a, TSNode meta_mapping, const char *source, - char *meta_name_buf, size_t meta_sz) { +// The identity scalars of one manifest document. +typedef struct { + char api_version[K8S_BUF_SIZE]; + char kind[K8S_BUF_SIZE]; + char name[K8S_BUF_SIZE]; + char ns[K8S_BUF_SIZE]; +} k8s_doc_id_t; + +// Read metadata.name and metadata.namespace from a metadata block_mapping. +static void extract_metadata(CBMArena *a, TSNode meta_mapping, const char *source, + k8s_doc_id_t *id) { if (ts_node_is_null(meta_mapping) || strcmp(ts_node_type(meta_mapping), "block_mapping") != 0) { return; } - uint32_t mn = ts_node_child_count(meta_mapping); - for (uint32_t mi = 0; mi < mn; mi++) { - TSNode mpair = ts_node_child(meta_mapping, mi); + k8s_kids_t pairs = k8s_kids(meta_mapping); + TSNode mpair; + while (k8s_kid(&pairs, false, &mpair)) { if (strcmp(ts_node_type(mpair), "block_mapping_pair") != 0) { continue; } @@ -231,43 +329,25 @@ static void extract_metadata_name(CBMArena *a, TSNode meta_mapping, const char * continue; } const char *mkey_text = get_scalar_text(a, mkey, source); - if (!mkey_text || strcmp(mkey_text, "name") != 0) { + bool is_name = mkey_text && strcmp(mkey_text, "name") == 0; + bool is_ns = mkey_text && strcmp(mkey_text, "namespace") == 0; + if (!is_name && !is_ns) { continue; } - TSNode mval = ts_node_named_child(mpair, SKIP_ONE); - if (ts_node_is_null(mval)) { - continue; - } - const char *meta_name = get_scalar_text(a, mval, source); - if (meta_name) { - snprintf(meta_name_buf, meta_sz, "%s", meta_name); + TSNode mval = unwrap_pair_value(mpair); + const char *text = ts_node_is_null(mval) ? NULL : get_scalar_text(a, mval, source); + if (text) { + snprintf(is_name ? id->name : id->ns, K8S_BUF_SIZE, "%s", text); } } } -// Unwrap a block_mapping_pair value through optional block_node. -static TSNode unwrap_pair_value(TSNode pair) { - TSNode val_node = ts_node_named_child(pair, SKIP_ONE); - if (ts_node_is_null(val_node)) { - return val_node; - } - if (strcmp(ts_node_type(val_node), "block_node") == 0) { - val_node = ts_node_named_child(val_node, 0); - } - return val_node; -} - -// Descend into the first block_mapping of a document and extract -// kind and metadata.name. Returns void; fills kind_buf and meta_name_buf. -static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, char *kind_buf, size_t kind_sz, - char *meta_name_buf, size_t meta_sz) { +// Read apiVersion, kind and metadata from a document's top-level mapping. +static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, k8s_doc_id_t *id) { CBMArena *a = ctx->arena; - kind_buf[0] = '\0'; - meta_name_buf[0] = '\0'; - - uint32_t n = ts_node_child_count(mapping); - for (uint32_t i = 0; i < n; i++) { - TSNode pair = ts_node_child(mapping, i); + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { continue; } @@ -279,61 +359,972 @@ static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, char *kind_b if (!key) { continue; } - TSNode val_node = unwrap_pair_value(pair); if (ts_node_is_null(val_node)) { continue; } - - if (strcmp(key, "kind") == 0) { - const char *v = get_scalar_text(a, val_node, ctx->source); - if (v) { - snprintf(kind_buf, kind_sz, "%s", v); - } - } else if (strcmp(key, "metadata") == 0) { - extract_metadata_name(a, val_node, ctx->source, meta_name_buf, meta_sz); + if (strcmp(key, "metadata") == 0) { + extract_metadata(a, val_node, ctx->source, id); + continue; + } + char *dst = strcmp(key, "kind") == 0 ? id->kind + : strcmp(key, "apiVersion") == 0 ? id->api_version + : NULL; + const char *v = dst ? get_scalar_text(a, val_node, ctx->source) : NULL; + if (v) { + snprintf(dst, K8S_BUF_SIZE, "%s", v); } } } +// --------------------------------------------------------------------------- +// Object references (PHASE1.md): facts the K8s pass resolves per bundle +// --------------------------------------------------------------------------- + +// A reference held by a scalar at a path, relative to the pod spec, to a +// container, or (kinds) to the whole object. +typedef struct { + const char *path; + const char *kind; + const char *via; +} k8s_leaf_ref_t; + +typedef struct { + const char *kinds; // "|A|B|" of the source kinds + const char *path; + const char *kind; + const char *via; + const char *edge; +} k8s_object_ref_t; + +// A reference held by a mapping's kind/name/namespace fields. +typedef struct { + const char *kinds; // "|A|B|" of the source kinds + const char *path; // the mapping's path + const char *kind; // the target kind; NULL = the mapping's own `kind` field + const char *allowed; // with kind NULL: "|A|" of the kinds that are objects + const char *dflt; // with kind NULL: the kind when the field is absent + const char *via; + const char *edge; +} k8s_mapping_ref_t; + +static const k8s_leaf_ref_t k8s_pod_refs[] = { + {"serviceAccountName", "ServiceAccount", "serviceAccountName"}, + {"serviceAccount", "ServiceAccount", "serviceAccountName"}, + {"priorityClassName", "PriorityClass", "priorityClassName"}, + {"imagePullSecrets[].name", "Secret", "imagePullSecrets"}, + {"volumes[].configMap.name", "ConfigMap", "volume"}, + {"volumes[].secret.secretName", "Secret", "volume"}, + {"volumes[].persistentVolumeClaim.claimName", "PersistentVolumeClaim", "volume"}, + {"volumes[].projected.sources[].configMap.name", "ConfigMap", "volume"}, + {"volumes[].projected.sources[].secret.name", "Secret", "volume"}, +}; + +static const k8s_leaf_ref_t k8s_container_refs[] = { + {"env[].valueFrom.configMapKeyRef.name", "ConfigMap", "env"}, + {"env[].valueFrom.secretKeyRef.name", "Secret", "env"}, + {"envFrom[].configMapRef.name", "ConfigMap", "envFrom"}, + {"envFrom[].secretRef.name", "Secret", "envFrom"}, +}; + +static const k8s_object_ref_t k8s_object_refs[] = { + {"|Ingress|", "spec.rules[].http.paths[].backend.service.name", "Service", "ingress", + "INFRA_MAPS"}, + {"|Ingress|", "spec.defaultBackend.service.name", "Service", "ingress", "INFRA_MAPS"}, + {"|Ingress|", "spec.tls[].secretName", "Secret", "tls", "USAGE"}, + {"|Ingress|", "spec.ingressClassName", "IngressClass", "ingressClassName", "USAGE"}, + {"|StatefulSet|", "spec.serviceName", "Service", "serviceName", "USAGE"}, + {"|PersistentVolumeClaim|", "spec.storageClassName", "StorageClass", "storageClassName", + "USAGE"}, +}; + +static const k8s_mapping_ref_t k8s_mapping_refs[] = { + {"|RoleBinding|ClusterRoleBinding|", "roleRef", NULL, "|Role|ClusterRole|", NULL, "roleRef", + "USAGE"}, + {"|RoleBinding|ClusterRoleBinding|", "subjects[]", NULL, "|ServiceAccount|", NULL, "subject", + "USAGE"}, + {"|HorizontalPodAutoscaler|", "spec.scaleTargetRef", NULL, NULL, NULL, "scaleTarget", "USAGE"}, + {"|ValidatingWebhookConfiguration|MutatingWebhookConfiguration|", + "webhooks[].clientConfig.service", "Service", NULL, NULL, "webhook", "INFRA_MAPS"}, + {"|APIService|", "spec.service", "Service", NULL, NULL, "apiservice", "INFRA_MAPS"}, + {"|HTTPRoute|GRPCRoute|TCPRoute|TLSRoute|UDPRoute|", "spec.rules[].backendRefs[]", NULL, NULL, + "Service", "route", "INFRA_MAPS"}, + {"|HTTPRoute|GRPCRoute|TCPRoute|TLSRoute|UDPRoute|", "spec.parentRefs[]", NULL, NULL, "Gateway", + "parentRef", "USAGE"}, +}; + +// The curated second table: the CRDs of Flux, Argo, Cluster API, +// cert-manager and Kubermatic, written from their API types. Source kinds +// are qualified by API group ("|g|" list), and a path may use `*`, which +// matches within one segment (`spec.cloud.*.credentialsReference`). +typedef struct { + const char *groups; + const char *kinds; + const char *path; + const char *kind; + const char *via; + const char *edge; +} k8s_crd_leaf_ref_t; + +typedef struct { + const char *groups; + const char *kinds; + const char *path; + const char *kind; // the target kind; NULL = the mapping's own `kind` field + const char *allowed; // with kind NULL: "|A|" of the kinds that are objects (NULL: any) + const char *dflt; // with kind NULL: the kind when the field is absent + const char *via; + const char *edge; +} k8s_crd_mapping_ref_t; + +#define K8S_FLUX_KS "|kustomize.toolkit.fluxcd.io|" +#define K8S_FLUX_HELM "|helm.toolkit.fluxcd.io|" +#define K8S_FLUX_SRC "|source.toolkit.fluxcd.io|" +#define K8S_FLUX_NOTE "|notification.toolkit.fluxcd.io|" +#define K8S_FLUX_IMAGE "|image.toolkit.fluxcd.io|" +#define K8S_ARGO "|argoproj.io|" +#define K8S_CAPI "|cluster.x-k8s.io|" +#define K8S_CAPI_BOOT "|bootstrap.cluster.x-k8s.io|" +#define K8S_CAPI_CP "|controlplane.cluster.x-k8s.io|" +#define K8S_CAPI_ADDONS "|addons.cluster.x-k8s.io|" +#define K8S_CM "|cert-manager.io|" +#define K8S_KKP "|kubermatic.k8c.io|" +#define K8S_KKP_APPS "|apps.kubermatic.k8c.io|" +#define K8S_FLUX_SOURCES "|GitRepository|OCIRepository|HelmRepository|Bucket|" +#define K8S_ISSUERS "|Issuer|ClusterIssuer|" + +static const k8s_crd_leaf_ref_t k8s_crd_leaf_refs[] = { + {K8S_FLUX_KS, "|Kustomization|", "spec.decryption.secretRef.name", "Secret", "secretRef", + "USAGE"}, + {K8S_FLUX_KS "helm.toolkit.fluxcd.io|", "|Kustomization|HelmRelease|", + "spec.kubeConfig.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_KS "helm.toolkit.fluxcd.io|", "|Kustomization|HelmRelease|", + "spec.serviceAccountName", "ServiceAccount", "serviceAccountName", "USAGE"}, + {K8S_FLUX_SRC, K8S_FLUX_SOURCES, "spec.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_SRC, K8S_FLUX_SOURCES, "spec.certSecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_SRC, K8S_FLUX_SOURCES, "spec.proxySecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_SRC, "|GitRepository|OCIRepository|", "spec.verify.secretRef.name", "Secret", + "secretRef", "USAGE"}, + {K8S_FLUX_SRC, "|OCIRepository|HelmRepository|", "spec.serviceAccountName", "ServiceAccount", + "serviceAccountName", "USAGE"}, + {K8S_FLUX_NOTE, "|Alert|", "spec.providerRef.name", "Provider", "providerRef", "USAGE"}, + {K8S_FLUX_NOTE, "|Provider|Receiver|", "spec.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_NOTE, "|Provider|", "spec.certSecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_IMAGE, "|ImagePolicy|", "spec.imageRepositoryRef.name", "ImageRepository", + "imageRepositoryRef", "USAGE"}, + {K8S_FLUX_IMAGE, "|ImageRepository|", "spec.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_FLUX_IMAGE, "|ImageRepository|", "spec.certSecretRef.name", "Secret", "secretRef", + "USAGE"}, + {K8S_FLUX_IMAGE, "|ImageRepository|", "spec.serviceAccountName", "ServiceAccount", + "serviceAccountName", "USAGE"}, + {K8S_ARGO, "|Application|", "spec.project", "AppProject", "project", "USAGE"}, + {K8S_ARGO, "|ApplicationSet|", "spec.template.spec.project", "AppProject", "project", "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.canary.canaryService", "Service", "rolloutService", + "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.canary.stableService", "Service", "rolloutService", + "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.blueGreen.activeService", "Service", "rolloutService", + "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.blueGreen.previewService", "Service", "rolloutService", + "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.canary.analysis.templates[].templateName", + "AnalysisTemplate", "analysis", "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.strategy.canary.steps[].analysis.templates[].templateName", + "AnalysisTemplate", "analysis", "USAGE"}, + {K8S_ARGO, "|Workflow|", "spec.workflowTemplateRef.name", "WorkflowTemplate", + "workflowTemplateRef", "USAGE"}, + {K8S_ARGO, "|CronWorkflow|", "spec.workflowSpec.workflowTemplateRef.name", "WorkflowTemplate", + "workflowTemplateRef", "USAGE"}, + {K8S_ARGO, "|Workflow|WorkflowTemplate|", "spec.templates[].steps[][].templateRef.name", + "WorkflowTemplate", "templateRef", "USAGE"}, + {K8S_ARGO, "|Workflow|WorkflowTemplate|", "spec.templates[].dag.tasks[].templateRef.name", + "WorkflowTemplate", "templateRef", "USAGE"}, + {K8S_CAPI, "|Cluster|", "spec.topology.class", "ClusterClass", "topologyClass", "USAGE"}, + {K8S_CAPI, "|Machine|MachineDeployment|MachineSet|MachinePool|MachineHealthCheck|", + "spec.clusterName", "Cluster", "clusterName", "USAGE"}, + {K8S_CAPI, "|Machine|", "spec.bootstrap.dataSecretName", "Secret", "dataSecretName", "USAGE"}, + {K8S_CAPI, "|MachineDeployment|MachineSet|MachinePool|", + "spec.template.spec.bootstrap.dataSecretName", "Secret", "dataSecretName", "USAGE"}, + {K8S_CAPI_BOOT, "|KubeadmConfig|", "spec.files[].contentFrom.secret.name", "Secret", "files", + "USAGE"}, + {K8S_CAPI_BOOT, "|KubeadmConfigTemplate|", "spec.template.spec.files[].contentFrom.secret.name", + "Secret", "files", "USAGE"}, + {K8S_CAPI_CP, "|KubeadmControlPlane|", "spec.kubeadmConfigSpec.files[].contentFrom.secret.name", + "Secret", "files", "USAGE"}, + {K8S_CAPI_CP, "|KubeadmControlPlaneTemplate|", + "spec.template.spec.kubeadmConfigSpec.files[].contentFrom.secret.name", "Secret", "files", + "USAGE"}, + {K8S_CM, "|Certificate|", "spec.secretName", "Secret", "secretName", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.ca.secretName", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.acme.privateKeySecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.acme.externalAccountBinding.keySecretRef.name", "Secret", + "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.acme.solvers[].dns01.*.*SecretRef.name", "Secret", "secretRef", + "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.vault.auth.tokenSecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.vault.auth.*.secretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.vault.caBundleSecretRef.name", "Secret", "secretRef", "USAGE"}, + {K8S_CM, K8S_ISSUERS, "spec.venafi.*.*Ref.name", "Secret", "secretRef", "USAGE"}, + {"|networking.k8s.io|extensions|gateway.networking.k8s.io|", "|Ingress|Gateway|", + "metadata.annotations.cert-manager.io/cluster-issuer", "ClusterIssuer", "issuer", "USAGE"}, + {"|networking.k8s.io|extensions|gateway.networking.k8s.io|", "|Ingress|Gateway|", + "metadata.annotations.cert-manager.io/issuer", "Issuer", "issuer", "USAGE"}, + {K8S_KKP_APPS, "|ApplicationInstallation|", "spec.applicationRef.name", "ApplicationDefinition", + "applicationRef", "USAGE"}, + {K8S_KKP_APPS, "|ApplicationDefinition|", + "spec.versions[].template.source.*.credentials.*.name", "Secret", "credentials", "USAGE"}, +}; + +static const k8s_crd_mapping_ref_t k8s_crd_mapping_refs[] = { + {K8S_FLUX_KS, "|Kustomization|", "spec.sourceRef", NULL, NULL, NULL, "sourceRef", "USAGE"}, + {K8S_FLUX_KS, "|Kustomization|", "spec.dependsOn[]", "Kustomization", NULL, NULL, "dependsOn", + "DEPENDS_ON"}, + {K8S_FLUX_KS, "|Kustomization|", "spec.postBuild.substituteFrom[]", NULL, "|ConfigMap|Secret|", + NULL, "substituteFrom", "USAGE"}, + {K8S_FLUX_KS, "|Kustomization|", "spec.healthChecks[]", NULL, NULL, NULL, "healthCheck", + "USAGE"}, + {K8S_FLUX_HELM, "|HelmRelease|", "spec.chart.spec.sourceRef", NULL, NULL, NULL, "sourceRef", + "USAGE"}, + {K8S_FLUX_HELM, "|HelmRelease|", "spec.chartRef", NULL, NULL, NULL, "chartRef", "USAGE"}, + {K8S_FLUX_HELM, "|HelmRelease|", "spec.dependsOn[]", "HelmRelease", NULL, NULL, "dependsOn", + "DEPENDS_ON"}, + {K8S_FLUX_HELM, "|HelmRelease|", "spec.valuesFrom[]", NULL, "|ConfigMap|Secret|", NULL, + "valuesFrom", "USAGE"}, + {K8S_FLUX_SRC, "|HelmChart|", "spec.sourceRef", NULL, NULL, NULL, "sourceRef", "USAGE"}, + {K8S_FLUX_SRC, "|GitRepository|", "spec.include[].repository", "GitRepository", NULL, NULL, + "include", "USAGE"}, + {K8S_FLUX_NOTE, "|Alert|", "spec.eventSources[]", NULL, NULL, NULL, "eventSource", "USAGE"}, + {K8S_FLUX_NOTE, "|Receiver|", "spec.resources[]", NULL, NULL, NULL, "resource", "USAGE"}, + {K8S_FLUX_IMAGE, "|ImageUpdateAutomation|", "spec.sourceRef", NULL, NULL, "GitRepository", + "sourceRef", "USAGE"}, + {K8S_ARGO, "|Rollout|", "spec.workloadRef", NULL, NULL, NULL, "workloadRef", "USAGE"}, + {K8S_CAPI, "|Cluster|", "spec.infrastructureRef", NULL, NULL, NULL, "infrastructureRef", + "USAGE"}, + {K8S_CAPI, "|Cluster|", "spec.controlPlaneRef", NULL, NULL, NULL, "controlPlaneRef", "USAGE"}, + {K8S_CAPI, "|Cluster|", "spec.topology.classRef", "ClusterClass", NULL, NULL, "topologyClass", + "USAGE"}, + {K8S_CAPI, "|Machine|", "spec.bootstrap.configRef", NULL, NULL, NULL, "bootstrap", "USAGE"}, + {K8S_CAPI, "|Machine|", "spec.infrastructureRef", NULL, NULL, NULL, "infrastructureRef", + "USAGE"}, + {K8S_CAPI, "|MachineDeployment|MachineSet|MachinePool|", + "spec.template.spec.bootstrap.configRef", NULL, NULL, NULL, "bootstrap", "USAGE"}, + {K8S_CAPI, "|MachineDeployment|MachineSet|MachinePool|", "spec.template.spec.infrastructureRef", + NULL, NULL, NULL, "infrastructureRef", "USAGE"}, + {K8S_CAPI_CP, "|KubeadmControlPlane|", "spec.machineTemplate.infrastructureRef", NULL, NULL, + NULL, "infrastructureRef", "USAGE"}, + {K8S_CAPI_CP, "|KubeadmControlPlane|", "spec.machineTemplate.spec.infrastructureRef", NULL, + NULL, NULL, "infrastructureRef", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.infrastructure.ref", NULL, NULL, NULL, "template", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.infrastructure.templateRef", NULL, NULL, NULL, "template", + "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.controlPlane.ref", NULL, NULL, NULL, "template", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.controlPlane.templateRef", NULL, NULL, NULL, "template", + "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.controlPlane.machineInfrastructure.ref", NULL, NULL, NULL, + "template", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.controlPlane.machineInfrastructure.templateRef", NULL, NULL, + NULL, "template", "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.workers.*[].template.*.ref", NULL, NULL, NULL, "template", + "USAGE"}, + {K8S_CAPI, "|ClusterClass|", "spec.workers.*[].*.templateRef", NULL, NULL, NULL, "template", + "USAGE"}, + {K8S_CAPI_ADDONS, "|ClusterResourceSet|", "spec.resources[]", NULL, "|ConfigMap|Secret|", NULL, + "resources", "USAGE"}, + {K8S_CM, "|Certificate|CertificateRequest|", "spec.issuerRef", NULL, NULL, "Issuer", + "issuerRef", "USAGE"}, + {K8S_KKP, "|KubermaticConfiguration|", "spec.ingress.certificateIssuer", NULL, NULL, NULL, + "certificateIssuer", "USAGE"}, + {K8S_KKP, "|Seed|", "spec.kubeconfig", "Secret", NULL, NULL, "kubeconfig", "USAGE"}, + {K8S_KKP, "|Seed|", "spec.etcdBackupRestore.destinations.*.credentials", "Secret", NULL, NULL, + "credentials", "USAGE"}, + {K8S_KKP, "|Cluster|", "spec.cloud.*.credentialsReference", "Secret", NULL, NULL, "credentials", + "USAGE"}, + {K8S_KKP, "|Addon|EtcdBackupConfig|EtcdRestore|", "spec.cluster", "Cluster", NULL, NULL, + "cluster", "USAGE"}, +}; + +// Does `path` match `pat`, where `*` matches any run within one segment? +static bool k8s_path_match(const char *pat, const char *path) { + const char *star = NULL; + const char *mark = NULL; + while (*path) { + if (*pat == '*') { + star = pat++; + mark = path; + } else if (*pat && *pat == *path) { + pat++; + path++; + } else if (star && *mark != '.') { + pat = star + SKIP_ONE; + path = ++mark; + } else { + return false; + } + } + while (*pat == '*') { + pat++; + } + return *pat == '\0'; +} + +// Is `kind` one of the "|A|B|" list? +static bool k8s_kind_in(const char *list, const char *kind) { + if (!list || !kind[0]) { + return false; + } + size_t n = strlen(kind); + for (const char *p = strstr(list, kind); p; p = strstr(p + SKIP_ONE, kind)) { + if (p[-SKIP_ONE] == '|' && p[n] == '|') { + return true; + } + } + return false; +} + +// Where a kind keeps its pod spec, "" for none. +static const char *k8s_pod_prefix(const char *kind) { + if (strcmp(kind, "Pod") == 0) { + return "spec"; + } + if (strcmp(kind, "CronJob") == 0) { + return "spec.jobTemplate.spec.template.spec"; + } + if (k8s_kind_in( + "|Deployment|StatefulSet|DaemonSet|ReplicaSet|ReplicationController|Job|Rollout|", + kind)) { + return "spec.template.spec"; + } + return ""; +} + +typedef struct { + CBMExtractCtx *ctx; + const char *group; // the document's API group + const char *kind; // the document's kind + int def_index; + char crd_group[K8S_BUF_SIZE]; + char crd_kind[K8S_BUF_SIZE]; + const char *deploy_ns; // a deploying object's target namespace + const char *hr_chart; // a HelmRelease's spec.chart.spec.chart + const char *hr_source; // ... and its sourceRef kind +} k8s_walk_t; + +// Is the document of a source kind in group list `groups`? +static bool k8s_source_is(const k8s_walk_t *w, const char *groups, const char *kinds) { + return k8s_kind_in(groups, w->group) && k8s_kind_in(kinds, w->kind); +} + +// A Flux Kustomization / Argo Application deploys a repository directory. +static void k8s_deploy_leaf(k8s_walk_t *w, const char *path, const char *value) { + bool flux = k8s_source_is(w, K8S_FLUX_KS, "|Kustomization|"); + bool argo = k8s_source_is(w, K8S_ARGO, "|Application|"); + if ((flux && strcmp(path, "spec.targetNamespace") == 0) || + (argo && strcmp(path, "spec.destination.namespace") == 0)) { + w->deploy_ns = strstr(value, "{{") ? NULL : value; + return; + } + if (k8s_source_is(w, K8S_FLUX_HELM, "|HelmRelease|")) { + if (strcmp(path, "spec.chart.spec.chart") == 0 && !strstr(value, "{{")) { + w->hr_chart = value; + } else if (strcmp(path, "spec.chart.spec.sourceRef.kind") == 0) { + w->hr_source = value; + } + return; + } + /* the chart an OCIRepository fetches (a HelmRelease's chartRef) */ + if (k8s_source_is(w, K8S_FLUX_SRC, "|OCIRepository|") && strcmp(path, "spec.url") == 0) { + if (!strstr(value, "{{") && !strstr(value, "${")) { + CBMK8sFact f = {.fact = CBM_K8S_FACT_SOURCE, + .def_index = w->def_index, + .name = cbm_arena_strdup(w->ctx->arena, value), + .via = "ocirepository"}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); + } + return; + } + if ((flux && strcmp(path, "spec.path") == 0) || + (argo && + (strcmp(path, "spec.source.path") == 0 || strcmp(path, "spec.sources[].path") == 0))) { + if (strstr(value, "{{")) { + return; + } + CBMK8sFact f = {.fact = CBM_K8S_FACT_DEPLOY, + .def_index = w->def_index, + .name = cbm_arena_strdup(w->ctx->arena, value), + .via = flux ? "flux" : "argo"}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); + } +} + +static void k8s_emit_ref(k8s_walk_t *w, const char *kind, const char *ns, const char *name, + const char *via, const char *edge) { + if (!kind || !kind[0] || !name || !name[0] || strstr(name, "{{") || strstr(kind, "{{")) { + return; + } + CBMArena *a = w->ctx->arena; + CBMK8sFact f = {.fact = CBM_K8S_FACT_REF, + .def_index = w->def_index, + .kind = cbm_arena_strdup(a, kind), + .ns = ns && ns[0] && ns[0] != '<' && !strstr(ns, "{{") ? cbm_arena_strdup(a, ns) + : NULL, + .name = cbm_arena_strdup(a, name), + .via = via, + .edge = edge}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, a, f); +} + +// A container's literal image: `path` ends in containers[].image, +// initContainers[].image or ephemeralContainers[].image at any depth (pod +// templates, CronJobs, the pod templates of custom resources). +static bool k8s_image_leaf(const char *path, const char *value) { + static const char *const LEAVES[] = {"containers[].image", "initContainers[].image", + "ephemeralContainers[].image"}; + if (!value[0] || strchr(value, '$') || strchr(value, ' ') || strstr(value, "{{")) { + return false; + } + size_t pl = strlen(path); + for (size_t i = 0; i < sizeof(LEAVES) / sizeof(LEAVES[0]); i++) { + size_t ll = strlen(LEAVES[i]); + if (pl >= ll && strcmp(path + pl - ll, LEAVES[i]) == 0 && + (pl == ll || path[pl - ll - SKIP_ONE] == '.')) { + return true; + } + } + return false; +} + +// A scalar at `path`: a leaf reference, or a CRD's group / kind. +static void k8s_leaf(k8s_walk_t *w, const char *path, const char *value) { + if (k8s_image_leaf(path, value)) { + CBMK8sFact f = {.fact = CBM_K8S_FACT_IMAGE, + .def_index = w->def_index, + .name = cbm_arena_strdup(w->ctx->arena, value), + .via = "container"}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); + } + const char *pod = k8s_pod_prefix(w->kind); + size_t pl = strlen(pod); + if (pl && strncmp(path, pod, pl) == 0 && path[pl] == '.') { + const char *rel = path + pl + SKIP_ONE; + for (size_t i = 0; i < sizeof(k8s_pod_refs) / sizeof(k8s_pod_refs[0]); i++) { + if (strcmp(rel, k8s_pod_refs[i].path) == 0) { + k8s_emit_ref(w, k8s_pod_refs[i].kind, NULL, value, k8s_pod_refs[i].via, "USAGE"); + } + } + static const char *const holders[] = {"containers[].", "initContainers[].", + "ephemeralContainers[]."}; + for (size_t h = 0; h < sizeof(holders) / sizeof(holders[0]); h++) { + size_t hl = strlen(holders[h]); + if (strncmp(rel, holders[h], hl) != 0) { + continue; + } + for (size_t i = 0; i < sizeof(k8s_container_refs) / sizeof(k8s_container_refs[0]); + i++) { + if (strcmp(rel + hl, k8s_container_refs[i].path) == 0) { + k8s_emit_ref(w, k8s_container_refs[i].kind, NULL, value, + k8s_container_refs[i].via, "USAGE"); + } + } + } + } + for (size_t i = 0; i < sizeof(k8s_object_refs) / sizeof(k8s_object_refs[0]); i++) { + const k8s_object_ref_t *r = &k8s_object_refs[i]; + if (k8s_kind_in(r->kinds, w->kind) && strcmp(path, r->path) == 0) { + k8s_emit_ref(w, r->kind, NULL, value, r->via, r->edge); + } + } + for (size_t i = 0; i < sizeof(k8s_crd_leaf_refs) / sizeof(k8s_crd_leaf_refs[0]); i++) { + const k8s_crd_leaf_ref_t *r = &k8s_crd_leaf_refs[i]; + if (k8s_source_is(w, r->groups, r->kinds) && k8s_path_match(r->path, path)) { + k8s_emit_ref(w, r->kind, NULL, value, r->via, r->edge); + } + } + k8s_deploy_leaf(w, path, value); + if (strcmp(w->kind, "CustomResourceDefinition") == 0) { + if (strcmp(path, "spec.group") == 0) { + snprintf(w->crd_group, sizeof(w->crd_group), "%s", value); + } else if (strcmp(path, "spec.names.kind") == 0) { + snprintf(w->crd_kind, sizeof(w->crd_kind), "%s", value); + } + } +} + +// Through block_node / flow_node wrappers to the content node. +static TSNode k8s_unwrap(TSNode node) { + for (int i = 0; i < PAIR_LEN && !ts_node_is_null(node); i++) { + const char *t = ts_node_type(node); + if (strcmp(t, "block_node") == 0) { + node = first_content_child(node); + } else if (strcmp(t, "flow_node") == 0) { + node = flow_node_value(node); + } else { + break; + } + } + return node; +} + +static bool k8s_is_mapping(TSNode n) { + const char *t = ts_node_type(n); + return strcmp(t, "block_mapping") == 0 || strcmp(t, "flow_mapping") == 0; +} + +static bool k8s_is_pair(TSNode n) { + const char *t = ts_node_type(n); + return strcmp(t, "block_mapping_pair") == 0 || strcmp(t, "flow_pair") == 0; +} + +// The scalar value of `field` in a mapping, or NULL. +static const char *k8s_field(k8s_walk_t *w, TSNode mapping, const char *field) { + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, true, &pair)) { + if (!k8s_is_pair(pair)) { + continue; + } + TSNode key = ts_node_named_child(pair, 0); + const char *k = + ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); + if (k && strcmp(k, field) == 0) { + k8s_kids_done(&pairs); + TSNode v = unwrap_pair_value(pair); + return ts_node_is_null(v) ? NULL : get_scalar_text(w->ctx->arena, v, w->ctx->source); + } + } + return NULL; +} + +// The content node of `field` in a mapping (null when absent or not a mapping). +static TSNode k8s_child(k8s_walk_t *w, TSNode mapping, const char *field) { + TSNode none = {0}; + if (ts_node_is_null(mapping) || !k8s_is_mapping(mapping)) { + return none; + } + k8s_kids_t pairs = k8s_kids(mapping); + TSNode pair; + while (k8s_kid(&pairs, true, &pair)) { + if (!k8s_is_pair(pair)) { + continue; + } + TSNode key = ts_node_named_child(pair, 0); + const char *k = + ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); + if (k && strcmp(k, field) == 0) { + k8s_kids_done(&pairs); + TSNode v = unwrap_pair_value(pair); + return ts_node_is_null(v) ? none : k8s_unwrap(v); + } + } + return none; +} + +// The items of a sequence node, unwrapped, comments skipped (iterate to the +// end: the cursor is released when the items are exhausted). +typedef k8s_kids_t k8s_iter_t; + +static k8s_iter_t k8s_iter(TSNode seq) { + TSNode none = {0}; + bool is_seq = !ts_node_is_null(seq) && (strcmp(ts_node_type(seq), "block_sequence") == 0 || + strcmp(ts_node_type(seq), "flow_sequence") == 0); + return k8s_kids(is_seq ? seq : none); +} + +static bool k8s_next(k8s_iter_t *it, TSNode *out) { + TSNode item; + while (k8s_kid(it, true, &item)) { + if (strcmp(ts_node_type(item), "block_sequence_item") == 0) { + item = first_content_child(item); + } + if (!ts_node_is_null(item) && strcmp(ts_node_type(item), "comment") != 0) { + *out = k8s_unwrap(item); + return true; + } + } + return false; +} + +// A scalar field of a node (NULL unless a mapping holding a scalar there). +static const char *k8s_text(k8s_walk_t *w, TSNode mapping, const char *field) { + TSNode v = k8s_child(w, mapping, field); + return ts_node_is_null(v) ? NULL : get_scalar_text(w->ctx->arena, v, w->ctx->source); +} + +// An exposed route "" to backend `name` (NULL: none). +static void k8s_emit_route(k8s_walk_t *w, const char *host, const char *path, const char *ns, + const char *name, const char *via) { + host = host && host[0] ? host : "*"; + path = path && path[0] ? path : "/"; + if (strstr(host, "{{") || strstr(path, "{{") || strchr(host, '/')) { + return; + } + CBMArena *a = w->ctx->arena; + CBMK8sFact f = {.fact = CBM_K8S_FACT_ROUTE, + .def_index = w->def_index, + .kind = name ? "Service" : NULL, + .ns = ns && ns[0] && ns[0] != '<' && !strstr(ns, "{{") ? cbm_arena_strdup(a, ns) + : NULL, + .name = name && !strstr(name, "{{") ? cbm_arena_strdup(a, name) : NULL, + .via = via, + .edge = "INFRA_MAPS", + .route = cbm_arena_sprintf(a, "%s%s%s", host, path[0] == '/' ? "" : "/", path)}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, a, f); +} + +// The routes an Ingress rule (spec.rules[] item) exposes: its host and each +// http path, sent to the path's backend Service. +static void k8s_ingress_rule(k8s_walk_t *w, TSNode rule) { + const char *host = k8s_text(w, rule, "host"); + k8s_iter_t paths = k8s_iter(k8s_child(w, k8s_child(w, rule, "http"), "paths")); + TSNode p; + while (k8s_next(&paths, &p)) { + TSNode backend = k8s_child(w, p, "backend"); + const char *svc = k8s_text(w, k8s_child(w, backend, "service"), "name"); + svc = svc ? svc : k8s_text(w, backend, "serviceName"); + k8s_emit_route(w, host, k8s_text(w, p, "path"), NULL, svc, "ingress"); + } +} + +// One HTTPRoute host x path: to each Service backend of the rule, else none. +static void k8s_httproute_target(k8s_walk_t *w, TSNode rule, const char *host, const char *path) { + k8s_iter_t backs = k8s_iter(k8s_child(w, rule, "backendRefs")); + TSNode b; + bool any = false; + while (k8s_next(&backs, &b)) { + const char *kind = k8s_text(w, b, "kind"); + if (kind && strcmp(kind, "Service") != 0) { + continue; + } + k8s_emit_route(w, host, path, k8s_text(w, b, "namespace"), k8s_text(w, b, "name"), + "httproute"); + any = true; + } + if (!any) { + k8s_emit_route(w, host, path, NULL, NULL, "httproute"); + } +} + +// The routes an HTTPRoute spec exposes: every hostname (else any) with +// every path match of a rule (else `/`), sent to the rule's Service backends. +static void k8s_httproute_spec(k8s_walk_t *w, TSNode spec) { + TSNode hostnames = k8s_child(w, spec, "hostnames"); + k8s_iter_t rules = k8s_iter(k8s_child(w, spec, "rules")); + TSNode rule; + while (k8s_next(&rules, &rule)) { + k8s_iter_t hosts = k8s_iter(hostnames); + TSNode h; + bool any_host = k8s_next(&hosts, &h); + do { + const char *host = any_host ? get_scalar_text(w->ctx->arena, h, w->ctx->source) : NULL; + k8s_iter_t matches = k8s_iter(k8s_child(w, rule, "matches")); + TSNode m; + bool any_match = false; + while (k8s_next(&matches, &m)) { + k8s_httproute_target(w, rule, host, k8s_text(w, k8s_child(w, m, "path"), "value")); + any_match = true; + } + if (!any_match) { + k8s_httproute_target(w, rule, host, NULL); + } + } while (any_host && k8s_next(&hosts, &h)); + } +} + +// The reference a mapping holds in its kind/name/namespace: to `kind`, or +// (NULL) to its own `kind` field, one of `allowed` (NULL: any), `dflt` +// when absent. +static void k8s_mapping_ref(k8s_walk_t *w, TSNode mapping, const char *kind, const char *allowed, + const char *dflt, const char *via, const char *edge) { + if (!kind) { + kind = k8s_field(w, mapping, "kind"); + kind = kind ? kind : dflt; + if (!kind || (allowed && !k8s_kind_in(allowed, kind))) { + return; + } + } + k8s_emit_ref(w, kind, k8s_field(w, mapping, "namespace"), k8s_field(w, mapping, "name"), via, + edge); +} + +// Where an env[] item's value comes from: "value", or its valueFrom kind +// (configMapKeyRef, secretKeyRef, fieldRef, resourceFieldRef, fileKeyRef; +// "valueFrom" for any other). The value itself is never read. +static const char *k8s_env_source(k8s_walk_t *w, TSNode item) { + static const char *const KINDS[] = {"configMapKeyRef", "secretKeyRef", "fieldRef", + "resourceFieldRef", "fileKeyRef"}; + k8s_kids_t pairs = k8s_kids(item); + TSNode pair; + while (k8s_kid(&pairs, true, &pair)) { + if (!k8s_is_pair(pair)) { + continue; + } + TSNode key = ts_node_named_child(pair, 0); + const char *k = + ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); + if (!k || strcmp(k, "valueFrom") != 0) { + continue; + } + k8s_kids_done(&pairs); + TSNode from = k8s_unwrap(unwrap_pair_value(pair)); + k8s_kids_t kinds = k8s_kids(from); + TSNode kp; + const char *kind = NULL; + while (k8s_kid(&kinds, true, &kp)) { + TSNode kk = k8s_is_pair(kp) ? ts_node_named_child(kp, 0) : (TSNode){0}; + const char *s = + ts_node_is_null(kk) ? NULL : get_scalar_text(w->ctx->arena, kk, w->ctx->source); + if (s && !kind) { + kind = s; + } + } + for (size_t i = 0; kind && i < sizeof(KINDS) / sizeof(KINDS[0]); i++) { + if (strcmp(kind, KINDS[i]) == 0) { + return KINDS[i]; + } + } + return "valueFrom"; + } + return "value"; +} + +// The environment variables a container (`path` ends in containers[], +// initContainers[] or ephemeralContainers[], any depth) sets: each env[] +// name, with where its value comes from -- never the value. +static void k8s_container_env(k8s_walk_t *w, const char *path, TSNode container) { + static const char *const ITEMS[] = {"containers[]", "initContainers[]", + "ephemeralContainers[]"}; + size_t pl = strlen(path); + bool at = false; + for (size_t i = 0; i < sizeof(ITEMS) / sizeof(ITEMS[0]) && !at; i++) { + size_t il = strlen(ITEMS[i]); + at = pl >= il && strcmp(path + pl - il, ITEMS[i]) == 0 && + (pl == il || path[pl - il - SKIP_ONE] == '.'); + } + if (!at) { + return; + } + const char *cname = k8s_text(w, container, "name"); + cname = cname && cname[0] && !strstr(cname, "{{") ? cname : NULL; + k8s_iter_t env = k8s_iter(k8s_child(w, container, "env")); + TSNode item; + while (k8s_next(&env, &item)) { + const char *key = k8s_is_mapping(item) ? k8s_text(w, item, "name") : NULL; + if (!key || !key[0] || strstr(key, "{{")) { + continue; + } + CBMK8sFact f = {.fact = CBM_K8S_FACT_ENV, + .def_index = w->def_index, + .name = key, + .container = cname, + .via = k8s_env_source(w, item), + .line = (int)ts_node_start_point(item).row + SKIP_ONE}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); + } +} + +// A mapping at `path` that holds a reference in its kind/name/namespace. +static void k8s_mapping(k8s_walk_t *w, const char *path, TSNode mapping) { + k8s_container_env(w, path, mapping); + for (size_t i = 0; i < sizeof(k8s_mapping_refs) / sizeof(k8s_mapping_refs[0]); i++) { + const k8s_mapping_ref_t *r = &k8s_mapping_refs[i]; + if (k8s_kind_in(r->kinds, w->kind) && strcmp(path, r->path) == 0) { + k8s_mapping_ref(w, mapping, r->kind, r->allowed, r->dflt, r->via, r->edge); + } + } + for (size_t i = 0; i < sizeof(k8s_crd_mapping_refs) / sizeof(k8s_crd_mapping_refs[0]); i++) { + const k8s_crd_mapping_ref_t *r = &k8s_crd_mapping_refs[i]; + if (k8s_source_is(w, r->groups, r->kinds) && k8s_path_match(r->path, path)) { + k8s_mapping_ref(w, mapping, r->kind, r->allowed, r->dflt, r->via, r->edge); + } + } + if (k8s_source_is(w, "|networking.k8s.io|extensions|", "|Ingress|") && + strcmp(path, "spec.rules[]") == 0) { + k8s_ingress_rule(w, mapping); + } + if (k8s_source_is(w, "|gateway.networking.k8s.io|", "|HTTPRoute|") && + strcmp(path, "spec") == 0) { + k8s_httproute_spec(w, mapping); + } +} + +// Walk `node` (reached at `path`) for references. +static void k8s_walk(k8s_walk_t *w, TSNode node, char *path, size_t len, size_t cap) { + node = k8s_unwrap(node); + if (ts_node_is_null(node)) { + return; + } + const char *type = ts_node_type(node); + if (k8s_is_mapping(node)) { + k8s_mapping(w, path, node); + k8s_kids_t pairs = k8s_kids(node); + TSNode pair; + while (k8s_kid(&pairs, true, &pair)) { + if (!k8s_is_pair(pair)) { + continue; + } + TSNode key = ts_node_named_child(pair, 0); + const char *k = + ts_node_is_null(key) ? NULL : get_scalar_text(w->ctx->arena, key, w->ctx->source); + int add = k ? snprintf(path + len, cap - len, "%s%s", len ? "." : "", k) : -1; + if (add <= 0 || len + (size_t)add >= cap) { + path[len] = '\0'; // a path too long for any reference path: not one + continue; + } + TSNode value = unwrap_pair_value(pair); + TSNode inner = k8s_unwrap(value); + const char *scalar = ts_node_is_null(inner) + ? NULL + : get_scalar_text(w->ctx->arena, inner, w->ctx->source); + if (scalar) { + k8s_leaf(w, path, scalar); + } else if (!ts_node_is_null(value)) { + k8s_walk(w, value, path, len + (size_t)add, cap); + } + path[len] = '\0'; + } + return; + } + if (strcmp(type, "block_sequence") == 0 || strcmp(type, "flow_sequence") == 0) { + int add = snprintf(path + len, cap - len, "[]"); + if (add <= 0 || len + (size_t)add >= cap) { + path[len] = '\0'; + return; + } + k8s_kids_t items = k8s_kids(node); + TSNode item; + while (k8s_kid(&items, true, &item)) { + if (strcmp(ts_node_type(item), "block_sequence_item") == 0) { + item = first_content_child(item); + } + if (!ts_node_is_null(item) && strcmp(ts_node_type(item), "comment") != 0) { + k8s_walk(w, item, path, len + (size_t)add, cap); + } + } + path[len] = '\0'; + } +} + +// The references, deploys and CRD declaration of document def `def_index`. +static void k8s_emit_facts(CBMExtractCtx *ctx, TSNode mapping, const char *group, const char *kind, + int def_index) { + k8s_walk_t w = {.ctx = ctx, .group = group, .kind = kind, .def_index = def_index}; + char path[CBM_SZ_512] = ""; + int first = ctx->result->k8s_facts.count; + k8s_walk(&w, mapping, path, 0, sizeof(path)); + /* a HelmRelease deploys its chart: a path of a Git/Bucket source, else + * a chart name of a Helm repository (kind: the source kind) */ + if (w.hr_chart) { + CBMK8sFact f = {.fact = CBM_K8S_FACT_DEPLOY, + .def_index = def_index, + .kind = w.hr_source ? cbm_arena_strdup(ctx->arena, w.hr_source) : NULL, + .name = cbm_arena_strdup(ctx->arena, w.hr_chart), + .via = "helmrelease"}; + cbm_k8sfact_push(&ctx->result->k8s_facts, ctx->arena, f); + } + /* the target namespace may follow the path in the document */ + for (int i = first; w.deploy_ns && i < ctx->result->k8s_facts.count; i++) { + CBMK8sFact *f = &ctx->result->k8s_facts.items[i]; + if (f->fact == CBM_K8S_FACT_DEPLOY) { + f->ns = cbm_arena_strdup(ctx->arena, w.deploy_ns); + } + } + if (w.crd_group[0] && w.crd_kind[0]) { + CBMK8sFact f = {.fact = CBM_K8S_FACT_CRD, + .def_index = def_index, + .kind = cbm_arena_strdup(ctx->arena, w.crd_kind), + .name = cbm_arena_strdup(ctx->arena, w.crd_group)}; + cbm_k8sfact_push(&ctx->result->k8s_facts, ctx->arena, f); + } +} + +// "." with the extension kept: the File node's QN +// without ".__file__" (IDENTITY.md). Unique per file, unlike the module QN, +// which drops the extension (`a.yaml` / `a.yml`, `x/` / `x.yaml`). +static const char *k8s_file_base(CBMArena *a, const char *project, const char *rel_path) { + project = project ? project : ""; + rel_path = rel_path ? rel_path : ""; + size_t plen = strlen(project); + char *buf = cbm_arena_alloc(a, plen + strlen(rel_path) + PAIR_LEN); + if (!buf) { + return NULL; + } + char *o = buf; + memcpy(o, project, plen); + o += plen; + bool seg_start = true; + for (const char *p = rel_path; *p; p++) { + char c = *p == '\\' ? '/' : *p; + if (c == '/') { + seg_start = true; + continue; + } + if (seg_start) { + *o++ = '.'; + seg_start = false; + } + *o++ = c; + } + *o = '\0'; + return buf; +} + +// The API group of an apiVersion: the part before '/', "core" for the core +// group ("v1"), "?" when the document names none. The version is not part of +// an object's identity. +static void k8s_api_group(const char *api_version, char *out, size_t out_sz) { + if (api_version[0] == '<') { // templated (a skeleton's symbolic `<...>`) + snprintf(out, out_sz, "?"); + return; + } + const char *slash = strchr(api_version, '/'); + if (slash) { + snprintf(out, out_sz, "%.*s", (int)(slash - api_version), api_version); + } else { + snprintf(out, out_sz, "%s", api_version[0] ? "core" : "?"); + } +} + +// One Resource def per document: `Kind/name`, QN +// `::///` (IDENTITY.md). static void extract_k8s_manifest(CBMExtractCtx *ctx) { CBMArena *a = ctx->arena; + const char *base = k8s_file_base(a, ctx->project, ctx->rel_path); + if (!base) { + return; + } - TSNode root = ctx->root; - uint32_t root_n = ts_node_child_count(root); - for (uint32_t si = 0; si < root_n; si++) { - TSNode stream_child = ts_node_child(root, si); + k8s_kids_t docs = k8s_kids(ctx->root); + TSNode stream_child; + while (k8s_kid(&docs, false, &stream_child)) { if (strcmp(ts_node_type(stream_child), "document") != 0) { continue; } - TSNode mapping = ts_node_named_child(stream_child, 0); + TSNode mapping = unwrap_block_mapping(stream_child); if (ts_node_is_null(mapping)) { continue; } - if (strcmp(ts_node_type(mapping), "block_node") == 0) { - mapping = ts_node_named_child(mapping, 0); - } - if (ts_node_is_null(mapping) || strcmp(ts_node_type(mapping), "block_mapping") != 0) { + + k8s_doc_id_t id; + memset(&id, 0, sizeof(id)); + extract_k8s_scalars(ctx, mapping, &id); + + // Skip malformed manifests (no kind or no metadata.name), and a + // templated kind (a skeleton's symbolic `<...>`): unidentifiable. + if (id.kind[0] == '\0' || id.name[0] == '\0' || id.kind[0] == '<') { continue; } - - char kind_buf[K8S_BUF_SIZE] = {0}; - char meta_name_buf[K8S_BUF_SIZE] = {0}; - extract_k8s_scalars(ctx, mapping, kind_buf, sizeof(kind_buf), meta_name_buf, - sizeof(meta_name_buf)); - - // Skip malformed manifests (no kind or no metadata.name) - if (kind_buf[0] == '\0' || meta_name_buf[0] == '\0') { - continue; + // A templated namespace (`<.Release.Namespace>`) is the deploy's. + if (id.ns[0] == '<') { + id.ns[0] = '\0'; } - char def_name[RESULT_BUF_SIZE]; - snprintf(def_name, sizeof(def_name), "%s/%s", kind_buf, meta_name_buf); + char group[K8S_BUF_SIZE]; + k8s_api_group(id.api_version, group, sizeof(group)); CBMDefinition def = {0}; - def.name = cbm_arena_strdup(a, def_name); - def.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->module_qn, def_name); + def.name = cbm_arena_sprintf(a, "%s/%s", id.kind, id.name); + def.qualified_name = cbm_arena_sprintf(a, "%s::%s/%s/%s/%s", base, group, id.kind, + id.ns[0] ? id.ns : "-", id.name); // "Resource" is the canonical def label for a K8s resource kind. It is a // valid graph label and is what the K8s pipeline pass (pass_k8s.c) filters // on to upsert Resource nodes and emit INFRA_MAPS edges. @@ -342,8 +1333,7 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { def.start_line = ts_node_start_point(mapping).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(mapping).row + TS_LINE_OFFSET; cbm_defs_push(&ctx->result->defs, a, def); - - break; // Only the first document per file + k8s_emit_facts(ctx, mapping, group, id.kind, ctx->result->defs.count - SKIP_ONE); } } diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index b252f039c..1b77f0bdc 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -1651,7 +1651,29 @@ bool cbm_is_module_level(TSNode node, CBMLanguage lang) { // Mirrors Go's fqn.Compute(): project + path_parts_dotted + name // Internal helper: find extension start in basename (returns length without ext) +/* A container build file named for its variant ("Dockerfile.debug") keeps + * the suffix (no language extension; dropping it gives the variant the QN of + * the plain Dockerfile beside it). Mirrors strip_file_extension in + * src/pipeline/fqn.c -- both must agree. */ +static bool build_file_variant(const char *s, size_t len) { + size_t b = len; + while (b > 0 && s[b - SKIP_ONE] != '/') { + b--; + } + static const char *const BASES[] = {"Dockerfile.", "Containerfile."}; + for (size_t i = 0; i < sizeof(BASES) / sizeof(BASES[0]); i++) { + size_t bl = strlen(BASES[i]); + if (len - b > bl && strncmp(s + b, BASES[i], bl) == 0) { + return true; + } + } + return false; +} + static size_t strip_ext_len(const char *s, size_t len) { + if (build_file_variant(s, len)) { + return len; + } for (size_t i = len; i > 0; i--) { if (s[i - SKIP_ONE] == '.') { /* A dot at the very start of a filename segment (index 0, or right diff --git a/scripts/check-lsp-originality.sh b/scripts/check-lsp-originality.sh index a50af9de4..56c0fdcde 100755 --- a/scripts/check-lsp-originality.sh +++ b/scripts/check-lsp-originality.sh @@ -64,8 +64,27 @@ REFS=( # Perl (PR #461) was authored clean-room. We scan against PerlNavigator (MIT) # — the leading OSS Perl language server — for defensive copy-detection. "perl|https://github.com/bscan/PerlNavigator|server/src" + # Infrastructure hybrid LSPs (K8s YAML, Kustomize bundles, Flux/Argo deploy + # scopes, the Go operator extension) were written from the Kubernetes, Flux, + # Argo, Cluster API, cert-manager and Kubermatic API types. Scanned against + # the YAML/K8s language server, helm-ls, kustomize, Flux's kustomization + # generator and controller-tools (the +groupName markers). + "yaml|https://github.com/redhat-developer/yaml-language-server|src" + "helm|https://github.com/mrjosh/helm-ls|internal" + "kustomize|https://github.com/kubernetes-sigs/kustomize|api/krusty,api/internal/target" + "flux|https://github.com/fluxcd/pkg|kustomize" + "kube|https://github.com/kubernetes-sigs/controller-tools|pkg" + # The Bash (+ Make) layer (deploy commands, constant propagation) was written + # from the bash, GNU make, kubectl and helm documentation. Scanned against + # bash-language-server (MIT), the leading OSS bash language server. + "bash|https://github.com/bash-lsp/bash-language-server|server/src" ) +# Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too). +INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c + src/pipeline/pass_shell.c src/pipeline/pass_terraform.c + src/pipeline/pass_ci.c src/pipeline/pass_docker.c) + ONLY_LANG=""; REFRESH=0; LIST_ONLY=0 while [ $# -gt 0 ]; do case "$1" in @@ -87,6 +106,9 @@ fi LSP_SRC=() while IFS= read -r f; do LSP_SRC+=("$f"); done < <( find "$LSP_DIR" -type f \( -name '*.c' -o -name '*.h' \) -not -path '*/generated/*' | sort) +for f in "${INFRA_SRC[@]}"; do + [ -f "$ROOT/$f" ] && LSP_SRC+=("$ROOT/$f") +done if [ "${#LSP_SRC[@]}" -eq 0 ]; then echo "$TAG ERROR: no LSP sources under $LSP_DIR" >&2; exit 2 fi diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index cae49fa0c..4bae148b4 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -64,14 +64,13 @@ src/pipeline/pass_cross_repo.c 8 src/pipeline/pass_definitions.c 11 src/pipeline/pass_enrichment.c 15 src/pipeline/pass_ensemble_routing.c 10 -src/pipeline/pass_envscan.c 2 src/pipeline/pass_githistory.c 27 src/pipeline/pass_importance.c 4 src/pipeline/pass_infrascan.c 2 src/pipeline/pass_k8s.c 6 src/pipeline/pass_lsp_cross.c 49 src/pipeline/pass_parallel.c 31 -src/pipeline/pass_pkgmap.c 82 +src/pipeline/pass_pkgmap.c 81 src/pipeline/pass_semantic.c 3 src/pipeline/pass_tests.c 3 src/pipeline/pass_usages.c 4 diff --git a/src/cli/cli.c b/src/cli/cli.c index 14fb053c8..a90a448be 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1567,7 +1567,8 @@ static const char skill_content[] = "CALLS, HTTP_CALLS, ASYNC_CALLS, SPAWNS, DATA_FLOWS, IMPORTS, DEFINES, DEFINES_METHOD,\n" "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, REFERENCES_FILE,\n" "MENTIONS (doc comment -> referenced code), FILE_CHANGES_WITH, SIMILAR_TO,\n" - "SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE\n" + "SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n" + "INFRA_MAPS (K8s Service -> selected workloads), DEPENDS_ON (chart / manifest -> dependency)\n" "\n" "## Cypher Examples (for query_graph)\n" "```\n" diff --git a/src/discover/language.c b/src/discover/language.c index eb1284c06..520416493 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -93,6 +93,7 @@ static const ext_entry_t EXT_TABLE[] = { /* Dockerfile */ {".dockerfile", CBM_LANG_DOCKERFILE}, + {".Dockerfile", CBM_LANG_DOCKERFILE}, /* Elixir */ {".ex", CBM_LANG_ELIXIR}, @@ -152,9 +153,11 @@ static const ext_entry_t EXT_TABLE[] = { /* Haskell */ {".hs", CBM_LANG_HASKELL}, - /* HCL / Terraform */ + /* HCL / Terraform / OpenTofu */ {".hcl", CBM_LANG_HCL}, {".tf", CBM_LANG_HCL}, + {".tofu", CBM_LANG_HCL}, + {".tfvars", CBM_LANG_HCL}, /* HTML */ {".htm", CBM_LANG_HTML}, @@ -722,6 +725,7 @@ typedef struct { static const filename_entry_t FILENAME_TABLE[] = { {"CMakeLists.txt", CBM_LANG_CMAKE}, + {"Containerfile", CBM_LANG_DOCKERFILE}, {"Dockerfile", CBM_LANG_DOCKERFILE}, {"openapi.json", CBM_LANG_OPENAPI}, /* API descriptions: their operations, not each key */ {"swagger.json", CBM_LANG_OPENAPI}, @@ -733,6 +737,7 @@ static const filename_entry_t FILENAME_TABLE[] = { {"meson_options.txt", CBM_LANG_MESON}, {"kustomization.yaml", CBM_LANG_KUSTOMIZE}, {"kustomization.yml", CBM_LANG_KUSTOMIZE}, + {"Kustomization", CBM_LANG_KUSTOMIZE}, /* kustomize's extensionless name */ /* Note: FILENAME_TABLE uses case-sensitive strcmp, so mixed-case variants * (e.g. "Kustomization.yaml") are not matched here. They fall through to * CBM_LANG_YAML and are re-classified by cbm_is_kustomize_file() in @@ -969,6 +974,35 @@ CBMLanguage cbm_language_for_extension(const char *ext) { return lang_for_extension(cbm_get_user_lang_config(), ext); } +/* A container build file named for its variant: "Dockerfile.dev", + * "Dockerfile-test", "Containerfile.arm64". Not when the suffix is a language + * extension of its own (Dockerfile.j2 is a Jinja template) nor a BuildKit + * ignore file (Dockerfile.dockerignore). */ +static bool lang_dockerfile_variant(const cbm_userconfig_t *ucfg, const char *filename) { + static const char *const BASES[] = {"Dockerfile", "Containerfile"}; + static const char DOCKERIGNORE[] = ".dockerignore"; + for (size_t i = 0; i < sizeof(BASES) / sizeof(BASES[0]); i++) { + size_t bl = strlen(BASES[i]); + /* the base first: only a name that holds it has a character after it */ + if (strncmp(filename, BASES[i], bl) != 0) { + continue; + } + char sep = filename[bl]; + if ((sep != '.' && sep != '-' && sep != '_') || !filename[bl + SKIP_ONE]) { + continue; + } + size_t fl = strlen(filename); + if (fl >= sizeof(DOCKERIGNORE) - SKIP_ONE && + strcmp(filename + fl - (sizeof(DOCKERIGNORE) - SKIP_ONE), DOCKERIGNORE) == 0) { + return false; + } + const char *last_dot = strrchr(filename, '.'); + return !last_dot || last_dot < filename + bl || + lang_for_extension(ucfg, last_dot) == CBM_LANG_COUNT; + } + return false; +} + static CBMLanguage lang_for_filename(const cbm_userconfig_t *ucfg, const char *filename) { if (!filename || !filename[0]) { return CBM_LANG_COUNT; @@ -983,12 +1017,15 @@ static CBMLanguage lang_for_filename(const cbm_userconfig_t *ucfg, const char *f /* DotEnv variant filenames (".env.local", ".env.production", …): the * filename starts with ".env." but its last "extension" (e.g. ".local") - * is not a real language extension. Match the dotenv convention used by - * pass_envscan/pass_infrascan (".env" exact, ".env." prefix, "*.env" - * suffix) so file-index routing agrees with direct extraction. */ + * is not a real language extension. Match the dotenv convention (".env" + * exact, ".env." prefix, "*.env" suffix) so file-index routing agrees with + * direct extraction. */ if (strncmp(filename, ".env.", SLEN(".env.")) == 0) { return CBM_LANG_DOTENV; } + if (lang_dockerfile_variant(ucfg, filename)) { + return CBM_LANG_DOCKERFILE; + } /* Fall back to extension-based lookup. * For compound extensions (e.g. ".blade.php") defined in the user config, diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index f8272e63a..13f0db3d7 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -105,6 +105,11 @@ struct cbm_gbuf { bool by_id_off; int64_t by_id_cap; + /* Same-QN collision spans, written on export (cbm_gbuf_settle_variants). + * key: the surviving node's own QN string, value: gb_span_vec_t*. NULL + * until the first collision. */ + CBMHashTable *variant_spans; + /* Secondary node indexes */ CBMHashTable *nodes_by_label; /* key: label, value: (node_ptr_array_t*) */ CBMHashTable *nodes_by_name; /* key: name, value: (node_ptr_array_t*) */ @@ -672,10 +677,16 @@ cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path, return gb; } +static void gb_free_span_vec(const char *qn, void *value, void *ud); + void cbm_gbuf_free(cbm_gbuf_t *gb) { if (!gb) { return; } + if (gb->variant_spans) { + cbm_ht_foreach(gb->variant_spans, gb_free_span_vec, NULL); + cbm_ht_free(gb->variant_spans); + } /* Free each individually-allocated node */ for (int i = 0; i < gb->nodes.count; i++) { @@ -897,14 +908,32 @@ static void gb_yy_free(void *ctx, void *block) { static const yyjson_alc gb_yy_alc = {gb_yy_malloc, gb_yy_realloc, gb_yy_free, NULL}; -enum { GB_VARIANTS_MAX = 64 }; - typedef struct { - const char *file; + const char *file; /* interned in the buffer's pool */ int start; int end; } gb_span_t; +/* One side of a same-QN collision. */ +typedef struct { + const char *label; + const char *props; + const char *file; + int start; + int end; +} gb_def_t; + +/* Every definition span that collapsed into one node, gathered while the + * buffer is built and written into the node's `variants` once, on export + * (cbm_gbuf_settle_variants). Rewriting a growing JSON list on every + * collision was quadratic in the collisions of one QN: 7,586 junit + * `testcase` elements in one file. */ +typedef struct { + gb_span_t *items; + size_t count; + size_t cap; +} gb_span_vec_t; + static bool gb_variant_label(const char *label) { static const char *const labels[] = {"Function", "Method", "Class", "Interface", "Enum", "Struct", "Trait", "Type", "Macro"}; @@ -929,99 +958,260 @@ static int gb_span_cmp(const void *a, const void *b) { return c; } -/* A definition's spans: its `variants` property, else its own span. */ -static int gb_spans_of(yyjson_doc *doc, const char *file, int start, int end, gb_span_t *out, - int n) { +/* The non-empty `variants` list of a properties document, or NULL. */ +static yyjson_val *gb_variants_list(yyjson_doc *doc) { yyjson_val *list = doc ? yyjson_obj_get(yyjson_doc_get_root(doc), "variants") : NULL; - if (yyjson_is_arr(list) && yyjson_arr_size(list) > 0) { + return yyjson_is_arr(list) && yyjson_arr_size(list) > 0 ? list : NULL; +} + +static bool gb_props_have_variants(const char *props) { + return props && strstr(props, "\"variants\"") != NULL; +} + +/* Does this side carry a span: a variants list, or a definition's own span? */ +static bool gb_def_has_spans(const gb_def_t *d) { + return gb_props_have_variants(d->props) || (gb_variant_label(d->label) && d->start > 0); +} + +static void gb_span_push(gb_span_vec_t *vec, gb_span_t span) { + if (vec->count == vec->cap) { + enum { GB_SPANS_MIN = 4 }; + size_t cap = vec->cap ? vec->cap * PAIR_LEN : GB_SPANS_MIN; + gb_span_t *grown = cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, vec->items, cap * sizeof(*grown)); + if (!grown) { + return; /* a lost span costs precision, never data */ + } + vec->items = grown; + vec->cap = cap; + } + vec->items[vec->count++] = span; +} + +/* Append a side's spans: its `variants` list, else its own span when it is a + * definition. File paths are interned, so the spans outlive the props. */ +static void gb_collect_spans(cbm_gbuf_t *gb, gb_span_vec_t *vec, const gb_def_t *d) { + if (gb_props_have_variants(d->props)) { + yyjson_doc *doc = yyjson_read_opts((char *)d->props, strlen(d->props), 0, &gb_yy_alc, NULL); + yyjson_val *list = gb_variants_list(doc); + if (list) { + size_t i; + size_t max; + yyjson_val *v; + yyjson_arr_foreach(list, i, max, v) { + const char *f = yyjson_get_str(yyjson_obj_get(v, "file_path")); + int st = (int)yyjson_get_int(yyjson_obj_get(v, "start_line")); + int en = (int)yyjson_get_int(yyjson_obj_get(v, "end_line")); + if (f && st > 0) { + gb_span_push(vec, (gb_span_t){gb_intern(gb, f), st, en}); + } + } + yyjson_doc_free(doc); + return; + } + yyjson_doc_free(doc); + } + if (gb_variant_label(d->label) && d->start > 0) { + gb_span_push(vec, (gb_span_t){gb_intern(gb, d->file ? d->file : ""), d->start, d->end}); + } +} + +/* Append spans gathered in another buffer (a worker's), re-interned here. */ +static void gb_append_spans(cbm_gbuf_t *gb, gb_span_vec_t *vec, const gb_span_vec_t *from) { + for (size_t i = 0; i < from->count; i++) { + gb_span_t span = from->items[i]; + span.file = gb_intern(gb, span.file); + gb_span_push(vec, span); + } +} + +/* A same-QN collision: gather both sides' spans for the surviving node, + * whichever label wins and however many there are. Call it before the + * winner is decided, while `existing` still holds its own fields. + * `arrival_spans` are the arrival's spans gathered in a worker buffer (they + * already include its own); NULL when it brings none. */ +static void gb_note_collision(cbm_gbuf_t *gb, const cbm_gbuf_node_t *existing, + const gb_def_t *arrival, const gb_span_vec_t *arrival_spans) { + gb_def_t prior = {existing->label, existing->properties_json, existing->file_path, + existing->start_line, existing->end_line}; + bool arrival_has = arrival_spans ? arrival_spans->count > 0 : gb_def_has_spans(arrival); + if (!existing->qualified_name || (!gb_def_has_spans(&prior) && !arrival_has)) { + return; + } + if (!gb->variant_spans) { + gb->variant_spans = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, 0); + if (!gb->variant_spans) { + return; + } + } + gb_span_vec_t *vec = cbm_ht_get(gb->variant_spans, existing->qualified_name); + if (!vec) { + vec = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(*vec)); + if (!vec) { + return; + } + gb_collect_spans(gb, vec, &prior); + cbm_ht_set(gb->variant_spans, existing->qualified_name, vec); + } + if (arrival_spans) { + gb_append_spans(gb, vec, arrival_spans); + } else { + gb_collect_spans(gb, vec, arrival); + } +} + +/* A worker node copied into this buffer brings the spans it gathered there. */ +static void gb_adopt_spans(cbm_gbuf_t *gb, const cbm_gbuf_node_t *node, const gb_span_vec_t *from) { + if (!node || !node->qualified_name || !from || from->count == 0) { + return; + } + if (!gb->variant_spans) { + gb->variant_spans = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, 0); + if (!gb->variant_spans) { + return; + } + } + gb_span_vec_t *vec = cbm_ht_get(gb->variant_spans, node->qualified_name); + if (!vec) { + vec = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(*vec)); + if (!vec) { + return; + } + cbm_ht_set(gb->variant_spans, node->qualified_name, vec); + } + gb_append_spans(gb, vec, from); +} + +static void gb_span_vec_free(gb_span_vec_t *vec) { + if (vec) { + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, vec->items); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, vec); + } +} + +/* A node leaves the QN index: its gathered spans go with it, so a later node + * of the same QN starts from its own. */ +static void gb_drop_variant_spans(cbm_gbuf_t *gb, const char *qn) { + if (gb->variant_spans && qn) { + gb_span_vec_free(cbm_ht_delete(gb->variant_spans, qn)); + } +} + +/* Write a node's gathered spans into its properties: sorted, distinct, the + * list left out only when it would say no more than the node's own span. */ +static void gb_write_variants(cbm_gbuf_node_t *node, gb_span_vec_t *vec) { + qsort(vec->items, vec->count, sizeof(vec->items[0]), gb_span_cmp); + size_t unique = 0; + for (size_t i = 0; i < vec->count; i++) { + if (unique == 0 || gb_span_cmp(&vec->items[unique - SKIP_ONE], &vec->items[i]) != 0) { + vec->items[unique++] = vec->items[i]; + } + } + gb_span_t own = {node->file_path ? node->file_path : "", node->start_line, node->end_line}; + bool own_is_span = gb_variant_label(node->label) && node->start_line > 0; + bool only_own = + own_is_span ? unique == SKIP_ONE && gb_span_cmp(&vec->items[0], &own) == 0 : unique == 0; + if (only_own && !gb_props_have_variants(node->properties_json)) { + return; + } + /* Numbers stay as written: a pass printed them (importance 0.000000). */ + const char *props = node->properties_json; + yyjson_doc *doc = props ? yyjson_read_opts((char *)props, strlen(props), + YYJSON_READ_NUMBER_AS_RAW, &gb_yy_alc, NULL) + : NULL; + yyjson_mut_doc *mut = yyjson_mut_doc_new(&gb_yy_alc); + yyjson_mut_val *root = NULL; + if (mut) { + root = doc && yyjson_is_obj(yyjson_doc_get_root(doc)) + ? yyjson_val_mut_copy(mut, yyjson_doc_get_root(doc)) + : yyjson_mut_obj(mut); + } + yyjson_mut_val *list = root && !only_own ? yyjson_mut_arr(mut) : NULL; + for (size_t i = 0; list && i < unique; i++) { + yyjson_mut_val *v = yyjson_mut_arr_add_obj(mut, list); + yyjson_mut_obj_add_strcpy(mut, v, "file_path", vec->items[i].file); + yyjson_mut_obj_add_int(mut, v, "start_line", vec->items[i].start); + yyjson_mut_obj_add_int(mut, v, "end_line", vec->items[i].end); + } + if (root && (only_own || list)) { + yyjson_mut_obj_remove_key(root, "variants"); + if (list) { + yyjson_mut_obj_add_val(mut, root, "variants", list); + } + yyjson_mut_doc_set_root(mut, root); + char *out = yyjson_mut_write_opts(mut, 0, &gb_yy_alc, NULL, NULL); + if (out) { + cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->properties_json); + node->properties_json = out; + } + } + yyjson_mut_doc_free(mut); + yyjson_doc_free(doc); +} + +static void gb_settle_one(const char *qn, void *value, void *ud) { + cbm_gbuf_t *gb = ud; + gb_span_vec_t *vec = value; + cbm_gbuf_node_t *node = gb->node_by_qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL; + if (node && vec) { + gb_write_variants(node, vec); + } + gb_span_vec_free(vec); +} + +static void gb_free_span_vec(const char *qn, void *value, void *ud) { + (void)qn; + (void)ud; + gb_span_vec_free(value); +} + +void cbm_gbuf_foreach_span(const cbm_gbuf_t *gb, const cbm_gbuf_node_t *n, cbm_gbuf_span_fn fn, + void *ud) { + if (!gb || !n || !fn) { + return; + } + const gb_span_vec_t *vec = gb->variant_spans && n->qualified_name + ? cbm_ht_get(gb->variant_spans, n->qualified_name) + : NULL; + if (vec && vec->count > 0) { + for (size_t i = 0; i < vec->count; i++) { + fn(vec->items[i].file, vec->items[i].start, vec->items[i].end, ud); + } + return; + } + if (gb_props_have_variants(n->properties_json)) { + yyjson_doc *doc = yyjson_read_opts((char *)n->properties_json, strlen(n->properties_json), + 0, &gb_yy_alc, NULL); + yyjson_val *list = gb_variants_list(doc); size_t i; size_t max; yyjson_val *v; + bool any = false; yyjson_arr_foreach(list, i, max, v) { const char *f = yyjson_get_str(yyjson_obj_get(v, "file_path")); int s = (int)yyjson_get_int(yyjson_obj_get(v, "start_line")); int e = (int)yyjson_get_int(yyjson_obj_get(v, "end_line")); - if (f && s > 0 && n < GB_VARIANTS_MAX) { - out[n++] = (gb_span_t){f, s, e}; + if (f && s > 0) { + fn(f, s, e, ud); + any = true; } } - return n; + yyjson_doc_free(doc); + if (any) { + return; + } } - if (start > 0 && n < GB_VARIANTS_MAX) { - out[n++] = (gb_span_t){file ? file : "", start, end}; + if (n->file_path && n->start_line > 0) { + fn(n->file_path, n->start_line, n->end_line, ud); } - return n; } -/* The survivor's properties with the union of both definitions' variant - * spans. Returns a GBUF_STRING-owned JSON string, or NULL when there is - * nothing to record (fewer than two distinct spans) or on any failure — the - * survivor then keeps its own properties: a lost variants list costs - * precision, never data. */ -static char *gb_variants_props(const char *survivor_props, const char *a_props, const char *a_file, - int a_start, int a_end, const char *b_props, const char *b_file, - int b_start, int b_end) { - if (a_start <= 0 || b_start <= 0) { - return NULL; - } - yyjson_doc *a_doc = - a_props ? yyjson_read_opts((char *)a_props, strlen(a_props), 0, &gb_yy_alc, NULL) : NULL; - yyjson_doc *b_doc = - b_props ? yyjson_read_opts((char *)b_props, strlen(b_props), 0, &gb_yy_alc, NULL) : NULL; - gb_span_t spans[GB_VARIANTS_MAX * 2]; - int n = gb_spans_of(a_doc, a_file, a_start, a_end, spans, 0); - n = gb_spans_of(b_doc, b_file, b_start, b_end, spans, n); - qsort(spans, (size_t)n, sizeof(spans[0]), gb_span_cmp); - int unique = 0; - for (int i = 0; i < n; i++) { - if (unique == 0 || gb_span_cmp(&spans[unique - 1], &spans[i]) != 0) { - spans[unique++] = spans[i]; - } - } - char *out = NULL; - if (unique >= 2) { - yyjson_mut_doc *mut = yyjson_mut_doc_new(&gb_yy_alc); - yyjson_doc *s_doc = survivor_props - ? yyjson_read_opts((char *)survivor_props, strlen(survivor_props), - 0, &gb_yy_alc, NULL) - : NULL; - yyjson_mut_val *root = s_doc && yyjson_is_obj(yyjson_doc_get_root(s_doc)) - ? yyjson_val_mut_copy(mut, yyjson_doc_get_root(s_doc)) - : yyjson_mut_obj(mut); - yyjson_mut_val *list = yyjson_mut_arr(mut); - for (int i = 0; root && list && i < unique; i++) { - yyjson_mut_val *v = yyjson_mut_arr_add_obj(mut, list); - yyjson_mut_obj_add_strcpy(mut, v, "file_path", spans[i].file); - yyjson_mut_obj_add_int(mut, v, "start_line", spans[i].start); - yyjson_mut_obj_add_int(mut, v, "end_line", spans[i].end); - } - if (mut && root && list) { - yyjson_mut_obj_remove_key(root, "variants"); - yyjson_mut_obj_add_val(mut, root, "variants", list); - yyjson_mut_doc_set_root(mut, root); - out = yyjson_mut_write_opts(mut, 0, &gb_yy_alc, NULL, NULL); - } - yyjson_doc_free(s_doc); - yyjson_mut_doc_free(mut); - } - yyjson_doc_free(a_doc); - yyjson_doc_free(b_doc); - return out; -} - -/* Record the variant spans of a same-QN collision on the surviving node. */ -static void gb_record_variants(cbm_gbuf_node_t *survivor, const char *a_label, const char *a_props, - const char *a_file, int a_start, int a_end, const char *b_label, - const char *b_props, const char *b_file, int b_start, int b_end) { - if (!gb_variant_label(a_label) || !b_label || strcmp(a_label, b_label) != 0) { +void cbm_gbuf_settle_variants(cbm_gbuf_t *gb) { + if (!gb || !gb->variant_spans) { return; } - char *merged = gb_variants_props(survivor->properties_json, a_props, a_file, a_start, a_end, - b_props, b_file, b_start, b_end); - if (merged) { - cbm_free(CBM_MEM_CLASS_GBUF_STRING, survivor->properties_json); - survivor->properties_json = merged; - } + cbm_ht_foreach(gb->variant_spans, gb_settle_one, gb); + cbm_ht_free(gb->variant_spans); + gb->variant_spans = NULL; } /* ── Node operations ─────────────────────────────────────────────── */ @@ -1099,20 +1289,13 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name if (c == 0) { c = strcmp(existing->label ? existing->label : "", label ? label : ""); } + gb_def_t arrival = {label, properties_json, file_path, start_line, end_line}; + gb_note_collision(gb, existing, &arrival, NULL); if (c > 0) { - gb_record_variants(existing, existing->label, existing->properties_json, - existing->file_path, existing->start_line, existing->end_line, label, - properties_json, file_path, start_line, end_line); return existing->id; /* existing entity is the canonical winner */ } - /* The arrival wins: its properties, plus every variant span both carry. */ - char *variant_props = NULL; - if (gb_variant_label(label) && existing->label && strcmp(existing->label, label) == 0) { - variant_props = - gb_variants_props(properties_json, existing->properties_json, existing->file_path, - existing->start_line, existing->end_line, properties_json, - file_path, start_line, end_line); - } + /* The arrival wins: its properties; the variant spans both carry are + * written on export (cbm_gbuf_settle_variants). */ /* Update in-place. name/properties are strdup'd BEFORE freeing old ones * (callers may pass existing->name as an argument). label/file_path are * interned: gb_intern returns a stable pool pointer (idempotent even when @@ -1122,9 +1305,7 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name * label/name — cbm_gbuf_find_by_label/name then missed or mis-listed it, * which is how the flickering Function set reached the semantic pass). */ char *new_name = heap_strdup(name); - char *new_props = variant_props ? variant_props - : properties_json ? heap_strdup(properties_json) - : NULL; + char *new_props = properties_json ? heap_strdup(properties_json) : NULL; const char *new_label_interned = gb_intern(gb, label); bool label_changed = !existing->label || !new_label_interned || strcmp(existing->label, new_label_interned) != 0; @@ -1269,6 +1450,7 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label) { intptr_to_ptr(SKIP_ONE)); /* Remove from primary indexes */ + gb_drop_variant_spans(gb, n->qualified_name); cbm_ht_delete(gb->node_by_qn, n->qualified_name); if (n->id >= 0 && n->id < gb->by_id_cap) { gb->by_id[n->id] = NULL; @@ -1316,6 +1498,7 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) { remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_name, n->name), n->id); /* Remove from primary indexes */ + gb_drop_variant_spans(gb, n->qualified_name); cbm_ht_delete(gb->node_by_qn, n->qualified_name); if (n->id >= 0 && n->id < gb->by_id_cap) { gb->by_id[n->id] = NULL; @@ -1348,6 +1531,75 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) { return deleted_count; } +/* Whether an edge of `types` (NULL-terminated) touches node `id`. */ +static bool gbuf_node_touched(const cbm_gbuf_t *gb, int64_t id, const char *const *types) { + for (int t = 0; types[t]; t++) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + if ((cbm_gbuf_find_edges_by_target_type(gb, id, types[t], &edges, &count) == 0 && + count > 0) || + (cbm_gbuf_find_edges_by_source_type(gb, id, types[t], &edges, &count) == 0 && + count > 0)) { + return true; + } + } + return false; +} + +int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) { + if (!gb || !label || !edge_types || !edge_types[0]) { + return CBM_NOT_FOUND; + } + node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label); + if (!arr || arr->count == 0) { + return 0; + } + /* The victims first: the per-node unlinking below edits `arr`. */ + cbm_gbuf_node_t **victims = + cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, (size_t)arr->count * sizeof(*victims)); + if (!victims) { + return CBM_NOT_FOUND; + } + int n = 0; + for (int i = 0; i < arr->count; i++) { + const char *qn = arr->items[i]->qualified_name; + cbm_gbuf_node_t *node = qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL; + if (node && !gbuf_node_touched(gb, node->id, edge_types)) { + victims[n++] = node; + } + } + if (n == 0) { + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, victims); + return 0; + } + CBMHashTable *deleted_set = cbm_ht_create((uint32_t)n); + for (int i = 0; i < n; i++) { + cbm_gbuf_node_t *node = victims[i]; + if (!node->qualified_name) { + continue; + } + char id_buf[CBM_SZ_32]; + make_id_key(id_buf, sizeof(id_buf), node->id); + cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf), + intptr_to_ptr(SKIP_ONE)); + remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_label, node->label), node->id); + remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_name, node->name), node->id); + gb_drop_variant_spans(gb, node->qualified_name); + cbm_ht_delete(gb->node_by_qn, node->qualified_name); + if (node->id >= 0 && node->id < gb->by_id_cap) { + gb->by_id[node->id] = NULL; + } + /* As delete_by_file: a NULL QN fails the dump's liveness check. */ + cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->qualified_name); + node->qualified_name = NULL; + } + cascade_delete_edges(gb, deleted_set); + cbm_ht_foreach(deleted_set, free_index_key, NULL); + cbm_ht_free(deleted_set); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, victims); + return n; +} + int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project) { if (!gb || !db_path || !project) { return CBM_NOT_FOUND; @@ -1636,16 +1888,15 @@ int cbm_gbuf_edge_count_by_type(const cbm_gbuf_t *gb, const char *type) { return arr ? arr->count : 0; } -int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type) { - if (!gb || !type) { - return CBM_NOT_FOUND; - } +/* Remove the edges `drop` selects from the array and the dedup index, then + * rebuild the secondary indexes. */ +typedef bool (*gbuf_edge_pred_t)(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud); - /* Remove edges of the given type from array and dedup index */ +static int gbuf_delete_edges_if(cbm_gbuf_t *gb, gbuf_edge_pred_t drop, const void *ud) { int write_idx = 0; for (int i = 0; i < gb->edges.count; i++) { cbm_gbuf_edge_t *e = gb->edges.items[i]; - if (strcmp(e->type, type) == 0) { + if (drop(gb, e, ud)) { char key[EDGE_KEY_BUF]; make_edge_key(key, sizeof(key), e->source_id, e->target_id, e->type, e->properties_json); @@ -1667,6 +1918,68 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type) { return 0; } +static bool gbuf_edge_is_type(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud) { + (void)gb; + return strcmp(e->type, (const char *)ud) == 0; +} + +int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type) { + if (!gb || !type) { + return CBM_NOT_FOUND; + } + return gbuf_delete_edges_if(gb, gbuf_edge_is_type, type); +} + +typedef struct { + const char *type; + const char *label; +} gbuf_type_label_t; + +static bool gbuf_edge_from_label(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud) { + const gbuf_type_label_t *tl = ud; + if (strcmp(e->type, tl->type) != 0) { + return false; + } + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(gb, e->source_id); + return src && src->label && strcmp(src->label, tl->label) == 0; +} + +int cbm_gbuf_delete_edges_from_label(cbm_gbuf_t *gb, const char *type, const char *label) { + if (!gb || !type || !label) { + return CBM_NOT_FOUND; + } + gbuf_type_label_t tl = {type, label}; + return gbuf_delete_edges_if(gb, gbuf_edge_from_label, &tl); +} + +static int gbuf_ptr_cmp(const void *k, const void *v) { + uintptr_t a = (uintptr_t)*(const cbm_gbuf_edge_t *const *)k; + uintptr_t b = (uintptr_t)*(const cbm_gbuf_edge_t *const *)v; + return (a > b) - (a < b); +} + +typedef struct { + const cbm_gbuf_edge_t **edges; + int n; +} gbuf_edge_set_t; + +static bool gbuf_edge_in_set(const cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e, const void *ud) { + (void)gb; + const gbuf_edge_set_t *s = ud; + return bsearch(&e, s->edges, (size_t)s->n, sizeof(*s->edges), gbuf_ptr_cmp) != NULL; +} + +/* By the edges themselves, not their ids: after the parallel resolve merge, + * cbm_gbuf_set_next_id can hand an id out twice. */ +int cbm_gbuf_delete_edges(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **edges, int n) { + if (!gb || !edges || n <= 0) { + return 0; + } + qsort((void *)edges, (size_t)n, sizeof(*edges), gbuf_ptr_cmp); + gbuf_edge_set_t s = {edges, n}; + return gbuf_delete_edges_if(gb, gbuf_edge_in_set, &s); +} + /* ── Merge ───────────────────────────────────────────────────────── */ /* Free remap hash table entries (key = heap string, value = heap int64_t*) */ @@ -1679,7 +1992,8 @@ static void free_remap_entry(const char *key, void *val, void *ud) { /* Handle QN collision: update dst node fields (src wins), record remap if IDs differ. * label/file_path are re-interned into dst's pool (sn's pointers belong to src). */ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, - const cbm_gbuf_node_t *sn, CBMHashTable **remap) { + const cbm_gbuf_node_t *sn, const gb_span_vec_t *sn_spans, + CBMHashTable **remap) { /* Same guard as cbm_gbuf_upsert_node: a per-file "Module" def coming from a * worker-local gbuf must not touch the structural directory node ("Project" * root or "Folder") that shares its QN in a directory-based-module language @@ -1718,22 +2032,12 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, c = strcmp(existing->label ? existing->label : "", sn->label ? sn->label : ""); } bool sn_wins = c <= 0; - if (!sn_wins) { - gb_record_variants(existing, existing->label, existing->properties_json, - existing->file_path, existing->start_line, existing->end_line, - sn->label, sn->properties_json, sn->file_path, sn->start_line, - sn->end_line); - } + gb_def_t src_def = {sn->label, sn->properties_json, sn->file_path, sn->start_line, + sn->end_line}; + gb_note_collision(dst, existing, &src_def, sn_spans); if (sn_wins) { - /* src's properties, plus every variant span both carry. */ - char *variant_props = NULL; - if (gb_variant_label(sn->label) && existing->label && - strcmp(existing->label, sn->label) == 0) { - variant_props = gb_variants_props(sn->properties_json, existing->properties_json, - existing->file_path, existing->start_line, - existing->end_line, sn->properties_json, - sn->file_path, sn->start_line, sn->end_line); - } + /* src's properties; the variant spans both carry are written on + * export (cbm_gbuf_settle_variants). */ /* Keep the secondary indexes consistent when the surviving * label/name changes (the old code left the node listed under its * original label/name, so find_by_label/name mis-listed it). */ @@ -1758,10 +2062,9 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, existing->file_path = (char *)gb_intern(dst, sn->file_path); existing->start_line = sn->start_line; existing->end_line = sn->end_line; - if (variant_props || sn->properties_json) { + if (sn->properties_json) { cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->properties_json); - existing->properties_json = - variant_props ? variant_props : heap_strdup(sn->properties_json); + existing->properties_json = heap_strdup(sn->properties_json); } if (label_changed) { node_ptr_array_t *by_label = get_or_create_node_array( @@ -1862,11 +2165,16 @@ int cbm_gbuf_merge(cbm_gbuf_t *dst, cbm_gbuf_t *src) { continue; } + /* The spans src gathered for this node travel with it; they are + * written once, when dst is exported. */ + const gb_span_vec_t *spans = + src->variant_spans ? cbm_ht_get(src->variant_spans, sn->qualified_name) : NULL; cbm_gbuf_node_t *existing = cbm_ht_get(dst->node_by_qn, sn->qualified_name); if (existing) { - merge_update_existing(dst, existing, sn, &remap); + merge_update_existing(dst, existing, sn, spans, &remap); } else { merge_copy_new_node(dst, sn); + gb_adopt_spans(dst, cbm_ht_get(dst->node_by_qn, sn->qualified_name), spans); } } @@ -2155,6 +2463,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { if (!gb || !path) { return CBM_NOT_FOUND; } + cbm_gbuf_settle_variants(gb); CBM_PROF_START(t_count); int live_count = count_live_nodes(gb); @@ -2272,6 +2581,7 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) { if (!gb || !store) { return CBM_NOT_FOUND; } + cbm_gbuf_settle_variants(gb); /* Upsert project */ cbm_store_upsert_project(store, gb->project, gb->root_path); @@ -2346,6 +2656,7 @@ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) { if (!gb || !store) { return CBM_NOT_FOUND; } + cbm_gbuf_settle_variants(gb); /* Begin bulk mode — no project wipe */ cbm_store_begin(store); diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 0553f47f0..83470f5ab 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -117,6 +117,13 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label); * Used by incremental indexing to remove stale nodes before re-extraction. */ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path); +/* Delete every `label` node that no edge of `edge_types` (NULL-terminated) + * touches, as source or target (with any edges it still has); returns how + * many. For nodes that exist only through such edges and carry no file + * (Process, Image): incremental indexing drops them when their last referrer + * is gone, as a full index never makes them. */ +int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types); + /* Bulk-load all nodes and edges for a project from an existing SQLite DB * into this graph buffer. Returns 0 on success. */ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project); @@ -159,6 +166,13 @@ int cbm_gbuf_edge_count_by_type(const cbm_gbuf_t *gb, const char *type); /* Delete all edges of a type. */ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type); +/* Delete the edges of `type` whose source node has label `label`. */ +int cbm_gbuf_delete_edges_from_label(cbm_gbuf_t *gb, const char *type, const char *label); + +/* Delete edges[0..n) (pointers found in this buffer; the array is sorted in + * place). */ +int cbm_gbuf_delete_edges(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **edges, int n); + /* ── Vector storage (for semantic embeddings) ───────────────────── */ /* Store an int8-quantized vector for a node. The vector data is copied. @@ -195,4 +209,17 @@ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store); * (the old value is kept). */ int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json); +/* Write the definition spans gathered from same-QN collisions into each + * surviving node's `variants` property. The dump, flush and merge entry + * points do it themselves; call it directly only to read a node's + * properties_json from the buffer after collisions. */ +void cbm_gbuf_settle_variants(cbm_gbuf_t *gb); + +/* Visit every definition span node `n` stands for: the spans gathered from + * same-QN collisions (not yet written, or already in its `variants`), else + * its own. For passes that place a source line inside a definition. */ +typedef void (*cbm_gbuf_span_fn)(const char *file, int start, int end, void *ud); +void cbm_gbuf_foreach_span(const cbm_gbuf_t *gb, const cbm_gbuf_node_t *n, cbm_gbuf_span_fn fn, + void *ud); + #endif /* CBM_GRAPH_BUFFER_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 412e92269..73afd3373 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -566,6 +566,8 @@ static const tool_def_t TOOLS[] = { "\"query\":{\"type\":\"string\"}," "\"label\":{\"type\":\"string\"},\"name_pattern\":{\"type\":\"string\"},\"qn_pattern\":{" "\"type\":\"string\"},\"file_pattern\":{\"type\":\"string\"}," + "\"kind\":{\"type\":\"string\",\"description\":\"Resource kind, e.g. Deployment.\"}," + "\"namespace\":{\"type\":\"string\",\"description\":\"Resource namespace.\"}," "\"relationship\":{\"type\":\"string\"},\"min_degree\":{\"type\":\"integer\"}," "\"max_degree\":{\"type\":\"integer\"},\"exclude_entry_points\":{\"type\":\"boolean\"}," "\"include_connected\":{\"type\":\"boolean\"},\"semantic_query\":{" @@ -623,8 +625,12 @@ static const tool_def_t TOOLS[] = { "\"cursor\":{\"type\":\"string\",\"description\":\"Pass next/next_cursor with traversal " "args unchanged; budget may increase.\"}," "\"mode\":{" - "\"type\":\"string\",\"enum\":[\"calls\",\"data_flow\",\"cross_service\"],\"default\":" - "\"calls\",\"description\":\"calls, argument-aware data_flow, or service edges.\"}," + "\"type\":\"string\",\"enum\":[\"calls\",\"data_flow\",\"cross_service\",\"topology\"," + "\"deploy\",\"run\"]," + "\"default\":\"calls\",\"description\":\"calls; argument-aware data_flow; service edges; " + "topology: infra objects, routes, controllers (from a Resource/Route); deploy: what CI " + "jobs and scripts run and deploy; run: what a Service runs (image, Dockerfile, program, " + "env).\"}," "\"parameter_name\":{\"type\":\"string\",\"description\":\"data_flow parameter filter.\"}," "\"edge_types\":{\"type\":\"array\",\"items\":{" "\"type\":\"string\"}},\"risk_labels\":{\"type\":\"boolean\",\"default\":false," @@ -1553,6 +1559,23 @@ char *cbm_mcp_get_string_arg(const char *args_json, const char *key) { return result; } +/* String argument `key` copied into `buf` (truncated to fit): buf, or NULL + * when absent or empty. Nothing to free. */ +static const char *mcp_string_arg_buf(const char *args_json, const char *key, char *buf, + size_t buf_sz) { + buf[0] = '\0'; + yyjson_doc *doc = yyjson_read(args_json, strlen(args_json), 0); + if (!doc) { + return NULL; + } + yyjson_val *val = yyjson_obj_get(yyjson_doc_get_root(doc), key); + if (val && yyjson_is_str(val)) { + snprintf(buf, buf_sz, "%s", yyjson_get_str(val)); + } + yyjson_doc_free(doc); + return buf[0] ? buf : NULL; +} + static char *canonicalize_repo_path_if_exists(char *repo_path) { if (!repo_path) { return NULL; @@ -3943,6 +3966,8 @@ enum { BM25_BIND_FILE = 6, BM25_BIND_LABEL = 7, BM25_BIND_EXACT = 8, + BM25_BIND_KIND = 9, + BM25_BIND_NS = 10, BM25_SQL_AUTO_LEN = -1, /* Inner FTS5 candidate cap. SQLite can early-terminate a plain FTS5 query * (no JOIN/WHERE on outer table) of the form: @@ -4200,8 +4225,9 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total, * Returns NULL if FTS5 is unavailable or the query produced no usable tokens, * in which case the caller falls back to the regex-based search path. */ static char *bm25_search(cbm_store_t *store, const char *project, const char *query, - const char *file_pattern, const char *label, int limit, int offset, - bool tree_format, size_t max_output_bytes) { + const char *file_pattern, const char *label, const char *kind, + const char *ns, int limit, int offset, bool tree_format, + size_t max_output_bytes) { sqlite3 *db = cbm_store_get_db(store); if (!db) { return NULL; @@ -4264,6 +4290,14 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu * in the structural mode (2026-09-16 probe: `label=Class` was ignored * here and five Methods came back). MIRRORED in the count query. */ " AND (?7 IS NULL OR n.label = ?7) " + /* Infra filters, guarded (json_extract raises on malformed JSON). + * MIRRORED in the count query. */ + " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) THEN " + " json_extract(n.properties,'$.kind') END) = ?9) " + " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) THEN " + " json_extract(n.properties,'$.obj_key') END) LIKE ('%/%/' || " + " replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),'_','\\_') || '/%') " + " ESCAPE '\\') " /* rank ties are common (boosted floats) — the id tie-break makes * offset pages contractually stable across calls. */ "ORDER BY rank, n.id " @@ -4290,25 +4324,42 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu sqlite3_bind_null(stmt, BM25_BIND_LABEL); } sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + if (kind && kind[0]) { + sqlite3_bind_text(stmt, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(stmt, BM25_BIND_KIND); + } + if (ns && ns[0]) { + sqlite3_bind_text(stmt, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(stmt, BM25_BIND_NS); + } /* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT. * Uses the identical subquery structure so the FTS5 early-exit applies here too. */ int total = 0; { - const char *count_sql = "SELECT COUNT(*) FROM (" - " SELECT fts.rowid FROM (" - " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1" - " ORDER BY " BM25_WEIGHTS " LIMIT ?3" - " ) fts " - " JOIN nodes n ON n.id = fts.rowid " - " WHERE n.project = ?2 " - /* MIRRORS the ranked query's filter verbatim — same weights, same - * label exclusions. Changing one alone reports a total that does - * not describe the rows returned. */ - " AND n.label NOT IN ('File','Folder','Variable','Project')" - " AND (?6 IS NULL OR n.file_path LIKE ?6)" - " AND (?7 IS NULL OR n.label = ?7)" - ")"; + const char *count_sql = + "SELECT COUNT(*) FROM (" + " SELECT fts.rowid FROM (" + " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1" + " ORDER BY " BM25_WEIGHTS " LIMIT ?3" + " ) fts " + " JOIN nodes n ON n.id = fts.rowid " + " WHERE n.project = ?2 " + /* MIRRORS the ranked query's filter verbatim — same weights, same + * label exclusions. Changing one alone reports a total that does + * not describe the rows returned. */ + " AND n.label NOT IN ('File','Folder','Variable','Project')" + " AND (?6 IS NULL OR n.file_path LIKE ?6)" + " AND (?7 IS NULL OR n.label = ?7)" + " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) " + " THEN json_extract(n.properties,'$.kind') END) = ?9)" + " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) " + " THEN json_extract(n.properties,'$.obj_key') END) " + " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%')," + " '_','\\_') || '/%') ESCAPE '\\')" + ")"; sqlite3_stmt *cs = NULL; if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) { sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, @@ -4328,6 +4379,17 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu } else { sqlite3_bind_null(cs, BM25_BIND_LABEL); } + if (kind && kind[0]) { + sqlite3_bind_text(cs, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, + MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(cs, BM25_BIND_KIND); + } + if (ns && ns[0]) { + sqlite3_bind_text(cs, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + } else { + sqlite3_bind_null(cs, BM25_BIND_NS); + } if (sqlite3_step(cs) == SQLITE_ROW) { total = sqlite3_column_int(cs, 0); } @@ -5306,8 +5368,12 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { if (query && query[0]) { char *q_file_pattern = cbm_mcp_get_string_arg(args, "file_pattern"); char *q_label = cbm_mcp_get_string_arg(args, "label"); - char *bm25_json = bm25_search(store, project, query, q_file_pattern, q_label, limit, offset, - !json_format, max_output_bytes); + char q_kind_buf[CBM_SZ_256]; + char q_ns_buf[CBM_SZ_256]; + const char *q_kind = mcp_string_arg_buf(args, "kind", q_kind_buf, sizeof(q_kind_buf)); + const char *q_ns = mcp_string_arg_buf(args, "namespace", q_ns_buf, sizeof(q_ns_buf)); + char *bm25_json = bm25_search(store, project, query, q_file_pattern, q_label, q_kind, q_ns, + limit, offset, !json_format, max_output_bytes); free(q_file_pattern); free(q_label); if (bm25_json) { @@ -5321,6 +5387,10 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { free(query); char *label = cbm_mcp_get_string_arg(args, "label"); + char kind_buf[CBM_SZ_256]; + char ns_buf[CBM_SZ_256]; + const char *kind_arg = mcp_string_arg_buf(args, "kind", kind_buf, sizeof(kind_buf)); + const char *ns_arg = mcp_string_arg_buf(args, "namespace", ns_buf, sizeof(ns_buf)); char *name_pattern = cbm_mcp_get_string_arg(args, "name_pattern"); char *qn_pattern = cbm_mcp_get_string_arg(args, "qn_pattern"); char *file_pattern = cbm_mcp_get_string_arg(args, "file_pattern"); @@ -5353,6 +5423,8 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { .offset = offset, .min_degree = min_degree, .max_degree = max_degree, + .kind = kind_arg, + .namespace_name = ns_arg, }; const char *fields[SG_MAX_EXTRA_FIELDS]; yyjson_doc *fields_owner = NULL; @@ -8651,6 +8723,22 @@ static yyjson_doc *resolve_trace_edge_types(const char *args, const char *mode, "HTTP_CALLS", "ASYNC_CALLS", "DATA_FLOWS", "CALLS", "CROSS_HTTP_CALLS", "CROSS_ASYNC_CALLS", "CROSS_CHANNEL", "CROSS_GRPC_CALLS", "CROSS_GRAPHQL_CALLS", "CROSS_TRPC_CALLS"}; + /* topology: traffic (INFRA_MAPS), object references (USAGE), ordering + * (DEPENDS_ON), controllers (HANDLES), what an object declares + * (DEFINES: an Ingress's routes) and what it deploys (DEPLOYS: a Flux + * Kustomization's path, a HelmRelease's chart) */ + static const char *mode_topology[] = {"INFRA_MAPS", "USAGE", "DEPENDS_ON", + "HANDLES", "DEFINES", "DEPLOYS"}; + /* deploy: what a CI job, script or Make rule starts (SPAWNS: scripts, + * make targets, programs), calls (CALLS: shell functions) and puts on a + * cluster (DEPLOYS) -- outbound from the job, inbound from a chart */ + static const char *mode_deploy[] = {"DEPLOYS", "SPAWNS", "CALLS"}; + /* run: what a Service or workload runs -- traffic to the workload + * (INFRA_MAPS), the Image its container runs (USAGE), the Dockerfile + * that builds it and what that builds on (DEPENDS_ON), the program the + * ENTRYPOINT starts (SPAWNS) and the environment each container is + * given (CONFIGURES) */ + static const char *mode_run[] = {"INFRA_MAPS", "USAGE", "DEPENDS_ON", "SPAWNS", "CONFIGURES"}; *out_count = 0; @@ -8683,6 +8771,15 @@ static yyjson_doc *resolve_trace_edge_types(const char *args, const char *mode, } else if (mode && strcmp(mode, "cross_service") == 0) { defaults = mode_cross_svc; n_defaults = (int)(sizeof(mode_cross_svc) / sizeof(mode_cross_svc[0])); + } else if (mode && strcmp(mode, "topology") == 0) { + defaults = mode_topology; + n_defaults = (int)(sizeof(mode_topology) / sizeof(mode_topology[0])); + } else if (mode && strcmp(mode, "deploy") == 0) { + defaults = mode_deploy; + n_defaults = (int)(sizeof(mode_deploy) / sizeof(mode_deploy[0])); + } else if (mode && strcmp(mode, "run") == 0) { + defaults = mode_run; + n_defaults = (int)(sizeof(mode_run) / sizeof(mode_run[0])); } for (int i = 0; i < n_defaults; i++) { out_types[i] = defaults[i]; diff --git a/src/mcp/test_impact_engine_reach.c b/src/mcp/test_impact_engine_reach.c index dd581a4f5..49a1142cc 100644 --- a/src/mcp/test_impact_engine_reach.c +++ b/src/mcp/test_impact_engine_reach.c @@ -22,8 +22,9 @@ * reached, never expanded (their caller is the runner's main). */ #define TE_RUNNER_ROLE_PREFIX "tf_maybe_run_" -/* SPAWNS only carries reach from a Process node, which enters the walk - * through te_spawn_lift alone. */ +/* SPAWNS carries reach from a Process node, which enters the walk through + * te_spawn_lift alone, and from a script of the repository (its Module / + * File, pass_shell.c) to whoever runs it. */ static const char *const te_walk_types[] = {"CALLS", "ASYNC_CALLS", "USAGE", "CALL_REFERENCE", "READS", "SPAWNS"}; diff --git a/src/pipeline/fqn.c b/src/pipeline/fqn.c index 96e959a0b..f391ce1e9 100644 --- a/src/pipeline/fqn.c +++ b/src/pipeline/fqn.c @@ -63,9 +63,21 @@ static char *join_segments(const char **segments, int count) { } /* Strip file extension from the last path component. */ +/* A container build file named for its variant ("Dockerfile.debug") keeps + * the suffix: it is no language extension, and dropping it would give the + * variant the QN of the plain Dockerfile beside it. Mirrors strip_ext_len in + * internal/cbm/helpers.c -- both must agree. */ +static bool fqn_build_file_variant(const char *base) { + return strncmp(base, "Dockerfile.", strlen("Dockerfile.")) == 0 || + strncmp(base, "Containerfile.", strlen("Containerfile.")) == 0; +} + static void strip_file_extension(char *path) { char *last_slash = strrchr(path, '/'); char *start = last_slash ? last_slash + SKIP_ONE : path; + if (fqn_build_file_variant(start)) { + return; + } char *ext = strrchr(start, '.'); if (ext) { *ext = '\0'; diff --git a/src/pipeline/pass_ci.c b/src/pipeline/pass_ci.c new file mode 100644 index 000000000..c8b9a4467 --- /dev/null +++ b/src/pipeline/pass_ci.c @@ -0,0 +1,1186 @@ +/* + * pass_ci.c — the command lines CI configurations run, handed to the shell + * layer (pass_shell.c), which links what they deploy and start. + * + * PHASE 3 P3.5. Read with tree-sitter-yaml: + * - GitHub Actions: a workflow (.github/workflows/.yml|yaml at the + * root -- GitHub reads no other) and a composite action (any + * action.yml|yaml with runs.using: composite). A step's run: is a shell + * text unless a shell other than bash / sh is named (step > job + * defaults.run > workflow defaults.run), or none is and the job's runs-on + * names a Windows runner (whose default is pwsh). It starts in the + * workspace -- the root -- or the working-directory (same precedence); env + * literals (workflow < job < step) are variables, a value holding `${{` an + * unknown one. In the text, `${{ github.workspace }}` is + * $GITHUB_WORKSPACE, `${{ github.action_path }}` $GITHUB_ACTION_PATH, any + * other `${{ }}` an unknown variable (its line breaks kept). + * - Prow: a file with a top-level presubmits / postsubmits / periodics key + * (in-repo job config). A job container's command + args is exec'd, no + * shell: the words are literal, a kubelet `$(VAR)` unknown; `bash -c TEXT` + * runs TEXT as shell. A decorated job starts in the repository's checkout + * (the root); undecorated, with an extra_ref taking the working directory, + * or with a container workingDir, the directory is unknown. + * Scalars: plain, quoted and literal block (|) values; a folded (>) block or + * one with an indentation indicator is skipped. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "pipeline/pipeline_internal.h" +#include "cbm.h" +#include "lang_specs.h" + +#include +#include +#include +#include +#include + +static const char CI_GHA[] = "github-actions"; +static const char CI_PROW[] = "prow"; + +static const char CI_EXPR_OPEN[] = "${{"; +#define CI_EXPR_OPEN_LEN (sizeof(CI_EXPR_OPEN) - SKIP_ONE) +static const char CI_QUOTED_QUOTE[] = "'\\''"; +#define CI_QUOTED_QUOTE_LEN (sizeof(CI_QUOTED_QUOTE) - SKIP_ONE) +/* A rewritten run: text is at most this many times the original: the + * shortest expression `${{}}` (5 bytes) becomes `${__ci_expr}` (12). */ +enum { CI_TEXT_GROWTH = 3 }; + +typedef struct { + CBMArena *a; + const char *src; + const char *rel; + cbm_ci_cmd_fn fn; + cbm_build_site_fn site_fn; /* a build-site walk's callback */ + cbm_compose_service_fn service_fn; /* a compose walk's callback */ + void *ud; +} ci_t; + +/* name, value pairs; a value NULL: unknown. */ +typedef struct { + const char **kv; + int n; +} ci_env_t; + +/* What a step inherits from its job and workflow. */ +typedef struct { + ci_env_t env; + const char *wd; /* working-directory; NULL: none named */ + const char *shell; /* NULL: none named */ + bool windows; + const char *action_dir; /* a composite action's directory */ +} ci_scope_t; + +typedef struct { + const char *key; + TSNode value; /* may be null: `key:` without a value */ + int row; /* the key's 0-based row */ +} ci_pair_t; + +static bool ci_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +/* The content of a node, through block_node / flow_node wrappers and past + * comments, anchors and tags; null for nothing (an alias stays an alias). */ +static TSNode ci_content(TSNode n) { + while (ci_type(n, "block_node") || ci_type(n, "flow_node")) { + TSNode next = {0}; + uint32_t k = ts_node_named_child_count(n); + for (uint32_t i = 0; i < k; i++) { + TSNode c = ts_node_named_child(n, i); + if (!ci_type(c, "comment") && !ci_type(c, "anchor") && !ci_type(c, "tag")) { + next = c; + break; + } + } + n = next; + } + return n; +} + +/* A flow scalar's value: line breaks folded (a single one to a space, n + * empty lines to n newlines), each line trimmed; `q` the quote (0: plain) + * whose escapes apply. */ +static const char *ci_fold(CBMArena *a, const char *s, size_t n, char q) { + char *out = cbm_arena_alloc(a, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t o = 0; + size_t i = 0; + while (i < n) { + char ch = s[i]; + if (ch == '\n' || ch == '\r') { + while (o > 0 && (out[o - SKIP_ONE] == ' ' || out[o - SKIP_ONE] == '\t')) { + o--; + } + size_t j = i; + int breaks = 0; + while (j < n && (s[j] == ' ' || s[j] == '\t' || s[j] == '\r' || s[j] == '\n')) { + breaks += s[j] == '\n'; + j++; + } + if (breaks > SKIP_ONE) { + for (int k = SKIP_ONE; k < breaks; k++) { + out[o++] = '\n'; + } + } else { + out[o++] = ' '; + } + i = j; + continue; + } + if (q == '"' && ch == '\\' && i + SKIP_ONE < n) { + char x = s[i + SKIP_ONE]; + i += PAIR_LEN; + if (x == '\n' || x == '\r') { + while (i < n && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) { + i++; + } + continue; + } + out[o++] = x == 'n' ? '\n' : x == 't' ? '\t' : x; + continue; + } + if (q == '\'' && ch == '\'' && i + SKIP_ONE < n && s[i + SKIP_ONE] == '\'') { + out[o++] = '\''; + i += PAIR_LEN; + continue; + } + out[o++] = ch; + i++; + } + out[o] = '\0'; + return out; +} + +/* A literal block scalar's value (s at its `|`); NULL for a folded block or + * an indentation indicator. */ +static const char *ci_literal(CBMArena *a, const char *s, size_t n) { + if (n == 0 || s[0] != '|') { + return NULL; + } + size_t i = SKIP_ONE; + while (i < n && s[i] != '\n' && s[i] != '#') { + if (s[i] >= '1' && s[i] <= '9') { + return NULL; + } + i++; + } + while (i < n && s[i] != '\n') { + i++; + } + i += i < n ? SKIP_ONE : 0; + size_t indent = 0; + for (size_t j = i; j < n;) { + size_t k = j; + while (k < n && s[k] == ' ') { + k++; + } + if (k < n && s[k] != '\n' && s[k] != '\r') { + indent = k - j; + break; + } + while (k < n && s[k] != '\n') { + k++; + } + j = k + SKIP_ONE; + } + char *out = cbm_arena_alloc(a, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t o = 0; + while (i < n) { + size_t e = i; + while (e < n && s[e] != '\n') { + e++; + } + size_t k = i; + while (k < e && k - i < indent && s[k] == ' ') { + k++; + } + if (k - i == indent) { + memcpy(out + o, s + k, e - k); + o += e - k; + } + if (o > 0 && out[o - SKIP_ONE] == '\r') { + o--; + } + if (e < n) { + out[o++] = '\n'; + } + i = e + SKIP_ONE; + } + out[o] = '\0'; + return out; +} + +/* A scalar's value and the 0-based row of its first character; NULL: no + * scalar this reader decodes. */ +static const char *ci_scalar(const ci_t *c, TSNode n, int *row) { + n = ci_content(n); + if (ts_node_is_null(n)) { + return NULL; + } + uint32_t b = ts_node_start_byte(n); + size_t len = ts_node_end_byte(n) - b; + const char *s = c->src + b; + if (row) { + *row = (int)ts_node_start_point(n).row; + } + if (ci_type(n, "plain_scalar")) { + return ci_fold(c->a, s, len, 0); + } + if (len >= PAIR_LEN && ci_type(n, "double_quote_scalar")) { + return ci_fold(c->a, s + SKIP_ONE, len - PAIR_LEN, '"'); + } + if (len >= PAIR_LEN && ci_type(n, "single_quote_scalar")) { + return ci_fold(c->a, s + SKIP_ONE, len - PAIR_LEN, '\''); + } + if (ci_type(n, "block_scalar")) { + if (row) { + (*row)++; + } + return ci_literal(c->a, s, len); + } + return NULL; +} + +/* The pairs of mapping `m` with a scalar key; *n their count. */ +static ci_pair_t *ci_pairs(const ci_t *c, TSNode m, int *n) { + *n = 0; + m = ci_content(m); + if (!ci_type(m, "block_mapping") && !ci_type(m, "flow_mapping")) { + return NULL; + } + uint32_t k = ts_node_named_child_count(m); + ci_pair_t *out = cbm_arena_alloc(c->a, (k + SKIP_ONE) * sizeof(*out)); + if (!out) { + return NULL; + } + for (uint32_t i = 0; i < k; i++) { + TSNode p = ts_node_named_child(m, i); + if (!ci_type(p, "block_mapping_pair") && !ci_type(p, "flow_pair")) { + continue; + } + int row = 0; + const char *key = ci_scalar(c, ts_node_child_by_field_name(p, TS_FIELD("key")), &row); + if (key) { + out[(*n)++] = (ci_pair_t){key, ts_node_child_by_field_name(p, TS_FIELD("value")), row}; + } + } + return out; +} + +/* The value of `key` in mapping `m` (the first such pair); null: none. */ +static TSNode ci_get(const ci_t *c, TSNode m, const char *key) { + int n = 0; + ci_pair_t *pairs = ci_pairs(c, m, &n); + for (int i = 0; i < n; i++) { + if (strcmp(pairs[i].key, key) == 0) { + return pairs[i].value; + } + } + TSNode none = {0}; + return none; +} + +static const char *ci_get_str(const ci_t *c, TSNode m, const char *key) { + return ci_scalar(c, ci_get(c, m, key), NULL); +} + +/* The item contents of a sequence; *n their count. */ +static TSNode *ci_items(const ci_t *c, TSNode seq, int *n) { + *n = 0; + seq = ci_content(seq); + bool block = ci_type(seq, "block_sequence"); + if (!block && !ci_type(seq, "flow_sequence")) { + return NULL; + } + uint32_t k = ts_node_named_child_count(seq); + TSNode *out = cbm_arena_alloc(c->a, (k + SKIP_ONE) * sizeof(*out)); + if (!out) { + return NULL; + } + for (uint32_t i = 0; i < k; i++) { + TSNode it = ts_node_named_child(seq, i); + if (block && ci_type(it, "block_sequence_item")) { + TSNode v = {0}; + for (uint32_t j = 0; j < ts_node_named_child_count(it); j++) { + TSNode x = ts_node_named_child(it, j); + if (!ci_type(x, "comment")) { + v = x; + break; + } + } + out[(*n)++] = ci_content(v); + } else if (!block && ci_type(it, "flow_node")) { + out[(*n)++] = ci_content(it); + } + } + return out; +} + +/* `base` and then the pairs of env mapping `m`: a non-scalar or `${{` value + * is unknown, an absent one empty. */ +static ci_env_t ci_env_add(const ci_t *c, const ci_env_t *base, TSNode m) { + int n = 0; + ci_pair_t *pairs = ci_pairs(c, m, &n); + int bn = base ? base->n : 0; + ci_env_t out = {cbm_arena_alloc(c->a, (size_t)(bn + n + SKIP_ONE) * PAIR_LEN * sizeof(char *)), + 0}; + if (!out.kv) { + return base ? *base : out; + } + for (int i = 0; i < bn; i++) { + out.kv[PAIR_LEN * out.n] = base->kv[PAIR_LEN * i]; + out.kv[PAIR_LEN * out.n + SKIP_ONE] = base->kv[PAIR_LEN * i + SKIP_ONE]; + out.n++; + } + for (int i = 0; i < n; i++) { + const char *v = ts_node_is_null(pairs[i].value) ? "" : ci_scalar(c, pairs[i].value, NULL); + out.kv[PAIR_LEN * out.n] = pairs[i].key; + out.kv[PAIR_LEN * out.n + SKIP_ONE] = v && !strstr(v, "${{") ? v : NULL; + out.n++; + } + return out; +} + +/* ── GitHub Actions ────────────────────────────────────────────────── */ + +/* The n bytes at b are exactly `word`. */ +static bool ci_span_is(const char *b, size_t n, const char *word) { + return strlen(word) == n && strncmp(b, word, n) == 0; +} + +/* The run: text with its `${{ }}` expressions replaced (see the header), + * CBM_SZ_16 zero bytes past its end. */ +static const char *ci_gha_text(CBMArena *a, const char *t, size_t *len) { + size_t n = strlen(t); + char *out = cbm_arena_alloc(a, n * CI_TEXT_GROWTH + CBM_SZ_16 + CBM_SZ_16); + if (!out) { + return NULL; + } + size_t o = 0; + size_t i = 0; + while (i < n) { + const char *open = t + i + CI_EXPR_OPEN_LEN; + const char *end = + strncmp(t + i, CI_EXPR_OPEN, CI_EXPR_OPEN_LEN) == 0 ? strstr(open, "}}") : NULL; + if (!end) { + out[o++] = t[i++]; + continue; + } + const char *b = open; + const char *e = end; + while (b < e && (*b == ' ' || *b == '\t')) { + b++; + } + while (e > b && (e[-SKIP_ONE] == ' ' || e[-SKIP_ONE] == '\t')) { + e--; + } + size_t il = (size_t)(e - b); + const char *rep = "${__ci_expr}"; + if (ci_span_is(b, il, "github.workspace")) { + rep = "${GITHUB_WORKSPACE}"; + } else if (ci_span_is(b, il, "github.action_path")) { + rep = "${GITHUB_ACTION_PATH}"; + } + size_t rl = strlen(rep); + memcpy(out + o, rep, rl); + o += rl; + for (const char *p = open; p < end; p++) { + if (*p == '\n') { + out[o++] = '\n'; + } + } + i = (size_t)(end - t) + PAIR_LEN; + } + memset(out + o, 0, CBM_SZ_16); + *len = o; + return out; +} + +/* A shell GitHub runs a step with as bash: `bash ...` / `sh ...`. */ +static bool ci_bash_shell(const char *sh) { + size_t k = strncmp(sh, "bash", strlen("bash")) == 0 ? strlen("bash") + : strncmp(sh, "sh", strlen("sh")) == 0 ? strlen("sh") + : 0; + if (!k) { + return false; + } + char ch = sh[k]; + return !(ch == '_' || (ch >= '0' && ch <= '9') || ((ch | ' ') >= 'a' && (ch | ' ') <= 'z')); +} + +/* `s` contains `word` (lowercase ASCII), ignoring case. */ +static bool ci_has_word(const char *s, const char *word) { + size_t wl = strlen(word); + for (const char *p = s; p && *p; p++) { + size_t k = 0; + while (k < wl && p[k] && (char)(p[k] | ' ') == word[k]) { + k++; + } + if (k == wl) { + return true; + } + } + return false; +} + +/* runs-on names a Windows runner: a label, or any label of a list. */ +static bool ci_windows(const ci_t *c, TSNode runs_on) { + int n = 0; + TSNode *items = ci_items(c, runs_on, &n); + if (!items) { + return ci_has_word(ci_scalar(c, runs_on, NULL), "windows"); + } + for (int i = 0; i < n; i++) { + if (ci_has_word(ci_scalar(c, items[i], NULL), "windows")) { + return true; + } + } + return false; +} + +static const char *ci_run_default(const ci_t *c, TSNode m, const char *key) { + return ci_get_str(c, ci_get(c, ci_get(c, m, "defaults"), "run"), key); +} + +static void ci_gha_steps(const ci_t *c, TSNode steps, const char *job, const ci_scope_t *sc) { + int n = 0; + TSNode *items = ci_items(c, steps, &n); + for (int i = 0; i < n; i++) { + int row = 0; + const char *run = ci_scalar(c, ci_get(c, items[i], "run"), &row); + if (!run) { + continue; + } + const char *shell = ci_get_str(c, items[i], "shell"); + shell = shell ? shell : sc->shell; + if (shell && shell[0] ? !ci_bash_shell(shell) : sc->windows) { + continue; + } + const char *wd = ci_get_str(c, items[i], "working-directory"); + wd = wd ? wd : sc->wd; + if (!wd) { + wd = ""; + } else if (strstr(wd, "${{") || wd[0] == '/' || wd[0] == '~') { + wd = NULL; + } + ci_env_t env = ci_env_add(c, &sc->env, ci_get(c, items[i], "env")); + size_t len = 0; + const char *text = ci_gha_text(c->a, run, &len); + if (!text) { + continue; + } + cbm_ci_cmd_t cmd = {.ci = CI_GHA, + .job = job, + .text = text, + .len = len, + .row = row, + .wd = wd, + .action_dir = sc->action_dir, + .env = env.kv, + .nenv = env.n, + .exec = false}; + c->fn(&cmd, c->ud); + } +} + +static void ci_workflow(const ci_t *c, TSNode root) { + ci_env_t wenv = ci_env_add(c, NULL, ci_get(c, root, "env")); + const char *wwd = ci_run_default(c, root, "working-directory"); + const char *wsh = ci_run_default(c, root, "shell"); + int n = 0; + ci_pair_t *jobs = ci_pairs(c, ci_get(c, root, "jobs"), &n); + for (int i = 0; i < n; i++) { + TSNode j = jobs[i].value; + const char *wd = ci_run_default(c, j, "working-directory"); + const char *sh = ci_run_default(c, j, "shell"); + ci_scope_t sc = {.env = ci_env_add(c, &wenv, ci_get(c, j, "env")), + .wd = wd ? wd : wwd, + .shell = sh ? sh : wsh, + .windows = ci_windows(c, ci_get(c, j, "runs-on")), + .action_dir = NULL}; + ci_gha_steps(c, ci_get(c, j, "steps"), jobs[i].key, &sc); + } +} + +static void ci_action(const ci_t *c, TSNode root) { + TSNode runs = ci_get(c, root, "runs"); + const char *using = ci_get_str(c, runs, "using"); + if (!using || strcmp(using, "composite") != 0) { + return; + } + const char *slash = strrchr(c->rel, '/'); + ci_scope_t sc = {.action_dir = + slash ? cbm_arena_strndup(c->a, c->rel, (size_t)(slash - c->rel)) : ""}; + ci_gha_steps(c, ci_get(c, runs, "steps"), NULL, &sc); +} + +/* ── Prow ──────────────────────────────────────────────────────────── */ + +static bool ci_is_shell(const char *w) { + static const char *const shells[] = {"bash", "sh", "zsh", "ksh", "dash", NULL}; + const char *slash = strrchr(w, '/'); + const char *base = slash ? slash + SKIP_ONE : w; + for (int i = 0; shells[i]; i++) { + if (strcmp(base, shells[i]) == 0) { + return true; + } + } + return false; +} + +/* argv as one shell line of quoted words (a `$(VAR)` word unknown), with + * CBM_SZ_16 zero bytes past its end. */ +static const char *ci_exec_line(CBMArena *a, const char **w, int n, size_t *len) { + size_t cap = CBM_SZ_16 + CBM_SZ_16; + for (int i = 0; i < n; i++) { + cap += strlen(w[i]) * CI_QUOTED_QUOTE_LEN + CBM_SZ_16; + } + char *out = cbm_arena_alloc(a, cap); + if (!out) { + return NULL; + } + size_t o = 0; + for (int i = 0; i < n; i++) { + if (i) { + out[o++] = ' '; + } + if (strstr(w[i], "$(")) { + static const char unknown[] = "\"$__k8s_var\""; + memcpy(out + o, unknown, sizeof(unknown) - SKIP_ONE); + o += sizeof(unknown) - SKIP_ONE; + continue; + } + out[o++] = '\''; + for (const char *p = w[i]; *p; p++) { + if (*p == '\'') { + memcpy(out + o, CI_QUOTED_QUOTE, CI_QUOTED_QUOTE_LEN); + o += CI_QUOTED_QUOTE_LEN; + } else { + out[o++] = *p; + } + } + out[o++] = '\''; + } + memset(out + o, 0, CBM_SZ_16); + *len = o; + return out; +} + +static void ci_prow_container(const ci_t *c, TSNode ct, const char *job, bool root_wd) { + int nc = 0; + int na = 0; + TSNode *cmd = ci_items(c, ci_get(c, ct, "command"), &nc); + TSNode *args = ci_items(c, ci_get(c, ct, "args"), &na); + int n = nc + na; + if (n == 0) { + return; + } + const char **w = cbm_arena_alloc(c->a, (size_t)n * sizeof(*w)); + int *rows = cbm_arena_alloc(c->a, (size_t)n * sizeof(*rows)); + if (!w || !rows) { + return; + } + for (int i = 0; i < n; i++) { + w[i] = ci_scalar(c, i < nc ? cmd[i] : args[i - nc], &rows[i]); + if (!w[i]) { + return; + } + } + const char *wdir = ci_get_str(c, ct, "workingDir"); + cbm_ci_cmd_t out = {.ci = CI_PROW, .job = job, .wd = root_wd && !(wdir && wdir[0]) ? "" : NULL}; + if (ci_is_shell(w[0])) { + int k = SKIP_ONE; + while (k < n && strcmp(w[k], "-c") != 0) { + k++; + } + if (k < n) { + if (k + SKIP_ONE >= n) { + return; + } + size_t tl = strlen(w[k + SKIP_ONE]); + char *text = cbm_arena_alloc(c->a, tl + CBM_SZ_16); + if (!text) { + return; + } + memcpy(text, w[k + SKIP_ONE], tl); + memset(text + tl, 0, CBM_SZ_16); + out.text = text; + out.len = tl; + out.row = rows[k + SKIP_ONE]; + c->fn(&out, c->ud); + return; + } + } + out.text = ci_exec_line(c->a, w, n, &out.len); + out.row = rows[0]; + out.exec = true; + if (out.text) { + c->fn(&out, c->ud); + } +} + +static void ci_prow_job(const ci_t *c, TSNode j) { + const char *name = ci_get_str(c, j, "name"); + const char *decorate = ci_get_str(c, j, "decorate"); + bool root_wd = decorate && strcmp(decorate, "true") == 0; + int n = 0; + TSNode *refs = ci_items(c, ci_get(c, j, "extra_refs"), &n); + for (int i = 0; i < n; i++) { + const char *wd = ci_get_str(c, refs[i], "workdir"); + root_wd = root_wd && !(wd && strcmp(wd, "true") == 0); + } + TSNode *cts = ci_items(c, ci_get(c, ci_get(c, j, "spec"), "containers"), &n); + for (int i = 0; i < n; i++) { + ci_prow_container(c, cts[i], name, root_wd); + } +} + +static void ci_prow_jobs(const ci_t *c, TSNode seq) { + int n = 0; + TSNode *jobs = ci_items(c, seq, &n); + for (int i = 0; i < n; i++) { + ci_prow_job(c, jobs[i]); + } +} + +static void ci_prow(const ci_t *c, TSNode root) { + static const char *const keys[] = {"presubmits", "postsubmits", "periodics", NULL}; + for (int k = 0; keys[k]; k++) { + TSNode v = ci_content(ci_get(c, root, keys[k])); + int n = 0; + ci_pair_t *repos = ci_pairs(c, v, &n); + if (!repos) { + ci_prow_jobs(c, v); + continue; + } + for (int i = 0; i < n; i++) { + ci_prow_jobs(c, repos[i].value); + } + } +} + +/* ── Entry ─────────────────────────────────────────────────────────── */ + +static bool ci_yaml_ext(const char *p) { + const char *dot = strrchr(p, '.'); + return dot && (strcmp(dot, ".yml") == 0 || strcmp(dot, ".yaml") == 0); +} + +static bool ci_workflow_path(const char *rel) { + static const char dir[] = ".github/workflows/"; + return strncmp(rel, dir, sizeof(dir) - SKIP_ONE) == 0 && + !strchr(rel + sizeof(dir) - SKIP_ONE, '/') && ci_yaml_ext(rel); +} + +static bool ci_action_path(const char *rel) { + const char *slash = strrchr(rel, '/'); + const char *base = slash ? slash + SKIP_ONE : rel; + return strcmp(base, "action.yml") == 0 || strcmp(base, "action.yaml") == 0; +} + +/* ── Build configurations (PHASE 4 P4.3) ────────────────────────────── + * Which Dockerfile builds which image, as a skaffold or compose file says: + * - skaffold: every build.artifacts[] item (top level, profiles) of a + * document whose apiVersion is skaffold/...: image, context (default the + * file's directory), docker.dockerfile (default Dockerfile, relative to + * the context); an artifact of another builder (jib, ko, bazel, + * buildpacks, custom, kaniko) is none. + * - compose: services. with build (a context, or context / + * dockerfile) and image; ${X}, ${X:-d}, ${X-d} and $X from the project's + * .env (a variable without a value: none). */ + +static bool ci_build_ident(const char *s, size_t n) { + if (n == 0 || !(isalpha((unsigned char)s[0]) || s[0] == '_')) { + return false; + } + for (size_t i = SKIP_ONE; i < n; i++) { + if (!isalnum((unsigned char)s[i]) && s[i] != '_') { + return false; + } + } + return true; +} + +/* Value v[0, *len) without the quotes around it (*len shortened). */ +static const char *ci_unquote(const char *v, size_t *len) { + size_t l = *len; + if (l >= PAIR_LEN && (v[0] == '"' || v[0] == '\'') && v[l - SKIP_ONE] == v[0]) { + *len = l - PAIR_LEN; + return v + SKIP_ONE; + } + return v; +} + +/* One .env line s[0, n): KEY=VALUE (an `export ` prefix, quotes around the + * value removed) appended to env; a comment, a blank or another line: none. */ +static void ci_dotenv_line(CBMArena *a, const char *s, size_t n, ci_env_t *env) { + static const char EXPORT[] = "export "; + size_t b = 0; + while (b < n && isspace((unsigned char)s[b])) { + b++; + } + while (n > b && isspace((unsigned char)s[n - SKIP_ONE])) { + n--; + } + if (n - b > sizeof(EXPORT) - SKIP_ONE && + strncmp(s + b, EXPORT, sizeof(EXPORT) - SKIP_ONE) == 0) { + b += sizeof(EXPORT) - SKIP_ONE; + } + const char *eq = b < n && s[b] != '#' ? memchr(s + b, '=', n - b) : NULL; + size_t kl = eq ? (size_t)(eq - (s + b)) : 0; + if (!eq || !ci_build_ident(s + b, kl)) { + return; + } + size_t v = b + kl + SKIP_ONE; + while (v < n && isspace((unsigned char)s[v])) { + v++; + } + size_t vl = n - v; + const char *val = ci_unquote(s + v, &vl); + env->kv[PAIR_LEN * env->n] = cbm_arena_strndup(a, s + b, kl); + env->kv[PAIR_LEN * env->n + SKIP_ONE] = cbm_arena_strndup(a, val, vl); + env->n++; +} + +/* KEY=VALUE pairs of a .env text (quotes removed; a later key wins). */ +static ci_env_t ci_dotenv(CBMArena *a, const char *text) { + size_t lines = SKIP_ONE; + for (const char *q = text; *q; q++) { + lines += *q == '\n'; + } + ci_env_t env = {cbm_arena_alloc(a, lines * PAIR_LEN * sizeof(char *)), 0}; + for (const char *p = text; env.kv && p && *p;) { + const char *e = strchr(p, '\n'); + ci_dotenv_line(a, p, e ? (size_t)(e - p) : strlen(p), &env); + p = e ? e + SKIP_ONE : NULL; + } + return env; +} + +static const char *ci_env_lookup(const ci_env_t *env, const char *name, size_t nl) { + for (int i = env->n - SKIP_ONE; i >= 0; i--) { + const char *k = env->kv[(size_t)i * PAIR_LEN]; + if (k && strlen(k) == nl && strncmp(k, name, nl) == 0) { + return env->kv[(size_t)i * PAIR_LEN + SKIP_ONE]; + } + } + return NULL; +} + +/* `s` with ${X} / ${X:-d} / ${X-d} / $X expanded from env; NULL when a + * variable has no value (and no default). */ +static const char *ci_interp(CBMArena *a, const char *s, const ci_env_t *env) { + if (!s || !strchr(s, '$')) { + return s; + } + size_t cap = strlen(s) + SKIP_ONE; + for (int i = 0; i < env->n; i++) { + const char *v = env->kv[(size_t)i * PAIR_LEN + SKIP_ONE]; + cap += v ? strlen(v) : 0; + } + char *out = cbm_arena_alloc(a, cap); + size_t len = 0; + for (const char *p = s; out && *p;) { + if (*p != '$') { + out[len++] = *p++; + continue; + } + bool braced = p[SKIP_ONE] == '{'; + const char *name = p + (braced ? PAIR_LEN : SKIP_ONE); + size_t nl = 0; + while (isalnum((unsigned char)name[nl]) || name[nl] == '_') { + nl++; + } + if (nl == 0 || !ci_build_ident(name, nl)) { + return NULL; + } + const char *after = name + nl; + const char *dflt = NULL; + size_t dl = 0; + bool colon = false; + if (braced) { + const char *close = strchr(after, '}'); + if (!close) { + return NULL; + } + if (after[0] == ':' && after[SKIP_ONE] == '-') { + colon = true; + dflt = after + PAIR_LEN; + } else if (after[0] == '-') { + dflt = after + SKIP_ONE; + } else if (after != close) { + return NULL; + } + dl = dflt ? (size_t)(close - dflt) : 0; + after = close + SKIP_ONE; + } + const char *v = ci_env_lookup(env, name, nl); + const char *use = v; + size_t ul = v ? strlen(v) : 0; + if (!v || (colon && !v[0])) { + if (!dflt) { + return NULL; + } + use = dflt; + ul = dl; + } + if (len + ul + SKIP_ONE > cap) { + return NULL; + } + memcpy(out + len, use, ul); + len += ul; + p = after; + } + if (out) { + out[len] = '\0'; + } + return out; +} + +/* Repository path of `rel` under repository directory `base`, normalised; + * NULL when it leaves the repository or is absolute. */ +static const char *ci_repo_path(CBMArena *a, const char *base, const char *rel) { + if (!rel || rel[0] == '/' || strstr(rel, "://")) { + return NULL; + } + size_t cap = strlen(base) + strlen(rel) + PAIR_LEN; + char *out = cbm_arena_alloc(a, cap); + if (!out) { + return NULL; + } + size_t len = 0; + const char *parts[PAIR_LEN] = {base, rel}; + for (int k = 0; k < PAIR_LEN; k++) { + for (const char *s = parts[k]; *s;) { + while (*s == '/') { + s++; + } + const char *e = s; + while (*e && *e != '/') { + e++; + } + size_t sl = (size_t)(e - s); + if (sl == PAIR_LEN && s[0] == '.' && s[SKIP_ONE] == '.') { + if (len == 0) { + return NULL; + } + while (len > 0 && out[len - SKIP_ONE] != '/') { + len--; + } + len = len > 0 ? len - SKIP_ONE : 0; + } else if (sl > 0 && !(sl == SKIP_ONE && s[0] == '.')) { + if (len > 0) { + out[len++] = '/'; + } + memcpy(out + len, s, sl); + len += sl; + } + s = e; + } + } + out[len] = '\0'; + return out; +} + +static void ci_build_emit(const ci_t *c, const char *via, const char *image, const char *cdir, + const char *dockerfile, int row) { + const char *df = cdir ? ci_repo_path(c->a, cdir, dockerfile) : NULL; + if (!image || !df) { + return; + } + cbm_build_site_t site = {.via = via, .image = image, .dockerfile = df, .line = row + SKIP_ONE}; + c->site_fn(&site, c->ud); +} + +static void ci_skaffold_artifact(const ci_t *c, TSNode art, const char *dir) { + static const char *const OTHER[] = {"jib", "ko", "bazel", "buildpacks", + "custom", "kaniko", NULL}; + for (int i = 0; OTHER[i]; i++) { + if (!ts_node_is_null(ci_get(c, art, OTHER[i]))) { + return; + } + } + int row = 0; + const char *image = ci_scalar(c, ci_get(c, art, "image"), &row); + const char *ctx = ci_get_str(c, art, "context"); + const char *df = ci_get_str(c, ci_get(c, art, "docker"), "dockerfile"); + const char *cdir = ci_repo_path(c->a, dir, ctx ? ctx : "."); + ci_build_emit(c, "skaffold", image, cdir, df ? df : "Dockerfile", row); +} + +/* Every build.artifacts list under `n` (any depth: top level, profiles). */ +static void ci_skaffold_walk(const ci_t *c, TSNode n, const char *dir, int depth) { + enum { CI_SKAFFOLD_DEPTH = 16 }; + if (depth > CI_SKAFFOLD_DEPTH) { + return; + } + int np = 0; + ci_pair_t *pairs = ci_pairs(c, n, &np); + for (int i = 0; i < np; i++) { + if (strcmp(pairs[i].key, "build") == 0) { + int na = 0; + TSNode *arts = ci_items(c, ci_get(c, pairs[i].value, "artifacts"), &na); + for (int k = 0; k < na; k++) { + ci_skaffold_artifact(c, arts[k], dir); + } + } + ci_skaffold_walk(c, pairs[i].value, dir, depth + SKIP_ONE); + } + int ni = 0; + TSNode *items = ci_items(c, n, &ni); + for (int i = 0; i < ni; i++) { + ci_skaffold_walk(c, items[i], dir, depth + SKIP_ONE); + } +} + +/* The Dockerfile a compose build names (a context, or context / + * dockerfile), as a repository path; NULL: none or unknown. */ +static const char *ci_compose_dockerfile(const ci_t *c, TSNode build, const char *dir, + const ci_env_t *env) { + if (ts_node_is_null(build)) { + return NULL; + } + const char *ctx = ci_scalar(c, build, NULL); + const char *df = "Dockerfile"; + if (!ctx) { + ctx = ci_get_str(c, build, "context"); + const char *d = ci_get_str(c, build, "dockerfile"); + ctx = ctx ? ctx : "."; + df = d ? d : df; + } + ctx = ci_interp(c->a, ctx, env); + df = ci_interp(c->a, df, env); + const char *cdir = ctx && df ? ci_repo_path(c->a, dir, ctx) : NULL; + return cdir ? ci_repo_path(c->a, cdir, df) : NULL; +} + +/* depends_on (a list, or a mapping's keys) and links ("svc" / "svc:alias"). */ +static int ci_compose_deps(const ci_t *c, TSNode svc, const char ***out) { + int nd = 0; + int nl = 0; + TSNode dep = ci_get(c, svc, "depends_on"); + TSNode *items = ci_items(c, dep, &nd); + ci_pair_t *pairs = items ? NULL : ci_pairs(c, dep, &nd); + TSNode *links = ci_items(c, ci_get(c, svc, "links"), &nl); + const char **deps = cbm_arena_alloc(c->a, (size_t)(nd + nl + SKIP_ONE) * sizeof(*deps)); + int n = 0; + for (int i = 0; deps && i < nd; i++) { + const char *d = items ? ci_scalar(c, items[i], NULL) : pairs[i].key; + if (d) { + deps[n++] = d; + } + } + for (int i = 0; deps && i < nl; i++) { + const char *l = ci_scalar(c, links[i], NULL); + const char *colon = l ? strchr(l, ':') : NULL; + if (l) { + deps[n++] = colon ? cbm_arena_strndup(c->a, l, (size_t)(colon - l)) : l; + } + } + *out = deps; + return n; +} + +/* The variable names a service's environment sets: a mapping's keys, a + * list's "NAME=value" / "NAME" items (the name only -- a value is never + * kept), with their 1-based lines. */ +static int ci_compose_env(const ci_t *c, TSNode svc, const char ***out, const int **lines) { + int n = 0; + TSNode env = ci_get(c, svc, "environment"); + TSNode *items = ci_items(c, env, &n); + ci_pair_t *pairs = items ? NULL : ci_pairs(c, env, &n); + const char **names = cbm_arena_alloc(c->a, (size_t)(n + SKIP_ONE) * sizeof(*names)); + int *rows = cbm_arena_alloc(c->a, (size_t)(n + SKIP_ONE) * sizeof(*rows)); + int k = 0; + for (int i = 0; names && rows && i < n; i++) { + int row = 0; + const char *name = NULL; + if (items) { + const char *s = ci_scalar(c, items[i], &row); + const char *eq = s ? strchr(s, '=') : NULL; + name = eq ? cbm_arena_strndup(c->a, s, (size_t)(eq - s)) : s; + } else { + name = pairs[i].key; + row = pairs[i].row; + } + if (name && name[0]) { + names[k] = name; + rows[k++] = row + SKIP_ONE; + } + } + *out = names; + *lines = rows; + return k; +} + +static void ci_compose(const ci_t *c, TSNode root, const char *dir, const ci_env_t *env) { + int ns = 0; + ci_pair_t *svcs = ci_pairs(c, ci_get(c, root, "services"), &ns); + for (int i = 0; i < ns; i++) { + TSNode svc = ci_content(svcs[i].value); + if (!ci_type(svc, "block_mapping") && !ci_type(svc, "flow_mapping")) { + continue; + } + int row = 0; + const char *raw = ci_scalar(c, ci_get(c, svc, "image"), &row); + const char *image = raw ? ci_interp(c->a, raw, env) : NULL; + const char *df = ci_compose_dockerfile(c, ci_get(c, svc, "build"), dir, env); + if (c->site_fn && image && df) { + cbm_build_site_t site = { + .via = "compose", .image = image, .dockerfile = df, .line = row + SKIP_ONE}; + c->site_fn(&site, c->ud); + } + if (c->service_fn) { + cbm_compose_service_t s = {.name = svcs[i].key, + .image = image, + .dockerfile = df, + .line = (int)ts_node_start_point(svc).row, + .end_line = (int)ts_node_end_point(svc).row + SKIP_ONE}; + s.ndeps = ci_compose_deps(c, svc, &s.deps); + s.nenv = ci_compose_env(c, svc, &s.env, &s.env_lines); + c->service_fn(&s, c->ud); + } + } +} + +bool cbm_build_path(const char *rel) { + if (!rel || !ci_yaml_ext(rel)) { + return false; + } + const char *slash = strrchr(rel, '/'); + const char *base = slash ? slash + SKIP_ONE : rel; + if (strncmp(base, "skaffold", strlen("skaffold")) == 0) { + return true; + } + const char *b = + strncmp(base, "docker-", strlen("docker-")) == 0 ? base + strlen("docker-") : base; + size_t cl = strlen("compose"); + return strncmp(b, "compose", cl) == 0 && (b[cl] == '.' || b[cl] == '-' || b[cl] == '_'); +} + +static void ci_build_walk(CBMArena *a, const char *rel, const char *src, size_t len, + const char *dotenv, ci_t proto); + +void cbm_build_sites(CBMArena *a, const char *rel, const char *src, size_t len, const char *dotenv, + cbm_build_site_fn fn, void *ud) { + ci_build_walk(a, rel, src, len, dotenv, (ci_t){.site_fn = fn, .ud = ud}); +} + +void cbm_compose_services(CBMArena *a, const char *rel, const char *src, size_t len, + const char *dotenv, cbm_compose_service_fn fn, void *ud) { + const char *slash = strrchr(rel, '/'); + const char *base = slash ? slash + SKIP_ONE : rel; + if (strncmp(base, "skaffold", strlen("skaffold")) == 0) { + return; + } + ci_build_walk(a, rel, src, len, dotenv, (ci_t){.service_fn = fn, .ud = ud}); +} + +static void ci_build_walk(CBMArena *a, const char *rel, const char *src, size_t len, + const char *dotenv, ci_t proto) { + if (!cbm_build_path(rel)) { + return; + } + TSParser *parser = ts_parser_new(); + if (!parser) { + return; + } + TSTree *tree = NULL; + if (ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_YAML))) { + tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + } + const char *slash = strrchr(rel, '/'); + const char *dir = slash ? cbm_arena_strndup(a, rel, (size_t)(slash - rel)) : ""; + const char *base = slash ? slash + SKIP_ONE : rel; + bool skaffold = strncmp(base, "skaffold", strlen("skaffold")) == 0; + ci_env_t env = {0}; + if (!skaffold && dotenv) { + env = ci_dotenv(a, dotenv); + } + if (tree && dir) { + ci_t c = proto; + c.a = a; + c.src = src; + c.rel = rel; + TSNode stream = ts_tree_root_node(tree); + for (uint32_t i = 0; i < ts_node_named_child_count(stream); i++) { + TSNode doc = ts_node_named_child(stream, i); + if (!ci_type(doc, "document")) { + continue; + } + TSNode root = {0}; + for (uint32_t k = 0; k < ts_node_named_child_count(doc); k++) { + TSNode x = ts_node_named_child(doc, k); + if (!ci_type(x, "comment")) { + root = x; + break; + } + } + if (skaffold && c.site_fn) { + const char *api = ci_get_str(&c, root, "apiVersion"); + if (api && strncmp(api, "skaffold/", strlen("skaffold/")) == 0) { + ci_skaffold_walk(&c, root, dir, 0); + } + } else if (!skaffold) { + ci_compose(&c, root, dir, &env); + } + } + } + if (tree) { + ts_tree_delete(tree); + } + ts_parser_delete(parser); +} + +bool cbm_ci_path(const char *rel) { + return rel && (ci_workflow_path(rel) || ci_action_path(rel)); +} + +void cbm_ci_commands(CBMArena *a, const char *rel, const char *src, size_t len, cbm_ci_cmd_fn fn, + void *ud) { + if (!ci_yaml_ext(rel)) { + return; + } + TSParser *parser = ts_parser_new(); + if (!parser) { + return; + } + TSTree *tree = NULL; + if (ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_YAML))) { + tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + } + if (tree) { + ci_t c = {.a = a, .src = src, .rel = rel, .fn = fn, .ud = ud}; + TSNode stream = ts_tree_root_node(tree); + for (uint32_t i = 0; i < ts_node_named_child_count(stream); i++) { + TSNode doc = ts_node_named_child(stream, i); + if (!ci_type(doc, "document")) { + continue; + } + TSNode root = {0}; + for (uint32_t k = 0; k < ts_node_named_child_count(doc); k++) { + TSNode x = ts_node_named_child(doc, k); + if (ci_type(x, "block_node") || ci_type(x, "flow_node")) { + root = ci_content(x); + break; + } + } + if (ci_workflow_path(rel)) { + ci_workflow(&c, root); + } else if (ci_action_path(rel)) { + ci_action(&c, root); + } else { + ci_prow(&c, root); + } + } + ts_tree_delete(tree); + } + ts_parser_delete(parser); +} diff --git a/src/pipeline/pass_configlink.c b/src/pipeline/pass_configlink.c index 1ee3ecf4b..5c1b41654 100644 --- a/src/pipeline/pass_configlink.c +++ b/src/pipeline/pass_configlink.c @@ -12,6 +12,7 @@ #include "pipeline/pipeline_internal.h" #include "graph_buffer/graph_buffer.h" #include "foundation/hash_table.h" +#include "foundation/arena.h" #include "foundation/log.h" #include "foundation/compat.h" @@ -63,18 +64,17 @@ static bool is_dep_section(const char *s) { /* ── Strategy 1: Config Key → Code Symbol ───────────────────────── */ -/* Canonical candidate order (determinism). Both collectors below fill a - * fixed-capacity array and stop at max_out; the label indexes they walk are - * gbuf insertion order = parallel-extraction merge order, which varies run to - * run. On a repo with more candidates than the cap, sorting by a pure content - * key first is what keeps the surviving set — and therefore the emitted - * CONFIGURES edges — a function of the inputs rather than of worker - * scheduling. Tie-breaks stay content-only: node ids are handed out in merge - * order, so an id tie-break belongs in no canonical comparator. */ +/* Canonical candidate order (determinism). The label indexes the collectors + * walk are gbuf insertion order = parallel-extraction merge order, which + * varies run to run. Sorting by a pure content key first keeps the candidate + * order -- and so the order edges are inserted in -- a function of the inputs + * rather than of worker scheduling. (The collectors once stopped at a cap, + * where this order also decided WHICH candidates survived; no cap cuts them + * now.) Tie-breaks stay content-only: node ids are handed out in merge order, + * so an id tie-break belongs in no canonical comparator. */ enum { - CANON_CMP_LESS = -1, /* qsort: left sorts before right */ - CANON_CMP_GREATER = 1, /* qsort: left sorts after right */ - CANON_CAP_BUF = 32 /* decimal rendering of a cap value */ + CANON_CMP_LESS = -1, /* qsort: left sorts before right */ + CANON_CMP_GREATER = 1 /* qsort: left sorts after right */ }; static int cmp_node_ptr_canonical(const void *pa, const void *pb) { @@ -100,14 +100,6 @@ static int cmp_node_ptr_canonical(const void *pa, const void *pb) { return strcmp(na, nb); } -/* A filled-to-capacity collector dropped candidates; say so rather than - * truncating silently. */ -static void log_candidate_truncation(const char *side, int cap) { - char cap_buf[CANON_CAP_BUF]; - snprintf(cap_buf, sizeof(cap_buf), "%d", cap); - cbm_log_info("configlinker.truncated", "side", side, "cap", cap_buf); -} - /* Heap copy of `nodes` sorted by cmp_node_ptr_canonical. Returns NULL (and * leaves the caller on the unsorted borrowed array) only on allocation * failure, which degrades determinism but never correctness. */ @@ -124,21 +116,105 @@ static const cbm_gbuf_node_t **canonical_node_copy(const cbm_gbuf_node_t *const return sorted; } +/* Infrastructure files: YAML, JSON and HCL (Terraform, OpenTofu, tfvars). A + * key_symbol pair with either side in one links by exact name and only inside + * the config file's component (user decision 2026-10-10): in 17 infra + * repositories 4 of 40 sampled key_symbol edges were right, all of them exact + * and within one directory tree; the rest were substring and cross-component + * coincidences. Other config formats keep exact + substring matching. */ +static bool cl_is_infra_path(const char *path) { + static const char *const exts[] = {".yaml", ".yml", ".json", ".tf", + ".hcl", ".tofu", ".tfvars", NULL}; + const char *dot = path ? strrchr(path, '.') : NULL; + const char *slash = path ? strrchr(path, '/') : NULL; + if (!dot || (slash && dot < slash)) { + return false; + } + for (int i = 0; exts[i]; i++) { + if (strcmp(dot, exts[i]) == 0) { + return true; + } + } + return false; +} + +/* Length of the config file's component directory: its own directory, or the + * one around it when that is a conventional config folder; 0 = the repo root. */ +static size_t cl_component_len(const char *path) { + static const char *const config_dirs[] = {"config", "configs", "conf", "settings", "etc", NULL}; + const char *slash = path ? strrchr(path, '/') : NULL; + if (!slash) { + return 0; + } + size_t dir_len = (size_t)(slash - path); + const char *base = path; + for (const char *p = path; p < slash; p++) { + if (*p == '/') { + base = p + SKIP_ONE; + } + } + size_t base_len = (size_t)(slash - base); + for (int i = 0; config_dirs[i]; i++) { + if (base_len == strlen(config_dirs[i]) && strncmp(base, config_dirs[i], base_len) == 0) { + return base == path ? 0 : (size_t)(base - path) - SKIP_ONE; + } + } + return dir_len; +} + typedef struct { int64_t node_id; - char normalized[CBM_SZ_256]; - char name[CBM_SZ_256]; -} config_entry_t; + const char *file_path; /* borrowed from the graph buffer for the pass */ + const char *name; /* borrowed */ + char *normalized; /* in the strategy's arena */ + size_t len; + size_t component_len; /* config entries: cl_component_len(file_path) */ + bool infra; +} cl_entry_t; + +typedef struct { + cl_entry_t *items; + int count; + int cap; +} cl_entries_t; + +static bool cl_entries_push(cl_entries_t *entries, const cbm_gbuf_node_t *node, const char *norm, + CBMArena *arena) { + if (entries->count == entries->cap) { + int cap = entries->cap ? entries->cap * PAIR_LEN : CBM_SZ_1K; + cl_entry_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, entries->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + entries->items = grown; + entries->cap = cap; + } + char *normalized = cbm_arena_strdup(arena, norm); + if (!normalized) { + return false; + } + entries->items[entries->count++] = (cl_entry_t){ + .node_id = node->id, + .file_path = node->file_path ? node->file_path : "", + .name = node->name ? node->name : "", + .normalized = normalized, + .len = strlen(normalized), + .component_len = cl_component_len(node->file_path), + .infra = cl_is_infra_path(node->file_path), + }; + return true; +} /* Collect config Variable nodes with ≥2 tokens, each ≥3 chars. */ -static int collect_config_entries(const cbm_gbuf_node_t *const *vars, int var_count, - config_entry_t *out, int max_out) { - int n = 0; +static bool collect_config_entries(const cbm_gbuf_node_t *const *vars, int var_count, + cl_entries_t *out, CBMArena *arena) { const cbm_gbuf_node_t **sorted = canonical_node_copy(vars, var_count); if (sorted) { vars = sorted; } - for (int i = 0; i < var_count && n < max_out; i++) { + bool ok = true; + for (int i = 0; i < var_count && ok; i++) { if (!cbm_has_config_extension(vars[i]->file_path)) { continue; } @@ -161,78 +237,168 @@ static int collect_config_entries(const cbm_gbuf_node_t *const *vars, int var_co } p = end ? end + SKIP_ONE : p + tlen; } - if (!all_long) { - continue; + if (all_long) { + ok = cl_entries_push(out, vars[i], norm, arena); } - - out[n].node_id = vars[i]->id; - snprintf(out[n].normalized, sizeof(out[n].normalized), "%s", norm); - snprintf(out[n].name, sizeof(out[n].name), "%s", vars[i]->name); - n++; - } - if (n == max_out) { - log_candidate_truncation("config", max_out); } free((void *)sorted); - return n; + return ok; +} + +static bool cl_is_hcl_path(const char *path) { + static const char *const exts[] = {".tf", ".hcl", ".tofu", ".tfvars", NULL}; + const char *dot = path ? strrchr(path, '.') : NULL; + for (int i = 0; dot && exts[i]; i++) { + if (strcmp(dot, exts[i]) == 0) { + return true; + } + } + return false; +} + +/* The name an HCL symbol matches a config key by. A config value sets a + * Terraform/OpenTofu `variable` block, whose Class is named `variable.`: + * it matches by . Outputs, modules, resources and a .tfvars file's own + * assignments are not what a config key sets; they take no part (NULL). */ +static const char *cl_hcl_variable_name(const cbm_gbuf_node_t *node) { + static const char prefix[] = "variable."; + if (!node->label || strcmp(node->label, "Class") != 0 || !node->name || + strncmp(node->name, prefix, sizeof(prefix) - SKIP_ONE) != 0) { + return NULL; + } + return node->name + sizeof(prefix) - SKIP_ONE; } /* Collect code nodes (Function/Variable/Class/Struct) not from config files. */ -typedef struct { - int64_t node_id; - char normalized[CBM_SZ_256]; -} code_entry_t; - -static int collect_code_entries(cbm_gbuf_t *gb, code_entry_t *out, int max_out) { - int n = 0; +static bool collect_code_entries(cbm_gbuf_t *gb, cl_entries_t *out, CBMArena *arena) { /* "Struct" alongside "Class": a config key may name a Go/Rust/Swift/D struct * type, which is now labelled "Struct" — keep it linkable. */ static const char *labels[] = {"Function", "Variable", "Class", "Struct", NULL}; - for (int li = 0; labels[li] && n < max_out; li++) { + bool ok = true; + for (int li = 0; labels[li] && ok; li++) { const cbm_gbuf_node_t **nodes = NULL; int count = 0; if (cbm_gbuf_find_by_label(gb, labels[li], &nodes, &count) != 0) { continue; } - /* Canonical order before the cap — see cmp_node_ptr_canonical. The cap - * spans the whole label list, so a later label can be cut mid-group; - * sorting per group keeps that cut a pure function of content. */ + /* Canonical order -- see cmp_node_ptr_canonical. */ const cbm_gbuf_node_t **sorted = canonical_node_copy(nodes, count); const cbm_gbuf_node_t *const *scan = sorted ? sorted : nodes; - for (int i = 0; i < count && n < max_out; i++) { + for (int i = 0; i < count && ok; i++) { if (cbm_has_config_extension(scan[i]->file_path)) { continue; } + const char *match_name = scan[i]->name; + if (cl_is_hcl_path(scan[i]->file_path)) { + match_name = cl_hcl_variable_name(scan[i]); + if (!match_name) { + continue; + } + } + char norm[CBM_SZ_256]; - int tokens = cbm_normalize_config_key(scan[i]->name, norm, sizeof(norm)); + int tokens = cbm_normalize_config_key(match_name, norm, sizeof(norm)); if (tokens == 0 || norm[0] == '\0') { continue; } - - out[n].node_id = scan[i]->id; - snprintf(out[n].normalized, sizeof(out[n].normalized), "%s", norm); - n++; + ok = cl_entries_push(out, scan[i], norm, arena); } /* gbuf data is borrowed — only the sorted copy is owned */ free((void *)sorted); } - if (n == max_out) { - log_candidate_truncation("code", max_out); - } - return n; + return ok; } -/* The key x symbol comparison of strategy 1, per config key: every pair's - * verdict depends only on its two strings, so rows run in parallel and the - * edges are inserted afterwards in the original (config, code) order. One - * thread compared up to 4,096 x 8,192 pairs -- configlink's 0.93 s on the Go - * corpus (profile, 2026-09-17). A pair whose lengths rule both verdicts out - * skips the string calls: an exact match needs equal lengths, a substring one - * no longer than the code key. */ +/* Code entries (indices, ascending) under one index key. */ +typedef struct { + int *items; + int count; + int cap; +} cl_posting_t; + +static bool cl_posting_add(CBMHashTable *index, const char *key, int idx) { + cl_posting_t *posting = cbm_ht_get(index, key); + if (!posting) { + posting = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*posting)); + if (!posting) { + return false; + } + cbm_ht_set(index, key, posting); + } + if (posting->count && posting->items[posting->count - 1] == idx) { + return true; /* the same trigram twice in one name */ + } + if (posting->count == posting->cap) { + int cap = posting->cap ? posting->cap * PAIR_LEN : CBM_SZ_16; + int *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, posting->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + posting->items = grown; + posting->cap = cap; + } + posting->items[posting->count++] = idx; + return true; +} + +static void cl_posting_free(const char *key, void *value, void *userdata) { + (void)key; + (void)userdata; + cl_posting_t *posting = value; + cbm_free(CBM_MEM_CLASS_OTHER, posting->items); + cbm_free(CBM_MEM_CLASS_OTHER, posting); +} + +enum { CL_TRIGRAM = 3 }; + +/* Exact index: normalized name -> code entries. Trigram index: every + * 3-character run of a non-infra code entry's normalized name -> entries, so a + * key's substring candidates are the entries holding its rarest trigram + * instead of every code entry (cost follows the matches, no cap needed). */ +static bool cl_build_indexes(const cl_entries_t *code, CBMHashTable *exact, CBMHashTable *trigrams, + CBMArena *arena) { + for (int k = 0; k < code->count; k++) { + const cl_entry_t *e = &code->items[k]; + if (!cl_posting_add(exact, e->normalized, k)) { + return false; + } + if (e->infra) { + continue; /* infra symbols link by exact name only */ + } + for (size_t i = 0; i + CL_TRIGRAM <= e->len; i++) { + char tri[CL_TRIGRAM + SKIP_ONE]; + memcpy(tri, e->normalized + i, CL_TRIGRAM); + tri[CL_TRIGRAM] = '\0'; + const char *key = cbm_ht_get_key(trigrams, tri); + if (!key) { + key = cbm_arena_strdup(arena, tri); + if (!key) { + return false; + } + } + if (!cl_posting_add(trigrams, key, k)) { + return false; + } + } + } + return true; +} + +/* A config key's component holds the code file. */ +static bool cl_in_component(const cl_entry_t *config, const cl_entry_t *code) { + size_t n = config->component_len; + return n == 0 || + (strncmp(code->file_path, config->file_path, n) == 0 && code->file_path[n] == '/'); +} + +/* The key x symbol matches of strategy 1, per config key: exact matches from + * the exact index, substring matches among the entries holding the key's + * rarest trigram. Rows run in parallel; each row's matches are in ascending + * code order, and the edges are inserted afterwards in (config, code) order. */ typedef struct { int co; double confidence; @@ -245,29 +411,78 @@ typedef struct { } key_match_list_t; typedef struct { - const config_entry_t *config; - const size_t *config_len; - int config_count; - const code_entry_t *code; - const size_t *code_len; - int code_count; + const cl_entries_t *config; + const cl_entries_t *code; + const CBMHashTable *exact; + const CBMHashTable *trigrams; key_match_list_t *lists; _Atomic int next; _Atomic bool failed; } key_match_job_t; -static double key_pair_confidence(const char *config_norm, size_t config_len, const char *code_norm, - size_t code_len) { - if (config_len > code_len) { - return 0.0; +static bool key_match_push(key_match_list_t *list, int co, double confidence) { + if (list->count == list->cap) { + int cap = list->cap ? list->cap * PAIR_LEN : CBM_SZ_16; + key_match_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, list->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + list->items = grown; + list->cap = cap; } - if (config_len == code_len && memcmp(config_norm, code_norm, code_len) == 0) { - return CONF_KEY_EXACT; /* Exact match */ + list->items[list->count].co = co; + list->items[list->count].confidence = confidence; + list->count++; + return true; +} + +/* The trigram posting of `key` with the fewest entries; NULL when some + * trigram occurs in no entry (then no entry holds the key). */ +static const cl_posting_t *cl_rarest_trigram(const CBMHashTable *trigrams, const char *key, + size_t len) { + const cl_posting_t *rarest = NULL; + for (size_t i = 0; i + CL_TRIGRAM <= len; i++) { + char tri[CL_TRIGRAM + SKIP_ONE]; + memcpy(tri, key + i, CL_TRIGRAM); + tri[CL_TRIGRAM] = '\0'; + const cl_posting_t *posting = cbm_ht_get(trigrams, tri); + if (!posting) { + return NULL; + } + if (!rarest || posting->count < rarest->count) { + rarest = posting; + } } - if (strstr(code_norm, config_norm) != NULL) { - return CONF_KEY_SUBSTRING; /* Substring match */ + return rarest; +} + +static bool key_match_row(const key_match_job_t *job, int ci, key_match_list_t *list) { + const cl_entry_t *cfg = &job->config->items[ci]; + const cl_posting_t *exact = cbm_ht_get(job->exact, cfg->normalized); + const cl_posting_t *subs = + cfg->infra ? NULL : cl_rarest_trigram(job->trigrams, cfg->normalized, cfg->len); + int ei = 0; + int si = 0; + int ecount = exact ? exact->count : 0; + int scount = subs ? subs->count : 0; + /* Merge both ascending candidate lists so the row stays in code order. */ + while (ei < ecount || si < scount) { + bool take_exact = si >= scount || (ei < ecount && exact->items[ei] <= subs->items[si]); + int co = take_exact ? exact->items[ei++] : subs->items[si++]; + const cl_entry_t *code = &job->code->items[co]; + double confidence = 0.0; + if (take_exact) { + bool infra = cfg->infra || code->infra; + confidence = (!infra || cl_in_component(cfg, code)) ? CONF_KEY_EXACT : 0.0; + } else if (code->len > cfg->len && strstr(code->normalized, cfg->normalized)) { + confidence = CONF_KEY_SUBSTRING; + } + if (confidence > 0.0 && !key_match_push(list, co, confidence)) { + return false; + } } - return 0.0; + return true; } static void key_match_worker(int worker_id, void *arg) { @@ -275,30 +490,12 @@ static void key_match_worker(int worker_id, void *arg) { key_match_job_t *job = (key_match_job_t *)arg; while (!atomic_load_explicit(&job->failed, memory_order_relaxed)) { int ci = atomic_fetch_add_explicit(&job->next, SKIP_ONE, memory_order_relaxed); - if (ci >= job->config_count) { + if (ci >= job->config->count) { break; } - key_match_list_t *list = &job->lists[ci]; - for (int co = 0; co < job->code_count; co++) { - double confidence = key_pair_confidence(job->config[ci].normalized, job->config_len[ci], - job->code[co].normalized, job->code_len[co]); - if (confidence <= 0.0) { - continue; - } - if (list->count == list->cap) { - int cap = list->cap ? list->cap * PAIR_LEN : CBM_SZ_16; - key_match_t *grown = - cbm_realloc(CBM_MEM_CLASS_OTHER, list->items, (size_t)cap * sizeof(*grown)); - if (!grown) { - atomic_store_explicit(&job->failed, true, memory_order_relaxed); - return; - } - list->items = grown; - list->cap = cap; - } - list->items[list->count].co = co; - list->items[list->count].confidence = confidence; - list->count++; + if (!key_match_row(job, ci, &job->lists[ci])) { + atomic_store_explicit(&job->failed, true, memory_order_relaxed); + return; } } } @@ -311,38 +508,30 @@ static int strategy_key_symbols(cbm_gbuf_t *gb) { return 0; } - config_entry_t config_entries[CBM_SZ_4K]; - int config_count = collect_config_entries(vars, var_count, config_entries, CBM_SZ_4K); - - if (config_count == 0) { - return 0; - } - - code_entry_t code_entries[CBM_SZ_8K]; - int code_count = collect_code_entries(gb, code_entries, CBM_SZ_8K); - + CBMArena arena; + cbm_arena_init(&arena); + cl_entries_t config = {0}; + cl_entries_t code = {0}; + CBMHashTable *exact = NULL; + CBMHashTable *trigrams = NULL; + key_match_list_t *lists = NULL; int edge_count = 0; - size_t *config_len = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)config_count * sizeof(size_t)); - size_t *code_len = - cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)(code_count > 0 ? code_count : 1) * sizeof(size_t)); - key_match_list_t *lists = - cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)config_count * sizeof(key_match_list_t)); - bool parallel_ok = config_len && code_len && lists; - if (parallel_ok) { - for (int ci = 0; ci < config_count; ci++) { - config_len[ci] = strlen(config_entries[ci].normalized); - } - for (int co = 0; co < code_count; co++) { - code_len[co] = strlen(code_entries[co].normalized); - } + bool ok = collect_config_entries(vars, var_count, &config, &arena); + if (ok && config.count > 0) { + ok = collect_code_entries(gb, &code, &arena); + exact = ok ? cbm_ht_create(0) : NULL; + trigrams = exact ? cbm_ht_create(0) : NULL; + ok = trigrams && cl_build_indexes(&code, exact, trigrams, &arena); + lists = ok ? cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)config.count * sizeof(*lists)) : NULL; + ok = lists != NULL; + } + if (ok && lists) { key_match_job_t job = { - .config = config_entries, - .config_len = config_len, - .config_count = config_count, - .code = code_entries, - .code_len = code_len, - .code_count = code_count, + .config = &config, + .code = &code, + .exact = exact, + .trigrams = trigrams, .lists = lists, }; atomic_init(&job.next, 0); @@ -351,59 +540,44 @@ static int strategy_key_symbols(cbm_gbuf_t *gb) { cbm_parallel_for( workers, key_match_worker, &job, (cbm_parallel_for_opts_t){.max_workers = workers, .force_pthreads = false}); - parallel_ok = !atomic_load_explicit(&job.failed, memory_order_relaxed); + ok = !atomic_load_explicit(&job.failed, memory_order_relaxed); } - if (parallel_ok) { - for (int ci = 0; ci < config_count; ci++) { + if (ok && lists) { + for (int ci = 0; ci < config.count; ci++) { for (int m = 0; m < lists[ci].count; m++) { const key_match_t *km = &lists[ci].items[m]; char props[CBM_SZ_512]; snprintf(props, sizeof(props), "{\"strategy\":\"key_symbol\",\"confidence\":%.2f,\"config_key\":\"%s\"}", - km->confidence, config_entries[ci].name); - - cbm_gbuf_insert_edge(gb, code_entries[km->co].node_id, config_entries[ci].node_id, + km->confidence, config.items[ci].name); + cbm_gbuf_insert_edge(gb, code.items[km->co].node_id, config.items[ci].node_id, "CONFIGURES", props); edge_count++; } } + } else if (!ok) { + char cfg_buf[CBM_SZ_16]; + char code_buf[CBM_SZ_16]; + snprintf(cfg_buf, sizeof(cfg_buf), "%d", config.count); + snprintf(code_buf, sizeof(code_buf), "%d", code.count); + cbm_log_warn("configlinker.key_symbol.oom", "config", cfg_buf, "code", code_buf); } - for (int ci = 0; lists && ci < config_count; ci++) { + + for (int ci = 0; lists && ci < config.count; ci++) { cbm_free(CBM_MEM_CLASS_OTHER, lists[ci].items); } cbm_free(CBM_MEM_CLASS_OTHER, lists); - cbm_free(CBM_MEM_CLASS_OTHER, config_len); - cbm_free(CBM_MEM_CLASS_OTHER, code_len); - if (parallel_ok) { - return edge_count; + if (exact) { + cbm_ht_foreach(exact, cl_posting_free, NULL); + cbm_ht_free(exact); } - - /* Allocation failed: the one-thread comparison, as before. */ - for (int ci = 0; ci < config_count; ci++) { - for (int co = 0; co < code_count; co++) { - double confidence = 0.0; - - if (strcmp(config_entries[ci].normalized, code_entries[co].normalized) == 0) { - /* Exact match */ - confidence = CONF_KEY_EXACT; - } else if (strstr(code_entries[co].normalized, config_entries[ci].normalized) != NULL) { - /* Substring match */ - confidence = CONF_KEY_SUBSTRING; - } - - if (confidence > 0.0) { - char props[CBM_SZ_512]; - snprintf(props, sizeof(props), - "{\"strategy\":\"key_symbol\",\"confidence\":%.2f,\"config_key\":\"%s\"}", - confidence, config_entries[ci].name); - - cbm_gbuf_insert_edge(gb, code_entries[co].node_id, config_entries[ci].node_id, - "CONFIGURES", props); - edge_count++; - } - } + if (trigrams) { + cbm_ht_foreach(trigrams, cl_posting_free, NULL); + cbm_ht_free(trigrams); } - + cbm_free(CBM_MEM_CLASS_OTHER, config.items); + cbm_free(CBM_MEM_CLASS_OTHER, code.items); + cbm_arena_destroy(&arena); return edge_count; } diff --git a/src/pipeline/pass_cross_repo.c b/src/pipeline/pass_cross_repo.c index 09384a8e9..1f57d1c25 100644 --- a/src/pipeline/pass_cross_repo.c +++ b/src/pipeline/pass_cross_repo.c @@ -33,7 +33,6 @@ enum { CR_PATH_BUF = 4096, CR_QN_BUF = 512, CR_PROPS_BUF = 2048, - CR_MAX_EDGES = 4096, CR_DB_EXT_LEN = 3, /* strlen(".db") */ CR_INIT_CAP = 32, CR_MAX_PROJECTS = 4096, @@ -481,9 +480,9 @@ static bool cr_path_matches_template(const char *concrete, const char *templ) { * the route QN into out_qn and return the handler id; returns 0 on no match. * * COST: this scans every Route node of the target project once per unmatched - * HTTP_CALLS edge — O(calls × routes) per project pair. Acceptable while both - * factors stay small (calls are capped at CR_MAX_EDGES and it only runs for - * edges the exact lookup missed); revisit with a prepared template index if + * HTTP_CALLS edge — O(calls × routes) per project pair. It only runs for edges + * the exact lookup missed; no row count stops a scan (a cap decided which + * calls were matched at all); revisit with a prepared template index if * cross-repo matching ever shows up in profiles. (#523) */ static int64_t find_route_handler_fuzzy(cbm_store_t *target_store, const char *concrete_path, const char *method, char *out_qn, size_t out_qn_sz, @@ -504,11 +503,8 @@ static int64_t find_route_handler_fuzzy(cbm_store_t *target_store, const char *c return 0; } int64_t found = 0; - int scanned = 0; int step_rc = SQLITE_DONE; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { const char *qn = (const char *)sqlite3_column_text(s, 0); if (!qn || strncmp(qn, "__route__", CR_ROUTE_PREFIX_LEN) != 0) { continue; @@ -550,7 +546,7 @@ static int64_t find_route_handler_fuzzy(cbm_store_t *target_store, const char *c break; } } - if (!cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { *failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { @@ -713,12 +709,9 @@ static cr_match_result_t match_http_routes(cbm_store_t *src_store, const char *s } int count = 0; - int scanned = 0; int step_rc = SQLITE_DONE; bool failed = false; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { int64_t caller_id = sqlite3_column_int64(s, 0); int64_t route_id = sqlite3_column_int64(s, SKIP_ONE); const char *props = (const char *)sqlite3_column_text(s, PAIR_LEN); @@ -772,7 +765,7 @@ static cr_match_result_t match_http_routes(cbm_store_t *src_store, const char *s count++; } - if (!failed && !cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!failed && !cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { @@ -804,12 +797,9 @@ static cr_match_result_t match_async_routes(cbm_store_t *src_store, const char * } int count = 0; - int scanned = 0; int step_rc = SQLITE_DONE; bool failed = false; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { int64_t caller_id = sqlite3_column_int64(s, 0); int64_t route_id = sqlite3_column_int64(s, SKIP_ONE); const char *props = (const char *)sqlite3_column_text(s, PAIR_LEN); @@ -854,7 +844,7 @@ static cr_match_result_t match_async_routes(cbm_store_t *src_store, const char * } count++; } - if (!failed && !cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!failed && !cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { @@ -969,12 +959,9 @@ static cr_match_result_t match_channels(cbm_store_t *src_store, const char *src_ } int count = 0; - int scanned = 0; int step_rc = SQLITE_DONE; bool failed = false; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { const char *channel_name = (const char *)sqlite3_column_text(s, SKIP_ONE); const char *channel_qn = (const char *)sqlite3_column_text(s, PAIR_LEN); if (!channel_name || !channel_qn) { @@ -1007,7 +994,7 @@ static cr_match_result_t match_channels(cbm_store_t *src_store, const char *src_ count++; } } - if (!failed && !cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!failed && !cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { @@ -1084,12 +1071,9 @@ static cr_match_result_t match_typed_routes(cbm_store_t *src_store, const char * } int count = 0; - int scanned = 0; int step_rc = SQLITE_DONE; bool failed = false; - while (scanned < CR_MAX_EDGES && !cr_cancel_requested(ctx) && - (step_rc = sqlite3_step(s)) == SQLITE_ROW) { - scanned++; + while (!cr_cancel_requested(ctx) && (step_rc = sqlite3_step(s)) == SQLITE_ROW) { int64_t caller_id = sqlite3_column_int64(s, 0); int64_t route_id = sqlite3_column_int64(s, SKIP_ONE); const char *props = (const char *)sqlite3_column_text(s, PAIR_LEN); @@ -1139,7 +1123,7 @@ static cr_match_result_t match_typed_routes(cbm_store_t *src_store, const char * } count++; } - if (!failed && !cr_cancel_requested(ctx) && scanned < CR_MAX_EDGES && step_rc != SQLITE_DONE) { + if (!failed && !cr_cancel_requested(ctx) && step_rc != SQLITE_DONE) { failed = true; } if (sqlite3_finalize(s) != SQLITE_OK) { diff --git a/src/pipeline/pass_docker.c b/src/pipeline/pass_docker.c new file mode 100644 index 000000000..4a2d83c2a --- /dev/null +++ b/src/pipeline/pass_docker.c @@ -0,0 +1,1974 @@ +/* + * pass_docker.c — the container build layer. + * + * PHASE 4 P4.1a: what a Dockerfile builds on. Each base it names DEPENDS_ON + * an Image node -- an edge from the Dockerfile's Module: + * FROM [AS ] {via:from} + * COPY --from= ... {via:copy-from} + * A stage of the same file is no image (FROM builder, COPY --from=builder or + * --from=0), nor is `scratch`. ${ARG} references are expanded from the ARG + * defaults in scope (global ARGs before the first FROM; a stage's own ARG and + * ENV); a reference without a value leaves the image unknown, and an unknown + * image is no edge. + * + * Image identity is the normalised repository: docker.io for an image + * without a registry host, library/ for an official one-segment image; + * tag and digest stay on the edge. + * + * PHASE 4 P4.1b: what the container runs. The last stage's ENTRYPOINT (else + * CMD) SPAWNS -- from the Dockerfile's Module {api:dockerfile}: + * the repository file it runs a script / source copied into the image + * (an interpreter's script, a path, a CMD + * file under an inherited ENTRYPOINT) + * the Go `main` of the package a binary `go build -o` made in a stage + * and the image copied in + * a Process any other program + * Container paths map back through COPY: a context source is looked up under + * the Dockerfile's directory and the repository root -- the one where it is + * the file (or Go main package) wanted, else nothing; COPY --from= + * continues in that stage. WORKDIR makes paths absolute. `sh -c`, a variable + * program and arguments to an inherited ENTRYPOINT are no edge. + * + * PHASE 4 P4.5: the environment a container is given. Each variable a K8s + * container's env, a compose service's environment or the last stage's ENV + * (with the stages it is built FROM) sets is an EnvVar of that container + * (image, for ENV) -- `#env//`, owned by the + * declaring file -- the owner CONFIGURES {strategy:env_set, source}. The + * global EnvVar a program's readers share is not joined: the same name is + * read by other programs (measured: 23 % of such joins in scope). Values are + * never read into the graph. + * Run at the end of the K8s pass. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/limits.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "cbm.h" +#include "helpers.h" +#include "lang_specs.h" + +#include +#include +#include +#include +#include +#include + +enum { DK_MAX_STAGES = 64, DK_MAX_WORDS = 256, DK_MAX_DEPTH = 8 }; + +typedef struct dk_copy { + const char *from; /* --from= value; NULL: none, or unknown */ + bool has_from; /* --from given */ + const char **srcs; /* NULL entries: unknown */ + int nsrc; + const char *dst; /* as written; NULL: unknown */ + const char *workdir; + struct dk_copy *next; /* the stage's previous COPY */ +} dk_copy_t; + +typedef struct dk_build { + const char *out; /* go build -o; NULL: none or unknown */ + const char *pkg; /* NULL: unknown */ + const char *workdir; + struct dk_build *next; /* the stage's previous build */ +} dk_build_t; + +typedef struct { + const char **argv; + int argc; + int line; + const char *workdir; + bool set; +} dk_entry_t; + +typedef struct dk_envkey { + const char *name; + int line; + struct dk_envkey *next; /* the stage's previous ENV key */ +} dk_envkey_t; + +typedef struct { + dk_copy_t *copies; /* newest first */ + dk_build_t *builds; /* newest first */ + dk_envkey_t *envs; /* newest first */ + int parent; /* the stage FROM builds on; -1: an image */ +} dk_stage_t; + +typedef struct { + CBMHashTable *files; /* repo path -> File node */ + CBMHashTable *gomain; /* repo dir -> the `main` Function of its package */ +} dk_index_t; + +typedef struct dk_var { + const char *name; + const char *value; /* NULL: declared without a value */ + struct dk_var *next; +} dk_var_t; + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMArena *a; + const char *src; + const char *file; + int64_t module; + dk_var_t *global; /* ARGs before the first FROM */ + dk_var_t *local; /* the current stage's ARG / ENV */ + const char *stages[DK_MAX_STAGES]; /* stage names; NULL: unnamed */ + int stage; /* index of the current stage; -1 before FROM */ + int edges; + const char *dir; /* the Dockerfile's directory ("" at the root) */ + const char *workdir; /* the current stage's WORKDIR */ + dk_stage_t st[DK_MAX_STAGES]; /* COPY / go build records */ + dk_entry_t entry; /* the current stage's ENTRYPOINT */ + dk_entry_t cmd; /* the current stage's CMD */ + const dk_index_t *ix; +} dk_t; + +static bool dk_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +static const char *dk_text(const dk_t *d, TSNode n) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + char *s = cbm_arena_alloc(d->a, (size_t)(e - b) + SKIP_ONE); + if (s) { + memcpy(s, d->src + b, e - b); + s[e - b] = '\0'; + } + return s; +} + +/* A value as written: quotes of a quoted string removed. */ +static const char *dk_value(const dk_t *d, TSNode n) { + const char *s = dk_text(d, n); + size_t l = s ? strlen(s) : 0; + if (l >= PAIR_LEN && (s[0] == '"' || s[0] == '\'') && s[l - SKIP_ONE] == s[0]) { + char *v = cbm_arena_alloc(d->a, l - SKIP_ONE); + if (v) { + memcpy(v, s + SKIP_ONE, l - PAIR_LEN); + v[l - PAIR_LEN] = '\0'; + } + return v; + } + return s; +} + +static const dk_var_t *dk_find(const dk_var_t *v, const char *name, size_t nl) { + for (; v; v = v->next) { + if (strlen(v->name) == nl && strncmp(v->name, name, nl) == 0) { + return v; + } + } + return NULL; +} + +static void dk_set(dk_t *d, dk_var_t **list, const char *name, const char *value) { + dk_var_t *v = cbm_arena_alloc(d->a, sizeof(*v)); + if (v) { + *v = (dk_var_t){.name = name, .value = value, .next = *list}; + *list = v; + } +} + +/* ASCII case-insensitive equality (stage names, `scratch`). */ +static bool dk_ieq(const char *a, const char *b) { + for (; *a && *b; a++, b++) { + if (tolower((unsigned char)*a) != tolower((unsigned char)*b)) { + return false; + } + } + return *a == *b; +} + +static bool dk_name_char(char c) { + return isalnum((unsigned char)c) || c == '_'; +} + +/* `text` with $X / ${X} / ${X:-d} / ${X-d} expanded from `vars`; NULL when a + * reference has no value. */ +static const char *dk_expand(dk_t *d, const dk_var_t *vars, const char *text) { + size_t cap = strlen(text) + SKIP_ONE; + size_t len = 0; + char *out = cbm_arena_alloc(d->a, cap); + for (const char *p = text; out && *p;) { + if (*p != '$') { + if (len + PAIR_LEN > cap) { + char *grown = cbm_arena_alloc(d->a, cap * PAIR_LEN); + if (!grown) { + return NULL; + } + memcpy(grown, out, len); + out = grown; + cap *= PAIR_LEN; + } + out[len++] = *p++; + continue; + } + const char *name = p + SKIP_ONE; + bool braced = *name == '{'; + if (braced) { + name++; + } + size_t nl = 0; + while (dk_name_char(name[nl])) { + nl++; + } + if (nl == 0) { + return NULL; + } + const char *after = name + nl; + const char *dflt = NULL; + size_t dl = 0; + if (braced) { + const char *close = strchr(after, '}'); + if (!close) { + return NULL; + } + if ((after[0] == ':' && after[SKIP_ONE] == '-') || after[0] == '-') { + dflt = after + (after[0] == ':' ? PAIR_LEN : SKIP_ONE); + dl = (size_t)(close - dflt); + } else if (after != close) { + return NULL; /* ${X:+y} and other forms: unknown */ + } + after = close + SKIP_ONE; + } + const dk_var_t *v = dk_find(vars, name, nl); + const char *val = v && v->value ? v->value : NULL; + size_t vl = val ? strlen(val) : dl; + if (!val && !dflt) { + return NULL; + } + const char *from = val ? val : dflt; + if (len + vl + SKIP_ONE > cap) { + size_t ncap = (len + vl + SKIP_ONE) * PAIR_LEN; + char *grown = cbm_arena_alloc(d->a, ncap); + if (!grown) { + return NULL; + } + memcpy(grown, out, len); + out = grown; + cap = ncap; + } + memcpy(out + len, from, vl); + len += vl; + p = after; + } + if (out) { + out[len] = '\0'; + } + return out; +} + +/* The repository identity of image reference `ref` into out. */ +/* Only characters an image reference can hold (name, :tag, @digest) -- a + * template's placeholder (`<.Values.image>`, ``) or an expression is + * no image. */ +static bool dk_ref_chars(const char *ref) { + for (const char *p = ref; *p; p++) { + if (!isalnum((unsigned char)*p) && !strchr("._-/:@+", *p)) { + return false; + } + } + return ref[0] != '\0'; +} + +static bool dk_normalize(const char *ref, char *out, size_t sz) { + if (!dk_ref_chars(ref)) { + return false; + } + char r[CBM_SZ_512]; + snprintf(r, sizeof(r), "%s", ref); + char *at = strchr(r, '@'); + if (at) { + *at = '\0'; + } + char *slash = strrchr(r, '/'); + char *colon = strchr(slash ? slash : r, ':'); + if (colon) { + *colon = '\0'; + } + if (!r[0] || strchr(r, ' ') || strchr(r, '$')) { + return false; + } + char *first_end = strchr(r, '/'); + if (!first_end) { + snprintf(out, sz, "docker.io/library/%s", r); + return true; + } + size_t fl = (size_t)(first_end - r); + bool host = memchr(r, '.', fl) || memchr(r, ':', fl) || + (fl == strlen("localhost") && strncmp(r, "localhost", fl) == 0); + if (!host) { + snprintf(out, sz, "docker.io/%s", r); + return true; + } + const char *rest = first_end + SKIP_ONE; + bool hub = + (fl == strlen("docker.io") && strncmp(r, "docker.io", fl) == 0) || + (fl == strlen("index.docker.io") && strncmp(r, "index.docker.io", fl) == 0) || + (fl == strlen("registry-1.docker.io") && strncmp(r, "registry-1.docker.io", fl) == 0); + if (hub) { + snprintf(out, sz, "docker.io/%s%s", strchr(rest, '/') ? "" : "library/", rest); + } else { + snprintf(out, sz, "%.*s/%s", (int)fl, r, rest); + } + return true; +} + +/* An earlier stage of this file: its name, or a stage index. */ +static bool dk_is_stage(const dk_t *d, const char *ref) { + bool digits = ref[0] != '\0'; + for (const char *p = ref; *p; p++) { + digits = digits && isdigit((unsigned char)*p); + } + if (digits) { + return true; + } + for (int i = 0; i < d->stage && i < DK_MAX_STAGES; i++) { + if (d->stages[i] && dk_ieq(d->stages[i], ref)) { + return true; + } + } + return false; +} + +/* The stage a --from names: an index, else the last stage of that name. */ +static int dk_stage_of(const dk_t *d, const char *ref) { + if (!ref || !ref[0]) { + return -1; + } + bool digits = true; + for (const char *p = ref; *p; p++) { + digits = digits && isdigit((unsigned char)*p); + } + if (digits) { + int k = atoi(ref); + return k < DK_MAX_STAGES ? k : -1; + } + int found = -1; + for (int i = 0; i <= d->stage && i < DK_MAX_STAGES; i++) { + if (d->stages[i] && dk_ieq(d->stages[i], ref)) { + found = i; + } + } + return found; +} + +int64_t cbm_pipeline_image_node(cbm_pipeline_ctx_t *ctx, const char *ref) { + char repo[CBM_SZ_512]; + if (!ref || dk_ieq(ref, "scratch") || !dk_normalize(ref, repo, sizeof(repo))) { + return 0; + } + char qn[CBM_SZ_1K]; + snprintf(qn, sizeof(qn), "__image__%s", repo); + char esc_repo[CBM_SZ_1K]; + cbm_json_escape(esc_repo, (int)sizeof(esc_repo), repo); + char node_props[CBM_SZ_2K]; + snprintf(node_props, sizeof(node_props), "{\"repository\":\"%s\"}", esc_repo); + return cbm_gbuf_upsert_node(ctx->gbuf, "Image", repo, qn, "", 0, 0, node_props); +} + +/* An environment variable name: [A-Za-z_][A-Za-z0-9_.-]*. */ +static bool dk_env_key(const char *k) { + if (!k || !(isalpha((unsigned char)k[0]) || k[0] == '_')) { + return false; + } + for (const char *p = k + SKIP_ONE; *p; p++) { + if (!isalnum((unsigned char)*p) && *p != '_' && *p != '.' && *p != '-') { + return false; + } + } + return strlen(k) < CBM_SZ_256; +} + +int64_t cbm_pipeline_env_setter(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *owner, + const char *container, const char *key, const char *source, + int line) { + if (!owner || !owner->qualified_name || !dk_env_key(key) || !source) { + return 0; + } + char qn[CBM_SZ_2K]; + int n = snprintf(qn, sizeof(qn), "%s#env/%s/%s", owner->qualified_name, + container ? container : "-", key); + if (n <= 0 || (size_t)n >= sizeof(qn)) { + return 0; + } + char esc_c[CBM_SZ_512]; + cbm_json_escape(esc_c, (int)sizeof(esc_c), container ? container : ""); + char scope[CBM_SZ_1K]; + if (container) { + snprintf(scope, sizeof(scope), "\"scope\":\"container\",\"container\":\"%s\"", esc_c); + } else { + snprintf(scope, sizeof(scope), "\"scope\":\"image\""); + } + char node_props[CBM_SZ_2K]; + snprintf(node_props, sizeof(node_props), "{\"env_key\":\"%s\",%s,\"source\":\"%s\"}", key, + scope, source); + int64_t id = + cbm_gbuf_upsert_node(ctx->gbuf, "EnvVar", key, qn, owner->file_path ? owner->file_path : "", + line, line, node_props); + if (id <= 0 || id == owner->id) { + return 0; + } + char props[CBM_SZ_2K]; + snprintf(props, sizeof(props), "{\"strategy\":\"env_set\",%s,\"source\":\"%s\",\"line\":%d}", + scope, source, line); + cbm_gbuf_insert_edge(ctx->gbuf, owner->id, id, "CONFIGURES", props); + return id; +} + +/* DEPENDS_ON from the Dockerfile to the Image of resolved reference `ref`. */ +static void dk_link(dk_t *d, const char *ref, const char *via, int line) { + int64_t img = cbm_pipeline_image_node(d->ctx, ref); + if (img <= 0) { + return; + } + char esc_ref[CBM_SZ_1K]; + cbm_json_escape(esc_ref, (int)sizeof(esc_ref), ref); + char props[CBM_SZ_2K]; + snprintf(props, sizeof(props), "{\"via\":\"%s\",\"image\":\"%s\",\"stage\":%d,\"line\":%d}", + via, esc_ref, d->stage, line); + cbm_gbuf_insert_edge(d->ctx->gbuf, d->module, img, "DEPENDS_ON", props); + d->edges++; +} + +static int dk_line(TSNode n) { + return (int)ts_node_start_point(n).row + SKIP_ONE; +} + +static void dk_arg(dk_t *d, TSNode ins) { + TSNode nm = ts_node_child_by_field_name(ins, "name", (uint32_t)strlen("name")); + TSNode df = ts_node_child_by_field_name(ins, "default", (uint32_t)strlen("default")); + if (ts_node_is_null(nm)) { + return; + } + const char *name = dk_text(d, nm); + if (!name) { + return; + } + if (d->stage < 0) { + dk_set(d, &d->global, name, ts_node_is_null(df) ? NULL : dk_value(d, df)); + return; + } + const char *value = NULL; + if (!ts_node_is_null(df)) { + const char *raw = dk_value(d, df); + value = raw ? dk_expand(d, d->local, raw) : NULL; + } else { + const dk_var_t *g = dk_find(d->global, name, strlen(name)); + value = g ? g->value : NULL; + } + dk_set(d, &d->local, name, value); +} + +static void dk_env(dk_t *d, TSNode ins) { + if (d->stage < 0) { + return; + } + for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) { + TSNode pair = ts_node_named_child(ins, i); + if (!dk_type(pair, "env_pair")) { + continue; + } + TSNode nm = ts_node_child_by_field_name(pair, "name", (uint32_t)strlen("name")); + TSNode vl = ts_node_child_by_field_name(pair, "value", (uint32_t)strlen("value")); + if (ts_node_is_null(nm)) { + continue; + } + const char *raw = ts_node_is_null(vl) ? NULL : dk_value(d, vl); + const char *name = dk_text(d, nm); + dk_set(d, &d->local, name, raw ? dk_expand(d, d->local, raw) : NULL); + dk_envkey_t *k = d->stage < DK_MAX_STAGES ? cbm_arena_alloc(d->a, sizeof(*k)) : NULL; + if (k && name) { + *k = (dk_envkey_t){name, dk_line(pair), d->st[d->stage].envs}; + d->st[d->stage].envs = k; + } + } +} + +static void dk_from(dk_t *d, TSNode ins) { + TSNode spec = {0}; + TSNode alias = ts_node_child_by_field_name(ins, "as", (uint32_t)strlen("as")); + for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) { + if (dk_type(ts_node_named_child(ins, i), "image_spec")) { + spec = ts_node_named_child(ins, i); + } + } + const char *ref = ts_node_is_null(spec) ? NULL : dk_expand(d, d->global, dk_text(d, spec)); + d->stage++; + d->local = NULL; + d->workdir = "/"; + d->entry.set = false; + d->cmd.set = false; + bool stage = ref && dk_is_stage(d, ref); + if (ref && !stage) { + dk_link(d, ref, "from", dk_line(ins)); + } + if (d->stage < DK_MAX_STAGES) { + d->st[d->stage].parent = stage ? dk_stage_of(d, ref) : -1; + d->stages[d->stage] = ts_node_is_null(alias) ? NULL : dk_text(d, alias); + } +} + +/* JSON string `s` (quoted) unquoted; \\ and \" unescaped. */ +static const char *dk_json_string(dk_t *d, const char *s) { + size_t l = s ? strlen(s) : 0; + if (l < PAIR_LEN || s[0] != '"') { + return s; + } + char *out = cbm_arena_alloc(d->a, l); + size_t n = 0; + for (size_t i = SKIP_ONE; out && i + SKIP_ONE < l; i++) { + if (s[i] == '\\' && i + PAIR_LEN < l) { + i++; + } + out[n++] = s[i]; + } + if (out) { + out[n] = '\0'; + } + return out; +} + +/* The strings of a json_string_array node. */ +static int dk_json_array(dk_t *d, TSNode arr, const char **out, int max) { + int n = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(arr) && n < max; i++) { + TSNode js = ts_node_named_child(arr, i); + if (dk_type(js, "json_string")) { + out[n++] = dk_json_string(d, dk_text(d, js)); + } + } + return n; +} + +static void dk_copy(dk_t *d, TSNode ins) { + static const char FROM[] = "--from="; + const char *words[DK_MAX_WORDS]; + int nw = 0; + dk_copy_t *c = cbm_arena_alloc(d->a, sizeof(*c)); + if (c) { + *c = (dk_copy_t){.workdir = d->workdir}; + } + for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) { + TSNode p = ts_node_named_child(ins, i); + if (dk_type(p, "path") && nw < DK_MAX_WORDS) { + const char *t = dk_text(d, p); + words[nw++] = t ? dk_expand(d, d->local, t) : NULL; + continue; + } + if (dk_type(p, "json_string_array")) { + const char *raw[DK_MAX_WORDS]; + int n = dk_json_array(d, p, raw, DK_MAX_WORDS - nw); + for (int k = 0; k < n; k++) { + words[nw++] = raw[k] ? dk_expand(d, d->local, raw[k]) : NULL; + } + continue; + } + if (!dk_type(p, "param")) { + continue; + } + const char *t = dk_text(d, p); + if (!t || strncmp(t, FROM, sizeof(FROM) - SKIP_ONE) != 0) { + continue; + } + const char *ref = dk_expand(d, d->local, t + sizeof(FROM) - SKIP_ONE); + if (c) { + c->has_from = true; + c->from = ref; + } + if (ref && ref[0] && !dk_is_stage(d, ref)) { + dk_link(d, ref, "copy-from", dk_line(ins)); + } + } + if (!c || nw < PAIR_LEN || d->stage >= DK_MAX_STAGES) { + return; + } + c->srcs = cbm_arena_alloc(d->a, (size_t)(nw - SKIP_ONE) * sizeof(*c->srcs)); + if (!c->srcs) { + return; + } + memcpy((void *)c->srcs, words, (size_t)(nw - SKIP_ONE) * sizeof(*c->srcs)); + c->nsrc = nw - SKIP_ONE; + c->dst = words[nw - SKIP_ONE]; + c->next = d->st[d->stage].copies; + d->st[d->stage].copies = c; +} + +/* ── Paths in the image and in the repository ────────────────────────── */ + +static const char *dk_join(dk_t *d, const char *a, const char *b) { + if (!a || !a[0]) { + return b; + } + if (!b || !b[0]) { + return a; + } + size_t al = strlen(a); + size_t bl = strlen(b); + char *s = cbm_arena_alloc(d->a, al + bl + PAIR_LEN); + if (s) { + memcpy(s, a, al); + size_t n = al; + if (a[al - SKIP_ONE] != '/') { + s[n++] = '/'; + } + memcpy(s + n, b, bl); + s[n + bl] = '\0'; + } + return s; +} + +/* `p` with "." and empty segments dropped and ".." applied; an absolute path + * stays absolute ("/" at most), a relative one climbing out keeps "..". */ +static const char *dk_norm(dk_t *d, const char *p) { + if (!p) { + return NULL; + } + size_t cap = strlen(p) + PAIR_LEN; + char *out = cbm_arena_alloc(d->a, cap); + if (!out) { + return NULL; + } + bool abs = p[0] == '/'; + size_t len = 0; + if (abs) { + out[len++] = '/'; + } + size_t base = len; + for (const char *s = p; *s;) { + while (*s == '/') { + s++; + } + const char *e = s; + while (*e && *e != '/') { + e++; + } + size_t sl = (size_t)(e - s); + if (sl == 0) { + break; + } + bool dot = sl == SKIP_ONE && s[0] == '.'; + bool up = sl == PAIR_LEN && s[0] == '.' && s[SKIP_ONE] == '.'; + if (up) { + size_t k = len; + while (k > base && out[k - SKIP_ONE] != '/') { + k--; + } + bool last_up = len - k == PAIR_LEN && out[k] == '.' && out[k + SKIP_ONE] == '.'; + if (len > base && !last_up) { + len = k > base ? k - SKIP_ONE : base; + } else if (!abs) { + if (len > base) { + out[len++] = '/'; + } + out[len++] = '.'; + out[len++] = '.'; + } + } else if (!dot) { + if (len > base) { + out[len++] = '/'; + } + memcpy(out + len, s, sl); + len += sl; + } + s = e; + } + out[len] = '\0'; + return out; +} + +/* Container path `p` made absolute against `workdir`, normalised. */ +static const char *dk_absolute(dk_t *d, const char *p, const char *workdir) { + if (!p) { + return NULL; + } + return dk_norm(d, p[0] == '/' ? p : dk_join(d, workdir && workdir[0] ? workdir : "/", p)); +} + +static const char *dk_base(const char *p) { + const char *s = strrchr(p, '/'); + return s ? s + SKIP_ONE : p; +} + +/* The directory of normalised path `p` ("/" for "/x", "" for "x"). */ +static const char *dk_dir(dk_t *d, const char *p) { + const char *s = strrchr(p, '/'); + if (!s) { + return ""; + } + if (s == p) { + return "/"; + } + char *out = cbm_arena_alloc(d->a, (size_t)(s - p) + SKIP_ONE); + if (out) { + memcpy(out, p, (size_t)(s - p)); + out[s - p] = '\0'; + } + return out; +} + +static bool dk_ends(const char *s, const char *suffix) { + size_t l = strlen(s); + size_t k = strlen(suffix); + return l >= k && strcmp(s + l - k, suffix) == 0; +} + +enum { DK_WANT_FILE, DK_WANT_GOMAIN }; + +/* The repository path of context source `src` (+ `rel`) that is what `want` + * asks for, the one such path over the context candidates (the Dockerfile's + * directory, the repository root); NULL when none or two. */ +static const char *dk_repo(dk_t *d, const char *src, const char *rel, int want) { + const char *ctxs[PAIR_LEN] = {d->dir, ""}; + int nctx = d->dir[0] ? PAIR_LEN : SKIP_ONE; + CBMHashTable *t = want == DK_WANT_FILE ? d->ix->files : d->ix->gomain; + const char *hit = NULL; + int hits = 0; + while (*src == '/') { + src++; + } + for (int i = 0; t && i < nctx; i++) { + const char *base = dk_join(d, ctxs[i], src); + const char *p = dk_norm(d, rel && rel[0] ? dk_join(d, base, rel) : base); + if (!p || strncmp(p, "..", PAIR_LEN) == 0 || !cbm_ht_get(t, p)) { + continue; + } + if (!hit || strcmp(hit, p) != 0) { + hits++; + hit = p; + } + } + return hits == SKIP_ONE ? hit : NULL; +} + +typedef struct { + const char *repo; /* a repository path, or */ + const dk_build_t *build; /* the go build that made the path */ + int build_stage; +} dk_found_t; + +/* Where container path `path` of `stage` came from: a repository path that is + * what `want` asks for, or a go build of the stage; false: nothing known. */ +static bool dk_origin(dk_t *d, int stage, const char *path, int want, int depth, dk_found_t *out) { + if (depth > DK_MAX_DEPTH || stage < 0 || stage >= DK_MAX_STAGES || !path) { + return false; + } + for (const dk_build_t *b = d->st[stage].builds; b; b = b->next) { + const char *o = b->out ? dk_absolute(d, b->out, b->workdir) : NULL; + if (o && strcmp(o, path) == 0) { + out->build = b; + out->build_stage = stage; + return true; + } + } + for (const dk_copy_t *c = d->st[stage].copies; c; c = c->next) { + bool known = c->dst != NULL; + for (int s = 0; known && s < c->nsrc; s++) { + known = c->srcs[s] != NULL; + } + if (!known) { + continue; + } + const char *dst = dk_absolute(d, c->dst, c->workdir); + int src_stage = c->has_from ? dk_stage_of(d, c->from) : -1; + if (!dst || (c->has_from && src_stage < 0)) { + continue; /* out of an image, or an unknown --from */ + } + size_t dl = strlen(dst); + bool into_dir = c->nsrc > SKIP_ONE || dk_ends(c->dst, "/"); + const char *tail = NULL; + if (strcmp(dst, "/") == 0) { + tail = strcmp(path, "/") != 0 ? path + SKIP_ONE : NULL; + } else if (strncmp(path, dst, dl) == 0 && path[dl] == '/') { + tail = path + dl + SKIP_ONE; + } + for (int s = 0; s < c->nsrc; s++) { + /* the source itself, a path inside a directory source, or a file + * source copied into the directory dst */ + const char *rels[PAIR_LEN + SKIP_ONE]; + int nc = 0; + if (strcmp(path, dst) == 0 && !into_dir) { + rels[nc++] = ""; + } + if (tail) { + rels[nc++] = tail; + const char *sn = dk_norm(d, c->srcs[s]); + if (sn && strcmp(dk_base(sn), tail) == 0) { + rels[nc++] = ""; + } + } + for (int k = 0; k < nc; k++) { + if (src_stage >= 0) { + const char *from = rels[k][0] ? dk_join(d, c->srcs[s], rels[k]) : c->srcs[s]; + if (dk_origin(d, src_stage, dk_absolute(d, from, "/"), want, depth + SKIP_ONE, + out)) { + return true; + } + continue; + } + const char *hit = dk_repo(d, c->srcs[s], rels[k], want); + if (hit) { + out->repo = hit; + return true; + } + } + } + } + return false; +} + +static const cbm_gbuf_node_t *dk_gomain_node(const dk_t *d, const char *dir) { + return dir && d->ix->gomain ? cbm_ht_get(d->ix->gomain, dir) : NULL; +} + +/* The `main` of the Go package build `b` (of `stage`) compiled: a file + * package through that file; a directory through the directory, else through + * the .go files copied straight into it (all from one repository dir). */ +static const cbm_gbuf_node_t *dk_gomain(dk_t *d, const dk_build_t *b, int stage) { + const char *pkg = b->pkg; + if (!pkg || strstr(pkg, "...") || strchr(pkg, '$')) { + return NULL; + } + dk_found_t f = {0}; + if (dk_ends(pkg, ".go")) { + if (!dk_origin(d, stage, dk_absolute(d, pkg, b->workdir), DK_WANT_FILE, 0, &f) || !f.repo) { + return NULL; + } + return dk_gomain_node(d, dk_dir(d, f.repo)); + } + const char *cdir = dk_absolute(d, pkg, b->workdir); + if (dk_origin(d, stage, cdir, DK_WANT_GOMAIN, 0, &f) && f.repo) { + return dk_gomain_node(d, f.repo); + } + const char *found = NULL; + bool many = false; + for (const dk_copy_t *c = d->st[stage].copies; c && cdir; c = c->next) { + if (c->has_from || !c->dst || c->nsrc != SKIP_ONE || !c->srcs[0] || + !dk_ends(c->srcs[0], ".go") || dk_ends(c->dst, "/")) { + continue; + } + const char *into = dk_absolute(d, c->dst, c->workdir); + if (!into || strcmp(dk_dir(d, into), cdir) != 0) { + continue; + } + const char *hit = dk_repo(d, c->srcs[0], "", DK_WANT_FILE); + const char *hd = hit ? dk_dir(d, hit) : NULL; + if (hd) { + many = many || (found && strcmp(found, hd) != 0); + found = hd; + } + } + return found && !many ? dk_gomain_node(d, found) : NULL; +} + +/* The words of shell text [p, e): split at blanks outside quotes, quotes + * and backslash escapes removed (a backslash-newline is a blank). */ +static int dk_words(dk_t *d, const char *p, const char *e, const char **out, int max) { + int n = 0; + while (p < e && n < max) { + while (p < e && (isspace((unsigned char)*p) || + (*p == '\\' && p + SKIP_ONE < e && p[SKIP_ONE] == '\n'))) { + p += *p == '\\' ? PAIR_LEN : SKIP_ONE; + } + if (p >= e) { + break; + } + char *w = cbm_arena_alloc(d->a, (size_t)(e - p) + SKIP_ONE); + if (!w) { + return n; + } + size_t len = 0; + char quote = 0; + while (p < e && (quote || !isspace((unsigned char)*p))) { + char c = *p++; + if (quote) { + if (c == quote) { + quote = 0; + } else if (c == '\\' && quote == '"' && p < e && + (*p == '"' || *p == '\\' || *p == '$' || *p == '`')) { + w[len++] = *p++; + } else { + w[len++] = c; + } + } else if (c == '\'' || c == '"') { + quote = c; + } else if (c == '\\' && p < e) { + if (*p == '\n') { + p++; + break; + } + w[len++] = *p++; + } else { + w[len++] = c; + } + } + w[len] = '\0'; + out[n++] = w; + } + return n; +} + +/* `go build` flags that take the next word as their value (go help build). */ +static bool dk_go_value_flag(const char *f) { + static const char *const FLAGS[] = { + "-o", "-ldflags", "-gcflags", "-asmflags", "-gccgoflags", "-tags", + "-mod", "-modfile", "-p", "-pkgdir", "-buildmode", "-installsuffix", + "-overlay", "-pgo", "-C", "-toolexec", "-compiler", NULL}; + for (int i = 0; FLAGS[i]; i++) { + if (strcmp(f, FLAGS[i]) == 0) { + return true; + } + } + return false; +} + +/* Each `go build [-o OUT] [PKG]` of one command list; OUT and PKG see the + * stage's ARG / ENV. */ +static void dk_go_builds(dk_t *d, const char **w, int n) { + for (int j = 0; j + SKIP_ONE < n; j++) { + if (strcmp(w[j], "go") != 0 || strcmp(w[j + SKIP_ONE], "build") != 0) { + continue; + } + const char *out = NULL; + const char *pkg = NULL; + for (int k = j + PAIR_LEN; k < n; k++) { + const char *a = w[k]; + if (strcmp(a, "-o") == 0 && k + SKIP_ONE < n) { + out = w[++k]; + } else if (strncmp(a, "-o=", strlen("-o=")) == 0) { + out = a + strlen("-o="); + } else if (a[0] == '-') { + k += dk_go_value_flag(a) ? SKIP_ONE : 0; + } else if (!pkg) { + pkg = a; + } + } + dk_build_t *b = cbm_arena_alloc(d->a, sizeof(*b)); + if (!b) { + return; + } + *b = (dk_build_t){.out = out ? dk_expand(d, d->local, out) : NULL, + .pkg = dk_expand(d, d->local, pkg ? pkg : "."), + .workdir = d->workdir, + .next = d->st[d->stage].builds}; + d->st[d->stage].builds = b; + } +} + +static void dk_run(dk_t *d, TSNode ins) { + if (d->stage >= DK_MAX_STAGES) { + return; + } + for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) { + TSNode sc = ts_node_named_child(ins, i); + if (!dk_type(sc, "shell_command")) { + continue; + } + /* command lists split at && || ; (as written, quotes or not) */ + const char *p = d->src + ts_node_start_byte(sc); + const char *end = d->src + ts_node_end_byte(sc); + while (p < end) { + const char *q = p; + while (q < end && + !(q + SKIP_ONE < end && ((q[0] == '&' && q[SKIP_ONE] == '&') || + (q[0] == '|' && q[SKIP_ONE] == '|'))) && + *q != ';') { + q++; + } + const char *w[DK_MAX_WORDS]; + int n = dk_words(d, p, q, w, DK_MAX_WORDS); + dk_go_builds(d, w, n); + p = q < end ? q + (*q == ';' ? SKIP_ONE : PAIR_LEN) : end; + } + } +} + +static void dk_workdir(dk_t *d, TSNode ins) { + for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) { + TSNode pn = ts_node_named_child(ins, i); + if (!dk_type(pn, "path")) { + continue; + } + const char *raw = dk_text(d, pn); + const char *v = raw ? dk_expand(d, d->local, raw) : NULL; + const char *w = v ? v : raw; + if (w) { + d->workdir = dk_absolute(d, w, d->workdir); + } + return; + } +} + +/* ENTRYPOINT / CMD of the current stage: its argv, as written. */ +static void dk_entry(dk_t *d, TSNode ins, dk_entry_t *e) { + const char *w[DK_MAX_WORDS]; + int n = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) { + TSNode c = ts_node_named_child(ins, i); + if (dk_type(c, "json_string_array")) { + n = dk_json_array(d, c, w, DK_MAX_WORDS); + } else if (dk_type(c, "shell_command")) { + n = dk_words(d, d->src + ts_node_start_byte(c), d->src + ts_node_end_byte(c), w, + DK_MAX_WORDS); + } + } + const char **argv = cbm_arena_alloc(d->a, (size_t)(n ? n : SKIP_ONE) * sizeof(*argv)); + if (!argv) { + return; + } + memcpy((void *)argv, w, (size_t)n * sizeof(*argv)); + *e = (dk_entry_t){ + .argv = argv, .argc = n, .line = dk_line(ins), .workdir = d->workdir, .set = true}; +} + +static bool dk_interpreter(const char *base) { + static const char *const NAMES[] = {"node", "php", "ruby", "perl", "bun", + "deno", "bash", "sh", NULL}; + for (int i = 0; NAMES[i]; i++) { + if (strcmp(base, NAMES[i]) == 0) { + return true; + } + } + if (strncmp(base, "python", strlen("python")) != 0) { + return false; + } + for (const char *p = base + strlen("python"); *p; p++) { + if (!isdigit((unsigned char)*p) && *p != '.') { + return false; + } + } + return true; +} + +/* SPAWNS from the Dockerfile to what its last stage runs. */ +static void dk_spawn(dk_t *d) { + if (d->stage < 0 || d->stage >= DK_MAX_STAGES || (!d->entry.set && !d->cmd.set)) { + return; + } + const dk_entry_t *e = d->entry.set ? &d->entry : &d->cmd; + const char *argv[DK_MAX_WORDS]; + int argc = 0; + for (int i = 0; i < e->argc; i++) { + if (e->argv[i] && e->argv[i][0]) { + argv[argc++] = e->argv[i]; + } + } + if (argc == 0 || strchr(argv[0], '$') || argv[0][0] == '-') { + return; /* unknown, or arguments to an inherited ENTRYPOINT */ + } + const char *prog = argv[0]; + const char *base = dk_base(prog); + if ((strcmp(base, "sh") == 0 || strcmp(base, "bash") == 0) && argc > SKIP_ONE && + strcmp(argv[SKIP_ONE], "-c") == 0) { + return; + } + dk_found_t f = {0}; + const char *path = NULL; + const cbm_gbuf_node_t *main_fn = NULL; + if (dk_interpreter(base)) { + const char *script = NULL; + for (int i = SKIP_ONE; i < argc && !script; i++) { + script = argv[i][0] != '-' ? argv[i] : NULL; + } + if (script && !strchr(script, '$') && + dk_origin(d, d->stage, dk_absolute(d, script, e->workdir), DK_WANT_FILE, 0, &f)) { + path = f.repo; + } + } else if (strchr(prog, '/')) { + if (dk_origin(d, d->stage, dk_absolute(d, prog, e->workdir), DK_WANT_FILE, 0, &f)) { + path = f.repo; + main_fn = f.build ? dk_gomain(d, f.build, f.build_stage) : NULL; + } + } else if (!d->entry.set && + dk_origin(d, d->stage, dk_absolute(d, prog, e->workdir), DK_WANT_FILE, 0, &f)) { + path = f.repo; /* a CMD file under an inherited ENTRYPOINT (distroless) */ + } + int64_t tgt = 0; + if (path) { + const char *mqn = cbm_fqn_module(d->a, d->ctx->project_name, path); + const cbm_gbuf_node_t *m = mqn ? cbm_gbuf_find_by_qn(d->ctx->gbuf, mqn) : NULL; + const cbm_gbuf_node_t *file = m ? NULL : cbm_ht_get(d->ix->files, path); + tgt = m ? m->id : file ? file->id : 0; + } else if (main_fn) { + tgt = main_fn->id; + } else { + char esc_base[CBM_SZ_512]; + cbm_json_escape(esc_base, (int)sizeof(esc_base), base); + char qn[CBM_SZ_512]; + snprintf(qn, sizeof(qn), "__process__%s", base); + char node_props[CBM_SZ_1K]; + snprintf(node_props, sizeof(node_props), "{\"program\":\"%s\"}", esc_base); + tgt = cbm_gbuf_upsert_node(d->ctx->gbuf, "Process", base, qn, "", 0, 0, node_props); + } + if (tgt <= 0 || tgt == d->module) { + return; + } + char esc_prog[CBM_SZ_1K]; + cbm_json_escape(esc_prog, (int)sizeof(esc_prog), prog); + char props[CBM_SZ_2K]; + snprintf(props, sizeof(props), + "{\"api\":\"dockerfile\",\"instruction\":\"%s\",\"program\":\"%s\",\"line\":%d," + "\"confidence\":1.00}", + d->entry.set ? "ENTRYPOINT" : "CMD", esc_prog, e->line); + cbm_gbuf_insert_edge(d->ctx->gbuf, d->module, tgt, "SPAWNS", props); + d->edges++; +} + +/* The environment the image of the last stage carries: the ENV keys of that + * stage and of the earlier stages it is built FROM (a key's nearest + * definition wins) -- image-scoped EnvVars the Dockerfile sets. */ +static void dk_envs(dk_t *d) { + const cbm_gbuf_node_t *mod = cbm_gbuf_find_by_id(d->ctx->gbuf, d->module); + if (!mod || d->stage < 0 || d->stage >= DK_MAX_STAGES) { + return; + } + CBMHashTable *seen = cbm_ht_create(0); + int s = d->stage; + for (int depth = 0; seen && s >= 0 && depth < DK_MAX_STAGES; depth++) { + for (const dk_envkey_t *k = d->st[s].envs; k; k = k->next) { + if (!cbm_ht_get(seen, k->name)) { + cbm_ht_set(seen, k->name, (void *)k); + if (cbm_pipeline_env_setter(d->ctx, mod, NULL, k->name, "dockerfile", k->line) > + 0) { + d->edges++; + } + } + } + int p = d->st[s].parent; + s = p >= 0 && p < s ? p : -1; + } + cbm_ht_free(seen); +} + +static void dk_file(dk_t *d, TSNode root) { + for (uint32_t i = 0; i < ts_node_named_child_count(root); i++) { + TSNode ins = ts_node_named_child(root, i); + if (dk_type(ins, "arg_instruction")) { + dk_arg(d, ins); + } else if (dk_type(ins, "env_instruction")) { + dk_env(d, ins); + } else if (dk_type(ins, "from_instruction")) { + dk_from(d, ins); + } else if ((dk_type(ins, "copy_instruction") || dk_type(ins, "add_instruction")) && + d->stage >= 0) { + dk_copy(d, ins); + } else if (dk_type(ins, "workdir_instruction") && d->stage >= 0) { + dk_workdir(d, ins); + } else if (dk_type(ins, "run_instruction") && d->stage >= 0) { + dk_run(d, ins); + } else if (dk_type(ins, "entrypoint_instruction") && d->stage >= 0) { + dk_entry(d, ins, &d->entry); + } else if (dk_type(ins, "cmd_instruction") && d->stage >= 0) { + dk_entry(d, ins, &d->cmd); + } + } + dk_spawn(d); + dk_envs(d); +} + +static char *dk_read(const char *path, size_t *len) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + enum { DK_TS_PAD = 16 }; /* tree-sitter's lexer reads past the end */ + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + DK_TS_PAD); + if (!buf) { + (void)fclose(f); + return NULL; + } + size_t got = fread(buf, SKIP_ONE, (size_t)size, f); + (void)fclose(f); + memset(buf + got, 0, DK_TS_PAD); + *len = got; + return buf; +} + +/* Does the line [b, e) hold a parser directive `# escape=...`? */ +static bool dk_escape_directive(const char *s, size_t b, size_t e) { + size_t i = b + SKIP_ONE; + while (i < e && (s[i] == ' ' || s[i] == '\t')) { + i++; + } + static const char kw[] = "escape"; + for (size_t k = 0; kw[k]; k++, i++) { + if (i >= e || tolower((unsigned char)s[i]) != kw[k]) { + return false; + } + } + while (i < e && (s[i] == ' ' || s[i] == '\t')) { + i++; + } + return i < e && s[i] == '='; +} + +/* Docker drops comment lines before it joins continued lines; the grammar + * ends an instruction at one (kubermatic: `ENV A=1 \` / `# url` / `B=2`). + * A comment line inside a continuation becomes a bare continuation (`#` -> + * `\`, the rest blank) -- rows and offsets stay. A file with an escape + * directive is left as it is. */ +static void dk_uncomment(char *s, size_t len) { + bool cont = false; + bool top = true; /* still in the leading directives */ + for (size_t i = 0; i < len;) { + size_t e = i; + while (e < len && s[e] != '\n') { + e++; + } + size_t b = i; + while (b < e && (s[b] == ' ' || s[b] == '\t')) { + b++; + } + if (b < e && s[b] == '#') { + if (top && dk_escape_directive(s, b, e)) { + return; + } + if (cont) { + s[b] = '\\'; + for (size_t k = b + SKIP_ONE; k < e && s[k] != '\r'; k++) { + s[k] = ' '; + } + } + } else if (b < e && s[b] != '\r') { + size_t t = e; + while (t > b && + (s[t - SKIP_ONE] == ' ' || s[t - SKIP_ONE] == '\t' || s[t - SKIP_ONE] == '\r')) { + t--; + } + cont = s[t - SKIP_ONE] == '\\'; + top = false; + } else { + top = false; + } + i = e + SKIP_ONE; + } +} + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMArena *a; + const char *file; + int edges; +} dk_builds_t; + +/* An Image a build configuration builds DEPENDS_ON {via:build} the + * Dockerfile it is built from. */ +static void dk_build_site(const cbm_build_site_t *s, void *ud) { + dk_builds_t *b = ud; + const char *mqn = cbm_fqn_module(b->a, b->ctx->project_name, s->dockerfile); + const cbm_gbuf_node_t *mod = mqn ? cbm_gbuf_find_by_qn(b->ctx->gbuf, mqn) : NULL; + if (!mod || !mod->file_path || strcmp(mod->file_path, s->dockerfile) != 0) { + return; + } + int64_t img = cbm_pipeline_image_node(b->ctx, s->image); + if (img <= 0) { + return; + } + char esc_file[CBM_SZ_1K]; + char esc_img[CBM_SZ_1K]; + cbm_json_escape(esc_file, (int)sizeof(esc_file), b->file); + cbm_json_escape(esc_img, (int)sizeof(esc_img), s->image); + char props[CBM_SZ_4K]; + snprintf(props, sizeof(props), + "{\"via\":\"build\",\"tool\":\"%s\",\"file\":\"%s\",\"image\":\"%s\",\"line\":%d}", + s->via, esc_file, esc_img, s->line); + cbm_gbuf_insert_edge(b->ctx->gbuf, img, mod->id, "DEPENDS_ON", props); + b->edges++; +} + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMArena *a; + const char *file; + const char *base; /* the compose file's module QN */ + int64_t file_node; + int pass; /* 0: make the service nodes, 1: their edges */ + int edges; +} dk_compose_t; + +static const char *dk_compose_qn(const dk_compose_t *c, const char *name) { + return cbm_arena_sprintf(c->a, "%s::compose/Service/-/%s", c->base, name); +} + +static void dk_compose_edge(dk_compose_t *c, int64_t src, int64_t tgt, const char *type, + const char *props) { + if (src > 0 && tgt > 0 && src != tgt) { + cbm_gbuf_insert_edge(c->ctx->gbuf, src, tgt, type, props); + c->edges++; + } +} + +/* A compose service: a Resource {ecosystem:compose, kind:Service} the file + * DEFINES; it DEPENDS_ON the services it depends on / links, USAGE + * {ref:image} its image and DEPENDS_ON {via:build} the Dockerfile it builds. */ +static void dk_compose_service(const cbm_compose_service_t *s, void *ud) { + dk_compose_t *c = ud; + const char *qn = dk_compose_qn(c, s->name); + if (!qn) { + return; + } + if (c->pass == 0) { + char esc[CBM_SZ_512]; + cbm_json_escape(esc, (int)sizeof(esc), s->name); + char props[CBM_SZ_1K]; + snprintf(props, sizeof(props), + "{\"ecosystem\":\"compose\",\"kind\":\"Service\",\"service\":\"%s\"}", esc); + int64_t id = cbm_gbuf_upsert_node(c->ctx->gbuf, "Resource", + cbm_arena_sprintf(c->a, "Service/%s", s->name), qn, + c->file, s->line + SKIP_ONE, s->end_line, props); + dk_compose_edge(c, c->file_node, id, "DEFINES", "{}"); + const cbm_gbuf_node_t *self = id > 0 ? cbm_gbuf_find_by_id(c->ctx->gbuf, id) : NULL; + for (int i = 0; self && i < s->nenv; i++) { + if (cbm_pipeline_env_setter(c->ctx, self, s->name, s->env[i], "compose", + s->env_lines[i]) > 0) { + c->edges++; + } + } + return; + } + const cbm_gbuf_node_t *self = cbm_gbuf_find_by_qn(c->ctx->gbuf, qn); + if (!self) { + return; + } + for (int i = 0; i < s->ndeps; i++) { + const char *dq = dk_compose_qn(c, s->deps[i]); + const cbm_gbuf_node_t *dep = dq ? cbm_gbuf_find_by_qn(c->ctx->gbuf, dq) : NULL; + dk_compose_edge(c, self->id, dep ? dep->id : 0, "DEPENDS_ON", "{\"via\":\"depends_on\"}"); + } + if (s->image) { + char esc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), s->image); + char props[CBM_SZ_2K]; + snprintf(props, sizeof(props), "{\"ref\":\"image\",\"image\":\"%s\"}", esc); + dk_compose_edge(c, self->id, cbm_pipeline_image_node(c->ctx, s->image), "USAGE", props); + } + if (s->dockerfile) { + const char *mqn = cbm_fqn_module(c->a, c->ctx->project_name, s->dockerfile); + const cbm_gbuf_node_t *mod = mqn ? cbm_gbuf_find_by_qn(c->ctx->gbuf, mqn) : NULL; + if (mod && mod->file_path && strcmp(mod->file_path, s->dockerfile) == 0) { + dk_compose_edge(c, self->id, mod->id, "DEPENDS_ON", + "{\"via\":\"build\",\"tool\":\"compose\"}"); + } + } +} + +/* The .env of the directory of `rel` under the repository, NULL when none. */ +static char *dk_read_dotenv(const cbm_pipeline_ctx_t *ctx, const char *rel) { + if (!ctx->repo_path) { + return NULL; + } + const char *slash = strrchr(rel, '/'); + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%.*s%s.env", ctx->repo_path, slash ? (int)(slash - rel) : 0, + rel, slash ? "/" : ""); + size_t len = 0; + return dk_read(path, &len); +} + +/* Every build configuration of the repository -- read from disk through the + * File nodes, so an incremental run (which recomputes these edges whole) + * sees the unchanged ones too. */ +static int dk_link_builds(cbm_pipeline_ctx_t *ctx, CBMArena *a) { + const cbm_gbuf_node_t **files = NULL; + int n = 0; + if (!ctx->repo_path || cbm_gbuf_find_by_label(ctx->gbuf, "File", &files, &n) != 0) { + return 0; + } + dk_builds_t b = {.ctx = ctx, .a = a}; + for (int i = 0; i < n; i++) { + const char *rel = files[i]->file_path; + if (!rel || !cbm_build_path(rel)) { + continue; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); + size_t len = 0; + char *src = dk_read(path, &len); + if (!src) { + continue; + } + char *dotenv = dk_read_dotenv(ctx, rel); + b.file = rel; + cbm_build_sites(a, rel, src, len, dotenv, dk_build_site, &b); + dk_compose_t cs = {.ctx = ctx, + .a = a, + .file = rel, + .base = cbm_fqn_compute(a, ctx->project_name, rel, NULL), + .file_node = files[i]->id}; + for (cs.pass = 0; cs.base && cs.pass < PAIR_LEN; cs.pass++) { + cbm_compose_services(a, rel, src, len, dotenv, dk_compose_service, &cs); + } + b.edges += cs.edges; + if (dotenv) { + cbm_free(CBM_MEM_CLASS_OTHER, dotenv); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + return b.edges; +} + +static bool dk_node_before(const cbm_gbuf_node_t *a, const cbm_gbuf_node_t *b) { + int c = strcmp(a->file_path, b->file_path); + return c < 0 || (c == 0 && a->start_line < b->start_line); +} + +/* The repository's File nodes by path, and the `main` of each Go main + * package by directory (the first by file and line). */ +static void dk_index_build(cbm_pipeline_ctx_t *ctx, CBMArena *a, dk_index_t *ix) { + ix->files = cbm_ht_create(0); + ix->gomain = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (ix->files && cbm_gbuf_find_by_label(ctx->gbuf, "File", &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + if (nodes[i]->file_path) { + cbm_ht_set(ix->files, nodes[i]->file_path, (void *)nodes[i]); + } + } + } + if (!ix->gomain || cbm_gbuf_find_by_name(ctx->gbuf, "main", &nodes, &n) != 0) { + return; + } + for (int i = 0; i < n; i++) { + const cbm_gbuf_node_t *fn = nodes[i]; + const char *fp = fn->file_path; + if (!fp || strcmp(fn->label, "Function") != 0 || !dk_ends(fp, ".go") || + dk_ends(fp, "_test.go")) { + continue; + } + const char *slash = strrchr(fp, '/'); + char *dir = cbm_arena_alloc(a, slash ? (size_t)(slash - fp) + SKIP_ONE : SKIP_ONE); + if (!dir) { + continue; + } + size_t dl = slash ? (size_t)(slash - fp) : 0; + memcpy(dir, fp, dl); + dir[dl] = '\0'; + const cbm_gbuf_node_t *have = cbm_ht_get(ix->gomain, dir); + if (!have || dk_node_before(fn, have)) { + cbm_ht_set(ix->gomain, dir, (void *)fn); + } + } +} + +/* ── Image tag sub-identity (P4.2b) ─────────────────────────────── */ + +_Atomic uint64_t g_dk_hub_edges = 0; /* complexity guard: Image edges a run examines */ + +/* A plain JSON string property (an image reference holds no escapes). */ +static bool dk_prop(const char *props, const char *key, char *out, size_t out_sz) { + char pat[CBM_SZ_64]; + snprintf(pat, sizeof(pat), "\"%s\":\"", key); + const char *p = props ? strstr(props, pat) : NULL; + if (!p) { + return false; + } + p += strlen(pat); + const char *e = strchr(p, '"'); + if (!e || (size_t)(e - p) >= out_sz || memchr(p, '\\', (size_t)(e - p))) { + return false; + } + memcpy(out, p, (size_t)(e - p)); + out[e - p] = '\0'; + return true; +} + +/* The tag of image reference `ref`: after the ':' that follows its last + * '/' (a registry port is no tag), up to a '@' digest; "" for none. */ +static void dk_tag(const char *ref, char *out, size_t sz) { + out[0] = '\0'; + const char *slash = strrchr(ref, '/'); + const char *colon = strchr(slash ? slash : ref, ':'); + if (colon) { + snprintf(out, sz, "%.*s", (int)strcspn(colon + SKIP_ONE, "@"), colon + SKIP_ONE); + } +} + +/* The repository of an Image node's name ("" or ":"). */ +static void dk_repo_of(const char *name, char *out, size_t sz) { + const char *slash = strrchr(name, '/'); + const char *colon = strchr(slash ? slash : name, ':'); + snprintf(out, sz, "%.*s", (int)(colon ? (size_t)(colon - name) : strlen(name)), name); +} + +static bool dk_sep(char c) { + return c == '-' || c == '_' || c == '.'; +} + +/* Does `comp` occur in `tag` as whole separator-delimited parts? */ +static bool dk_tag_has(const char *tag, const char *comp) { + size_t cl = strlen(comp); + for (const char *p = strstr(tag, comp); p; p = strstr(p + SKIP_ONE, comp)) { + if ((p == tag || dk_sep(p[-SKIP_ONE])) && (p[cl] == '\0' || dk_sep(p[cl]))) { + return true; + } + } + return false; +} + +/* An edge of a repository's Images, copied before any edge moves. */ +typedef struct { + const cbm_gbuf_edge_t *edge; /* stable until deleted */ + int64_t src; + int64_t tgt; + const char *type; + const char *props; + const char *tag; + int comp; /* its component; -1: none */ +} dk_hub_edge_t; + +typedef struct { + dk_hub_edge_t *v; + int n; + int cap; +} dk_hub_edges_t; + +typedef struct { + const cbm_gbuf_edge_t **v; + int n; + int cap; +} dk_drop_t; + +typedef struct { + int64_t id; + const char *name; + const char *repo; +} dk_img_t; + +static int dk_img_cmp(const void *x, const void *y) { + const dk_img_t *a = x; + const dk_img_t *b = y; + int c = strcmp(a->repo, b->repo); + return c ? c : strcmp(a->name, b->name); +} + +/* Grow an arena array of `n` items of `size` to hold one more. */ +static void *dk_grow(CBMArena *a, void *v, int n, int *cap, size_t size) { + if (n < *cap) { + return v; + } + int nc = *cap ? *cap * PAIR_LEN : CBM_SZ_16; + void *g = cbm_arena_alloc(a, (size_t)nc * size); + if (g && n) { + memcpy(g, v, (size_t)n * size); + } + if (g) { + *cap = nc; + } + return g; +} + +static void dk_hub_push(CBMArena *a, dk_hub_edges_t *l, const cbm_gbuf_edge_t *e, const char *tag) { + dk_hub_edge_t *v = dk_grow(a, l->v, l->n, &l->cap, sizeof(*v)); + if (!v) { + return; + } + l->v = v; + l->v[l->n++] = (dk_hub_edge_t){e, + e->source_id, + e->target_id, + cbm_arena_strdup(a, e->type), + cbm_arena_strdup(a, e->properties_json), + cbm_arena_strdup(a, tag), + -1}; +} + +/* Is an edge of `type` with properties `p` a reference to an image -- + * USAGE {ref:image}, or DEPENDS_ON {via:from|copy-from}? */ +static bool dk_hub_ref(const char *type, const char *p) { + char v[CBM_SZ_32]; + if (strcmp(type, "USAGE") == 0) { + return dk_prop(p, "ref", v, sizeof(v)) && strcmp(v, "image") == 0; + } + return dk_prop(p, "via", v, sizeof(v)) && + (strcmp(v, "from") == 0 || strcmp(v, "copy-from") == 0); +} + +/* The edges of Image `img`: build edges out (DEPENDS_ON {via:build}) and + * references in, each with the tag of its `image` reference. */ +static void dk_hub_collect(cbm_pipeline_ctx_t *ctx, CBMArena *a, int64_t img, + dk_hub_edges_t *builds, dk_hub_edges_t *refs) { + const cbm_gbuf_edge_t **es = NULL; + int n = 0; + char ref[CBM_SZ_1K]; + char tag[CBM_SZ_256]; + if (cbm_gbuf_find_edges_by_source_type(ctx->gbuf, img, "DEPENDS_ON", &es, &n) == 0) { + for (int i = 0; i < n; i++) { + char via[CBM_SZ_32]; + if (!dk_prop(es[i]->properties_json, "via", via, sizeof(via)) || + strcmp(via, "build") != 0) { + continue; + } + tag[0] = '\0'; + if (dk_prop(es[i]->properties_json, "image", ref, sizeof(ref))) { + dk_tag(ref, tag, sizeof(tag)); + } + dk_hub_push(a, builds, es[i], tag); + } + atomic_fetch_add_explicit(&g_dk_hub_edges, (uint64_t)n, memory_order_relaxed); + } + static const char *const REF_TYPES[] = {"USAGE", "DEPENDS_ON"}; + for (size_t t = 0; t < sizeof(REF_TYPES) / sizeof(REF_TYPES[0]); t++) { + if (cbm_gbuf_find_edges_by_target_type(ctx->gbuf, img, REF_TYPES[t], &es, &n) != 0) { + continue; + } + atomic_fetch_add_explicit(&g_dk_hub_edges, (uint64_t)n, memory_order_relaxed); + for (int i = 0; i < n; i++) { + if (dk_hub_ref(REF_TYPES[t], es[i]->properties_json) && + dk_prop(es[i]->properties_json, "image", ref, sizeof(ref))) { + dk_tag(ref, tag, sizeof(tag)); + dk_hub_push(a, refs, es[i], tag); + } + } + } +} + +/* The number of distinct Dockerfiles the build edges target. */ +static int dk_hub_dockerfiles(CBMArena *a, const dk_hub_edges_t *b) { + CBMHashTable *seen = cbm_ht_create(0); + int n = 0; + for (int i = 0; seen && i < b->n; i++) { + const char *k = cbm_arena_sprintf(a, "%lld", (long long)b->v[i].tgt); + if (k && !cbm_ht_has(seen, k)) { + cbm_ht_set(seen, k, (void *)SKIP_ONE); + n++; + } + } + if (seen) { + cbm_ht_free(seen); + } + return n; +} + +/* The common prefix / suffix of the build tags, cut back to a separator and + * leaving every tag a non-empty middle where it can: *pre, *suf. */ +static void dk_hub_affixes(const dk_hub_edges_t *b, size_t *pre, size_t *suf) { + const char *t0 = b->v[0].tag; + size_t l0 = strlen(t0); + size_t p = l0; + size_t s = l0; + for (int i = SKIP_ONE; i < b->n; i++) { + const char *t = b->v[i].tag; + size_t tl = strlen(t); + size_t k = 0; + while (k < p && k < tl && t[k] == t0[k]) { + k++; + } + p = k; + k = 0; + while (k < s && k < tl && t[tl - SKIP_ONE - k] == t0[l0 - SKIP_ONE - k]) { + k++; + } + s = k; + } + while (p > 0 && !dk_sep(t0[p - SKIP_ONE])) { + p--; + } + while (s > 0 && !dk_sep(t0[l0 - s])) { + s--; + } + for (int i = 0; i < b->n; i++) { + size_t tl = strlen(b->v[i].tag); + while (s > 0 && p + s >= tl) { + for (s--; s > 0 && !dk_sep(t0[l0 - s]); s--) {} + } + while (p > 0 && p + s >= tl) { + for (p--; p > 0 && !dk_sep(t0[p - SKIP_ONE]); p--) {} + } + } + *pre = p; + *suf = s; +} + +/* Components of a repository built from several Dockerfiles: each build + * tag without the tags' common prefix / suffix (otel's latest-frontend, + * latest-cart -> frontend, cart). A component built from exactly one + * Dockerfile, holding no unknown part, is kept (comps[k] non-NULL); each + * build edge's comp is set. Returns the number of component slots (0: + * none kept). */ +static int dk_hub_components(CBMArena *a, dk_hub_edges_t *b, size_t pre, size_t suf, + const char ***comps) { + const char **names = cbm_arena_alloc(a, (size_t)b->n * sizeof(*names)); + int64_t *dfs = cbm_arena_alloc(a, (size_t)b->n * sizeof(*dfs)); + CBMHashTable *index = cbm_ht_create(0); /* component -> slot + 1 */ + int nc = 0; + for (int i = 0; names && dfs && index && i < b->n; i++) { + const char *t = b->v[i].tag; + size_t tl = strlen(t); + const char *c = tl > pre + suf ? cbm_arena_strndup(a, t + pre, tl - pre - suf) : NULL; + if (!c || strpbrk(c, "$*{}()`\"\\")) { + continue; + } + int k = (int)(intptr_t)cbm_ht_get(index, c) - SKIP_ONE; + if (k < 0) { + k = nc++; + names[k] = c; + dfs[k] = b->v[i].tgt; + cbm_ht_set(index, c, (void *)(intptr_t)(k + SKIP_ONE)); + } else if (names[k] && dfs[k] != b->v[i].tgt) { + names[k] = NULL; /* one name, two Dockerfiles: no identity */ + } + b->v[i].comp = k; + } + if (index) { + cbm_ht_free(index); + } + if (!names || !dfs) { + return 0; + } + int kept = 0; + for (int k = 0; k < nc; k++) { + kept += names[k] ? SKIP_ONE : 0; + } + for (int i = 0; i < b->n; i++) { + if (b->v[i].comp >= 0 && !names[b->v[i].comp]) { + b->v[i].comp = -1; + } + } + *comps = names; + return kept ? nc : 0; +} + +/* The component a reference's tag names: the longest kept one occurring in + * it as whole parts; -1 for none, or for two of that length. */ +static int dk_hub_match(const char *tag, const char *const *comps, int nc) { + int best = -1; + size_t bl = 0; + bool tie = false; + for (int k = 0; k < nc; k++) { + if (!comps[k] || !dk_tag_has(tag, comps[k])) { + continue; + } + size_t l = strlen(comps[k]); + if (l > bl) { + best = k; + bl = l; + tie = false; + } else if (l == bl) { + tie = true; + } + } + return tie ? -1 : best; +} + +/* The Image of a component: `:` (`*-frontend`: the + * prefix's version masked) {repository, component, tag_pattern}. */ +static int64_t dk_hub_image(cbm_pipeline_ctx_t *ctx, const char *repo, const char *comp, + char pre_sep, char suf_sep) { + char head[PAIR_LEN + SKIP_ONE] = {0}; + char tail[PAIR_LEN + SKIP_ONE] = {0}; + if (pre_sep) { + head[0] = '*'; + head[SKIP_ONE] = pre_sep; + } + if (suf_sep) { + tail[0] = suf_sep; + tail[SKIP_ONE] = '*'; + } + char pattern[CBM_SZ_512]; + char name[CBM_SZ_1K]; + char qn[CBM_SZ_2K]; + snprintf(pattern, sizeof(pattern), "%s%s%s", head, comp, tail); + snprintf(name, sizeof(name), "%s:%s", repo, pattern); + snprintf(qn, sizeof(qn), "__image__%s", name); + char esc_r[CBM_SZ_1K]; + char esc_c[CBM_SZ_1K]; + char esc_p[CBM_SZ_1K]; + cbm_json_escape(esc_r, (int)sizeof(esc_r), repo); + cbm_json_escape(esc_c, (int)sizeof(esc_c), comp); + cbm_json_escape(esc_p, (int)sizeof(esc_p), pattern); + char props[CBM_SZ_4K]; + snprintf(props, sizeof(props), + "{\"repository\":\"%s\",\"component\":\"%s\",\"tag_pattern\":\"%s\"}", esc_r, esc_c, + esc_p); + return cbm_gbuf_upsert_node(ctx->gbuf, "Image", name, qn, "", 0, 0, props); +} + +/* Move edge `e` to (src, tgt): insert the copy, queue `e` for deletion. */ +static int dk_hub_move(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_hub_edge_t *e, int64_t src, + int64_t tgt, dk_drop_t *drop) { + if ((src == e->src && tgt == e->tgt) || src <= 0 || tgt <= 0) { + return 0; + } + const cbm_gbuf_edge_t **v = dk_grow(a, (void *)drop->v, drop->n, &drop->cap, sizeof(*v)); + if (!v) { + return 0; + } + drop->v = v; + drop->v[drop->n++] = e->edge; + cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, e->type, e->props); + return SKIP_ONE; +} + +/* One repository's Images (imgs[0..n)): split it by component when it is + * built from several Dockerfiles, else keep (or bring back) everything on + * the repository Image. Returns the edges moved. */ +static int dk_hub_repo(cbm_pipeline_ctx_t *ctx, CBMArena *a, const dk_img_t *imgs, int n, + dk_drop_t *drop) { + dk_hub_edges_t builds = {0}; + dk_hub_edges_t refs = {0}; + bool has_sub = false; + for (int i = 0; i < n; i++) { + has_sub = has_sub || strcmp(imgs[i].name, imgs[i].repo) != 0; + dk_hub_collect(ctx, a, imgs[i].id, &builds, &refs); + } + const char **comps = NULL; + size_t pre = 0; + size_t suf = 0; + int nc = 0; + if (builds.n > SKIP_ONE && dk_hub_dockerfiles(a, &builds) > SKIP_ONE) { + dk_hub_affixes(&builds, &pre, &suf); + nc = dk_hub_components(a, &builds, pre, suf, &comps); + } + if (nc == 0 && !has_sub) { + return 0; + } + int64_t repo_img = cbm_pipeline_image_node(ctx, imgs[0].repo); + int64_t *sub = nc ? cbm_arena_alloc(a, (size_t)nc * sizeof(*sub)) : NULL; + const char *t0 = builds.n ? builds.v[0].tag : ""; + char pre_sep = pre ? t0[pre - SKIP_ONE] : '\0'; + char suf_sep = suf ? t0[strlen(t0) - suf] : '\0'; + for (int k = 0; k < nc && sub; k++) { + sub[k] = comps[k] ? dk_hub_image(ctx, imgs[0].repo, comps[k], pre_sep, suf_sep) : 0; + } + int moved = 0; + for (int i = 0; i < builds.n; i++) { + int k = builds.v[i].comp; + int64_t src = sub && k >= 0 && sub[k] > 0 ? sub[k] : repo_img; + moved += dk_hub_move(ctx, a, &builds.v[i], src, builds.v[i].tgt, drop); + } + for (int i = 0; i < refs.n; i++) { + int k = sub ? dk_hub_match(refs.v[i].tag, comps, nc) : -1; + int64_t tgt = k >= 0 && sub[k] > 0 ? sub[k] : repo_img; + moved += dk_hub_move(ctx, a, &refs.v[i], refs.v[i].src, tgt, drop); + } + return moved; +} + +/* P4.2b: a repository built from more than one Dockerfile (otel builds + * every service as ghcr.io/open-telemetry/demo:-) is no + * single image. Each component of its build tags built from one Dockerfile + * becomes an Image of its own, which its build edges leave from; every + * reference (K8s container, compose service, FROM) whose tag names a + * component uses that Image; all else stays on the repository Image. + * Recomputed whole each run from the edges in the buffer. */ +static int dk_image_hubs(cbm_pipeline_ctx_t *ctx, CBMArena *a) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "Image", &nodes, &n) != 0 || n == 0) { + return 0; + } + dk_img_t *imgs = cbm_arena_alloc(a, (size_t)n * sizeof(*imgs)); + if (!imgs) { + return 0; + } + for (int i = 0; i < n; i++) { + char repo[CBM_SZ_1K]; + dk_repo_of(nodes[i]->name, repo, sizeof(repo)); + imgs[i] = (dk_img_t){nodes[i]->id, cbm_arena_strdup(a, nodes[i]->name), + cbm_arena_strdup(a, repo)}; + } + qsort(imgs, (size_t)n, sizeof(*imgs), dk_img_cmp); + dk_drop_t drop = {0}; + int moved = 0; + for (int g = 0; g < n;) { + int h = g + SKIP_ONE; + while (h < n && strcmp(imgs[h].repo, imgs[g].repo) == 0) { + h++; + } + moved += dk_hub_repo(ctx, a, imgs + g, h - g, &drop); + g = h; + } + cbm_gbuf_delete_edges(ctx->gbuf, drop.v, drop.n); + return moved; +} + +void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count) { + TSParser *parser = NULL; + CBMArena arena; + cbm_arena_init_lazy(&arena, CBM_SZ_4K); + dk_index_t ix = {0}; + int edges = 0; + for (int i = 0; i < file_count; i++) { + if (files[i].language != CBM_LANG_DOCKERFILE) { + continue; + } + if (!ix.files) { + dk_index_build(ctx, &arena, &ix); + } + const char *mqn = cbm_fqn_module(&arena, ctx->project_name, files[i].rel_path); + const cbm_gbuf_node_t *mod = mqn ? cbm_gbuf_find_by_qn(ctx->gbuf, mqn) : NULL; + if (!mod) { + continue; + } + size_t len = 0; + char *src = dk_read(files[i].path, &len); + if (!src) { + continue; + } + dk_uncomment(src, len); + if (!parser) { + parser = ts_parser_new(); + } + TSTree *tree = NULL; + if (parser && ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_DOCKERFILE))) { + tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + } + if (tree) { + const char *slash = strrchr(files[i].rel_path, '/'); + char *dir = cbm_arena_alloc( + &arena, slash ? (size_t)(slash - files[i].rel_path) + SKIP_ONE : SKIP_ONE); + if (dir) { + size_t dl = slash ? (size_t)(slash - files[i].rel_path) : 0; + memcpy(dir, files[i].rel_path, dl); + dir[dl] = '\0'; + } + dk_t d = {.ctx = ctx, + .a = &arena, + .src = src, + .file = files[i].rel_path, + .module = mod->id, + .stage = -1, + .dir = dir ? dir : "", + .workdir = "/", + .ix = &ix}; + dk_file(&d, ts_tree_root_node(tree)); + edges += d.edges; + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + if (parser) { + ts_parser_delete(parser); + } + if (ix.files) { + cbm_ht_free(ix.files); + } + if (ix.gomain) { + cbm_ht_free(ix.gomain); + } + edges += dk_link_builds(ctx, &arena); + edges += dk_image_hubs(ctx, &arena); + cbm_arena_destroy(&arena); + if (edges > 0) { + char buf[CBM_SZ_32]; + snprintf(buf, sizeof(buf), "%d", edges); + cbm_log_info("pass.docker.edges", "edges", buf); + } +} diff --git a/src/pipeline/pass_envscan.c b/src/pipeline/pass_envscan.c deleted file mode 100644 index 88bd2e685..000000000 --- a/src/pipeline/pass_envscan.c +++ /dev/null @@ -1,625 +0,0 @@ -/* - * pass_envscan.c — Environment URL scanner. - * - * Walks a project directory, scans config files (Dockerfile, .env, shell, - * YAML, TOML, Terraform, .properties) for environment variable assignments - * where the value is a URL. Filters out secrets. - * - * Port of internal/pipeline/envscan.go:ScanProjectEnvURLs(). - */ -#include "foundation/constants.h" - -enum { - ENV_REGEX_MAX = 5, - ENV_GRP_1 = 1, - ENV_GRP_2 = 2, - ENV_GRP_3 = 3, - ENV_GRP_4 = 4, - ENV_GRP_5 = 5, - ENV_EXT_LEN = 4, /* strlen(".env") */ -}; - -#define SLEN(s) (sizeof(s) - 1) -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_internal.h" -#include "foundation/log.h" - -#include -#include "foundation/compat_fs.h" -#include "foundation/compat_regex.h" -#include "foundation/platform.h" /* cbm_normalize_path_sep */ -#include -#include -#include -#include - -#ifdef _WIN32 -#include "foundation/win_utf8.h" /* cbm_path_to_wide */ -#include -/* Canonical paths on both sides come from the same resolver, so casing already - * agrees; comparing case-insensitively removes the doubt at no cost. */ -#define envscan_path_ncmp _strnicmp -#else -#include -#include -#define envscan_path_ncmp strncmp -#endif - -/* ── Regex patterns (compiled lazily) ──────────────────────────── */ - -static cbm_regex_t dockerfile_re; /* ENV|ARG KEY=VALUE or KEY VALUE */ -static cbm_regex_t yaml_kv_re; /* key: "https://..." */ -static cbm_regex_t yaml_setenv_re; /* --set-env-vars KEY=VALUE */ -static cbm_regex_t terraform_re; /* default|value = "https://..." */ -static cbm_regex_t shell_re; /* [export] KEY=https://... */ -static cbm_regex_t envfile_re; /* KEY=https://... */ -static cbm_regex_t toml_re; /* key = "https://..." */ -static cbm_regex_t properties_re; /* key=https://... */ -static int patterns_compiled = 0; - -/* POSIX ERE doesn't support \w or \S — use bracket expressions */ -#define W "[A-Za-z0-9_]" /* word char */ -#define NW "[^ \t\"']" /* non-whitespace, non-quote */ - -static void compile_patterns(void) { - if (patterns_compiled) { - return; - } - - cbm_regcomp(&dockerfile_re, "^(ENV|ARG)[[:space:]]+(" W "+)[= ](.*)", CBM_REG_EXTENDED); - cbm_regcomp(&yaml_kv_re, "(" W "+):[[:space:]]*[\"']?(https?://" NW "+)", CBM_REG_EXTENDED); - cbm_regcomp(&yaml_setenv_re, "--set-env-vars[[:space:]]+(" W "+)=([^ \t]+)", CBM_REG_EXTENDED); - cbm_regcomp(&terraform_re, "(default|value)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", - CBM_REG_EXTENDED); - cbm_regcomp(&shell_re, "(export[[:space:]]+)?(" W "+)=[\"']?(https?://" NW "+)", - CBM_REG_EXTENDED); - cbm_regcomp(&envfile_re, "^(" W "+)=(https?://[^ \t]+)", CBM_REG_EXTENDED); - cbm_regcomp(&toml_re, "(" W "+)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", - CBM_REG_EXTENDED); - cbm_regcomp(&properties_re, "(" W "+)[[:space:]]*=[[:space:]]*(https?://[^ \t]+)", - CBM_REG_EXTENDED); - - patterns_compiled = SKIP_ONE; -} - -#undef W -#undef NW - -/* ── File type detection ───────────────────────────────────────── */ - -static int is_dockerfile_name(const char *name) { - /* Case-insensitive check */ - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - if (strcmp(lower, "dockerfile") == 0) { - return SKIP_ONE; - } -#define DOCKERFILE_SUFFIX_LEN 11 /* strlen("dockerfile.") == strlen(".dockerfile") */ - if (strncmp(lower, "dockerfile.", DOCKERFILE_SUFFIX_LEN) == 0) { - return SKIP_ONE; - } - if (len > DOCKERFILE_SUFFIX_LEN && - strcmp(lower + len - DOCKERFILE_SUFFIX_LEN, ".dockerfile") == 0) { - return SKIP_ONE; - } - return 0; -} - -static int is_env_file_name(const char *name) { - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - if (strcmp(lower, ".env") == 0) { - return SKIP_ONE; - } - if (strncmp(lower, ".env.", SLEN(".env.")) == 0) { - return SKIP_ONE; - } - if (len > ENV_EXT_LEN && strcmp(lower + len - ENV_EXT_LEN, ".env") == 0) { - return SKIP_ONE; - } - return 0; -} - -static int is_secret_file(const char *name) { - char lower[CBM_SZ_256]; - size_t len = strlen(name); - if (len >= sizeof(lower)) { - return 0; - } - for (size_t i = 0; i <= len; i++) { - lower[i] = (char)tolower((unsigned char)name[i]); - } - - static const char *patterns[] = { - "service_account", "credentials", "key.json", "key.pem", "id_rsa", - "id_ed25519", ".pem", ".key", NULL}; - for (int i = 0; patterns[i]; i++) { - if (strstr(lower, patterns[i])) { - return SKIP_ONE; - } - } - return 0; -} - -/* ── Ignored directories ───────────────────────────────────────── */ - -static int is_ignored_dir(const char *name) { - static const char *dirs[] = { - ".git", "node_modules", ".svn", ".hg", "__pycache__", "vendor", ".terraform", ".cache", - ".idea", ".vscode", "dist", "build", ".next", ".nuxt", "target", NULL}; - for (int i = 0; dirs[i]; i++) { - if (strcmp(name, dirs[i]) == 0) { - return SKIP_ONE; - } - } - return 0; -} - -/* ── File type enum ────────────────────────────────────────────── */ - -typedef enum { - FT_UNKNOWN = 0, - FT_DOCKERFILE, - FT_YAML, - FT_TERRAFORM, - FT_SHELL, - FT_ENVFILE, - FT_TOML, - FT_PROPERTIES, -} file_type_t; - -static file_type_t detect_file_type(const char *name) { - if (is_dockerfile_name(name)) { - return FT_DOCKERFILE; - } - if (is_env_file_name(name)) { - return FT_ENVFILE; - } - - const char *ext = strrchr(name, '.'); - if (!ext) { - return FT_UNKNOWN; - } - - if (strcmp(ext, ".yaml") == 0 || strcmp(ext, ".yml") == 0) { - return FT_YAML; - } - if (strcmp(ext, ".tf") == 0 || strcmp(ext, ".hcl") == 0) { - return FT_TERRAFORM; - } - if (strcmp(ext, ".sh") == 0 || strcmp(ext, ".bash") == 0 || strcmp(ext, ".zsh") == 0) { - return FT_SHELL; - } - if (strcmp(ext, ".toml") == 0) { - return FT_TOML; - } - if (strcmp(ext, ".properties") == 0 || strcmp(ext, ".cfg") == 0 || strcmp(ext, ".ini") == 0) { - return FT_PROPERTIES; - } - - return FT_UNKNOWN; -} - -/* ── Line scanner ──────────────────────────────────────────────── */ - -/* Extract key/value from a regex match with two capture groups. - * Returns 1 on success, 0 if groups are empty or too large. */ -static int extract_kv_groups(const char *trimmed, const cbm_regmatch_t *m, int key_grp, int val_grp, - char *key_out, size_t key_sz, char *val_out, size_t val_sz) { - int klen = (m[key_grp].rm_eo - m[key_grp].rm_so); - int vlen = (m[val_grp].rm_eo - m[val_grp].rm_so); - if (klen <= 0 || klen >= (int)key_sz || vlen <= 0 || vlen >= (int)val_sz) { - return 0; - } - memcpy(key_out, trimmed + m[key_grp].rm_so, klen); - key_out[klen] = '\0'; - memcpy(val_out, trimmed + m[val_grp].rm_so, vlen); - val_out[vlen] = '\0'; - return SKIP_ONE; -} - -/* Try to scan a Dockerfile line. */ -static int scan_dockerfile_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&dockerfile_re, line, ENV_GRP_4, m, 0) != 0) { - return 0; - } - if (!extract_kv_groups(line, m, ENV_GRP_2, ENV_GRP_3, key, ksz, val, vsz)) { - return 0; - } - size_t vl = strlen(val); - while (vl > 0 && (val[vl - SKIP_ONE] == '"' || val[vl - SKIP_ONE] == '\'')) { - val[--vl] = '\0'; - } - return SKIP_ONE; -} - -/* Try to scan a YAML line. */ -static int scan_yaml_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&yaml_kv_re, line, ENV_GRP_3, m, 0) == 0 && - extract_kv_groups(line, m, ENV_GRP_1, ENV_GRP_2, key, ksz, val, vsz)) { - return SKIP_ONE; - } - if (cbm_regexec(&yaml_setenv_re, line, ENV_GRP_3, m, 0) == 0 && - extract_kv_groups(line, m, ENV_GRP_1, ENV_GRP_2, key, ksz, val, vsz)) { - return SKIP_ONE; - } - return 0; -} - -/* Try to scan a Terraform line. */ -static int scan_terraform_line(const char *line, char *key, size_t ksz, char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(&terraform_re, line, ENV_GRP_3, m, 0) != 0) { - return 0; - } - int vlen = (m[ENV_GRP_2].rm_eo - m[ENV_GRP_2].rm_so); - if (vlen <= 0 || vlen >= (int)vsz) { - return 0; - } - strncpy(key, "_tf_default", ksz - SKIP_ONE); - key[ksz - SKIP_ONE] = '\0'; - memcpy(val, line + m[ENV_GRP_2].rm_so, vlen); - val[vlen] = '\0'; - return SKIP_ONE; -} - -/* Try single-regex scan (shell, envfile, toml, properties). */ -static int scan_regex_line(cbm_regex_t *re, const char *line, int kg, int vg, char *key, size_t ksz, - char *val, size_t vsz) { - cbm_regmatch_t m[ENV_REGEX_MAX]; - if (cbm_regexec(re, line, ENV_GRP_5, m, 0) == 0 && - extract_kv_groups(line, m, kg, vg, key, ksz, val, vsz)) { - return SKIP_ONE; - } - return 0; -} - -static int scan_line(const char *line, file_type_t ft, char *key_out, size_t key_sz, char *val_out, - size_t val_sz) { - const char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - if (*trimmed == '#' || (trimmed[0] == '/' && trimmed[SKIP_ONE] == '/')) { - return 0; - } - - switch (ft) { - case FT_DOCKERFILE: - return scan_dockerfile_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_YAML: - return scan_yaml_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_TERRAFORM: - return scan_terraform_line(trimmed, key_out, key_sz, val_out, val_sz); - case FT_SHELL: - return scan_regex_line(&shell_re, trimmed, ENV_GRP_2, ENV_GRP_3, key_out, key_sz, val_out, - val_sz); - case FT_ENVFILE: - return scan_regex_line(&envfile_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, val_out, - val_sz); - case FT_TOML: - return scan_regex_line(&toml_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, val_out, - val_sz); - case FT_PROPERTIES: - return scan_regex_line(&properties_re, trimmed, ENV_GRP_1, ENV_GRP_2, key_out, key_sz, - val_out, val_sz); - default: - return 0; - } -} - -/* ── Containment ───────────────────────────────────────────────── */ - -/* Join dir + name, rejecting rather than truncating. - * - * Silent truncation here was not merely a lost path. The root-relative suffix - * used to be derived as `full_path + strlen(root_path)`, so once the real root - * was longer than this fixed buffer that pointer landed past the end of the - * array — out-of-bounds arithmetic on every entry below it. Refusing the path is - * what keeps envscan_relative_suffix in bounds by construction. */ -static bool envscan_join_path(char *out, size_t out_sz, const char *dir, const char *name) { - if (out_sz == 0) { - return false; - } - int written = snprintf(out, out_sz, "%s/%s", dir, name); - if (written <= 0 || (size_t)written >= out_sz) { - out[0] = '\0'; - cbm_log_warn("envscan.path_too_long", "dir", dir, "name", name); - return false; - } - return true; -} - -/* Root-relative suffix of a path built beneath root_path. - * - * The prefix comparison is not a formality: it succeeding is precisely what - * proves `path` holds at least root_len bytes, which is what makes - * `path + root_len` a valid pointer. Never compute this suffix from an - * unverified strlen. Returns NULL when `path` is not under the root. */ -static const char *envscan_relative_suffix(const char *path, const char *root_path, - size_t root_len) { - if (strncmp(path, root_path, root_len) != 0) { - return NULL; - } - const char *rel = path + root_len; - while (*rel == '/') { - rel++; - } - return rel; -} - -/* Stat WITHOUT following links, so a repository-controlled symlink cannot lead - * the walk outside the project. The previous stat() resolved the link, reported - * the target's S_ISDIR, and the walk happily descended wherever it pointed. - * Returns 0 on success, CBM_NOT_FOUND to skip the entry. - * Mirrors pass_pkgmap.c's pkgmap_safe_stat and discover.c's safe_stat. */ -static int envscan_safe_stat(const char *abs_path, struct stat *st) { -#ifdef _WIN32 - /* Windows has no lstat; a junction or directory symlink surfaces as a - * reparse point, which is the same escape hatch, so screen the attribute. - * The wide stat also keeps non-ASCII paths off the ANSI CRT. */ - wchar_t *wide = cbm_path_to_wide(abs_path); - if (!wide) { - return CBM_NOT_FOUND; - } - DWORD attributes = GetFileAttributesW(wide); - if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - free(wide); - return CBM_NOT_FOUND; - } - struct _stat64 wst; - int ret = _wstat64(wide, &wst); - free(wide); - if (ret != 0) { - return CBM_NOT_FOUND; - } - st->st_mode = wst.st_mode; - st->st_size = wst.st_size; - st->st_mtime = wst.st_mtime; - return 0; -#else - if (lstat(abs_path, st) != 0) { - return CBM_NOT_FOUND; - } - if (S_ISLNK(st->st_mode)) { - return CBM_NOT_FOUND; - } - return 0; -#endif -} - -/* True when `candidate` still resolves to somewhere under `canonical_root`. - * - * Belt to envscan_safe_stat's braces. The link screen refuses the symlink - * itself; this catches escapes a mode check cannot see — a resolved path that - * leaves the root for any other reason. `canonical_root` is resolved once per - * scan because cbm_canonical_path costs a syscall per component. */ -static bool envscan_within_root(const char *candidate, const char *canonical_root) { - char resolved[CBM_SZ_4K]; - if (!cbm_canonical_path(candidate, resolved, sizeof(resolved))) { - return false; - } -#ifdef _WIN32 - cbm_normalize_path_sep(resolved); -#endif - size_t root_len = strlen(canonical_root); - if (root_len == 0 || envscan_path_ncmp(resolved, canonical_root, root_len) != 0) { - return false; - } - /* Demand a real component boundary, so root "/repo" does not admit - * "/repo-elsewhere". */ - return canonical_root[root_len - 1] == '/' || resolved[root_len] == '\0' || - resolved[root_len] == '/'; -} - -/* ── Public API ────────────────────────────────────────────────── */ - -/* Open a candidate config file without following a symlink where the platform - * allows it. Mode "r" is preserved from the previous cbm_fopen call so the - * per-platform newline handling the line loop already copes with is unchanged. */ -static FILE *envscan_open_file(const char *full_path) { -#ifdef _WIN32 - /* No O_NOFOLLOW equivalent here; reparse points were screened by - * envscan_safe_stat. cbm_fopen is required for UTF-8 → _wfopen mapping. */ - return cbm_fopen(full_path, "r"); -#else - /* O_NOFOLLOW closes the window between the lstat above and this open: if the - * entry turned into a symlink in between, the open fails rather than reading - * through it. A raw open is correct on POSIX — paths are bytes there, and the - * cbm_fopen rule exists for Windows' wide-API mapping. */ - int flags = O_RDONLY | O_CLOEXEC; -#ifdef O_NOFOLLOW - flags |= O_NOFOLLOW; -#endif - int descriptor = open(full_path, flags); - if (descriptor < 0) { - return NULL; - } - FILE *stream = fdopen(descriptor, "r"); - if (!stream) { - (void)close(descriptor); - } - return stream; -#endif -} - -/* Scan a single file for env URL bindings. Returns number of bindings added. */ -static int scan_env_file(const char *full_path, const char *rel, file_type_t ft, - cbm_env_binding_t *out, int max_out) { - FILE *f = envscan_open_file(full_path); - if (!f) { - return 0; - } - - struct stat fst; - if (fstat(fileno(f), &fst) != 0 || fst.st_size > (long)CBM_SZ_1K * CBM_SZ_1K) { - (void)fclose(f); - return 0; - } - - int count = 0; - char line[CBM_SZ_2K]; - while (fgets(line, sizeof(line), f) && count < max_out) { - size_t ll = strlen(line); - while (ll > 0 && (line[ll - SKIP_ONE] == '\n' || line[ll - SKIP_ONE] == '\r')) { - line[--ll] = '\0'; - } - - char key[CBM_SZ_128]; - char value[CBM_SZ_512]; - if (!scan_line(line, ft, key, sizeof(key), value, sizeof(value))) { - continue; - } - if (strncmp(value, "http://", SLEN("http://")) != 0 && - strncmp(value, "https://", SLEN("https://")) != 0) { - continue; - } - if (cbm_is_secret_binding(key, value) || cbm_is_secret_value(value)) { - continue; - } - - strncpy(out[count].key, key, sizeof(out[count].key) - 1); - out[count].key[sizeof(out[count].key) - SKIP_ONE] = '\0'; - strncpy(out[count].value, value, sizeof(out[count].value) - 1); - out[count].value[sizeof(out[count].value) - SKIP_ONE] = '\0'; - strncpy(out[count].file_path, rel, sizeof(out[count].file_path) - 1); - out[count].file_path[sizeof(out[count].file_path) - SKIP_ONE] = '\0'; - count++; - } - (void)fclose(f); - return count; -} - -/* Process a single directory entry for env scanning. Returns bindings added. - * `root_len` is strlen of the root the walk started from, verified against - * path_stack[0] by the caller; `canonical_root` is that root resolved once. */ -static int process_env_entry(cbm_dirent_t *ent, const char *dir_path, const char *root_path, - size_t root_len, const char *canonical_root, cbm_env_binding_t *out, - int max_out, char path_stack[][CBM_SZ_512], int *stack_top, - char **excluded_dirs, int excluded_count) { - char full_path[CBM_SZ_512]; - if (!envscan_join_path(full_path, sizeof(full_path), dir_path, ent->name)) { - return 0; - } - const char *rel = envscan_relative_suffix(full_path, root_path, root_len); - if (!rel) { - return 0; - } - - struct stat st; - if (envscan_safe_stat(full_path, &st) != 0) { - return 0; - } - if (S_ISDIR(st.st_mode)) { - if (is_ignored_dir(ent->name) || - cbm_pipeline_relpath_is_excluded(rel, excluded_dirs, excluded_count) || - *stack_top >= CBM_SZ_256) { - return 0; - } - if (!envscan_within_root(full_path, canonical_root)) { - cbm_log_warn("envscan.dir_outside_root", "path", full_path, "root", canonical_root); - return 0; - } - /* Checked copy. The stack rows are the same width as full_path, so this - * cannot truncate today — but check anyway rather than lean on that - * coincidence: a truncated row is exactly what reintroduced the - * out-of-bounds suffix arithmetic. */ - int written = snprintf(path_stack[*stack_top], sizeof(path_stack[0]), "%s", full_path); - if (written <= 0 || (size_t)written >= sizeof(path_stack[0])) { - cbm_log_warn("envscan.path_too_long", "dir", dir_path, "name", ent->name); - return 0; - } - (*stack_top)++; - return 0; - } - /* Only regular files are config files. Without this, the lstat above still - * lets a FIFO through, and opening one blocks the pass until a writer - * appears. */ - if (!S_ISREG(st.st_mode)) { - return 0; - } - if (is_secret_file(ent->name)) { - return 0; - } - file_type_t ft = detect_file_type(ent->name); - if (ft == FT_UNKNOWN) { - return 0; - } - return scan_env_file(full_path, rel, ft, out, max_out); -} - -int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t *out, int max_out, - char **excluded_dirs, int excluded_count) { - if (!root_path || !out || max_out <= 0) { - return 0; - } - compile_patterns(); - - int count = 0; - char path_stack[CBM_SZ_256][CBM_SZ_512]; - int stack_top = SKIP_ONE; - /* Reject an over-long root instead of truncating it in. The old strncpy left - * path_stack[0] SHORTER than root_path, and every suffix pointer derived from - * strlen(root_path) was then past the end of its buffer. Establishing - * root_len from the copy that actually fit is what removes that class. */ - int written = snprintf(path_stack[0], sizeof(path_stack[0]), "%s", root_path); - if (written <= 0 || (size_t)written >= sizeof(path_stack[0])) { - cbm_log_warn("envscan.root_too_long", "root", root_path); - return 0; - } - size_t root_len = (size_t)written; - - /* Resolved once per scan: every directory the walk wants to descend is - * checked against this, and resolving costs a syscall per component. */ - char canonical_root[CBM_SZ_4K]; - if (!cbm_canonical_path(root_path, canonical_root, sizeof(canonical_root))) { - /* Unresolvable root — nothing to walk. Previously opendir simply failed - * on the first iteration, so the result is the same: zero bindings. */ - return 0; - } -#ifdef _WIN32 - cbm_normalize_path_sep(canonical_root); -#endif - - while (stack_top > 0 && count < max_out) { - stack_top--; - char dir_path[CBM_SZ_512]; - int copied = snprintf(dir_path, sizeof(dir_path), "%s", path_stack[stack_top]); - if (copied <= 0 || (size_t)copied >= sizeof(dir_path)) { - continue; - } - - cbm_dir_t *d = cbm_opendir(dir_path); - if (!d) { - continue; - } - cbm_dirent_t *ent; - while ((ent = cbm_readdir(d)) && count < max_out) { - count += process_env_entry(ent, dir_path, root_path, root_len, canonical_root, - out + count, max_out - count, path_stack, &stack_top, - excluded_dirs, excluded_count); - } - cbm_closedir(d); - } - return count; -} - -int cbm_scan_project_env_urls(const char *root_path, cbm_env_binding_t *out, int max_out) { - return cbm_scan_project_env_urls_excluded(root_path, out, max_out, NULL, 0); -} diff --git a/src/pipeline/pass_infrascan.c b/src/pipeline/pass_infrascan.c index a9b2418fd..4b51149e3 100644 --- a/src/pipeline/pass_infrascan.c +++ b/src/pipeline/pass_infrascan.c @@ -1,35 +1,24 @@ /* - * pass_infrascan.c — Infrastructure file detection and parsing helpers. + * pass_infrascan.c — Infrastructure file helpers. * - * Pure helper functions for detecting infrastructure file types, - * secret filtering, and parsing Dockerfiles, .env files, shell scripts, - * and Terraform files. + * Kustomization / K8s manifest detection, secret filtering, the infra QN + * helper, and the Helm Chart.yaml parser. */ #include "foundation/constants.h" enum { - IS_ENV_EXT = 4, /* strlen(".env") */ IS_CONTENT_LIMIT = 4096, - IS_MAX_STAGES = 16, - IS_MAX_PORTS = 16, - IS_MAX_SOURCES = 16, - IS_TOKEN_MIN_1 = 16, /* min alnum for AWS key */ - IS_TOKEN_MIN_2 = 20, /* min alnum for GH classic token */ - IS_FROM_SKIP = 4, /* strlen("FROM") */ - IS_EXPOSE_SKIP = 6, /* strlen("EXPOSE") */ - IS_RUN_SKIP = 3, /* strlen("RUN") / strlen("ENV") / strlen("ARG") */ - IS_CMD_SKIP = 4, /* strlen("CMD ") */ - IS_USER_SKIP = 5, /* strlen("USER ") */ - IS_EXPORT_SKIP = 7, /* strlen("export ") */ - IS_WORKDIR_SKIP = 8, /* strlen("WORKDIR ") */ - IS_ENTRYPOINT_OFF = 10, /* strlen("ENTRYPOINT") */ - IS_AS_SKIP = 3, /* strlen("AS ") */ - IS_SOURCE_DOT = 2, /* strlen(". ") */ - IS_QUOTE_TRIM = 2, /* strip surrounding quotes */ + IS_TOKEN_MIN_1 = 16, /* min alnum for AWS key */ + IS_TOKEN_MIN_2 = 20, /* min alnum for GH classic token */ + SECRET_AKIA_LEN = 4, /* strlen("AKIA") */ + SECRET_SK_LEN = 3, /* strlen("sk-") */ + SECRET_GHP_LEN = 4, /* strlen("ghp_") */ + SECRET_GLPAT_LEN = 6, /* strlen("glpat-") */ + SECRET_XOX_LEN = 3, /* strlen("xox"); the token-type letter follows */ + SECRET_XOX_DASH = 4, /* offset of the '-' in "xoxb-" */ + SECRET_XOX_BODY = 5, /* strlen("xoxb-") */ }; -#define IS_NOT_FOUND (-1) -#define SLEN(s) (sizeof(s) - 1) #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" @@ -38,14 +27,6 @@ enum { #include #include -/* ── Constants ───────────────────────────────────────────────────── */ - -/* String length constants for keyword matching */ -#define LEN_DOCKERFILE 11 /* strlen("dockerfile.") == strlen(".dockerfile") */ -#define LEN_DOCKER_COMPOSE 14 /* strlen("docker-compose") */ -#define LEN_HEALTHCHECK 11 /* strlen("HEALTHCHECK") */ -#define LEN_DOCKER_COMPOSE_SKIP 15 /* strlen("docker-compose") + space */ - /* Minimum alnum chars after prefix for secret detection */ #define GITHUB_PAT_MIN_ALNUM 36 /* ghp_ + 36 alnum chars */ @@ -95,124 +76,8 @@ static int count_alnum_dash(const char *s) { return n; } -/* Skip whitespace, return pointer past it */ -static const char *skip_ws(const char *p) { - while (*p == ' ' || *p == '\t') { - p++; - } - return p; -} - -/* Extract a word ([a-zA-Z0-9_]+) into buf. Returns length. */ -static int extract_word(const char *p, char *buf, size_t buf_sz) { - int n = 0; - while ((isalnum((unsigned char)p[n]) || p[n] == '_') && (size_t)n < buf_sz - SKIP_ONE) { - buf[n] = p[n]; - n++; - } - buf[n] = '\0'; - return n; -} - -/* Extract a non-space token into buf. Returns length. */ -static int extract_token(const char *p, char *buf, size_t buf_sz) { - int n = 0; - while (p[n] && p[n] != ' ' && p[n] != '\t' && (size_t)n < buf_sz - SKIP_ONE) { - buf[n] = p[n]; - n++; - } - buf[n] = '\0'; - return n; -} - -/* Trim trailing whitespace in-place */ -static void rtrim(char *s) { - size_t len = strlen(s); - while (len > 0 && (s[len - SKIP_ONE] == ' ' || s[len - SKIP_ONE] == '\t' || - s[len - SKIP_ONE] == '\r' || s[len - SKIP_ONE] == '\n')) { - s[--len] = '\0'; - } -} - /* ── File identification ────────────────────────────────────────── */ -bool cbm_is_dockerfile(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strcmp(lower, "dockerfile") == 0) { - return true; - } - if (strncmp(lower, "dockerfile.", LEN_DOCKERFILE) == 0) { - return true; - } - size_t len = strlen(lower); - if (len > LEN_DOCKERFILE && strcmp(lower + len - LEN_DOCKERFILE, ".dockerfile") == 0) { - return true; - } - return false; -} - -bool cbm_is_compose_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - bool prefix_match = (strncmp(lower, "docker-compose", LEN_DOCKER_COMPOSE) == 0) || - (strcmp(lower, "compose.yml") == 0) || (strcmp(lower, "compose.yaml") == 0); - if (!prefix_match) { - return false; - } - - const char *ext = strrchr(lower, '.'); - if (!ext) { - return false; - } - return (strcmp(ext, ".yml") == 0 || strcmp(ext, ".yaml") == 0); -} - -bool cbm_is_cloudbuild_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strncmp(lower, "cloudbuild", SLEN("cloudbuild")) != 0) { - return false; - } - const char *ext = strrchr(lower, '.'); - if (!ext) { - return false; - } - return (strcmp(ext, ".yml") == 0 || strcmp(ext, ".yaml") == 0); -} - -bool cbm_is_env_file(const char *name) { - if (!name) { - return false; - } - char lower[CBM_SZ_256]; - to_lower(name, lower, sizeof(lower)); - - if (strcmp(lower, ".env") == 0) { - return true; - } - if (strncmp(lower, ".env.", SLEN(".env.")) == 0) { - return true; - } - size_t len = strlen(lower); - if (len > IS_ENV_EXT && strcmp(lower + len - IS_ENV_EXT, ".env") == 0) { - return true; - } - return false; -} - bool cbm_is_kustomize_file(const char *name) { if (!name) { return false; @@ -237,14 +102,6 @@ bool cbm_is_k8s_manifest(const char *name, const char *content) { return ci_strstr(buf, "apiVersion:") != NULL; } -bool cbm_is_shell_script(const char *name, const char *ext) { - (void)name; - if (!ext) { - return false; - } - return (strcmp(ext, ".sh") == 0 || strcmp(ext, ".bash") == 0 || strcmp(ext, ".zsh") == 0); -} - /* ── Secret detection ───────────────────────────────────────────── */ static bool key_is_secret(const char *key) { @@ -276,35 +133,35 @@ bool cbm_is_secret_value(const char *value) { /* AKIA + 16 alnum (AWS key) */ p = ci_strstr(value, "AKIA"); - if (p && count_alnum(p + IS_CMD_SKIP) >= IS_TOKEN_MIN_1) { + if (p && count_alnum(p + SECRET_AKIA_LEN) >= IS_TOKEN_MIN_1) { return true; } /* sk- + 20 alnum (API key) */ p = ci_strstr(value, "sk-"); - if (p && count_alnum(p + IS_RUN_SKIP) >= IS_TOKEN_MIN_2) { + if (p && count_alnum(p + SECRET_SK_LEN) >= IS_TOKEN_MIN_2) { return true; } /* ghp_ + 36 alnum (GitHub PAT) */ p = ci_strstr(value, "ghp_"); - if (p && count_alnum(p + IS_CMD_SKIP) >= GITHUB_PAT_MIN_ALNUM) { + if (p && count_alnum(p + SECRET_GHP_LEN) >= GITHUB_PAT_MIN_ALNUM) { return true; } /* glpat- + 20 alnum/dash (GitLab PAT) */ p = ci_strstr(value, "glpat-"); - if (p && count_alnum_dash(p + IS_EXPOSE_SKIP) >= IS_TOKEN_MIN_2) { + if (p && count_alnum_dash(p + SECRET_GLPAT_LEN) >= IS_TOKEN_MIN_2) { return true; } /* xox[bps]- (Slack token) */ p = ci_strstr(value, "xox"); - if (p && p[IS_RUN_SKIP] != '\0' && - (tolower((unsigned char)p[IS_RUN_SKIP]) == 'b' || - tolower((unsigned char)p[IS_RUN_SKIP]) == 'p' || - tolower((unsigned char)p[IS_RUN_SKIP]) == 's') && - p[IS_CMD_SKIP] == '-' && count_alnum_dash(p + IS_USER_SKIP) >= SKIP_ONE) { + if (p && p[SECRET_XOX_LEN] != '\0' && + (tolower((unsigned char)p[SECRET_XOX_LEN]) == 'b' || + tolower((unsigned char)p[SECRET_XOX_LEN]) == 'p' || + tolower((unsigned char)p[SECRET_XOX_LEN]) == 's') && + p[SECRET_XOX_DASH] == '-' && count_alnum_dash(p + SECRET_XOX_BODY) >= SKIP_ONE) { return true; } @@ -321,892 +178,6 @@ bool cbm_is_secret_binding(const char *key, const char *value) { return false; } -/* ── Clean JSON brackets ────────────────────────────────────────── */ - -/* Strip JSON array brackets, quotes, and normalize whitespace. */ -static void clean_json_array_inner(const char *s, size_t len, char *out, size_t out_sz) { - size_t pos = 0; - bool in_space = false; - for (size_t i = SKIP_ONE; i < len - SKIP_ONE && pos < out_sz - SKIP_ONE; i++) { - char c = s[i]; - if (c == '"') { - continue; - } - if (c == ',') { - c = ' '; - } - if (c == ' ' || c == '\t') { - if (!in_space && pos > 0) { - out[pos++] = ' '; - in_space = true; - } - } else { - out[pos++] = c; - in_space = false; - } - } - while (pos > 0 && out[pos - SKIP_ONE] == ' ') { - pos--; - } - out[pos] = '\0'; -} - -void cbm_clean_json_brackets(const char *s, char *out, size_t out_sz) { - if (!s || !out || out_sz == 0) { - return; - } - - size_t len = strlen(s); - if (len >= PAIR_LEN && s[0] == '[' && s[len - SKIP_ONE] == ']') { - clean_json_array_inner(s, len, out, out_sz); - } else { - snprintf(out, out_sz, "%s", s); - } -} - -/* ── Dockerfile parser ──────────────────────────────────────────── */ - -static void df_parse_from(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "FROM", SLEN("FROM")) != 0) { - return; - } - const char *p = line + IS_FROM_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - /* Extract image */ - if (r->stage_count >= IS_MAX_STAGES) { - return; - } - int idx = r->stage_count; - extract_token(p, r->stage_images[idx], sizeof(r->stage_images[idx])); - - /* Advance past image */ - while (*p && *p != ' ' && *p != '\t') { - p++; - } - p = skip_ws(p); - - /* Check for AS */ - r->stage_names[idx][0] = '\0'; - if (strncasecmp(p, "AS", SLEN("AS")) == 0 && (p[PAIR_LEN] == ' ' || p[PAIR_LEN] == '\t')) { - p = skip_ws(p + IS_AS_SKIP); - extract_word(p, r->stage_names[idx], sizeof(r->stage_names[idx])); - } - r->stage_count++; -} - -static void df_parse_expose(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "EXPOSE", SLEN("EXPOSE")) != 0) { - return; - } - const char *p = line + IS_EXPOSE_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - /* Parse space-separated ports */ - while (*p && r->port_count < IS_MAX_PORTS) { - char port[CBM_SZ_32]; - int n = extract_token(p, port, sizeof(port)); - if (n == 0) { - break; - } - - /* Strip protocol suffix (e.g. 8080/tcp) */ - char *slash = strchr(port, '/'); - if (slash) { - *slash = '\0'; - } - - snprintf(r->exposed_ports[r->port_count], sizeof(r->exposed_ports[0]), "%s", port); - r->port_count++; - - p += n; - p = skip_ws(p); - } -} - -static void df_parse_env(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "ENV", SLEN("ENV")) != 0) { - return; - } - const char *p = line + IS_RUN_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - if (r->env_count >= CBM_SZ_64) { - return; - } - - /* Extract key */ - char key[CBM_SZ_128]; - int klen = extract_word(p, key, sizeof(key)); - if (klen == 0) { - return; - } - p += klen; - - /* Separator: = or space */ - if (*p != '=' && *p != ' ' && *p != '\t') { - return; - } - p++; - /* Skip additional whitespace after separator */ - while (*p == ' ' || *p == '\t') { - p++; - } - - /* Value is rest of line (trimmed) */ - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", p); - rtrim(value); - - /* Filter secrets */ - if (cbm_is_secret_binding(key, value)) { - return; - } - - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void df_parse_arg(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "ARG", SLEN("ARG")) != 0) { - return; - } - const char *p = line + IS_RUN_SKIP; - if (*p != ' ' && *p != '\t') { - return; - } - p = skip_ws(p); - - if (r->build_arg_count >= CBM_SZ_32) { - return; - } - extract_word(p, r->build_args[r->build_arg_count], sizeof(r->build_args[0])); - if (r->build_args[r->build_arg_count][0]) { - r->build_arg_count++; - } -} - -/* Parse a Dockerfile CMD/ENTRYPOINT value: strip brackets, trim. */ -static void df_parse_json_cmd(const char *line, int prefix_len, char *out, size_t out_sz) { - const char *p = skip_ws(line + prefix_len); - char raw[CBM_SZ_512]; - snprintf(raw, sizeof(raw), "%s", p); - rtrim(raw); - cbm_clean_json_brackets(raw, out, out_sz); -} - -static void df_parse_directives(const char *line, cbm_dockerfile_result_t *r) { - if (strncasecmp(line, "WORKDIR", SLEN("WORKDIR")) == 0 && - (line[IS_EXPORT_SKIP] == ' ' || line[IS_EXPORT_SKIP] == '\t')) { - const char *p = skip_ws(line + IS_WORKDIR_SKIP); - snprintf(r->workdir, sizeof(r->workdir), "%s", p); - rtrim(r->workdir); - return; - } - if (strncasecmp(line, "CMD", SLEN("CMD")) == 0 && - (line[IS_RUN_SKIP] == ' ' || line[IS_RUN_SKIP] == '\t')) { - df_parse_json_cmd(line, IS_CMD_SKIP, r->cmd, sizeof(r->cmd)); - return; - } - if (strncasecmp(line, "ENTRYPOINT", SLEN("ENTRYPOINT")) == 0 && - (line[IS_ENTRYPOINT_OFF] == ' ' || line[IS_ENTRYPOINT_OFF] == '\t')) { - df_parse_json_cmd(line, LEN_HEALTHCHECK, r->entrypoint, sizeof(r->entrypoint)); - return; - } - if (strncasecmp(line, "USER", SLEN("USER")) == 0 && - (line[IS_CMD_SKIP] == ' ' || line[IS_CMD_SKIP] == '\t')) { - const char *p = skip_ws(line + IS_USER_SKIP); - extract_word(p, r->user, sizeof(r->user)); - return; - } - if (strncasecmp(line, "HEALTHCHECK", LEN_HEALTHCHECK) == 0 && - (line[LEN_HEALTHCHECK] == ' ' || line[LEN_HEALTHCHECK] == '\t')) { - const char *cmd_pos = ci_strstr(line + LEN_HEALTHCHECK, "CMD"); - if (cmd_pos) { - cmd_pos = skip_ws(cmd_pos + IS_RUN_SKIP); - snprintf(r->healthcheck, sizeof(r->healthcheck), "%s", cmd_pos); - rtrim(r->healthcheck); - } - } -} - -int cbm_parse_dockerfile_source(const char *source, cbm_dockerfile_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - /* Process line by line */ - const char *p = source; - char line[CBM_SZ_4K]; - - while (*p) { - /* Extract one line */ - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - /* Trim */ - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - /* Skip empty lines and comments */ - if (*trimmed == '\0' || *trimmed == '#') { - continue; - } - - df_parse_from(trimmed, out); - df_parse_expose(trimmed, out); - df_parse_env(trimmed, out); - df_parse_arg(trimmed, out); - df_parse_directives(trimmed, out); - } - - /* No stages = empty/invalid Dockerfile */ - if (out->stage_count == 0) { - return CBM_NOT_FOUND; - } - - /* base_image = last stage's image (use intermediate copy to avoid restrict overlap) */ - { - char tmp[sizeof(out->base_image)]; - snprintf(tmp, sizeof(tmp), "%s", out->stage_images[out->stage_count - SKIP_ONE]); - memcpy(out->base_image, tmp, sizeof(out->base_image)); - } - - return 0; -} - -/* ── Dotenv parser ──────────────────────────────────────────────── */ - -/* Strip matching surrounding quotes from a value string in-place. */ -static void strip_surrounding_quotes(char *value) { - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } -} - -/* Parse a single dotenv line: KEY=VALUE. Returns 1 if env var added. */ -static int parse_dotenv_line(const char *trimmed, cbm_dotenv_result_t *out) { - char key[CBM_SZ_128]; - int klen = 0; - if (isalpha((unsigned char)trimmed[0]) || trimmed[0] == '_') { - klen = extract_word(trimmed, key, sizeof(key)); - } - if (klen == 0 || trimmed[klen] != '=') { - return 0; - } - - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", trimmed + klen + SKIP_ONE); - rtrim(value); - strip_surrounding_quotes(value); - - if (cbm_is_secret_binding(key, value) || out->env_count >= CBM_SZ_64) { - return 0; - } - snprintf(out->env_vars[out->env_count].key, sizeof(out->env_vars[0].key), "%s", key); - snprintf(out->env_vars[out->env_count].value, sizeof(out->env_vars[0].value), "%s", value); - out->env_count++; - return SKIP_ONE; -} - -int cbm_parse_dotenv_source(const char *source, cbm_dotenv_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - - while (*p) { - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - if (*trimmed == '\0' || *trimmed == '#') { - continue; - } - - parse_dotenv_line(trimmed, out); - } - - return (out->env_count > 0) ? 0 : IS_NOT_FOUND; -} - -/* ── Shell script parser ────────────────────────────────────────── */ - -static void shell_parse_export(const char *line, cbm_shell_result_t *r) { - /* export VAR=VALUE */ - if (strncmp(line, "export", SLEN("export")) != 0 || - (line[IS_EXPOSE_SKIP] != ' ' && line[IS_EXPOSE_SKIP] != '\t')) { - return; - } - const char *p = skip_ws(line + IS_EXPORT_SKIP); - - char key[CBM_SZ_128]; - int klen = extract_word(p, key, sizeof(key)); - if (klen == 0 || p[klen] != '=') { - return; - } - - const char *val_start = p + klen + SKIP_ONE; - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", val_start); - rtrim(value); - - /* Strip surrounding quotes */ - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } - - if (cbm_is_secret_binding(key, value)) { - return; - } - if (r->env_count >= CBM_SZ_64) { - return; - } - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void shell_parse_plain_var(const char *line, cbm_shell_result_t *r) { - /* VAR=VALUE (only if no spaces in line — avoids matching commands) */ - if (strchr(line, ' ') || strchr(line, '\t')) { - return; - } - - char key[CBM_SZ_128]; - int klen = extract_word(line, key, sizeof(key)); - if (klen == 0 || line[klen] != '=') { - return; - } - - const char *val_start = line + klen + SKIP_ONE; - char value[CBM_SZ_512]; - snprintf(value, sizeof(value), "%s", val_start); - rtrim(value); - - /* Strip surrounding quotes */ - size_t vlen = strlen(value); - if (vlen >= CBM_QUOTE_PAIR && ((value[0] == '"' && value[vlen - SKIP_ONE] == '"') || - (value[0] == '\'' && value[vlen - SKIP_ONE] == '\''))) { - memmove(value, value + CBM_QUOTE_OFFSET, vlen - CBM_QUOTE_PAIR); - value[vlen - IS_QUOTE_TRIM] = '\0'; - } - - if (cbm_is_secret_binding(key, value)) { - return; - } - if (r->env_count >= CBM_SZ_64) { - return; - } - snprintf(r->env_vars[r->env_count].key, sizeof(r->env_vars[0].key), "%s", key); - snprintf(r->env_vars[r->env_count].value, sizeof(r->env_vars[0].value), "%s", value); - r->env_count++; -} - -static void shell_parse_source(const char *line, cbm_shell_result_t *r) { - /* source or . */ - const char *p = NULL; - if (strncmp(line, "source", SLEN("source")) == 0 && - (line[IS_EXPOSE_SKIP] == ' ' || line[IS_EXPOSE_SKIP] == '\t')) { - p = skip_ws(line + IS_EXPORT_SKIP); - } else if (line[0] == '.' && (line[SKIP_ONE] == ' ' || line[SKIP_ONE] == '\t')) { - p = skip_ws(line + IS_SOURCE_DOT); - } - if (!p) { - return; - } - - if (r->source_count >= IS_MAX_SOURCES) { - return; - } - - /* Strip surrounding quotes */ - char path[CBM_SZ_256]; - snprintf(path, sizeof(path), "%s", p); - rtrim(path); - size_t plen = strlen(path); - if (plen >= CBM_QUOTE_PAIR && ((path[0] == '"' && path[plen - SKIP_ONE] == '"') || - (path[0] == '\'' && path[plen - SKIP_ONE] == '\''))) { - memmove(path, path + CBM_QUOTE_OFFSET, plen - CBM_QUOTE_PAIR); - path[plen - IS_QUOTE_TRIM] = '\0'; - } - - /* Extract just the path token (no trailing args) */ - char *space = strchr(path, ' '); - if (space) { - *space = '\0'; - } - - snprintf(r->sources[r->source_count], sizeof(r->sources[0]), "%s", path); - r->source_count++; -} - -static void shell_parse_docker(const char *line, cbm_shell_result_t *r) { - /* docker or docker-compose */ - const char *p = NULL; - const char *tool = NULL; - - if (strncmp(line, "docker-compose", LEN_DOCKER_COMPOSE) == 0 && - (line[LEN_DOCKER_COMPOSE] == ' ' || line[LEN_DOCKER_COMPOSE] == '\t')) { - tool = "docker-compose"; - p = skip_ws(line + LEN_DOCKER_COMPOSE_SKIP); - } else if (strncmp(line, "docker", SLEN("docker")) == 0 && - (line[IS_EXPOSE_SKIP] == ' ' || line[IS_EXPOSE_SKIP] == '\t')) { - tool = "docker"; - p = skip_ws(line + IS_EXPORT_SKIP); - } - if (!tool || !p || !*p) { - return; - } - - if (r->docker_cmd_count >= IS_MAX_STAGES) { - return; - } - - char subcmd[CBM_SZ_64]; - extract_word(p, subcmd, sizeof(subcmd)); - if (subcmd[0]) { - snprintf(r->docker_cmds[r->docker_cmd_count], sizeof(r->docker_cmds[0]), "%s %s", tool, - subcmd); - r->docker_cmd_count++; - } -} - -/* Check for shebang on first non-empty line. Returns true if line was consumed. */ -static bool shell_try_shebang(const char *trimmed, cbm_shell_result_t *out, bool *shebang_checked) { - if (*shebang_checked) { - return false; - } - if (trimmed[0] == '#' && trimmed[SKIP_ONE] == '!') { - const char *sb = trimmed + PAIR_LEN; - while (*sb == ' ') { - sb++; - } - snprintf(out->shebang, sizeof(out->shebang), "%s", sb); - rtrim(out->shebang); - *shebang_checked = true; - return true; - } - *shebang_checked = true; - return false; -} - -/* Dispatch a single trimmed shell line to the appropriate parser. */ -static void shell_dispatch_line(const char *trimmed, cbm_shell_result_t *out) { - if (strncmp(trimmed, "export", SLEN("export")) == 0) { - shell_parse_export(trimmed, out); - return; - } - if (strncmp(trimmed, "source", SLEN("source")) == 0 || - (trimmed[0] == '.' && trimmed[SKIP_ONE] == ' ')) { - shell_parse_source(trimmed, out); - return; - } - if (strncmp(trimmed, "docker", SLEN("docker")) == 0) { - shell_parse_docker(trimmed, out); - } - shell_parse_plain_var(trimmed, out); -} - -int cbm_parse_shell_source(const char *source, cbm_shell_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - bool shebang_checked = false; - - while (*p) { - const char *eol = strchr(p, '\n'); - size_t line_len = eol ? (size_t)(eol - p) : strlen(p); - if (line_len >= sizeof(line)) { - line_len = sizeof(line) - SKIP_ONE; - } - memcpy(line, p, line_len); - line[line_len] = '\0'; - p = eol ? eol + SKIP_ONE : p + line_len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - - if (*trimmed == '\0') { - continue; - } - - if (shell_try_shebang(trimmed, out, &shebang_checked)) { - continue; - } - - if (*trimmed == '#') { - continue; - } - - shell_dispatch_line(trimmed, out); - } - - bool has_content = out->shebang[0] != '\0' || out->env_count > 0 || out->source_count > 0 || - out->docker_cmd_count > 0; - return has_content ? 0 : IS_NOT_FOUND; -} - -/* ── Terraform parser ───────────────────────────────────────────── */ - -typedef enum { - TF_BLOCK_NONE = 0, - TF_BLOCK_VARIABLE, - TF_BLOCK_MODULE, - TF_BLOCK_TERRAFORM -} tf_block_kind_t; - -/* Extract a double-quoted string value after = sign. - * Handles: key = "value" and key = value */ -static void tf_extract_quoted(const char *line, const char *prefix, char *out, size_t out_sz) { - const char *p = skip_ws(line); - size_t plen = strlen(prefix); - if (strncmp(p, prefix, plen) != 0) { - return; - } - p = skip_ws(p + plen); - if (*p != '=') { - return; - } - p = skip_ws(p + SKIP_ONE); - - /* Strip optional quotes */ - if (*p == '"') { - p++; - const char *end = strchr(p, '"'); - size_t vlen = end ? (size_t)(end - p) : strlen(p); - if (vlen >= out_sz) { - vlen = out_sz - SKIP_ONE; - } - memcpy(out, p, vlen); - out[vlen] = '\0'; - } else { - extract_token(p, out, out_sz); - } -} - -/* Count occurrences of a character in a string */ -static int count_char(const char *s, char c) { - int n = 0; - for (; *s; s++) { - if (*s == c) { - n++; - } - } - return n; -} - -/* Extract double-quoted identifier after keyword. - * Matches: keyword "identifier" → sets out to "identifier" - * Returns number of chars consumed from p, or 0 if no match. */ -static int tf_extract_ident(const char *p, const char *keyword, char *out, size_t out_sz) { - size_t klen = strlen(keyword); - if (strncmp(p, keyword, klen) != 0) { - return 0; - } - const char *after = skip_ws(p + klen); - if (*after != '"') { - return 0; - } - after++; - const char *end = strchr(after, '"'); - if (!end) { - return 0; - } - size_t ilen = (size_t)(end - after); - if (ilen >= out_sz) { - ilen = out_sz - SKIP_ONE; - } - memcpy(out, after, ilen); - out[ilen] = '\0'; - return (int)(end - p + SKIP_ONE); -} - -/* Parse a second quoted string after the first ident (for resource/data blocks). */ -static void tf_extract_second_quoted(const char *trimmed, int consumed, char *out, size_t outsz) { - const char *rest = skip_ws(trimmed + consumed); - out[0] = '\0'; - if (*rest == '"') { - rest++; - const char *end = strchr(rest, '"'); - if (end) { - size_t len = (size_t)(end - rest); - if (len >= outsz) { - len = outsz - SKIP_ONE; - } - memcpy(out, rest, len); - out[len] = '\0'; - } - } -} - -/* Try to parse a simple "keyword ident" block (variable, output, provider). */ -static bool tf_try_simple_block(const char *trimmed, const char *keyword, char *names, int name_sz, - int *count, int max_count) { - char ident[CBM_SZ_128]; - int consumed = tf_extract_ident(trimmed, keyword, ident, sizeof(ident)); - if (consumed <= 0 || *count >= max_count) { - return false; - } - snprintf(names + ((size_t)(*count) * (size_t)name_sz), name_sz, "%s", ident); - (*count)++; - return true; -} - -/* Try to match terraform/locals keyword blocks. */ -static bool tf_try_keyword_block(const char *trimmed, tf_block_kind_t *cur_block, - cbm_terraform_result_t *out) { - if (strncmp(trimmed, "terraform", SLEN("terraform")) == 0 && strstr(trimmed, "{")) { - *cur_block = TF_BLOCK_TERRAFORM; - return true; - } - if (strncmp(trimmed, "locals", SLEN("locals")) == 0 && strstr(trimmed, "{")) { - out->has_locals = true; - return true; - } - return false; -} - -/* Detect a top-level block header. Returns true if line was consumed as a header. */ -static bool tf_detect_block_header(const char *trimmed, cbm_terraform_result_t *out, - tf_block_kind_t *cur_block, int *cur_var_idx, int *cur_mod_idx) { - char ident1[CBM_SZ_128]; - - /* resource "type" "name" */ - { - int rc = tf_extract_ident(trimmed, "resource", ident1, sizeof(ident1)); - if (rc > 0 && out->resource_count < CBM_SZ_32) { - char ident2[CBM_SZ_128]; - tf_extract_second_quoted(trimmed, rc, ident2, sizeof(ident2)); - snprintf(out->resources[out->resource_count].type, sizeof(out->resources[0].type), "%s", - ident1); - snprintf(out->resources[out->resource_count].name, sizeof(out->resources[0].name), "%s", - ident2); - out->resource_count++; - return true; - } - } - - /* variable "name" */ - int consumed = tf_extract_ident(trimmed, "variable", ident1, sizeof(ident1)); - if (consumed > 0 && out->variable_count < CBM_SZ_32) { - *cur_block = TF_BLOCK_VARIABLE; - *cur_var_idx = out->variable_count; - snprintf(out->variables[*cur_var_idx].name, sizeof(out->variables[0].name), "%s", ident1); - out->variable_count++; - return true; - } - - /* output "name" */ - if (tf_try_simple_block(trimmed, "output", out->outputs[0], (int)sizeof(out->outputs[0]), - &out->output_count, CBM_SZ_32)) { - return true; - } - - /* provider "name" */ - if (tf_try_simple_block(trimmed, "provider", out->providers[0], (int)sizeof(out->providers[0]), - &out->provider_count, IS_MAX_STAGES)) { - return true; - } - - /* module "name" */ - consumed = tf_extract_ident(trimmed, "module", ident1, sizeof(ident1)); - if (consumed > 0 && out->module_count < IS_MAX_STAGES) { - *cur_block = TF_BLOCK_MODULE; - *cur_mod_idx = out->module_count; - snprintf(out->modules[*cur_mod_idx].tf_name, sizeof(out->modules[0].tf_name), "%s", ident1); - out->module_count++; - return true; - } - - /* data "type" "name" */ - { - int dc = tf_extract_ident(trimmed, "data", ident1, sizeof(ident1)); - if (dc > 0 && out->data_source_count < IS_MAX_STAGES) { - char ident2[CBM_SZ_128]; - tf_extract_second_quoted(trimmed, dc, ident2, sizeof(ident2)); - snprintf(out->data_sources[out->data_source_count].type, - sizeof(out->data_sources[0].type), "%s", ident1); - snprintf(out->data_sources[out->data_source_count].name, - sizeof(out->data_sources[0].name), "%s", ident2); - out->data_source_count++; - return true; - } - } - - return tf_try_keyword_block(trimmed, cur_block, out); -} - -/* Extract attributes from a line inside a variable block. */ -static void tf_parse_variable_attrs(const char *trimmed, cbm_tf_variable_t *v) { - char def_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "default", def_val, sizeof(def_val)); - if (def_val[0] && !cbm_is_secret_binding(v->name, def_val)) { - snprintf(v->default_val, sizeof(v->default_val), "%s", def_val); - } - char type_val[CBM_SZ_64] = {0}; - tf_extract_quoted(trimmed, "type", type_val, sizeof(type_val)); - if (type_val[0]) { - snprintf(v->type, sizeof(v->type), "%s", type_val); - } - char desc_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "description", desc_val, sizeof(desc_val)); - if (desc_val[0]) { - snprintf(v->description, sizeof(v->description), "%s", desc_val); - } -} - -/* Extract attributes inside a block (variable/module/terraform). */ -static void tf_parse_block_attrs(const char *trimmed, cbm_terraform_result_t *out, - tf_block_kind_t cur_block, int cur_var_idx, int cur_mod_idx) { - switch (cur_block) { - case TF_BLOCK_VARIABLE: - if (cur_var_idx >= 0) { - tf_parse_variable_attrs(trimmed, &out->variables[cur_var_idx]); - } - break; - case TF_BLOCK_MODULE: - if (cur_mod_idx >= 0) { - char src_val[CBM_SZ_256] = {0}; - tf_extract_quoted(trimmed, "source", src_val, sizeof(src_val)); - if (src_val[0]) { - snprintf(out->modules[cur_mod_idx].source, sizeof(out->modules[0].source), "%s", - src_val); - } - } - break; - case TF_BLOCK_TERRAFORM: { - const char *bp = skip_ws(trimmed); - char backend_name[CBM_SZ_128] = {0}; - tf_extract_ident(bp, "backend", backend_name, sizeof(backend_name)); - if (backend_name[0]) { - snprintf(out->backend, sizeof(out->backend), "%s", backend_name); - } - break; - } - default: - break; - } -} - -/* Check if any terraform content was parsed. */ -static bool tf_result_has_content(const cbm_terraform_result_t *out) { - return out->resource_count > 0 || out->variable_count > 0 || out->output_count > 0 || - out->provider_count > 0 || out->module_count > 0 || out->data_source_count > 0 || - out->backend[0] != '\0' || out->has_locals; -} - -/* Read next non-empty, non-comment line. Advances *pp. Returns trimmed or NULL. */ -static char *tf_next_line(const char **pp, char *line, size_t line_sz) { - const char *p = *pp; - const char *eol = strchr(p, '\n'); - size_t len = eol ? (size_t)(eol - p) : strlen(p); - if (len >= line_sz) { - len = line_sz - SKIP_ONE; - } - memcpy(line, p, len); - line[len] = '\0'; - *pp = eol ? eol + SKIP_ONE : p + len; - - char *trimmed = line; - while (*trimmed == ' ' || *trimmed == '\t') { - trimmed++; - } - rtrim(trimmed); - if (*trimmed == '\0' || *trimmed == '#' || (trimmed[0] == '/' && trimmed[SKIP_ONE] == '/')) { - return NULL; - } - return trimmed; -} - -int cbm_parse_terraform_source(const char *source, cbm_terraform_result_t *out) { - if (!source || !out) { - return CBM_NOT_FOUND; - } - memset(out, 0, sizeof(*out)); - - const char *p = source; - char line[CBM_SZ_4K]; - int brace_depth = 0; - tf_block_kind_t cur_block = TF_BLOCK_NONE; - int cur_var_idx = IS_NOT_FOUND; - int cur_mod_idx = IS_NOT_FOUND; - - while (*p) { - char *trimmed = tf_next_line(&p, line, sizeof(line)); - if (!trimmed) { - continue; - } - - brace_depth += count_char(trimmed, '{') - count_char(trimmed, '}'); - - if (brace_depth <= SKIP_ONE && cur_block == TF_BLOCK_NONE) { - if (tf_detect_block_header(trimmed, out, &cur_block, &cur_var_idx, &cur_mod_idx)) { - continue; - } - } - - if (cur_block != TF_BLOCK_NONE && brace_depth >= SKIP_ONE && brace_depth <= PAIR_LEN) { - tf_parse_block_attrs(trimmed, out, cur_block, cur_var_idx, cur_mod_idx); - } - - if (brace_depth == 0 && cur_block != TF_BLOCK_NONE) { - cur_block = TF_BLOCK_NONE; - cur_var_idx = IS_NOT_FOUND; - cur_mod_idx = IS_NOT_FOUND; - } - } - - return tf_result_has_content(out) ? 0 : IS_NOT_FOUND; -} - /* ── Infra QN helper ────────────────────────────────────────────── */ char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *infra_type, @@ -1272,14 +243,51 @@ static const char *helm_match_key(const char *content, const char *eol, const ch return content + kl + 1; } +/* Append an empty dependency entry; returns it, or NULL when out of memory. */ +static cbm_helm_dep_t *helm_push_dep(cbm_helm_chart_t *out) { + if (out->dep_count == out->dep_cap) { + enum { HELM_DEPS_MIN = 8 }; + int cap = out->dep_cap ? out->dep_cap * 2 : HELM_DEPS_MIN; + cbm_helm_dep_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, out->deps, (size_t)cap * sizeof(*grown)); + if (!grown) { + return NULL; + } + out->deps = grown; + out->dep_cap = cap; + } + cbm_helm_dep_t *dep = &out->deps[out->dep_count++]; + dep->name[0] = '\0'; + dep->repository[0] = '\0'; + return dep; +} + +void cbm_helm_chart_free(cbm_helm_chart_t *hc) { + if (!hc) { + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, hc->deps); + hc->deps = NULL; + hc->dep_count = 0; + hc->dep_cap = 0; +} + int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out) { - if (!source || !out) { + if (!out) { return -1; } out->chart_name[0] = '\0'; + out->deps = NULL; out->dep_count = 0; + out->dep_cap = 0; + if (!source) { + return -1; + } bool in_deps = false; + cbm_helm_dep_t *cur = NULL; /* the dependency entry being read */ + int item_indent = -1; /* indent of the entries' "- " */ + int key_col = -1; /* column of an entry's keys */ const char *p = source; while (*p) { const char *eol = strchr(p, '\n'); @@ -1296,26 +304,35 @@ int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out) { if (indent == 0) { /* New top-level key ends any dependencies block. */ in_deps = (helm_match_key(content, eol, "dependencies") != NULL); + cur = NULL; + item_indent = -1; + key_col = -1; const char *nv = helm_match_key(content, eol, "name"); if (!in_deps && nv && out->chart_name[0] == '\0') { helm_copy_scalar(nv, eol, out->chart_name, CBM_HELM_NAME_MAX); } } else if (in_deps) { - /* List item: optional leading "- ", then look for name:. */ + /* A list item ("- ") at the entries' indent opens an entry; its + * keys share the dash line or follow at the same key column. + * Deeper lines (import-values, tags) belong to the entry. */ const char *q = content; - if (*q == '-') { + bool opens = *q == '-' && (item_indent < 0 || indent == item_indent); + if (opens) { + item_indent = indent; q++; while (q < eol && *q == ' ') { q++; } + key_col = indent + (int)(q - content); + cur = helm_push_dep(out); } - const char *nv = helm_match_key(q, eol, "name"); - if (nv && out->dep_count < CBM_HELM_MAX_DEPS) { - char buf[CBM_HELM_NAME_MAX]; - helm_copy_scalar(nv, eol, buf, CBM_HELM_NAME_MAX); - if (buf[0]) { - memcpy(out->deps[out->dep_count], buf, CBM_HELM_NAME_MAX); - out->dep_count++; + if (cur && (opens || indent == key_col)) { + const char *nv = helm_match_key(q, eol, "name"); + const char *rv = helm_match_key(q, eol, "repository"); + if (nv) { + helm_copy_scalar(nv, eol, cur->name, CBM_HELM_NAME_MAX); + } else if (rv) { + helm_copy_scalar(rv, eol, cur->repository, CBM_HELM_REPO_MAX); } } } @@ -1325,5 +342,13 @@ int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out) { } p = eol + 1; } + /* Keep only named entries (a dependency without a name names nothing). */ + int kept = 0; + for (int i = 0; i < out->dep_count; i++) { + if (out->deps[i].name[0]) { + out->deps[kept++] = out->deps[i]; + } + } + out->dep_count = kept; return (out->chart_name[0] || out->dep_count > 0) ? 0 : -1; } diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 3a6982ef6..2ba210f49 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -1,14 +1,12 @@ /* - * pass_k8s.c — Pipeline pass for Kubernetes manifest and Kustomize overlay processing. + * pass_k8s.c — Pipeline pass for Kubernetes manifests, Helm charts and dependency manifests. * - * For each discovered YAML file: - * 1. Check if it is a kustomize overlay (kustomization.yaml / kustomization.yml) - * → emit a Module node and IMPORTS edges for each resources/bases/patches entry - * 2. Else if it is a generic k8s manifest (apiVersion: detected) - * → emit one Resource node per file (first document only — multi-document YAML is not yet - * supported) + * For each discovered YAML file that is a generic k8s manifest (apiVersion: detected) + * → emit one Resource node per file (first document only — multi-document YAML is not yet + * supported). Kustomizations are not handled here: their entries are IMPORTS of the + * file itself, resolved by the import resolver (resolve_kustomize_entry). * - * Depends on: pass_infrascan.c (cbm_is_kustomize_file, cbm_is_k8s_manifest, cbm_infra_qn), + * Depends on: pass_infrascan.c (cbm_is_k8s_manifest, cbm_infra_qn), * extraction layer (cbm.h), graph_buffer, pipeline internals. */ #include "foundation/constants.h" @@ -25,6 +23,7 @@ #include "helpers.h" /* cbm_kind_in_set_free_cache */ #include "foundation/slab_alloc.h" /* cbm_slab_destroy_thread */ #include "foundation/platform.h" /* cbm_default_worker_count */ +#include "foundation/str_util.h" /* cbm_json_escape */ #include "pipeline/worker_pool.h" #include @@ -32,6 +31,8 @@ #include #include +#define K8S_SLEN(s) (sizeof(s) - 1) + /* ── Internal helpers ────────────────────────────────────────────── */ /* Read entire file into heap-allocated buffer. Returns NULL on error. @@ -86,88 +87,44 @@ static const char *k8s_basename(const char *path) { return p ? p + SKIP_ONE : path; } -/* ── Kustomize handler ───────────────────────────────────────────── */ - -/* `result` is the cached parallel-pass result or, for an uncached file, the - * re-extraction the preparation step made; the caller owns it. */ -static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *rel_path, - const CBMFileResult *result) { - /* Emit Module node for this kustomize overlay file */ - char *mod_qn = cbm_infra_qn(ctx->project_name, rel_path, "kustomize", NULL); - if (!mod_qn) { - return; - } - - int64_t mod_id = cbm_gbuf_upsert_node(ctx->gbuf, "Module", k8s_basename(rel_path), mod_qn, - rel_path, SKIP_ONE, 0, "{\"source\":\"kustomize\"}"); - free(mod_qn); - - if (mod_id <= 0) { - return; - } - - /* With an extraction result, emit IMPORTS edges for - * resources/bases/patches/components entries */ - int import_count = 0; - const CBMFileResult *res = result; - - if (res) { - for (int j = 0; j < res->imports.count; j++) { - CBMImport *imp = &res->imports.items[j]; - if (!imp->module_path) { - continue; - } - - /* Compute target file QN */ - char *target_qn = - cbm_pipeline_fqn_compute(ctx->project_name, imp->module_path, "__file__"); - if (!target_qn) { - continue; - } - - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_qn(ctx->gbuf, target_qn); - free(target_qn); - - if (target) { - cbm_gbuf_insert_edge(ctx->gbuf, mod_id, target->id, "IMPORTS", - "{\"via\":\"kustomize\"}"); - import_count++; - } - } - } - - cbm_log_info("pass.k8s.kustomize", "file", rel_path, "imports", itoa_k8s(import_count)); -} - /* ── K8s cross-manifest label-selector matching ────────────────────── - * A Service routes traffic to the workload Pods whose labels match the - * Service's spec.selector. We record, per manifest, the Resource node id plus - * its selector label-values (Services) and pod label-values (workloads), then - * after all manifests are processed connect each Service to the workload(s) it - * targets via an INFRA_MAPS edge. This models the runtime traffic path the same - * way k8s itself resolves Service → Endpoints by label. - * - * Matching is by label *value*: a Service selector `app: frontend` matches a - * workload that either carries the pod label value "frontend" OR is named - * "frontend" (the ubiquitous app==name convention, used when the pod template - * omits explicit labels). */ -enum { K8S_MAX_LABELS = 16, K8S_MAX_RECORDS = 512, K8S_LABEL_LEN = 128 }; + * A Service routes traffic to the Pods whose labels carry EVERY pair of the + * Service's spec.selector, in the Service's own namespace -- the way k8s + * resolves Service → Endpoints. Per manifest we record the Resource node, its + * namespace and the label pairs that take part: a Service's spec.selector, a + * workload's pod-template labels, a bare Pod's metadata.labels. After all + * manifests are read, an inverted index from "key=value" to workloads turns + * each Service into a scan of the workloads carrying its rarest pair rather + * than of every workload; each match becomes an INFRA_MAPS edge + * (Service → workload). */ +enum { K8S_LABEL_LEN = 128, K8S_GROW_MIN = 64 }; + +typedef enum { + K8S_REC_OTHER = 0, + K8S_REC_SERVICE, /* pairs = spec.selector */ + K8S_REC_WORKLOAD, /* pairs = pod-template labels */ + K8S_REC_POD, /* pairs = metadata.labels */ + K8S_REC_PDB, /* pairs = spec.selector.matchLabels */ + K8S_REC_NETPOL, /* pairs = spec.podSelector.matchLabels */ +} k8s_rec_kind_t; typedef struct { int64_t node_id; char name[K8S_LABEL_LEN]; - bool is_service; - bool is_workload; - char selector_vals[K8S_MAX_LABELS][K8S_LABEL_LEN]; /* Service spec.selector values */ - int n_selector; - char label_vals[K8S_MAX_LABELS][K8S_LABEL_LEN]; /* workload pod-template label values */ - int n_label; + char ns[K8S_LABEL_LEN]; /* metadata.namespace; "" when unset */ + k8s_rec_kind_t kind; + int pair_off; /* this record's pairs: pairs[pair_off .. pair_off + pair_n) */ + int pair_n; + bool has_expr; /* its label selector also has matchExpressions */ } k8s_record_t; typedef struct { k8s_record_t *items; int count; int cap; + char **pairs; /* "key=value", heap-owned */ + int pair_count; + int pair_cap; } k8s_record_array_t; /* Count leading-space indentation of a line (tabs are invalid YAML indent). */ @@ -219,26 +176,186 @@ static int k8s_split_kv(const char *t, char *key, size_t key_sz, char *val, size return 1; } -static void k8s_add_val(char dst[][K8S_LABEL_LEN], int *n, const char *v) { - if (*n >= K8S_MAX_LABELS || !v || !v[0]) { - return; +/* Which label block a leaf `key: value` sits in, by its parent path. */ +typedef enum { + K8S_PAIR_NONE = 0, + K8S_PAIR_SELECTOR, /* spec.selector (a Service's selector is a flat map) */ + K8S_PAIR_TEMPLATE_LABEL, /* spec.template.metadata.labels, also under a CronJob's jobTemplate */ + K8S_PAIR_META_LABEL, /* metadata.labels */ + K8S_PAIR_MATCH_LABELS, /* spec.selector.matchLabels (a PodDisruptionBudget's) */ + K8S_PAIR_POD_SELECTOR, /* spec.podSelector.matchLabels (a NetworkPolicy's) */ +} k8s_pair_block_t; + +static k8s_pair_block_t k8s_pair_block(const char *parent_path) { + if (strcmp(parent_path, "spec.selector") == 0) { + return K8S_PAIR_SELECTOR; } - snprintf(dst[*n], K8S_LABEL_LEN, "%s", v); - (*n)++; + if (strcmp(parent_path, "spec.selector.matchLabels") == 0) { + return K8S_PAIR_MATCH_LABELS; + } + if (strcmp(parent_path, "spec.podSelector.matchLabels") == 0) { + return K8S_PAIR_POD_SELECTOR; + } + if (strcmp(parent_path, "spec.template.metadata.labels") == 0 || + strcmp(parent_path, "spec.jobTemplate.spec.template.metadata.labels") == 0) { + return K8S_PAIR_TEMPLATE_LABEL; + } + if (strcmp(parent_path, "metadata.labels") == 0) { + return K8S_PAIR_META_LABEL; + } + return K8S_PAIR_NONE; } -/* Scan a single-document k8s manifest's text for the resource name, selector - * label-values (Service) and pod-template label-values (workload). A small - * indentation path-stack distinguishes spec.selector from - * spec.template.metadata.labels and from top-level metadata.name. */ -static void k8s_scan_labels(const char *source, k8s_record_t *rec) { +static k8s_rec_kind_t k8s_rec_kind(const char *kind) { + static const char *const workloads[] = { + "Deployment", "StatefulSet", "DaemonSet", "ReplicaSet", "ReplicationController", + "Job", "CronJob", "Rollout", NULL}; + if (strcmp(kind, "Service") == 0) { + return K8S_REC_SERVICE; + } + if (strcmp(kind, "PodDisruptionBudget") == 0) { + return K8S_REC_PDB; + } + if (strcmp(kind, "NetworkPolicy") == 0) { + return K8S_REC_NETPOL; + } + if (strcmp(kind, "Pod") == 0) { + return K8S_REC_POD; + } + for (int i = 0; workloads[i]; i++) { + if (strcmp(kind, workloads[i]) == 0) { + return K8S_REC_WORKLOAD; + } + } + return K8S_REC_OTHER; +} + +/* The block whose pairs a record of this kind contributes. */ +static k8s_pair_block_t k8s_kind_block(k8s_rec_kind_t kind) { + switch (kind) { + case K8S_REC_SERVICE: + return K8S_PAIR_SELECTOR; + case K8S_REC_WORKLOAD: + return K8S_PAIR_TEMPLATE_LABEL; + case K8S_REC_POD: + return K8S_PAIR_META_LABEL; + case K8S_REC_PDB: + return K8S_PAIR_MATCH_LABELS; + case K8S_REC_NETPOL: + return K8S_PAIR_POD_SELECTOR; + case K8S_REC_OTHER: + default: + return K8S_PAIR_NONE; + } +} + +/* One scanned `key: value` with the block it sits in. */ +typedef struct { + k8s_pair_block_t block; + char *kv; /* "key=value", heap-owned until moved into the record array */ +} k8s_scanned_pair_t; + +typedef struct { + k8s_scanned_pair_t *items; + int count; + int cap; +} k8s_scanned_t; + +static char *k8s_join_kv(const char *key, const char *val) { + size_t kl = strlen(key); + size_t vl = strlen(val); + char *kv = cbm_alloc(CBM_MEM_CLASS_OTHER, kl + vl + 2); + if (!kv) { + return NULL; + } + memcpy(kv, key, kl); + kv[kl] = '='; + memcpy(kv + kl + 1, val, vl + 1); + return kv; +} + +static void k8s_scanned_push(k8s_scanned_t *s, k8s_pair_block_t block, const char *key, + const char *val) { + if (s->count == s->cap) { + int cap = s->cap ? s->cap * 2 : K8S_GROW_MIN; + k8s_scanned_pair_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, s->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + s->items = grown; + s->cap = cap; + } + char *kv = k8s_join_kv(key, val); + if (kv) { + s->items[s->count++] = (k8s_scanned_pair_t){.block = block, .kv = kv}; + } +} + +/* Dotted path of the open block keys, e.g. "spec.template.metadata.labels". */ +static void k8s_parent_path(char *out, size_t out_sz, const char (*keys)[64], int depth) { + out[0] = '\0'; + size_t w = 0; + for (int i = 0; i < depth && w + 1 < out_sz; i++) { + int n = snprintf(out + w, out_sz - w, "%s%s", i ? "." : "", keys[i]); + if (n < 0) { + break; + } + w += (size_t)n < out_sz - w ? (size_t)n : out_sz - w - 1; + } +} + +/* `key: { a: b, c: d }` — a flow-style map is the same label map written on + * one line. One the line buffer cut short (no closing brace) is skipped + * whole: a selector missing pairs would select too much. */ +static void k8s_scan_flow_map(k8s_scanned_t *out, const char (*keys)[64], int depth, + const char *key, const char *val) { + size_t n = strlen(val); + if (n < PAIR_LEN || val[0] != '{' || val[n - SKIP_ONE] != '}') { + return; + } + char path[CBM_SZ_512]; + k8s_parent_path(path, sizeof(path), keys, depth); + size_t w = strlen(path); + snprintf(path + w, sizeof(path) - w, "%s%s", w ? "." : "", key); + k8s_pair_block_t block = k8s_pair_block(path); + if (block == K8S_PAIR_NONE) { + return; + } + char inner[CBM_SZ_512]; + snprintf(inner, sizeof(inner), "%.*s", (int)(n - PAIR_LEN), val + SKIP_ONE); + for (char *item = inner; item;) { + char *comma = strchr(item, ','); + if (comma) { + *comma = '\0'; + } + while (*item == ' ') { + item++; + } + char k[64]; + char v[K8S_LABEL_LEN]; + if (k8s_split_kv(item, k, sizeof(k), v, sizeof(v)) && v[0]) { + size_t kl = strlen(k); + while (kl > 0 && k[kl - SKIP_ONE] == ' ') { + k[--kl] = '\0'; + } + k8s_scanned_push(out, block, k, v); + } + item = comma ? comma + SKIP_ONE : NULL; + } +} + +/* Scan ONE manifest document's text for its name, namespace, kind and + * label pairs into `rec` / `out`. A small indentation path-stack gives every + * leaf its parent path, which tells spec.selector from the pod template's + * labels and from the object's own metadata.labels. */ +static void k8s_scan_labels(const char *source, k8s_record_t *rec, k8s_scanned_t *out) { enum { K8S_PATH_DEPTH = 12 }; - struct { - int indent; - char key[64]; - } stack[K8S_PATH_DEPTH]; + int indents[K8S_PATH_DEPTH]; + char keys[K8S_PATH_DEPTH][64]; int depth = 0; bool got_name = false; + bool seen_key = false; const char *p = source; while (p && *p) { @@ -249,53 +366,62 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec) { memcpy(line, p, cp); line[cp] = '\0'; - /* End of first YAML document — stop (one Resource per file). */ + /* End of the document — stop (the caller hands over one document). */ const char *trimmed = line; while (*trimmed == ' ') { trimmed++; } + /* A `---` before any key opens the first document; it does not end it. */ if (strncmp(trimmed, "---", 3) == 0 && line == trimmed) { - break; + if (seen_key) { + break; + } + if (!eol) { + break; + } + p = eol + 1; + continue; } if (trimmed[0] && trimmed[0] != '#') { + seen_key = true; int ind = k8s_indent(line); - while (depth > 0 && stack[depth - 1].indent >= ind) { + while (depth > 0 && indents[depth - 1] >= ind) { depth--; } char key[64]; - char val[K8S_LABEL_LEN]; + char val[CBM_SZ_512]; /* a whole flow map fits: the line is the limit */ if (k8s_split_kv(trimmed, key, sizeof(key), val, sizeof(val))) { - /* Build the current dotted path for context decisions. */ - bool under_selector = (depth >= 1 && strcmp(stack[depth - 1].key, "selector") == 0); - bool under_labels = (depth >= 1 && strcmp(stack[depth - 1].key, "labels") == 0); - bool under_metadata = (depth >= 1 && strcmp(stack[depth - 1].key, "metadata") == 0); - + if (strcmp(key, "matchExpressions") == 0) { + char parent[CBM_SZ_512]; + k8s_parent_path(parent, sizeof(parent), (const char (*)[64])keys, depth); + rec->has_expr = rec->has_expr || strcmp(parent, "spec.selector") == 0 || + strcmp(parent, "spec.podSelector") == 0; + } if (val[0] == '\0') { /* Block-opening key: push onto the path stack. */ if (depth < K8S_PATH_DEPTH) { - stack[depth].indent = ind; - snprintf(stack[depth].key, sizeof(stack[depth].key), "%s", key); + indents[depth] = ind; + snprintf(keys[depth], sizeof(keys[depth]), "%s", key); depth++; } - } else { - /* Leaf key: value. */ - if (ind == 0 && strcmp(key, "kind") == 0) { - rec->is_service = (strcmp(val, "Service") == 0); - rec->is_workload = - (strcmp(val, "Deployment") == 0 || strcmp(val, "StatefulSet") == 0 || - strcmp(val, "DaemonSet") == 0 || strcmp(val, "ReplicaSet") == 0 || - strcmp(val, "ReplicationController") == 0 || strcmp(val, "Pod") == 0 || - strcmp(val, "Job") == 0 || strcmp(val, "CronJob") == 0); - } else if (!got_name && under_metadata && strcmp(key, "name") == 0) { + } else if (ind == 0 && strcmp(key, "kind") == 0) { + rec->kind = k8s_rec_kind(val); + } else if (val[0] == '{') { + k8s_scan_flow_map(out, (const char (*)[64])keys, depth, key, val); + } else if (depth == 1 && strcmp(keys[0], "metadata") == 0) { + if (!got_name && strcmp(key, "name") == 0) { snprintf(rec->name, sizeof(rec->name), "%s", val); got_name = true; - } else if (under_selector) { - /* Service spec.selector matchLabels values. */ - k8s_add_val(rec->selector_vals, &rec->n_selector, val); - } else if (under_labels) { - /* Pod-template / metadata labels values. */ - k8s_add_val(rec->label_vals, &rec->n_label, val); + } else if (strcmp(key, "namespace") == 0) { + snprintf(rec->ns, sizeof(rec->ns), "%s", val); + } + } else { + char parent[CBM_SZ_512]; + k8s_parent_path(parent, sizeof(parent), (const char (*)[64])keys, depth); + k8s_pair_block_t block = k8s_pair_block(parent); + if (block != K8S_PAIR_NONE) { + k8s_scanned_push(out, block, key, val); } } } @@ -308,18 +434,614 @@ static void k8s_scan_labels(const char *source, k8s_record_t *rec) { } } -/* True if any of the service's selector values matches the workload. */ -static bool k8s_selector_matches(const k8s_record_t *svc, const k8s_record_t *wl) { - for (int s = 0; s < svc->n_selector; s++) { - const char *sv = svc->selector_vals[s]; - if (!sv[0]) { - continue; +static bool k8s_records_reserve(k8s_record_array_t *recs, int pair_extra) { + if (recs->count == recs->cap) { + int cap = recs->cap ? recs->cap * 2 : K8S_GROW_MIN; + k8s_record_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, recs->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; } - if (wl->name[0] && strcmp(sv, wl->name) == 0) { + recs->items = grown; + recs->cap = cap; + } + if (recs->pair_count + pair_extra > recs->pair_cap) { + int cap = recs->pair_cap ? recs->pair_cap : K8S_GROW_MIN; + while (cap < recs->pair_count + pair_extra) { + cap *= 2; + } + char **grown = cbm_realloc(CBM_MEM_CLASS_OTHER, recs->pairs, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + recs->pairs = grown; + recs->pair_cap = cap; + } + return true; +} + +/* Record the manifest behind Resource `node_id`: scan its text, keep the + * pairs its kind contributes, drop the rest. */ +static void k8s_record_manifest(k8s_record_array_t *recs, int64_t node_id, const char *source) { + k8s_record_t rec = {.node_id = node_id}; + k8s_scanned_t scanned = {0}; + k8s_scan_labels(source, &rec, &scanned); + k8s_pair_block_t keep = k8s_kind_block(rec.kind); + int kept = 0; + for (int i = 0; i < scanned.count; i++) { + kept += scanned.items[i].block == keep; + } + if (rec.kind != K8S_REC_OTHER && k8s_records_reserve(recs, kept)) { + rec.pair_off = recs->pair_count; + for (int i = 0; i < scanned.count; i++) { + if (scanned.items[i].block == keep) { + recs->pairs[recs->pair_count++] = scanned.items[i].kv; + scanned.items[i].kv = NULL; + } + } + rec.pair_n = recs->pair_count - rec.pair_off; + recs->items[recs->count++] = rec; + } + for (int i = 0; i < scanned.count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, scanned.items[i].kv); + } + cbm_free(CBM_MEM_CLASS_OTHER, scanned.items); +} + +/* Record the document at lines [start_line, end_line] (1-based, inclusive) of + * a multi-document manifest: every object's selector or pod labels come from + * its own document. */ +static void k8s_record_document(k8s_record_array_t *recs, int64_t node_id, const char *source, + int start_line, int end_line) { + const char *begin = source; + for (int line = 1; begin && line < start_line; line++) { + begin = strchr(begin, '\n'); + begin = begin ? begin + SKIP_ONE : NULL; + } + if (!begin) { + return; + } + const char *end = begin; + for (int line = start_line; *end && line <= end_line; line++) { + const char *eol = strchr(end, '\n'); + end = eol ? eol + SKIP_ONE : end + strlen(end); + } + size_t n = (size_t)(end - begin); + char *doc = cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE); + if (!doc) { + return; + } + memcpy(doc, begin, n); + doc[n] = '\0'; + k8s_record_manifest(recs, node_id, doc); + cbm_free(CBM_MEM_CLASS_OTHER, doc); +} + +/* Built-in kinds that live outside any namespace (IDENTITY.md). A CRD's own + * scope (spec.scope) is the curated / generated table's job. */ +static bool k8s_cluster_scoped(const char *kind) { + static const char *const kinds[] = {"Namespace", + "Node", + "PersistentVolume", + "ClusterRole", + "ClusterRoleBinding", + "CustomResourceDefinition", + "StorageClass", + "PriorityClass", + "ValidatingWebhookConfiguration", + "MutatingWebhookConfiguration", + "APIService", + "CSIDriver", + "CSINode", + "IngressClass", + "RuntimeClass", + "VolumeAttachment", + "ValidatingAdmissionPolicy", + "ValidatingAdmissionPolicyBinding", + "ClusterIssuer"}; + for (size_t i = 0; i < sizeof(kinds) / sizeof(kinds[0]); i++) { + if (strcmp(kind, kinds[i]) == 0) { return true; } - for (int l = 0; l < wl->n_label; l++) { - if (strcmp(sv, wl->label_vals[l]) == 0) { + } + return false; +} + +/* A Resource's properties from its identity key, the part of its QN after + * "::": `///` (IDENTITY.md). + * `obj_key` is the logical identity: the namespace it lives in, `_` for a + * cluster-scoped kind, `?` when the deploy decides; `effective_ns` (when + * not NULL) is the namespace a kustomize build gives it. NULL for a QN + * without a well-formed key (the caller keeps the plain source tag). */ +static char *k8s_resource_props(const char *qn, const char *effective_ns) { + enum { K8S_KEY_PARTS = 4, K8S_ESC = 6 * K8S_LABEL_LEN }; + const char *key = qn ? strstr(qn, "::") : NULL; + if (!key) { + return NULL; + } + key += K8S_SLEN("::"); + char part[K8S_KEY_PARTS][K8S_LABEL_LEN]; + int parts = 0; + for (const char *p = key; parts < K8S_KEY_PARTS; parts++) { + const char *slash = parts < K8S_KEY_PARTS - SKIP_ONE ? strchr(p, '/') : NULL; + size_t len = slash ? (size_t)(slash - p) : strlen(p); + if (len == 0 || len >= K8S_LABEL_LEN || (!slash && parts < K8S_KEY_PARTS - SKIP_ONE)) { + return NULL; + } + memcpy(part[parts], p, len); + part[parts][len] = '\0'; + p = slash ? slash + SKIP_ONE : p + len; + } + const char *group = part[0]; + const char *kind = part[1]; + const char *ns = strcmp(part[2], "-") == 0 ? "" : part[2]; + const char *name = part[3]; + const char *effective = k8s_cluster_scoped(kind) ? "_" + : effective_ns ? effective_ns + : ns[0] ? ns + : "?"; + char esc[K8S_KEY_PARTS + SKIP_ONE][K8S_ESC]; + cbm_json_escape(esc[0], K8S_ESC, group); + cbm_json_escape(esc[1], K8S_ESC, kind); + cbm_json_escape(esc[2], K8S_ESC, ns); + cbm_json_escape(esc[3], K8S_ESC, name); + cbm_json_escape(esc[4], K8S_ESC, effective); + size_t n = K8S_KEY_PARTS * K8S_ESC + K8S_ESC + CBM_SZ_256; + char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, n); + if (!props) { + return NULL; + } + int w = + snprintf(props, n, "{\"source\":\"k8s\",\"group\":\"%s\",\"kind\":\"%s\"", esc[0], esc[1]); + if (ns[0]) { + w += snprintf(props + w, n - (size_t)w, ",\"namespace\":\"%s\"", esc[2]); + } + snprintf(props + w, n - (size_t)w, ",\"obj_key\":\"%s/%s/%s/%s\"}", esc[0], esc[1], esc[4], + esc[3]); + return props; +} + +static void k8s_records_free(k8s_record_array_t *recs) { + for (int i = 0; i < recs->pair_count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, recs->pairs[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, recs->pairs); + cbm_free(CBM_MEM_CLASS_OTHER, recs->items); + *recs = (k8s_record_array_t){0}; +} + +/* Workloads (record indices, ascending) that carry one "key=value" pair. */ +typedef struct { + int *items; + int count; + int cap; +} k8s_posting_t; + +static void k8s_posting_free(const char *key, void *value, void *userdata) { + (void)key; + (void)userdata; + k8s_posting_t *posting = value; + cbm_free(CBM_MEM_CLASS_OTHER, posting->items); + cbm_free(CBM_MEM_CLASS_OTHER, posting); +} + +static bool k8s_posting_add(CBMHashTable *index, const char *kv, int rec_idx) { + k8s_posting_t *posting = cbm_ht_get(index, kv); + if (!posting) { + posting = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*posting)); + if (!posting) { + return false; + } + cbm_ht_set(index, kv, posting); + } + if (posting->count && posting->items[posting->count - 1] == rec_idx) { + return true; /* the same pair twice in one workload */ + } + if (posting->count == posting->cap) { + int cap = posting->cap ? posting->cap * 2 : K8S_GROW_MIN; + int *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, posting->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return false; + } + posting->items = grown; + posting->cap = cap; + } + posting->items[posting->count++] = rec_idx; + return true; +} + +static bool k8s_has_pair(const k8s_record_array_t *recs, const k8s_record_t *rec, const char *kv) { + for (int i = 0; i < rec->pair_n; i++) { + if (strcmp(recs->pairs[rec->pair_off + i], kv) == 0) { + return true; + } + } + return false; +} + +/* ── Bundles: what is deployed together (IDENTITY.md) ────────────── */ + +/* A Service selects pods of its own deployment, never a same-labelled copy + * elsewhere in the repository. A bundle is a kustomize build root (a + * kustomization no other one includes) with every manifest its entries + * reach; outside kustomize, the Helm chart a file sits in, else its + * directory. A base several overlays include belongs to each of them. */ + +typedef struct { + char *path; /* the kustomization file */ + char *ns; /* its `namespace:`, NULL when unset */ + int *kids; /* kustomizations its entries name (indices) */ + int kid_n; + int kid_cap; + const char **files; /* manifests its entries name (graph-owned paths) */ + int file_n; + int file_cap; + bool referenced; /* included by another kustomization */ +} k8s_kust_t; + +typedef struct { + k8s_kust_t *items; + int count; + int cap; +} k8s_kusts_t; + +/* One bundle a manifest belongs to, with the namespace the bundle gives + * its objects (NULL: their own). `bundle` is an interned name, compared by + * pointer. */ +typedef struct { + const char *bundle; + const char *ns; +} k8s_member_t; + +typedef struct { + k8s_member_t *items; + int count; + int cap; +} k8s_members_t; + +typedef struct { + CBMHashTable *names; /* bundle name -> itself (owned key) */ + CBMHashTable *members; /* manifest path -> k8s_members_t* */ + CBMHashTable *charts; /* Helm chart directory -> itself (owned key) */ + CBMHashTable *deploys; /* manifest path -> k8s_members_t* of its deploy scopes */ +} k8s_bundles_t; + +static bool k8s_grow(void **items, int *cap, int count, size_t elem) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : K8S_GROW_MIN; + void *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, *items, (size_t)ncap * elem); + if (!grown) { + return false; + } + *items = grown; + *cap = ncap; + return true; +} + +/* The top-level `namespace:` of a kustomization, NULL when it sets none. */ +static char *k8s_kustomize_namespace(const char *source) { + for (const char *p = source; p && *p;) { + const char *eol = strchr(p, '\n'); + size_t len = eol ? (size_t)(eol - p) : strlen(p); + if (strncmp(p, "namespace:", K8S_SLEN("namespace:")) == 0) { + char line[CBM_SZ_512]; + snprintf(line, sizeof(line), "%.*s", (int)len, p); + char key[64]; + char val[K8S_LABEL_LEN]; + if (k8s_split_kv(line, key, sizeof(key), val, sizeof(val)) && val[0]) { + return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, val); + } + return NULL; + } + p = eol ? eol + SKIP_ONE : NULL; + } + return NULL; +} + +static void k8s_kusts_push(k8s_kusts_t *ks, const char *path, char *ns) { + if (!k8s_grow((void **)&ks->items, &ks->cap, ks->count, sizeof(*ks->items))) { + cbm_free(CBM_MEM_CLASS_OTHER, ns); + return; + } + ks->items[ks->count++] = + (k8s_kust_t){.path = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, path), .ns = ns}; +} + +static void k8s_kusts_free(k8s_kusts_t *ks) { + for (int i = 0; i < ks->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].path); + cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].ns); + cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].kids); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)ks->items[i].files); + } + cbm_free(CBM_MEM_CLASS_OTHER, ks->items); + *ks = (k8s_kusts_t){0}; +} + +static int k8s_cmp_str(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +static int k8s_cmp_int(const void *a, const void *b) { + int x = *(const int *)a; + int y = *(const int *)b; + return (x > y) - (x < y); +} + +static int k8s_cmp_kust(const void *a, const void *b) { + return strcmp(((const k8s_kust_t *)a)->path, ((const k8s_kust_t *)b)->path); +} + +/* index + 1 of the kustomization at `path`, 0 when none. */ +static int k8s_kust_at(CBMHashTable *by_path, const char *path) { + intptr_t at = (intptr_t)cbm_ht_get(by_path, path); + return (int)at; +} + +/* The kustomization file inside directory `dir`, as kustomize looks for it. */ +static int k8s_kust_in_dir(CBMHashTable *by_path, const char *dir) { + static const char *const names[] = {"kustomization.yaml", "kustomization.yml", "Kustomization"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + int k = k8s_kust_at(by_path, path); + if (k) { + return k; + } + } + return 0; +} + +/* Wire each kustomization to what its entries name, from the IMPORTS edges + * the kustomize resolver drew: a directory means the kustomization in it, + * a file another kustomization or a manifest. */ +static void k8s_kusts_wire(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, CBMHashTable *by_path) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + if (cbm_gbuf_find_edges_by_type(ctx->gbuf, "IMPORTS", &edges, &count) != 0) { + return; + } + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, edges[i]->source_id); + int k = src && src->file_path ? k8s_kust_at(by_path, src->file_path) : 0; + const cbm_gbuf_node_t *tgt = k ? cbm_gbuf_find_by_id(ctx->gbuf, edges[i]->target_id) : NULL; + if (!tgt || !tgt->file_path) { + continue; + } + k8s_kust_t *kust = &ks->items[k - SKIP_ONE]; + int kid = tgt->label && strcmp(tgt->label, "Folder") == 0 + ? k8s_kust_in_dir(by_path, tgt->file_path) + : k8s_kust_at(by_path, tgt->file_path); + if (kid && kid != k) { + if (k8s_grow((void **)&kust->kids, &kust->kid_cap, kust->kid_n, sizeof(int))) { + kust->kids[kust->kid_n++] = kid - SKIP_ONE; + ks->items[kid - SKIP_ONE].referenced = true; + } + } else if (!kid && strcmp(tgt->label ? tgt->label : "", "Folder") != 0) { + if (k8s_grow((void **)&kust->files, &kust->file_cap, kust->file_n, + sizeof(*kust->files))) { + kust->files[kust->file_n++] = tgt->file_path; + } + } + } + for (int i = 0; i < ks->count; i++) { + qsort(ks->items[i].files, (size_t)ks->items[i].file_n, sizeof(*ks->items[i].files), + k8s_cmp_str); + qsort(ks->items[i].kids, (size_t)ks->items[i].kid_n, sizeof(int), k8s_cmp_int); + } +} + +static const char *k8s_bundle_name(k8s_bundles_t *b, const char *kind, const char *where) { + char name[CBM_SZ_1K]; + snprintf(name, sizeof(name), "%s:%s", kind, where); + const char *known = cbm_ht_get_key(b->names, name); + if (known) { + return known; + } + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, name); + if (owned) { + cbm_ht_set(b->names, owned, owned); + } + return owned; +} + +static void k8s_member_add(CBMHashTable *table, const char *file, const char *bundle, + const char *ns) { + k8s_members_t *m = cbm_ht_get(table, file); + if (!m) { + m = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*m)); + if (!m) { + return; + } + cbm_ht_set(table, file, m); + } + for (int i = 0; i < m->count; i++) { + if (m->items[i].bundle == bundle) { + return; /* reached twice from one root: the first path's namespace */ + } + } + if (k8s_grow((void **)&m->items, &m->cap, m->count, sizeof(*m->items))) { + m->items[m->count++] = (k8s_member_t){bundle, ns}; + } +} + +/* Every build root's manifests, each with the namespace the outermost + * kustomization on its path sets (kustomize applies the root's namespace + * transformer last, so it wins). */ +static void k8s_bundles_from_kusts(k8s_bundles_t *b, k8s_kusts_t *ks) { + typedef struct { + int kust; + const char *ns; + } frame_t; + int *seen = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(ks->count ? ks->count : SKIP_ONE) * sizeof(int)); + frame_t *stack = NULL; + int depth = 0; + int cap = 0; + for (int r = 0; seen && r < ks->count; r++) { + if (ks->items[r].referenced) { + continue; + } + const char *bundle = k8s_bundle_name(b, "kustomize", ks->items[r].path); + if (!bundle || !k8s_grow((void **)&stack, &cap, depth, sizeof(*stack))) { + continue; + } + stack[depth++] = (frame_t){r, ks->items[r].ns}; + while (depth > 0) { + frame_t f = stack[--depth]; + if (seen[f.kust] == r + SKIP_ONE) { + continue; + } + seen[f.kust] = r + SKIP_ONE; + const k8s_kust_t *k = &ks->items[f.kust]; + for (int i = 0; i < k->file_n; i++) { + k8s_member_add(b->members, k->files[i], bundle, f.ns); + } + for (int i = k->kid_n - SKIP_ONE; i >= 0; i--) { + if (k8s_grow((void **)&stack, &cap, depth, sizeof(*stack))) { + int kid = k->kids[i]; + stack[depth++] = (frame_t){kid, f.ns ? f.ns : ks->items[kid].ns}; + } + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, stack); + cbm_free(CBM_MEM_CLASS_OTHER, seen); +} + +static void k8s_dirname(const char *path, char *out, size_t out_sz) { + const char *slash = strrchr(path, '/'); + snprintf(out, out_sz, "%.*s", slash ? (int)(slash - path) : 0, path); +} + +/* The bundles of the manifest at `file`: its kustomize build roots; else + * the Helm chart it sits in; else its directory. */ +static const k8s_members_t *k8s_bundles_of(k8s_bundles_t *b, const char *file, + k8s_members_t *single) { + const k8s_members_t *m = cbm_ht_get(b->members, file); + if (m && m->count > 0) { + return m; + } + char dir[CBM_SZ_1K]; + k8s_dirname(file, dir, sizeof(dir)); + char up[CBM_SZ_1K]; + snprintf(up, sizeof(up), "%s", dir); + const char *bundle = NULL; + while (!bundle) { + if (cbm_ht_get(b->charts, up)) { + bundle = k8s_bundle_name(b, "chart", up); + break; + } + if (!up[0]) { + break; + } + char parent[CBM_SZ_1K]; + k8s_dirname(up, parent, sizeof(parent)); + snprintf(up, sizeof(up), "%s", parent); + } + if (!bundle) { + bundle = k8s_bundle_name(b, "dir", dir); + } + single->items[0] = (k8s_member_t){bundle, NULL}; + single->count = bundle ? SKIP_ONE : 0; + return single; +} + +static void k8s_free_owned_key(const char *key, void *value, void *ud) { + (void)value; + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static void k8s_free_members(const char *key, void *value, void *ud) { + (void)key; + (void)ud; + k8s_members_t *m = value; + cbm_free(CBM_MEM_CLASS_OTHER, m->items); + cbm_free(CBM_MEM_CLASS_OTHER, m); +} + +static void k8s_bundles_free(k8s_bundles_t *b) { + if (b->members) { + cbm_ht_foreach(b->members, k8s_free_members, NULL); + cbm_ht_free(b->members); + } + if (b->deploys) { + cbm_ht_foreach(b->deploys, k8s_free_members, NULL); + cbm_ht_free(b->deploys); + } + if (b->names) { + cbm_ht_foreach(b->names, k8s_free_owned_key, NULL); + cbm_ht_free(b->names); + } + if (b->charts) { + cbm_ht_foreach(b->charts, k8s_free_owned_key, NULL); + cbm_ht_free(b->charts); + } + *b = (k8s_bundles_t){0}; +} + +/* A Helm chart's directory bundles the plain manifests below it. */ +static void k8s_chart_dir_add(k8s_bundles_t *b, const char *chart_yaml) { + char dir[CBM_SZ_1K]; + k8s_dirname(chart_yaml, dir, sizeof(dir)); + if (b->charts && !cbm_ht_get(b->charts, dir)) { + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir); + if (owned) { + cbm_ht_set(b->charts, owned, owned); + } + } +} + +/* Build the kustomize bundles (call once every manifest and kustomization is + * known and the IMPORTS edges exist). */ +static void k8s_bundles_build(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b) { + b->names = cbm_ht_create(0); + b->members = cbm_ht_create(0); + if (!b->charts) { + b->charts = cbm_ht_create(0); + } + if (!b->names || !b->members || !b->charts) { + return; + } + qsort(ks->items, (size_t)ks->count, sizeof(*ks->items), k8s_cmp_kust); + CBMHashTable *by_path = cbm_ht_create(0); + if (!by_path) { + return; + } + for (int i = 0; i < ks->count; i++) { + cbm_ht_set(by_path, ks->items[i].path, (void *)(intptr_t)(i + SKIP_ONE)); + } + k8s_kusts_wire(ctx, ks, by_path); + cbm_ht_free(by_path); + k8s_bundles_from_kusts(b, ks); +} + +/* A Service and a workload meet when they share a bundle whose namespaces + * agree: two set namespaces must be equal; an unset one is the deploy's. */ +static bool k8s_share_bundle(k8s_bundles_t *b, const cbm_pipeline_ctx_t *ctx, + const k8s_record_t *svc, const k8s_record_t *wl) { + const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_id(ctx->gbuf, svc->node_id); + const cbm_gbuf_node_t *wn = cbm_gbuf_find_by_id(ctx->gbuf, wl->node_id); + if (!sn || !wn || !sn->file_path || !wn->file_path) { + return false; + } + k8s_member_t one_s[SKIP_ONE]; + k8s_member_t one_w[SKIP_ONE]; + k8s_members_t single_s = {.items = one_s, .cap = SKIP_ONE}; + k8s_members_t single_w = {.items = one_w, .cap = SKIP_ONE}; + const k8s_members_t *ms = k8s_bundles_of(b, sn->file_path, &single_s); + const k8s_members_t *mw = k8s_bundles_of(b, wn->file_path, &single_w); + for (int i = 0; i < ms->count; i++) { + for (int j = 0; j < mw->count; j++) { + if (ms->items[i].bundle != mw->items[j].bundle) { + continue; + } + const char *ns_s = ms->items[i].ns ? ms->items[i].ns : svc->ns; + const char *ns_w = mw->items[j].ns ? mw->items[j].ns : wl->ns; + if (!ns_s[0] || !ns_w[0] || strcmp(ns_s, ns_w) == 0) { return true; } } @@ -327,30 +1049,1179 @@ static bool k8s_selector_matches(const k8s_record_t *svc, const k8s_record_t *wl return false; } -/* After all manifests are recorded, connect each Service to the workload(s) its - * selector targets via an INFRA_MAPS edge (Service Resource → workload Resource). */ -static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs) { - int edges = 0; - for (int i = 0; i < recs->count; i++) { - const k8s_record_t *svc = &recs->items[i]; - if (!svc->is_service || svc->n_selector == 0 || svc->node_id <= 0) { +/* The declared namespace inside a Resource QN's key ("" when `-`). */ +static void k8s_declared_ns(const char *qn, char *out, size_t out_sz) { + out[0] = '\0'; + const char *key = qn ? strstr(qn, "::") : NULL; + const char *p = key ? key + K8S_SLEN("::") : NULL; + for (int part = 0; p && part < PAIR_LEN; part++) { + p = strchr(p, '/'); + p = p ? p + SKIP_ONE : NULL; + } + const char *end = p ? strchr(p, '/') : NULL; + if (end && !(end - p == SKIP_ONE && p[0] == '-')) { + snprintf(out, out_sz, "%.*s", (int)(end - p), p); + } +} + +/* Objects a kustomize build deploys live in the namespace it gives them: + * when every bundle of the file agrees on one, that is the namespace of + * the object's logical key; bundles that disagree leave it to the deploy. */ +static void k8s_apply_effective_namespaces(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!b->members || cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &count) != 0) { + return; + } + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const k8s_members_t *m = n->file_path ? cbm_ht_get(b->members, n->file_path) : NULL; + if (!m || m->count == 0) { continue; } - for (int j = 0; j < recs->count; j++) { - const k8s_record_t *wl = &recs->items[j]; - if (i == j || !wl->is_workload || wl->node_id <= 0) { + char declared[K8S_LABEL_LEN]; + k8s_declared_ns(n->qualified_name, declared, sizeof(declared)); + const char *eff = NULL; + bool agree = true; + bool any_set = false; + for (int j = 0; j < m->count; j++) { + const char *v = m->items[j].ns ? m->items[j].ns : declared; + any_set = any_set || m->items[j].ns; + if (!v[0] || (eff && strcmp(eff, v) != 0)) { + agree = false; + } + eff = eff ? eff : v; + } + if (!any_set) { + continue; /* no build sets a namespace: the declared one stands */ + } + char *props = k8s_resource_props(n->qualified_name, agree ? eff : "?"); + if (props) { + (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)n, props); + cbm_free(CBM_MEM_CLASS_OTHER, props); + } + } +} + +/* Candidates examined by the resolvers below (pipeline_internal.h). */ +_Atomic uint64_t g_k8s_candidates = 0; + +/* ── Object references (PHASE1.md) ───────────────────────────────── */ + +/* A reference fact of a manifest, kept past its file's extraction. */ +typedef struct { + int64_t src; /* the referencing Resource */ + char *kind; /* target kind */ + char *ns; /* explicit target namespace, NULL = the source's */ + char *name; /* target metadata.name */ + const char *via; /* static: the extractor's table */ + const char *edge; /* static: "USAGE", "INFRA_MAPS" or "DEPENDS_ON" */ + char *route; /* an exposed route "" the edge leaves from, NULL = src */ +} k8s_ref_t; + +/* A CRD and the group/kind it declares. */ +typedef struct { + int64_t node; + char *group; + char *kind; +} k8s_crd_t; + +/* An object that deploys a repository directory (Flux Kustomization, Argo + * Application). */ +typedef struct { + int64_t node; + char *dir; /* normalised, "" = the repository root */ + char *ns; /* the namespace it deploys into, NULL = the objects' own */ + const char *via; /* static: "flux" / "argo" */ +} k8s_deploy_t; + +/* A HelmRelease and the chart it deploys: a repository path (Git / Bucket + * source), else a chart name of a Helm repository. */ +typedef struct { + int64_t node; + char *chart; + char *source; /* the sourceRef kind, NULL when unset */ +} k8s_hr_t; + +/* An environment variable an object's container sets (never its value). */ +typedef struct { + int64_t node; + char *key; + char *container; /* NULL: unnamed */ + const char *source; /* static: "value" or the valueFrom kind */ + int line; +} k8s_env_t; + +typedef struct { + k8s_ref_t *refs; + int ref_n; + int ref_cap; + k8s_crd_t *crds; + int crd_n; + int crd_cap; + k8s_deploy_t *deploys; + int dep_n; + int dep_cap; + k8s_hr_t *hrs; + int hr_n; + int hr_cap; + k8s_hr_t *srcs; /* an OCIRepository (node) and the URL it fetches (chart) */ + int src_n; + int src_cap; + k8s_hr_t *imgs; /* an object (node) and a container image it runs (chart) */ + int img_n; + int img_cap; + k8s_env_t *envs; + int env_n; + int env_cap; +} k8s_facts_t; + +static void k8s_facts_free(k8s_facts_t *f) { + for (int i = 0; i < f->ref_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].kind); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].ns); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].name); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].route); + } + for (int i = 0; i < f->crd_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->crds[i].group); + cbm_free(CBM_MEM_CLASS_OTHER, f->crds[i].kind); + } + for (int i = 0; i < f->dep_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->deploys[i].dir); + cbm_free(CBM_MEM_CLASS_OTHER, f->deploys[i].ns); + } + for (int i = 0; i < f->hr_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->hrs[i].chart); + cbm_free(CBM_MEM_CLASS_OTHER, f->hrs[i].source); + } + for (int i = 0; i < f->src_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->srcs[i].chart); + } + for (int i = 0; i < f->img_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->imgs[i].chart); + } + for (int i = 0; i < f->env_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, f->envs[i].key); + cbm_free(CBM_MEM_CLASS_OTHER, f->envs[i].container); + } + cbm_free(CBM_MEM_CLASS_OTHER, f->envs); + cbm_free(CBM_MEM_CLASS_OTHER, f->hrs); + cbm_free(CBM_MEM_CLASS_OTHER, f->srcs); + cbm_free(CBM_MEM_CLASS_OTHER, f->imgs); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs); + cbm_free(CBM_MEM_CLASS_OTHER, f->crds); + cbm_free(CBM_MEM_CLASS_OTHER, f->deploys); + *f = (k8s_facts_t){0}; +} + +/* A deploy path as a repository directory: "./a//b/" -> "a/b", "." -> ""; + * false for URLs, templates, variables and paths leaving the repository. */ +static bool k8s_norm_dir(const char *in, char *out, size_t out_sz) { + if (!in || strstr(in, "://") || strstr(in, "{{") || strchr(in, '$') || out_sz == 0) { + return false; + } + size_t n = 0; + out[0] = '\0'; + for (const char *p = in; *p;) { + while (*p == '/') { + p++; + } + size_t sl = strcspn(p, "/"); + if (sl == 0) { + break; + } + if (sl == SKIP_ONE && p[0] == '.') { + p += sl; + continue; + } + if (sl == PAIR_LEN && p[0] == '.' && p[SKIP_ONE] == '.') { + if (n == 0) { + return false; + } + char *slash = strrchr(out, '/'); + n = slash ? (size_t)(slash - out) : 0; + out[n] = '\0'; + p += sl; + continue; + } + if (n + SKIP_ONE + sl >= out_sz) { + return false; + } + if (n) { + out[n++] = '/'; + } + memcpy(out + n, p, sl); + n += sl; + out[n] = '\0'; + p += sl; + } + return true; +} + +/* Keep a file's facts, its def indices mapped to the Resource nodes. */ +static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64_t *node_of) { + for (int i = 0; i < res->k8s_facts.count; i++) { + const CBMK8sFact *x = &res->k8s_facts.items[i]; + if (x->def_index < 0 || x->def_index >= res->defs.count || node_of[x->def_index] <= 0) { + continue; + } + int64_t node = node_of[x->def_index]; + if (x->fact == CBM_K8S_FACT_DEPLOY && x->via && strcmp(x->via, "helmrelease") == 0) { + if (x->name && k8s_grow((void **)&f->hrs, &f->hr_cap, f->hr_n, sizeof(*f->hrs))) { + f->hrs[f->hr_n++] = + (k8s_hr_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + x->kind ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind) : NULL}; + } + } else if (x->fact == CBM_K8S_FACT_DEPLOY) { + char dir[CBM_SZ_1K]; + if (k8s_norm_dir(x->name, dir, sizeof(dir)) && + k8s_grow((void **)&f->deploys, &f->dep_cap, f->dep_n, sizeof(*f->deploys))) { + f->deploys[f->dep_n++] = (k8s_deploy_t){ + node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir), + x->ns ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->ns) : NULL, x->via}; + } + } else if (x->fact == CBM_K8S_FACT_SOURCE) { + if (x->name && k8s_grow((void **)&f->srcs, &f->src_cap, f->src_n, sizeof(*f->srcs))) { + f->srcs[f->src_n++] = + (k8s_hr_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), NULL}; + } + } else if (x->fact == CBM_K8S_FACT_IMAGE) { + if (x->name && k8s_grow((void **)&f->imgs, &f->img_cap, f->img_n, sizeof(*f->imgs))) { + f->imgs[f->img_n++] = + (k8s_hr_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), NULL}; + } + } else if (x->fact == CBM_K8S_FACT_ENV) { + if (x->name && k8s_grow((void **)&f->envs, &f->env_cap, f->env_n, sizeof(*f->envs))) { + f->envs[f->env_n++] = (k8s_env_t){ + node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + x->container ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->container) : NULL, x->via, + x->line}; + } + } else if (x->fact == CBM_K8S_FACT_CRD) { + if (k8s_grow((void **)&f->crds, &f->crd_cap, f->crd_n, sizeof(*f->crds))) { + f->crds[f->crd_n++] = + (k8s_crd_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind)}; + } + } else if (k8s_grow((void **)&f->refs, &f->ref_cap, f->ref_n, sizeof(*f->refs))) { + bool route = x->fact == CBM_K8S_FACT_ROUTE && x->route; + f->refs[f->ref_n++] = + (k8s_ref_t){.src = node, + .kind = x->kind ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->kind) : NULL, + .ns = x->ns ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->ns) : NULL, + .name = x->name ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name) : NULL, + .via = x->via, + .edge = x->edge, + .route = route ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->route) : NULL}; + } + } +} + +/* ── Deploy scopes (PHASE1 P1.3) ───────────────────────────────────── */ + +/* A Flux Kustomization or Argo Application deploys a repository directory: + * the build of a kustomization there, else every manifest below it, a + * sub-directory's kustomization by its build (as Flux generates one). The + * objects a root deploy reaches through such directories -- and the deploys + * among them, transitively -- form one deploy scope, a cluster: references + * meet across kustomize roots inside it (a Kustomization's dependsOn, a + * Certificate's ClusterIssuer). A path that names no directory of the + * repository (another source's layout) deploys nothing. */ + +typedef struct { + const char **items; /* graph-owned paths */ + int count; + int cap; +} k8s_paths_t; + +static void k8s_paths_push(k8s_paths_t *v, const char *p) { + if (k8s_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = p; + } +} + +static void k8s_paths_sort_unique(k8s_paths_t *v) { + if (v->count == 0) { + return; + } + qsort((void *)v->items, (size_t)v->count, sizeof(*v->items), k8s_cmp_str); + int w = SKIP_ONE; + for (int i = SKIP_ONE; i < v->count; i++) { + if (strcmp(v->items[w - SKIP_ONE], v->items[i]) != 0) { + v->items[w++] = v->items[i]; + } + } + v->count = w; +} + +/* The manifests the build of kustomization `root` reaches; `seen` marks the + * kustomizations visited under `mark`. */ +static void k8s_kust_closure(const k8s_kusts_t *ks, int root, int *seen, int mark, + k8s_paths_t *out) { + int *stack = NULL; + int depth = 0; + int cap = 0; + if (!k8s_grow((void **)&stack, &cap, depth, sizeof(int))) { + return; + } + stack[depth++] = root; + while (depth > 0) { + int k = stack[--depth]; + if (seen[k] == mark) { + continue; + } + seen[k] = mark; + for (int i = 0; i < ks->items[k].file_n; i++) { + k8s_paths_push(out, ks->items[k].files[i]); + } + for (int i = 0; i < ks->items[k].kid_n; i++) { + if (k8s_grow((void **)&stack, &cap, depth, sizeof(int))) { + stack[depth++] = ks->items[k].kids[i]; + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, stack); +} + +/* The manifests a deploy of directory `dir` applies; `manifests` is every + * manifest path, sorted (a directory's are one range). */ +static void k8s_dir_members(const k8s_kusts_t *ks, CBMHashTable *by_path, + const k8s_paths_t *manifests, const char *dir, int *seen, int mark, + k8s_paths_t *out, k8s_paths_t *units) { + size_t dl = strlen(dir); + int lo = 0; + int hi = dl ? manifests->count : 0; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + int c = strncmp(manifests->items[mid], dir, dl); + if (c < 0 || (c == 0 && manifests->items[mid][dl] < '/')) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (int i = lo; i < manifests->count; i++) { + const char *file = manifests->items[i]; + if (dl && (strncmp(file, dir, dl) != 0 || file[dl] != '/')) { + break; + } + /* the outermost kustomization between `dir` and the file applies it */ + char up[CBM_SZ_1K]; + k8s_dirname(file, up, sizeof(up)); + int top = 0; + for (;;) { + int k = k8s_kust_in_dir(by_path, up); + top = k ? k : top; + if (strlen(up) <= dl) { + break; + } + char parent[CBM_SZ_1K]; + k8s_dirname(up, parent, sizeof(parent)); + snprintf(up, sizeof(up), "%s", parent); + } + if (!top) { + k8s_paths_push(out, file); + if (units) { + k8s_paths_push(units, file); + } + } else if (seen[top - SKIP_ONE] != mark) { + if (units) { + k8s_paths_push(units, ks->items[top - SKIP_ONE].path); + } + k8s_kust_closure(ks, top - SKIP_ONE, seen, mark, out); + } + } + k8s_paths_sort_unique(out); +} + +/* A deploy's sort key: where it is declared, then which one. */ +typedef struct { + const char *file; + const char *qn; + const char *dir; + int deploy; +} k8s_deploy_key_t; + +static int k8s_cmp_deploy_key(const void *a, const void *b) { + const k8s_deploy_key_t *x = a; + const k8s_deploy_key_t *y = b; + int c = strcmp(x->file, y->file); + c = c ? c : strcmp(x->qn, y->qn); + c = c ? c : strcmp(x->dir, y->dir); + return c ? c : (x->deploy > y->deploy) - (x->deploy < y->deploy); +} + +/* The deploys declared in `file`: a range of `keys` (sorted by file). */ +static int k8s_deploys_in(const k8s_deploy_key_t *keys, int n, const char *file, int *first) { + int lo = 0; + int hi = n; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (strcmp(keys[mid].file, file) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + int end = lo; + while (end < n && strcmp(keys[end].file, file) == 0) { + end++; + } + *first = lo; + return end - lo; +} + +typedef struct { + int *items; + int count; + int cap; +} k8s_ints_t; + +/* Build the deploy scopes (after the kustomize bundles). */ +static void k8s_deploys_build(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b, + const k8s_facts_t *f) { + int n = f->dep_n; + const cbm_gbuf_node_t **res = NULL; + int rn = 0; + if (n == 0 || !b->names || cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &res, &rn) != 0) { + return; + } + b->deploys = cbm_ht_create(0); + k8s_paths_t manifests = {0}; + for (int i = 0; i < rn; i++) { + if (res[i]->file_path) { + k8s_paths_push(&manifests, res[i]->file_path); + } + } + k8s_paths_sort_unique(&manifests); + CBMHashTable *by_path = cbm_ht_create(0); + for (int i = 0; by_path && i < ks->count; i++) { + cbm_ht_set(by_path, ks->items[i].path, (void *)(intptr_t)(i + SKIP_ONE)); + } + size_t un = (size_t)n; + k8s_paths_t *members = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(*members)); + k8s_ints_t *adj = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(*adj)); + const cbm_gbuf_node_t **node = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(*node)); + k8s_deploy_key_t *keys = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(*keys)); + int *indeg = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(int)); + int *reached = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(int)); + int *visit = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(int)); + int *queue = cbm_calloc(CBM_MEM_CLASS_OTHER, un * sizeof(int)); + int *seen = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(ks->count ? ks->count : SKIP_ONE) * sizeof(int)); + int roots = 0; + if (b->deploys && by_path && members && adj && node && keys && indeg && reached && visit && + queue && seen) { + for (int d = 0; d < n; d++) { + node[d] = cbm_gbuf_find_by_id(ctx->gbuf, f->deploys[d].node); + keys[d] = (k8s_deploy_key_t){ + node[d] && node[d]->file_path ? node[d]->file_path : "", + node[d] && node[d]->qualified_name ? node[d]->qualified_name : "", + f->deploys[d].dir, d}; + k8s_dir_members(ks, by_path, &manifests, f->deploys[d].dir, seen, d + SKIP_ONE, + &members[d], NULL); + } + qsort(keys, un, sizeof(*keys), k8s_cmp_deploy_key); + /* deploy d reaches deploy e when e sits in a manifest d applies */ + for (int d = 0; d < n; d++) { + for (int i = 0; i < members[d].count; i++) { + int first = 0; + int cnt = k8s_deploys_in(keys, n, members[d].items[i], &first); + for (int j = first; j < first + cnt; j++) { + int e = keys[j].deploy; + if (e != d && + k8s_grow((void **)&adj[d].items, &adj[d].cap, adj[d].count, sizeof(int))) { + adj[d].items[adj[d].count++] = e; + indeg[e]++; + } + } + } + } + /* roots: deploys no other reaches; then, in order, any a cycle hides */ + for (int pass = 0; pass < PAIR_LEN; pass++) { + for (int oi = 0; oi < n; oi++) { + int r = keys[oi].deploy; + if (reached[r] || (pass == 0 && indeg[r] != 0)) { + continue; + } + const char *bundle = k8s_bundle_name( + b, "deploy", node[r] ? node[r]->qualified_name : f->deploys[r].dir); + if (!bundle) { + continue; + } + roots++; + int mark = pass * n + oi + SKIP_ONE; + int qh = 0; + int qt = 0; + queue[qt++] = r; + visit[r] = mark; + while (qh < qt) { + int d = queue[qh++]; + reached[d] = SKIP_ONE; + if (node[d] && node[d]->file_path) { + k8s_member_add(b->deploys, node[d]->file_path, bundle, NULL); + } + for (int i = 0; i < members[d].count; i++) { + k8s_member_add(b->deploys, members[d].items[i], bundle, f->deploys[d].ns); + } + for (int i = 0; i < adj[d].count; i++) { + int e = adj[d].items[i]; + if (visit[e] != mark) { + visit[e] = mark; + queue[qt++] = e; + } + } + } + } + } + } + for (int d = 0; members && d < n; d++) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)members[d].items); + } + for (int d = 0; adj && d < n; d++) { + cbm_free(CBM_MEM_CLASS_OTHER, adj[d].items); + } + cbm_free(CBM_MEM_CLASS_OTHER, members); + cbm_free(CBM_MEM_CLASS_OTHER, adj); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)node); + cbm_free(CBM_MEM_CLASS_OTHER, keys); + cbm_free(CBM_MEM_CLASS_OTHER, indeg); + cbm_free(CBM_MEM_CLASS_OTHER, reached); + cbm_free(CBM_MEM_CLASS_OTHER, visit); + cbm_free(CBM_MEM_CLASS_OTHER, queue); + cbm_free(CBM_MEM_CLASS_OTHER, seen); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)manifests.items); + if (by_path) { + cbm_ht_free(by_path); + } + cbm_log_info("pass.k8s.deploys", "deploys", itoa_k8s(n), "roots", itoa_k8s(roots)); +} + +/* ── Bundle-keyed candidate indexes ───────────────────────────────── */ + +/* Candidates are looked up per bundle the source shares, never by name + * across the repository: copies of one name in independent bundles would + * otherwise make every lookup visit every copy (quadratic in copies). */ + +typedef struct { + int64_t *items; + int count; + int cap; +} k8s_ids_t; + +static void k8s_ids_push(k8s_ids_t *v, int64_t id) { + if (k8s_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = id; + } +} + +static int k8s_cmp_id(const void *a, const void *b) { + int64_t x = *(const int64_t *)a; + int64_t y = *(const int64_t *)b; + return (x > y) - (x < y); +} + +static void k8s_ids_sort_unique(k8s_ids_t *v) { + if (v->count < PAIR_LEN) { + return; + } + qsort(v->items, (size_t)v->count, sizeof(*v->items), k8s_cmp_id); + int w = SKIP_ONE; + for (int i = SKIP_ONE; i < v->count; i++) { + if (v->items[i] != v->items[w - SKIP_ONE]) { + v->items[w++] = v->items[i]; + } + } + v->count = w; +} + +static void k8s_free_ids_entry(const char *key, void *value, void *ud) { + (void)ud; + k8s_ids_t *v = value; + cbm_free(CBM_MEM_CLASS_OTHER, v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +/* "\x1f": an interned bundle (by address) with an object + * name, a type or a label pair. */ +static void k8s_bkey(char *out, size_t out_sz, const char *bundle, const char *name) { + snprintf(out, out_sz, "%p\x1f%s", (const void *)bundle, name); +} + +static void k8s_index_add(CBMHashTable *ix, const char *key, int64_t id) { + k8s_ids_t *v = cbm_ht_get(ix, key); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!v || !owned) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, owned); + return; + } + cbm_ht_set(ix, owned, v); + } + if (v->count == 0 || v->items[v->count - SKIP_ONE] != id) { + k8s_ids_push(v, id); + } +} + +/* Calls `fn(bundle, ud)` for every bundle of `file`: its build roots / + * chart / dir and, when `deploys`, the deploy scopes reaching it. */ +typedef void (*k8s_scope_fn)(const char *bundle, void *ud); + +static void k8s_each_scope(k8s_bundles_t *b, const char *file, bool deploys, k8s_scope_fn fn, + void *ud) { + k8s_member_t one[SKIP_ONE]; + k8s_members_t single = {.items = one, .cap = SKIP_ONE}; + const k8s_members_t *m = k8s_bundles_of(b, file, &single); + for (int i = 0; i < m->count; i++) { + fn(m->items[i].bundle, ud); + } + const k8s_members_t *d = deploys && b->deploys ? cbm_ht_get(b->deploys, file) : NULL; + for (int i = 0; d && i < d->count; i++) { + fn(d->items[i].bundle, ud); + } +} + +typedef struct { + CBMHashTable *ix; + const char *name; + int64_t id; + k8s_ids_t *found; +} k8s_scope_job_t; + +static void k8s_scope_add(const char *bundle, void *ud) { + k8s_scope_job_t *j = ud; + char key[CBM_SZ_1K]; + k8s_bkey(key, sizeof(key), bundle, j->name); + k8s_index_add(j->ix, key, j->id); +} + +static void k8s_scope_find(const char *bundle, void *ud) { + k8s_scope_job_t *j = ud; + char key[CBM_SZ_1K]; + k8s_bkey(key, sizeof(key), bundle, j->name); + const k8s_ids_t *v = cbm_ht_get(j->ix, key); + for (int i = 0; v && i < v->count; i++) { + k8s_ids_push(j->found, v->items[i]); + } +} + +/* Index every Resource by "\x1f" over its bundles (and + * deploy scopes when `deploys`). */ +static CBMHashTable *k8s_index_objects(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, bool deploys) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!ix || cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &count) != 0) { + return ix; + } + for (int i = 0; i < count; i++) { + if (nodes[i]->file_path && nodes[i]->name) { + k8s_scope_job_t j = {ix, nodes[i]->name, nodes[i]->id, NULL}; + k8s_each_scope(b, nodes[i]->file_path, deploys, k8s_scope_add, &j); + } + } + return ix; +} + +/* The ids `name` has in any scope of `file`, sorted, unique. */ +static void k8s_find_in_scopes(k8s_bundles_t *b, CBMHashTable *ix, const char *file, + const char *name, bool deploys, k8s_ids_t *found) { + found->count = 0; + k8s_scope_job_t j = {ix, name, 0, found}; + k8s_each_scope(b, file, deploys, k8s_scope_find, &j); + k8s_ids_sort_unique(found); +} + +/* The namespace of a Resource's logical key ("" when unknown `?` or + * cluster-scoped `_`), else its declared one. */ +static void k8s_effective_ns(const cbm_gbuf_node_t *n, char *out, size_t out_sz) { + out[0] = '\0'; + const char *k = n->properties_json ? strstr(n->properties_json, "\"obj_key\":\"") : NULL; + if (!k) { + k8s_declared_ns(n->qualified_name, out, out_sz); + return; + } + k += K8S_SLEN("\"obj_key\":\""); + for (int part = 0; k && part < PAIR_LEN; part++) { /* past group and kind */ + k = strchr(k, '/'); + k = k ? k + SKIP_ONE : NULL; + } + const char *end = k ? strchr(k, '/') : NULL; + if (end && !(end - k == SKIP_ONE && (k[0] == '?' || k[0] == '_'))) { + snprintf(out, out_sz, "%.*s", (int)(end - k), k); + } +} + +/* Does the target at `tgt` answer `ref` from `src`: one bundle, and the + * namespace the reference asks for (explicit, else the source's own there) + * agrees with the target's there? A cluster-scoped kind has no namespace. */ +static bool k8s_ref_meets(k8s_bundles_t *b, const k8s_ref_t *ref, const cbm_gbuf_node_t *src, + const cbm_gbuf_node_t *tgt) { + k8s_member_t one_s[SKIP_ONE]; + k8s_member_t one_t[SKIP_ONE]; + k8s_members_t single_s = {.items = one_s, .cap = SKIP_ONE}; + k8s_members_t single_t = {.items = one_t, .cap = SKIP_ONE}; + const k8s_members_t *ms = k8s_bundles_of(b, src->file_path, &single_s); + const k8s_members_t *mt = k8s_bundles_of(b, tgt->file_path, &single_t); + char src_ns[K8S_LABEL_LEN]; + char tgt_ns[K8S_LABEL_LEN]; + k8s_declared_ns(src->qualified_name, src_ns, sizeof(src_ns)); + k8s_declared_ns(tgt->qualified_name, tgt_ns, sizeof(tgt_ns)); + bool cluster = k8s_cluster_scoped(ref->kind); + for (int i = 0; i < ms->count; i++) { + for (int j = 0; j < mt->count; j++) { + if (ms->items[i].bundle != mt->items[j].bundle) { continue; } - if (k8s_selector_matches(svc, wl)) { + const char *want = ref->ns ? ref->ns : ms->items[i].ns ? ms->items[i].ns : src_ns; + const char *have = mt->items[j].ns ? mt->items[j].ns : tgt_ns; + if (cluster || !want[0] || !have[0] || strcmp(want, have) == 0) { + return true; + } + } + } + /* or one deploy scope: the cluster a Flux / Argo root deploys */ + const k8s_members_t *ds = b->deploys ? cbm_ht_get(b->deploys, src->file_path) : NULL; + const k8s_members_t *dt = ds ? cbm_ht_get(b->deploys, tgt->file_path) : NULL; + if (!dt) { + return false; + } + k8s_effective_ns(src, src_ns, sizeof(src_ns)); + k8s_effective_ns(tgt, tgt_ns, sizeof(tgt_ns)); + for (int i = 0; i < ds->count; i++) { + for (int j = 0; j < dt->count; j++) { + if (ds->items[i].bundle != dt->items[j].bundle) { + continue; + } + const char *want = ref->ns ? ref->ns : ds->items[i].ns ? ds->items[i].ns : src_ns; + const char *have = dt->items[j].ns ? dt->items[j].ns : tgt_ns; + if (cluster || !want[0] || !have[0] || strcmp(want, have) == 0) { + return true; + } + } + } + return false; +} + +/* The Route node of a route `src` (an Ingress, an HTTPRoute) exposes, which + * `src` DEFINES; 0 on failure. Its QN is the route itself, so every object + * exposing one host and path shares it. */ +static int64_t k8s_route_node(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *src, + const k8s_ref_t *ref) { + enum { K8S_ROUTE_ESC = 6 * CBM_SZ_256 }; + const char *slash = strchr(ref->route, '/'); + if (!slash) { + return 0; + } + char qn[CBM_SZ_1K]; + char host[CBM_SZ_256]; + char esc_host[K8S_ROUTE_ESC]; + char esc_path[K8S_ROUTE_ESC]; + snprintf(qn, sizeof(qn), "__route__ingress__%s", ref->route); + snprintf(host, sizeof(host), "%.*s", (int)(slash - ref->route), ref->route); + cbm_json_escape(esc_host, sizeof(esc_host), host); + cbm_json_escape(esc_path, sizeof(esc_path), slash); + char props[2 * K8S_ROUTE_ESC + CBM_SZ_256]; + snprintf(props, sizeof(props), + "{\"source\":\"k8s\",\"via\":\"%s\",\"host\":\"%s\",\"path\":\"%s\"}", ref->via, + esc_host, esc_path); + int64_t id = cbm_gbuf_upsert_node(ctx->gbuf, "Route", ref->route, qn, src->file_path, + src->start_line, src->end_line, props); + if (id > 0) { + cbm_gbuf_insert_edge(ctx->gbuf, src->id, id, "DEFINES", "{}"); + } + return id; +} + +/* Draw every reference to the object(s) it names inside its bundle; + * an unresolved reference draws nothing. An exposed route's reference + * leaves from its Route node. */ +static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_facts_t *f) { + int edges = 0; + int unresolved = 0; + CBMHashTable *ix = f->ref_n > 0 ? k8s_index_objects(ctx, b, true) : NULL; + k8s_ids_t found = {0}; + const cbm_gbuf_node_t **cands = NULL; + int cand_cap = 0; + for (int i = 0; ix && i < f->ref_n; i++) { + const k8s_ref_t *ref = &f->refs[i]; + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, ref->src); + int64_t from = src ? src->id : 0; + if (ref->route) { + from = src && src->file_path ? k8s_route_node(ctx, src, ref) : 0; + if (from <= 0 || !ref->kind || !ref->name) { + continue; + } + } + char name[CBM_SZ_1K]; + snprintf(name, sizeof(name), "%s/%s", ref->kind, ref->name); + int n = 0; + if (src && src->file_path) { + k8s_find_in_scopes(b, ix, src->file_path, name, true, &found); + for (int c = 0; c < found.count; c++) { + const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(ctx->gbuf, found.items[c]); + if (t && k8s_grow((void **)&cands, &cand_cap, n, sizeof(*cands))) { + cands[n++] = t; + } + } + } + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)(n), memory_order_relaxed); + char props[CBM_SZ_256]; + snprintf(props, sizeof(props), "{\"%s\":\"%s\"}", + strcmp(ref->edge, "INFRA_MAPS") == 0 ? "kind" + : strcmp(ref->edge, "DEPENDS_ON") == 0 ? "via" + : "ref", + ref->via); + /* A name is one object per kind and namespace, so two declarations + * of it in one bundle are alternatives (versioned manifests side by + * side): the declaration in the reference's own file, when there is + * one, is the one it means. */ + bool own_file = false; + for (int c = 0; c < n && !own_file; c++) { + const cbm_gbuf_node_t *t = cands[c]; + own_file = t != src && t->label && strcmp(t->label, "Resource") == 0 && t->file_path && + strcmp(t->file_path, src->file_path) == 0 && k8s_ref_meets(b, ref, src, t); + } + int hits = 0; + for (int c = 0; c < n; c++) { + const cbm_gbuf_node_t *t = cands[c]; + if (t == src || !t->label || strcmp(t->label, "Resource") != 0 || !t->file_path || + (own_file && strcmp(t->file_path, src->file_path) != 0) || + !k8s_ref_meets(b, ref, src, t)) { + continue; + } + cbm_gbuf_insert_edge(ctx->gbuf, from, t->id, ref->edge, props); + hits++; + } + edges += hits; + unresolved += hits == 0; + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)cands); + cbm_free(CBM_MEM_CLASS_OTHER, found.items); + if (ix) { + cbm_ht_foreach(ix, k8s_free_ids_entry, NULL); + cbm_ht_free(ix); + } + if (f->ref_n > 0) { + cbm_log_info("pass.k8s.refs", "refs", itoa_k8s(f->ref_n), "edges", itoa_k8s(edges), + "unresolved", itoa_k8s(unresolved)); + } +} + +/* A CRD's group/kind, for grouping the CRDs of one type. */ +typedef struct { + char key[CBM_SZ_512]; + int crd; +} k8s_crd_key_t; + +static int k8s_cmp_crd_key(const void *a, const void *b) { + const k8s_crd_key_t *x = a; + const k8s_crd_key_t *y = b; + int c = strcmp(x->key, y->key); + return c ? c : (x->crd > y->crd) - (x->crd < y->crd); +} + +/* A custom resource is an instance of the CRD declaring its group and + * kind. A CRD vendored in several places: the copy deployed with the + * resource (one bundle) is the one; with no copy beside it, the resource + * stands for all of them. */ +/* A CRD node says which group/kind it declares (search, and the Go operator + * pass's CRD -> type link). */ +static void k8s_mark_crd(cbm_pipeline_ctx_t *ctx, const k8s_crd_t *crd) { + cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)cbm_gbuf_find_by_id(ctx->gbuf, crd->node); + const char *props = n ? n->properties_json : NULL; + size_t pl = props ? strlen(props) : 0; + if (!props || pl < PAIR_LEN || props[pl - SKIP_ONE] != '}' || strstr(props, "\"crd_kind\"")) { + return; + } + char group[K8S_LABEL_LEN * PAIR_LEN]; + char kind[K8S_LABEL_LEN * PAIR_LEN]; + cbm_json_escape(group, (int)sizeof(group), crd->group); + cbm_json_escape(kind, (int)sizeof(kind), crd->kind); + size_t n_out = pl + strlen(group) + strlen(kind) + CBM_SZ_64; + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, n_out); + if (out) { + snprintf(out, n_out, "%.*s,\"crd_group\":\"%s\",\"crd_kind\":\"%s\"}", (int)(pl - SKIP_ONE), + props, group, kind); + (void)cbm_gbuf_node_set_properties_json(n, out); + cbm_free(CBM_MEM_CLASS_OTHER, out); + } +} + +static void k8s_link_crds(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8s_facts_t *f) { + if (f->crd_n == 0) { + return; + } + for (int i = 0; i < f->crd_n; i++) { + k8s_mark_crd(ctx, &f->crds[i]); + } + /* group/kind -> the run of its CRDs in `order` (sorted by group/kind), so + * each resource visits only the CRDs of its own type. */ + k8s_crd_key_t *order = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)f->crd_n * sizeof(*order)); + CBMHashTable *by_type = cbm_ht_create(0); + if (!order || !by_type) { + cbm_free(CBM_MEM_CLASS_OTHER, order); + if (by_type) { + cbm_ht_free(by_type); + } + return; + } + for (int i = 0; i < f->crd_n; i++) { + snprintf(order[i].key, sizeof(order[i].key), "%s/%s", f->crds[i].group, f->crds[i].kind); + order[i].crd = i; + } + qsort(order, (size_t)f->crd_n, sizeof(*order), k8s_cmp_crd_key); + for (int i = f->crd_n - SKIP_ONE; i >= 0; i--) { + cbm_ht_set(by_type, order[i].key, (void *)(intptr_t)(i + SKIP_ONE)); /* first of run */ + } + /* "\x1f" -> the CRDs of that type in that bundle */ + CBMHashTable *in_bundle = cbm_ht_create(0); + for (int i = 0; in_bundle && i < f->crd_n; i++) { + const cbm_gbuf_node_t *crd = cbm_gbuf_find_by_id(ctx->gbuf, f->crds[order[i].crd].node); + if (crd && crd->file_path) { + k8s_scope_job_t j = {in_bundle, order[i].key, crd->id, NULL}; + k8s_each_scope(b, crd->file_path, false, k8s_scope_add, &j); + } + } + k8s_ids_t beside = {0}; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + int edges = 0; + if (in_bundle && cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const char *key = + nodes[i]->qualified_name ? strstr(nodes[i]->qualified_name, "::") : NULL; + if (!key || !nodes[i]->file_path) { + continue; + } + key += K8S_SLEN("::"); + const char *kind_end = strchr(key, '/'); + kind_end = kind_end ? strchr(kind_end + SKIP_ONE, '/') : NULL; + if (!kind_end) { + continue; + } + char type[CBM_SZ_512]; + snprintf(type, sizeof(type), "%.*s", (int)(kind_end - key), key); + int first = (int)(intptr_t)cbm_ht_get(by_type, type) - SKIP_ONE; + if (first < 0) { + continue; + } + k8s_find_in_scopes(b, in_bundle, nodes[i]->file_path, type, false, &beside); + if (beside.count > 0) { + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)beside.count, + memory_order_relaxed); + for (int r = 0; r < beside.count; r++) { + if (beside.items[r] != nodes[i]->id) { + cbm_gbuf_insert_edge(ctx->gbuf, nodes[i]->id, beside.items[r], "USAGE", + "{\"ref\":\"crd\"}"); + edges++; + } + } + continue; + } + int end = first; + while (end < f->crd_n && strcmp(order[end].key, type) == 0) { + end++; + } + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)(end - first), + memory_order_relaxed); + for (int r = first; r < end; r++) { + int64_t crd_id = f->crds[order[r].crd].node; + if (crd_id != nodes[i]->id) { + cbm_gbuf_insert_edge(ctx->gbuf, nodes[i]->id, crd_id, "USAGE", + "{\"ref\":\"crd\"}"); + edges++; + } + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, beside.items); + if (in_bundle) { + cbm_ht_foreach(in_bundle, k8s_free_ids_entry, NULL); + cbm_ht_free(in_bundle); + } + cbm_ht_free(by_type); + cbm_free(CBM_MEM_CLASS_OTHER, order); + if (edges > 0) { + cbm_log_info("pass.k8s.crds", "instances", itoa_k8s(edges)); + } +} + +/* Selector postings keyed by "\x1f" (owned keys): the + * workloads (record indices) of one bundle carrying one label pair. */ +static void k8s_posting_free_owned(const char *key, void *value, void *userdata) { + k8s_posting_free(key, value, userdata); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +typedef struct { + CBMHashTable *index; + const k8s_record_array_t *recs; + const k8s_record_t *rec; + int rec_idx; + k8s_ids_t *cand; /* record indices */ +} k8s_sel_job_t; + +static void k8s_sel_post(const char *bundle, void *ud) { + k8s_sel_job_t *j = ud; + for (int p = 0; p < j->rec->pair_n; p++) { + char key[CBM_SZ_1K]; + k8s_bkey(key, sizeof(key), bundle, j->recs->pairs[j->rec->pair_off + p]); + if (!cbm_ht_get(j->index, key)) { + k8s_posting_t *posting = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*posting)); + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!posting || !owned) { + cbm_free(CBM_MEM_CLASS_OTHER, posting); + cbm_free(CBM_MEM_CLASS_OTHER, owned); + continue; + } + cbm_ht_set(j->index, owned, posting); + } + (void)k8s_posting_add(j->index, key, j->rec_idx); + } +} + +/* The workloads of `bundle` carrying the selector's rarest pair there. */ +static void k8s_sel_rarest(const char *bundle, void *ud) { + k8s_sel_job_t *j = ud; + const k8s_posting_t *rarest = NULL; + for (int p = 0; p < j->rec->pair_n; p++) { + char key[CBM_SZ_1K]; + k8s_bkey(key, sizeof(key), bundle, j->recs->pairs[j->rec->pair_off + p]); + const k8s_posting_t *posting = cbm_ht_get(j->index, key); + if (!posting) { + return; + } + if (!rarest || posting->count < rarest->count) { + rarest = posting; + } + } + for (int c = 0; rarest && c < rarest->count; c++) { + k8s_ids_push(j->cand, rarest->items[c]); + } +} + +/* A workload a selector matched: its object name, declaring file and + * whether that is the selector's file. */ +typedef struct { + const char *name; /* "Kind/name" (graph-owned) */ + const char *ns; + bool own; + int rec; +} k8s_match_t; + +static int k8s_cmp_match(const void *a, const void *b) { + const k8s_match_t *x = a; + const k8s_match_t *y = b; + int c = strcmp(x->name, y->name); + c = c ? c : strcmp(x->ns, y->ns); + return c ? c : (x->rec > y->rec) - (x->rec < y->rec); +} + +/* Draw the selector's edge to each match. A name is one object per kind and + * namespace, so matches declaring one name in several files are alternatives + * (versioned manifests side by side): the one in the selector's file, when + * there is one, is the one it selects. */ +static int k8s_draw_matches(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs, + const k8s_record_t *svc, k8s_match_t *m, int n) { + qsort(m, (size_t)n, sizeof(*m), k8s_cmp_match); + int edges = 0; + for (int g = 0; g < n;) { + int end = g; + bool own = false; + while (end < n && strcmp(m[end].name, m[g].name) == 0 && strcmp(m[end].ns, m[g].ns) == 0) { + own = own || m[end].own; + end++; + } + for (int i = g; i < end; i++) { + if (own && !m[i].own) { + continue; + } + const k8s_record_t *wl = &recs->items[m[i].rec]; + if (svc->kind == K8S_REC_SERVICE) { char props[CBM_SZ_256]; snprintf(props, sizeof(props), "{\"kind\":\"selector\",\"service\":\"%s\",\"workload\":\"%s\"}", svc->name, wl->name); cbm_gbuf_insert_edge(ctx->gbuf, svc->node_id, wl->node_id, "INFRA_MAPS", props); - edges++; + } else { + cbm_gbuf_insert_edge(ctx->gbuf, svc->node_id, wl->node_id, "USAGE", + svc->kind == K8S_REC_PDB ? "{\"ref\":\"selector\"}" + : "{\"ref\":\"podSelector\"}"); } + edges++; + } + g = end; + } + return edges; +} + +/* Connect each Service to the workloads its selector targets via an INFRA_MAPS + * edge (Service Resource → workload Resource), inside a shared bundle; a + * PodDisruptionBudget or NetworkPolicy to the workloads it selects (USAGE). */ +static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t *recs, + k8s_bundles_t *bundles) { + CBMHashTable *index = cbm_ht_create(0); + if (!index) { + return; + } + k8s_match_t *matches = NULL; + int match_cap = 0; + for (int i = 0; i < recs->count; i++) { + const k8s_record_t *wl = &recs->items[i]; + const cbm_gbuf_node_t *wn = cbm_gbuf_find_by_id(ctx->gbuf, wl->node_id); + if ((wl->kind == K8S_REC_WORKLOAD || wl->kind == K8S_REC_POD) && wn && wn->file_path) { + k8s_sel_job_t j = {index, recs, wl, i, NULL}; + k8s_each_scope(bundles, wn->file_path, false, k8s_sel_post, &j); } } + k8s_ids_t cand = {0}; + int edges = 0; + for (int i = 0; i < recs->count; i++) { + /* a Service, PodDisruptionBudget or NetworkPolicy selector; one with + * matchExpressions selects fewer pods than its matchLabels, and an + * empty one every pod of the namespace: neither is drawn */ + const k8s_record_t *svc = &recs->items[i]; + if ((svc->kind != K8S_REC_SERVICE && svc->kind != K8S_REC_PDB && + svc->kind != K8S_REC_NETPOL) || + svc->pair_n == 0 || svc->has_expr) { + continue; + } + /* Candidates: per bundle of the selector, the workloads there + * carrying its rarest pair. */ + const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_id(ctx->gbuf, svc->node_id); + cand.count = 0; + if (sn && sn->file_path) { + k8s_sel_job_t j = {index, recs, svc, i, &cand}; + k8s_each_scope(bundles, sn->file_path, false, k8s_sel_rarest, &j); + } + k8s_ids_sort_unique(&cand); + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)cand.count, memory_order_relaxed); + int n = 0; + for (int c = 0; c < cand.count; c++) { + int rec = (int)cand.items[c]; + const k8s_record_t *wl = &recs->items[rec]; + bool all = k8s_share_bundle(bundles, ctx, svc, wl); + for (int p = 0; all && p < svc->pair_n; p++) { + all = k8s_has_pair(recs, wl, recs->pairs[svc->pair_off + p]); + } + const cbm_gbuf_node_t *wn = all ? cbm_gbuf_find_by_id(ctx->gbuf, wl->node_id) : NULL; + if (wn && wn->name && wn->file_path && + k8s_grow((void **)&matches, &match_cap, n, sizeof(*matches))) { + matches[n++] = + (k8s_match_t){wn->name, wl->ns, strcmp(wn->file_path, sn->file_path) == 0, rec}; + } + } + edges += k8s_draw_matches(ctx, recs, svc, matches, n); + } + cbm_free(CBM_MEM_CLASS_OTHER, matches); + cbm_free(CBM_MEM_CLASS_OTHER, cand.items); + cbm_ht_foreach(index, k8s_posting_free_owned, NULL); + cbm_ht_free(index); if (edges > 0) { cbm_log_info("pass.k8s.selectors", "linked", itoa_k8s(edges)); } @@ -359,11 +2230,11 @@ static void k8s_link_selectors(cbm_pipeline_ctx_t *ctx, const k8s_record_array_t /* ── K8s manifest handler ────────────────────────────────────────── */ /* source/src_len are the already-read file bytes (caller retains ownership and - * must free after this call returns). When `rec` is non-NULL it is populated - * with the first Resource's node id, name and label/selector values for later - * cross-manifest selector matching. */ + * must free after this call returns). Every Resource's own document is + * recorded in `recs` for cross-manifest selector matching. */ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, - const CBMFileResult *res, k8s_record_t *rec) { + const CBMFileResult *res, k8s_record_array_t *recs, + k8s_facts_t *facts) { int resource_count = 0; if (!res) { @@ -375,6 +2246,9 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; free(file_qn); + int64_t *node_of = + cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)(res->defs.count ? res->defs.count : SKIP_ONE) * sizeof(int64_t)); for (int d = 0; d < res->defs.count; d++) { CBMDefinition *def = &res->defs.items[d]; if (!def->label || strcmp(def->label, "Resource") != 0) { @@ -384,26 +2258,30 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c continue; } - int64_t node_id = - cbm_gbuf_upsert_node(ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, - (int)def->start_line, (int)def->end_line, "{\"source\":\"k8s\"}"); + char *props = k8s_resource_props(def->qualified_name, NULL); + int64_t node_id = cbm_gbuf_upsert_node( + ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, (int)def->start_line, + (int)def->end_line, props ? props : "{\"source\":\"k8s\"}"); + cbm_free(CBM_MEM_CLASS_OTHER, props); /* DEFINES edge: File → Resource */ if (file_node && node_id > 0) { cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}"); } - /* Capture the first Resource for cross-manifest selector matching. */ - if (rec && rec->node_id <= 0 && node_id > 0) { - rec->node_id = node_id; + /* Record its selector / pod-label pairs for Service → workload linking. */ + if (node_id > 0 && source) { + k8s_record_document(recs, node_id, source, (int)def->start_line, (int)def->end_line); + } + if (node_of) { + node_of[d] = node_id; } resource_count++; } - - /* Record selector / pod-label values for later Service → workload linking. */ - if (rec && rec->node_id > 0) { - k8s_scan_labels(source, rec); + if (node_of) { + k8s_facts_keep(facts, res, node_of); + cbm_free(CBM_MEM_CLASS_OTHER, node_of); } cbm_log_info("pass.k8s.manifest", "file", rel_path, "resources", itoa_k8s(resource_count)); @@ -415,17 +2293,133 @@ static bool is_helm_chart_file(const char *base) { return strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0; } -/* Emit a Chart node for a Chart.yaml and a DEPENDS_ON edge to a (shared, - * deduplicated) Chart node per declared dependency (#338). */ -static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source) { +/* A chart's `file://` dependency: the chart in that directory of this + * repository. Linked once every Chart.yaml has been read, so a dependency + * indexed after its dependent still resolves. One that names no chart of this + * repository (an absolute build-container path, a directory without a chart) + * stays a declared dependency on the external node, like a remote one. */ +typedef struct { + int64_t chart_id; + char name[CBM_HELM_NAME_MAX]; /* dependency name, for the external fallback */ + char *chart_rel; /* the depending Chart.yaml */ + char *target_qn; /* QN of the depended-on Chart node; NULL = unresolvable */ + char *props; /* edge properties */ +} k8s_local_dep_t; + +typedef struct { + k8s_local_dep_t *items; + int count; + int cap; +} k8s_local_deps_t; + +static bool k8s_is_file_repository(const char *repository) { + return strncmp(repository, "file://", K8S_SLEN("file://")) == 0; +} + +static char *k8s_dep_props(const char *repository) { + char escaped[CBM_HELM_REPO_MAX * 2]; + cbm_json_escape(escaped, (int)sizeof(escaped), repository); + size_t n = strlen(escaped) + sizeof("{\"repository\":\"\"}"); + char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, n); + if (props) { + snprintf(props, n, "{\"repository\":\"%s\"}", escaped); + } + return props; +} + +/* DEPENDS_ON from `chart_id` to the shared external Chart node of `name` + * (one per name and project, so charts depending on the same chart meet). */ +static bool k8s_external_dep(cbm_pipeline_ctx_t *ctx, int64_t chart_id, const char *name, + const char *chart_rel, const char *props) { + char dep_qn[CBM_SZ_512]; + snprintf(dep_qn, sizeof(dep_qn), "%s.__helm_dep__.%s", ctx->project_name, name); + int64_t dep_id = cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, dep_qn, chart_rel, SKIP_ONE, 0, + "{\"source\":\"helm\",\"external\":true}"); + if (dep_id <= 0) { + return false; + } + cbm_gbuf_insert_edge(ctx->gbuf, chart_id, dep_id, "DEPENDS_ON", props ? props : "{}"); + return true; +} + +/* QN of the chart a relative `file://` directory names, core-owned; NULL for + * an absolute path (it names the build host, not this repo). */ +static char *k8s_local_chart_qn(const char *project, const char *chart_rel, const char *dir) { + if (!dir[0] || dir[0] == '/') { + return NULL; + } + char rel[CBM_HELM_REPO_MAX + K8S_SLEN("./") + K8S_SLEN("/Chart.yaml")]; + snprintf(rel, sizeof(rel), "%s%s/Chart.yaml", strncmp(dir, "../", 3) == 0 ? "" : "./", dir); + /* fqn.c hands out libc-allocated strings: copy into the core, release them. */ + char *path = cbm_pipeline_resolve_relative_import(chart_rel, rel); + char *qn = path ? cbm_infra_qn(project, path, "helm-chart", NULL) : NULL; + free(path); + char *owned = qn ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, qn) : NULL; + free(qn); + return owned; +} + +static void k8s_local_deps_push(k8s_local_deps_t *deps, int64_t chart_id, const char *chart_rel, + const char *project, const cbm_helm_dep_t *dep) { + if (deps->count == deps->cap) { + int cap = deps->cap ? deps->cap * 2 : K8S_GROW_MIN; + k8s_local_dep_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, deps->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + deps->items = grown; + deps->cap = cap; + } + k8s_local_dep_t *d = &deps->items[deps->count++]; + *d = (k8s_local_dep_t){.chart_id = chart_id}; + d->target_qn = k8s_local_chart_qn(project, chart_rel, dep->repository + K8S_SLEN("file://")); + snprintf(d->name, sizeof(d->name), "%s", dep->name); + d->chart_rel = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, chart_rel); + d->props = k8s_dep_props(dep->repository); +} + +static void k8s_local_deps_free(k8s_local_deps_t *deps) { + for (int i = 0; i < deps->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, deps->items[i].target_qn); + cbm_free(CBM_MEM_CLASS_OTHER, deps->items[i].chart_rel); + cbm_free(CBM_MEM_CLASS_OTHER, deps->items[i].props); + } + cbm_free(CBM_MEM_CLASS_OTHER, deps->items); + *deps = (k8s_local_deps_t){0}; +} + +static void k8s_link_local_deps(cbm_pipeline_ctx_t *ctx, k8s_local_deps_t *deps) { + for (int i = 0; i < deps->count; i++) { + const k8s_local_dep_t *d = &deps->items[i]; + const cbm_gbuf_node_t *target = + d->target_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, d->target_qn) : NULL; + if (target && target->label && strcmp(target->label, "Chart") == 0 && + target->id != d->chart_id) { + cbm_gbuf_insert_edge(ctx->gbuf, d->chart_id, target->id, "DEPENDS_ON", d->props); + } else if (d->chart_rel) { + (void)k8s_external_dep(ctx, d->chart_id, d->name, d->chart_rel, d->props); + } + } + k8s_local_deps_free(deps); +} + +/* Emit a Chart node for a Chart.yaml and a DEPENDS_ON edge per declared + * dependency (#338): to the chart in this repository for a resolvable + * `file://` dependency (once all charts are read), otherwise to a shared + * external Chart node per dependency name, the repository on the edge. */ +static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, + k8s_local_deps_t *local_deps) { cbm_helm_chart_t hc; if (cbm_parse_helm_chart(source, &hc) != 0) { + cbm_helm_chart_free(&hc); return; } const char *cname = hc.chart_name[0] ? hc.chart_name : k8s_basename(rel_path); char *chart_qn = cbm_infra_qn(ctx->project_name, rel_path, "helm-chart", NULL); if (!chart_qn) { + cbm_helm_chart_free(&hc); return; } int64_t chart_id = cbm_gbuf_upsert_node(ctx->gbuf, "Chart", cname, chart_qn, rel_path, SKIP_ONE, @@ -441,18 +2435,16 @@ static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, con int dep_edges = 0; for (int i = 0; i < hc.dep_count && chart_id > 0; i++) { - /* Stable per-project QN so multiple charts depending on the same chart - * link to one shared dependency node. */ - char dep_qn[CBM_SZ_512]; - snprintf(dep_qn, sizeof(dep_qn), "%s.__helm_dep__.%s", ctx->project_name, hc.deps[i]); - int64_t dep_id = - cbm_gbuf_upsert_node(ctx->gbuf, "Chart", hc.deps[i], dep_qn, rel_path, SKIP_ONE, 0, - "{\"source\":\"helm\",\"external\":true}"); - if (dep_id > 0) { - cbm_gbuf_insert_edge(ctx->gbuf, chart_id, dep_id, "DEPENDS_ON", "{}"); - dep_edges++; + const cbm_helm_dep_t *dep = &hc.deps[i]; + if (k8s_is_file_repository(dep->repository)) { + k8s_local_deps_push(local_deps, chart_id, rel_path, ctx->project_name, dep); + continue; } + char *props = dep->repository[0] ? k8s_dep_props(dep->repository) : NULL; + dep_edges += k8s_external_dep(ctx, chart_id, dep->name, rel_path, props); + cbm_free(CBM_MEM_CLASS_OTHER, props); } + cbm_helm_chart_free(&hc); cbm_log_info("pass.k8s.helm", "file", rel_path, "deps", itoa_k8s(dep_edges)); } @@ -613,13 +2605,13 @@ static void handle_dep_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c * the sequential loop did. Sequentially the pass re-read and re-parsed every * YAML manifest on one thread -- 1.1 s of the Go corpus (6,275 manifests; * profile + waste sanitizer, 2026-09-17). */ -typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_KUSTOMIZE, K8S_HELM, K8S_MANIFEST } k8s_kind_t; +typedef enum { K8S_SKIP = 0, K8S_DEP, K8S_HELM, K8S_MANIFEST, K8S_KUSTOMIZE } k8s_kind_t; typedef struct { k8s_kind_t kind; char *source; /* DEP, HELM, MANIFEST */ int src_len; - CBMFileResult *res; /* MANIFEST, or KUSTOMIZE without a cached result */ + CBMFileResult *res; /* MANIFEST */ } k8s_prep_t; typedef struct { @@ -631,7 +2623,144 @@ typedef struct { _Atomic int next; } k8s_prep_job_t; -enum { K8S_CHUNK = 1024 }; +enum { K8S_CHUNK = 1024, K8S_TS_PAD = 16 }; + +/* ── Templated manifests (PHASE 2 P2.3) ────────────────────────────── */ + +/* A Helm chart template or Go-template addon is YAML only once rendered: + * `{{- if .Values.x }}` opens the document, `name: {{ include "app.fullname" + * . }}` is a flow mapping to a YAML parser. The skeleton is the text with + * every action rewritten, line for line (document lines stay exact): a line + * holding nothing but actions is blanked; an action inside text becomes a + * symbolic scalar -- `` for an include/template call, the + * first `.`/`$` path otherwise (`<.Values.image.tag>`), else ``. Objects + * then get the chart-scoped symbolic names IDENTITY.md describes, and a + * reference written with the same expression meets them in the chart. */ + +static bool k8s_tpl_ident(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* Does the action text at q (before e) call a named template: `include` / + * `template` then a quoted name? Fills `out` with ``. */ +static bool k8s_action_call(const char *a, const char *q, const char *e, char *out, size_t out_sz) { + size_t kl = strncmp(q, "include", K8S_SLEN("include")) == 0 ? K8S_SLEN("include") + : strncmp(q, "template", K8S_SLEN("template")) == 0 ? K8S_SLEN("template") + : 0; + if (!kl || q + kl >= e || q[kl] != ' ' || (q > a && k8s_tpl_ident(q[-SKIP_ONE]))) { + return false; + } + const char *r = q + kl; + while (r < e && *r == ' ') { + r++; + } + const char *ne = + r < e && *r == '"' ? memchr(r + SKIP_ONE, '"', (size_t)(e - r - SKIP_ONE)) : NULL; + if (!ne || ne == r + SKIP_ONE) { + return false; + } + snprintf(out, out_sz, "<%.*s>", (int)(ne - r - SKIP_ONE), r + SKIP_ONE); + return true; +} + +/* The symbolic token of the action text [a, e) into out (NUL-terminated). */ +static void k8s_action_token(const char *a, const char *e, char *out, size_t out_sz) { + for (const char *q = a; q < e; q++) { + if (k8s_action_call(a, q, e, out, out_sz)) { + return; + } + } + for (const char *q = a; q < e; q++) { + if ((*q == '.' || *q == '$') && (q == a || q[-SKIP_ONE] == ' ' || q[-SKIP_ONE] == '(')) { + const char *r = q + SKIP_ONE; + while (r < e && (k8s_tpl_ident(*r) || *r == '.' || *r == '$')) { + r++; + } + if (r > q + SKIP_ONE || *q == '$') { + snprintf(out, out_sz, "<%.*s>", (int)(r - q), q); + return; + } + } + } + snprintf(out, out_sz, ""); +} + +/* The end of the logical line starting at ls: its newline, or -- when an + * action starting on it spans lines -- the newline after that action. Sets + * *only_actions when the line holds actions and nothing else but blanks. */ +static const char *k8s_logical_line(const char *ls, const char *end, bool *only_actions) { + const char *le = memchr(ls, '\n', (size_t)(end - ls)); + le = le ? le : end; + bool blank = true; + bool any = false; + for (const char *p = ls; p < le;) { + if (p + SKIP_ONE < le && p[0] == '{' && p[SKIP_ONE] == '{') { + const char *ae = strstr(p + PAIR_LEN, "}}"); + if (!ae || ae >= end) { + blank = false; + break; + } + any = true; + p = ae + PAIR_LEN; + if (p > le) { + const char *nl = memchr(p, '\n', (size_t)(end - p)); + le = nl ? nl : end; + } + continue; + } + blank = blank && (*p == ' ' || *p == '\t' || *p == '\r'); + p++; + } + *only_actions = any && blank; + return le; +} + +/* The skeleton of templated text (cbm_alloc'd, *out_len set); NULL on OOM. */ +static char *k8s_skeleton(const char *src, int len, int *out_len) { + size_t cap = (size_t)len * PAIR_LEN + CBM_SZ_64 + K8S_TS_PAD; + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, cap); + if (!out) { + return NULL; + } + size_t w = 0; + const char *end = src + len; + for (const char *ls = src; ls < end;) { + bool only_actions = false; + const char *le = k8s_logical_line(ls, end, &only_actions); + for (const char *p = ls; p < le;) { + const char *ae = p + SKIP_ONE < le && p[0] == '{' && p[SKIP_ONE] == '{' + ? strstr(p + PAIR_LEN, "}}") + : NULL; + if (!ae || ae >= end) { + if (!only_actions) { + out[w++] = *p; + } + p++; + continue; + } + if (!only_actions) { + char tok[CBM_SZ_256]; + k8s_action_token(p + PAIR_LEN, ae, tok, sizeof(tok)); + size_t tl = strlen(tok); + memcpy(out + w, tok, tl); + w += tl; + } + for (const char *q = p; q < ae; q++) { /* the action's own newlines stay */ + if (*q == '\n') { + out[w++] = '\n'; + } + } + p = ae + PAIR_LEN; + } + if (le < end) { + out[w++] = '\n'; + } + ls = le < end ? le + SKIP_ONE : end; + } + memset(out + w, 0, K8S_TS_PAD); /* lexer lookahead, as k8s_read_file pads */ + *out_len = (int)w; + return out; +} static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { cbm_pipeline_ctx_t *ctx = job->ctx; @@ -643,19 +2772,10 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { if (is_gomod_file(base) || lang == CBM_LANG_GOMOD || is_requirements_file(base)) { out->source = k8s_read_file(path, &out->src_len); out->kind = out->source ? K8S_DEP : K8S_SKIP; - } else if (cbm_is_kustomize_file(base)) { - out->kind = K8S_KUSTOMIZE; - bool cached = ctx->result_cache && ctx->result_cache[i]; - if (!cached) { - /* Fall back to re-extraction */ - int src_len = 0; - char *source = k8s_read_file(path, &src_len); - if (source) { - out->res = cbm_extract_file(source, src_len, CBM_LANG_KUSTOMIZE, ctx->project_name, - rel, CBM_EXTRACT_BUDGET, NULL, NULL); - cbm_free(CBM_MEM_CLASS_OTHER, source); - } - } + } else if (lang == CBM_LANG_KUSTOMIZE) { + /* Its `namespace:` is the namespace of the objects it builds. */ + out->source = k8s_read_file(path, &out->src_len); + out->kind = out->source ? K8S_KUSTOMIZE : K8S_SKIP; } else if (lang == CBM_LANG_YAML || lang == CBM_LANG_K8S) { /* Read source once to classify (and reuse for uncached extraction). */ out->source = k8s_read_file(path, &out->src_len); @@ -669,6 +2789,15 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { * result: cached results were produced during the parallel YAML * pass and contain no "Resource" definitions. */ out->kind = K8S_MANIFEST; + if (strstr(out->source, "{{")) { + int sk_len = 0; + char *sk = k8s_skeleton(out->source, out->src_len, &sk_len); + if (sk) { + cbm_free(CBM_MEM_CLASS_OTHER, out->source); + out->source = sk; + out->src_len = sk_len; + } + } out->res = cbm_extract_file(out->source, out->src_len, CBM_LANG_K8S, ctx->project_name, rel, CBM_EXTRACT_BUDGET, NULL, NULL); } else { @@ -696,23 +2825,627 @@ static void k8s_prep_worker(int worker_id, void *arg) { cbm_kind_in_set_free_cache(); } +/* ── Helm includes (PHASE 2 P2.1) ──────────────────────────────────── */ + +/* A chart template's `include "name"` / `template "name"` calls a named + * template (`define "name"`, a Function of a .tpl file). Helm loads a chart's + * templates with those of every chart below it into one namespace, so the + * name resolves among the defines of the chart family -- the outermost + * enclosing chart and the charts below it -- and of the families its members + * depend on locally (file://). A dynamic name draws nothing. */ + +/* The nearest chart dir enclosing file `path` ("" for a root chart). */ +static bool k8s_chart_of(k8s_bundles_t *b, const char *path, char *out, size_t out_sz) { + char dir[CBM_SZ_1K]; + k8s_dirname(path, dir, sizeof(dir)); + for (;;) { + if (cbm_ht_get(b->charts, dir)) { + snprintf(out, out_sz, "%s", dir); + return true; + } + if (!dir[0]) { + return false; + } + char parent[CBM_SZ_1K]; + k8s_dirname(dir, parent, sizeof(parent)); + snprintf(dir, sizeof(dir), "%s", parent); + } +} + +/* The outermost chart dir enclosing chart dir `chart` (itself when none). */ +static void k8s_chart_family(k8s_bundles_t *b, const char *chart, char *out, size_t out_sz) { + snprintf(out, out_sz, "%s", chart); + char dir[CBM_SZ_1K]; + snprintf(dir, sizeof(dir), "%s", chart); + while (dir[0]) { + char parent[CBM_SZ_1K]; + k8s_dirname(dir, parent, sizeof(parent)); + snprintf(dir, sizeof(dir), "%s", parent); + if (cbm_ht_get(b->charts, dir)) { + snprintf(out, out_sz, "%s", dir); + } + } +} + +static bool k8s_has_suffix(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t m = strlen(suffix); + return n >= m && strcmp(s + n - m, suffix) == 0; +} + +static bool k8s_ident_char(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +typedef struct { + cbm_pipeline_ctx_t *ctx; + k8s_bundles_t *b; + CBMHashTable *defines; /* "\x1f" -> define Functions */ + CBMHashTable *deps; /* "" -> Chart nodes its members depend on */ + CBMHashTable *resources; /* file -> its Resource nodes */ + CBMHashTable *file_nodes; /* file -> its File node */ + CBMHashTable *yaml_defs; /* template file -> the defines written in it */ + CBMArena arena; /* define QNs */ + const char *file; + char family[CBM_SZ_1K]; /* the chart family of `file` */ + k8s_ids_t found; + int edges; + int unresolved; +} k8s_inc_t; + +/* The node a call on `line` of the file leaves from: the innermost document + * (Resource) spanning it, else the File. */ +static int64_t k8s_inc_source(k8s_inc_t *s, int line) { + const cbm_gbuf_node_t *best = NULL; + for (int pass = 0; pass < PAIR_LEN && !best; pass++) { + const k8s_ids_t *res = cbm_ht_get(pass == 0 ? s->yaml_defs : s->resources, s->file); + for (int i = 0; res && i < res->count; i++) { + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_id(s->ctx->gbuf, res->items[i]); + if (n && n->start_line <= line && line <= n->end_line && + (!best || n->end_line - n->start_line < best->end_line - best->start_line)) { + best = n; + } + } + } + if (best) { + return best->id; + } + const k8s_ids_t *f = cbm_ht_get(s->file_nodes, s->file); + return f && f->count > 0 ? f->items[0] : 0; +} + +static void k8s_inc_lookup(k8s_inc_t *s, const char *family, const char *name) { + char key[CBM_SZ_1K]; + snprintf(key, sizeof(key), "%s\x1f%s", family, name); + const k8s_ids_t *v = cbm_ht_get(s->defines, key); + for (int i = 0; v && i < v->count; i++) { + k8s_ids_push(&s->found, v->items[i]); + } +} + +/* One `include "name"` / `template "name"` call on `line`. */ +static void k8s_inc_call(k8s_inc_t *s, const char *name, int line, const char *via) { + s->found.count = 0; + k8s_inc_lookup(s, s->family, name); + const k8s_ids_t *deps = cbm_ht_get(s->deps, s->family); + for (int d = 0; deps && d < deps->count; d++) { + const cbm_gbuf_node_t *dep = cbm_gbuf_find_by_id(s->ctx->gbuf, deps->items[d]); + if (dep && dep->file_path) { + char chart[CBM_SZ_1K]; + char family[CBM_SZ_1K]; + k8s_dirname(dep->file_path, chart, sizeof(chart)); + k8s_chart_family(s->b, chart, family, sizeof(family)); + k8s_inc_lookup(s, family, name); + } + } + k8s_ids_sort_unique(&s->found); + atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)s->found.count, memory_order_relaxed); + int64_t src = s->found.count > 0 ? k8s_inc_source(s, line) : 0; + if (src <= 0) { + s->unresolved++; + return; + } + char esc[CBM_SZ_512]; + cbm_json_escape(esc, (int)sizeof(esc), name); + char props[CBM_SZ_1K]; + snprintf(props, sizeof(props), + "{\"callee\":\"%s\",\"strategy\":\"helm_include\",\"via\":\"%s\"}", esc, via); + for (int i = 0; i < s->found.count; i++) { + if (s->found.items[i] != src) { + cbm_gbuf_insert_edge(s->ctx->gbuf, src, s->found.items[i], "CALLS", props); + s->edges++; + } + } +} + +/* Every literal `include "name"` / `template "name"` inside the `{{ }}` + * actions of a template's text, with its line. */ +static void k8s_scan_includes(k8s_inc_t *s, const char *src) { + int line = SKIP_ONE; + const char *at = src; + for (const char *p = strstr(src, "{{"); p;) { + const char *e = strstr(p + PAIR_LEN, "}}"); + if (!e) { + break; + } + for (const char *q = p + PAIR_LEN; q < e; q++) { + const char *via = strncmp(q, "include", K8S_SLEN("include")) == 0 ? "include" + : strncmp(q, "template", K8S_SLEN("template")) == 0 ? "template" + : NULL; + if (!via || k8s_ident_char(q[-SKIP_ONE]) || q[-SKIP_ONE] == '.' || + q[-SKIP_ONE] == '$') { + continue; + } + const char *r = q + strlen(via); + if (r >= e || (*r != ' ' && *r != '\t' && *r != '\n' && *r != '\r')) { + continue; + } + while (r < e && (*r == ' ' || *r == '\t' || *r == '\n' || *r == '\r')) { + r++; + } + const char *ne = + r < e && *r == '"' ? memchr(r + SKIP_ONE, '"', (size_t)(e - r - SKIP_ONE)) : NULL; + if (!ne || ne == r + SKIP_ONE) { + continue; + } + for (; at < q; at++) { + line += *at == '\n'; + } + char name[CBM_SZ_256]; + snprintf(name, sizeof(name), "%.*s", (int)(ne - r - SKIP_ONE), r + SKIP_ONE); + k8s_inc_call(s, name, line, via); + q = ne; + } + p = strstr(e + PAIR_LEN, "{{"); + } +} + +static CBMHashTable *k8s_index_by_file(cbm_pipeline_ctx_t *ctx, const char *label) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!ix || cbm_gbuf_find_by_label(ctx->gbuf, label, &nodes, &count) != 0) { + return ix; + } + for (int i = 0; i < count; i++) { + if (nodes[i]->file_path) { + k8s_index_add(ix, nodes[i]->file_path, nodes[i]->id); + } + } + return ix; +} + +static void k8s_free_index(CBMHashTable *ix) { + if (ix) { + cbm_ht_foreach(ix, k8s_free_ids_entry, NULL); + cbm_ht_free(ix); + } +} + +/* Does the action text at `q` start with keyword `kw`? */ +static bool k8s_action_is(const char *q, const char *kw) { + size_t n = strlen(kw); + return strncmp(q, kw, n) == 0 && !k8s_ident_char(q[n]); +} + +/* A `define "name"` written in a template (outside a .tpl file the grammar + * reads): a Function like a .tpl define, spanning to its matching `end`. */ +static void k8s_yaml_define(k8s_inc_t *s, const char *name, int start, int end) { + char *qn = cbm_fqn_compute(&s->arena, s->ctx->project_name, s->file, name); + if (!qn) { + return; + } + int64_t id = cbm_gbuf_upsert_node(s->ctx->gbuf, "Function", name, qn, s->file, start, end, + "{\"source\":\"helm_template\"}"); + if (id <= 0) { + return; + } + const k8s_ids_t *f = cbm_ht_get(s->file_nodes, s->file); + if (f && f->count > 0) { + cbm_gbuf_insert_edge(s->ctx->gbuf, f->items[0], id, "DEFINES", "{}"); + } + char key[CBM_SZ_1K]; + snprintf(key, sizeof(key), "%s\x1f%s", s->family, name); + k8s_index_add(s->defines, key, id); + k8s_index_add(s->yaml_defs, s->file, id); +} + +/* Every `define "name"` ... `end` of a template's text: block actions (if, + * range, with, block, define) nest until their `end`. */ +static void k8s_scan_defines(k8s_inc_t *s, const char *src) { + int line = SKIP_ONE; + const char *at = src; + int depth = 0; + int start = 0; + char name[CBM_SZ_256] = ""; + for (const char *p = strstr(src, "{{"); p;) { + const char *e = strstr(p + PAIR_LEN, "}}"); + if (!e) { + break; + } + for (; at < p; at++) { + line += *at == '\n'; + } + const char *q = p + PAIR_LEN; + q += *q == '-'; + while (q < e && (*q == ' ' || *q == '\t' || *q == '\n' || *q == '\r')) { + q++; + } + if (depth == 0 && k8s_action_is(q, "define")) { + const char *r = q + K8S_SLEN("define"); + while (r < e && (*r == ' ' || *r == '\t')) { + r++; + } + const char *ne = + r < e && *r == '"' ? memchr(r + SKIP_ONE, '"', (size_t)(e - r - SKIP_ONE)) : NULL; + if (ne && ne > r + SKIP_ONE) { + snprintf(name, sizeof(name), "%.*s", (int)(ne - r - SKIP_ONE), r + SKIP_ONE); + depth = SKIP_ONE; + start = line; + } + } else if (depth > 0 && (k8s_action_is(q, "if") || k8s_action_is(q, "range") || + k8s_action_is(q, "with") || k8s_action_is(q, "block") || + k8s_action_is(q, "define"))) { + depth++; + } else if (depth > 0 && k8s_action_is(q, "end")) { + depth--; + if (depth == 0) { + for (; at < e; at++) { + line += *at == '\n'; + } + k8s_yaml_define(s, name, start, line); + } + } + p = strstr(e + PAIR_LEN, "{{"); + } +} + +/* A chart template file: .yaml / .yml / .json under a chart's templates/; + * fills its chart's family. */ +static bool k8s_template_family(k8s_bundles_t *b, const char *rel, char *family, size_t sz) { + char chart[CBM_SZ_1K]; + if (!rel || + !(k8s_has_suffix(rel, ".yaml") || k8s_has_suffix(rel, ".yml") || + k8s_has_suffix(rel, ".json")) || + !k8s_chart_of(b, rel, chart, sizeof(chart))) { + return false; + } + size_t cl = strlen(chart); + const char *sub = cl ? rel + cl + SKIP_ONE : rel; + if (strncmp(sub, "templates/", K8S_SLEN("templates/")) != 0) { + return false; + } + k8s_chart_family(b, chart, family, sz); + return true; +} + +/* Index the defines and dependencies, then scan every template of a chart. */ +static void k8s_includes_run(k8s_inc_t *s, const cbm_file_info_t *files, int file_count) { + /* defines by chart family: Functions of the charts' Go template files */ + const cbm_gbuf_node_t **fns = NULL; + int fn_n = 0; + if (cbm_gbuf_find_by_label(s->ctx->gbuf, "Function", &fns, &fn_n) == 0) { + for (int i = 0; i < fn_n; i++) { + char chart[CBM_SZ_1K]; + char family[CBM_SZ_1K]; + char key[CBM_SZ_1K]; + const char *fp = fns[i]->file_path; + if (!fp || !fns[i]->name || + !(k8s_has_suffix(fp, ".tpl") || k8s_has_suffix(fp, ".gotmpl")) || + !k8s_chart_of(s->b, fp, chart, sizeof(chart))) { + continue; + } + k8s_chart_family(s->b, chart, family, sizeof(family)); + snprintf(key, sizeof(key), "%s\x1f%s", family, fns[i]->name); + k8s_index_add(s->defines, key, fns[i]->id); + } + } + /* local dependencies: Chart -DEPENDS_ON-> Chart, by the source's family */ + const cbm_gbuf_edge_t **deps = NULL; + int dep_n = 0; + if (cbm_gbuf_find_edges_by_type(s->ctx->gbuf, "DEPENDS_ON", &deps, &dep_n) == 0) { + for (int i = 0; i < dep_n; i++) { + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(s->ctx->gbuf, deps[i]->source_id); + const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_id(s->ctx->gbuf, deps[i]->target_id); + if (!src || !tgt || !src->label || !tgt->label || strcmp(src->label, "Chart") != 0 || + strcmp(tgt->label, "Chart") != 0 || !src->file_path || !tgt->file_path) { + continue; + } + char chart[CBM_SZ_1K]; + char family[CBM_SZ_1K]; + k8s_dirname(src->file_path, chart, sizeof(chart)); + k8s_chart_family(s->b, chart, family, sizeof(family)); + k8s_index_add(s->deps, family, tgt->id); + } + } + /* every template of a chart: its own defines, then its includes */ + for (int pass = 0; pass < PAIR_LEN; pass++) { + for (int i = 0; i < file_count; i++) { + const char *rel = files[i].rel_path; + if (!k8s_template_family(s->b, rel, s->family, sizeof(s->family))) { + continue; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", s->ctx->repo_path, rel); + int len = 0; + char *src = k8s_read_file(path, &len); + if (!src) { + continue; + } + s->file = rel; + if (pass == 0) { + k8s_scan_defines(s, src); + } else { + k8s_scan_includes(s, src); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + } +} + +static void k8s_link_includes(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, + const cbm_file_info_t *files, int file_count) { + if (!b->charts || !ctx->repo_path) { + return; + } + k8s_inc_t s = {.ctx = ctx, + .b = b, + .defines = cbm_ht_create(0), + .deps = cbm_ht_create(0), + .resources = k8s_index_by_file(ctx, "Resource"), + .file_nodes = k8s_index_by_file(ctx, "File"), + .yaml_defs = cbm_ht_create(0)}; + cbm_arena_init(&s.arena); + if (s.defines && s.deps && s.resources && s.file_nodes && s.yaml_defs) { + k8s_includes_run(&s, files, file_count); + } + cbm_arena_destroy(&s.arena); + cbm_free(CBM_MEM_CLASS_OTHER, s.found.items); + k8s_free_index(s.defines); + k8s_free_index(s.deps); + k8s_free_index(s.resources); + k8s_free_index(s.file_nodes); + k8s_free_index(s.yaml_defs); + if (s.edges + s.unresolved > 0) { + cbm_log_info("pass.k8s.includes", "edges", itoa_k8s(s.edges), "unresolved", + itoa_k8s(s.unresolved)); + } +} + +/* ── DEPLOYS (PHASE 3 P3.1) ────────────────────────────────────────── */ + +/* What a GitOps object deploys, as DEPLOYS edges to existing nodes only: + * a Flux Kustomization / Argo Application -> the units its path applies (the + * outermost kustomizations below it -- their Kustomization node -- and the + * manifests outside any, their File node; a chart dir: its Chart); a + * HelmRelease -> its chart (a repository chart dir of a Git/Bucket source, + * else the shared external Chart of that name). */ + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMHashTable *files; /* file -> File node */ + CBMHashTable *charts; /* Chart.yaml path -> Chart node */ + CBMHashTable *kusts; /* kustomization path -> its Kustomization Resource */ + int edges; +} k8s_dep_link_t; + +static int64_t k8s_first_id(CBMHashTable *ix, const char *key) { + const k8s_ids_t *v = cbm_ht_get(ix, key); + return v && v->count > 0 ? v->items[0] : 0; +} + +static void k8s_deploys_edge(k8s_dep_link_t *l, int64_t src, int64_t tgt, const char *props) { + if (src > 0 && tgt > 0 && src != tgt) { + cbm_gbuf_insert_edge(l->ctx->gbuf, src, tgt, "DEPLOYS", props); + l->edges++; + } +} + +/* The Chart node of chart dir `dir`, 0 when it is none. */ +static int64_t k8s_chart_node(k8s_dep_link_t *l, const char *dir) { + static const char *const names[] = {"Chart.yaml", "Chart.yml"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + int64_t id = k8s_first_id(l->charts, path); + if (id > 0) { + return id; + } + } + return 0; +} + +/* The shared external Chart of `name` (the one chart dependencies use). */ +static int64_t k8s_external_chart(k8s_dep_link_t *l, const char *name, const char *file) { + char qn[CBM_SZ_512]; + snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", l->ctx->project_name, name); + const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(l->ctx->gbuf, qn); + if (known) { + return known->id; + } + return cbm_gbuf_upsert_node(l->ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, + "{\"source\":\"helm\",\"external\":true}"); +} + +/* A HelmRelease whose chartRef names OCIRepository `oci` (its USAGE edge + * {"ref":"chartRef"}, resolved with the references) deploys the chart that + * repository fetches: the shared external Chart of the URL's last segment. */ +static void k8s_link_chart_refs(k8s_dep_link_t *l, const k8s_hr_t *oci) { + const char *url = oci->chart; + size_t end = strlen(url); + while (end > 0 && url[end - SKIP_ONE] == '/') { + end--; + } + size_t start = end; + while (start > 0 && url[start - SKIP_ONE] != '/') { + start--; + } + if (start == 0 || start == end || end - start >= CBM_SZ_256) { + return; + } + char name[CBM_SZ_256]; + snprintf(name, sizeof(name), "%.*s", (int)(end - start), url + start); + char esc_name[CBM_SZ_512]; + char esc_url[CBM_SZ_1K]; + cbm_json_escape(esc_name, (int)sizeof(esc_name), name); + cbm_json_escape(esc_url, (int)sizeof(esc_url), url); + char props[CBM_SZ_2K]; + snprintf(props, sizeof(props), + "{\"via\":\"helmrelease\",\"chart\":\"%s\",\"external\":true,\"url\":\"%s\"}", + esc_name, esc_url); + const cbm_gbuf_edge_t **in = NULL; + int n = 0; + if (cbm_gbuf_find_edges_by_target_type(l->ctx->gbuf, oci->node, "USAGE", &in, &n) != 0) { + return; + } + for (int i = 0; i < n; i++) { + atomic_fetch_add_explicit(&g_k8s_candidates, SKIP_ONE, memory_order_relaxed); + const char *p = in[i]->properties_json; + const cbm_gbuf_node_t *hr = cbm_gbuf_find_by_id(l->ctx->gbuf, in[i]->source_id); + if (p && strstr(p, "\"ref\":\"chartRef\"") && hr && hr->file_path) { + k8s_deploys_edge(l, hr->id, k8s_external_chart(l, name, hr->file_path), props); + } + } +} + +/* A container image an object runs: USAGE {ref:image} to its Image node -- + * the node a Dockerfile building or building on that image joins. */ +static void k8s_link_images(cbm_pipeline_ctx_t *ctx, const k8s_facts_t *f) { + for (int i = 0; i < f->img_n; i++) { + int64_t img = cbm_pipeline_image_node(ctx, f->imgs[i].chart); + if (img <= 0) { + continue; + } + char esc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), f->imgs[i].chart); + char props[CBM_SZ_2K]; + snprintf(props, sizeof(props), "{\"ref\":\"image\",\"image\":\"%s\"}", esc); + cbm_gbuf_insert_edge(ctx->gbuf, f->imgs[i].node, img, "USAGE", props); + } +} + +/* The environment variables an object's containers set: container-scoped + * EnvVar nodes the object CONFIGURES {strategy:env_set}. */ +static void k8s_link_envs(cbm_pipeline_ctx_t *ctx, const k8s_facts_t *f) { + for (int i = 0; i < f->env_n; i++) { + const k8s_env_t *e = &f->envs[i]; + const cbm_gbuf_node_t *owner = cbm_gbuf_find_by_id(ctx->gbuf, e->node); + if (owner) { + cbm_pipeline_env_setter(ctx, owner, e->container ? e->container : "-", e->key, + e->source, e->line); + } + } +} + +static void k8s_link_deploys(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b, + const k8s_facts_t *f) { + if (f->dep_n == 0 && f->hr_n == 0 && f->src_n == 0) { + return; + } + k8s_dep_link_t l = {.ctx = ctx, + .files = k8s_index_by_file(ctx, "File"), + .charts = k8s_index_by_file(ctx, "Chart"), + .kusts = cbm_ht_create(0)}; + const cbm_gbuf_node_t **res = NULL; + int rn = 0; + k8s_paths_t manifests = {0}; + if (l.kusts && cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &res, &rn) == 0) { + for (int i = 0; i < rn; i++) { + if (!res[i]->file_path) { + continue; + } + k8s_paths_push(&manifests, res[i]->file_path); + if (res[i]->name && strcmp(res[i]->name, "Kustomization") == 0) { + k8s_index_add(l.kusts, res[i]->file_path, res[i]->id); + } + } + } + k8s_paths_sort_unique(&manifests); + CBMHashTable *by_path = cbm_ht_create(0); + for (int i = 0; by_path && i < ks->count; i++) { + cbm_ht_set(by_path, ks->items[i].path, (void *)(intptr_t)(i + SKIP_ONE)); + } + int *seen = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(ks->count ? ks->count : SKIP_ONE) * sizeof(int)); + k8s_paths_t members = {0}; + k8s_paths_t units = {0}; + for (int d = 0; l.files && l.charts && l.kusts && by_path && seen && d < f->dep_n; d++) { + const k8s_deploy_t *dp = &f->deploys[d]; + char esc[CBM_SZ_1K]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), dp->dir); + snprintf(props, sizeof(props), "{\"via\":\"%s\",\"path\":\"%s\"}", dp->via, esc); + int64_t chart = cbm_ht_get(b->charts, dp->dir) ? k8s_chart_node(&l, dp->dir) : 0; + if (chart > 0) { + k8s_deploys_edge(&l, dp->node, chart, props); + continue; + } + members.count = 0; + units.count = 0; + k8s_dir_members(ks, by_path, &manifests, dp->dir, seen, d + SKIP_ONE, &members, &units); + k8s_paths_sort_unique(&units); + for (int u = 0; u < units.count; u++) { + int64_t tgt = k8s_first_id(l.kusts, units.items[u]); + k8s_deploys_edge(&l, dp->node, tgt > 0 ? tgt : k8s_first_id(l.files, units.items[u]), + props); + } + } + for (int h = 0; l.files && l.charts && h < f->hr_n; h++) { + const k8s_hr_t *hr = &f->hrs[h]; + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, hr->node); + bool repo_path = hr->source && (strcmp(hr->source, "GitRepository") == 0 || + strcmp(hr->source, "Bucket") == 0); + char esc[CBM_SZ_1K]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), hr->chart); + snprintf(props, sizeof(props), "{\"via\":\"helmrelease\",\"chart\":\"%s\"%s}", esc, + repo_path ? "" : ",\"external\":true"); + char dir[CBM_SZ_1K]; + if (repo_path) { + if (k8s_norm_dir(hr->chart, dir, sizeof(dir)) && cbm_ht_get(b->charts, dir)) { + k8s_deploys_edge(&l, hr->node, k8s_chart_node(&l, dir), props); + } + } else if (src && src->file_path && !strchr(hr->chart, '/') && !strchr(hr->chart, '$')) { + k8s_deploys_edge(&l, hr->node, k8s_external_chart(&l, hr->chart, src->file_path), + props); + } + } + for (int s = 0; l.files && l.charts && s < f->src_n; s++) { + k8s_link_chart_refs(&l, &f->srcs[s]); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)members.items); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)units.items); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)manifests.items); + cbm_free(CBM_MEM_CLASS_OTHER, seen); + if (by_path) { + cbm_ht_free(by_path); + } + k8s_free_index(l.files); + k8s_free_index(l.charts); + k8s_free_index(l.kusts); + if (l.edges > 0) { + cbm_log_info("pass.k8s.deploys_edges", "edges", itoa_k8s(l.edges)); + } +} + int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { cbm_log_info("pass.start", "pass", "k8s", "files", itoa_k8s(file_count)); cbm_init(); - int kustomize_count = 0; int manifest_count = 0; int helm_count = 0; /* Collect per-manifest selector/label records for cross-manifest matching. */ k8s_record_array_t recs = {0}; - recs.items = cbm_calloc(CBM_MEM_CLASS_OTHER, K8S_MAX_RECORDS * sizeof(*recs.items)); - recs.cap = recs.items ? K8S_MAX_RECORDS : 0; + k8s_local_deps_t local_deps = {0}; + k8s_kusts_t kusts = {0}; + k8s_bundles_t bundles = {.charts = cbm_ht_create(0)}; + k8s_facts_t facts = {0}; k8s_prep_t *prep = cbm_calloc(CBM_MEM_CLASS_OTHER, K8S_CHUNK * sizeof(*prep)); if (!prep) { - cbm_free(CBM_MEM_CLASS_OTHER, recs.items); + k8s_bundles_free(&bundles); return CBM_NOT_FOUND; } int workers = cbm_default_worker_count(false); @@ -721,7 +3454,11 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int end = begin + K8S_CHUNK < file_count ? begin + K8S_CHUNK : file_count; if (cbm_pipeline_check_cancel(ctx)) { cbm_free(CBM_MEM_CLASS_OTHER, prep); - cbm_free(CBM_MEM_CLASS_OTHER, recs.items); + k8s_records_free(&recs); + k8s_local_deps_free(&local_deps); + k8s_kusts_free(&kusts); + k8s_bundles_free(&bundles); + k8s_facts_free(&facts); return CBM_NOT_FOUND; } memset(prep, 0, (size_t)K8S_CHUNK * sizeof(*prep)); @@ -740,30 +3477,23 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, if (!cancelled) { const char *rel = files[i].rel_path; const char *base = k8s_basename(rel); - CBMFileResult *cached = - (ctx->result_cache && ctx->result_cache[i]) ? ctx->result_cache[i] : NULL; switch (p->kind) { case K8S_DEP: handle_dep_manifest(ctx, rel, p->source, is_requirements_file(base) ? "pypi" : "gomod"); break; - case K8S_KUSTOMIZE: - handle_kustomize(ctx, rel, cached ? cached : p->res); - kustomize_count++; - break; case K8S_HELM: - handle_helm_chart(ctx, rel, p->source); + handle_helm_chart(ctx, rel, p->source, &local_deps); + k8s_chart_dir_add(&bundles, rel); helm_count++; break; - case K8S_MANIFEST: { - k8s_record_t *rec = (recs.count < recs.cap) ? &recs.items[recs.count] : NULL; - handle_k8s_manifest(ctx, rel, p->source, p->res, rec); - if (rec && rec->node_id > 0) { - recs.count++; - } + case K8S_KUSTOMIZE: + k8s_kusts_push(&kusts, rel, k8s_kustomize_namespace(p->source)); + break; + case K8S_MANIFEST: + handle_k8s_manifest(ctx, rel, p->source, p->res, &recs, &facts); manifest_count++; break; - } case K8S_SKIP: default: break; @@ -776,18 +3506,41 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, } if (cancelled) { cbm_free(CBM_MEM_CLASS_OTHER, prep); - cbm_free(CBM_MEM_CLASS_OTHER, recs.items); + k8s_records_free(&recs); + k8s_local_deps_free(&local_deps); + k8s_kusts_free(&kusts); + k8s_bundles_free(&bundles); + k8s_facts_free(&facts); return CBM_NOT_FOUND; } } cbm_free(CBM_MEM_CLASS_OTHER, prep); - /* Connect Services to the workloads their selectors target (INFRA_MAPS). */ - k8s_link_selectors(ctx, &recs); - cbm_free(CBM_MEM_CLASS_OTHER, recs.items); + /* Connect Services to the workloads their selectors target (INFRA_MAPS), + * inside the bundles that deploy them together. */ + k8s_bundles_build(ctx, &kusts, &bundles); + k8s_deploys_build(ctx, &kusts, &bundles, &facts); + k8s_apply_effective_namespaces(ctx, &bundles); + k8s_link_selectors(ctx, &recs, &bundles); + k8s_resolve_refs(ctx, &bundles, &facts); + k8s_link_crds(ctx, &bundles, &facts); + k8s_link_deploys(ctx, &kusts, &bundles, &facts); + k8s_link_images(ctx, &facts); + k8s_link_envs(ctx, &facts); + k8s_facts_free(&facts); + k8s_records_free(&recs); + /* Link `file://` chart dependencies now that every Chart.yaml is read, + * then the templates' includes, which see those dependencies' defines. */ + k8s_link_local_deps(ctx, &local_deps); + k8s_link_includes(ctx, &bundles, files, file_count); + k8s_bundles_free(&bundles); + k8s_kusts_free(&kusts); + /* Shell and Make deploy commands, against the nodes linked above. */ + cbm_pipeline_pass_shell(ctx, files, file_count); + cbm_pipeline_pass_terraform(ctx, files, file_count); + cbm_pipeline_pass_docker(ctx, files, file_count); - cbm_log_info("pass.done", "pass", "k8s", "kustomize", itoa_k8s(kustomize_count), "manifests", - itoa_k8s(manifest_count)); + cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count)); (void)helm_count; return 0; } diff --git a/src/pipeline/pass_operator.c b/src/pipeline/pass_operator.c new file mode 100644 index 000000000..85fd106d8 --- /dev/null +++ b/src/pipeline/pass_operator.c @@ -0,0 +1,2104 @@ +/* + * pass_operator.c — Go Kubernetes operators (PHASE1 P1.2). + * + * A controller-runtime operator ties Go code to Kubernetes kinds in three + * ways the type graph alone does not carry: + * - a controller is built `For(&v1.Widget{})`, `Owns(&v1.Gadget{})`, + * `Watches(&v1.X{}, ...)` or watches `source.Kind(cache, &v1.X{})`: + * HANDLES {via: controller, role} from the function that builds it to + * the Go type, resolved as Go does: `v1` is an import of the file, and + * the type is the struct of that name in the imported package's + * directory (unqualified: the file's own package); + * - a CRD's group/kind is backed by `type struct` in the package + * marked `+groupName=` (or `GroupName = ""`): USAGE + * {ref: crd-type} from the CRD Resource to that Struct; + * - `//go:embed ` puts files into the binary: REFERENCES_FILE + * {via: go:embed} from the Go file to each embedded file; + * - an installer loads a chart of the repository (helm.LoadChart( + * filepath.Join(opt.ChartsDirectory, ...)), the Helm SDK's loader): + * DEPLOYS {via: helm-sdk} from the function to the Chart (PHASE 3 P3.3). + * Runs after USAGE resolution; reads each Go file once, and the files that + * mention a chart again. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/hash_table.h" +#include "foundation/limits.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" + +#include +#include +#include +#include +#include + +#define OP_SLEN(lit) (sizeof(lit) - SKIP_ONE) + +enum { OP_GROW_MIN = 16, OP_IDENT_MAX = 128 }; + +/* Candidates examined by the resolvers below (pipeline_internal.h). */ +_Atomic uint64_t g_op_candidates = 0; + +/* Read a whole file into a core-owned NUL-terminated buffer, NULL on error + * or past the indexer's file-size policy. */ +static char *op_read_file(const char *path) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + SKIP_ONE); + if (buf) { + size_t n = fread(buf, SKIP_ONE, (size_t)size, f); + buf[n] = '\0'; + } + (void)fclose(f); + return buf; +} + +static void op_dirname(const char *path, char *out, size_t out_sz) { + const char *slash = strrchr(path, '/'); + snprintf(out, out_sz, "%.*s", slash ? (int)(slash - path) : 0, path); +} + +/* ── Functions per file ──────────────────────────────────────────────── */ + +/* One definition span of a function node: Go method QNs carry no receiver, + * so the methods of several types in one package (every reconciler's + * SetupWithManager) are one node standing for spans in several files. */ +typedef struct { + const cbm_gbuf_node_t *node; + int start; + int end; +} op_span_t; + +typedef struct { + op_span_t *items; + int count; + int cap; +} op_spans_t; + +static void op_spans_push(op_spans_t *v, op_span_t span) { + if (v->count == v->cap) { + int cap = v->cap ? v->cap * PAIR_LEN : OP_GROW_MIN; + op_span_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, v->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + v->items = grown; + v->cap = cap; + } + v->items[v->count++] = span; +} + +static void op_free_spans(const char *key, void *value, void *ud) { + (void)ud; + op_spans_t *v = value; + cbm_free(CBM_MEM_CLASS_OTHER, v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +typedef struct { + CBMHashTable *by_file; + const cbm_gbuf_node_t *node; +} op_index_ctx_t; + +static void op_index_span(const char *file, int start, int end, void *ud) { + op_index_ctx_t *c = ud; + op_spans_t *v = cbm_ht_get(c->by_file, file); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, file); + if (!v || !key) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, key); + return; + } + cbm_ht_set(c->by_file, key, v); + } + op_spans_push(v, (op_span_t){c->node, start, end}); +} + +/* file_path -> the spans of Function and Method nodes in it (owned keys). */ +static CBMHashTable *op_functions_by_file(const cbm_gbuf_t *gb) { + CBMHashTable *by_file = cbm_ht_create(0); + static const char *const labels[] = {"Function", "Method"}; + for (size_t l = 0; by_file && l < sizeof(labels) / sizeof(labels[0]); l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(gb, labels[l], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count; i++) { + op_index_ctx_t c = {by_file, nodes[i]}; + cbm_gbuf_foreach_span(gb, nodes[i], op_index_span, &c); + } + } + return by_file; +} + +/* Is span `a` a better enclosing function than `b`: shorter, then (equal + * spans of two nodes) the lower QN, so the choice never follows index order. */ +static bool op_span_better(const op_span_t *a, const op_span_t *b) { + int la = a->end - a->start; + int lb = b->end - b->start; + if (la != lb) { + return la < lb; + } + const char *qa = a->node->qualified_name ? a->node->qualified_name : ""; + const char *qb = b->node->qualified_name ? b->node->qualified_name : ""; + return strcmp(qa, qb) < 0; +} + +/* The innermost function of `file` spanning `line`. */ +static const cbm_gbuf_node_t *op_enclosing(CBMHashTable *by_file, const char *file, int line) { + const op_spans_t *v = cbm_ht_get(by_file, file); + const op_span_t *best = NULL; + for (int i = 0; v && i < v->count; i++) { + const op_span_t *sp = &v->items[i]; + if (sp->start <= line && line <= sp->end && (!best || op_span_better(sp, best))) { + best = sp; + } + } + return best ? best->node : NULL; +} + +/* ── Go modules: import path -> directory ─────────────────────────────── */ + +/* A go.mod of the repository: module path and the directory it roots. */ +typedef struct { + char *path; + char *dir; +} op_module_t; + +typedef struct { + op_module_t *items; + int count; + int cap; +} op_modules_t; + +static void op_modules_free(op_modules_t *m) { + for (int i = 0; i < m->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, m->items[i].path); + cbm_free(CBM_MEM_CLASS_OTHER, m->items[i].dir); + } + cbm_free(CBM_MEM_CLASS_OTHER, m->items); + *m = (op_modules_t){0}; +} + +/* Note the `module` line of a go.mod at `file`. */ +static void op_note_module(op_modules_t *m, const char *file, const char *text) { + for (const char *p = strstr(text, "module"); p; p = strstr(p + SKIP_ONE, "module")) { + if ((p > text && p[-SKIP_ONE] != '\n') || + (p[OP_SLEN("module")] != ' ' && p[OP_SLEN("module")] != '\t')) { + continue; + } + const char *q = p + OP_SLEN("module"); + while (*q == ' ' || *q == '\t' || *q == '"') { + q++; + } + size_t n = strcspn(q, " \t\r\n\"/"); + while (q[n] == '/') { /* module paths contain '/' */ + n += SKIP_ONE + strcspn(q + n + SKIP_ONE, " \t\r\n\"/"); + } + if (n == 0) { + return; + } + if (m->count == m->cap) { + int cap = m->cap ? m->cap * PAIR_LEN : OP_GROW_MIN; + op_module_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, m->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + m->items = grown; + m->cap = cap; + } + char path[CBM_SZ_1K]; + char dir[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%.*s", (int)n, q); + op_dirname(file, dir, sizeof(dir)); + m->items[m->count].path = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, path); + m->items[m->count].dir = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir); + m->count++; + return; + } +} + +/* The repository directory of import path `import`: below the go.mod whose + * module path is its longest prefix ("" when no module of the repository + * has it). */ +static bool op_import_dir(const op_modules_t *m, const char *import, char *out, size_t out_sz) { + const op_module_t *best = NULL; + for (int i = 0; i < m->count; i++) { + size_t pl = strlen(m->items[i].path); + if (strncmp(import, m->items[i].path, pl) == 0 && + (import[pl] == '\0' || import[pl] == '/') && (!best || pl > strlen(best->path))) { + best = &m->items[i]; + } + } + if (!best) { + return false; + } + const char *rest = import + strlen(best->path); + rest += *rest == '/'; + snprintf(out, out_sz, "%s%s%s", best->dir, best->dir[0] && rest[0] ? "/" : "", rest); + return true; +} + +/* ── Indexes: Go types by package dir, module roots ───────────────────── */ + +/* Graph-owned nodes. */ +typedef struct { + const cbm_gbuf_node_t **items; + int count; + int cap; +} op_nodes_t; + +static void op_nodes_push(op_nodes_t *v, const cbm_gbuf_node_t *n) { + if (v->count == v->cap) { + int cap = v->cap ? v->cap * PAIR_LEN : OP_GROW_MIN; + const cbm_gbuf_node_t **grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, (void *)v->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + v->items = grown; + v->cap = cap; + } + v->items[v->count++] = n; +} + +static void op_free_nodes_entry(const char *key, void *value, void *ud) { + (void)ud; + op_nodes_t *v = value; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +/* "\x1f": a directory (or module root) with a name. */ +static void op_key(char *out, size_t out_sz, const char *a, const char *b) { + snprintf(out, out_sz, "%s\x1f%s", a, b); +} + +/* "\x1f" -> the Struct / Class / Type nodes Name names in dir. */ +static CBMHashTable *op_types_by_dir(const cbm_gbuf_t *gb) { + CBMHashTable *ix = cbm_ht_create(0); + static const char *const labels[] = {"Struct", "Class", "Type"}; + for (size_t l = 0; ix && l < sizeof(labels) / sizeof(labels[0]); l++) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(gb, labels[l], &nodes, &count) != 0) { + continue; + } + for (int i = 0; i < count; i++) { + if (!nodes[i]->file_path || !nodes[i]->name) { + continue; + } + char dir[CBM_SZ_1K]; + char key[PAIR_LEN * CBM_SZ_1K]; + op_dirname(nodes[i]->file_path, dir, sizeof(dir)); + op_key(key, sizeof(key), dir, nodes[i]->name); + op_nodes_t *v = cbm_ht_get(ix, key); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!v || !owned) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, owned); + continue; + } + cbm_ht_set(ix, owned, v); + } + op_nodes_push(v, nodes[i]); + } + } + return ix; +} + +typedef struct { + const op_modules_t *mods; + CBMHashTable *dir_types; /* op_types_by_dir */ +} op_index_t; + +/* ── Controllers: For / Owns / Watches / source.Kind ──────────────────── */ + +static int op_line_of(const char *text, const char *at) { + int line = SKIP_ONE; + for (const char *p = text; p < at; p++) { + line += *p == '\n'; + } + return line; +} + +static bool op_is_quote(char c) { + return c == '"' || c == '\'' || c == '`'; +} + +/* Where a `//` comment starts in the line text [ls, le), outside string and + * rune literals; le when there is none. */ +static const char *op_comment_start(const char *ls, const char *le) { + char quote = 0; + for (const char *p = ls; p < le; p++) { + if (quote) { + if (*p == '\\' && quote != '`' && p + SKIP_ONE < le) { + p++; + } else if (*p == quote) { + quote = 0; + } + } else if (op_is_quote(*p)) { + quote = *p; + } else if (*p == '/' && p + SKIP_ONE < le && p[SKIP_ONE] == '/') { + return p; + } + } + return le; +} + +static const char *op_line_start(const char *text, const char *p) { + while (p > text && p[-SKIP_ONE] != '\n') { + p--; + } + return p; +} + +/* The code character before `at`, past blanks and `//` comments: gofmt + * puts a chain's '.' at the end of an earlier line, and comment lines may + * sit between. NULL at the start of the text. */ +static const char *op_prev_code(const char *text, const char *at) { + const char *le = at; + for (;;) { + const char *ls = op_line_start(text, le); + const char *end = op_comment_start(ls, le); + while (end > ls && + (end[-SKIP_ONE] == ' ' || end[-SKIP_ONE] == '\t' || end[-SKIP_ONE] == '\r')) { + end--; + } + if (end > ls) { + return end - SKIP_ONE; + } + if (ls == text) { + return NULL; + } + le = ls - SKIP_ONE; /* the previous line, without its '\n' */ + } +} + +/* The end of the argument list opened at `open` ('('), balanced; parentheses + * in `//` comments and literals do not count. */ +static const char *op_args_end(const char *open) { + int depth = 0; + char quote = 0; + for (const char *p = open; *p; p++) { + if (quote) { + if (*p == '\\' && quote != '`' && p[SKIP_ONE]) { + p++; + } else if (*p == quote || (*p == '\n' && quote != '`')) { + quote = 0; + } + continue; + } + if (op_is_quote(*p)) { + quote = *p; + continue; + } + if (*p == '/' && p[SKIP_ONE] == '/') { + p = strchr(p, '\n'); + if (!p) { + break; + } + continue; + } + depth += *p == '('; + depth -= *p == ')'; + if (depth == 0) { + return p; + } + } + return open + strlen(open); +} + +static bool op_ident_char(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +/* The import path a file binds to `alias` (an explicit alias, else the + * path's last segment, Go's usual package name); "" when none. */ +static void op_import_path(const char *text, const char *alias, char *out, size_t out_sz) { + out[0] = '\0'; + size_t al = strlen(alias); + for (const char *p = strstr(text, "import"); p; p = strstr(p + SKIP_ONE, "import")) { + if ((p > text && p[-SKIP_ONE] != '\n') || op_ident_char(p[OP_SLEN("import")])) { + continue; + } + const char *q = p + OP_SLEN("import"); + const char *end = q; + while (*end == ' ' || *end == '\t') { + end++; + } + end = *end == '(' ? strchr(end, ')') : strchr(end, '\n'); + end = end ? end : q + strlen(q); + /* each import line: [alias] "path" */ + for (const char *l = q; l < end;) { + while (l < end && (*l == ' ' || *l == '\t' || *l == '(' || *l == '\n' || *l == '\r')) { + l++; + } + const char *name = l; + while (l < end && op_ident_char(*l)) { + l++; + } + size_t nl = (size_t)(l - name); + while (l < end && (*l == ' ' || *l == '\t')) { + l++; + } + if (l >= end || *l != '"') { + while (l < end && *l != '\n') { + l++; + } + continue; + } + const char *ps = l + SKIP_ONE; + const char *pe = strchr(ps, '"'); + if (!pe || pe > end) { + break; + } + const char *seg = pe; + while (seg > ps && seg[-SKIP_ONE] != '/') { + seg--; + } + bool hit = nl ? (nl == al && strncmp(name, alias, al) == 0) + : ((size_t)(pe - seg) == al && strncmp(seg, alias, al) == 0); + if (hit) { + snprintf(out, out_sz, "%.*s", (int)(pe - ps), ps); + return; + } + l = pe + SKIP_ONE; + } + } +} + +/* Is directory `dir` the package at import path `path`: the path ends with + * the directory, on a segment boundary (the module path is its prefix). */ +static bool op_dir_is_import(const char *dir, const char *path) { + size_t dl = strlen(dir); + size_t pl = strlen(path); + if (dl == 0 || dl > pl || strcmp(path + pl - dl, dir) != 0) { + return false; + } + return dl == pl || path[pl - dl - SKIP_ONE] == '/'; +} + +/* HANDLES from `fn` to the Go type `type` resolved as Go resolves it: the + * package the file imports as `qual`, or (unqualified) the file's own -- one + * directory, looked up in the type index. Only an import no module of the + * repository has falls back to the same-named types whose directory ends + * the import path. */ +static int op_handle(cbm_gbuf_t *gb, const op_index_t *ix, const cbm_gbuf_node_t *fn, + const char *file, const char *text, const char *qual, const char *type, + const char *role) { + char import[CBM_SZ_1K] = ""; + char pkg_dir[CBM_SZ_1K] = ""; + bool in_repo = false; + if (qual[0]) { + op_import_path(text, qual, import, sizeof(import)); + if (!import[0]) { + return 0; + } + in_repo = op_import_dir(ix->mods, import, pkg_dir, sizeof(pkg_dir)); + } else { + op_dirname(file, pkg_dir, sizeof(pkg_dir)); + } + char props[CBM_SZ_128]; + snprintf(props, sizeof(props), "{\"via\":\"controller\",\"role\":\"%s\"}", role); + int linked = 0; + if (!qual[0] || in_repo) { + char key[PAIR_LEN * CBM_SZ_1K]; + op_key(key, sizeof(key), pkg_dir, type); + const op_nodes_t *v = cbm_ht_get(ix->dir_types, key); + atomic_fetch_add_explicit(&g_op_candidates, (uint64_t)(v ? v->count : 0), + memory_order_relaxed); + for (int i = 0; v && i < v->count; i++) { + cbm_gbuf_insert_edge(gb, fn->id, v->items[i]->id, "HANDLES", props); + linked++; + } + return linked; + } + const cbm_gbuf_node_t **types = NULL; + int tn = 0; + if (cbm_gbuf_find_by_name(gb, type, &types, &tn) != 0) { + return 0; + } + atomic_fetch_add_explicit(&g_op_candidates, (uint64_t)(tn), memory_order_relaxed); + for (int i = 0; i < tn; i++) { + const cbm_gbuf_node_t *t = types[i]; + if (!t->label || !t->file_path || + (strcmp(t->label, "Struct") != 0 && strcmp(t->label, "Class") != 0 && + strcmp(t->label, "Type") != 0)) { + continue; + } + char dir[CBM_SZ_1K]; + op_dirname(t->file_path, dir, sizeof(dir)); + if (op_dir_is_import(dir, import)) { + cbm_gbuf_insert_edge(gb, fn->id, t->id, "HANDLES", props); + linked++; + } + } + return linked; +} + +/* Every `&[pkg.]Type{` in the arguments of a controller builder call. */ +static int op_controller_calls(cbm_gbuf_t *gb, CBMHashTable *by_file, const op_index_t *ix, + const char *file, const char *text) { + /* method name + "(", called on a receiver outside a comment: the previous + * code char is '.', and Kind's receiver is the `source` package. */ + static const struct { + const char *name; + const char *recv; + const char *role; + } calls[] = {{"For(", NULL, "for"}, + {"Owns(", NULL, "owns"}, + {"Watches(", NULL, "watches"}, + {"Kind(", "source", "watches"}}; + int linked = 0; + for (size_t c = 0; c < sizeof(calls) / sizeof(calls[0]); c++) { + for (const char *p = strstr(text, calls[c].name); p; + p = strstr(p + SKIP_ONE, calls[c].name)) { + if (p > text && op_ident_char(p[-SKIP_ONE])) { + continue; /* part of a longer name */ + } + if (op_comment_start(op_line_start(text, p), p) < p) { + continue; + } + const char *dot = op_prev_code(text, p); + if (!dot || *dot != '.') { + continue; + } + if (calls[c].recv) { + size_t rl = strlen(calls[c].recv); + const char *r = dot - rl; + if (r < text || strncmp(r, calls[c].recv, rl) != 0 || + (r > text && op_ident_char(r[-SKIP_ONE]))) { + continue; + } + } + const char *open = p + strlen(calls[c].name) - SKIP_ONE; + const char *end = op_args_end(open); + const cbm_gbuf_node_t *fn = op_enclosing(by_file, file, op_line_of(text, p)); + if (!fn) { + continue; + } + for (const char *a = open; a < end; a++) { + if (*a == '/' && a[SKIP_ONE] == '/') { + const char *nl = strchr(a, '\n'); + a = nl && nl < end ? nl : end; + continue; + } + if (*a != '&') { + continue; + } + const char *q = a + SKIP_ONE; + const char *first = q; + const char *last = q; + while (q < end && (op_ident_char(*q) || *q == '.')) { + if (*q == '.') { + last = q + SKIP_ONE; + } + q++; + } + if (q < end && *q == '{' && q > last && (size_t)(q - last) < OP_IDENT_MAX && + (size_t)(last - first) < OP_IDENT_MAX) { + char type[OP_IDENT_MAX]; + char qual[OP_IDENT_MAX]; + snprintf(type, sizeof(type), "%.*s", (int)(q - last), last); + snprintf(qual, sizeof(qual), "%.*s", + last > first ? (int)(last - first - SKIP_ONE) : 0, first); + linked += op_handle(gb, ix, fn, file, text, qual, type, calls[c].role); + } + } + } + } + return linked; +} + +/* ── API groups: +groupName / GroupName ───────────────────────────────── */ + +/* A Go identifier-or-DNS token at p (the group name). */ +static void op_token(const char *p, char *out, size_t out_sz) { + size_t n = 0; + while (p[n] && (op_ident_char(p[n]) || p[n] == '.' || p[n] == '-') && n + SKIP_ONE < out_sz) { + n++; + } + snprintf(out, out_sz, "%.*s", (int)n, p); +} + +/* Owned strings. */ +typedef struct { + char **items; + int count; + int cap; +} op_strs_t; + +static void op_strs_push(op_strs_t *v, const char *s) { + if (v->count == v->cap) { + int cap = v->cap ? v->cap * PAIR_LEN : OP_GROW_MIN; + char **grown = cbm_realloc(CBM_MEM_CLASS_OTHER, v->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + v->items = grown; + v->cap = cap; + } + char *copy = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, s); + if (copy) { + v->items[v->count++] = copy; + } +} + +/* group -> the package directories that declare it (owned keys and lists); + * `seen` holds "group\x1fdir" (owned keys) so a repeat is one lookup. */ +static void op_note_group(CBMHashTable *groups, CBMHashTable *seen, const char *group, + const char *dir) { + char key[PAIR_LEN * CBM_SZ_1K]; + op_key(key, sizeof(key), group, dir); + if (!group[0] || cbm_ht_get(seen, key)) { + return; + } + char *seen_key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!seen_key) { + return; + } + cbm_ht_set(seen, seen_key, seen_key); + op_strs_t *dirs = cbm_ht_get(groups, group); + if (!dirs) { + dirs = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*dirs)); + char *gkey = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, group); + if (!dirs || !gkey) { + cbm_free(CBM_MEM_CLASS_OTHER, dirs); + cbm_free(CBM_MEM_CLASS_OTHER, gkey); + return; + } + cbm_ht_set(groups, gkey, dirs); + } + op_strs_push(dirs, dir); +} + +static void op_scan_groups(CBMHashTable *groups, CBMHashTable *seen, const char *file, + const char *text) { + char dir[CBM_SZ_1K]; + op_dirname(file, dir, sizeof(dir)); + for (const char *p = strstr(text, "+groupName="); p; p = strstr(p + SKIP_ONE, "+groupName=")) { + char g[CBM_SZ_256]; + op_token(p + OP_SLEN("+groupName="), g, sizeof(g)); + op_note_group(groups, seen, g, dir); + } + for (const char *p = strstr(text, "GroupName"); p; p = strstr(p + SKIP_ONE, "GroupName")) { + const char *q = p + OP_SLEN("GroupName"); + while (*q == ' ' || *q == '\t') { + q++; + } + if (*q != '=') { + continue; + } + q++; + while (*q == ' ' || *q == '\t') { + q++; + } + if (*q == '"') { + char g[CBM_SZ_256]; + op_token(q + SKIP_ONE, g, sizeof(g)); + op_note_group(groups, seen, g, dir); + } + } +} + +static void op_free_seen(const char *key, void *value, void *ud) { + (void)value; + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static void op_free_groups(const char *key, void *value, void *ud) { + (void)ud; + op_strs_t *dirs = value; + for (int i = 0; i < dirs->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, dirs->items[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, dirs->items); + cbm_free(CBM_MEM_CLASS_OTHER, dirs); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +/* A JSON string property's value (simple, unescaped scalars). */ +static bool op_prop(const char *props, const char *key, char *out, size_t out_sz) { + char pat[CBM_SZ_64]; + snprintf(pat, sizeof(pat), "\"%s\":\"", key); + const char *p = props ? strstr(props, pat) : NULL; + if (!p) { + return false; + } + p += strlen(pat); + const char *e = strchr(p, '"'); + if (!e) { + return false; + } + snprintf(out, out_sz, "%.*s", (int)(e - p), p); + return true; +} + +/* CRD -> the Go type declared under its group, in every package marking it: + * an API group is one identity, and its CRDs and Go types routinely sit in + * different modules (cluster-api's api/, kubermatic's sdk/). */ +static int op_link_crd_types(cbm_gbuf_t *gb, const op_index_t *ix, CBMHashTable *groups) { + const cbm_gbuf_node_t **res = NULL; + int count = 0; + int linked = 0; + if (cbm_gbuf_find_by_label(gb, "Resource", &res, &count) != 0) { + return 0; + } + for (int i = 0; i < count; i++) { + char group[CBM_SZ_256]; + char kind[CBM_SZ_256]; + if (!res[i]->file_path || + !op_prop(res[i]->properties_json, "crd_group", group, sizeof(group)) || + !op_prop(res[i]->properties_json, "crd_kind", kind, sizeof(kind))) { + continue; + } + char key[PAIR_LEN * CBM_SZ_1K]; + const op_strs_t *dirs = cbm_ht_get(groups, group); + if (!dirs) { + continue; + } + atomic_fetch_add_explicit(&g_op_candidates, (uint64_t)dirs->count, memory_order_relaxed); + for (int d = 0; d < dirs->count; d++) { + op_key(key, sizeof(key), dirs->items[d], kind); + const op_nodes_t *v = cbm_ht_get(ix->dir_types, key); + for (int t = 0; v && t < v->count; t++) { + if (v->items[t]->label && strcmp(v->items[t]->label, "Struct") == 0) { + cbm_gbuf_insert_edge(gb, res[i]->id, v->items[t]->id, "USAGE", + "{\"ref\":\"crd-type\"}"); + linked++; + } + } + } + } + return linked; +} + +/* ── go:embed ─────────────────────────────────────────────────────────── */ + +/* Go's path.Match for one segment: '*', '?', '[...]', '\\' escapes. */ +static bool op_match(const char *pat, const char *s) { + while (*pat) { + if (*pat == '*') { + pat++; + for (const char *t = s;; t++) { + if (op_match(pat, t)) { + return true; + } + if (!*t) { + return false; + } + } + } + if (!*s) { + return false; + } + if (*pat == '?') { + pat++; + s++; + continue; + } + if (*pat == '[') { + const char *q = pat + SKIP_ONE; + bool neg = *q == '^'; + q += neg; + bool hit = false; + while (*q && *q != ']') { + char lo = *q; + char hi = lo; + if (q[SKIP_ONE] == '-' && q[PAIR_LEN] && q[PAIR_LEN] != ']') { + hi = q[PAIR_LEN]; + q += PAIR_LEN; + } + hit = hit || (*s >= lo && *s <= hi); + q++; + } + if (*q != ']' || hit == neg) { + return false; + } + pat = q + SKIP_ONE; + s++; + continue; + } + if (*pat == '\\' && pat[SKIP_ONE]) { + pat++; + } + if (*pat != *s) { + return false; + } + pat++; + s++; + } + return *s == '\0'; +} + +/* Does `rel` (a path below the Go file's directory) match the embed + * pattern: segment by segment, a matched directory taking its files except + * hidden ones ('.', '_') unless `all`. */ +static bool op_embeds(const char *pattern, const char *rel, bool all) { + char pat[CBM_SZ_1K]; + char path[CBM_SZ_1K]; + snprintf(pat, sizeof(pat), "%s", pattern); + snprintf(path, sizeof(path), "%s", rel); + char *ps = pat; + char *rs = path; + while (ps && rs) { + char *pn = strchr(ps, '/'); + char *rn = strchr(rs, '/'); + if (pn) { + *pn = '\0'; + } + if (rn) { + *rn = '\0'; + } + if (!op_match(ps, rs)) { + return false; + } + ps = pn ? pn + SKIP_ONE : NULL; + rs = rn ? rn + SKIP_ONE : NULL; + if (!ps) { + /* pattern consumed: the file itself, or a file under a matched dir */ + for (char *seg = rs; seg && !all;) { + if (seg[0] == '.' || seg[0] == '_') { + return false; + } + char *next = strchr(seg, '/'); + seg = next ? next + SKIP_ONE : NULL; + } + return true; + } + } + return false; +} + +static int op_cmp_path(const void *a, const void *b) { + const cbm_gbuf_node_t *x = *(const cbm_gbuf_node_t *const *)a; + const cbm_gbuf_node_t *y = *(const cbm_gbuf_node_t *const *)b; + return strcmp(x->file_path, y->file_path); +} + +/* REFERENCES_FILE from the Go file to every file its //go:embed lines name; + * `files` is every File node sorted by path, so a directory's files are one + * range. */ +static int op_embed(cbm_gbuf_t *gb, const cbm_gbuf_node_t *go_file, const char *text, + const cbm_gbuf_node_t **files, int nfiles) { + char dir[CBM_SZ_1K]; + op_dirname(go_file->file_path, dir, sizeof(dir)); + size_t dl = strlen(dir); + int linked = 0; + for (const char *p = strstr(text, "//go:embed "); p; p = strstr(p + SKIP_ONE, "//go:embed ")) { + if (p != text && p[-SKIP_ONE] != '\n') { + continue; /* a directive starts its line */ + } + const char *eol = strchr(p, '\n'); + char line[CBM_SZ_1K]; + const char *args = p + OP_SLEN("//go:embed "); + snprintf(line, sizeof(line), "%.*s", eol ? (int)(eol - args) : (int)strlen(args), args); + for (char *tok = line; *tok;) { + while (*tok == ' ' || *tok == '\t' || *tok == '\r') { + tok++; + } + char *e = tok; + while (*e && *e != ' ' && *e != '\t' && *e != '\r') { + e++; + } + char save = *e; + *e = '\0'; + bool all = strncmp(tok, "all:", OP_SLEN("all:")) == 0; + const char *pattern = all ? tok + OP_SLEN("all:") : tok; + if (pattern[0] && pattern[0] != '"' && pattern[0] != '`') { + /* files below dir: the sorted range starting at "dir/" */ + int lo = 0; + int hi = dl ? nfiles : 0; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (strncmp(files[mid]->file_path, dir, dl) < 0 || + (strncmp(files[mid]->file_path, dir, dl) == 0 && + files[mid]->file_path[dl] < '/')) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (int i = lo; i < nfiles; i++) { + const char *fp = files[i]->file_path; + if (dl && (strncmp(fp, dir, dl) != 0 || fp[dl] != '/')) { + break; + } + const char *rel = dl ? fp + dl + SKIP_ONE : fp; + if (files[i] != go_file && op_embeds(pattern, rel, all)) { + cbm_gbuf_insert_edge(gb, go_file->id, files[i]->id, "REFERENCES_FILE", + "{\"via\":\"go:embed\"}"); + linked++; + } + } + } + *e = save; + tok = e; + } + } + return linked; +} + +/* ── The pass ─────────────────────────────────────────────────────────── */ + +static bool op_is_go(const char *path) { + size_t n = strlen(path); + return n > OP_SLEN(".go") && strcmp(path + n - OP_SLEN(".go"), ".go") == 0; +} + +/* ── Helm charts the Go code loads (PHASE 3 P3.3) ─────────────────────── */ + +/* An installer written in Go (kubermatic's) deploys charts it loads from a + * charts directory: helm.LoadChart(filepath.Join(opt.ChartsDirectory, + * MonitoringChartsPrefix, NodeExporterChartName)). A chart loader is the + * Helm SDK's loader.Load / loader.LoadDir, or a function of the repository + * whose body reads `/Chart.yaml` or hands its parameter to the SDK + * loader. Its call DEPLOYS {via: helm-sdk} the chart directory its argument + * names, evaluated over string literals, the package's constants (and an + * imported package's), filepath.Join / path.Join, + and a local assigned + * once: a leading unknown part (a charts root chosen at run time) leaves a + * known tail, linked when exactly one chart directory of the repository ends + * with it. */ + +typedef struct { + char *dir; /* package directory */ + char *name; /* function */ + int param; /* index of the chart-directory parameter */ +} gc_loader_t; + +typedef struct { + const char *dir; /* graph-owned chart dir path */ + int64_t id; +} gc_chart_t; + +typedef struct { + gc_chart_t *items; + int count; + int cap; +} gc_charts_t; + +/* One chart load whose argument evaluated: linked once all are read. */ +typedef struct { + int64_t fn; + int line; + char *tail; + char *root; /* NULL: no unknown root */ +} gc_site_t; + +typedef struct { + cbm_pipeline_ctx_t *ctx; + const op_modules_t *mods; + CBMHashTable *by_file; + const cbm_gbuf_node_t **files; /* sorted File nodes */ + int nfiles; + gc_loader_t *loaders; + int nloaders; + int cap_loaders; + char **noted; /* Go files that mention a chart */ + int nnoted; + int cap_noted; + CBMHashTable *consts; /* dir -> CBMHashTable name -> expression (owned) */ + CBMHashTable *charts; /* chart dir basename -> gc_charts_t */ + gc_site_t *sites; + int nsites; + int cap_sites; + int edges; +} gc_t; + +typedef struct { + char s[CBM_SZ_1K]; + bool known; + bool open_root; /* an unknown leading part: s is a tail */ + char root[CBM_SZ_256]; /* that part's source text (opt.ChartsDirectory) */ +} gc_val_t; + +enum { GC_ARGS_MAX = 8, GC_EVAL_DEPTH = 16 }; + +static bool gc_grow(void **items, int *cap, int count, size_t elem) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : OP_GROW_MIN; + void *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, *items, (size_t)ncap * elem); + if (!grown) { + return false; + } + *items = grown; + *cap = ncap; + return true; +} + +/* Past a string / rune literal or a comment starting at p; p when none. */ +static const char *gc_skip(const char *p) { + if (*p == '"' || *p == '\'') { + char q = *p++; + while (*p && *p != q && *p != '\n') { + p += (*p == '\\' && p[SKIP_ONE]) ? PAIR_LEN : SKIP_ONE; + } + return *p == q ? p + SKIP_ONE : p; + } + if (*p == '`') { + const char *e = strchr(p + SKIP_ONE, '`'); + return e ? e + SKIP_ONE : p + strlen(p); + } + if (p[0] == '/' && p[SKIP_ONE] == '/') { + const char *e = strchr(p, '\n'); + return e ? e : p + strlen(p); + } + if (p[0] == '/' && p[SKIP_ONE] == '*') { + const char *e = strstr(p + PAIR_LEN, "*/"); + return e ? e + PAIR_LEN : p + strlen(p); + } + return p; +} + +/* The '}' closing the block opened at `open`. */ +static const char *gc_block_end(const char *open) { + int depth = 0; + for (const char *p = open; *p;) { + const char *q = gc_skip(p); + if (q != p) { + p = q; + continue; + } + depth += *p == '{'; + depth -= *p == '}'; + if (depth == 0) { + return p; + } + p++; + } + return open + strlen(open); +} + +/* The top-level comma-separated parts of the list opened at `open` ('('). + * Returns their count (at most `max`); *end: the closing ')'. */ +static int gc_args(const char *open, const char *starts[], size_t lens[], int max, + const char **end) { + int n = 0; + int depth = 0; + const char *start = open + SKIP_ONE; + for (const char *p = open; *p;) { + const char *q = gc_skip(p); + if (q != p) { + p = q; + continue; + } + if (strchr("([{", *p)) { + depth++; + } else if (strchr(")]}", *p)) { + depth--; + if (depth == 0) { + if (n < max && p > start) { + starts[n] = start; + lens[n++] = (size_t)(p - start); + } + *end = p; + return n; + } + } else if (*p == ',' && depth == SKIP_ONE && n < max) { + starts[n] = start; + lens[n++] = (size_t)(p - start); + start = p + SKIP_ONE; + } + p++; + } + *end = open + strlen(open); + return n; +} + +static void gc_trim(const char **s, size_t *n) { + while (*n && strchr(" \t\r\n", **s)) { + (*s)++; + (*n)--; + } + while (*n && strchr(" \t\r\n", (*s)[*n - SKIP_ONE])) { + (*n)--; + } +} + +static bool gc_is_ident(const char *s, size_t n) { + if (n == 0 || (s[0] >= '0' && s[0] <= '9')) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!op_ident_char(s[i])) { + return false; + } + } + return true; +} + +static bool gc_ident_eq(const char *s, size_t n, const char *name) { + return strlen(name) == n && strncmp(s, name, n) == 0; +} + +/* The repository directory a file binds package qualifier `qual` to. */ +static bool gc_qual_dir(const gc_t *g, const char *text, const char *qual, size_t ql, char *out, + size_t out_sz) { + char alias[OP_IDENT_MAX]; + char import[CBM_SZ_1K]; + if (ql == 0 || ql >= sizeof(alias)) { + return false; + } + snprintf(alias, sizeof(alias), "%.*s", (int)ql, qual); + op_import_path(text, alias, import, sizeof(import)); + return import[0] && op_import_dir(g->mods, import, out, out_sz); +} + +static bool gc_qual_is(const char *text, const char *qual, size_t ql, const char *path) { + char alias[OP_IDENT_MAX]; + char import[CBM_SZ_1K]; + if (ql == 0 || ql >= sizeof(alias)) { + return false; + } + snprintf(alias, sizeof(alias), "%.*s", (int)ql, qual); + op_import_path(text, alias, import, sizeof(import)); + return strcmp(import, path) == 0; +} + +static const char GC_SDK_LOADER[] = "helm.sh/helm/v3/pkg/chart/loader"; + +/* The flag before a noted file's path: it imports the SDK loader, or not. */ +enum { GC_NOTED_SDK = 'S', GC_NOTED_PLAIN = '-' }; + +/* ── Loaders: functions whose parameter is a chart directory ── */ + +static void gc_note_loader(gc_t *g, const char *dir, const char *name, size_t nl, int param) { + for (int i = 0; i < g->nloaders; i++) { + if (strcmp(g->loaders[i].dir, dir) == 0 && gc_ident_eq(name, nl, g->loaders[i].name)) { + return; + } + } + if (!gc_grow((void **)&g->loaders, &g->cap_loaders, g->nloaders, sizeof(*g->loaders))) { + return; + } + char nm[OP_IDENT_MAX]; + snprintf(nm, sizeof(nm), "%.*s", (int)nl, name); + g->loaders[g->nloaders++] = (gc_loader_t){cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir), + cbm_mem_strdup(CBM_MEM_CLASS_OTHER, nm), param}; +} + +/* Does body [b, e) load a chart from the parameter named p: Join(p, + * "Chart.yaml") or the SDK loader on p. */ +static bool gc_body_loads(const char *text, const char *b, const char *e, const char *p, + size_t pl) { + for (const char *s = b; s < e;) { + const char *q = gc_skip(s); + if (q != s) { + s = q; + continue; + } + const char *open = NULL; + bool join = false; + if (strncmp(s, "filepath.Join(", OP_SLEN("filepath.Join(")) == 0) { + open = s + OP_SLEN("filepath.Join"); + join = true; + } else if (strncmp(s, "path.Join(", OP_SLEN("path.Join(")) == 0 && + (s == b || !op_ident_char(s[-SKIP_ONE]))) { + open = s + OP_SLEN("path.Join"); + join = true; + } else if ((strncmp(s, ".Load(", OP_SLEN(".Load(")) == 0 || + strncmp(s, ".LoadDir(", OP_SLEN(".LoadDir(")) == 0)) { + const char *qe = s; + const char *qs = qe; + while (qs > text && op_ident_char(qs[-SKIP_ONE])) { + qs--; + } + if (gc_qual_is(text, qs, (size_t)(qe - qs), GC_SDK_LOADER)) { + open = strchr(s, '('); + } + } + if (!open) { + s++; + continue; + } + const char *starts[GC_ARGS_MAX]; + size_t lens[GC_ARGS_MAX]; + const char *end = NULL; + int n = gc_args(open, starts, lens, GC_ARGS_MAX, &end); + for (int i = 0; i < n; i++) { + gc_trim(&starts[i], &lens[i]); + } + if (n > 0 && lens[0] == pl && strncmp(starts[0], p, pl) == 0) { + if (!join && n == SKIP_ONE) { + return true; + } + if (join && n == PAIR_LEN && + (gc_ident_eq(starts[SKIP_ONE], lens[SKIP_ONE], "\"Chart.yaml\"") || + gc_ident_eq(starts[SKIP_ONE], lens[SKIP_ONE], "\"Chart.yml\""))) { + return true; + } + } + s = end > s ? end : s + SKIP_ONE; + } + return false; +} + +/* Note the file's top-level functions (not methods) that load a chart from + * a parameter. */ +static void gc_summarize(gc_t *g, const char *file, const char *text) { + char dir[CBM_SZ_1K]; + op_dirname(file, dir, sizeof(dir)); + for (const char *p = strstr(text, "func "); p; p = strstr(p + SKIP_ONE, "func ")) { + if (p > text && p[-SKIP_ONE] != '\n') { + continue; + } + const char *name = p + OP_SLEN("func "); + const char *ne = name; + while (op_ident_char(*ne)) { + ne++; + } + if (ne == name || *ne != '(') { + continue; /* a method, or generic */ + } + const char *starts[GC_ARGS_MAX]; + size_t lens[GC_ARGS_MAX]; + const char *pend = NULL; + int np = gc_args(ne, starts, lens, GC_ARGS_MAX, &pend); + const char *open = pend; + while (*open && *open != '{' && *open != '\n') { + open++; + } + if (*open != '{') { + continue; + } + const char *close = gc_block_end(open); + for (int i = 0; i < np; i++) { + const char *ps = starts[i]; + size_t pl = lens[i]; + gc_trim(&ps, &pl); + size_t idl = 0; + while (idl < pl && op_ident_char(ps[idl])) { + idl++; + } + if (idl && gc_body_loads(text, open, close, ps, idl)) { + gc_note_loader(g, dir, name, (size_t)(ne - name), i); + } + } + } +} + +/* ── Constants of a package ── */ + +static void gc_free_expr(const char *key, void *value, void *ud) { + (void)ud; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); + cbm_free(CBM_MEM_CLASS_OTHER, value); +} + +static void gc_free_consts(const char *key, void *value, void *ud) { + (void)ud; + cbm_ht_foreach(value, gc_free_expr, NULL); + cbm_ht_free(value); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +/* One `NAME [TYPE] = expr` of a const declaration, line [ls, le). */ +static void gc_const_line(CBMHashTable *t, const char *ls, const char *le) { + while (ls < le && (*ls == ' ' || *ls == '\t')) { + ls++; + } + const char *ne = ls; + while (ne < le && op_ident_char(*ne)) { + ne++; + } + const char *eq = memchr(ne, '=', (size_t)(le - ne)); + if (ne == ls || !eq || (eq + SKIP_ONE < le && eq[SKIP_ONE] == '=')) { + return; + } + const char *ve = op_comment_start(eq + SKIP_ONE, le); + const char *vs = eq + SKIP_ONE; + size_t vl = (size_t)(ve - vs); + gc_trim(&vs, &vl); + char name[OP_IDENT_MAX]; + snprintf(name, sizeof(name), "%.*s", (int)(ne - ls), ls); + if (vl == 0 || cbm_ht_get(t, name)) { + return; + } + char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, name); + char *val = cbm_alloc(CBM_MEM_CLASS_OTHER, vl + SKIP_ONE); + if (!key || !val) { + cbm_free(CBM_MEM_CLASS_OTHER, key); + cbm_free(CBM_MEM_CLASS_OTHER, val); + return; + } + memcpy(val, vs, vl); + val[vl] = '\0'; + cbm_ht_set(t, key, val); +} + +static void gc_scan_consts(CBMHashTable *t, const char *text) { + for (const char *p = strstr(text, "const"); p; p = strstr(p + SKIP_ONE, "const")) { + if ((p > text && p[-SKIP_ONE] != '\n') || op_ident_char(p[OP_SLEN("const")])) { + continue; + } + const char *q = p + OP_SLEN("const"); + while (*q == ' ' || *q == '\t') { + q++; + } + if (*q == '(') { + const char *starts[SKIP_ONE]; + size_t lens[SKIP_ONE]; + const char *end = NULL; + (void)gc_args(q, starts, lens, 0, &end); + for (const char *l = q + SKIP_ONE; l < end;) { + const char *le = memchr(l, '\n', (size_t)(end - l)); + le = le ? le : end; + gc_const_line(t, l, le); + l = le + SKIP_ONE; + } + } else { + const char *le = strchr(q, '\n'); + gc_const_line(t, q, le ? le : q + strlen(q)); + } + } +} + +/* The constants of package directory `dir` (its non-test Go files). */ +static CBMHashTable *gc_consts(gc_t *g, const char *dir) { + CBMHashTable *t = cbm_ht_get(g->consts, dir); + if (t) { + return t; + } + t = cbm_ht_create(0); + char *key = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, dir); + if (!t || !key) { + if (t) { + cbm_ht_free(t); + } + cbm_free(CBM_MEM_CLASS_OTHER, key); + return NULL; + } + cbm_ht_set(g->consts, key, t); + size_t dl = strlen(dir); + for (int i = 0; i < g->nfiles; i++) { + const char *fp = g->files[i]->file_path; + const char *slash = strrchr(fp, '/'); + size_t fdl = slash ? (size_t)(slash - fp) : 0; + size_t fl = strlen(fp); + if (fdl != dl || strncmp(fp, dir, dl) != 0 || !op_is_go(fp) || + (fl > OP_SLEN("_test.go") && strcmp(fp + fl - OP_SLEN("_test.go"), "_test.go") == 0)) { + continue; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", g->ctx->repo_path, fp); + char *text = op_read_file(path); + if (text) { + gc_scan_consts(t, text); + cbm_free(CBM_MEM_CLASS_OTHER, text); + } + } + return t; +} + +/* ── Values ── */ + +typedef struct { + gc_t *g; + const char *text; /* the call's file */ + const char *dir; /* its package */ + const char *sig; /* the enclosing function's `func` .. its body, NULL at top level */ + const char *body; + const char *body_end; + const char *at; /* the call */ +} gc_scope_t; + +/* Is `name` a word of the enclosing function's signature (a parameter). */ +static bool gc_is_param(const gc_scope_t *sc, const char *name, size_t nl) { + for (const char *p = sc->sig; sc->sig && p + nl <= sc->body; p++) { + if (strncmp(p, name, nl) == 0 && !op_ident_char(p[-SKIP_ONE]) && !op_ident_char(p[nl])) { + return true; + } + } + return false; +} + +static void gc_eval(const gc_scope_t *sc, const char *s, size_t n, int depth, gc_val_t *out); + +static void gc_join(gc_val_t *acc, const gc_val_t *part, bool path) { + if (!acc->known) { + return; + } + bool empty = !acc->s[0]; + if (!part->known || part->open_root) { + if (!path || !empty) { + acc->known = false; /* an unknown part after a known one */ + return; + } + acc->open_root = true; /* still the leading, unknown root */ + if (!part->known) { + return; + } + } + size_t al = strlen(acc->s); + bool sep = path && al && part->s[0] && acc->s[al - SKIP_ONE] != '/' && part->s[0] != '/'; + snprintf(acc->s + al, sizeof(acc->s) - al, "%s%s", sep ? "/" : "", part->s); +} + +/* A local assigned exactly once in the enclosing function, before the call. */ +static bool gc_local(const gc_scope_t *sc, const char *name, size_t nl, const char **vs, + size_t *vlen) { + if (!sc->body) { + return false; + } + int seen = 0; + for (const char *p = sc->body; p < sc->body_end;) { + const char *q = gc_skip(p); + if (q != p) { + p = q; + continue; + } + if (strncmp(p, name, nl) == 0 && (p == sc->body || !op_ident_char(p[-SKIP_ONE])) && + !op_ident_char(p[nl]) && p[-SKIP_ONE] != '.') { + const char *a = p + nl; + while (*a == ' ' || *a == '\t') { + a++; + } + if (a[0] && strchr("+-*/%|&^", a[0]) && a[SKIP_ONE] == '=') { + return false; /* modified in place */ + } + bool assign = + (a[0] == ':' && a[SKIP_ONE] == '=') || (a[0] == '=' && a[SKIP_ONE] != '='); + if (assign) { + if (++seen > SKIP_ONE || p > sc->at) { + return false; + } + const char *v = a + (a[0] == ':' ? PAIR_LEN : SKIP_ONE); + const char *le = strchr(v, '\n'); + le = le ? le : v + strlen(v); + *vs = v; + *vlen = (size_t)(op_comment_start(v, le) - v); + } + } + p++; + } + return seen == SKIP_ONE; +} + +static void gc_const_value(const gc_scope_t *sc, const char *dir, const char *name, int depth, + gc_val_t *out) { + CBMHashTable *t = gc_consts(sc->g, dir); + const char *expr = t ? cbm_ht_get(t, name) : NULL; + if (!expr) { + out->known = false; + return; + } + gc_scope_t inner = {sc->g, sc->text, dir, NULL, NULL, NULL, NULL}; + gc_eval(&inner, expr, strlen(expr), depth + SKIP_ONE, out); +} + +static void gc_eval(const gc_scope_t *sc, const char *s, size_t n, int depth, gc_val_t *out) { + *out = (gc_val_t){.known = false}; + gc_trim(&s, &n); + if (n == 0 || depth > GC_EVAL_DEPTH) { + return; + } + /* a + b */ + int pd = 0; + for (size_t i = 0; i < n; i++) { + const char *q = gc_skip(s + i); + if (q != s + i) { + i = (size_t)(q - s) - SKIP_ONE; + continue; + } + pd += strchr("([{", s[i]) != NULL; + pd -= strchr(")]}", s[i]) != NULL; + if (s[i] == '+' && pd == 0) { + gc_val_t a; + gc_val_t b; + gc_eval(sc, s, i, depth + SKIP_ONE, &a); + gc_eval(sc, s + i + SKIP_ONE, n - i - SKIP_ONE, depth + SKIP_ONE, &b); + *out = a; + if (!b.known || b.open_root) { + out->known = false; + } else if (out->known) { + gc_join(out, &b, false); + } + return; + } + } + if ((s[0] == '"' || s[0] == '`') && gc_skip(s) == s + n) { + size_t w = 0; + for (size_t i = SKIP_ONE; i + SKIP_ONE < n && w + SKIP_ONE < sizeof(out->s); i++) { + if (s[0] == '"' && s[i] == '\\' && i + PAIR_LEN < n) { + i++; + } + out->s[w++] = s[i]; + } + out->s[w] = '\0'; + out->known = true; + return; + } + bool fp = n > OP_SLEN("filepath.Join(") && + strncmp(s, "filepath.Join(", OP_SLEN("filepath.Join(")) == 0; + bool pp = n > OP_SLEN("path.Join(") && strncmp(s, "path.Join(", OP_SLEN("path.Join(")) == 0; + if ((fp || pp) && s[n - SKIP_ONE] == ')') { + const char *open = s + (fp ? OP_SLEN("filepath.Join") : OP_SLEN("path.Join")); + const char *starts[GC_ARGS_MAX]; + size_t lens[GC_ARGS_MAX]; + const char *end = NULL; + int na = gc_args(open, starts, lens, GC_ARGS_MAX, &end); + if (end != s + n - SKIP_ONE || na == 0 || na == GC_ARGS_MAX) { + return; + } + *out = (gc_val_t){.known = true}; + for (int i = 0; i < na && out->known; i++) { + gc_val_t part; + gc_eval(sc, starts[i], lens[i], depth + SKIP_ONE, &part); + bool was_open = out->open_root; + gc_join(out, &part, true); + if (out->known && out->open_root && !was_open) { + if (part.known && part.root[0]) { + snprintf(out->root, sizeof(out->root), "%s", part.root); + } else { + const char *rs = starts[i]; + size_t rl = lens[i]; + gc_trim(&rs, &rl); + snprintf(out->root, sizeof(out->root), "%.*s", (int)rl, rs); + } + } else if (out->known && was_open && !part.known) { + out->root[0] = '\0'; /* several unknown parts: no one root */ + } + } + return; + } + if (gc_is_ident(s, n)) { + const char *vs = NULL; + size_t vl = 0; + char name[OP_IDENT_MAX]; + snprintf(name, sizeof(name), "%.*s", (int)n, s); + if (gc_local(sc, s, n, &vs, &vl)) { + gc_scope_t outer = *sc; + outer.at = vs; /* what the local reads, read before it */ + gc_eval(&outer, vs, vl, depth + SKIP_ONE, out); + return; + } + if (gc_is_param(sc, s, n)) { + return; /* an argument: whatever the caller passes */ + } + gc_const_value(sc, sc->dir, name, depth, out); + return; + } + const char *dot = memchr(s, '.', n); + if (dot && gc_is_ident(s, (size_t)(dot - s)) && + gc_is_ident(dot + SKIP_ONE, n - (size_t)(dot - s) - SKIP_ONE)) { + char dir[CBM_SZ_1K]; + char name[OP_IDENT_MAX]; + snprintf(name, sizeof(name), "%.*s", (int)(n - (size_t)(dot - s) - SKIP_ONE), + dot + SKIP_ONE); + if (gc_qual_dir(sc->g, sc->text, s, (size_t)(dot - s), dir, sizeof(dir))) { + gc_const_value(sc, dir, name, depth, out); + } + } +} + +/* ── Charts and edges ── */ + +static void gc_free_chart_list(const char *key, void *value, void *ud) { + (void)ud; + gc_charts_t *v = value; + cbm_free(CBM_MEM_CLASS_OTHER, v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static CBMHashTable *gc_chart_index(const cbm_gbuf_t *gb) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!ix || cbm_gbuf_find_by_label(gb, "Chart", &nodes, &count) != 0) { + return ix; + } + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + const char *qn = nodes[i]->qualified_name; + const char *slash = fp ? strrchr(fp, '/') : NULL; + const char *base = slash ? slash + SKIP_ONE : fp; + if (!fp || !qn || strstr(qn, ".__helm_dep__.") || + (strcmp(base, "Chart.yaml") != 0 && strcmp(base, "Chart.yml") != 0) || !slash) { + continue; + } + const char *db = slash; + while (db > fp && db[-SKIP_ONE] != '/') { + db--; + } + char key[CBM_SZ_256]; + snprintf(key, sizeof(key), "%.*s", (int)(slash - db), db); + gc_charts_t *v = cbm_ht_get(ix, key); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *k = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!v || !k) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, k); + continue; + } + cbm_ht_set(ix, k, v); + } + if (gc_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = (gc_chart_t){fp, nodes[i]->id}; + } + } + return ix; +} + +/* The one chart directory a value names: the tail itself, or the only chart + * directory ending with it. */ +static const gc_chart_t *gc_match(gc_t *g, const gc_val_t *v, char *dir, size_t dir_sz) { + const char *t = v->s; + while (t[0] == '.' && t[SKIP_ONE] == '/') { + t += PAIR_LEN; + } + while (*t == '/') { + t++; + } + snprintf(dir, dir_sz, "%s", t); + size_t dl = strlen(dir); + while (dl && dir[dl - SKIP_ONE] == '/') { + dir[--dl] = '\0'; + } + if (!dl) { + return NULL; + } + const char *base = strrchr(dir, '/'); + base = base ? base + SKIP_ONE : dir; + const gc_charts_t *list = cbm_ht_get(g->charts, base); + const gc_chart_t *hit = NULL; + for (int i = 0; list && i < list->count; i++) { + atomic_fetch_add_explicit(&g_op_candidates, SKIP_ONE, memory_order_relaxed); + const char *cd = list->items[i].dir; /* "/Chart.yaml" */ + size_t cl = (size_t)(strrchr(cd, '/') - cd); + bool same = cl == dl && strncmp(cd, dir, dl) == 0; + bool tail = cl > dl && cd[cl - dl - SKIP_ONE] == '/' && strncmp(cd + cl - dl, dir, dl) == 0; + if (same || tail) { + if (hit) { + return NULL; + } + hit = &list->items[i]; + } + } + if (hit) { + const char *cd = hit->dir; + snprintf(dir, dir_sz, "%.*s", (int)(strrchr(cd, '/') - cd), cd); + } + return hit; +} + +/* The body of the top-level function around `at`, else NULL; *sig: its + * `func` keyword. */ +static const char *gc_body_of(const char *text, const char *at, const char **end, + const char **sig) { + const char *best = NULL; + for (const char *p = strstr(text, "func "); p && p < at; p = strstr(p + SKIP_ONE, "func ")) { + if (p == text || p[-SKIP_ONE] == '\n') { + best = p; + } + } + if (!best) { + return NULL; + } + const char *open = best; + int pd = 0; + while (*open && !(*open == '{' && pd == 0)) { + const char *q = gc_skip(open); + if (q != open) { + open = q; + continue; + } + pd += *open == '('; + pd -= *open == ')'; + open++; + } + if (*open != '{') { + return NULL; + } + const char *close = gc_block_end(open); + if (close < at) { + return NULL; + } + *end = close; + *sig = best; + return open; +} + +static void gc_link_call(gc_t *g, const char *file, const char *text, const char *name_at, + const char *open, int param) { + const char *starts[GC_ARGS_MAX]; + size_t lens[GC_ARGS_MAX]; + const char *end = NULL; + int n = gc_args(open, starts, lens, GC_ARGS_MAX, &end); + if (param >= n) { + return; + } + char dir[CBM_SZ_1K]; + op_dirname(file, dir, sizeof(dir)); + const char *body_end = NULL; + const char *sig = NULL; + const char *body = gc_body_of(text, name_at, &body_end, &sig); + gc_scope_t sc = {g, text, dir, body ? sig : NULL, body, body_end, name_at}; + gc_val_t v; + gc_eval(&sc, starts[param], lens[param], 0, &v); + int line = op_line_of(text, name_at); + const cbm_gbuf_node_t *fn = v.known ? op_enclosing(g->by_file, file, line) : NULL; + if (!fn || !gc_grow((void **)&g->sites, &g->cap_sites, g->nsites, sizeof(*g->sites))) { + return; + } + g->sites[g->nsites++] = + (gc_site_t){fn->id, line, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, v.s), + v.open_root && v.root[0] ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, v.root) : NULL}; +} + +static void gc_edge(gc_t *g, const gc_site_t *s, const gc_chart_t *hit, const char *chart, + const char *root) { + char esc[CBM_SZ_1K]; + char resc[CBM_SZ_512]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), chart); + cbm_json_escape(resc, (int)sizeof(resc), root ? root : ""); + snprintf(props, sizeof(props), "{\"via\":\"helm-sdk\",\"chart\":\"%s\"%s%s%s,\"line\":%d}", esc, + root ? ",\"root\":\"" : "", root ? resc : "", root ? "\"" : "", s->line); + cbm_gbuf_insert_edge(g->ctx->gbuf, s->fn, hit->id, "DEPLOYS", props); + g->edges++; +} + +/* The repository dir a chart load's unknown root stood for: the matched + * chart directory without the tail. */ +static void gc_root_dir(const char *chart, const char *tail, char *out, size_t out_sz) { + while (tail[0] == '.' && tail[SKIP_ONE] == '/') { + tail += PAIR_LEN; + } + while (*tail == '/') { + tail++; + } + size_t cl = strlen(chart); + size_t tl = strlen(tail); + while (tl && tail[tl - SKIP_ONE] == '/') { + tl--; + } + size_t rl = cl > tl ? cl - tl - SKIP_ONE : 0; + snprintf(out, out_sz, "%.*s", (int)rl, chart); +} + +static const char GC_CONFLICT[] = "\x01"; + +/* Remember that root text `root` stood for directory `rdir`; two different + * directories for one root text: it stands for none. */ +static void gc_note_root(CBMHashTable *roots, const char *root, const char *rdir) { + const char *known = cbm_ht_get(roots, root); + if (!known) { + char *k = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, root); + char *d = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, rdir); + if (k && d) { + cbm_ht_set(roots, k, d); + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, k); + cbm_free(CBM_MEM_CLASS_OTHER, d); + return; + } + if (strcmp(known, GC_CONFLICT) != 0 && strcmp(known, rdir) != 0) { + char *c = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, GC_CONFLICT); + if (c) { + cbm_free(CBM_MEM_CLASS_OTHER, cbm_ht_set(roots, cbm_ht_get_key(roots, root), c)); + } + } +} + +/* Link the chart loads: a tail exactly one chart directory ends with; else, + * when the loads under the same unknown root (opt.ChartsDirectory) whose + * tail matched one directory all put that root at one place, /. */ +static void gc_link_sites(gc_t *g) { + CBMHashTable *roots = cbm_ht_create(0); /* root text -> its dir (owned) */ + for (int i = 0; roots && i < g->nsites; i++) { + const gc_site_t *s = &g->sites[i]; + gc_val_t v = {.known = true, .open_root = s->root != NULL}; + snprintf(v.s, sizeof(v.s), "%s", s->tail); + char chart[CBM_SZ_1K]; + if (s->root && gc_match(g, &v, chart, sizeof(chart))) { + char rdir[CBM_SZ_1K]; + gc_root_dir(chart, s->tail, rdir, sizeof(rdir)); + gc_note_root(roots, s->root, rdir); + } + } + for (int i = 0; roots && i < g->nsites; i++) { + const gc_site_t *s = &g->sites[i]; + gc_val_t v = {.known = true, .open_root = s->root != NULL}; + snprintf(v.s, sizeof(v.s), "%s", s->tail); + char chart[CBM_SZ_1K]; + const gc_chart_t *hit = gc_match(g, &v, chart, sizeof(chart)); + if (hit) { + gc_edge(g, s, hit, chart, NULL); + continue; + } + const char *rdir = s->root ? cbm_ht_get(roots, s->root) : NULL; + if (!rdir || strcmp(rdir, GC_CONFLICT) == 0) { + continue; + } + gc_val_t full = {.known = true}; + snprintf(full.s, sizeof(full.s), "%s%s%s", rdir, rdir[0] ? "/" : "", s->tail); + char want[CBM_SZ_1K]; + snprintf(want, sizeof(want), "%s", full.s); + hit = gc_match(g, &full, chart, sizeof(chart)); + if (hit && strcmp(chart, want) == 0) { + gc_edge(g, s, hit, chart, rdir); + } + } + if (roots) { + cbm_ht_foreach(roots, gc_free_expr, NULL); + cbm_ht_free(roots); + } +} + +/* The chart loads in one file: calls of a noted loader (qualified by an + * import of its package, or unqualified in it) and of the SDK loader. */ +static void gc_calls(gc_t *g, const char *file, const char *text) { + char dir[CBM_SZ_1K]; + op_dirname(file, dir, sizeof(dir)); + for (const char *p = text; *p;) { + const char *q = gc_skip(p); + if (q != p) { + p = q; + continue; + } + if (!op_ident_char(*p) || (p > text && op_ident_char(p[-SKIP_ONE]))) { + p++; + continue; + } + const char *ne = p; + while (op_ident_char(*ne)) { + ne++; + } + if (*ne != '(') { + p = ne; + continue; + } + size_t nl = (size_t)(ne - p); + const char *qual = NULL; + size_t ql = 0; + if (p > text && p[-SKIP_ONE] == '.') { + qual = p - SKIP_ONE; + while (qual > text && op_ident_char(qual[-SKIP_ONE])) { + qual--; + } + ql = (size_t)(p - SKIP_ONE - qual); + } + bool is_def = p >= text + OP_SLEN("func ") && + strncmp(p - OP_SLEN("func "), "func ", OP_SLEN("func ")) == 0; + int param = -1; + if (!is_def && qual && (gc_ident_eq(p, nl, "Load") || gc_ident_eq(p, nl, "LoadDir")) && + gc_qual_is(text, qual, ql, GC_SDK_LOADER)) { + param = 0; + } + char qdir[CBM_SZ_1K]; + bool resolved = !qual || gc_qual_dir(g, text, qual, ql, qdir, sizeof(qdir)); + for (int i = 0; !is_def && param < 0 && resolved && i < g->nloaders; i++) { + if (gc_ident_eq(p, nl, g->loaders[i].name) && + strcmp(g->loaders[i].dir, qual ? qdir : dir) == 0) { + param = g->loaders[i].param; + } + } + if (param >= 0) { + gc_link_call(g, file, text, p, ne, param); + } + p = ne; + } +} + +/* A Go file that mentions a chart: its loaders noted now; the file is read + * again for its loader calls once every loader is known. A file importing + * the SDK loader is marked: it is read again even when no loader of the + * repository exists. */ +static void gc_note(gc_t *g, const char *file, const char *text) { + if (!strstr(text, "Chart") && !strstr(text, "chart/loader")) { + return; + } + gc_summarize(g, file, text); + bool sdk = strstr(text, GC_SDK_LOADER) != NULL; + size_t fl = strlen(file); + char *entry = cbm_alloc(CBM_MEM_CLASS_OTHER, fl + PAIR_LEN); /* flag + path */ + if (!entry || !gc_grow((void **)&g->noted, &g->cap_noted, g->nnoted, sizeof(*g->noted))) { + cbm_free(CBM_MEM_CLASS_OTHER, entry); + return; + } + entry[0] = sdk ? GC_NOTED_SDK : GC_NOTED_PLAIN; + memcpy(entry + SKIP_ONE, file, fl + SKIP_ONE); + g->noted[g->nnoted++] = entry; +} + +static int gc_run(gc_t *g) { + if (g->nnoted == 0) { + return 0; + } + g->consts = cbm_ht_create(0); + g->charts = gc_chart_index(g->ctx->gbuf); + for (int i = 0; g->consts && g->charts && i < g->nnoted; i++) { + const char *noted = g->noted[i]; + if (!noted || (g->nloaders == 0 && noted[0] != GC_NOTED_SDK)) { + continue; /* no loader it could call */ + } + const char *file = noted + SKIP_ONE; + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", g->ctx->repo_path, file); + char *text = op_read_file(path); + if (text) { + gc_calls(g, file, text); + cbm_free(CBM_MEM_CLASS_OTHER, text); + } + } + if (g->charts) { + gc_link_sites(g); + } + if (g->consts) { + cbm_ht_foreach(g->consts, gc_free_consts, NULL); + cbm_ht_free(g->consts); + } + if (g->charts) { + cbm_ht_foreach(g->charts, gc_free_chart_list, NULL); + cbm_ht_free(g->charts); + } + return g->edges; +} + +static void gc_free(gc_t *g) { + for (int i = 0; i < g->nloaders; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, g->loaders[i].dir); + cbm_free(CBM_MEM_CLASS_OTHER, g->loaders[i].name); + } + cbm_free(CBM_MEM_CLASS_OTHER, g->loaders); + for (int i = 0; i < g->nnoted; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, g->noted[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)g->noted); + for (int i = 0; i < g->nsites; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, g->sites[i].tail); + cbm_free(CBM_MEM_CLASS_OTHER, g->sites[i].root); + } + cbm_free(CBM_MEM_CLASS_OTHER, g->sites); +} + +int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx) { + cbm_gbuf_t *gb = ctx->gbuf; + const cbm_gbuf_node_t **all = NULL; + int count = 0; + if (!gb || !ctx->repo_path || cbm_gbuf_find_by_label(gb, "File", &all, &count) != 0) { + return 0; + } + const cbm_gbuf_node_t **files = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(count ? count : SKIP_ONE) * sizeof(*files)); + if (!files) { + return CBM_NOT_FOUND; + } + int nfiles = 0; + for (int i = 0; i < count; i++) { + if (all[i]->file_path) { + files[nfiles++] = all[i]; + } + } + qsort((void *)files, (size_t)nfiles, sizeof(*files), op_cmp_path); + CBMHashTable *by_file = op_functions_by_file(gb); + CBMHashTable *groups = cbm_ht_create(0); + CBMHashTable *seen = cbm_ht_create(0); + op_modules_t mods = {0}; + for (int i = 0; i < nfiles; i++) { + const char *slash = strrchr(files[i]->file_path, '/'); + if (strcmp(slash ? slash + SKIP_ONE : files[i]->file_path, "go.mod") == 0) { + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, files[i]->file_path); + char *text = op_read_file(path); + if (text) { + op_note_module(&mods, files[i]->file_path, text); + cbm_free(CBM_MEM_CLASS_OTHER, text); + } + } + } + op_index_t ix = {.mods = &mods, .dir_types = op_types_by_dir(gb)}; + gc_t gc = {0}; + int handles = 0; + int embeds = 0; + bool ready = by_file && groups && seen && ix.dir_types; + for (int i = 0; ready && i < nfiles; i++) { + if (!op_is_go(files[i]->file_path) || cbm_pipeline_check_cancel(ctx)) { + continue; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, files[i]->file_path); + char *text = op_read_file(path); + if (!text) { + continue; + } + if (strstr(text, "controller-runtime") || strstr(text, "source.Kind(")) { + handles += op_controller_calls(gb, by_file, &ix, files[i]->file_path, text); + } + op_scan_groups(groups, seen, files[i]->file_path, text); + if (strstr(text, "//go:embed ")) { + embeds += op_embed(gb, files[i], text, files, nfiles); + } + gc_note(&gc, files[i]->file_path, text); + cbm_free(CBM_MEM_CLASS_OTHER, text); + } + int crd_types = ready ? op_link_crd_types(gb, &ix, groups) : 0; + gc.ctx = ctx; + gc.mods = &mods; + gc.by_file = by_file; + gc.files = files; + gc.nfiles = nfiles; + int deploys = ready ? gc_run(&gc) : 0; + gc_free(&gc); + if (ix.dir_types) { + cbm_ht_foreach(ix.dir_types, op_free_nodes_entry, NULL); + cbm_ht_free(ix.dir_types); + } + op_modules_free(&mods); + if (groups) { + cbm_ht_foreach(groups, op_free_groups, NULL); + cbm_ht_free(groups); + } + if (seen) { + cbm_ht_foreach(seen, op_free_seen, NULL); + cbm_ht_free(seen); + } + if (by_file) { + cbm_ht_foreach(by_file, op_free_spans, NULL); + cbm_ht_free(by_file); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)files); + if (handles + embeds + crd_types + deploys > 0) { + char h[CBM_SZ_32]; + char e[CBM_SZ_32]; + char c[CBM_SZ_32]; + char d[CBM_SZ_32]; + snprintf(h, sizeof(h), "%d", handles); + snprintf(e, sizeof(e), "%d", embeds); + snprintf(c, sizeof(c), "%d", crd_types); + snprintf(d, sizeof(d), "%d", deploys); + cbm_log_info("pass.operator", "handles", h, "embeds", e, "crd_types", c, "chart_deploys", + d); + } + return 0; +} diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index f12dd8d4e..f2a451f89 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -31,7 +31,8 @@ #include #include -/* Read an entire file into a malloc'd buffer. Returns NULL on failure. */ +/* Read an entire file into a core-owned buffer (class extract; release with + * cbm_free(CBM_MEM_CLASS_EXTRACT, ...)). Returns NULL on failure. */ static char *pkgmap_read_file(const char *path, int *out_len) { FILE *f = cbm_fopen(path, "rb"); if (!f) { @@ -44,7 +45,7 @@ static char *pkgmap_read_file(const char *path, int *out_len) { (void)fclose(f); return NULL; } - char *buf = (char *)malloc((size_t)size + SKIP_ONE); + char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)size + SKIP_ONE); if (!buf) { (void)fclose(f); return NULL; @@ -1201,7 +1202,7 @@ static int pkgmap_walk_dir(const char *abs_dir, const char *rel_dir, cbm_pkg_ent if (cbm_pkgmap_try_parse(name, rel_path, source, source_len, entries)) { parsed++; } - free(source); + cbm_free(CBM_MEM_CLASS_EXTRACT, source); } cbm_closedir(dir); return parsed; @@ -1247,7 +1248,7 @@ CBMHashTable *cbm_pkgmap_build_from_files(const cbm_file_info_t *files, int file continue; } cbm_pkgmap_try_parse(basename, files[i].rel_path, source, source_len, &entries); - free(source); + cbm_free(CBM_MEM_CLASS_EXTRACT, source); } CBMHashTable *map = cbm_pkgmap_build(&entries, SKIP_ONE, project_name); @@ -1282,7 +1283,7 @@ CBMHashTable *cbm_pkgmap_build_from_repo(const char *repo_path, const cbm_file_i continue; } cbm_pkgmap_try_parse(basename, files[i].rel_path, source, source_len, &entries); - free(source); + cbm_free(CBM_MEM_CLASS_EXTRACT, source); } int from_walk = cbm_pkgmap_scan_repo(repo_path, &entries, excluded_dirs, excluded_count); @@ -2213,6 +2214,63 @@ cbm_psr4_member_t cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char return *out_qn ? CBM_PSR4_MEMBER_RESOLVED : CBM_PSR4_MEMBER_UNRESOLVED; } +/* A kustomization entry naming a remote base -- a URL, a go-getter host path, + * an ssh remote or a `?ref=` pin -- lies outside the repository. */ +static bool kustomize_remote_entry(const char *entry) { + static const char *const hosts[] = {"github.com/", "gitlab.com/", "bitbucket.org/", "git@", + NULL}; + if (strstr(entry, "://") || strstr(entry, "?ref=") || strstr(entry, ".git/")) { + return true; + } + for (int i = 0; hosts[i]; i++) { + if (strncmp(entry, hosts[i], strlen(hosts[i])) == 0) { + return true; + } + } + return false; +} + +static bool kustomize_file_label(const char *label) { + return label && (strcmp(label, "Module") == 0 || strcmp(label, "File") == 0); +} + +/* A kustomization entry is a path relative to the kustomization's own + * directory -- a manifest file or a directory -- or a remote base; it never + * names a symbol. Resolve it exactly or not at all: the name-guess strategies + * bound a remote base's URL segment `deploy` to a shell function, component + * paths to Go structs, and an entry's basename to a same-named file beside + * the importer. */ +static const cbm_gbuf_node_t *resolve_kustomize_entry(const cbm_pipeline_ctx_t *ctx, + const char *source_rel, const char *entry) { + if (!entry[0] || kustomize_remote_entry(entry)) { + return NULL; + } + /* "x.yaml" and "./x.yaml" name the same file; make every entry relative. */ + bool dotted = entry[0] == '.' && (entry[1] == '/' || (entry[1] == '.' && entry[2] == '/')); + char rel[PKGMAP_PATH_BUF]; + snprintf(rel, sizeof(rel), "%s%s", dotted ? "" : "./", entry); + char *path = cbm_pipeline_resolve_relative_import(source_rel, rel); + if (!path) { + return NULL; + } + const cbm_gbuf_node_t *found = NULL; + char *qn = cbm_pipeline_fqn_module(ctx->project_name, path); + const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(ctx->gbuf, qn) : NULL; + free(qn); + if (n && kustomize_file_label(n->label)) { + found = n; + } else { + char *fqn = cbm_pipeline_fqn_folder(ctx->project_name, path); + n = fqn ? cbm_gbuf_find_by_qn(ctx->gbuf, fqn) : NULL; + free(fqn); + if (n && n->label && strcmp(n->label, "Folder") == 0) { + found = n; + } + } + free(path); + return found; +} + const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t *ctx, const char *source_rel, const char *source_file_qn, @@ -2222,6 +2280,18 @@ const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t return NULL; } + /* The importing file's language gates the strategies below. */ + const char *src_base = source_rel ? source_rel : ""; + for (const char *pb = src_base; *pb; pb++) { + if (*pb == '/' || *pb == '\\') { + src_base = pb + SKIP_ONE; + } + } + const CBMLanguage src_lang = cbm_language_for_filename(src_base); + if (src_lang == CBM_LANG_KUSTOMIZE) { + return resolve_kustomize_entry(ctx, source_rel, imp->module_path); + } + /* Prefer exact header-file nodes for C/C++ includes so same-stem source or * module nodes do not steal the edge target. */ const cbm_gbuf_node_t *header_target = @@ -2283,13 +2353,6 @@ const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t /* Name-guess fallbacks below (Strategy 1b sibling-file, Strategy 3 * symbol-name) are gated per importing-file language — see * cbm_import_symbol_fallback_allowed (#1934). */ - const char *src_base = source_rel ? source_rel : ""; - for (const char *pb = src_base; *pb; pb++) { - if (*pb == '/' || *pb == '\\') { - src_base = pb + SKIP_ONE; - } - } - const CBMLanguage src_lang = cbm_language_for_filename(src_base); const bool symbol_fallback_allowed = cbm_import_symbol_fallback_allowed(src_lang); /* Strategy 1b: sibling-file resolution for build/markup grammars whose diff --git a/src/pipeline/pass_route_nodes.c b/src/pipeline/pass_route_nodes.c index 51701851c..3bc472f6a 100644 --- a/src/pipeline/pass_route_nodes.c +++ b/src/pipeline/pass_route_nodes.c @@ -147,6 +147,44 @@ static const char *url_path(const char *url) { return path ? path : "/"; } +/* Route path `needle` inside URL path `hay`, ending where a segment does: + * `/items` is in `/api/items` and `/items/42`; `/a` is not in `/acidjunk` + * (fastapi: a `/a` prefix route matched every github.com/a... URL). */ +static bool path_has_route(const char *hay, const char *needle) { + size_t n = strlen(needle); + if (n == 0) { + return false; + } + for (const char *p = strstr(hay, needle); p; p = strstr(p + SKIP_ONE, needle)) { + char c = p[n]; + bool start = needle[0] == '/' || p == hay || p[-SKIP_ONE] == '/'; + bool end = needle[n - SKIP_ONE] == '/' || c == '\0' || c == '/' || c == '?' || c == '#'; + if (start && end) { + return true; + } + } + return false; +} + +/* A route prefix's segments (`items`, `api/v1`) as whole segments of a file + * path: `app/routers/items.py`, `items/views.py`; `a` is not in + * `docs_src/app_testing/main.py`. */ +static bool file_has_segments(const char *path, const char *segs) { + size_t n = strlen(segs); + if (n == 0) { + return false; + } + for (const char *p = strstr(path, segs); p; p = strstr(p + SKIP_ONE, segs)) { + char c = p[n]; + bool start = p == path || p[-SKIP_ONE] == '/'; + bool end = segs[n - SKIP_ONE] == '/' || c == '\0' || c == '/' || c == '.'; + if (start && end) { + return true; + } + } + return false; +} + /* Check if a trailing dash-segment is a hash/project-number (strippable). * Returns true if the segment is short alphanumeric but not a meaningful word. */ static bool is_hash_segment(const char *seg, size_t slen) { @@ -219,7 +257,7 @@ static bool is_broker_route(const char *qn) { static const char *const prefixes[] = { "__route__infra__", "__route__pubsub__", "__route__cloud_tasks__", "__route__async__", "__route__cloud_scheduler__", "__route__kafka__", - "__route__sqs__"}; + "__route__sqs__", "__route__ingress__"}; if (!qn) { return false; } @@ -284,9 +322,12 @@ static int match_one_infra_route(cbm_gbuf_t *gb, const cbm_gbuf_node_t *infra, } int path_match = - (strlen(handler_path) > SKIP_ONE && (strstr(infra_path, handler_path) != NULL || - strstr(handler_path, infra_path) != NULL)); - int root_svc_match = (strcmp(handler_path, "/") == 0); + (strlen(handler_path) > SKIP_ONE && path_has_route(infra_path, handler_path)) || + (strlen(infra_path) > SKIP_ONE && path_has_route(handler_path, infra_path)); + /* a root route stands for a URL's service only when its file is that + * service's; any prefix or ANY route at `/` matched every URL (a docs + * site's chat link took the root handlers) */ + int root_svc_match = file_matches && strcmp(handler_path, "/") == 0; if (path_match || root_svc_match) { if (infra_candidate_is_better(handler_route, best)) { best = handler_route; @@ -475,7 +516,7 @@ static int bridge_funcs_to_prefix(cbm_gbuf_t *gb, const cbm_gbuf_node_t *prefix_ if (!func->properties_json || !strstr(func->properties_json, "\"route_path\"")) { continue; } - if (prefix_segs && prefix_segs[0] && !strstr(func->file_path, prefix_segs)) { + if (prefix_segs && prefix_segs[0] && !file_has_segments(func->file_path, prefix_segs)) { continue; } cbm_gbuf_insert_edge(gb, func->id, prefix_route->id, "HANDLES", @@ -503,29 +544,37 @@ static void connect_prefix_to_decorators(cbm_gbuf_t *gb) { continue; } - const cbm_gbuf_edge_t **calls_in = NULL; - int calls_count = 0; - cbm_gbuf_find_edges_by_target_type(gb, prefix_route->id, "CALLS", &calls_in, &calls_count); - if (calls_count == 0) { - continue; - } - - const cbm_gbuf_node_t *registrar = cbm_gbuf_find_by_id(gb, calls_in[0]->source_id); - if (!registrar || !registrar->file_path) { - continue; - } - const char *last_slash = strrchr(registrar->file_path, '/'); - if (!last_slash) { - continue; - } - int dir_len = (int)(last_slash - registrar->file_path) + SKIP_ONE; - const char *prefix_path = prefix_route->name; const char *prefix_segs = (prefix_path && prefix_path[0] == '/') ? prefix_path + SKIP_ONE : prefix_path; + /* A root mount (`app.mount("/", ...)`) names no part of a handler's + * path, so nothing tells its handlers from the rest of the directory: + * bridging it linked every decorated function there (fastapi: 702 + * handlers to `/`, then to every infra URL through it). */ + if (!prefix_segs || !prefix_segs[0]) { + continue; + } - connected += - bridge_funcs_to_prefix(gb, prefix_route, registrar->file_path, dir_len, prefix_segs); + const cbm_gbuf_edge_t **calls_in = NULL; + int calls_count = 0; + cbm_gbuf_find_edges_by_target_type(gb, prefix_route->id, "CALLS", &calls_in, &calls_count); + + /* Every registrar's directory, not the first CALLS edge's: their + * order is the workers' merge order, so the first one moved between + * runs of the same corpus. Edges dedup, so the union is order-free. */ + for (int ci = 0; ci < calls_count; ci++) { + const cbm_gbuf_node_t *registrar = cbm_gbuf_find_by_id(gb, calls_in[ci]->source_id); + if (!registrar || !registrar->file_path) { + continue; + } + const char *last_slash = strrchr(registrar->file_path, '/'); + if (!last_slash) { + continue; + } + int dir_len = (int)(last_slash - registrar->file_path) + SKIP_ONE; + connected += bridge_funcs_to_prefix(gb, prefix_route, registrar->file_path, dir_len, + prefix_segs); + } } if (connected > 0) { diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c new file mode 100644 index 000000000..cce391672 --- /dev/null +++ b/src/pipeline/pass_shell.c @@ -0,0 +1,3761 @@ +/* + * pass_shell.c — the Bash (+ Make, CI) infrastructure layer (PHASE 3 P3.2, P3.5). + * + * A shell script or a Make recipe that puts manifests, a kustomization or a + * chart on a cluster DEPLOYS them: an edge from the function running the + * command (a bash function, a Make rule; top-level code: the Module) to the + * node of what it deploys -- existing nodes only: + * kubectl apply|create|replace -f F a File holding Kubernetes objects: + * F itself, F's *.yaml|*.yml|*.json + * (-R: below it), or a glob's matches + * kubectl ... -k D the Kustomization of D + * kustomize build D | ... | kubectl apply -f - (also kubectl kustomize D) + * helm install|upgrade [NAME] CHART the Chart of a chart dir, else the + * shared external Chart (repo/name, + * oci://.../name) + * helm template [NAME] CHART | ... | kubectl apply -f - + * find D -name G -exec kubectl apply -f {} \; D's manifests matching G + * + * A program started by a path (./_build/kubermatic-installer, ${DIR}/x, + * $(LOCALBIN)/controller-gen) SPAWNS the Process of its name, as + * pass_spawns.c draws a call that starts a program. A program that is a file + * of the repository -- the path names one, or a shell runs one + * (bash|sh|zsh|ksh|dash [-opts] FILE) -- SPAWNS that script instead: its + * Module (the source of its own top-level edges), else its File (P3.5). + * + * CI (P3.5): the command lines GitHub Actions steps and Prow job containers + * run (pass_ci.c reads them) evaluate the same way, from the step's + * directory with its env literals; their edges carry the CI and the job. + * + * Words are evaluated by constant propagation over the script, nothing is + * executed: assignments (local / export / readonly / declare, arrays and + * "${a[@]}"), ${X:-default}, $0 / BASH_SOURCE, $(dirname ..), $(basename ..), + * $(cd .. && pwd), $(realpath ..), $(pwd), $(git rev-parse --show-toplevel). + * A word with any other part is unknown, and an unknown target is no edge. + * A value set in a branch, a loop or the right side of && / || is unknown + * after it; a subshell's or pipeline element's changes do not leak; a + * function body is read where it is defined, and the globals it sets are + * unknown after it. cd / pushd / popd track the working directory. A + * relative path resolves against it; with none tracked (a script runs from + * wherever it is started), against the repository root and the script's + * directory, linked only when exactly one of the two names a target. + * + * Make: the Makefile's variables expand into each recipe line (recursively, + * with the last assignment; $(CURDIR) / $(shell pwd) are the Makefile's + * directory, where its recipes run), which then evaluates as a shell line of + * its own. A variable assigned under a conditional is unknown; an included + * .mk file runs in its includer's directory, so it tracks none. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/hash_table.h" +#include "foundation/limits.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "cbm.h" +#include "lang_specs.h" + +#include +#include +#include +#include +#include + +/* Candidate paths a target lookup scanned (complexity guard counter). */ +_Atomic uint64_t g_sh_candidates = 0; + +enum { + SH_GROW_MIN = 16, + SH_DEC = 10, + SH_NO_DEPTH = -1, +}; + +/* A repository-anchored absolute path value: "\x01hack/ci" is /hack/ci. */ +#define SH_ANCHOR '\x01' +static const char SH_ROOT[] = "\x01"; + +static bool sh_grow(void **items, int *cap, int count, size_t elem) { + if (count < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : SH_GROW_MIN; + void *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, *items, (size_t)ncap * elem); + if (!grown) { + return false; + } + *items = grown; + *cap = ncap; + return true; +} + +static const char *sh_basename(const char *p) { + const char *s = strrchr(p, '/'); + return s ? s + SKIP_ONE : p; +} + +static bool sh_eq(const char *a, const char *b) { + return a && b && strcmp(a, b) == 0; +} + +static bool sh_starts(const char *s, const char *prefix) { + return strncmp(s, prefix, strlen(prefix)) == 0; +} + +/* ── The graph's deploy targets ────────────────────────────────────── */ + +typedef struct { + const char **items; /* graph-owned paths */ + int count; + int cap; +} sh_paths_t; + +static void sh_paths_push(sh_paths_t *v, const char *p) { + if (sh_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = p; + } +} + +static int sh_cmp_str(const void *a, const void *b) { + return strcmp(*(const char *const *)a, *(const char *const *)b); +} + +static void sh_paths_sort_unique(sh_paths_t *v) { + if (v->count == 0) { + return; + } + qsort((void *)v->items, (size_t)v->count, sizeof(*v->items), sh_cmp_str); + int w = SKIP_ONE; + for (int i = SKIP_ONE; i < v->count; i++) { + if (strcmp(v->items[i], v->items[w - SKIP_ONE]) != 0) { + v->items[w++] = v->items[i]; + } + } + v->count = w; +} + +/* First index whose path is >= key. */ +static int sh_lower_bound(const sh_paths_t *v, const char *key) { + int lo = 0; + int hi = v->count; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (strcmp(v->items[mid], key) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + return lo; +} + +static bool sh_paths_has(const sh_paths_t *v, const char *p) { + int i = sh_lower_bound(v, p); + return i < v->count && strcmp(v->items[i], p) == 0; +} + +typedef struct { + cbm_pipeline_ctx_t *ctx; + sh_paths_t files; /* every File node's path */ + sh_paths_t manifests; /* files holding Kubernetes objects (.yaml/.yml/.json) */ + CBMHashTable *file_ids; /* path -> File node id */ + CBMHashTable *kusts; /* kustomization path -> its Kustomization Resource id */ + CBMHashTable *charts; /* Chart.yaml path -> Chart node id */ +} sh_index_t; + +static bool sh_manifest_ext(const char *path) { + const char *dot = strrchr(sh_basename(path), '.'); + return dot && + (strcmp(dot, ".yaml") == 0 || strcmp(dot, ".yml") == 0 || strcmp(dot, ".json") == 0); +} + +static bool sh_kustomize_base(const char *base) { + return cbm_is_kustomize_file(base) || strcmp(base, "Kustomization") == 0; +} + +static bool sh_index_build(sh_index_t *ix, cbm_pipeline_ctx_t *ctx) { + ix->ctx = ctx; + ix->file_ids = cbm_ht_create(0); + ix->kusts = cbm_ht_create(0); + ix->charts = cbm_ht_create(0); + if (!ix->file_ids || !ix->kusts || !ix->charts) { + return false; + } + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "File", &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + if (nodes[i]->file_path) { + sh_paths_push(&ix->files, nodes[i]->file_path); + cbm_ht_set(ix->file_ids, nodes[i]->file_path, (void *)(intptr_t)nodes[i]->id); + } + } + } + if (cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + const char *fp = nodes[i]->file_path; + if (!fp) { + continue; + } + bool kust_file = sh_kustomize_base(sh_basename(fp)); + if (kust_file && sh_eq(nodes[i]->name, "Kustomization")) { + cbm_ht_set(ix->kusts, fp, (void *)(intptr_t)nodes[i]->id); + } else if (!kust_file && sh_manifest_ext(fp)) { + sh_paths_push(&ix->manifests, fp); + } + } + } + if (cbm_gbuf_find_by_label(ctx->gbuf, "Chart", &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + const char *fp = nodes[i]->file_path; + const char *base = fp ? sh_basename(fp) : ""; + const char *qn = nodes[i]->qualified_name; + if ((strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0) && qn && + !strstr(qn, ".__helm_dep__.")) { + cbm_ht_set(ix->charts, fp, (void *)(intptr_t)nodes[i]->id); + } + } + } + sh_paths_sort_unique(&ix->files); + sh_paths_sort_unique(&ix->manifests); + return true; +} + +static void sh_index_free(sh_index_t *ix) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)ix->files.items); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)ix->manifests.items); + if (ix->file_ids) { + cbm_ht_free(ix->file_ids); + } + if (ix->kusts) { + cbm_ht_free(ix->kusts); + } + if (ix->charts) { + cbm_ht_free(ix->charts); + } +} + +/* The paths of `v` below directory `dir` ("" = the root): [*lo, *hi). */ +static void sh_range(const sh_paths_t *v, const char *dir, char *key, size_t key_sz, int *lo, + int *hi) { + if (!dir[0]) { + key[0] = '\0'; + *lo = 0; + *hi = v->count; + return; + } + snprintf(key, key_sz, "%s/", dir); + size_t kl = strlen(key); + *lo = sh_lower_bound(v, key); + int h = *lo; + while (h < v->count && strncmp(v->items[h], key, kl) == 0) { + h++; + } + *hi = h; +} + +static bool sh_dir_exists(const sh_index_t *ix, const char *dir) { + if (!dir[0]) { + return true; + } + char key[CBM_SZ_4K]; + int lo = 0; + int hi = 0; + sh_range(&ix->files, dir, key, sizeof(key), &lo, &hi); + return hi > lo; +} + +/* ── Path values ───────────────────────────────────────────────────── */ + +/* `rel` (a relative path: ./ and ../ allowed) inside repository directory + * `base` ("" = the root), normalized; NULL when it leaves the repository. */ +static char *sh_join(CBMArena *a, const char *base, const char *rel) { + size_t bl = strlen(base); + char *out = cbm_arena_alloc(a, bl + strlen(rel) + PAIR_LEN); + if (!out) { + return NULL; + } + memcpy(out, base, bl); + size_t n = bl; + out[n] = '\0'; + for (const char *p = rel; *p;) { + while (*p == '/') { + p++; + } + size_t sl = strcspn(p, "/"); + if (sl == 0) { + break; + } + if (sl == SKIP_ONE && p[0] == '.') { + p += sl; + continue; + } + if (sl == PAIR_LEN && p[0] == '.' && p[SKIP_ONE] == '.') { + if (n == 0) { + return NULL; + } + while (n > 0 && out[n - SKIP_ONE] != '/') { + n--; + } + if (n > 0) { + n--; + } + out[n] = '\0'; + p += sl; + continue; + } + if (n) { + out[n++] = '/'; + } + memcpy(out + n, p, sl); + n += sl; + out[n] = '\0'; + p += sl; + } + return out; +} + +static const char *sh_anchor(CBMArena *a, const char *repo_path) { + return repo_path ? cbm_arena_sprintf(a, "%c%s", SH_ANCHOR, repo_path) : NULL; +} + +/* dirname(1), on a value: textual, keeps an anchored path anchored. */ +static const char *sh_dirname(CBMArena *a, const char *v) { + bool anch = v[0] == SH_ANCHOR; + const char *p = anch ? v + SKIP_ONE : v; + size_t len = strlen(p); + while (len > SKIP_ONE && p[len - SKIP_ONE] == '/') { + len--; + } + size_t k = len; + while (k > 0 && p[k - SKIP_ONE] != '/') { + k--; + } + if (k == 0) { + return anch ? SH_ROOT : "."; + } + size_t dl = k - SKIP_ONE; + while (dl > 0 && p[dl - SKIP_ONE] == '/') { + dl--; + } + if (dl == 0) { + return anch ? SH_ROOT : "/"; + } + return anch ? cbm_arena_sprintf(a, "%c%.*s", SH_ANCHOR, (int)dl, p) + : cbm_arena_strndup(a, p, dl); +} + +/* basename(1), on a value. */ +static const char *sh_value_basename(CBMArena *a, const char *v) { + const char *p = v[0] == SH_ANCHOR ? v + SKIP_ONE : v; + size_t len = strlen(p); + while (len > SKIP_ONE && p[len - SKIP_ONE] == '/') { + len--; + } + size_t k = len; + while (k > 0 && p[k - SKIP_ONE] != '/') { + k--; + } + return cbm_arena_strndup(a, p + k, len - k); +} + +/* ── Hits: what a command deploys ──────────────────────────────────── */ + +typedef struct { + const char *file; /* the script (caller-owned path) */ + int line; + const char *via; /* static */ + int64_t tgt; /* the target node; 0: the external chart `ext` */ + const char *ext; /* pass-arena */ + const char *path; /* pass-arena: the deployed path or chart; a spawned script */ + const char *ci; /* static: the CI running the command; NULL: none */ + const char *job; /* pass-arena: its job; NULL: none */ + bool chart; + bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ + bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */ + bool build; /* the image `ext` is built from the Dockerfile `path` */ +} sh_hit_t; + +typedef struct { + sh_hit_t *items; + int count; + int cap; + CBMArena arena; + CBMHashTable *declarers; /* files running a container build: path -> path */ +} sh_hits_t; + +typedef struct { + int64_t id; + const char *path; /* graph-owned or file-arena */ +} sh_target_t; + +typedef struct { + sh_target_t *items; + int count; + int cap; +} sh_targets_t; + +static void sh_targets_push(sh_targets_t *v, int64_t id, const char *path) { + if (id > 0 && sh_grow((void **)&v->items, &v->cap, v->count, sizeof(*v->items))) { + v->items[v->count++] = (sh_target_t){id, path}; + } +} + +/* ── Shell state ───────────────────────────────────────────────────── */ + +typedef struct { + const char *s; /* scalar value; NULL: unknown */ + const char **items; /* array items (an item NULL: unknown); NULL: no array */ + int n; +} sh_val_t; + +static const sh_val_t SH_UNKNOWN_VAL = {NULL, NULL, 0}; + +typedef enum { SH_FREE, SH_AT, SH_LOST } sh_cwd_mode_t; + +typedef struct { + sh_cwd_mode_t mode; + const char *dir; /* SH_AT: repository path ("" = the root) */ +} sh_cwd_t; + +typedef struct { + const char *name; + const sh_val_t *prev; /* NULL: was unset */ + bool local; +} sh_undo_t; + +typedef struct { + int undo; + sh_cwd_t cwd; + int ndirs; +} sh_mark_t; + +typedef struct { + const char **names; + int n; + int cap; + bool cwd; +} sh_touch_t; + +typedef struct { + sh_index_t *ix; + sh_hits_t *hits; + CBMArena *a; /* this file's values */ + const char *src; /* the parsed text */ + const char *file; /* the script / Makefile path */ + const char *script; /* $0: anchored script path; NULL: none (Make) */ + const char *sdir; /* the script's directory */ + int line_base; /* Make / CI: the text's first row */ + const char *ci; /* the CI running the text (static); NULL: none */ + const char *job; /* its job; NULL: none */ + bool exec; /* an argv exec'd without a shell (Prow) */ + int in_func; + CBMHashTable *env; /* name -> const sh_val_t * */ + sh_undo_t *undo; + int nundo; + int cap_undo; + sh_cwd_t cwd; + sh_cwd_t *dirs; /* pushd stack */ + int ndirs; + int cap_dirs; +} sh_t; + +static void sh_set(sh_t *s, const char *name, const sh_val_t *v, bool local) { + if (!sh_grow((void **)&s->undo, &s->cap_undo, s->nundo, sizeof(*s->undo))) { + return; + } + s->undo[s->nundo++] = (sh_undo_t){name, cbm_ht_get(s->env, name), local}; + cbm_ht_set(s->env, name, (void *)v); +} + +static sh_mark_t sh_mark(const sh_t *s) { + return (sh_mark_t){s->nundo, s->cwd, s->ndirs}; +} + +static bool sh_cwd_same(sh_cwd_t a, sh_cwd_t b) { + return a.mode == b.mode && (a.mode != SH_AT || strcmp(a.dir, b.dir) == 0); +} + +static void sh_rewind(sh_t *s, sh_mark_t m) { + while (s->nundo > m.undo) { + const sh_undo_t *u = &s->undo[--s->nundo]; + if (u->prev) { + cbm_ht_set(s->env, u->name, (void *)u->prev); + } else { + (void)cbm_ht_delete(s->env, u->name); + } + } + s->cwd = m.cwd; + s->ndirs = m.ndirs; +} + +static void sh_touch_add(sh_touch_t *t, const char *name) { + if (sh_grow((void **)&t->names, &t->cap, t->n, sizeof(*t->names))) { + t->names[t->n++] = name; + } +} + +/* End one alternative of a conditional: what it set is noted, then undone. */ +static void sh_clause_end(sh_t *s, sh_mark_t m, sh_touch_t *t) { + for (int i = m.undo; i < s->nundo; i++) { + sh_touch_add(t, s->undo[i].name); + } + if (!sh_cwd_same(s->cwd, m.cwd) || s->ndirs != m.ndirs) { + t->cwd = true; + } + sh_rewind(s, m); +} + +/* After the conditional: whatever any alternative set is unknown. */ +static void sh_group_end(sh_t *s, sh_touch_t *t) { + for (int i = 0; i < t->n; i++) { + sh_set(s, t->names[i], &SH_UNKNOWN_VAL, false); + } + if (t->cwd) { + s->cwd = (sh_cwd_t){SH_LOST, NULL}; + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)t->names); + *t = (sh_touch_t){0}; +} + +/* ── Word evaluation ───────────────────────────────────────────────── */ + +static const char *sh_text(sh_t *s, TSNode n) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + return cbm_arena_strndup(s->a, s->src + b, e - b); +} + +static bool sh_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +/* A growable string in the file arena. */ +typedef struct { + char *buf; + size_t len; + size_t cap; + bool bad; /* an unknown part */ +} sh_str_t; + +static void sh_str_add(CBMArena *a, sh_str_t *b, const char *p, size_t n) { + if (b->bad || !p) { + b->bad = true; + return; + } + if (b->len + n + SKIP_ONE > b->cap) { + size_t ncap = (b->cap ? b->cap : SH_GROW_MIN) * PAIR_LEN; + while (ncap < b->len + n + SKIP_ONE) { + ncap *= PAIR_LEN; + } + char *nb = cbm_arena_alloc(a, ncap); + if (!nb) { + b->bad = true; + return; + } + if (b->len) { + memcpy(nb, b->buf, b->len); + } + b->buf = nb; + b->cap = ncap; + } + memcpy(b->buf + b->len, p, n); + b->len += n; + b->buf[b->len] = '\0'; +} + +static const char *sh_str_done(sh_str_t *b) { + if (b->bad) { + return NULL; + } + return b->buf ? b->buf : ""; +} + +/* Text with backslash escapes removed; in_dq: only the double-quote ones. */ +static void sh_add_unescaped(CBMArena *a, sh_str_t *b, const char *p, size_t n, bool in_dq) { + size_t i = 0; + while (i < n) { + size_t j = i; + while (j < n && p[j] != '\\') { + j++; + } + sh_str_add(a, b, p + i, j - i); + if (j + SKIP_ONE >= n) { + if (j < n) { + sh_str_add(a, b, p + j, SKIP_ONE); + } + return; + } + char c = p[j + SKIP_ONE]; + if (c == '\n') { + i = j + PAIR_LEN; + continue; + } + if (!in_dq || c == '"' || c == '\\' || c == '$' || c == '`') { + sh_str_add(a, b, p + j + SKIP_ONE, SKIP_ONE); + } else { + sh_str_add(a, b, p + j, PAIR_LEN); + } + i = j + PAIR_LEN; + } +} + +static const char *sh_eval(sh_t *s, TSNode n); + +/* The cwd as a value: the anchored directory, or NULL. */ +static const char *sh_cwd_value(sh_t *s) { + return s->cwd.mode == SH_AT ? sh_anchor(s->a, s->cwd.dir) : NULL; +} + +/* A variable's value; *found: it is set (or a special that exists). */ +static const sh_val_t *sh_lookup(sh_t *s, const char *name, bool *found) { + *found = true; + if (strcmp(name, "0") == 0 || strcmp(name, "BASH_SOURCE") == 0) { + if (!s->script) { + return &SH_UNKNOWN_VAL; + } + sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); + if (v) { + *v = (sh_val_t){s->script, NULL, 0}; + } + return v ? v : &SH_UNKNOWN_VAL; + } + if (strcmp(name, "PWD") == 0) { + sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); + if (v) { + *v = (sh_val_t){sh_cwd_value(s), NULL, 0}; + } + return v ? v : &SH_UNKNOWN_VAL; + } + if (!name[0] || (strchr("123456789@*#?$!-", name[0]) && + (name[SKIP_ONE] == '\0' || (name[0] >= '1' && name[0] <= '9')))) { + return &SH_UNKNOWN_VAL; /* positional and special parameters */ + } + const sh_val_t *v = cbm_ht_get(s->env, name); + if (!v) { + *found = false; + return &SH_UNKNOWN_VAL; + } + return v; +} + +/* Concatenate the children of `n` in [from, to), source gaps included. */ +static void sh_concat(sh_t *s, sh_str_t *b, TSNode n, uint32_t from, uint32_t to, bool in_dq) { + uint32_t pos = from; + for (uint32_t i = 0; i < ts_node_child_count(n) && !b->bad; i++) { + TSNode c = ts_node_child(n, i); + uint32_t cb = ts_node_start_byte(c); + uint32_t ce = ts_node_end_byte(c); + if (cb < from || ce > to) { + continue; + } + if (cb > pos) { + sh_add_unescaped(s->a, b, s->src + pos, cb - pos, in_dq); + } + if (!ts_node_is_named(c)) { + if (!(in_dq && strcmp(ts_node_type(c), "\"") == 0)) { + sh_str_add(s->a, b, s->src + cb, ce - cb); + } + } else if (sh_type(c, "string_content")) { + sh_add_unescaped(s->a, b, s->src + cb, ce - cb, in_dq); + } else { + const char *v = sh_eval(s, c); + sh_str_add(s->a, b, v, v ? strlen(v) : 0); + } + pos = ce; + } + if (pos < to && !b->bad) { + sh_add_unescaped(s->a, b, s->src + pos, to - pos, in_dq); + } +} + +/* ${...}: a plain variable, an element 0, or a default (:- - := =). */ +static const char *sh_eval_expansion(sh_t *s, TSNode n) { + TSNode name = {0}; + const char *op = NULL; + uint32_t op_end = 0; + uint32_t close = ts_node_end_byte(n); + uint32_t cc = ts_node_child_count(n); + for (uint32_t i = 0; i < cc; i++) { + TSNode c = ts_node_child(n, i); + const char *t = ts_node_type(c); + if (ts_node_is_named(c)) { + if (ts_node_is_null(name)) { + name = c; + } else if (!op) { + return NULL; /* ${X%pattern} and the like: no plain value */ + } + continue; + } + if (strcmp(t, "${") == 0) { + continue; + } + if (strcmp(t, "}") == 0) { + close = ts_node_start_byte(c); + continue; + } + if (op) { + continue; /* inside the default */ + } + if (ts_node_is_null(name)) { + return NULL; /* ${#X}, ${!X}: not the value */ + } + if (strcmp(t, ":-") != 0 && strcmp(t, "-") != 0 && strcmp(t, ":=") != 0 && + strcmp(t, "=") != 0) { + return NULL; /* ${X%..}, ${X/..}, ${X:+..}, ... */ + } + op = t; + op_end = ts_node_end_byte(c); + } + if (ts_node_is_null(name)) { + return NULL; + } + const char *nm = NULL; + int index = SH_NO_DEPTH; + if (sh_type(name, "subscript")) { + TSNode base = ts_node_child_by_field_name(name, TS_FIELD("name")); + TSNode idx = ts_node_child_by_field_name(name, TS_FIELD("index")); + if (ts_node_is_null(base) || ts_node_is_null(idx)) { + return NULL; + } + const char *it = sh_text(s, idx); + if (!it || strcmp(it, "0") != 0) { + return NULL; + } + nm = sh_text(s, base); + index = 0; + } else if (sh_type(name, "variable_name") || sh_type(name, "special_variable_name")) { + nm = sh_text(s, name); + } else { + return NULL; + } + if (!nm) { + return NULL; + } + bool found = false; + const sh_val_t *v = sh_lookup(s, nm, &found); + const char *val = v->s; + if (index == 0 && v->items) { + val = v->n > 0 ? v->items[0] : ""; + } + if (!op) { + return found ? val : NULL; + } + bool colon = op[0] == ':'; + bool use_default = !found || (colon && val && !val[0]); + if (!use_default) { + return val; + } + sh_str_t b = {0}; + sh_concat(s, &b, n, op_end, close, true); + const char *d = sh_str_done(&b); + if (d && strchr(op, '=')) { + sh_val_t *nv = cbm_arena_alloc(s->a, sizeof(*nv)); + if (nv) { + *nv = (sh_val_t){d, NULL, 0}; + sh_set(s, nm, nv, false); + } + } + return d; +} + +/* A value as a repository path: anchored, or relative to a tracked + * directory. NULL when it is neither. */ +static const char *sh_cd_target(sh_t *s, const char *v, sh_cwd_t *out); + +/* realpath / readlink -f: the anchored path, when it is decidable. */ +static const char *sh_absolute(sh_t *s, const char *v) { + if (!v) { + return NULL; + } + if (v[0] == SH_ANCHOR) { + return sh_anchor(s->a, sh_join(s->a, "", v + SKIP_ONE)); + } + if (v[0] == '/' || s->cwd.mode != SH_AT) { + return NULL; + } + return sh_anchor(s->a, sh_join(s->a, s->cwd.dir, v)); +} + +typedef struct { + const char **v; + bool *split; + uint32_t *span; /* source bytes of the word: [2i] start, [2i+1] end */ + int n; + int cap; +} sh_argv_t; + +static void sh_argv_build(sh_t *s, TSNode cmd, sh_argv_t *out); + +static void sh_argv_free(sh_argv_t *a) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)a->v); + cbm_free(CBM_MEM_CLASS_OTHER, a->split); + cbm_free(CBM_MEM_CLASS_OTHER, a->span); + *a = (sh_argv_t){0}; +} + +/* The command a statement runs: the body of a redirection. */ +static TSNode sh_unwrap(TSNode n) { + while (sh_type(n, "redirected_statement")) { + TSNode body = ts_node_child_by_field_name(n, TS_FIELD("body")); + if (ts_node_is_null(body)) { + break; + } + n = body; + } + return n; +} + +/* The first non-option argument after argv[0]. */ +static const char *sh_first_operand(const sh_argv_t *a, int from) { + for (int i = from; i < a->n; i++) { + if (a->v[i] && a->v[i][0] == '-' && a->v[i][SKIP_ONE]) { + continue; + } + return a->v[i]; + } + return NULL; +} + +/* $(dirname ..), $(basename ..), $(pwd), $(realpath ..), $(readlink -f ..), + * $(git rev-parse --show-toplevel). */ +static const char *sh_eval_subst_command(sh_t *s, TSNode cmd) { + sh_argv_t a = {0}; + sh_argv_build(s, cmd, &a); + const char *out = NULL; + const char *name = a.n > 0 ? a.v[0] : NULL; + if (sh_eq(name, "dirname") && a.n == PAIR_LEN && a.v[SKIP_ONE]) { + out = sh_dirname(s->a, a.v[SKIP_ONE]); + } else if (sh_eq(name, "basename") && a.n == PAIR_LEN && a.v[SKIP_ONE]) { + out = sh_value_basename(s->a, a.v[SKIP_ONE]); + } else if (sh_eq(name, "pwd")) { + out = sh_cwd_value(s); + } else if (sh_eq(name, "realpath") || + (sh_eq(name, "readlink") && a.n > PAIR_LEN && a.v[SKIP_ONE] && + (sh_eq(a.v[SKIP_ONE], "-f") || sh_eq(a.v[SKIP_ONE], "-e") || + sh_eq(a.v[SKIP_ONE], "-m")))) { + out = sh_absolute(s, sh_first_operand(&a, SKIP_ONE)); + } else if (sh_eq(name, "git") && a.n == 3 && sh_eq(a.v[SKIP_ONE], "rev-parse") && + sh_eq(a.v[PAIR_LEN], "--show-toplevel")) { + out = SH_ROOT; + } + sh_argv_free(&a); + return out; +} + +/* `cd DIR` as the left half of $(cd DIR && pwd): the directory, or NULL. */ +static const char *sh_subst_cd(sh_t *s, TSNode cd) { + cd = sh_unwrap(cd); + if (!sh_type(cd, "command")) { + return NULL; + } + sh_argv_t a = {0}; + sh_argv_build(s, cd, &a); + const char *out = NULL; + if (a.n > 0 && sh_eq(a.v[0], "cd")) { + sh_cwd_t to = {SH_LOST, NULL}; + (void)sh_cd_target(s, sh_first_operand(&a, SKIP_ONE), &to); + out = to.mode == SH_AT ? sh_anchor(s->a, to.dir) : NULL; + } + sh_argv_free(&a); + return out; +} + +static bool sh_is_pwd(sh_t *s, TSNode n) { + n = sh_unwrap(n); + if (!sh_type(n, "command")) { + return false; + } + sh_argv_t a = {0}; + sh_argv_build(s, n, &a); + bool pwd = a.n > 0 && sh_eq(a.v[0], "pwd"); + sh_argv_free(&a); + return pwd; +} + +static const char *sh_eval_subst(sh_t *s, TSNode n) { + TSNode st[PAIR_LEN]; + int k = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (sh_type(c, "comment")) { + continue; + } + if (k == PAIR_LEN) { + return NULL; + } + st[k++] = c; + } + if (k == SKIP_ONE && sh_type(st[0], "list") && ts_node_named_child_count(st[0]) == PAIR_LEN) { + TSNode l = st[0]; + st[0] = ts_node_named_child(l, 0); + st[SKIP_ONE] = ts_node_named_child(l, SKIP_ONE); + k = PAIR_LEN; + } + if (k == PAIR_LEN) { + return sh_is_pwd(s, st[SKIP_ONE]) ? sh_subst_cd(s, st[0]) : NULL; + } + if (k == SKIP_ONE) { + TSNode c = sh_unwrap(st[0]); + return sh_type(c, "command") ? sh_eval_subst_command(s, c) : NULL; + } + return NULL; +} + +static const char *sh_eval(sh_t *s, TSNode n) { + if (ts_node_is_null(n)) { + return NULL; + } + const char *t = ts_node_type(n); + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + if (strcmp(t, "word") == 0 || strcmp(t, "number") == 0 || strcmp(t, "extglob_pattern") == 0) { + if (s->src[b] == '~') { + return NULL; + } + sh_str_t out = {0}; + sh_add_unescaped(s->a, &out, s->src + b, e - b, false); + return sh_str_done(&out); + } + if (strcmp(t, "raw_string") == 0) { + return e - b >= PAIR_LEN ? cbm_arena_strndup(s->a, s->src + b + SKIP_ONE, e - b - PAIR_LEN) + : NULL; + } + if (strcmp(t, "string") == 0 || strcmp(t, "concatenation") == 0) { + sh_str_t out = {0}; + bool dq = strcmp(t, "string") == 0; + uint32_t from = dq ? b + SKIP_ONE : b; + uint32_t to = dq && e > b + SKIP_ONE ? e - SKIP_ONE : e; + sh_concat(s, &out, n, from, to, dq); + return sh_str_done(&out); + } + if (strcmp(t, "simple_expansion") == 0) { + TSNode v = ts_node_named_child(n, 0); + const char *nm = ts_node_is_null(v) ? NULL : sh_text(s, v); + if (!nm) { + return NULL; + } + bool found = false; + const sh_val_t *val = sh_lookup(s, nm, &found); + return found ? val->s : NULL; + } + if (strcmp(t, "expansion") == 0) { + return sh_eval_expansion(s, n); + } + if (strcmp(t, "command_substitution") == 0) { + return sh_eval_subst(s, n); + } + return NULL; +} + +/* A command word: its value; an option whose value is unknown is still that + * option (--context="${ctx}" -> "--context="). */ +static const char *sh_eval_word(sh_t *s, TSNode w) { + const char *v = sh_eval(s, w); + if (v || s->src[ts_node_start_byte(w)] != '-') { + return v; + } + uint32_t b = ts_node_start_byte(w); + uint32_t e = b; + while (e < ts_node_end_byte(w) && !strchr("$\"'`\\", s->src[e])) { + e++; + } + return cbm_arena_strndup(s->a, s->src + b, e - b); +} + +/* "${name[@]}" / ${name[*]} as a whole word: the array, NULL when not. */ +static const sh_val_t *sh_array_word(sh_t *s, TSNode w) { + if (sh_type(w, "string") && ts_node_named_child_count(w) == SKIP_ONE) { + w = ts_node_named_child(w, 0); + } + if (!sh_type(w, "expansion") || ts_node_named_child_count(w) != SKIP_ONE) { + return NULL; + } + TSNode sub = ts_node_named_child(w, 0); + if (!sh_type(sub, "subscript")) { + return NULL; + } + TSNode base = ts_node_child_by_field_name(sub, TS_FIELD("name")); + TSNode idx = ts_node_child_by_field_name(sub, TS_FIELD("index")); + const char *it = ts_node_is_null(idx) ? NULL : sh_text(s, idx); + if (ts_node_is_null(base) || !it || (strcmp(it, "@") != 0 && strcmp(it, "*") != 0)) { + return NULL; + } + bool found = false; + const sh_val_t *v = sh_lookup(s, sh_text(s, base), &found); + return v; +} + +/* An unquoted expansion may split into several words. */ +static bool sh_may_split(TSNode w) { + const char *t = ts_node_type(w); + if (strcmp(t, "simple_expansion") == 0 || strcmp(t, "expansion") == 0 || + strcmp(t, "command_substitution") == 0) { + return true; + } + if (strcmp(t, "concatenation") == 0) { + for (uint32_t i = 0; i < ts_node_named_child_count(w); i++) { + if (sh_may_split(ts_node_named_child(w, i))) { + return true; + } + } + } + return false; +} + +static void sh_argv_push(sh_argv_t *a, const char *v, bool split, uint32_t b, uint32_t e) { + if (a->n >= a->cap) { + int ncap = a->cap ? a->cap * PAIR_LEN : SH_GROW_MIN; + const char **nv = + cbm_realloc(CBM_MEM_CLASS_OTHER, (void *)a->v, (size_t)ncap * sizeof(*nv)); + if (!nv) { + return; + } + a->v = nv; + bool *ns = cbm_realloc(CBM_MEM_CLASS_OTHER, a->split, (size_t)ncap * sizeof(*ns)); + if (!ns) { + return; + } + a->split = ns; + uint32_t *nsp = + cbm_realloc(CBM_MEM_CLASS_OTHER, a->span, (size_t)ncap * PAIR_LEN * sizeof(*nsp)); + if (!nsp) { + return; + } + a->span = nsp; + a->cap = ncap; + } + a->v[a->n] = v; + a->split[a->n] = split; + a->span[PAIR_LEN * a->n] = b; + a->span[PAIR_LEN * a->n + SKIP_ONE] = e; + a->n++; +} + +static void sh_argv_word(sh_t *s, TSNode w, sh_argv_t *out) { + uint32_t b = ts_node_start_byte(w); + uint32_t e = ts_node_end_byte(w); + const sh_val_t *arr = sh_array_word(s, w); + if (arr) { + if (!arr->items) { + sh_argv_push(out, NULL, true, b, e); + return; + } + for (int i = 0; i < arr->n; i++) { + sh_argv_push(out, arr->items[i], false, b, e); + } + return; + } + const char *v = sh_eval_word(s, w); + sh_argv_push(out, v, v ? false : sh_may_split(w), b, e); +} + +/* argv of a command: its name and arguments, evaluated (NULL: unknown). */ +static void sh_argv_build(sh_t *s, TSNode cmd, sh_argv_t *out) { + uint32_t cc = ts_node_child_count(cmd); + for (uint32_t i = 0; i < cc; i++) { + const char *field = ts_node_field_name_for_child(cmd, i); + TSNode c = ts_node_child(cmd, i); + if (sh_eq(field, "name")) { + TSNode w = ts_node_named_child(c, 0); + if (!ts_node_is_null(w)) { + sh_argv_word(s, w, out); + } + } else if (sh_eq(field, "argument")) { + sh_argv_word(s, c, out); + } + } +} + +/* ── Paths to targets ──────────────────────────────────────────────── */ + +typedef struct { + bool recursive; + const char *name; /* find -name glob; NULL: any */ + int maxdepth; /* find -maxdepth; SH_NO_DEPTH: none */ +} sh_find_t; + +typedef int (*sh_target_fn)(sh_t *s, const char *path, const sh_find_t *f, sh_targets_t *out); + +/* Candidate repository paths of a value: the anchored path; a relative one + * in the tracked directory, or -- none tracked -- in the root and in the + * script's directory. */ +static int sh_candidates(sh_t *s, const char *v, const char *out[PAIR_LEN]) { + if (!v || !v[0] || strstr(v, "://")) { + return 0; + } + if (v[0] == SH_ANCHOR) { + out[0] = sh_join(s->a, "", v + SKIP_ONE); + return out[0] ? SKIP_ONE : 0; + } + if (v[0] == '/' || v[0] == '~') { + return 0; + } + if (s->cwd.mode == SH_LOST) { + return 0; + } + if (s->cwd.mode == SH_AT) { + out[0] = sh_join(s->a, s->cwd.dir, v); + return out[0] ? SKIP_ONE : 0; + } + int n = 0; + const char *r = sh_join(s->a, "", v); + const char *d = sh_join(s->a, s->sdir, v); + if (r) { + out[n++] = r; + } + if (d && (!r || strcmp(d, r) != 0)) { + out[n++] = d; + } + return n; +} + +/* The targets of value `v`: of its one candidate, or of the only one of two + * candidates that has any. */ +static void sh_targets_of(sh_t *s, const char *v, sh_target_fn fn, const sh_find_t *f, + sh_targets_t *out) { + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, v, c); + int mark = out->count; + int found = 0; + for (int i = 0; i < n; i++) { + if (fn(s, c[i], f, out) > 0) { + found++; + } + } + if (found > SKIP_ONE) { + out->count = mark; + } +} + +static bool sh_glob_seg(const char *p, size_t pn, const char *t, size_t tn) { + size_t pi = 0; + size_t ti = 0; + size_t star = (size_t)-1; + size_t mark = 0; + while (ti < tn) { + if (pi < pn && (p[pi] == '?' || p[pi] == t[ti])) { + pi++; + ti++; + } else if (pi < pn && p[pi] == '*') { + star = pi++; + mark = ti; + } else if (star != (size_t)-1) { + pi = star + SKIP_ONE; + ti = ++mark; + } else { + return false; + } + } + while (pi < pn && p[pi] == '*') { + pi++; + } + return pi == pn; +} + +/* A path glob: * and ? within a segment, segment for segment. */ +static bool sh_glob_path(const char *pat, const char *path) { + for (;;) { + size_t pl = strcspn(pat, "/"); + size_t tl = strcspn(path, "/"); + if (!sh_glob_seg(pat, pl, path, tl)) { + return false; + } + if (!pat[pl] || !path[tl]) { + return !pat[pl] && !path[tl]; + } + pat += pl + SKIP_ONE; + path += tl + SKIP_ONE; + } +} + +static int sh_add_manifest(sh_t *s, const char *p, sh_targets_t *out) { + int64_t id = (int64_t)(intptr_t)cbm_ht_get(s->ix->file_ids, p); + if (id <= 0) { + return 0; + } + sh_targets_push(out, id, p); + return SKIP_ONE; +} + +/* kubectl -f PATH: the manifest, a directory's manifests, a glob's. */ +static int sh_t_files(sh_t *s, const char *path, const sh_find_t *f, sh_targets_t *out) { + const sh_paths_t *m = &s->ix->manifests; + char key[CBM_SZ_4K]; + int lo = 0; + int hi = 0; + int added = 0; + if (strpbrk(path, "*?[")) { + if (strchr(path, '[')) { + return 0; + } + const char *g = strpbrk(path, "*?"); + size_t dl = (size_t)(g - path); + while (dl > 0 && path[dl - SKIP_ONE] != '/') { + dl--; + } + char dir[CBM_SZ_4K]; + snprintf(dir, sizeof(dir), "%.*s", (int)(dl ? dl - SKIP_ONE : 0), path); + sh_range(m, dir, key, sizeof(key), &lo, &hi); + for (int i = lo; i < hi; i++) { + atomic_fetch_add_explicit(&g_sh_candidates, SKIP_ONE, memory_order_relaxed); + if (sh_glob_path(path, m->items[i])) { + added += sh_add_manifest(s, m->items[i], out); + } + } + return added; + } + if (sh_paths_has(m, path)) { + return sh_add_manifest(s, path, out); + } + sh_range(m, path, key, sizeof(key), &lo, &hi); + size_t kl = strlen(key); + for (int i = lo; i < hi; i++) { + atomic_fetch_add_explicit(&g_sh_candidates, SKIP_ONE, memory_order_relaxed); + if (f->recursive || !strchr(m->items[i] + kl, '/')) { + added += sh_add_manifest(s, m->items[i], out); + } + } + return added; +} + +/* find DIR -name G: DIR's manifests (to -maxdepth) whose name matches. */ +static int sh_t_found(sh_t *s, const char *dir, const sh_find_t *f, sh_targets_t *out) { + const sh_paths_t *m = &s->ix->manifests; + char key[CBM_SZ_4K]; + int lo = 0; + int hi = 0; + int added = 0; + if (!sh_dir_exists(s->ix, dir)) { + return 0; + } + sh_range(m, dir, key, sizeof(key), &lo, &hi); + size_t kl = strlen(key); + for (int i = lo; i < hi; i++) { + atomic_fetch_add_explicit(&g_sh_candidates, SKIP_ONE, memory_order_relaxed); + const char *rest = m->items[i] + kl; + int depth = SKIP_ONE; + for (const char *p = rest; *p; p++) { + depth += *p == '/'; + } + const char *base = sh_basename(rest); + if ((f->maxdepth == SH_NO_DEPTH || depth <= f->maxdepth) && + (!f->name || sh_glob_seg(f->name, strlen(f->name), base, strlen(base)))) { + added += sh_add_manifest(s, m->items[i], out); + } + } + return added; +} + +/* -k DIR / kustomize build DIR: the Kustomization of DIR. */ +static int sh_t_kust(sh_t *s, const char *dir, const sh_find_t *f, sh_targets_t *out) { + (void)f; + static const char *const names[] = {"kustomization.yaml", "kustomization.yml", "Kustomization"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + const char *p = cbm_arena_sprintf(s->a, "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + const char *key = p ? cbm_ht_get_key(s->ix->kusts, p) : NULL; + if (key) { + sh_targets_push(out, (int64_t)(intptr_t)cbm_ht_get(s->ix->kusts, key), dir); + return SKIP_ONE; + } + } + return 0; +} + +/* A chart directory: its Chart. */ +static int sh_t_chart(sh_t *s, const char *dir, const sh_find_t *f, sh_targets_t *out) { + (void)f; + static const char *const names[] = {"Chart.yaml", "Chart.yml"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + const char *p = cbm_arena_sprintf(s->a, "%s%s%s", dir, dir[0] ? "/" : "", names[i]); + const char *key = p ? cbm_ht_get_key(s->ix->charts, p) : NULL; + if (key) { + sh_targets_push(out, (int64_t)(intptr_t)cbm_ht_get(s->ix->charts, key), dir); + return SKIP_ONE; + } + } + return 0; +} + +/* Where `cd v` lands: *out = AT an existing directory, else LOST. */ +static const char *sh_cd_target(sh_t *s, const char *v, sh_cwd_t *out) { + *out = (sh_cwd_t){SH_LOST, NULL}; + if (!v) { + return NULL; + } + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, v, c); + const char *hit = NULL; + for (int i = 0; i < n; i++) { + if (sh_dir_exists(s->ix, c[i])) { + if (hit) { + return NULL; + } + hit = c[i]; + } + } + if (hit) { + *out = (sh_cwd_t){SH_AT, hit}; + } + return hit; +} + +/* ── Deploy commands ───────────────────────────────────────────────── */ + +/* Record hit `x` of command `cmd`: its file, line and CI context are the + * state's; ext / path are copied. */ +static void sh_hit_push(sh_t *s, TSNode cmd, sh_hit_t x) { + sh_hits_t *h = s->hits; + if (!sh_grow((void **)&h->items, &h->cap, h->count, sizeof(*h->items))) { + return; + } + x.file = s->file; + x.line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base; + x.ext = x.ext ? cbm_arena_strdup(&h->arena, x.ext) : NULL; + x.path = x.path ? cbm_arena_strdup(&h->arena, x.path) : NULL; + x.ci = s->ci; + x.job = s->job ? cbm_arena_strdup(&h->arena, s->job) : NULL; + h->items[h->count++] = x; +} + +static void sh_hit(sh_t *s, TSNode cmd, const char *via, const sh_target_t *t, const char *ext, + bool chart) { + sh_hit_push(s, cmd, + (sh_hit_t){.via = via, + .tgt = t ? t->id : 0, + .ext = ext, + .path = t ? t->path : ext, + .chart = chart}); +} + +static void sh_hit_all(sh_t *s, TSNode cmd, const char *via, const sh_targets_t *ts, bool chart) { + for (int i = 0; i < ts->count; i++) { + sh_hit(s, cmd, via, &ts->items[i], NULL, chart); + } +} + +static bool sh_in(const char *v, const char *const *set) { + for (int i = 0; v && set[i]; i++) { + if (strcmp(v, set[i]) == 0) { + return true; + } + } + return false; +} + +static bool sh_numberish(const char *v) { + if (!v[0] || v[0] < '0' || v[0] > '9') { + return false; + } + for (const char *p = v; *p; p++) { + if (!((*p >= '0' && *p <= '9') || *p == '.' || strchr("smhd", *p))) { + return false; + } + } + return true; +} + +/* Word i of argv is an option, assignment or count a wrapper takes before + * the program: by its value, or -- the value unknown -- by its text + * (env OUT="$TMP/x" prog: the assignment is no program). */ +static bool sh_wrapper_arg(const sh_t *s, const sh_argv_t *a, int i) { + const char *v = a->v[i]; + if (v) { + return v[0] == '-' || strchr(v, '=') || sh_numberish(v); + } + uint32_t b = a->span[PAIR_LEN * i]; + uint32_t e = a->span[PAIR_LEN * i + SKIP_ONE]; + return e > b && (s->src[b] == '-' || memchr(s->src + b, '=', e - b)); +} + +/* The index of the program argv runs, past wrappers (time, sudo, retry N, + * timeout D, env A=B, ...); -1 when argv has none. Its value may be unknown + * (NULL). */ +static int sh_program(const sh_t *s, const sh_argv_t *a) { + static const char *const wrappers[] = {"time", "sudo", "exec", "nice", "nohup", + "command", "env", "timeout", "retry", NULL}; + int i = 0; + while (i < a->n) { + if (!a->v[i] || !sh_in(sh_basename(a->v[i]), wrappers)) { + return i; + } + if (strcmp(a->v[i], "command") == 0 && i + SKIP_ONE < a->n && + (sh_eq(a->v[i + SKIP_ONE], "-v") || sh_eq(a->v[i + SKIP_ONE], "-V"))) { + return -1; /* command -v X: is there an X, not a run of it */ + } + i++; + while (i < a->n && sh_wrapper_arg(s, a, i)) { + i++; + } + } + return -1; +} + +/* kubectl's options that take a separate value (before the verb). */ +static const char *const SH_KUBECTL_VALUED[] = {"-n", + "--namespace", + "--context", + "--kubeconfig", + "--cluster", + "--user", + "-l", + "--selector", + "-o", + "--output", + "--server", + "-s", + "--token", + "--as", + "--as-group", + "--request-timeout", + "--cache-dir", + "--certificate-authority", + "--client-certificate", + "--client-key", + NULL}; + +/* helm's options that take a separate value. */ +static const char *const SH_HELM_VALUED[] = {"-n", + "--namespace", + "-f", + "--values", + "--set", + "--set-string", + "--set-file", + "--set-json", + "--set-literal", + "--version", + "--timeout", + "--kubeconfig", + "--kube-context", + "--repo", + "--post-renderer", + "--post-renderer-args", + "--description", + "--username", + "--password", + "--ca-file", + "--cert-file", + "--key-file", + "--keyring", + "--history-max", + "--name-template", + "--registry-config", + "--repository-config", + "--repository-cache", + "-o", + "--output", + "--api-versions", + "-a", + "--kube-version", + "--labels", + "-l", + "--show-only", + "-s", + "--output-dir", + "--release-name", + "--kube-apiserver", + "--kube-as-user", + "--kube-as-group", + "--kube-ca-file", + "--kube-token", + "--burst-limit", + "--qps", + NULL}; + +typedef enum { SH_SRC_NONE, SH_SRC_KUST, SH_SRC_CHART } sh_src_kind_t; + +typedef struct { + sh_src_kind_t kind; + const char *v; + const char *via; +} sh_src_t; + +typedef struct { + int verb; /* index of the verb in argv; -1 none */ + const char **files; + int nfiles; + const char **kusts; + int nkusts; + int cap_files; + int cap_kusts; + bool recursive; +} sh_kubectl_t; + +static void sh_kubectl_free(sh_kubectl_t *k) { + cbm_free(CBM_MEM_CLASS_OTHER, (void *)k->files); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)k->kusts); +} + +static void sh_kubectl_add(const char ***v, int *n, int *cap, const char *p) { + if (sh_grow((void **)v, cap, *n, sizeof(**v))) { + (*v)[(*n)++] = p; + } +} + +/* kubectl [options] VERB ...: the verb and its -f / -k / -R. */ +static void sh_kubectl_parse(const sh_argv_t *a, int from, sh_kubectl_t *k) { + k->verb = -1; + int i = from; + while (i < a->n && k->verb < 0) { + const char *w = a->v[i]; + if (!w) { + return; + } + if (w[0] != '-') { + k->verb = i++; + break; + } + i += (sh_in(w, SH_KUBECTL_VALUED) && !strchr(w, '=')) ? PAIR_LEN : SKIP_ONE; + } + for (; i < a->n; i++) { + const char *w = a->v[i]; + if (!w || w[0] != '-') { + continue; + } + const char *next = i + SKIP_ONE < a->n ? a->v[i + SKIP_ONE] : NULL; + if (strcmp(w, "--recursive") == 0) { + k->recursive = true; + } else if (strcmp(w, "--filename") == 0 || strcmp(w, "--kustomize") == 0) { + bool fl = w[PAIR_LEN] == 'f'; + sh_kubectl_add(fl ? &k->files : &k->kusts, fl ? &k->nfiles : &k->nkusts, + fl ? &k->cap_files : &k->cap_kusts, next); + i++; + } else if (sh_starts(w, "--filename=")) { + sh_kubectl_add(&k->files, &k->nfiles, &k->cap_files, strchr(w, '=') + SKIP_ONE); + } else if (sh_starts(w, "--kustomize=")) { + sh_kubectl_add(&k->kusts, &k->nkusts, &k->cap_kusts, strchr(w, '=') + SKIP_ONE); + } else if (w[SKIP_ONE] != '-') { + /* a cluster of short options: -f, -k, -R, -Rf */ + size_t len = strlen(w); + bool ok = len > SKIP_ONE; + for (size_t c = SKIP_ONE; c < len; c++) { + ok = ok && ((w[c] >= 'a' && w[c] <= 'z') || (w[c] >= 'A' && w[c] <= 'Z')); + } + if (!ok) { + continue; + } + if (strchr(w, 'R')) { + k->recursive = true; + } + char last = w[len - SKIP_ONE]; + if (last == 'f' || last == 'k') { + sh_kubectl_add(last == 'f' ? &k->files : &k->kusts, + last == 'f' ? &k->nfiles : &k->nkusts, + last == 'f' ? &k->cap_files : &k->cap_kusts, next); + i++; + } + } + } +} + +static void sh_deploy_src(sh_t *s, TSNode cmd, const sh_src_t *src); + +/* kubectl: apply / create / replace -f / -k, or `kubectl kustomize D` as a + * pipe's source. */ +static void sh_kubectl(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog, const sh_src_t *in, + sh_src_t *out) { + sh_kubectl_t k = {0}; + sh_kubectl_parse(a, prog + SKIP_ONE, &k); + const char *verb = k.verb >= 0 ? a->v[k.verb] : NULL; + if (sh_eq(verb, "kustomize")) { + const char *d = sh_first_operand(a, k.verb + SKIP_ONE); + *out = (sh_src_t){SH_SRC_KUST, d ? d : ".", "kustomize"}; + } else if (sh_eq(verb, "apply") || sh_eq(verb, "create") || sh_eq(verb, "replace")) { + sh_find_t f = {k.recursive, NULL, SH_NO_DEPTH}; + for (int i = 0; i < k.nfiles; i++) { + if (sh_eq(k.files[i], "-")) { + sh_deploy_src(s, cmd, in); + continue; + } + sh_targets_t ts = {0}; + sh_targets_of(s, k.files[i], sh_t_files, &f, &ts); + sh_hit_all(s, cmd, "kubectl", &ts, false); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + } + for (int i = 0; i < k.nkusts; i++) { + sh_targets_t ts = {0}; + sh_targets_of(s, k.kusts[i], sh_t_kust, &f, &ts); + sh_hit_all(s, cmd, "kubectl", &ts, false); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + } + } + sh_kubectl_free(&k); +} + +/* The chart a helm word names: a chart dir, else an external chart. */ +static void sh_chart(sh_t *s, TSNode cmd, const char *v) { + if (!v) { + return; + } + if (sh_starts(v, "oci://")) { + const char *ext = sh_value_basename(s->a, v + strlen("oci://")); + if (ext && ext[0] && !strchr(ext, ':')) { + sh_hit(s, cmd, "helm", NULL, ext, true); + } + return; + } + sh_targets_t ts = {0}; + sh_targets_of(s, v, sh_t_chart, NULL, &ts); + if (ts.count > 0) { + sh_hit_all(s, cmd, "helm", &ts, true); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + return; + } + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + /* repo/name: one slash, a plain name on each side */ + const char *slash = strchr(v, '/'); + if (!slash || strchr(slash + SKIP_ONE, '/') || v[0] == '.' || slash == v || !slash[SKIP_ONE]) { + return; + } + for (const char *p = v; *p; p++) { + if (!(*p == '/' || *p == '.' || *p == '_' || *p == '-' || (*p >= '0' && *p <= '9') || + (*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z'))) { + return; + } + } + sh_hit(s, cmd, "helm", NULL, slash + SKIP_ONE, true); +} + +/* helm install|upgrade [NAME] CHART; helm template [NAME] CHART as a source. */ +static void sh_helm(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog, sh_src_t *out) { + const char *verb = NULL; + int pos[PAIR_LEN] = {0, 0}; + int npos = 0; + int first_split = SH_NO_DEPTH; /* an unknown unquoted word: maybe several */ + bool gen = false; + for (int i = prog + SKIP_ONE; i < a->n; i++) { + const char *w = a->v[i]; + if (w && w[0] == '-') { + gen = gen || strcmp(w, "--generate-name") == 0 || strcmp(w, "-g") == 0; + if (sh_in(w, SH_HELM_VALUED) && !strchr(w, '=')) { + i++; + } + continue; + } + if (!verb) { + if (!w) { + return; + } + verb = w; + continue; + } + if (!w && a->split[i] && first_split == SH_NO_DEPTH) { + first_split = npos; + } + if (npos < PAIR_LEN) { + pos[npos] = i; + } + npos++; + } + if (!verb) { + return; + } + if (strcmp(verb, "install") == 0 || strcmp(verb, "upgrade") == 0) { + int at = gen ? 0 : SKIP_ONE; + if (npos > at && (first_split == SH_NO_DEPTH || first_split > at)) { + sh_chart(s, cmd, a->v[pos[at]]); + } + } else if (strcmp(verb, "template") == 0 && npos > 0 && + (first_split == SH_NO_DEPTH || first_split > SKIP_ONE)) { + int at = npos >= PAIR_LEN ? SKIP_ONE : 0; + *out = (sh_src_t){SH_SRC_CHART, a->v[pos[at]], "helm"}; + } +} + +/* What a `kubectl apply -f -` reading the pipe deploys. */ +static void sh_deploy_src(sh_t *s, TSNode cmd, const sh_src_t *src) { + if (!src || src->kind == SH_SRC_NONE || !src->v) { + return; + } + if (src->kind == SH_SRC_CHART) { + sh_chart(s, cmd, src->v); + return; + } + sh_targets_t ts = {0}; + sh_targets_of(s, src->v, sh_t_kust, NULL, &ts); + sh_hit_all(s, cmd, src->via, &ts, false); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); +} + +/* find DIR [-type f] [-name G] [-maxdepth N] -exec kubectl apply -f {} \; */ +static void sh_find(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog) { + int i = prog + SKIP_ONE; + if (i >= a->n || !a->v[i] || a->v[i][0] == '-') { + return; + } + const char *root = a->v[i++]; + sh_find_t f = {false, NULL, SH_NO_DEPTH}; + while (i < a->n) { + const char *w = a->v[i]; + const char *next = i + SKIP_ONE < a->n ? a->v[i + SKIP_ONE] : NULL; + if (!w) { + return; + } + if (strcmp(w, "-exec") == 0) { + break; + } + if (strcmp(w, "-name") == 0 && next) { + f.name = next; + } else if (strcmp(w, "-maxdepth") == 0 && next && sh_numberish(next)) { + f.maxdepth = (int)strtol(next, NULL, SH_DEC); + } else if (!(strcmp(w, "-type") == 0 && sh_eq(next, "f"))) { + return; + } + i += PAIR_LEN; /* each test takes one value */ + } + /* -exec PROGRAM ... {} ; */ + sh_argv_t e = {0}; + for (i++; i < a->n && !sh_eq(a->v[i], ";") && !sh_eq(a->v[i], "+"); i++) { + sh_argv_push(&e, a->v[i], a->split[i], a->span[PAIR_LEN * i], + a->span[PAIR_LEN * i + SKIP_ONE]); + } + int p = sh_program(s, &e); + if (p >= 0 && e.v[p] && strcmp(sh_basename(e.v[p]), "kubectl") == 0) { + sh_kubectl_t k = {0}; + sh_kubectl_parse(&e, p + SKIP_ONE, &k); + const char *verb = k.verb >= 0 ? e.v[k.verb] : NULL; + if ((sh_eq(verb, "apply") || sh_eq(verb, "create") || sh_eq(verb, "replace")) && + k.nfiles == SKIP_ONE && k.nkusts == 0 && sh_eq(k.files[0], "{}")) { + sh_targets_t ts = {0}; + sh_targets_of(s, root, sh_t_found, &f, &ts); + sh_hit_all(s, cmd, "kubectl", &ts, false); + cbm_free(CBM_MEM_CLASS_OTHER, ts.items); + } + sh_kubectl_free(&k); + } + sh_argv_free(&e); +} + +/* One command of a pipeline (or a lone one): its deploys; *out: what it + * writes to the pipe, when that is a kustomize build or a helm template. */ +static void sh_deploy_command(sh_t *s, TSNode cmd, const sh_argv_t *a, const sh_src_t *in, + sh_src_t *out) { + *out = (sh_src_t){SH_SRC_NONE, NULL, NULL}; + int prog = sh_program(s, a); + if (prog < 0 || !a->v[prog]) { + return; + } + const char *base = sh_basename(a->v[prog]); + if (strcmp(base, "kubectl") == 0) { + sh_kubectl(s, cmd, a, prog, in, out); + } else if (strcmp(base, "helm") == 0) { + sh_helm(s, cmd, a, prog, out); + } else if (strcmp(base, "kustomize") == 0) { + const char *verb = sh_first_operand(a, prog + SKIP_ONE); + if (sh_eq(verb, "build")) { + int at = SH_NO_DEPTH; + for (int i = prog + SKIP_ONE; i < a->n; i++) { + if (a->v[i] == verb) { + at = i; + break; + } + } + const char *d = at >= 0 ? sh_first_operand(a, at + SKIP_ONE) : NULL; + *out = (sh_src_t){SH_SRC_KUST, d ? d : ".", "kustomize"}; + } + } else if (strcmp(base, "find") == 0) { + sh_find(s, cmd, a, prog); + } +} + +/* The File node a repository path names; 0: none. */ +static int64_t sh_repo_file_id(const sh_t *s, const char *path) { + return (int64_t)(intptr_t)cbm_ht_get(s->ix->file_ids, path); +} + +static const char *const SH_SHELLS[] = {"bash", "sh", "zsh", "ksh", "dash", NULL}; + +/* The one candidate of `path` that names a File of the repository (its + * File node in *id); NULL: none, or two. */ +static const char *sh_unique_file(sh_t *s, const char *path, int64_t *id) { + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, path, c); + const char *hit = NULL; + int found = 0; + for (int i = 0; i < n; i++) { + int64_t f = sh_repo_file_id(s, c[i]); + if (f > 0) { + found++; + *id = f; + hit = c[i]; + } + } + return found == SKIP_ONE ? hit : NULL; +} + +/* The program is a script of the repository -- its path names a file of it, + * or a shell runs one (bash [-opts] FILE; -c TEXT is no file) -- resolved to + * exactly one file: that script is spawned. */ +static void sh_spawn_script(sh_t *s, TSNode cmd, const sh_argv_t *a, int p) { + const char *v = a->v[p]; + const char *path = NULL; + if (sh_in(sh_basename(v), SH_SHELLS)) { + int j = p + SKIP_ONE; + while (j < a->n && a->v[j] && a->v[j][0] == '-' && !sh_eq(a->v[j], "-c")) { + j++; + } + path = j < a->n && !sh_eq(a->v[j], "-c") ? a->v[j] : NULL; + } else if (strchr(v, '/') && !strpbrk(v, "*?[")) { + path = v; + } + if (!path) { + return; + } + int64_t id = 0; + const char *hit = sh_unique_file(s, path, &id); + if (hit) { + sh_hit_push(s, cmd, + (sh_hit_t){.via = s->exec ? "exec" : "shell", + .tgt = id, + .ext = sh_basename(hit), + .path = hit, + .spawn = true}); + } +} + +/* ── make TARGET (P3.5b) ───────────────────────────────────────────── */ + +static const char *const SH_MAKES[] = {"make", "gmake", NULL}; +static const char *const SH_MAKEFILES[] = {"GNUmakefile", "makefile", "Makefile", NULL}; + +typedef enum { MK_OPT_NONE, MK_OPT_DIR, MK_OPT_FILE, MK_OPT_ARG } mk_opt_t; + +/* A long option of make that takes a value (v: n bytes, no `=...`). */ +static mk_opt_t sh_make_long(const char *v, size_t n) { + static const struct { + const char *name; + mk_opt_t kind; + } opts[] = { + {"--directory", MK_OPT_DIR}, {"--file", MK_OPT_FILE}, {"--makefile", MK_OPT_FILE}, + {"--include-dir", MK_OPT_ARG}, {"--old-file", MK_OPT_ARG}, {"--assume-old", MK_OPT_ARG}, + {"--what-if", MK_OPT_ARG}, {"--new-file", MK_OPT_ARG}, {"--assume-new", MK_OPT_ARG}}; + for (size_t i = 0; i < sizeof(opts) / sizeof(opts[0]); i++) { + if (strlen(opts[i].name) == n && strncmp(v, opts[i].name, n) == 0) { + return opts[i].kind; + } + } + return MK_OPT_NONE; +} + +/* A command-line variable: NAME=, NAME:=, NAME::=, NAME+=, NAME?=, NAME!=. */ +static bool sh_make_assignment(const char *v) { + if (!(v[0] == '_' || ((v[0] | ' ') >= 'a' && (v[0] | ' ') <= 'z'))) { + return false; + } + size_t i = SKIP_ONE; + while (v[i] && (v[i] == '_' || v[i] == '.' || v[i] == '-' || (v[i] >= '0' && v[i] <= '9') || + ((v[i] | ' ') >= 'a' && (v[i] | ' ') <= 'z'))) { + i++; + } + while (v[i] == ' ' || v[i] == '\t') { + i++; + } + if (v[i] == ':') { + while (v[i] == ':') { + i++; + } + return v[i] == '='; + } + return v[i] == '=' || (v[i] && strchr("+?!", v[i]) && v[i + SKIP_ONE] == '='); +} + +/* -j N / -l N: N is a count. */ +static bool sh_make_count(const char *v) { + if (!v || !v[0]) { + return false; + } + for (const char *p = v; *p; p++) { + if (!(*p == '.' || (*p >= '0' && *p <= '9'))) { + return false; + } + } + return true; +} + +/* -C DIR after `base`: DIR from base, an absolute (anchored) DIR alone. */ +static const char *sh_make_join(CBMArena *a, const char *base, const char *dir) { + if (dir[0] == SH_ANCHOR || dir[0] == '/' || sh_eq(base, ".")) { + return dir; + } + return cbm_arena_sprintf(a, "%s/%s", base, dir); +} + +/* The makefile make reads in directory `dir` (GNUmakefile, makefile, + * Makefile, the first there is), for the one candidate of `dir` that has + * one; NULL otherwise. */ +static const char *sh_make_default(sh_t *s, const char *dir) { + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, dir, c); + const char *hit = NULL; + int found = 0; + for (int i = 0; i < n; i++) { + for (int k = 0; SH_MAKEFILES[k]; k++) { + const char *p = sh_join(s->a, c[i], SH_MAKEFILES[k]); + if (p && sh_repo_file_id(s, p) > 0) { + found++; + hit = p; + break; + } + } + } + return found == SKIP_ONE ? hit : NULL; +} + +/* make [-C DIR] [-f FILE] [VAR=V] TARGET...: each TARGET SPAWNS the rule of + * that name the makefile reads -- the first -f, else the directory's (-C is + * cumulative) -- resolved when linking, through its includes. An unknown -C + * or -f value is no edge, an unknown word no target; no target (the default + * goal) is no edge. */ +static void sh_make(sh_t *s, TSNode cmd, const sh_argv_t *a, int p) { + const char *dir = "."; + const char *file = NULL; + const char **targets = cbm_arena_alloc(s->a, (size_t)(a->n + SKIP_ONE) * sizeof(*targets)); + if (!targets) { + return; + } + int nt = 0; + bool rest = false; + for (int j = p + SKIP_ONE; j < a->n; j++) { + const char *v = a->v[j]; + if (!v) { + continue; + } + if (rest || v[0] != '-' || sh_eq(v, "-")) { + if (rest || !sh_make_assignment(v)) { + targets[nt++] = v; + } + continue; + } + if (sh_eq(v, "--")) { + rest = true; + continue; + } + const char *arg = NULL; + mk_opt_t kind = MK_OPT_NONE; + if (v[SKIP_ONE] == '-') { + const char *eq = strchr(v, '='); + kind = sh_make_long(v, eq ? (size_t)(eq - v) : strlen(v)); + arg = eq ? eq + SKIP_ONE : NULL; + if (kind != MK_OPT_NONE && !eq) { + j++; + arg = j < a->n ? a->v[j] : NULL; + } + } else { + for (size_t k = SKIP_ONE; v[k]; k++) { + if (strchr("CfIoW", v[k])) { + kind = v[k] == 'C' ? MK_OPT_DIR : v[k] == 'f' ? MK_OPT_FILE : MK_OPT_ARG; + arg = v[k + SKIP_ONE] ? v + k + SKIP_ONE : NULL; + if (!arg) { + j++; + arg = j < a->n ? a->v[j] : NULL; + } + break; + } + if (v[k] == 'j' || v[k] == 'l') { + if (!v[k + SKIP_ONE] && j + SKIP_ONE < a->n && + sh_make_count(a->v[j + SKIP_ONE])) { + j++; + } + break; + } + } + } + if (kind == MK_OPT_DIR || kind == MK_OPT_FILE) { + if (!arg) { + return; + } + if (kind == MK_OPT_DIR) { + dir = sh_make_join(s->a, dir, arg); + } else if (!file) { + file = arg; + } + } + } + int64_t id = 0; + const char *mkfile = + file ? sh_unique_file(s, sh_make_join(s->a, dir, file), &id) : sh_make_default(s, dir); + for (int i = 0; mkfile && i < nt; i++) { + sh_hit_push(s, cmd, + (sh_hit_t){.via = s->exec ? "exec" : "shell", + .ext = targets[i], + .path = mkfile, + .spawn = true, + .make = true}); + } +} + +/* A program started by a path (./_build/x, ${DIR}/x, $(LOCALBIN)/x) SPAWNS + * the Process of its name -- the value's last segment, or, the value + * unknown, the word's literal last segment. A path naming a file of the + * repository is a script (sh_spawn_script), not a Process. */ +static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) { + int p = sh_program(s, a); + if (p < 0) { + return; + } + const char *v = a->v[p]; + const char *name = NULL; + if (v) { + if (sh_in(sh_basename(v), SH_MAKES)) { + sh_make(s, cmd, a, p); + } + sh_spawn_script(s, cmd, a, p); + if (!strchr(v, '/') || strpbrk(v, "*?[")) { + return; + } + const char *c[PAIR_LEN] = {NULL, NULL}; + int n = sh_candidates(s, v, c); + for (int i = 0; i < n; i++) { + if (sh_repo_file_id(s, c[i]) > 0) { + return; + } + } + name = sh_value_basename(s->a, v); + } else { + uint32_t b = a->span[PAIR_LEN * p]; + uint32_t e = a->span[PAIR_LEN * p + SKIP_ONE]; + while (e > b && (s->src[e - SKIP_ONE] == '"' || s->src[e - SKIP_ONE] == '\'')) { + e--; + } + uint32_t k = e; + while (k > b && s->src[k - SKIP_ONE] != '/') { + k--; + } + if (k == b) { + return; + } + name = cbm_arena_strndup(s->a, s->src + k, e - k); + if (name && strpbrk(name, "$`*?[]{}()\"' \t\\")) { + return; + } + } + if (!name || !name[0] || strcmp(name, ".") == 0 || strcmp(name, "..") == 0) { + return; + } + sh_hit_push(s, cmd, (sh_hit_t){.via = s->exec ? "exec" : "shell", .ext = name, .spawn = true}); +} + +/* ── Statements ────────────────────────────────────────────────────── */ + +static void sh_walk(sh_t *s, TSNode n); + +static void sh_walk_children(sh_t *s, TSNode n) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + sh_walk(s, ts_node_named_child(n, i)); + } +} + +/* The commands inside $( ) and <( ) of a word, a value or a redirect run + * too, each in a subshell of its own. */ +static void sh_walk_substs(sh_t *s, TSNode n) { + if (sh_type(n, "command_substitution") || sh_type(n, "process_substitution")) { + sh_mark_t m = sh_mark(s); + sh_walk_children(s, n); + sh_rewind(s, m); + return; + } + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + sh_walk_substs(s, ts_node_named_child(n, i)); + } +} + +static void sh_assign(sh_t *s, TSNode n, bool local) { + TSNode name = ts_node_child_by_field_name(n, TS_FIELD("name")); + TSNode value = ts_node_child_by_field_name(n, TS_FIELD("value")); + if (!ts_node_is_null(value)) { + sh_walk_substs(s, value); + } + if (ts_node_is_null(name)) { + return; + } + if (sh_type(name, "subscript")) { + TSNode base = ts_node_child_by_field_name(name, TS_FIELD("name")); + if (!ts_node_is_null(base)) { + sh_set(s, sh_text(s, base), &SH_UNKNOWN_VAL, local); + } + return; + } + const char *nm = sh_text(s, name); + sh_val_t *v = cbm_arena_alloc(s->a, sizeof(*v)); + if (!nm || !v) { + return; + } + *v = (sh_val_t){"", NULL, 0}; + if (sh_type(value, "array")) { + uint32_t cnt = ts_node_named_child_count(value); + const char **items = cbm_arena_alloc(s->a, (cnt ? cnt : SKIP_ONE) * sizeof(*items)); + if (!items) { + return; + } + int k = 0; + for (uint32_t i = 0; i < cnt; i++) { + TSNode c = ts_node_named_child(value, i); + if (!sh_type(c, "comment")) { + items[k++] = sh_eval_word(s, c); + } + } + *v = (sh_val_t){k > 0 ? items[0] : "", items, k}; + } else if (!ts_node_is_null(value)) { + v->s = sh_eval(s, value); + } + bool append = false; + for (uint32_t i = 0; i < ts_node_child_count(n); i++) { + TSNode c = ts_node_child(n, i); + append = append || (!ts_node_is_named(c) && strcmp(ts_node_type(c), "+=") == 0); + } + if (append) { + bool found = false; + const sh_val_t *prev = sh_lookup(s, nm, &found); + if (found) { + v->s = (prev->s && v->s) ? cbm_arena_sprintf(s->a, "%s%s", prev->s, v->s) : NULL; + v->items = NULL; + } + } + sh_set(s, nm, v, local); +} + +/* local / declare / typeset / export / readonly NAME[=VALUE] ... */ +static void sh_declare(sh_t *s, TSNode n) { + TSNode kw = ts_node_child(n, 0); + const char *k = ts_node_is_null(kw) ? "" : ts_node_type(kw); + bool local = strcmp(k, "local") == 0 || + (s->in_func && (strcmp(k, "declare") == 0 || strcmp(k, "typeset") == 0)); + bool declares = local || strcmp(k, "declare") == 0 || strcmp(k, "typeset") == 0; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (sh_type(c, "variable_assignment")) { + sh_assign(s, c, local); + } else if (declares && sh_type(c, "variable_name")) { + sh_set(s, sh_text(s, c), &SH_UNKNOWN_VAL, local); + } + } +} + +/* cd / pushd / popd; true when the command is one. */ +static bool sh_chdir(sh_t *s, const sh_argv_t *a) { + const char *name = a->n > 0 ? a->v[0] : NULL; + if (sh_eq(name, "popd")) { + s->cwd = s->ndirs > 0 ? s->dirs[--s->ndirs] : (sh_cwd_t){SH_LOST, NULL}; + return true; + } + if (!sh_eq(name, "cd") && !sh_eq(name, "pushd")) { + return false; + } + if (sh_eq(name, "pushd")) { + if (sh_grow((void **)&s->dirs, &s->cap_dirs, s->ndirs, sizeof(*s->dirs))) { + s->dirs[s->ndirs++] = s->cwd; + } + } + const char *to = sh_first_operand(a, SKIP_ONE); + sh_cwd_t next = {SH_LOST, NULL}; + if (to && strcmp(to, "-") != 0) { + (void)sh_cd_target(s, to, &next); + } + s->cwd = next; + return true; +} + +/* A container build's flags that take a separate value. */ +static const char *const SH_BUILD_VALUED[] = {"--build-arg", + "--target", + "--platform", + "--label", + "--cache-from", + "--cache-to", + "--secret", + "--ssh", + "-o", + "--output", + "--network", + "--add-host", + "--iidfile", + "--progress", + "--builder", + "--build-context", + "--shm-size", + "--ulimit", + "-m", + "--memory", + "--memory-swap", + "--cpu-shares", + "-c", + "--cpuset-cpus", + "--cpuset-mems", + "--cgroup-parent", + "--isolation", + "--metadata-file", + "--annotation", + "--attest", + "--security-opt", + "--format", + "--jobs", + "--os", + "--arch", + "--variant", + "--manifest", + "--authfile", + "--cert-dir", + "--creds", + "--allow", + "--call", + NULL}; + +enum { SH_BUILD_TAGS = 16, SH_IMAGE_NEST = 16 }; + +/* The repository of image word `i` whose value is unknown: the word up to + * the ':' of its tag (or '@' of a digest) after its last '/', outside + * $( ) ${ } and backticks, evaluated alone -- the repository is the image's + * identity, a tag of unknown value keeps it. NULL: no tag to cut, or the + * repository is unknown too. */ +static const char *sh_image_repo(sh_t *s, TSNode cmd, const sh_argv_t *a, int i) { + uint32_t b = a->span[(size_t)i * PAIR_LEN]; + uint32_t e = a->span[(size_t)i * PAIR_LEN + SKIP_ONE]; + char closers[SH_IMAGE_NEST]; + int depth = 0; + uint32_t cut = 0; + for (uint32_t k = b; k < e; k++) { + char c = s->src[k]; + bool open2 = k + SKIP_ONE < e && c == '$' && + (s->src[k + SKIP_ONE] == '(' || s->src[k + SKIP_ONE] == '{'); + if (depth > 0 && c == closers[depth - SKIP_ONE]) { + depth--; + } else if ((open2 || c == '`') && depth < SH_IMAGE_NEST) { + closers[depth++] = c == '`' ? '`' : (s->src[k + SKIP_ONE] == '(' ? ')' : '}'); + k += open2 ? SKIP_ONE : 0; + } else if (depth == 0 && c == '/') { + cut = 0; + } else if (depth == 0 && (c == ':' || c == '@') && cut == 0) { + cut = k; + } + } + if (cut == 0) { + return NULL; + } + TSNode w = ts_node_descendant_for_byte_range(cmd, b, e); + while (!ts_node_is_null(w) && (ts_node_start_byte(w) != b || ts_node_end_byte(w) != e)) { + w = ts_node_parent(w); + } + sh_str_t out = {0}; + if (sh_type(w, "concatenation")) { + sh_concat(s, &out, w, b, cut, false); + } else if (sh_type(w, "string")) { + sh_concat(s, &out, w, b + SKIP_ONE, cut, true); + } else { + return NULL; + } + return sh_str_done(&out); +} + +/* docker / podman / nerdctl [buildx | image | builder] build, buildah bud: + * each tagged image is built from the Dockerfile -f names (against the cwd), + * else /Dockerfile -- one file of the repository. */ +static void sh_build(sh_t *s, TSNode cmd, const sh_argv_t *a) { + int p = sh_program(s, a); + const char *tool = p >= 0 && a->v[p] ? sh_basename(a->v[p]) : NULL; + int i = p + SKIP_ONE; + if (tool && strcmp(tool, "buildah") == 0) { + if (i >= a->n || !a->v[i] || + !(sh_eq(a->v[i], "bud") || sh_eq(a->v[i], "build") || + sh_eq(a->v[i], "build-using-dockerfile"))) { + return; + } + i++; + } else if (tool && (strcmp(tool, "docker") == 0 || strcmp(tool, "podman") == 0 || + strcmp(tool, "nerdctl") == 0)) { + if (i < a->n && sh_eq(a->v[i], "build")) { + i++; + } else if (i + SKIP_ONE < a->n && + (sh_eq(a->v[i], "buildx") || sh_eq(a->v[i], "image") || + sh_eq(a->v[i], "builder")) && + sh_eq(a->v[i + SKIP_ONE], "build")) { + i += PAIR_LEN; + } else { + return; + } + } else { + return; + } + if (s->hits->declarers) { + cbm_ht_set(s->hits->declarers, s->file, (void *)s->file); + } + int tags[SH_BUILD_TAGS]; + const char *tag_vals[SH_BUILD_TAGS]; + int nt = 0; + const char *dfile = NULL; + bool dfile_known = true; + const char *ctx = NULL; + bool ctx_seen = false; + for (; i < a->n; i++) { + const char *w = a->v[i]; + if (w && (sh_eq(w, "-t") || sh_eq(w, "--tag")) && i + SKIP_ONE < a->n) { + if (nt < SH_BUILD_TAGS) { + tags[nt] = i + SKIP_ONE; + tag_vals[nt++] = a->v[i + SKIP_ONE]; + } + i++; + } else if (w && (sh_eq(w, "-f") || sh_eq(w, "--file")) && i + SKIP_ONE < a->n) { + dfile = a->v[i + SKIP_ONE]; + dfile_known = dfile != NULL; + i++; + } else if (w && strncmp(w, "--tag=", strlen("--tag=")) == 0) { + if (nt < SH_BUILD_TAGS && w[strlen("--tag=")]) { + tags[nt] = -SKIP_ONE; + tag_vals[nt++] = w + strlen("--tag="); + } + } else if (w && strncmp(w, "--file=", strlen("--file=")) == 0) { + dfile = w + strlen("--file="); + } else if (w && w[0] == '-') { + i += !strchr(w, '=') && sh_in(w, SH_BUILD_VALUED) ? SKIP_ONE : 0; + } else if (!ctx_seen) { + ctx_seen = true; + ctx = w; + } + } + if (nt == 0 || !ctx || !dfile_known || strcmp(ctx, "-") == 0 || strstr(ctx, "://") || + strncmp(ctx, "git@", strlen("git@")) == 0) { + return; + } + const char *path = dfile ? dfile : sh_join(s->a, ctx, "Dockerfile"); + int64_t id = 0; + const char *hit = path ? sh_unique_file(s, path, &id) : NULL; + for (int k = 0; hit && k < nt; k++) { + const char *img = tag_vals[k]; + if (!img && tags[k] >= 0) { + img = sh_image_repo(s, cmd, a, tags[k]); + } + if (img && img[0]) { + sh_hit_push(s, cmd, + (sh_hit_t){.via = "docker-build", .ext = img, .path = hit, .build = true}); + } + } +} + +static void sh_command(sh_t *s, TSNode cmd) { + sh_walk_substs(s, cmd); + sh_argv_t a = {0}; + sh_argv_build(s, cmd, &a); + if (!sh_chdir(s, &a)) { + sh_src_t out; + sh_deploy_command(s, cmd, &a, NULL, &out); + sh_spawn(s, cmd, &a); + sh_build(s, cmd, &a); + } + sh_argv_free(&a); +} + +/* The redirects of a statement: `cat <in_func++; + if (sh_type(body, "compound_statement")) { + sh_walk_children(s, body); + } else { + sh_walk(s, body); + } + s->in_func--; + sh_touch_t t = {0}; + CBMHashTable *locals = cbm_ht_create(0); + for (int i = m.undo; locals && i < s->nundo; i++) { + if (s->undo[i].local) { + cbm_ht_set(locals, s->undo[i].name, (void *)s->undo[i].name); + } + } + for (int i = m.undo; i < s->nundo; i++) { + if (!locals || !cbm_ht_get(locals, s->undo[i].name)) { + sh_touch_add(&t, s->undo[i].name); + } + } + if (locals) { + cbm_ht_free(locals); + } + t.cwd = !sh_cwd_same(s->cwd, m.cwd) || s->ndirs != m.ndirs; + sh_rewind(s, m); + sh_group_end(s, &t); +} + +static void sh_walk(sh_t *s, TSNode n) { + const char *t = ts_node_type(n); + if (strcmp(t, "program") == 0 || strcmp(t, "compound_statement") == 0 || + strcmp(t, "negated_command") == 0) { + sh_walk_children(s, n); + } else if (strcmp(t, "variable_assignment") == 0) { + sh_assign(s, n, false); + } else if (strcmp(t, "variable_assignments") == 0) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + sh_assign(s, ts_node_named_child(n, i), false); + } + } else if (strcmp(t, "declaration_command") == 0) { + sh_declare(s, n); + } else if (strcmp(t, "unset_command") == 0) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (sh_type(c, "variable_name") || sh_type(c, "word")) { + sh_set(s, sh_text(s, c), &SH_UNKNOWN_VAL, false); + } + } + } else if (strcmp(t, "command") == 0) { + sh_command(s, n); + } else if (strcmp(t, "redirected_statement") == 0) { + TSNode body = ts_node_child_by_field_name(n, TS_FIELD("body")); + if (!ts_node_is_null(body)) { + sh_walk(s, body); + } + sh_mark_t m = sh_mark(s); + sh_redirects(s, n); + sh_rewind(s, m); + } else if (strcmp(t, "pipeline") == 0) { + sh_pipeline(s, n); + } else if (strcmp(t, "list") == 0) { + sh_list(s, n); + } else if (strcmp(t, "subshell") == 0) { + sh_mark_t m = sh_mark(s); + sh_walk_children(s, n); + sh_rewind(s, m); + } else if (strcmp(t, "if_statement") == 0) { + sh_if(s, n); + } else if (strcmp(t, "case_statement") == 0) { + sh_case(s, n); + } else if (strcmp(t, "for_statement") == 0 || strcmp(t, "c_style_for_statement") == 0 || + strcmp(t, "while_statement") == 0) { + sh_loop(s, n); + } else if (strcmp(t, "function_definition") == 0) { + sh_function(s, n); + } else { + sh_walk_substs(s, n); /* [[ -n "$(cmd)" ]], ((...)), a bare word */ + } +} + +/* ── Files ─────────────────────────────────────────────────────────── */ + +static bool sh_mentions_deploy_tool(const char *src, size_t n) { + static const char *const tools[] = {"kubectl", "helm", "kustomize", "docker", + "podman", "buildah", "nerdctl"}; + for (size_t i = 0; i < n; i++) { + char c = src[i] | ' '; + if (!strchr("khdpbn", c)) { + continue; + } + for (size_t t = 0; t < sizeof(tools) / sizeof(tools[0]); t++) { + size_t tl = strlen(tools[t]); + size_t j = 0; + while (j < tl && i + j < n && (src[i + j] | ' ') == tools[t][j]) { + j++; + } + if (j == tl) { + return true; + } + } + } + return false; +} + +static char *sh_read(const char *path, size_t *len) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + enum { SH_TS_PAD = 16 }; /* tree-sitter's lexer reads past the end */ + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + SH_TS_PAD); + if (!buf) { + (void)fclose(f); + return NULL; + } + size_t got = fread(buf, SKIP_ONE, (size_t)size, f); + (void)fclose(f); + memset(buf + got, 0, SH_TS_PAD); + *len = got; + return buf; +} + +typedef struct { + sh_index_t *ix; + sh_hits_t *hits; + TSParser *parser; + bool ix_ready; + bool ix_ok; +} sh_pass_t; + +static void sh_state_init(sh_t *s, sh_pass_t *p, CBMArena *a, const char *rel) { + *s = (sh_t){.ix = p->ix, .hits = p->hits, .a = a, .file = rel}; + s->env = cbm_ht_create(0); + const char *dir = sh_dirname(a, rel); + s->sdir = (dir && strcmp(dir, ".") != 0) ? dir : ""; +} + +static void sh_state_free(sh_t *s) { + if (s->env) { + cbm_ht_free(s->env); + } + cbm_free(CBM_MEM_CLASS_OTHER, s->undo); + cbm_free(CBM_MEM_CLASS_OTHER, s->dirs); +} + +/* Parse `text` as bash and walk it. */ +static void sh_run(sh_t *s, TSParser *parser, const char *text, size_t len) { + if (!ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_BASH))) { + return; + } + TSTree *tree = ts_parser_parse_string(parser, NULL, text, (uint32_t)len); + if (!tree) { + return; + } + s->src = text; + sh_walk(s, ts_tree_root_node(tree)); + ts_tree_delete(tree); +} + +static void sh_script(sh_pass_t *p, const char *rel, const char *src, size_t len) { + CBMArena a; + cbm_arena_init(&a); + sh_t s; + sh_state_init(&s, p, &a, rel); + if (s.env) { + s.script = sh_anchor(&a, rel); + s.cwd = (sh_cwd_t){SH_FREE, NULL}; + sh_run(&s, p->parser, src, len); + } + sh_state_free(&s); + cbm_arena_destroy(&a); +} + +/* ── Make ──────────────────────────────────────────────────────────── */ + +typedef struct { + const char *raw; /* the assigned text */ + const char *expanded; /* memo; NULL: not yet */ + bool known; + bool busy; +} mk_var_t; + +/* One recipe line as make runs it: a tab-led line and its \-continuations, + * without the @ - + prefixes. */ +typedef struct { + uint32_t b; /* source bytes [b, e) */ + uint32_t e; + uint32_t row; /* 0-based row of b */ +} mk_line_t; + +typedef struct mk mk_t; + +/* An include directive's expanded file names (mk_closure). */ +typedef void (*mk_include_fn)(mk_t *m, const char *names); + +struct mk { + CBMArena *a; + const char *src; + CBMHashTable *vars; /* name -> mk_var_t */ + bool makefile; /* a Makefile (its recipes run in its dir), not an included .mk */ + mk_line_t *lines; + int nlines; + int cap_lines; + mk_include_fn on_include; /* NULL: includes are not followed */ + void *ud; +}; + +static const char MK_UNKNOWN[] = "${__cbm_unknown}"; + +static const char *mk_expand(mk_t *m, const char *text); + +static const char *mk_lookup(mk_t *m, const char *name) { + if (strcmp(name, "CURDIR") == 0) { + return m->makefile ? "${__cbm_curdir}" : MK_UNKNOWN; + } + mk_var_t *v = cbm_ht_get(m->vars, name); + if (!v && strcmp(name, "MAKE") == 0) { + return "make"; /* make's own command, unless the makefile sets it */ + } + if (!v || !v->known || v->busy) { + return MK_UNKNOWN; + } + if (!v->expanded) { + v->busy = true; + v->expanded = mk_expand(m, v->raw); + v->busy = false; + } + return v->expanded ? v->expanded : MK_UNKNOWN; +} + +static const char *mk_trim(CBMArena *a, const char *p, size_t n) { + while (n && (*p == ' ' || *p == '\t')) { + p++; + n--; + } + while (n && (p[n - SKIP_ONE] == ' ' || p[n - SKIP_ONE] == '\t')) { + n--; + } + return cbm_arena_strndup(a, p, n); +} + +/* `cmd` is exactly the words `words` (NULL-terminated), however spaced. */ +static bool mk_words(const char *cmd, const char *const *words) { + const char *p = cmd; + for (int i = 0; words[i]; i++) { + while (*p == ' ' || *p == '\t') { + p++; + } + size_t wl = strlen(words[i]); + if (strncmp(p, words[i], wl) != 0 || (p[wl] && p[wl] != ' ' && p[wl] != '\t')) { + return false; + } + p += wl; + } + while (*p == ' ' || *p == '\t') { + p++; + } + return *p == '\0'; +} + +/* $(NAME) / ${NAME} / $(shell pwd) / $(abspath X): its expansion. */ +static const char *mk_reference(mk_t *m, const char *ref) { + static const char *const pwd[] = {"pwd", NULL}; + static const char *const pwd_p[] = {"pwd", "-P", NULL}; + static const char *const toplevel[] = {"git", "rev-parse", "--show-toplevel", NULL}; + size_t rl = strcspn(ref, " \t"); + if (!ref[rl]) { + return mk_lookup(m, ref); + } + const char *fn = cbm_arena_strndup(m->a, ref, rl); + const char *arg = mk_trim(m->a, ref + rl, strlen(ref + rl)); + if (!fn || !arg) { + return MK_UNKNOWN; + } + if (strcmp(fn, "shell") == 0) { + const char *cmd = mk_expand(m, arg); + if (cmd && (mk_words(cmd, pwd) || mk_words(cmd, pwd_p))) { + return m->makefile ? "${__cbm_curdir}" : MK_UNKNOWN; + } + if (cmd && mk_words(cmd, toplevel)) { + return "${__cbm_root}"; + } + return MK_UNKNOWN; + } + if (strcmp(fn, "abspath") == 0 || strcmp(fn, "realpath") == 0) { + const char *v = mk_expand(m, arg); + return v && !strchr(v, ' ') ? v : MK_UNKNOWN; + } + return MK_UNKNOWN; +} + +/* Make's expansion of `text`: references replaced, $$ -> $, what is not + * known -> a shell variable no line sets. */ +static const char *mk_expand(mk_t *m, const char *text) { + sh_str_t b = {0}; + size_t n = strlen(text); + size_t i = 0; + while (i < n && !b.bad) { + if (text[i] != '$') { + size_t j = i; + while (j < n && text[j] != '$') { + j++; + } + sh_str_add(m->a, &b, text + i, j - i); + i = j; + continue; + } + if (i + SKIP_ONE >= n) { + sh_str_add(m->a, &b, "$", SKIP_ONE); + i++; + continue; + } + char c = text[i + SKIP_ONE]; + if (c == '$') { + sh_str_add(m->a, &b, "$", SKIP_ONE); + i += PAIR_LEN; + continue; + } + if (c != '(' && c != '{') { + sh_str_add(m->a, &b, MK_UNKNOWN, strlen(MK_UNKNOWN)); + i += PAIR_LEN; + continue; + } + char close = c == '(' ? ')' : '}'; + int depth = 0; + size_t j = i + SKIP_ONE; + for (; j < n; j++) { + if (text[j] == c) { + depth++; + } else if (text[j] == close && --depth == 0) { + break; + } + } + if (j >= n) { + sh_str_add(m->a, &b, MK_UNKNOWN, strlen(MK_UNKNOWN)); + break; + } + const char *ref = cbm_arena_strndup(m->a, text + i + PAIR_LEN, j - i - PAIR_LEN); + const char *v = ref ? mk_reference(m, ref) : MK_UNKNOWN; + sh_str_add(m->a, &b, v, strlen(v)); + i = j + SKIP_ONE; + } + return sh_str_done(&b); +} + +/* A variable's value as make reads it: each backslash-newline, with the + * blanks around it, is one space (a value expanded into a recipe line adds + * no lines). */ +static const char *mk_fold(CBMArena *a, const char *p, size_t n) { + char *out = cbm_arena_alloc(a, n + SKIP_ONE); + if (!out) { + return NULL; + } + size_t o = 0; + for (size_t i = 0; i < n; i++) { + size_t k = i + SKIP_ONE; + k += k < n && p[k] == '\r'; + if (p[i] != '\\' || k >= n || p[k] != '\n') { + out[o++] = p[i]; + continue; + } + while (o > 0 && (out[o - SKIP_ONE] == ' ' || out[o - SKIP_ONE] == '\t')) { + o--; + } + out[o++] = ' '; + i = k; + while (i + SKIP_ONE < n && (p[i + SKIP_ONE] == ' ' || p[i + SKIP_ONE] == '\t')) { + i++; + } + } + return mk_trim(a, out, o); +} + +static void mk_assign(mk_t *m, TSNode n, bool cond) { + TSNode name = ts_node_child_by_field_name(n, TS_FIELD("name")); + TSNode value = ts_node_child_by_field_name(n, TS_FIELD("value")); + if (ts_node_is_null(name)) { + return; + } + uint32_t nb = ts_node_start_byte(name); + const char *nm = mk_trim(m->a, m->src + nb, ts_node_end_byte(name) - nb); + const char *raw = ""; + if (!ts_node_is_null(value)) { + uint32_t vb = ts_node_start_byte(value); + raw = mk_fold(m->a, m->src + vb, ts_node_end_byte(value) - vb); + } + const char *op = "="; + for (uint32_t i = 0; i < ts_node_child_count(n); i++) { + TSNode c = ts_node_child(n, i); + if (!ts_node_is_named(c) && strchr(ts_node_type(c), '=')) { + op = ts_node_type(c); + } + } + if (!nm || !raw) { + return; + } + mk_var_t *old = cbm_ht_get(m->vars, nm); + if (strcmp(op, "?=") == 0 && old) { + return; + } + mk_var_t *v = cbm_arena_alloc(m->a, sizeof(*v)); + if (!v) { + return; + } + *v = (mk_var_t){raw, NULL, !cond && strcmp(op, "!=") != 0, false}; + if (strcmp(op, "+=") == 0 && old) { + v->known = v->known && old->known; + v->raw = cbm_arena_sprintf(m->a, "%s %s", old->raw, raw); + } + cbm_ht_set(m->vars, nm, v); +} + +/* Variables and recipe lines of a makefile; `cond`: under a conditional. */ +/* The recipe lines in the bytes of a recipe_line node, split as make splits + * them: tree-sitter-make can let one node run over several lines (a quote in + * `"Don't ..."` does it). */ +static void mk_split_lines(mk_t *m, TSNode node) { + const char *src = m->src; + uint32_t p = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + uint32_t row = ts_node_start_point(node).row; + while (p < end) { + uint32_t b = p; + uint32_t brow = row; + uint32_t q = p; + for (;;) { + while (q < end && src[q] != '\n') { + q++; + } + uint32_t k = (q > p && src[q - SKIP_ONE] == '\r') ? q - SKIP_ONE : q; + bool cont = k > p && src[k - SKIP_ONE] == '\\'; + if (!cont || q >= end) { + break; + } + q++; + row++; + p = q; + } + uint32_t e = q; + p = q < end ? q + SKIP_ONE : q; + row++; + while (e > b && src[e - SKIP_ONE] == '\r') { + e--; + } + while (b < e && src[b] && strchr(" \t@-+", src[b])) { + b++; + } + if (b < e && src[b] != '#' && + sh_grow((void **)&m->lines, &m->cap_lines, m->nlines, sizeof(*m->lines))) { + m->lines[m->nlines++] = (mk_line_t){b, e, brow}; + } + } +} + +static int mk_cmp_line(const void *a, const void *b) { + uint32_t x = ((const mk_line_t *)a)->b; + uint32_t y = ((const mk_line_t *)b)->b; + return (x > y) - (x < y); +} + +static void mk_collect(mk_t *m, TSNode n, bool cond); +static void mk_collect_error(mk_t *m, TSNode err, bool cond); + +static void mk_collect_one(mk_t *m, TSNode c, bool cond) { + const char *t = ts_node_type(c); + if (strcmp(t, "variable_assignment") == 0) { + mk_assign(m, c, cond); + } else if (strcmp(t, "recipe_line") == 0) { + mk_split_lines(m, c); + } else if (strcmp(t, "rule") == 0 || strcmp(t, "recipe") == 0) { + mk_collect(m, c, cond); + } else if (strcmp(t, "ERROR") == 0) { + mk_collect_error(m, c, cond); + } else if (strcmp(t, "define_directive") == 0) { + TSNode name = ts_node_child_by_field_name(c, TS_FIELD("name")); + if (!ts_node_is_null(name)) { + uint32_t nb = ts_node_start_byte(name); + const char *nm = mk_trim(m->a, m->src + nb, ts_node_end_byte(name) - nb); + mk_var_t *v = cbm_arena_alloc(m->a, sizeof(*v)); + if (nm && v) { + *v = (mk_var_t){"", NULL, false, false}; + cbm_ht_set(m->vars, nm, v); + } + } + } else if (strstr(t, "conditional") || strstr(t, "if") == t || + strcmp(t, "else_directive") == 0 || strcmp(t, "elsif_directive") == 0) { + mk_collect(m, c, true); + } else if (strcmp(t, "include_directive") == 0) { + TSNode names = ts_node_child_by_field_name(c, TS_FIELD("filenames")); + if (m->on_include && !ts_node_is_null(names)) { + uint32_t nb = ts_node_start_byte(names); + const char *raw = mk_trim(m->a, m->src + nb, ts_node_end_byte(names) - nb); + const char *v = raw ? mk_expand(m, raw) : NULL; + if (v) { + m->on_include(m, v); + } + } + } else if (strstr(t, "directive")) { + mk_collect(m, c, cond); + } +} + +static void mk_collect(mk_t *m, TSNode n, bool cond) { + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + mk_collect_one(m, ts_node_named_child(n, i), cond); + } +} + +/* +1 for a line opening a conditional or a define, -1 for its end, else 0. */ +static int mk_directive_delta(const char *p, const char *end) { + static const struct { + const char *kw; + int delta; + } kws[] = {{"ifeq", SKIP_ONE}, {"ifneq", SKIP_ONE}, {"ifdef", SKIP_ONE}, {"ifndef", SKIP_ONE}, + {"define", SKIP_ONE}, {"endif", -1}, {"endef", -1}}; + while (p < end && (*p == ' ' || *p == '\t')) { + p++; + } + for (size_t i = 0; i < sizeof(kws) / sizeof(kws[0]); i++) { + size_t kl = strlen(kws[i].kw); + if ((size_t)(end - p) >= kl && strncmp(p, kws[i].kw, kl) == 0 && + (p + kl == end || strchr(" \t(\r", p[kl]))) { + return kws[i].delta; + } + } + return 0; +} + +/* A region tree-sitter-make could not parse (an indented, nested ifeq does + * it) still holds rules and assignments; an assignment counts as certain + * only outside the conditionals its lines open. */ +static void mk_collect_error(mk_t *m, TSNode err, bool cond) { + uint32_t scan = ts_node_start_byte(err); + uint32_t end = ts_node_end_byte(err); + int depth = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(err); i++) { + TSNode c = ts_node_named_child(err, i); + uint32_t cb = ts_node_start_byte(c); + while (scan < cb && scan < end) { + uint32_t le = scan; + while (le < end && m->src[le] != '\n') { + le++; + } + depth += mk_directive_delta(m->src + scan, m->src + le); + depth = depth < 0 ? 0 : depth; + scan = le + SKIP_ONE; + } + mk_collect_one(m, c, cond || depth > 0); + } +} + +static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t len) { + if (!ts_parser_set_language(p->parser, cbm_ts_language(CBM_LANG_MAKEFILE))) { + return; + } + TSTree *tree = ts_parser_parse_string(p->parser, NULL, src, (uint32_t)len); + if (!tree) { + return; + } + CBMArena a; + cbm_arena_init(&a); + const char *base = sh_basename(rel); + mk_t m = {.a = &a, + .src = src, + .vars = cbm_ht_create(0), + .makefile = strcmp(base, "Makefile") == 0 || strcmp(base, "GNUmakefile") == 0 || + strcmp(base, "makefile") == 0}; + if (m.vars) { + mk_collect(&m, ts_tree_root_node(tree), false); + } + if (m.nlines > SKIP_ONE) { + qsort(m.lines, (size_t)m.nlines, sizeof(*m.lines), mk_cmp_line); + } + for (int r = 0; m.vars && r < m.nlines; r++) { + if (r > 0 && m.lines[r].b == m.lines[r - SKIP_ONE].b) { + continue; /* nested nodes covered it already */ + } + uint32_t b = m.lines[r].b; + uint32_t e = m.lines[r].e; + const char *raw = cbm_arena_strndup(&a, src + b, e - b); + const char *text = raw ? mk_expand(&m, raw) : NULL; + if (!text || (!strchr(text, '/') && !strstr(text, "sh") && !strstr(text, "make") && + !sh_mentions_deploy_tool(text, strlen(text)))) { + continue; /* no deploy, program by path, shell or make */ + } + size_t tl = strlen(text); + char *buf = cbm_arena_alloc(&a, tl + CBM_SZ_16); + if (!buf) { + continue; + } + memcpy(buf, text, tl); + memset(buf + tl, 0, CBM_SZ_16); + sh_t s; + sh_state_init(&s, p, &a, rel); + if (s.env) { + s.line_base = (int)m.lines[r].row; + sh_val_t *root = cbm_arena_alloc(&a, sizeof(*root)); + sh_val_t *cur = cbm_arena_alloc(&a, sizeof(*cur)); + if (root && cur) { + *root = (sh_val_t){SH_ROOT, NULL, 0}; + *cur = (sh_val_t){sh_anchor(&a, s.sdir), NULL, 0}; + cbm_ht_set(s.env, "__cbm_root", root); + if (m.makefile) { + cbm_ht_set(s.env, "__cbm_curdir", cur); + } + } + s.cwd = m.makefile ? (sh_cwd_t){SH_AT, s.sdir} : (sh_cwd_t){SH_FREE, NULL}; + sh_run(&s, p->parser, buf, tl); + } + sh_state_free(&s); + } + if (m.vars) { + cbm_ht_free(m.vars); + } + cbm_free(CBM_MEM_CLASS_OTHER, m.lines); + cbm_arena_destroy(&a); + ts_tree_delete(tree); +} + +/* ── make targets: the rules a makefile reads (P3.5b) ──────────────── */ + +typedef struct { + sh_pass_t *p; + cbm_pipeline_ctx_t *ctx; + CBMArena *a; + CBMHashTable *seen; /* the files read */ + CBMHashTable *vars; /* shared by them, as make shares them */ + const char **order; /* the files, in reading order */ + int n; + int cap; + const char *mkdir; /* make's directory ("" = the root) */ +} mk_closure_t; + +static void mk_closure_file(mk_closure_t *cl, const char *rel); + +/* An include directive: each name -- from make's directory ($(CURDIR) is + * it), or the root ($(shell git rev-parse --show-toplevel)) -- is read in + * turn, recursively, where the directive stands. A name not fully known is + * skipped. */ +static void mk_closure_include(mk_t *m, const char *names) { + static const char curdir[] = "${__cbm_curdir}"; + static const char root[] = "${__cbm_root}"; + mk_closure_t *cl = m->ud; + const char *p = names; + for (;;) { + while (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') { + p++; + } + size_t wl = strcspn(p, " \t\r\n"); + if (!wl) { + return; + } + const char *w = cbm_arena_strndup(cl->a, p, wl); + p += wl; + const char *base = cl->mkdir; + if (w && strncmp(w, curdir, sizeof(curdir) - SKIP_ONE) == 0) { + w += sizeof(curdir) - SKIP_ONE; + } else if (w && strncmp(w, root, sizeof(root) - SKIP_ONE) == 0) { + w += sizeof(root) - SKIP_ONE; + base = ""; + } else if (w && w[0] == '/') { + continue; + } + const char *path = w && !strchr(w, '$') ? sh_join(cl->a, base, w) : NULL; + if (path) { + mk_closure_file(cl, path); + } + } +} + +static void mk_closure_file(mk_closure_t *cl, const char *rel) { + if (cbm_ht_get(cl->seen, rel) || + !sh_grow((void **)&cl->order, &cl->cap, cl->n, sizeof(*cl->order))) { + return; + } + cbm_ht_set(cl->seen, rel, (void *)rel); + cl->order[cl->n++] = rel; + const char *abs = cbm_arena_sprintf(cl->a, "%s/%s", cl->ctx->repo_path, rel); + size_t len = 0; + char *src = abs ? sh_read(abs, &len) : NULL; + if (!src) { + return; + } + TSTree *tree = NULL; + if (ts_parser_set_language(cl->p->parser, cbm_ts_language(CBM_LANG_MAKEFILE))) { + tree = ts_parser_parse_string(cl->p->parser, NULL, src, (uint32_t)len); + } + if (tree) { + mk_t m = {.a = cl->a, + .src = src, + .vars = cl->vars, + .makefile = true, + .on_include = mk_closure_include, + .ud = cl}; + mk_collect(&m, ts_tree_root_node(tree), false); + cbm_free(CBM_MEM_CLASS_OTHER, m.lines); + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); +} + +static const char *mk_rule_key(CBMArena *a, const char *file, const char *name) { + return cbm_arena_sprintf(a, "%s\x1f%s", file, name); +} + +/* Each make hit's target: the rule node of that name in the makefile, else + * in the files it includes, in reading order (the first rule of a file). + * Read here, when linking, so an unchanged makefile is read too. */ +static void sh_make_resolve(sh_pass_t *p, cbm_pipeline_ctx_t *ctx, sh_hits_t *h) { + CBMArena a; + cbm_arena_init(&a); + CBMHashTable *closures = cbm_ht_create(0); /* makefile -> mk_closure_t */ + CBMHashTable *want = cbm_ht_create(0); /* every file a closure reads */ + CBMHashTable *rules = cbm_ht_create(0); /* file \x1f name -> rule node */ + for (int i = 0; closures && want && i < h->count; i++) { + const sh_hit_t *x = &h->items[i]; + if (!x->make || cbm_ht_get(closures, x->path)) { + continue; + } + mk_closure_t *cl = cbm_arena_alloc(&a, sizeof(*cl)); + if (!cl) { + continue; + } + const char *dir = sh_dirname(&a, x->path); + *cl = (mk_closure_t){.p = p, + .ctx = ctx, + .a = &a, + .seen = cbm_ht_create(0), + .vars = cbm_ht_create(0), + .mkdir = dir && strcmp(dir, ".") != 0 ? dir : ""}; + if (cl->seen && cl->vars) { + mk_closure_file(cl, x->path); + } + for (int k = 0; k < cl->n; k++) { + cbm_ht_set(want, cl->order[k], (void *)cl->order[k]); + } + cbm_ht_set(closures, x->path, cl); + } + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (rules && want && cbm_ht_count(want) > 0 && + cbm_gbuf_find_by_label(ctx->gbuf, "Function", &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + if (!fp || !nodes[i]->name || !cbm_ht_get(want, fp)) { + continue; + } + const char *key = mk_rule_key(&a, fp, nodes[i]->name); + const cbm_gbuf_node_t *old = key ? cbm_ht_get(rules, key) : NULL; + if (key && (!old || nodes[i]->start_line < old->start_line)) { + cbm_ht_set(rules, key, (void *)nodes[i]); + } + } + } + for (int i = 0; rules && closures && i < h->count; i++) { + sh_hit_t *x = &h->items[i]; + const mk_closure_t *cl = x->make ? cbm_ht_get(closures, x->path) : NULL; + for (int k = 0; cl && k < cl->n && x->tgt <= 0; k++) { + atomic_fetch_add_explicit(&g_sh_candidates, SKIP_ONE, memory_order_relaxed); + const char *key = mk_rule_key(&a, cl->order[k], x->ext); + const cbm_gbuf_node_t *rule = key ? cbm_ht_get(rules, key) : NULL; + x->tgt = rule ? rule->id : 0; + } + } + for (int i = 0; closures && i < h->count; i++) { + mk_closure_t *cl = h->items[i].make ? cbm_ht_get(closures, h->items[i].path) : NULL; + if (cl) { + cbm_ht_delete(closures, h->items[i].path); + cbm_ht_free(cl->seen); + cbm_ht_free(cl->vars); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)cl->order); + } + } + if (closures) { + cbm_ht_free(closures); + } + if (want) { + cbm_ht_free(want); + } + if (rules) { + cbm_ht_free(rules); + } + cbm_arena_destroy(&a); +} + +/* ── Edges ─────────────────────────────────────────────────────────── */ + +typedef struct { + const char *file; + const cbm_gbuf_node_t *node; +} sh_span_t; + +static int sh_cmp_span(const void *a, const void *b) { + const sh_span_t *x = a; + const sh_span_t *y = b; + return strcmp(x->file, y->file); +} + +typedef struct { + const cbm_gbuf_node_t **items; + int count; + int cap; +} sh_srcs_t; + +static int sh_cmp_node_qn(const void *a, const void *b) { + const cbm_gbuf_node_t *x = *(const cbm_gbuf_node_t *const *)a; + const cbm_gbuf_node_t *y = *(const cbm_gbuf_node_t *const *)b; + return strcmp(x->qualified_name ? x->qualified_name : "", + y->qualified_name ? y->qualified_name : ""); +} + +/* The innermost Function around `line` in `file`, else its Module, else its + * File node. Into `twins` (when given): every Function of exactly that + * span, in QN order -- the targets of one Make rule share it, and the + * rule's recipe runs for each of them. */ +static int64_t sh_source(cbm_pipeline_ctx_t *ctx, const sh_span_t *spans, int n, + CBMHashTable *modules, const char *file, int line, sh_srcs_t *twins) { + int lo = 0; + int hi = n; + while (lo < hi) { + int mid = lo + (hi - lo) / PAIR_LEN; + if (strcmp(spans[mid].file, file) < 0) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + const cbm_gbuf_node_t *best = NULL; + for (int i = lo; i < n && strcmp(spans[i].file, file) == 0; i++) { + const cbm_gbuf_node_t *f = spans[i].node; + if (f->start_line > line || f->end_line < line) { + continue; + } + int w = f->end_line - f->start_line; + int bw = best ? best->end_line - best->start_line : 0; + if (!best || w < bw || + (w == bw && strcmp(f->qualified_name ? f->qualified_name : "", + best->qualified_name ? best->qualified_name : "") < 0)) { + best = f; + } + } + if (best) { + for (int i = lo; twins && i < n && strcmp(spans[i].file, file) == 0; i++) { + const cbm_gbuf_node_t *f = spans[i].node; + if (f->start_line == best->start_line && f->end_line == best->end_line && + sh_grow((void **)&twins->items, &twins->cap, twins->count, sizeof(*twins->items))) { + twins->items[twins->count++] = f; + } + } + if (twins && twins->count > SKIP_ONE) { + qsort((void *)twins->items, (size_t)twins->count, sizeof(*twins->items), + sh_cmp_node_qn); + } + return best->id; + } + int64_t mod = (int64_t)(intptr_t)cbm_ht_get(modules, file); + if (mod > 0) { + return mod; + } + const cbm_gbuf_node_t *fn = cbm_pipeline_file_node(ctx->gbuf, ctx->project_name, file); + return fn ? fn->id : 0; +} + +/* The shared external Chart of `name` (the one chart dependencies use). */ +static int64_t sh_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, const char *file) { + char qn[CBM_SZ_512]; + snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", ctx->project_name, name); + const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(ctx->gbuf, qn); + if (known) { + return known->id; + } + return cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, + "{\"source\":\"helm\",\"external\":true}"); +} + +/* A hit's CI context as JSON members (`,"ci":..,"job":..`); "" for none. */ +static void sh_ci_props(const sh_hit_t *x, char *out, size_t sz) { + out[0] = '\0'; + if (!x->ci) { + return; + } + char job[CBM_SZ_512]; + cbm_json_escape(job, (int)sizeof(job), x->job ? x->job : ""); + if (x->job) { + snprintf(out, sz, ",\"ci\":\"%s\",\"job\":\"%s\"", x->ci, job); + } else { + snprintf(out, sz, ",\"ci\":\"%s\"", x->ci); + } +} + +/* SPAWNS to the rule `x->tgt` of a make target, the script `x->tgt` (its + * Module, else its File) or the Process of `x->ext`, as pass_spawns.c draws + * it. */ +static int sh_link_spawn(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x, + CBMHashTable *modules) { + char program[CBM_SZ_128]; + cbm_json_escape(program, (int)sizeof(program), x->make ? "make" : x->ext); + char path[CBM_SZ_2K]; + path[0] = '\0'; + int64_t tgt = 0; + if (x->make) { + char esc[CBM_SZ_1K]; + char target[CBM_SZ_512]; + cbm_json_escape(esc, (int)sizeof(esc), x->path); + cbm_json_escape(target, (int)sizeof(target), x->ext); + snprintf(path, sizeof(path), ",\"path\":\"%s\",\"target\":\"%s\"", esc, target); + tgt = x->tgt; + } else if (x->tgt > 0) { + int64_t mod = (int64_t)(intptr_t)cbm_ht_get(modules, x->path); + tgt = mod > 0 ? mod : x->tgt; + char esc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), x->path); + snprintf(path, sizeof(path), ",\"path\":\"%s\"", esc); + } else { + char qn[CBM_SZ_256]; + snprintf(qn, sizeof(qn), "__process__%s", x->ext); + char node_props[CBM_SZ_256]; + snprintf(node_props, sizeof(node_props), "{\"program\":\"%s\"}", program); + tgt = cbm_gbuf_upsert_node(ctx->gbuf, "Process", x->ext, qn, "", 0, 0, node_props); + } + if (tgt <= 0 || tgt == src) { + return 0; + } + char ci[CBM_SZ_1K]; + sh_ci_props(x, ci, sizeof(ci)); + char props[CBM_SZ_4K]; + snprintf(props, sizeof(props), + "{\"api\":\"%s\",\"program\":\"%s\"%s%s,\"line\":%d,\"confidence\":1.00}", x->via, + program, path, ci, x->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "SPAWNS", props); + return SKIP_ONE; +} + +/* The edge hit `x` draws from source `src`; 1 when drawn. */ +/* An image a build command makes DEPENDS_ON {via:build} the Dockerfile + * it is built from (edges recomputed whole every run, as the image belongs + * to no file). */ +static int sh_link_build(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, CBMHashTable *modules) { + int64_t mod = (int64_t)(intptr_t)cbm_ht_get(modules, x->path); + int64_t img = mod > 0 ? cbm_pipeline_image_node(ctx, x->ext) : 0; + if (img <= 0) { + return 0; + } + char esc_file[CBM_SZ_1K]; + char esc_img[CBM_SZ_1K]; + char ci[CBM_SZ_1K]; + char props[CBM_SZ_4K]; + cbm_json_escape(esc_file, (int)sizeof(esc_file), x->file); + cbm_json_escape(esc_img, (int)sizeof(esc_img), x->ext); + sh_ci_props(x, ci, sizeof(ci)); + snprintf(props, sizeof(props), + "{\"via\":\"build\",\"tool\":\"docker-build\",\"file\":\"%s\",\"image\":\"%s\"%s," + "\"line\":%d}", + esc_file, esc_img, ci, x->line); + cbm_gbuf_insert_edge(ctx->gbuf, img, mod, "DEPENDS_ON", props); + return SKIP_ONE; +} + +static int sh_link_hit(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, int64_t src, + CBMHashTable *modules) { + if (x->spawn) { + return src > 0 ? sh_link_spawn(ctx, src, x, modules) : 0; + } + int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file); + if (src <= 0 || tgt <= 0 || src == tgt) { + return 0; + } + char esc[CBM_SZ_1K]; + char ci[CBM_SZ_1K]; + char props[CBM_SZ_4K]; + cbm_json_escape(esc, (int)sizeof(esc), x->path ? x->path : ""); + sh_ci_props(x, ci, sizeof(ci)); + snprintf(props, sizeof(props), "{\"via\":\"%s\",\"%s\":\"%s\"%s%s,\"line\":%d}", x->via, + x->chart ? "chart" : "path", esc, x->tgt > 0 ? "" : ",\"external\":true", ci, x->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "DEPLOYS", props); + return SKIP_ONE; +} + +static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) { + CBMHashTable *files = cbm_ht_create(0); + CBMHashTable *modules = cbm_ht_create(0); + if (!files || !modules) { + if (files) { + cbm_ht_free(files); + } + if (modules) { + cbm_ht_free(modules); + } + return 0; + } + for (int i = 0; i < h->count; i++) { + const sh_hit_t *x = &h->items[i]; + cbm_ht_set(files, x->file, (void *)x->file); + if ((x->spawn && x->tgt > 0) || x->build) { + cbm_ht_set(files, x->path, + (void *)x->path); /* a spawned script's / Dockerfile's Module */ + } + } + sh_span_t *spans = NULL; + int n = 0; + int cap = 0; + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(ctx->gbuf, "Function", &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + const char *key = fp ? cbm_ht_get_key(files, fp) : NULL; + if (key && sh_grow((void **)&spans, &cap, n, sizeof(*spans))) { + spans[n++] = (sh_span_t){key, nodes[i]}; + } + } + } + if (n > SKIP_ONE) { + qsort(spans, (size_t)n, sizeof(*spans), sh_cmp_span); + } + if (cbm_gbuf_find_by_label(ctx->gbuf, "Module", &nodes, &count) == 0) { + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + const char *key = fp ? cbm_ht_get_key(files, fp) : NULL; + if (key && !cbm_ht_get(modules, key)) { + cbm_ht_set(modules, key, (void *)(intptr_t)nodes[i]->id); + } + } + } + int edges = 0; + sh_srcs_t twins = {0}; + for (int i = 0; i < h->count; i++) { + const sh_hit_t *x = &h->items[i]; + if (x->build) { + edges += sh_link_build(ctx, x, modules); + continue; + } + twins.count = 0; + int64_t one = sh_source(ctx, spans, n, modules, x->file, x->line, &twins); + for (int k = 0; k < (twins.count ? twins.count : SKIP_ONE); k++) { + edges += sh_link_hit(ctx, x, twins.count ? twins.items[k]->id : one, modules); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)twins.items); + cbm_free(CBM_MEM_CLASS_OTHER, spans); + cbm_ht_free(files); + cbm_ht_free(modules); + return edges; +} + +/* ── CI configurations (pass_ci.c) ─────────────────────────────────── */ + +typedef struct { + sh_pass_t *p; + const char *rel; + CBMArena *a; +} sh_ci_t; + +static void sh_ci_var(sh_t *s, const char *name, const char *value) { + sh_val_t *v = value ? cbm_arena_alloc(s->a, sizeof(*v)) : NULL; + if (v) { + *v = (sh_val_t){value, NULL, 0}; + } + sh_set(s, name, v ? v : &SH_UNKNOWN_VAL, false); +} + +/* A command line CI runs: evaluated from its directory with its variables; + * $0 is unknown. */ +static void sh_ci_cmd(const cbm_ci_cmd_t *cmd, void *ud) { + sh_ci_t *x = ud; + sh_t s; + sh_state_init(&s, x->p, x->a, x->rel); + if (s.env) { + s.line_base = cmd->row; + s.ci = cmd->ci; + s.job = cmd->job; + s.exec = cmd->exec; + const char *wd = cmd->wd ? sh_join(x->a, "", cmd->wd) : NULL; + s.cwd = wd ? (sh_cwd_t){SH_AT, wd} : (sh_cwd_t){SH_LOST, NULL}; + if (strcmp(cmd->ci, "github-actions") == 0) { + sh_ci_var(&s, "GITHUB_WORKSPACE", SH_ROOT); + if (cmd->action_dir) { + sh_ci_var(&s, "GITHUB_ACTION_PATH", sh_anchor(x->a, cmd->action_dir)); + } + } + for (int i = 0; i < cmd->nenv; i++) { + sh_ci_var(&s, cmd->env[PAIR_LEN * i], cmd->env[PAIR_LEN * i + SKIP_ONE]); + } + sh_run(&s, x->p->parser, cmd->text, cmd->len); + } + sh_state_free(&s); +} + +static void sh_ci_file(sh_pass_t *p, const char *rel, const char *src, size_t len) { + CBMArena a; + cbm_arena_init(&a); + sh_ci_t x = {.p = p, .rel = rel, .a = &a}; + cbm_ci_commands(&a, rel, src, len, sh_ci_cmd, &x); + cbm_arena_destroy(&a); +} + +/* Prow job configs: the files whose top-level keys (YAML Variables) include + * presubmits / postsubmits / periodics. */ +static CBMHashTable *sh_prow_files(cbm_pipeline_ctx_t *ctx) { + static const char *const keys[] = {"presubmits", "postsubmits", "periodics", NULL}; + CBMHashTable *out = NULL; + for (int k = 0; keys[k]; k++) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_name(ctx->gbuf, keys[k], &nodes, &n) != 0) { + continue; + } + for (int i = 0; i < n; i++) { + const char *fp = nodes[i]->file_path; + if (!fp || !sh_eq(nodes[i]->label, "Variable")) { + continue; + } + out = out ? out : cbm_ht_create(0); + if (out) { + cbm_ht_set(out, fp, (void *)fp); + } + } + } + return out; +} + +/* One script, Makefile or CI configuration `rel` (on disk at `path`). */ +static void sh_pass_file(sh_pass_t *p, cbm_pipeline_ctx_t *ctx, CBMHashTable *prow, const char *rel, + const char *path, CBMLanguage lang) { + bool ci = cbm_ci_path(rel) || (prow && cbm_ht_get(prow, rel)); + if (lang != CBM_LANG_BASH && lang != CBM_LANG_MAKEFILE && !ci) { + return; + } + size_t len = 0; + char *src = sh_read(path, &len); + if (!src) { + return; + } + if (!p->ix_ready) { + p->ix_ready = true; + p->ix_ok = sh_index_build(p->ix, ctx); + p->parser = ts_parser_new(); + } + if (p->ix_ok && p->parser) { + if (ci) { + sh_ci_file(p, rel, src, len); + } else if (lang == CBM_LANG_BASH) { + sh_script(p, rel, src, len); + } else { + sh_makefile(p, rel, src, len); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, src); +} + +/* Mark the File node of each file running a container build + * {"declares_build":true} -- resolved or not: an incremental run reads these + * files again, as the image build edges are recomputed whole + * (pipeline_incremental.c). The mark lives and dies with its file's node. */ +static void sh_mark_declarers(cbm_pipeline_ctx_t *ctx, CBMHashTable *declarers) { + static const char MARK[] = "\"declares_build\":true"; + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_ht_count(declarers) == 0 || + cbm_gbuf_find_by_label(ctx->gbuf, "File", &nodes, &n) != 0) { + return; + } + for (int i = 0; i < n; i++) { + cbm_gbuf_node_t *f = (cbm_gbuf_node_t *)nodes[i]; + const char *props = f->properties_json ? f->properties_json : "{}"; + if (!f->file_path || !cbm_ht_get(declarers, f->file_path) || strstr(props, MARK)) { + continue; + } + size_t pl = strlen(props); + char buf[CBM_SZ_4K]; + bool empty = pl < PAIR_LEN || strcmp(props, "{}") == 0; + if (props[pl - SKIP_ONE] != '}' || pl + sizeof(MARK) + PAIR_LEN > sizeof(buf)) { + continue; + } + snprintf(buf, sizeof(buf), "%.*s%s%s}", (int)(pl - SKIP_ONE), empty ? "{" : props, + empty ? "" : ",", MARK); + (void)cbm_gbuf_node_set_properties_json(f, buf); + } +} + +void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count) { + sh_index_t ix = {0}; + sh_hits_t hits = {0}; + cbm_arena_init(&hits.arena); + hits.declarers = cbm_ht_create(0); + sh_pass_t p = {.ix = &ix, .hits = &hits, .parser = NULL}; + CBMHashTable *prow = sh_prow_files(ctx); + CBMHashTable *seen = ctx->build_decl_count > 0 ? cbm_ht_create(0) : NULL; + for (int i = 0; i < file_count; i++) { + if (seen) { + cbm_ht_set(seen, files[i].rel_path, (void *)files[i].rel_path); + } + sh_pass_file(&p, ctx, prow, files[i].rel_path, files[i].path, files[i].language); + } + /* An incremental run recomputes every image's build edges: the unchanged + * files that declared builds before are read again (pipeline_incremental.c). */ + for (int i = 0; seen && ctx->repo_path && i < ctx->build_decl_count; i++) { + const char *rel = ctx->build_decl_files[i]; + if (!rel || cbm_ht_get(seen, rel)) { + continue; + } + cbm_ht_set(seen, rel, (void *)rel); + const char *slash = strrchr(rel, '/'); + CBMLanguage lang = cbm_language_for_filename(slash ? slash + SKIP_ONE : rel); + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, rel); + sh_pass_file(&p, ctx, prow, rel, path, + lang == CBM_LANG_MAKEFILE ? CBM_LANG_MAKEFILE : CBM_LANG_BASH); + } + if (seen) { + cbm_ht_free(seen); + } + if (prow) { + cbm_ht_free(prow); + } + if (hits.count > 0 && p.ix_ok && p.parser) { + sh_make_resolve(&p, ctx, &hits); + } + int edges = hits.count > 0 ? sh_link(ctx, &hits) : 0; + if (hits.declarers) { + sh_mark_declarers(ctx, hits.declarers); + cbm_ht_free(hits.declarers); + } + if (p.parser) { + ts_parser_delete(p.parser); + } + cbm_free(CBM_MEM_CLASS_OTHER, hits.items); + cbm_arena_destroy(&hits.arena); + if (p.ix_ready) { + sh_index_free(&ix); + } + if (edges > 0) { + char buf[CBM_SZ_32]; + snprintf(buf, sizeof(buf), "%d", edges); + cbm_log_info("pass.shell.deploys", "edges", buf); + } +} diff --git a/src/pipeline/pass_terraform.c b/src/pipeline/pass_terraform.c new file mode 100644 index 000000000..309f6c99c --- /dev/null +++ b/src/pipeline/pass_terraform.c @@ -0,0 +1,449 @@ +/* + * pass_terraform.c — the Terraform / OpenTofu infrastructure layer. + * + * PHASE 3 P3.4: a `resource "helm_release"` DEPLOYS its chart -- an edge from + * the resource block (its node `resource.helm_release.`) to existing + * nodes only: + * chart = "./charts/x" / "${path.module}/charts/x" the Chart of that + * directory, relative to + * the module's directory + * repository = "", chart = "" the shared external + * chart = "oci://registry/.../" Chart of that name + * Values are evaluated from string literals and ${path.module}; any other + * expression is unknown, and an unknown chart is no edge. Run at the end of + * the K8s pass, over the same files. + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/hash_table.h" +#include "foundation/limits.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "cbm.h" +#include "helpers.h" +#include "lang_specs.h" + +#include +#include +#include +#include +#include + +/* Blocks a helm_release lookup examined (complexity guard counter). */ +_Atomic uint64_t g_tf_candidates = 0; + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMArena *arena; /* lookup QNs */ + CBMHashTable *charts; /* Chart.yaml path -> local Chart node id */ + const char *src; + const char *file; + char mdir[CBM_SZ_1K]; /* the module's directory */ + int edges; +} tf_t; + +static bool tf_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +static const char *tf_basename(const char *p) { + const char *s = strrchr(p, '/'); + return s ? s + SKIP_ONE : p; +} + +/* Append [p, p+n) to out (size sz); false when it does not fit. */ +static bool tf_add(char *out, size_t sz, size_t *len, const char *p, size_t n) { + if (*len + n + SKIP_ONE > sz) { + return false; + } + memcpy(out + *len, p, n); + *len += n; + out[*len] = '\0'; + return true; +} + +/* A string value: a literal, or a template of literals and ${path.module}. + * false when any part is something else. */ +static bool tf_string(const tf_t *t, TSNode n, char *out, size_t sz) { + size_t len = 0; + out[0] = '\0'; + while (tf_type(n, "expression") || tf_type(n, "literal_value") || tf_type(n, "template_expr") || + tf_type(n, "string_lit") || tf_type(n, "quoted_template")) { + if (tf_type(n, "string_lit") || tf_type(n, "quoted_template")) { + break; + } + if (ts_node_named_child_count(n) != SKIP_ONE) { + return false; + } + n = ts_node_named_child(n, 0); + } + if (!tf_type(n, "string_lit") && !tf_type(n, "quoted_template")) { + return false; + } + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + uint32_t b = ts_node_start_byte(c); + uint32_t e = ts_node_end_byte(c); + if (tf_type(c, "quoted_template_start") || tf_type(c, "quoted_template_end")) { + continue; + } + if (tf_type(c, "template_literal")) { + if (!tf_add(out, sz, &len, t->src + b, e - b)) { + return false; + } + continue; + } + if (!tf_type(c, "template_interpolation")) { + return false; + } + /* ${path.module} only */ + TSNode expr = {0}; + for (uint32_t j = 0; j < ts_node_named_child_count(c); j++) { + if (tf_type(ts_node_named_child(c, j), "expression")) { + expr = ts_node_named_child(c, j); + } + } + if (ts_node_is_null(expr)) { + return false; + } + uint32_t xb = ts_node_start_byte(expr); + uint32_t xe = ts_node_end_byte(expr); + static const char PATH_MODULE[] = "path.module"; + if (xe - xb != sizeof(PATH_MODULE) - SKIP_ONE || + strncmp(t->src + xb, PATH_MODULE, xe - xb) != 0) { + return false; + } + /* an absolute marker: the module directory, from the repo root */ + if (!tf_add(out, sz, &len, "\x01", SKIP_ONE) || + !tf_add(out, sz, &len, t->mdir, strlen(t->mdir))) { + return false; + } + } + return true; +} + +/* A path inside the module directory: "./a/../b" -> "/b"; NULL past + * the repository root. A value starting with the \x01 marker is already + * from the root. */ +static bool tf_repo_path(const char *mdir, const char *v, char *out, size_t sz) { + bool anchored = v[0] == '\x01'; + const char *rel = anchored ? v + SKIP_ONE : v; + char buf[CBM_SZ_2K]; + if (anchored) { + snprintf(buf, sizeof(buf), "%s", rel); + } else { + snprintf(buf, sizeof(buf), "%s%s%s", mdir, mdir[0] ? "/" : "", rel); + } + size_t n = 0; + out[0] = '\0'; + for (const char *p = buf; *p;) { + while (*p == '/') { + p++; + } + size_t sl = strcspn(p, "/"); + if (sl == 0) { + break; + } + if (sl == SKIP_ONE && p[0] == '.') { + p += sl; + continue; + } + if (sl == PAIR_LEN && p[0] == '.' && p[SKIP_ONE] == '.') { + if (n == 0) { + return false; + } + while (n > 0 && out[n - SKIP_ONE] != '/') { + n--; + } + if (n > 0) { + n--; + } + out[n] = '\0'; + p += sl; + continue; + } + if (n + sl + PAIR_LEN > sz) { + return false; + } + if (n) { + out[n++] = '/'; + } + memcpy(out + n, p, sl); + n += sl; + out[n] = '\0'; + p += sl; + } + return true; +} + +/* Chart.yaml path -> its (local) Chart node. */ +static CBMHashTable *tf_chart_index(cbm_pipeline_ctx_t *ctx) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (!ix || cbm_gbuf_find_by_label(ctx->gbuf, "Chart", &nodes, &count) != 0) { + return ix; + } + for (int i = 0; i < count; i++) { + const char *fp = nodes[i]->file_path; + const char *qn = nodes[i]->qualified_name; + const char *base = fp ? tf_basename(fp) : ""; + if (fp && qn && !strstr(qn, ".__helm_dep__.") && + (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0)) { + cbm_ht_set(ix, fp, (void *)(intptr_t)nodes[i]->id); + } + } + return ix; +} + +/* The Chart node of repository chart directory `dir`, 0 when none. */ +static int64_t tf_local_chart(const tf_t *t, const char *dir) { + static const char *const names[] = {"Chart.yaml", "Chart.yml"}; + for (size_t k = 0; t->charts && k < sizeof(names) / sizeof(names[0]); k++) { + char want[CBM_SZ_2K]; + snprintf(want, sizeof(want), "%s%s%s", dir, dir[0] ? "/" : "", names[k]); + int64_t id = (int64_t)(intptr_t)cbm_ht_get(t->charts, want); + if (id > 0) { + return id; + } + } + return 0; +} + +/* The shared external Chart of `name` (the one chart dependencies use). */ +static int64_t tf_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, const char *file) { + char qn[CBM_SZ_512]; + snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", ctx->project_name, name); + const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(ctx->gbuf, qn); + if (known) { + return known->id; + } + return cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, + "{\"source\":\"helm\",\"external\":true}"); +} + +/* The node of block `resource.helm_release.` in the file, by the QN + * its definition got -- not among every block of that name: modules that + * share a release name (`this`) would make each lookup scan all of them. */ +static int64_t tf_block_node(const tf_t *t, const char *name) { + char full[CBM_SZ_512]; + snprintf(full, sizeof(full), "resource.helm_release.%s", name); + const char *qn = cbm_fqn_compute(t->arena, t->ctx->project_name, t->file, full); + const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(t->ctx->gbuf, qn) : NULL; + atomic_fetch_add_explicit(&g_tf_candidates, SKIP_ONE, memory_order_relaxed); + return n && n->file_path && strcmp(n->file_path, t->file) == 0 ? n->id : 0; +} + +static bool tf_plain_name(const char *s) { + if (!s[0]) { + return false; + } + for (const char *p = s; *p; p++) { + if (!((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') || + *p == '-' || *p == '_' || *p == '.')) { + return false; + } + } + return true; +} + +/* One `resource "helm_release" "" { ... }` block. */ +static void tf_helm_release(tf_t *t, TSNode block, const char *name) { + TSNode body = {0}; + for (uint32_t i = 0; i < ts_node_named_child_count(block); i++) { + if (tf_type(ts_node_named_child(block, i), "body")) { + body = ts_node_named_child(block, i); + } + } + if (ts_node_is_null(body)) { + return; + } + char chart[CBM_SZ_1K] = ""; + bool has_chart = false; + bool has_repo = false; /* a repository, even of unknown value: not a local chart */ + for (uint32_t i = 0; i < ts_node_named_child_count(body); i++) { + TSNode a = ts_node_named_child(body, i); + if (!tf_type(a, "attribute") || ts_node_named_child_count(a) < PAIR_LEN) { + continue; + } + TSNode key = ts_node_named_child(a, 0); + TSNode val = ts_node_named_child(a, SKIP_ONE); + uint32_t kb = ts_node_start_byte(key); + uint32_t ke = ts_node_end_byte(key); + size_t kl = ke - kb; + if (kl == strlen("chart") && strncmp(t->src + kb, "chart", kl) == 0) { + has_chart = tf_string(t, val, chart, sizeof(chart)); + } else if (kl == strlen("repository") && strncmp(t->src + kb, "repository", kl) == 0) { + has_repo = true; + } + } + if (!has_chart) { + return; + } + int64_t src = tf_block_node(t, name); + int64_t tgt = 0; + bool external = false; + char path[CBM_SZ_1K] = ""; + static const char OCI[] = "oci://"; + if (strncmp(chart, OCI, sizeof(OCI) - SKIP_ONE) == 0) { + const char *base = tf_basename(chart); + if (tf_plain_name(base)) { + snprintf(path, sizeof(path), "%s", base); + external = true; + } + } else if (has_repo) { + if (tf_plain_name(chart)) { + snprintf(path, sizeof(path), "%s", chart); + external = true; + } + } else if (tf_repo_path(t->mdir, chart, path, sizeof(path))) { + tgt = tf_local_chart(t, path); + } + if (external) { + tgt = tf_external_chart(t->ctx, path, t->file); + } + if (src <= 0 || tgt <= 0) { + return; + } + char esc[CBM_SZ_1K]; + char props[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), path); + snprintf(props, sizeof(props), "{\"via\":\"terraform\",\"chart\":\"%s\"%s,\"line\":%d}", esc, + external ? ",\"external\":true" : "", (int)ts_node_start_point(block).row + SKIP_ONE); + cbm_gbuf_insert_edge(t->ctx->gbuf, src, tgt, "DEPLOYS", props); + t->edges++; +} + +/* The text of a string_lit label: "helm_release" -> helm_release. */ +static bool tf_label(const tf_t *t, TSNode n, char *out, size_t sz) { + if (!tf_type(n, "string_lit")) { + return false; + } + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "template_literal")) { + uint32_t b = ts_node_start_byte(c); + snprintf(out, sz, "%.*s", (int)(ts_node_end_byte(c) - b), t->src + b); + return true; + } + } + return false; +} + +static void tf_file(tf_t *t, TSNode root) { + TSNode body = {0}; /* comments are children of the file too */ + for (uint32_t i = 0; i < ts_node_named_child_count(root) && ts_node_is_null(body); i++) { + if (tf_type(ts_node_named_child(root, i), "body")) { + body = ts_node_named_child(root, i); + } + } + for (uint32_t i = 0; !ts_node_is_null(body) && i < ts_node_named_child_count(body); i++) { + TSNode b = ts_node_named_child(body, i); + if (!tf_type(b, "block") || ts_node_named_child_count(b) < 3) { + continue; + } + TSNode kw = ts_node_named_child(b, 0); + uint32_t kb = ts_node_start_byte(kw); + uint32_t ke = ts_node_end_byte(kw); + char type[CBM_SZ_128]; + char name[CBM_SZ_256]; + if (ke - kb != strlen("resource") || strncmp(t->src + kb, "resource", ke - kb) != 0 || + !tf_label(t, ts_node_named_child(b, SKIP_ONE), type, sizeof(type)) || + !tf_label(t, ts_node_named_child(b, PAIR_LEN), name, sizeof(name))) { + continue; + } + if (strcmp(type, "helm_release") == 0) { + tf_helm_release(t, b, name); + } + } +} + +static char *tf_read(const char *path, size_t *len) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + enum { TF_TS_PAD = 16 }; /* tree-sitter's lexer reads past the end */ + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + TF_TS_PAD); + if (!buf) { + (void)fclose(f); + return NULL; + } + size_t got = fread(buf, SKIP_ONE, (size_t)size, f); + (void)fclose(f); + memset(buf + got, 0, TF_TS_PAD); + *len = got; + return buf; +} + +static bool tf_is_module_file(const char *rel) { + const char *dot = strrchr(tf_basename(rel), '.'); + return dot && (strcmp(dot, ".tf") == 0 || strcmp(dot, ".tofu") == 0); +} + +void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count) { + TSParser *parser = NULL; + CBMHashTable *charts = NULL; + CBMArena arena; + cbm_arena_init_lazy(&arena, CBM_SZ_4K); + int edges = 0; + for (int i = 0; i < file_count; i++) { + if (files[i].language != CBM_LANG_HCL || !tf_is_module_file(files[i].rel_path)) { + continue; + } + size_t len = 0; + char *src = tf_read(files[i].path, &len); + if (!src) { + continue; + } + if (strstr(src, "helm_release")) { + if (!parser) { + parser = ts_parser_new(); + charts = tf_chart_index(ctx); + } + TSTree *tree = NULL; + if (parser && ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_HCL))) { + tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len); + } + if (tree) { + tf_t t = {.ctx = ctx, + .arena = &arena, + .charts = charts, + .src = src, + .file = files[i].rel_path}; + const char *slash = strrchr(files[i].rel_path, '/'); + snprintf(t.mdir, sizeof(t.mdir), "%.*s", + slash ? (int)(slash - files[i].rel_path) : 0, files[i].rel_path); + tf_file(&t, ts_tree_root_node(tree)); + edges += t.edges; + ts_tree_delete(tree); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + if (parser) { + ts_parser_delete(parser); + } + if (charts) { + cbm_ht_free(charts); + } + cbm_arena_destroy(&arena); + if (edges > 0) { + char buf[CBM_SZ_32]; + snprintf(buf, sizeof(buf), "%d", edges); + cbm_log_info("pass.terraform.deploys", "edges", buf); + } +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 196349a18..f8f72916f 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1593,6 +1593,53 @@ static void *gh_compute_thread_fn(void *arg) { return NULL; } +/* ── Secret hygiene: config values that are secret material stay out ─── */ + +#define INFRA_SLEN(lit) (sizeof(lit) - SKIP_ONE) + +/* A file of values, not declarations: .tfvars assigns Terraform variables, + * .env the process environment -- where deployments keep their secrets. */ +static bool infra_values_file(const char *fp) { + const char *slash = strrchr(fp, '/'); + const char *base = slash ? slash + SKIP_ONE : fp; + size_t n = strlen(base); + static const char *const suffixes[] = {".tfvars", ".tfvars.json", ".env"}; + for (size_t i = 0; i < sizeof(suffixes) / sizeof(suffixes[0]); i++) { + size_t s = strlen(suffixes[i]); + if (n >= s && strcmp(base + n - s, suffixes[i]) == 0) { + return true; + } + } + return strcmp(base, ".env") == 0 || strncmp(base, ".env.", INFRA_SLEN(".env.")) == 0; +} + +/* A manifest's data / stringData / binaryData hold configuration payload + * (a Secret keeps its values there); a value under them never declares an + * endpoint the service exposes. */ +static bool infra_payload_key(const char *key_path) { + if (!key_path) { + return false; + } + size_t first = strcspn(key_path, "."); + return (first == INFRA_SLEN("data") && strncmp(key_path, "data", first) == 0) || + (first == INFRA_SLEN("stringData") && strncmp(key_path, "stringData", first) == 0) || + (first == INFRA_SLEN("binaryData") && strncmp(key_path, "binaryData", first) == 0); +} + +/* `url` as a route: scheme, host and path. Userinfo (`user:pass@`), the query + * and the fragment are where credentials and tokens ride; none is part of + * the endpoint. */ +static void infra_url_route_form(const char *url, char *out, size_t out_sz) { + const char *scheme_end = strstr(url, "://"); + const char *host = scheme_end ? scheme_end + INFRA_SLEN("://") : url; + const char *path = host + strcspn(host, "/?#"); + const char *at = memchr(host, '@', (size_t)(path - host)); + const char *host_start = at ? at + SKIP_ONE : host; + size_t path_len = strcspn(path, "?#"); + snprintf(out, out_sz, "%.*s%.*s%.*s", (int)(host - url), url, (int)(path - host_start), + host_start, (int)path_len, path); +} + /* Extract Route nodes from URL strings found in config files (YAML, HCL, TOML). * These are infrastructure-defined endpoints (Cloud Scheduler, Terraform). */ /* Process infra bindings: topic→URL pairs from IaC configs. @@ -1601,10 +1648,17 @@ static void *gh_compute_thread_fn(void *arg) { /* Process one infra binding: create Route node + INFRA_MAPS edge. */ static int process_one_infra_binding(cbm_gbuf_t *gbuf, const CBMInfraBinding *ib, const char *rel_path) { + /* Secret hygiene, as for config routes: a values file or a token-shaped + * URL mints nothing; a kept URL loses its credentials and query. */ + if (infra_values_file(rel_path) || cbm_is_secret_value(ib->target_url)) { + return 0; + } + char url[CBM_ROUTE_QN_SIZE]; + infra_url_route_form(ib->target_url, url, sizeof(url)); char url_route_qn[CBM_ROUTE_QN_SIZE]; - snprintf(url_route_qn, sizeof(url_route_qn), "__route__infra__%s", ib->target_url); - int64_t url_route_id = cbm_gbuf_upsert_node(gbuf, "Route", ib->target_url, url_route_qn, - rel_path, 0, 0, "{\"source\":\"infra\"}"); + snprintf(url_route_qn, sizeof(url_route_qn), "__route__infra__%s", url); + int64_t url_route_id = cbm_gbuf_upsert_node(gbuf, "Route", url, url_route_qn, rel_path, 0, 0, + "{\"source\":\"infra\"}"); char topic_route_qn[CBM_ROUTE_QN_SIZE]; snprintf(topic_route_qn, sizeof(topic_route_qn), "__route__%s__%s", ib->broker ? ib->broker : "async", ib->source_name); @@ -1625,7 +1679,7 @@ static int process_one_infra_binding(cbm_gbuf_t *gbuf, const CBMInfraBinding *ib } char props[CBM_SZ_512]; snprintf(props, sizeof(props), "{\"broker\":\"%s\",\"topic\":\"%s\",\"endpoint\":\"%s\"}", - ib->broker ? ib->broker : "async", ib->source_name, ib->target_url); + ib->broker ? ib->broker : "async", ib->source_name, url); cbm_gbuf_insert_edge(gbuf, topic_route_id, url_route_id, "INFRA_MAPS", props); return SKIP_ONE; } @@ -1711,21 +1765,32 @@ static bool is_ci_tooling_config(const char *fp) { return false; } -/* True when a YAML key path denotes an UPSTREAM dependency, CONFIG value, or - * HEALTHCHECK target rather than an endpoint this service exposes. Such URLs - * (auth JWKS, downstream service base URLs, package-registry URLs, healthcheck - * curl targets) are NOT routes the service serves and must not mint Route nodes - * (#521). Exposed-endpoint keys (push_endpoint, post_url, callback, webhook) - * are intentionally absent here so they still produce infra Route nodes. */ -static bool is_upstream_config_key(const char *key_path) { +/* True when a config key names an endpoint THIS service exposes -- a URL other + * systems call on it: a push subscription's endpoint, a post/callback/webhook + * URL. Only such keys mint infra Route nodes. Every other config URL is a + * reference: an upstream dependency, a registry or repository, a docs or + * runbook link, a changelog's pull-request link, a kubeconfig server. A deny + * list of those (#521) could not keep up: in 17 infra repositories almost + * every infra Route came from such a key (deckhouse: 6,240 `pull_request` + * links). The leaf key decides, compared ignoring case, `_` and `-`. */ +static bool is_exposed_endpoint_key(const char *key_path) { if (!key_path) { - /* No key context (e.g. flat string) — keep prior behaviour and mint. */ return false; } - static const char *const deny[] = {"jwks", "registry", "registries", "healthcheck", - "upstream", "_service_url", "auth", NULL}; - for (int i = 0; deny[i]; i++) { - if (strstr(key_path, deny[i]) != NULL) { + const char *leaf = strrchr(key_path, '.'); + leaf = leaf ? leaf + SKIP_ONE : key_path; + char norm[CBM_SZ_64]; + size_t n = 0; + for (const char *p = leaf; *p && n + 1 < sizeof(norm); p++) { + if (*p != '_' && *p != '-') { + norm[n++] = (char)tolower((unsigned char)*p); + } + } + norm[n] = '\0'; + static const char *const endpoints[] = {"pushendpoint", "posturl", "callback", "callbackurl", + "webhook", "webhookurl", NULL}; + for (int i = 0; endpoints[i]; i++) { + if (strcmp(norm, endpoints[i]) == 0) { return true; } } @@ -1737,12 +1802,14 @@ static void try_upsert_infra_route(cbm_gbuf_t *gbuf, const CBMStringRef *sr, con if (sr->kind != CBM_STRREF_URL || !sr->value || !strstr(sr->value, "://")) { return; } - /* Skip upstream/config/healthcheck URLs — they are not exposed routes (#521). */ - if (is_upstream_config_key(sr->key_path)) { + /* Only an endpoint this service exposes is a route it serves (#521). */ + if (!is_exposed_endpoint_key(sr->key_path)) { return; } + char url[CBM_ROUTE_QN_SIZE]; + infra_url_route_form(sr->value, url, sizeof(url)); char route_qn[CBM_ROUTE_QN_SIZE]; - snprintf(route_qn, sizeof(route_qn), "__route__infra__%s", sr->value); + snprintf(route_qn, sizeof(route_qn), "__route__infra__%s", url); char route_props[CBM_SZ_512]; if (sr->key_path) { snprintf(route_props, sizeof(route_props), "{\"source\":\"infra\",\"key_path\":\"%s\"}", @@ -1750,14 +1817,14 @@ static void try_upsert_infra_route(cbm_gbuf_t *gbuf, const CBMStringRef *sr, con } else { snprintf(route_props, sizeof(route_props), "{\"source\":\"infra\"}"); } - cbm_gbuf_upsert_node(gbuf, "Route", sr->value, route_qn, fp, 0, 0, route_props); + cbm_gbuf_upsert_node(gbuf, "Route", url, route_qn, fp, 0, 0, route_props); } /* A URL string_ref that does NOT denote a route the service serves: a value * containing whitespace is a command/sentence with an embedded URL (e.g. a * Docker healthcheck `curl --fail http://... || exit 1`); a NULL key_path is a - * context-less/duplicate ref; an upstream/config/healthcheck key is an external - * dependency, not an exposed route. (#521) */ + * context-less/duplicate ref; any key but an exposed-endpoint key is a + * reference, not an exposed route. (#521) */ static bool route_sr_denied(const CBMStringRef *sr) { if (!sr->value || strchr(sr->value, ' ')) { return true; @@ -1765,7 +1832,12 @@ static bool route_sr_denied(const CBMStringRef *sr) { if (!sr->key_path) { return true; } - return is_upstream_config_key(sr->key_path); + /* Secret hygiene: payload values and token-shaped values are denied by + * value, so no sibling ref of the same URL mints it either. */ + if (infra_payload_key(sr->key_path) || cbm_is_secret_value(sr->value)) { + return true; + } + return !is_exposed_endpoint_key(sr->key_path); } static void cbm_pipeline_extract_infra_routes(const cbm_pipeline_ctx_t *ctx, cbm_gbuf_t *gbuf, @@ -1786,7 +1858,8 @@ static void cbm_pipeline_extract_infra_routes(const cbm_pipeline_ctx_t *ctx, cbm CBMHashTable *denied = cbm_ht_create(16); for (int pass = 0; pass < 2; pass++) { for (int i = 0; i < file_count; i++) { - if (!is_infra_file(files[i].rel_path) || is_ci_tooling_config(files[i].rel_path)) { + if (!is_infra_file(files[i].rel_path) || is_ci_tooling_config(files[i].rel_path) || + infra_values_file(files[i].rel_path)) { continue; } bool loaded = false; @@ -1835,6 +1908,9 @@ static void predump_sem(cbm_pipeline_ctx_t *ctx) { static void predump_cfg(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_configlink(ctx); } +static void predump_operator(cbm_pipeline_ctx_t *ctx) { + (void)cbm_pipeline_pass_operator(ctx); +} static void predump_doclinks(cbm_pipeline_ctx_t *ctx) { cbm_pipeline_pass_doclinks(ctx); } @@ -2077,6 +2153,7 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { } passes[] = { {predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false}, + {predump_operator, "operator", false}, {predump_doclinks, "doclinks", false}, {predump_route, "route_match", false}, {predump_ensemble, "ensemble_routing", false}, @@ -3616,6 +3693,9 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (check_cancel(p)) { return CBM_NOT_FOUND; } + /* A repository Image whose edges all went to its components (pass_docker.c + * P4.2b) is left without any: it goes, as on the incremental routes. */ + (void)cbm_gbuf_delete_unreferenced(p->gbuf, "Image", DK_IMAGE_EDGES); return rc; } diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index e7f5ee40a..0ccbbce48 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -247,6 +247,120 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const * return 0; } +int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label, + const char *const *edge_types) { + sqlite3 *db = cbm_store_get_db(store); + if (!db || !edge_types) { + return CBM_NOT_FOUND; + } + /* One statement: a pair of NOT EXISTS per listed type (target, source), + * each served by an (id, type) index. */ + char sql[CBM_SZ_2K]; + int len = snprintf(sql, sizeof(sql), "DELETE FROM nodes WHERE project = ?1 AND label = ?2"); + int t = 0; + for (; edge_types[t] && len > 0 && (size_t)len < sizeof(sql); t++) { + len += snprintf(sql + len, sizeof(sql) - (size_t)len, + " AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.project = ?1" + " AND e.target_id = nodes.id AND e.type = ?%d)" + " AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.project = ?1" + " AND e.source_id = nodes.id AND e.type = ?%d)", + t + 3, t + 3); + } + if (t == 0 || len <= 0 || (size_t)len >= sizeof(sql)) { + return CBM_NOT_FOUND; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + return CBM_NOT_FOUND; + } + sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT); + for (int i = 0; i < t; i++) { + sqlite3_bind_text(stmt, i + 3, edge_types[i], CBM_NOT_FOUND, SQLITE_TRANSIENT); + } + int rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND; +} + +/* The first column of `sql` (?1 = the project) as arena strings in *out. */ +static int delta_query_strings(cbm_store_t *store, const char *project, const char *sql, + CBMArena *a, const char ***out) { + *out = NULL; + sqlite3 *db = cbm_store_get_db(store); + if (!db) { + return CBM_NOT_FOUND; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + return CBM_NOT_FOUND; + } + sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); + int n = 0; + int cap = 0; + const char **items = NULL; + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *f = (const char *)sqlite3_column_text(stmt, 0); + if (!f) { + continue; + } + if (n == cap) { + int ncap = cap ? cap * PAIR_LEN : CBM_SZ_16; + const char **grown = cbm_arena_alloc(a, (size_t)ncap * sizeof(*grown)); + if (!grown) { + break; + } + if (n) { + memcpy((void *)grown, items, (size_t)n * sizeof(*grown)); + } + items = grown; + cap = ncap; + } + items[n++] = cbm_arena_strdup(a, f); + } + sqlite3_finalize(stmt); + *out = items; + return n; +} + +int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena *a, + const char ***out) { + return delta_query_strings(store, project, + "SELECT file_path FROM nodes WHERE project = ?1 AND label = 'File'" + " AND json_extract(properties, '$.declares_build') = 1", + a, out); +} + +int cbm_delta_image_components(cbm_store_t *store, const char *project, CBMArena *a, + const char ***out) { + return delta_query_strings(store, project, + "SELECT qualified_name FROM nodes WHERE project = ?1 AND" + " label = 'Image' AND json_extract(properties, '$.component')" + " IS NOT NULL ORDER BY qualified_name", + a, out); +} + +int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, const char *type, + const char *label) { + sqlite3 *db = cbm_store_get_db(store); + if (!db) { + return CBM_NOT_FOUND; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, + "DELETE FROM edges WHERE project = ?1 AND type = ?2 AND source_id IN" + " (SELECT id FROM nodes WHERE project = ?1 AND label = ?3)", + CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + return CBM_NOT_FOUND; + } + sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, type, CBM_NOT_FOUND, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 3, label, CBM_NOT_FOUND, SQLITE_TRANSIENT); + int rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND; +} + /* Every project node becomes a proxy. The first cut filtered by label and * the fail-closed patch immediately found the counterexamples (synthetic * Decorator nodes on django; Macro is six of the kernel's 8.5M nodes) — any @@ -412,6 +526,7 @@ typedef struct { sqlite3 *db; const char *project; int64_t max_db_id; + const cbm_gbuf_t *gbuf; sqlite3_stmt *node_stmt; sqlite3_stmt *edge_stmt; sqlite3_stmt *qn_lookup; @@ -525,6 +640,17 @@ static void delta_patch_node(const cbm_gbuf_node_t *node, void *userdata) { ctx->nodes++; } +/* An edge of the class recomputed wholesale on every run (an Image's build + * edges): the staging graph dropped its old set, so it is written even + * between pre-existing nodes. */ +static bool delta_edge_recomputed(const delta_patch_ctx_t *ctx, const cbm_gbuf_edge_t *edge) { + if (!ctx->gbuf || strcmp(edge->type, CBM_BUILD_EDGE_TYPE) != 0) { + return false; + } + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, edge->source_id); + return src && src->label && strcmp(src->label, CBM_BUILD_EDGE_SOURCE) == 0; +} + static void delta_patch_edge(const cbm_gbuf_edge_t *edge, void *userdata) { delta_patch_ctx_t *ctx = (delta_patch_ctx_t *)userdata; if (ctx->failed) { @@ -533,7 +659,7 @@ static void delta_patch_edge(const cbm_gbuf_edge_t *edge, void *userdata) { int64_t src = delta_remap_get(ctx, edge->source_id); int64_t tgt = delta_remap_get(ctx, edge->target_id); if (src <= ctx->max_db_id && tgt <= ctx->max_db_id && src == edge->source_id && - tgt == edge->target_id) { + tgt == edge->target_id && !delta_edge_recomputed(ctx, edge)) { return; /* both endpoints pre-existed unremapped: already recorded */ } sqlite3_reset(ctx->edge_stmt); @@ -596,7 +722,10 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i if (cbm_store_begin(store) != CBM_STORE_OK) { return CBM_NOT_FOUND; } - delta_patch_ctx_t ctx = {.db = db, .project = project, .max_db_id = max_db_id}; + /* This writes the buffer's nodes itself: settle the collision spans first, + * as the buffer's own dump and flush do. */ + cbm_gbuf_settle_variants(gbuf); + delta_patch_ctx_t ctx = {.db = db, .project = project, .max_db_id = max_db_id, .gbuf = gbuf}; if (sqlite3_prepare_v2(db, "SELECT id FROM nodes WHERE project = ?1" " AND qualified_name = ?2", diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index a220b4a1d..a04cc7044 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1164,6 +1164,68 @@ static bool incr_edge_type_is_recomputed(const char *type) { strcmp(type, "FILE_CHANGES_WITH") == 0 || strcmp(type, "DATA_FLOWS") == 0); } +static int incr_strcmp(const void *x, const void *y) { + return strcmp(*(const char *const *)x, *(const char *const *)y); +} + +/* Are the component Images this run built (pass_docker.c P4.2b: an Image + * with a `component` property and a build edge in `gb`) exactly `before` + * (sorted QNs)? */ +static bool incr_image_components_equal(const cbm_gbuf_t *gb, const char *const *before, int nb, + CBMArena *a) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_label(gb, "Image", &nodes, &n) != 0 || n == 0) { + return nb == 0; + } + const char **now = cbm_arena_alloc(a, (size_t)n * sizeof(*now)); + if (!now) { + return false; + } + int k = 0; + for (int i = 0; i < n; i++) { + const cbm_gbuf_edge_t **out = NULL; + int ne = 0; + if (nodes[i]->properties_json && strstr(nodes[i]->properties_json, "\"component\":") && + cbm_gbuf_find_edges_by_source_type(gb, nodes[i]->id, "DEPENDS_ON", &out, &ne) == 0 && + ne > 0) { + now[k++] = nodes[i]->qualified_name; + } + } + if (k != nb) { + return false; + } + qsort((void *)now, (size_t)k, sizeof(*now), incr_strcmp); + for (int i = 0; i < k; i++) { + if (strcmp(now[i], before[i]) != 0) { + return false; + } + } + return true; +} + +/* The files whose File node the shell pass marked as running a container + * build (pass_shell.c), from the full-load buffer, into arena `a`; returns + * how many. */ +static int incr_build_decl_files(const cbm_gbuf_t *gb, CBMArena *a, const char ***out) { + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + *out = NULL; + if (cbm_gbuf_find_by_label(gb, "File", &nodes, &n) != 0 || n == 0) { + return 0; + } + const char **items = cbm_arena_alloc(a, (size_t)n * sizeof(*items)); + int k = 0; + for (int i = 0; items && i < n; i++) { + const char *p = nodes[i]->properties_json; + if (nodes[i]->file_path && p && strstr(p, "\"declares_build\":true")) { + items[k++] = cbm_arena_strdup(a, nodes[i]->file_path); + } + } + *out = items; + return k; +} + /* cbm_gbuf_foreach_edge visitor: snapshot inbound cross-file edges into * changed files so they survive the purge and can be re-linked afterward. */ static void incr_capture_inbound_edge(const cbm_gbuf_edge_t *edge, void *userdata) { @@ -2020,6 +2082,14 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file return rc < 0 ? rc : CBM_NOT_FOUND; } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + rc = cbm_pipeline_pass_operator(ctx); + cbm_log_info("pass.timing", "pass", "incr_operator", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + if (rc < 0 || cbm_pipeline_check_cancel(ctx)) { + return rc < 0 ? rc : CBM_NOT_FOUND; + } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); rc = cbm_pipeline_pass_doclinks(ctx); cbm_log_info("pass.timing", "pass", "incr_doclinks", "elapsed_ms", @@ -2346,6 +2416,90 @@ static const char *closure_add_variant_partners(cbm_store_t *store, const char * return decline; } +/* Whether the directory of `rel` (whose basename is at `base`) holds a + * compose file: compose interpolates that directory's .env into it, so the + * .env changes what its services build and run (pass_ci.c). */ +static bool closure_dir_has_compose(const char *repo_path, const char *rel, const char *base) { + char dir[CBM_SZ_4K]; + int n = snprintf(dir, sizeof(dir), "%s/%.*s", repo_path, (int)(base - rel), rel); + if (n < 0 || (size_t)n >= sizeof(dir)) { + return true; /* unknown: decline */ + } + cbm_dir_t *d = cbm_opendir(dir); + if (!d) { + return false; + } + bool found = false; + cbm_dirent_t *e; + while (!found && (e = cbm_readdir(d)) != NULL) { + found = !e->is_dir && cbm_build_path(e->name) && + strncmp(e->name, "skaffold", strlen("skaffold")) != 0; + } + cbm_closedir(d); + return found; +} + +/* Kubernetes inputs reach past their own file (pass_k8s.c): a kustomization + * decides which manifests deploy together and in which namespace, a + * Chart.yaml links other charts, and a manifest's Services select workloads + * in other files of its bundle. The repair re-runs the K8s pass over its plan + * only, so a delta touching any of them -- a file that held objects, or holds + * them now -- declines to the full rebuild. Returns the reason, or NULL. */ +static const char *closure_k8s_decline(cbm_store_t *store, const char *project, + const char *repo_path, char **paths, int n_changed, + int n_deleted) { + for (int i = 0; i < n_changed + n_deleted; i++) { + const char *rel = paths[i]; + const char *slash = strrchr(rel, '/'); + const char *base = slash ? slash + SKIP_ONE : rel; + if (cbm_is_kustomize_file(base) || strcmp(base, "Kustomization") == 0) { + return "k8s_bundle_input_changed"; + } + if (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0) { + return "helm_chart_changed"; + } + if (strcmp(base, ".env") == 0 && closure_dir_has_compose(repo_path, rel, base)) { + return "compose_env_changed"; + } + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_file(store, project, rel, &nodes, &count) != CBM_STORE_OK) { + return "k8s_lookup_failed"; + } + /* A compose service (pass_docker.c) is a Resource too, but it links + * only inside its file and to the Images / Dockerfiles every run + * recomputes: no bundle reaches past it. */ + bool held = false; + for (int j = 0; j < count && !held; j++) { + held = nodes[j].label && strcmp(nodes[j].label, "Resource") == 0 && + !(nodes[j].properties_json && + strstr(nodes[j].properties_json, "\"ecosystem\":\"compose\"")); + } + cbm_store_free_nodes(nodes, count); + if (held) { + return "k8s_manifest_changed"; + } + const char *ext = strrchr(base, '.'); + if (i < n_changed && ext && (strcmp(ext, ".yaml") == 0 || strcmp(ext, ".yml") == 0)) { + enum { K8S_PEEK = 4096 }; + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", repo_path, rel); + char head[K8S_PEEK + SKIP_ONE]; + size_t n = 0; + FILE *f = cbm_fopen(path, "rb"); + if (f) { + n = fread(head, SKIP_ONE, K8S_PEEK, f); + (void)fclose(f); + } + head[n] = '\0'; + if (cbm_is_k8s_manifest(base, head)) { + return "k8s_manifest_changed"; + } + } + } + return NULL; +} + /* Decide whether this manifest delta is closure-repairable, and if so build * the plan. Returns 1 with *plan filled, or 0 (decline → FULL) with the * reason logged. Every early exit declines; nothing here mutates state. */ @@ -2468,6 +2622,11 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p goto done; } } + decline = closure_k8s_decline(store, project, cbm_pipeline_repo_path(p), changed_paths, + n_changed, n_deleted); + if (decline) { + goto done; + } /* Load the previous generation's surfaces and probe the changed files' * fresh ones. Missing rows fail closed. */ @@ -2763,6 +2922,10 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char cbm_doc_link_row_t *doc_rows = NULL; int doc_row_count = 0; bool doc_failed = false; + CBMArena decl_arena; /* the files that declared image builds before */ + cbm_arena_init_lazy(&decl_arena, CBM_SZ_4K); + const char **decl_files = NULL; + int decl_count = 0; cbm_clock_gettime(CLOCK_MONOTONIC, &t); if (cbm_delta_stage_clone(db_path, &stage) != 0) { @@ -2775,6 +2938,14 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char goto out; } cbm_log_info("delta.staging_open", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + /* before the purge drops any of their edges */ + decl_count = cbm_delta_build_decl_files(staging, project, &decl_arena, &decl_files); + decl_count = decl_count > 0 ? decl_count : 0; + const char **hub_images = NULL; + int hub_count = cbm_delta_image_components(staging, project, &decl_arena, &hub_images); + if (hub_count < 0) { + goto out; + } /* Purge set = closure files + deleted files. */ int purge_count = ci + deleted_count; @@ -2958,6 +3129,8 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char .excluded_count = excluded_count, .doc_link_base = doc_base, .doc_link_base_count = doc_base_count, + .build_decl_files = decl_files, + .build_decl_count = decl_count, }; for (int i = 0; i < ci; i++) { char *file_qn = cbm_pipeline_fqn_compute(project, changed_files[i].rel_path, "__file__"); @@ -3002,6 +3175,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char } goto out; } + /* A repository split differently (pass_docker.c P4.2b) leaves the + * references of files not repaired here on the old component Images: + * only a full rebuild re-points them. */ + if (!incr_image_components_equal(gbuf, hub_images, hub_count, &decl_arena)) { + cbm_log_info("delta.fallback", "reason", "image_components_changed"); + goto out; + } cbm_log_info("delta.repair", "files", itoa_buf(ci), "elapsed_ms", itoa_buf((int)elapsed_ms(t))); /* Doc-link rows: the previous rows of every file not re-extracted (or @@ -3017,11 +3197,26 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char } } + /* An Image's build edges (Image -> Dockerfile, pass_docker.c) belong to + * the skaffold / compose file declaring them, not to either end: the K8s + * pass just recomputed them from every build file, so they replace the + * staging set wholesale. */ + if (cbm_delta_drop_edges_from_label(staging, project, CBM_BUILD_EDGE_TYPE, + CBM_BUILD_EDGE_SOURCE) < 0) { + goto out; + } cbm_clock_gettime(CLOCK_MONOTONIC, &t); if (cbm_delta_patch(staging, project, gbuf, max_db_id, snapshot, snapshot_count) != 0) { goto out; } cbm_log_info("delta.patch_done", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + /* The proxy buffer holds no edges, so only the patched staging graph can + * tell a Process node no SPAWNS reaches any more (its last spawner was + * purged), or an Image nothing builds on, builds or runs. */ + if (cbm_delta_drop_unreferenced(staging, project, "Process", SH_PROCESS_EDGES) < 0 || + cbm_delta_drop_unreferenced(staging, project, "Image", DK_IMAGE_EDGES) < 0) { + goto out; + } /* Importance, in SQL, over the now-complete staging graph. This is the * FIRST point on this route where the whole project is queryable: the @@ -3200,6 +3395,7 @@ out: #endif } cbm_delta_free_snapshot(snapshot, snapshot_count); + cbm_arena_destroy(&decl_arena); surface_name_set_free(stale_surface_set); cbm_doclinks_free_scopes(doc_base, doc_base_count); cbm_doclinks_free_rows(doc_fresh, doc_fresh_count); @@ -3531,6 +3727,16 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil legacy_doc_load(&legacy_doc, store, project, changed_files, ci, deleted, deleted_count); cbm_store_close(store); + /* An Image's build edges (Image -> Dockerfile, pass_docker.c) belong to + * the skaffold / compose file declaring them, not to either end: the K8s + * pass recomputes them all from every build file, so none is kept (nor + * captured below). */ + CBMArena decl_arena; /* the files that declared image builds before */ + cbm_arena_init_lazy(&decl_arena, CBM_SZ_4K); + const char **decl_files = NULL; + int decl_count = incr_build_decl_files(existing, &decl_arena, &decl_files); + (void)cbm_gbuf_delete_edges_from_label(existing, CBM_BUILD_EDGE_TYPE, CBM_BUILD_EDGE_SOURCE); + /* Snapshot inbound cross-file edges into changed files BEFORE purging, so * the cascade delete doesn't permanently drop edges whose source lives in * an unchanged (never-re-parsed) file. Re-linked after re-resolution. */ @@ -3622,6 +3828,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } } + ctx.build_decl_files = decl_files; + ctx.build_decl_count = decl_count; int phase_rc = run_extract_resolve(&ctx, changed_files, ci, NULL); if (phase_rc == 0) { phase_rc = cbm_pipeline_pass_k8s(&ctx, changed_files, ci); @@ -3630,6 +3838,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil phase_rc = cbm_pipeline_check_cancel(&ctx); } if (phase_rc == 0) { + /* A Process node exists only as a SPAWNS target (pass_spawns.c, + * pass_shell.c), an Image only through its DEPENDS_ON / USAGE edges + * (pass_docker.c); neither carries a file: when the last file naming + * one is gone, nothing else removes it -- a full index never makes + * it. This buffer holds the whole graph. */ + (void)cbm_gbuf_delete_unreferenced(existing, "Process", SH_PROCESS_EDGES); + (void)cbm_gbuf_delete_unreferenced(existing, "Image", DK_IMAGE_EDGES); phase_rc = run_postpasses(&ctx, changed_files, ci, project, true); } @@ -3661,6 +3876,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil free_mode_skipped(mode_skipped, mode_skipped_count); free(saved_adr); cbm_gbuf_free(existing); + cbm_arena_destroy(&decl_arena); legacy_doc_free(&legacy_doc); return CBM_PIPELINE_ABORT_PRESERVE_DB; } @@ -3773,6 +3989,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil free_mode_skipped(mode_skipped, mode_skipped_count); free(saved_adr); cbm_gbuf_free(existing); + cbm_arena_destroy(&decl_arena); cbm_doclinks_free_rows(doc_rows, doc_row_count); return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_PIPELINE_ABORT_PRESERVE_DB; @@ -3816,6 +4033,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_store_free_coverage(old_cov, old_cov_count); free_mode_skipped(mode_skipped, mode_skipped_count); cbm_gbuf_free(existing); + cbm_arena_destroy(&decl_arena); cbm_log_info("incremental.done", "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); return persist_rc; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 92e7e315d..bc6387b64 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -308,6 +308,11 @@ void cbm_pkg_entries_free(cbm_pkg_entries_t *e); typedef struct { const char *project_name; /* borrowed from pipeline */ const char *repo_path; /* borrowed from pipeline */ + /* An incremental run: the unchanged files whose earlier run declared + * image builds (docker build ...), read again by the shell pass, as + * those edges are recomputed whole. Borrowed; NULL in a full run. */ + const char *const *build_decl_files; + int build_decl_count; cbm_gbuf_t *gbuf; /* owned by pipeline */ cbm_registry_t *registry; /* owned by pipeline */ atomic_int *cancelled; /* pointer to pipeline's cancelled flag */ @@ -906,11 +911,6 @@ int cbm_parse_compile_commands(const char *json_data, const char *repo_path, cha /* ── Infrascan helpers (pass_infrascan.c) ─────────────────────────── */ /* File identification helpers */ -bool cbm_is_dockerfile(const char *name); -bool cbm_is_compose_file(const char *name); -bool cbm_is_cloudbuild_file(const char *name); -bool cbm_is_env_file(const char *name); -bool cbm_is_shell_script(const char *name, const char *ext); bool cbm_is_kustomize_file(const char *name); bool cbm_is_k8s_manifest(const char *name, const char *content); @@ -918,114 +918,24 @@ bool cbm_is_k8s_manifest(const char *name, const char *content); bool cbm_is_secret_binding(const char *key, const char *value); bool cbm_is_secret_value(const char *value); -/* Clean JSON array brackets from CMD/ENTRYPOINT values. - * E.g. ["./app", "--flag"] → ./app --flag - * Writes result to out (up to out_sz). */ -void cbm_clean_json_brackets(const char *s, char *out, size_t out_sz); - -/* Key-value pair for environment variables / config entries */ +/* Helm Chart.yaml parse result: chart name + its dependencies (#338). */ +enum { CBM_HELM_NAME_MAX = 128, CBM_HELM_REPO_MAX = 512 }; typedef struct { - char key[CBM_SZ_128]; - char value[CBM_SZ_512]; -} cbm_env_kv_t; - -/* Dockerfile parsing result */ -typedef struct { - char base_image[CBM_SZ_256]; - char stage_images[CBM_SZ_16][CBM_SZ_256]; - char stage_names[CBM_SZ_16][CBM_SZ_128]; - int stage_count; - char exposed_ports[CBM_SZ_16][CBM_SZ_32]; - int port_count; - cbm_env_kv_t env_vars[CBM_SZ_64]; - int env_count; - char build_args[CBM_SZ_32][CBM_SZ_128]; - int build_arg_count; - char workdir[CBM_SZ_256]; - char cmd[CBM_SZ_512]; - char entrypoint[CBM_SZ_512]; - char healthcheck[CBM_SZ_512]; - char user[CBM_SZ_64]; -} cbm_dockerfile_result_t; - -/* Dotenv parsing result */ -typedef struct { - cbm_env_kv_t env_vars[CBM_SZ_64]; - int env_count; -} cbm_dotenv_result_t; - -/* Shell script parsing result */ -typedef struct { - char shebang[CBM_SZ_256]; - cbm_env_kv_t env_vars[CBM_SZ_64]; - int env_count; - char sources[CBM_SZ_16][CBM_SZ_256]; - int source_count; - char docker_cmds[CBM_SZ_16][CBM_SZ_256]; - int docker_cmd_count; -} cbm_shell_result_t; - -/* Terraform variable */ -typedef struct { - char name[CBM_SZ_128]; - char type[CBM_SZ_64]; - char default_val[CBM_SZ_256]; - char description[CBM_SZ_256]; -} cbm_tf_variable_t; - -/* Terraform resource / data source */ -typedef struct { - char type[CBM_SZ_128]; - char name[CBM_SZ_128]; -} cbm_tf_resource_t; - -/* Terraform module */ -typedef struct { - char tf_name[CBM_SZ_128]; - char source[CBM_SZ_256]; -} cbm_tf_module_t; - -/* Terraform parsing result */ -typedef struct { - cbm_tf_resource_t resources[CBM_SZ_32]; - int resource_count; - cbm_tf_variable_t variables[CBM_SZ_32]; - int variable_count; - char outputs[CBM_SZ_32][CBM_SZ_128]; - int output_count; - char providers[CBM_SZ_16][CBM_SZ_128]; - int provider_count; - cbm_tf_module_t modules[CBM_SZ_16]; - int module_count; - cbm_tf_resource_t data_sources[CBM_SZ_16]; - int data_source_count; - char backend[CBM_SZ_128]; - bool has_locals; -} cbm_terraform_result_t; - -/* Parse a Dockerfile from source text. Returns 0 if parsed, -1 if empty/invalid. */ -int cbm_parse_dockerfile_source(const char *source, cbm_dockerfile_result_t *out); - -/* Parse a .env file from source text. Returns 0 if parsed, -1 if empty. */ -int cbm_parse_dotenv_source(const char *source, cbm_dotenv_result_t *out); - -/* Parse a shell script from source text. Returns 0 if parsed, -1 if empty. */ -int cbm_parse_shell_source(const char *source, cbm_shell_result_t *out); - -/* Parse a Terraform file from source text. Returns 0 if parsed, -1 if empty. */ -int cbm_parse_terraform_source(const char *source, cbm_terraform_result_t *out); - -/* Helm Chart.yaml parse result: chart name + dependency chart names (#338). */ -enum { CBM_HELM_MAX_DEPS = 128, CBM_HELM_NAME_MAX = 128 }; + char name[CBM_HELM_NAME_MAX]; + char repository[CBM_HELM_REPO_MAX]; /* "" when the entry names none */ +} cbm_helm_dep_t; typedef struct { char chart_name[CBM_HELM_NAME_MAX]; - char deps[CBM_HELM_MAX_DEPS][CBM_HELM_NAME_MAX]; + cbm_helm_dep_t *deps; /* heap-owned; release with cbm_helm_chart_free */ int dep_count; + int dep_cap; } cbm_helm_chart_t; -/* Parse a Helm Chart.yaml: top-level `name:` and `dependencies:` list names. - * Returns 0 if parsed (name or deps found), -1 otherwise. */ +/* Parse a Helm Chart.yaml: top-level `name:` and every `dependencies:` entry's + * name and repository. Returns 0 if parsed (name or deps found), -1 otherwise; + * either way the caller releases `out` with cbm_helm_chart_free. */ int cbm_parse_helm_chart(const char *source, cbm_helm_chart_t *out); +void cbm_helm_chart_free(cbm_helm_chart_t *hc); /* Build an infrastructure QN. Caller must free the returned string. */ char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *infra_type, @@ -1160,6 +1070,116 @@ int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project); /* Pre-dump pass: config ↔ code linking. */ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx); +/* Go Kubernetes operators (pass_operator.c): controller-runtime controllers + * HANDLES the kinds they reconcile, CRDs -> their Go types, //go:embed files. + * Runs after USAGE resolution. */ +int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx); + +/* Shell scripts, Make recipes and the commands CI runs (pass_shell.c): a + * command that puts manifests, kustomizations or charts on a cluster DEPLOYS + * them; one that starts a program by a path, or a script of the repository, + * SPAWNS it. Run at the end of the K8s pass, over the same files. */ +void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); + +/* One command line a CI configuration runs (pass_ci.c). */ +typedef struct { + const char *ci; /* "github-actions" | "prow" */ + const char *job; /* the job; NULL: none (a composite action's step) */ + const char *text; /* shell text; CBM_SZ_16 zero bytes past len */ + size_t len; /* bytes of text */ + int row; /* 0-based source row of the text's first line */ + const char *wd; /* start directory, repo-relative ("" = root); NULL: unknown */ + const char *action_dir; /* a composite action's directory; NULL: none */ + const char **env; /* name, value pairs (a value NULL: unknown) */ + int nenv; /* pairs in env */ + bool exec; /* an argv exec'd without a shell, quoted into text */ +} cbm_ci_cmd_t; + +typedef void (*cbm_ci_cmd_fn)(const cbm_ci_cmd_t *cmd, void *ud); + +/* `rel` is a CI configuration by its path: a GitHub workflow + * (.github/workflows/.yml|yaml) or a composite action (action.yml|yaml). */ +bool cbm_ci_path(const char *rel); + +/* Hand each command line the CI configuration `rel` (YAML text src) runs to + * fn: GitHub Actions run: steps for a workflow or composite action, else + * Prow job containers. Strings live in arena `a`. */ +void cbm_ci_commands(CBMArena *a, const char *rel, const char *src, size_t len, cbm_ci_cmd_fn fn, + void *ud); + +/* An image a build configuration builds from a Dockerfile (pass_ci.c). */ +typedef struct { + const char *via; /* "skaffold" / "compose" */ + const char *image; /* the reference, variables expanded */ + const char *dockerfile; /* repository path */ + int line; +} cbm_build_site_t; +typedef void (*cbm_build_site_fn)(const cbm_build_site_t *site, void *ud); + +/* A compose service (pass_ci.c): its image (a reference, variables + * expanded; NULL: none or unknown), the Dockerfile its build names + * (repository path; NULL: no build, or unknown), the service names its + * depends_on and links list, and the names of the environment variables its + * environment sets (map keys / list items' names -- never a value), with + * their 1-based lines. */ +typedef struct { + const char *name; + const char *image; + const char *dockerfile; + const char **deps; + int ndeps; + const char **env; + const int *env_lines; + int nenv; + int line; + int end_line; +} cbm_compose_service_t; +typedef void (*cbm_compose_service_fn)(const cbm_compose_service_t *svc, void *ud); + +/* Hand fn each service of compose file `rel` (YAML text src; `dotenv` the + * project's .env text or NULL). Nothing for a file that is no compose file. */ +void cbm_compose_services(CBMArena *a, const char *rel, const char *src, size_t len, + const char *dotenv, cbm_compose_service_fn fn, void *ud); + +/* Whether `rel` names a build configuration: skaffold*.yaml|yml, + * [docker-]compose[.x].yaml|yml. */ +bool cbm_build_path(const char *rel); + +/* Hand fn each image build configuration `rel` (YAML text src) declares: + * skaffold build.artifacts (top level and profiles; the docker builder), + * compose services with build and image. `dotenv` (may be NULL): the text of + * the compose project's .env, whose variables compose interpolates. */ +void cbm_build_sites(CBMArena *a, const char *rel, const char *src, size_t len, const char *dotenv, + cbm_build_site_fn fn, void *ud); + +/* Terraform / OpenTofu modules (pass_terraform.c): a helm_release DEPLOYS its + * chart. Run at the end of the K8s pass, over the same files. */ +void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count); + +/* The Image node of image reference `ref` (its normalised repository: + * docker.io for no registry host, library/ for an official image; tag and + * digest dropped), made when new; 0 for scratch or no image. */ +int64_t cbm_pipeline_image_node(cbm_pipeline_ctx_t *ctx, const char *ref); + +/* Environment variable `key` that `owner` sets (pass_docker.c): a K8s or + * compose Resource for its container `container`, a Dockerfile Module for + * its image (container NULL). Makes the container-scoped EnvVar + * `#env//` {env_key, scope, container, + * source}, owned by the owner's file at `line`, and owner -CONFIGURES + * {strategy:env_set, source}-> it. No edge to the global EnvVar of the key: + * the same name read elsewhere is another program's. `source` says where the + * value comes from; the value is never stored. 0 when `key` is no variable + * name ([A-Za-z_][A-Za-z0-9_.-]*). */ +int64_t cbm_pipeline_env_setter(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *owner, + const char *container, const char *key, const char *source, + int line); + +/* Container builds (pass_docker.c): a Dockerfile DEPENDS_ON the Images it + * builds on (FROM, COPY --from). Run at the end of the K8s pass. */ +void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count); + /* Pre-dump pass: markdown → file REFERENCES_FILE linking. */ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx); @@ -1236,25 +1256,13 @@ extern _Atomic uint64_t g_importance_name_visits; * closure-delta route actually took that path instead of passing vacuously. */ extern _Atomic uint64_t g_importance_store_rows; -/* ── Env URL scanner (pass_envscan.c) ────────────────────────────── */ - -typedef struct { - char key[CBM_SZ_128]; - char value[CBM_SZ_512]; - char file_path[CBM_SZ_256]; -} cbm_env_binding_t; - -/* Scan a project directory for environment variable assignments with URL values. - * Walks the filesystem, scans Dockerfiles, shell scripts, .env, YAML, TOML, - * Terraform, and .properties files. Filters out secrets. - * Returns number of bindings written to out (up to max_out). - * NOTE: this walker currently has no production callers — it is exercised - * only by tests. The _excluded variant honors discovery exclusions for - * consistency with the pkgmap/path-alias walks (#792); the plain variant - * scans unexcluded (NULL exclusion list). */ -int cbm_scan_project_env_urls(const char *root_path, cbm_env_binding_t *out, int max_out); -int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t *out, int max_out, - char **excluded_dirs, int excluded_count); +/* Work counters of the infra resolvers, gated by the complexity suite: + * candidate objects the K8s pass examines for references, selectors and + * CRDs (pass_k8s.c), and candidate Go types / package dirs the operator pass + * examines (pass_operator.c). Independent copies must grow them linearly. */ +extern _Atomic uint64_t g_k8s_candidates; +extern _Atomic uint64_t g_op_candidates; +extern _Atomic uint64_t g_sh_candidates; /* ── Incremental pipeline (pipeline_incremental.c) ───────────────── */ @@ -1404,6 +1412,41 @@ int cbm_delta_snapshot_inbound(cbm_store_t *store, const char *project, const ch void cbm_delta_free_snapshot(cbm_delta_saved_edge_t *items, int count); int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const *paths, int path_count); +/* Delete the project's `label` nodes no edge of `edge_types` (NULL-terminated) + * touches as source or target (a Process whose last spawner was purged, an + * Image no Dockerfile or workload names any more); returns how many, or -1 + * on failure. */ +int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label, + const char *const *edge_types); +/* An Image's build edges (Image -DEPENDS_ON {via:build}-> Dockerfile, + * pass_docker.c) belong to the build configuration declaring them, not to + * either end: every run recomputes them whole from all build files, the + * incremental routes drop the old set first and write the new one even + * between pre-existing nodes. */ +#define CBM_BUILD_EDGE_TYPE "DEPENDS_ON" +#define CBM_BUILD_EDGE_SOURCE "Image" + +/* The files whose File node the shell pass marked as running a container + * build ({"declares_build":true}), into arena `a`; returns how many, or -1 + * on failure. */ +int cbm_delta_build_decl_files(cbm_store_t *store, const char *project, CBMArena *a, + const char ***out); + +/* The QNs of the component Images a split repository gave its components + * (pass_docker.c P4.2b: Images with a `component` property), sorted, into + * arena `a`; returns how many, or -1 on failure. A closure repair compares + * them with the run's own: references in files it does not repair point at + * the old ones. */ +int cbm_delta_image_components(cbm_store_t *store, const char *project, CBMArena *a, + const char ***out); + +/* Delete the project's `type` edges whose source node has label `label`; + * returns how many, or -1 on failure. */ +int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, const char *type, + const char *label); +/* The edges that keep a file-less shared node alive. */ +#define SH_PROCESS_EDGES ((const char *const[]){"SPAWNS", NULL}) +#define DK_IMAGE_EDGES ((const char *const[]){"DEPENDS_ON", "USAGE", NULL}) /* Pre-seed proxy nodes with their REAL database ids and move the gbuf id * watermark above MAX(id); returns that max id, or -1 on failure. */ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf); diff --git a/src/store/store.c b/src/store/store.c index f2f40e11a..7a2cbe616 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -6571,6 +6571,33 @@ static void where_add_like_hints(const char *column, const char *pattern, char * } } +/* Infra filters: a Resource's kind property and the namespace of its + * obj_key "group/Kind/ns/name" (no part holds a '/', so `%/%//%` pins + * the third; the pattern is built in SQL from the bound name, LIKE + * metacharacters escaped). Properties are guarded: json_extract raises on + * malformed JSON. */ +static void search_where_infra(const cbm_search_params_t *params, char *where, int where_sz, + int *wlen, int *nparams, search_bind_t *binds, int *bind_idx) { + char clause[CBM_SZ_256]; + if (params->kind && params->kind[0]) { + snprintf(clause, sizeof(clause), + "(CASE WHEN json_valid(n.properties) THEN json_extract(n.properties,'$.kind') " + "END) = ?%d", + *bind_idx + SKIP_ONE); + *wlen = where_append(where, where_sz, *wlen, nparams, clause); + where_bind_text(binds, bind_idx, params->kind); + } + if (params->namespace_name && params->namespace_name[0]) { + snprintf(clause, sizeof(clause), + "(CASE WHEN json_valid(n.properties) THEN json_extract(n.properties,'$.obj_key') " + "END) LIKE ('%%/%%/' || replace(replace(replace(?%d,'\\','\\\\'),'%%','\\%%')," + "'_','\\_') || '/%%') ESCAPE '\\'", + *bind_idx + SKIP_ONE); + *wlen = where_append(where, where_sz, *wlen, nparams, clause); + where_bind_text(binds, bind_idx, params->namespace_name); + } +} + /* Build basic WHERE clauses: project, label, name, file, qn patterns. */ static int search_where_basic(const cbm_search_params_t *params, char *where, int where_sz, int *wlen, int *nparams, search_bind_t *binds, int *bind_idx, @@ -6630,6 +6657,7 @@ static int search_where_basic(const cbm_search_params_t *params, char *where, in where_bind_text(binds, bind_idx, lp); } } + search_where_infra(params, where, where_sz, wlen, nparams, binds, bind_idx); return *nparams; } diff --git a/src/store/store.h b/src/store/store.h index 8721853c5..da4026d29 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -24,7 +24,13 @@ typedef struct cbm_store_read_scope cbm_store_read_scope_t; #define CBM_STORE_OK 0 #define CBM_STORE_ERR (-1) #define CBM_STORE_NOT_FOUND (-2) -#define CBM_INDEX_FORMAT_VERSION 1 +/* Persisted index format. A bump routes every existing index through one full + * rebuild (pipeline.c, format_change_reindex) that keeps ADR/project metadata. + * 1 #1108: File-node QNs keep the file extension. + * 2 Kubernetes objects: one Resource per YAML document, QN + * ::///; a + * Dockerfile. keeps its suffix in its QNs. */ +#define CBM_INDEX_FORMAT_VERSION 2 #define CBM_STORE_CANCELLED (-3) #define CBM_STORE_SCAN_LIMIT (-4) #define CBM_STORE_CALLBACK_ERR (-5) @@ -226,6 +232,8 @@ typedef struct { const char *sort_by; /* "relevance" / "name" / "degree", NULL = relevance */ bool case_sensitive; const char **exclude_labels; /* NULL-terminated array, or NULL */ + const char *kind; /* a Resource's kind property ("Deployment"), NULL = any */ + const char *namespace_name; /* the namespace of a Resource's obj_key, NULL = any */ } cbm_search_params_t; typedef struct { diff --git a/tests/fixtures/complexity/Bash/Makefile b/tests/fixtures/complexity/Bash/Makefile new file mode 100644 index 000000000..518656be5 --- /dev/null +++ b/tests/fixtures/complexity/Bash/Makefile @@ -0,0 +1,9 @@ +include common.mk + +.PHONY: all deploy + +all: build + $(MAKE) deploy + +deploy: + ./deploy.sh diff --git a/tests/fixtures/complexity/Bash/action.yml b/tests/fixtures/complexity/Bash/action.yml new file mode 100644 index 000000000..2f517178f --- /dev/null +++ b/tests/fixtures/complexity/Bash/action.yml @@ -0,0 +1,9 @@ +name: deploy +description: build and deploy +runs: + using: composite + steps: + - run: make -C "${{ github.action_path }}" all + shell: bash + - run: ${{ github.action_path }}/deploy.sh + shell: bash diff --git a/tests/fixtures/complexity/Bash/charts/web/Chart.yaml b/tests/fixtures/complexity/Bash/charts/web/Chart.yaml new file mode 100644 index 000000000..dc9e11e05 --- /dev/null +++ b/tests/fixtures/complexity/Bash/charts/web/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: web +version: 0.1.0 diff --git a/tests/fixtures/complexity/Bash/common.mk b/tests/fixtures/complexity/Bash/common.mk new file mode 100644 index 000000000..dc18e50aa --- /dev/null +++ b/tests/fixtures/complexity/Bash/common.mk @@ -0,0 +1,2 @@ +build: + ./scripts/build.sh diff --git a/tests/fixtures/complexity/Bash/deploy.sh b/tests/fixtures/complexity/Bash/deploy.sh new file mode 100644 index 000000000..3adf2dc97 --- /dev/null +++ b/tests/fixtures/complexity/Bash/deploy.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" + +./scripts/build.sh +kubectl apply -f manifests/ +helm upgrade --install web ./charts/web diff --git a/tests/fixtures/complexity/Bash/manifests/app.yaml b/tests/fixtures/complexity/Bash/manifests/app.yaml new file mode 100644 index 000000000..32f00c515 --- /dev/null +++ b/tests/fixtures/complexity/Bash/manifests/app.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web +spec: + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + containers: + - name: web + image: nginx:1.27 +--- +apiVersion: v1 +kind: Service +metadata: + name: web +spec: + selector: + app: web + ports: + - port: 80 diff --git a/tests/fixtures/complexity/Bash/scripts/build.sh b/tests/fixtures/complexity/Bash/scripts/build.sh new file mode 100644 index 000000000..083a16db1 --- /dev/null +++ b/tests/fixtures/complexity/Bash/scripts/build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +echo building diff --git a/tests/fixtures/complexity/Dockerfile/Dockerfile b/tests/fixtures/complexity/Dockerfile/Dockerfile new file mode 100644 index 000000000..985fd1488 --- /dev/null +++ b/tests/fixtures/complexity/Dockerfile/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:3.20 AS base +ENV TZ=UTC +FROM base +ENV APP_HOME=/app \ + # where the script lives + APP_MODE=web +COPY run.sh /usr/local/bin/run.sh +ENTRYPOINT ["/usr/local/bin/run.sh"] diff --git a/tests/fixtures/complexity/Dockerfile/Dockerfile.worker b/tests/fixtures/complexity/Dockerfile/Dockerfile.worker new file mode 100644 index 000000000..35c273d65 --- /dev/null +++ b/tests/fixtures/complexity/Dockerfile/Dockerfile.worker @@ -0,0 +1,4 @@ +FROM alpine:3.20 +ENV APP_MODE=worker +COPY run.sh /usr/local/bin/run.sh +CMD ["/usr/local/bin/run.sh"] diff --git a/tests/fixtures/complexity/Dockerfile/compose.yaml b/tests/fixtures/complexity/Dockerfile/compose.yaml new file mode 100644 index 000000000..eb41e0006 --- /dev/null +++ b/tests/fixtures/complexity/Dockerfile/compose.yaml @@ -0,0 +1,20 @@ +services: + web: + image: acme/cx-app:1-web + build: . + depends_on: + - db + environment: + DB_HOST: db + LOG_LEVEL: info + worker: + image: acme/cx-app:1-worker + build: + context: . + dockerfile: Dockerfile.worker + depends_on: + - db + environment: + - QUEUE + db: + image: postgres:16 diff --git a/tests/fixtures/complexity/Dockerfile/run.sh b/tests/fixtures/complexity/Dockerfile/run.sh new file mode 100644 index 000000000..5382ea94c --- /dev/null +++ b/tests/fixtures/complexity/Dockerfile/run.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "running $APP_MODE" diff --git a/tests/fixtures/complexity/Go/charts/operator/Chart.yaml b/tests/fixtures/complexity/Go/charts/operator/Chart.yaml new file mode 100644 index 000000000..a0cccb37b --- /dev/null +++ b/tests/fixtures/complexity/Go/charts/operator/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: operator +version: 0.1.0 diff --git a/tests/fixtures/complexity/Go/controller.go b/tests/fixtures/complexity/Go/controller.go new file mode 100644 index 000000000..bc88e26b7 --- /dev/null +++ b/tests/fixtures/complexity/Go/controller.go @@ -0,0 +1,14 @@ +package op + +import ( + ctrl "sigs.k8s.io/controller-runtime" +) + +type WidgetReconciler struct{} + +func (r *WidgetReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&Widget{}). + Owns(&Gadget{}). + Complete(r) +} diff --git a/tests/fixtures/complexity/Go/crd.yaml b/tests/fixtures/complexity/Go/crd.yaml new file mode 100644 index 000000000..9e1d2174f --- /dev/null +++ b/tests/fixtures/complexity/Go/crd.yaml @@ -0,0 +1,10 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.embedded.example.org +spec: + group: embedded.example.org + names: + kind: Widget + plural: widgets + scope: Namespaced diff --git a/tests/fixtures/complexity/Go/embed.go b/tests/fixtures/complexity/Go/embed.go new file mode 100644 index 000000000..99d9f2142 --- /dev/null +++ b/tests/fixtures/complexity/Go/embed.go @@ -0,0 +1,6 @@ +package op + +import "embed" + +//go:embed crd.yaml +var CRDs embed.FS diff --git a/tests/fixtures/complexity/Go/go.mod b/tests/fixtures/complexity/Go/go.mod new file mode 100644 index 000000000..2e5105206 --- /dev/null +++ b/tests/fixtures/complexity/Go/go.mod @@ -0,0 +1,3 @@ +module example.com/op + +go 1.22 diff --git a/tests/fixtures/complexity/Go/install.go b/tests/fixtures/complexity/Go/install.go new file mode 100644 index 000000000..89c7180c8 --- /dev/null +++ b/tests/fixtures/complexity/Go/install.go @@ -0,0 +1,8 @@ +package op + +import "helm.sh/helm/v3/pkg/chart/loader" + +func deployOperator() error { + _, err := loader.Load("charts/operator") + return err +} diff --git a/tests/fixtures/complexity/Go/types.go b/tests/fixtures/complexity/Go/types.go new file mode 100644 index 000000000..625acb8e5 --- /dev/null +++ b/tests/fixtures/complexity/Go/types.go @@ -0,0 +1,10 @@ +// +groupName=example.com +package op + +type Widget struct { + Name string +} + +type Gadget struct { + Name string +} diff --git a/tests/fixtures/complexity/HCL/charts/app/Chart.yaml b/tests/fixtures/complexity/HCL/charts/app/Chart.yaml new file mode 100644 index 000000000..4121cad31 --- /dev/null +++ b/tests/fixtures/complexity/HCL/charts/app/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: app +version: 0.1.0 diff --git a/tests/fixtures/complexity/HCL/main.tf b/tests/fixtures/complexity/HCL/main.tf new file mode 100644 index 000000000..14b399a79 --- /dev/null +++ b/tests/fixtures/complexity/HCL/main.tf @@ -0,0 +1,10 @@ +resource "helm_release" "this" { + name = "app" + chart = "${path.module}/charts/app" +} + +resource "helm_release" "ingress" { + name = "ingress" + repository = "https://kubernetes.github.io/ingress-nginx" + chart = "ingress-nginx" +} diff --git a/tests/fixtures/complexity/Kubernetes/app.yaml b/tests/fixtures/complexity/Kubernetes/app.yaml new file mode 100644 index 000000000..c4d0a839d --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/app.yaml @@ -0,0 +1,117 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web +spec: + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + serviceAccountName: web + volumes: + - name: config + configMap: + name: web-config + containers: + - name: web + image: example.com/web:1.0 + envFrom: + - secretRef: + name: web-env +--- +apiVersion: v1 +kind: Service +metadata: + name: web +spec: + selector: + app: web + ports: + - port: 80 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: web-config +data: + mode: production +--- +apiVersion: v1 +kind: Secret +metadata: + name: web-env +type: Opaque +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: web +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: web +spec: + rules: + - host: web.example.com + http: + paths: + - path: /api + pathType: Prefix + backend: + service: + name: web + port: + number: 80 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: web +spec: + minAvailable: 1 + selector: + matchLabels: + app: web +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: web +spec: + podSelector: + matchLabels: + app: web +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: web +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: web + minReplicas: 1 + maxReplicas: 3 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: web +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: web +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: web +subjects: + - kind: ServiceAccount + name: web diff --git a/tests/fixtures/complexity/Kubernetes/crd.yaml b/tests/fixtures/complexity/Kubernetes/crd.yaml new file mode 100644 index 000000000..e2617023a --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/crd.yaml @@ -0,0 +1,37 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.k8s.example.org +spec: + group: k8s.example.org + names: + kind: Widget + plural: widgets + scope: Namespaced +--- +apiVersion: k8s.example.org/v1 +kind: Widget +metadata: + name: sample +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: web +spec: + ca: + secretName: web-ca +--- +apiVersion: v1 +kind: Secret +metadata: + name: web-ca +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: web-tls +spec: + secretName: web-tls + issuerRef: + name: web diff --git a/tests/fixtures/complexity/Kubernetes/helmrelease.yaml b/tests/fixtures/complexity/Kubernetes/helmrelease.yaml new file mode 100644 index 000000000..33d74c55e --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/helmrelease.yaml @@ -0,0 +1,8 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: web +spec: + chartRef: + kind: OCIRepository + name: web-chart diff --git a/tests/fixtures/complexity/Kubernetes/kustomization.yaml b/tests/fixtures/complexity/Kubernetes/kustomization.yaml new file mode 100644 index 000000000..aea332b3f --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: shop +resources: + - app.yaml + - crd.yaml + - ocirepository.yaml + - helmrelease.yaml diff --git a/tests/fixtures/complexity/Kubernetes/ocirepository.yaml b/tests/fixtures/complexity/Kubernetes/ocirepository.yaml new file mode 100644 index 000000000..56289128e --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/ocirepository.yaml @@ -0,0 +1,9 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: web-chart +spec: + interval: 15m + ref: + tag: 1.2.3 + url: oci://ghcr.io/acme/charts/web-app diff --git a/tests/test_complexity.c b/tests/test_complexity.c index f20eac16e..775141cfc 100644 --- a/tests/test_complexity.c +++ b/tests/test_complexity.c @@ -67,6 +67,13 @@ * lsp_resolve.h — re-declared here to avoid pulling that header's statics). */ extern _Atomic uint64_t g_lsp_tail_lookups; extern _Atomic uint64_t g_lsp_tail_candidates; +/* Infra resolver candidate counters (pass_k8s.c, pass_operator.c, + * pass_shell.c, pass_terraform.c). */ +extern _Atomic uint64_t g_k8s_candidates; +extern _Atomic uint64_t g_op_candidates; +extern _Atomic uint64_t g_sh_candidates; +extern _Atomic uint64_t g_tf_candidates; +extern _Atomic uint64_t g_dk_hub_edges; /* ── Corpus scale ────────────────────────────────────────────────────── * Small on purpose: the gate reads exponents from ratios, not magnitudes. @@ -166,8 +173,9 @@ static const CxTemplate CX_TEMPLATES[] = { }; enum { CX_TEMPLATE_COUNT = sizeof(CX_TEMPLATES) / sizeof(CX_TEMPLATES[0]) }; -/* Fixture-dir provider: tests/fixtures/complexity// — every file in - * it is copied verbatim into each module dir. Lets a new language join the +/* Fixture-dir provider: tests/fixtures/complexity// — every file + * and directory in it (dot-names aside) is copied verbatim into each module + * dir. Lets a new language join the * guard by dropping fixtures, with no edit to this suite. */ static bool cx_fixture_dir_for(CBMLanguage lang, char *out, size_t cap) { const char *name = cbm_language_name(lang); @@ -203,6 +211,36 @@ static int cx_copy_file(const char *src, const char *dst) { return 0; } +/* A fixture entry into `dst`: a file, or a directory with its contents. */ +static int cx_copy_entry(const char *src, const char *dst, bool is_dir) { + if (!is_dir) { + return cx_copy_file(src, dst); + } + if (th_mkdir_p(dst) != 0) { + return -1; + } + cbm_dir_t *d = cbm_opendir(src); + if (!d) { + return -1; + } + int rc = 0; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(d)) != NULL) { + if (entry->name[0] == '.') { + continue; + } + char from[1024]; + char to[1024]; + snprintf(from, sizeof(from), "%s/%s", src, entry->name); + snprintf(to, sizeof(to), "%s/%s", dst, entry->name); + if (cx_copy_entry(from, to, entry->is_dir) != 0) { + rc = -1; + } + } + cbm_closedir(d); + return rc; +} + /* ── Corpus builder ──────────────────────────────────────────────────── */ static int cx_build_corpus(const char *root, int k_modules) { @@ -230,15 +268,8 @@ static int cx_build_corpus(const char *root, int k_modules) { * k module copies. Distinct paths make distinct modules/QNs, which is all * the independence argument needs. */ for (int lang = 0; lang < CBM_LANG_COUNT; lang++) { - bool templated = false; - for (int t = 0; t < CX_TEMPLATE_COUNT; t++) { - if (CX_TEMPLATES[t].lang == (CBMLanguage)lang) { - templated = true; - } - } - if (templated) { - continue; - } + /* a templated language may add fixtures too (Go: the operator + * corpus); they land in their own fx_ root */ char fixdir[512]; if (!cx_fixture_dir_for((CBMLanguage)lang, fixdir, sizeof(fixdir))) { continue; @@ -261,7 +292,7 @@ static int cx_build_corpus(const char *root, int k_modules) { char src[1024]; snprintf(src, sizeof(src), "%s/%s", fixdir, entry->name); snprintf(path, sizeof(path), "%s/%s", dir, entry->name); - (void)cx_copy_file(src, path); + (void)cx_copy_entry(src, path, entry->is_dir); } } cbm_closedir(d); @@ -342,6 +373,25 @@ static void cx_pass_sink(const char *line) { g_cx_passes[n].ms = atol(ee + 11); } +/* Fixture-provided infra corpora (tests/fixtures/complexity//), scoped + * by their copy root: Kubernetes manifests with a kustomization (and a Flux + * HelmRelease whose chartRef names an OCIRepository), a Go operator + * (controller, API types with +groupName, an embedded CRD, a chart loader), a + * shell deploy chain (composite CI action -> make -> scripts -> kubectl / helm + * of the copy's own manifests and chart), and Terraform helm_releases (one of + * a chart dir of the module, one of an external chart). An API + * group/kind is one identity across a repository: copies declaring the same + * one ARE joined (a CRD links to every package marking its group), so the + * fixtures' CRDs use groups no copy's Go code declares -- the guard measures + * accidental growth, not that by-design join. The container layer (Docker) + * has Dockerfiles with stage ENV, an ENTRYPOINT script and a compose project + * building two components of one repository; an image repository is one + * identity too, so every copy's builds of it join one Image -- that join + * must stay linear (the component split examines its edges). */ +static const char *const CX_INFRA_DIRS[] = {"fx_Kubernetes", "fx_Go", "fx_Bash", "fx_HCL", + "fx_Dockerfile"}; +enum { CX_INFRA_COUNT = sizeof(CX_INFRA_DIRS) / sizeof(CX_INFRA_DIRS[0]) }; + typedef struct { int nodes; int edges; @@ -357,6 +407,13 @@ typedef struct { uint64_t fallback_rows; uint64_t imp_nodes; /* symbols scored by the importance pass */ uint64_t imp_name_visits; /* same-name-group members visited by that pass */ + uint64_t k8s_candidates; /* objects the K8s resolvers examined */ + uint64_t op_candidates; /* Go types / dirs the operator pass examined */ + uint64_t sh_candidates; /* paths / make rules the shell layer examined */ + uint64_t tf_candidates; /* blocks the Terraform layer examined */ + uint64_t dk_hub_edges; /* Image edges the component split examined */ + int infra_nodes[CX_INFRA_COUNT]; + int infra_edges[CX_INFRA_COUNT]; double wall_s; } CxMetrics; @@ -379,6 +436,11 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { uint64_t fb0 = cbm_pp_lsp_linear_fallback_rows(); uint64_t in0 = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed); uint64_t iv0 = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed); + uint64_t kc0 = atomic_load_explicit(&g_k8s_candidates, memory_order_relaxed); + uint64_t oc0 = atomic_load_explicit(&g_op_candidates, memory_order_relaxed); + uint64_t sc0 = atomic_load_explicit(&g_sh_candidates, memory_order_relaxed); + uint64_t fc0 = atomic_load_explicit(&g_tf_candidates, memory_order_relaxed); + uint64_t dh0 = atomic_load_explicit(&g_dk_hub_edges, memory_order_relaxed); atomic_store_explicit(&g_cx_pass_count, 0, memory_order_relaxed); cbm_log_set_sink_ex(cx_pass_sink, CBM_LOG_SINK_TEE); @@ -416,6 +478,11 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->imp_nodes = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed) - in0; out->imp_name_visits = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed) - iv0; + out->k8s_candidates = atomic_load_explicit(&g_k8s_candidates, memory_order_relaxed) - kc0; + out->op_candidates = atomic_load_explicit(&g_op_candidates, memory_order_relaxed) - oc0; + out->sh_candidates = atomic_load_explicit(&g_sh_candidates, memory_order_relaxed) - sc0; + out->tf_candidates = atomic_load_explicit(&g_tf_candidates, memory_order_relaxed) - fc0; + out->dk_hub_edges = atomic_load_explicit(&g_dk_hub_edges, memory_order_relaxed) - dh0; cbm_store_t *s = cbm_store_open_path(db_path); if (!s) { @@ -427,6 +494,10 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->lang_nodes[t] = cbm_store_count_nodes_scoped(s, project, CX_TEMPLATES[t].dirname); out->lang_edges[t] = cbm_store_count_edges_scoped(s, project, CX_TEMPLATES[t].dirname); } + for (int i = 0; i < CX_INFRA_COUNT; i++) { + out->infra_nodes[i] = cbm_store_count_nodes_scoped(s, project, CX_INFRA_DIRS[i]); + out->infra_edges[i] = cbm_store_count_edges_scoped(s, project, CX_INFRA_DIRS[i]); + } cbm_store_close(s); return 0; } @@ -529,6 +600,59 @@ TEST(complexity_replicated_modules_scale_linearly) { PASS(); } +/* The infra resolvers (K8s references / selectors / CRDs / chartRefs, the Go + * operator's controller kinds, CRD types and loaded charts, the shell layer's + * manifest dirs and make rules, Terraform's helm_release blocks) examine + * candidates by name. Independent + * copies share every name, so a candidate set that is not scoped to the + * bundle / package / module grows with the copy count: the examined count + * goes ~4x per doubling while the edges stay ~2x. Both are gated, on the + * fixture corpora's own subtrees, and neither may be vacuous. */ +TEST(complexity_infra_resolution_is_linear) { + if (cx_measure_pair() != 0) { + FAIL("failed to build/run the complexity corpus pair"); + } + const CxMetrics *a = &g_cx_base; + const CxMetrics *b = &g_cx_doubled; + printf(" k8s_candidates %llu -> %llu op_candidates %llu -> %llu\n", + (unsigned long long)a->k8s_candidates, (unsigned long long)b->k8s_candidates, + (unsigned long long)a->op_candidates, (unsigned long long)b->op_candidates); + ASSERT_GT((long long)a->k8s_candidates, 0); + ASSERT_GT((long long)a->op_candidates, 0); + double kr = cx_ratio((double)b->k8s_candidates, (double)a->k8s_candidates); + double orr = cx_ratio((double)b->op_candidates, (double)a->op_candidates); + printf(" k8s_candidates ratio %.2f op_candidates ratio %.2f\n", kr, orr); + ASSERT_TRUE(kr >= CX_RATIO_LO && kr <= CX_RATIO_HI); + ASSERT_TRUE(orr >= CX_RATIO_LO && orr <= CX_RATIO_HI); + printf(" sh_candidates %llu -> %llu tf_candidates %llu -> %llu\n", + (unsigned long long)a->sh_candidates, (unsigned long long)b->sh_candidates, + (unsigned long long)a->tf_candidates, (unsigned long long)b->tf_candidates); + ASSERT_GT((long long)a->sh_candidates, 0); + ASSERT_GT((long long)a->tf_candidates, 0); + double sr = cx_ratio((double)b->sh_candidates, (double)a->sh_candidates); + double fr = cx_ratio((double)b->tf_candidates, (double)a->tf_candidates); + printf(" sh_candidates ratio %.2f tf_candidates ratio %.2f\n", sr, fr); + ASSERT_TRUE(sr >= CX_RATIO_LO && sr <= CX_RATIO_HI); + ASSERT_TRUE(fr >= CX_RATIO_LO && fr <= CX_RATIO_HI); + printf(" dk_hub_edges %llu -> %llu\n", (unsigned long long)a->dk_hub_edges, + (unsigned long long)b->dk_hub_edges); + ASSERT_GT((long long)a->dk_hub_edges, 0); + double dr = cx_ratio((double)b->dk_hub_edges, (double)a->dk_hub_edges); + printf(" dk_hub_edges ratio %.2f\n", dr); + ASSERT_TRUE(dr >= CX_RATIO_LO && dr <= CX_RATIO_HI); + for (int i = 0; i < CX_INFRA_COUNT; i++) { + ASSERT_GT(a->infra_nodes[i], 0); + ASSERT_GT(a->infra_edges[i], 0); + double nr = cx_ratio(b->infra_nodes[i], a->infra_nodes[i]); + double er = cx_ratio(b->infra_edges[i], a->infra_edges[i]); + printf(" %-14s nodes %d -> %d (%.2f) edges %d -> %d (%.2f)\n", CX_INFRA_DIRS[i], + a->infra_nodes[i], b->infra_nodes[i], nr, a->infra_edges[i], b->infra_edges[i], er); + ASSERT_TRUE(nr >= CX_RATIO_LO && nr <= CX_RATIO_HI); + ASSERT_TRUE(er >= CX_RATIO_LO && er <= CX_RATIO_HI); + } + PASS(); +} + /* Σ per-file registry work must be linear in independent copies. This is the * counter that measured ~4x/doubling on the v0.10.x Java path (defs_per_file * tracked defs_total — #1669). Guarded against vacuous passes: the base run @@ -845,6 +969,7 @@ SUITE(complexity) { RUN_TEST(complexity_rescript_error_recovery_lexing_is_linear); RUN_TEST(complexity_replicated_modules_scale_linearly); RUN_TEST(complexity_perfile_registry_work_is_linear); + RUN_TEST(complexity_infra_resolution_is_linear); RUN_TEST(complexity_importance_scoring_is_linear); RUN_TEST(complexity_shared_package_growth_stays_linear); RUN_TEST(complexity_throughput_report_written); diff --git a/tests/test_configlink.c b/tests/test_configlink.c index 1b1ea216a..c7248a8bc 100644 --- a/tests/test_configlink.c +++ b/tests/test_configlink.c @@ -160,14 +160,16 @@ TEST(configlink_key_symbol_short_key_skipped) { } /* Go: TestConfigKeySymbol_CamelCaseNormalization - * config.json has maxRetryCount, handler.go has getMaxRetryCount() */ + * config.json has max_retry_count, handler.go has maxRetryCount: camelCase and + * snake_case normalize to the same key. (A JSON key links by exact name only, + * see configlink_key_symbol_infra_exact_only, so the symbol names it exactly.) */ TEST(configlink_key_symbol_camel_case) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); - cbm_gbuf_upsert_node(gb, "Variable", "maxRetryCount", "test.config.maxRetryCount", + cbm_gbuf_upsert_node(gb, "Variable", "max_retry_count", "test.config.max_retry_count", "config.json", 0, 0, NULL); - cbm_gbuf_upsert_node(gb, "Function", "getMaxRetryCount", "test.handler.getMaxRetryCount", + cbm_gbuf_upsert_node(gb, "Function", "maxRetryCount", "test.handler.maxRetryCount", "handler.go", 0, 0, NULL); run_configlink(gb, "test", NULL); @@ -180,6 +182,124 @@ TEST(configlink_key_symbol_camel_case) { PASS(); } +static int count_key_symbol_edges(cbm_gbuf_t *gb) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "CONFIGURES", &edges, &count); + int n = 0; + for (int i = 0; i < count; i++) { + if (edges[i]->properties_json && + strstr(edges[i]->properties_json, "\"strategy\":\"key_symbol\"")) { + n++; + } + } + return n; +} + +/* Keys of infrastructure files (YAML, JSON, HCL, tfvars) link by exact name + * only: the substring guess linked a chart's `node_selector` to every + * function whose name contains it. In 17 infra repositories 4 of 40 sampled + * key_symbol edges were right, all exact and in the same directory tree. */ +TEST(configlink_key_symbol_infra_exact_only) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + cbm_gbuf_upsert_node(gb, "Variable", "node_selector", "test.charts.app.values.node_selector", + "charts/app/values.yaml", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Function", "GetPodNodeSelectorConfig", + "test.charts.app.gen.GetPodNodeSelectorConfig", "charts/app/gen.go", 0, 0, + NULL); + run_configlink(gb, "test", NULL); + ASSERT_EQ(count_key_symbol_edges(gb), 0); + cbm_gbuf_free(gb); + PASS(); +} + +/* ...and only within the config file's own component (its directory tree): + * a stage's JSON schema does not configure an unrelated module's variable. */ +TEST(configlink_key_symbol_infra_scoped_to_component) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + int64_t cfg = cbm_gbuf_upsert_node(gb, "Variable", "cluster_name", + "test.svc_a.config.cluster_name", "svc-a/config.yaml", 0, + 0, NULL); + int64_t inside = cbm_gbuf_upsert_node(gb, "Function", "clusterName", + "test.svc_a.cmd.main.clusterName", "svc-a/cmd/main.go", + 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Function", "clusterName", "test.svc_b.main.clusterName", + "svc-b/main.go", 0, 0, NULL); + run_configlink(gb, "test", NULL); + ASSERT_EQ(count_key_symbol_edges(gb), 1); + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "CONFIGURES", &edges, &count); + ASSERT_EQ(count, 1); + ASSERT_EQ(edges[0]->source_id, inside); + ASSERT_EQ(edges[0]->target_id, cfg); + cbm_gbuf_free(gb); + PASS(); +} + +/* A conventional config directory (config/, conf/, settings/, ...) belongs to + * the component around it. */ +TEST(configlink_key_symbol_config_dir_component) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + cbm_gbuf_upsert_node(gb, "Variable", "cluster_name", "test.config.app.cluster_name", + "config/app.yaml", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Function", "clusterName", "test.pkg.app.clusterName", "pkg/app.go", + 0, 0, NULL); + run_configlink(gb, "test", NULL); + ASSERT_EQ(count_key_symbol_edges(gb), 1); + cbm_gbuf_free(gb); + PASS(); +} + +/* A Terraform/OpenTofu variable is what a config value sets: a tfvars.json key + * links to `variable ""` of its component (the block's Class is named + * `variable.`), not to outputs, modules or resources, and not to a + * variable of another component (kubespray: contrib/terraform/gcp/tfvars.json + * sets contrib/terraform/gcp/modules/.../variables.tf). */ +TEST(configlink_key_symbol_terraform_variable) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + int64_t cfg = cbm_gbuf_upsert_node(gb, "Variable", "ssh_whitelist", + "test.gcp.tfvars.ssh_whitelist", + "contrib/terraform/gcp/tfvars.json", 0, 0, NULL); + int64_t var = cbm_gbuf_upsert_node( + gb, "Class", "variable.ssh_whitelist", "test.gcp.modules.cluster.variable.ssh_whitelist", + "contrib/terraform/gcp/modules/cluster/variables.tf", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Class", "output.ssh_whitelist", + "test.gcp.modules.cluster.output.ssh_whitelist", + "contrib/terraform/gcp/modules/cluster/outputs.tf", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Class", "variable.ssh_whitelist", + "test.hetzner.variable.ssh_whitelist", + "contrib/terraform/hetzner/variables.tf", 0, 0, NULL); + run_configlink(gb, "test", NULL); + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "CONFIGURES", &edges, &count); + ASSERT_EQ(count, 1); + ASSERT_EQ(edges[0]->source_id, var); + ASSERT_EQ(edges[0]->target_id, cfg); + cbm_gbuf_free(gb); + PASS(); +} + +/* No count of config keys decides what links: the collector stopped at 4,096. */ +TEST(configlink_key_symbol_no_config_cap) { + enum { KEYS = 4200 }; + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + for (int i = 0; i < KEYS; i++) { + char name[CBM_SZ_64]; + char qn[CBM_SZ_128]; + snprintf(name, sizeof(name), "limit_alpha_%04d", i); + snprintf(qn, sizeof(qn), "test.config.%s", name); + cbm_gbuf_upsert_node(gb, "Variable", name, qn, "config.toml", 0, 0, NULL); + snprintf(qn, sizeof(qn), "test.main.%s", name); + cbm_gbuf_upsert_node(gb, "Function", name, qn, "main.go", 0, 0, NULL); + } + run_configlink(gb, "test", NULL); + ASSERT_EQ(count_key_symbol_edges(gb), KEYS); + cbm_gbuf_free(gb); + PASS(); +} + /* Go: TestConfigKeySymbol_NoFalsePositive * config.toml has url — 1-token key, should not match parseURL */ TEST(configlink_key_symbol_no_false_positive) { @@ -366,17 +486,18 @@ static cbm_gbuf_t *build_oversized_corpus(bool reverse) { return gb; } -/* REGRESSION: collect_config_entries/collect_code_entries fill fixed-capacity - * arrays and stop at the cap, walking gbuf label indexes in insertion order — - * which under parallel extraction is worker-merge order and varies run to run. - * Sorting candidates canonically before the cap is what keeps the surviving - * set, and therefore the emitted CONFIGURES edges, a pure function of the - * inputs. This test stands in for the scheduling variance by feeding the same - * corpus in two insertion orders: the emitted edge set must be identical. */ +/* REGRESSION: the collectors walk gbuf label indexes in insertion order -- + * under parallel extraction worker-merge order, which varies run to run. + * They filled fixed-capacity arrays and stopped at a cap, so the surviving + * set depended on that order until candidates were sorted first; now no cap + * cuts the set at all. This test stands in for the scheduling variance by + * feeding the same corpus in two insertion orders: the emitted edge set must + * be identical -- and complete, all 9,000 candidates past the old 8,192 cap. */ TEST(configlink_candidate_truncation_is_order_independent) { cbm_gbuf_t *fwd = build_oversized_corpus(false); run_configlink(fwd, "test", NULL); char *fp_fwd = configures_fingerprint(fwd); + int fwd_edges = count_key_symbol_edges(fwd); cbm_gbuf_t *rev = build_oversized_corpus(true); run_configlink(rev, "test", NULL); @@ -384,7 +505,7 @@ TEST(configlink_candidate_truncation_is_order_independent) { ASSERT_TRUE(fp_fwd != NULL); ASSERT_TRUE(fp_rev != NULL); - ASSERT_TRUE(fp_fwd[0] != '\0'); /* the cap must actually have been exercised */ + ASSERT_EQ(fwd_edges, CL_DET_CANDIDATES); ASSERT_STR_EQ(fp_fwd, fp_rev); free(fp_fwd); @@ -403,6 +524,11 @@ SUITE(configlink) { RUN_TEST(configlink_key_symbol_short_key_skipped); RUN_TEST(configlink_key_symbol_camel_case); RUN_TEST(configlink_key_symbol_no_false_positive); + RUN_TEST(configlink_key_symbol_infra_exact_only); + RUN_TEST(configlink_key_symbol_infra_scoped_to_component); + RUN_TEST(configlink_key_symbol_config_dir_component); + RUN_TEST(configlink_key_symbol_no_config_cap); + RUN_TEST(configlink_key_symbol_terraform_variable); /* Strategy 2: Dependency → Import */ RUN_TEST(configlink_dep_import_package_json); diff --git a/tests/test_cross_repo.c b/tests/test_cross_repo.c index 23d74fb5b..9d318f8c0 100644 --- a/tests/test_cross_repo.c +++ b/tests/test_cross_repo.c @@ -574,7 +574,10 @@ static bool cross_repo_seed_bounded_scan(const cross_repo_fixture_t *fixture, return ok; } -TEST(cross_repo_scan_bound_counts_examined_rows_not_matches) { +/* No row count decides which calls are matched: the scans stopped after 4,096 + * rows, so a call examined after 4,096 others never got its CROSS_HTTP_CALLS + * edge even though the target serves its route. */ +TEST(cross_repo_matches_calls_past_4096_rows) { cross_repo_fixture_t fixture; bool setup = cross_repo_fixture_begin(&fixture) && cross_repo_seed_bounded_scan(&fixture, "bounded-source", "bounded-target"); @@ -589,7 +592,7 @@ TEST(cross_repo_scan_bound_counts_examined_rows_not_matches) { ASSERT_FALSE(result.failed); ASSERT_EQ(result.projects_scanned, 1); - ASSERT_EQ(result.http_edges, 0); + ASSERT_EQ(result.http_edges, 1); PASS(); } @@ -952,7 +955,7 @@ SUITE(cross_repo) { RUN_TEST(cross_repo_named_pre768_target_fails_before_cleanup_issue2133); RUN_TEST(cross_repo_status_distinguishes_never_run_from_ran_without_links); RUN_TEST(cross_repo_wildcard_reports_skipped_pre768_store); - RUN_TEST(cross_repo_scan_bound_counts_examined_rows_not_matches); + RUN_TEST(cross_repo_matches_calls_past_4096_rows); RUN_TEST(cross_repo_propagates_delete_failure); RUN_TEST(cross_repo_failed_bidirectional_insert_is_not_counted); RUN_TEST(cross_repo_cancel_mid_run_keeps_completed_target_and_stops_before_later_target); diff --git a/tests/test_fqn.c b/tests/test_fqn.c index f9d17fa0c..013bea6a9 100644 --- a/tests/test_fqn.c +++ b/tests/test_fqn.c @@ -7,6 +7,8 @@ #include "test_framework.h" #include "../src/pipeline/pipeline.h" #include "../src/foundation/str_util.h" +#include "../src/foundation/arena.h" +#include "helpers.h" #include #include @@ -27,6 +29,25 @@ /* ── Basic: project + path + name ─────────────────────────────── */ +/* A Dockerfile variant keeps its suffix in both QN implementations (the + * pipeline's fqn.c, the extractor's helpers.c) -- else it gets the QN of the + * plain Dockerfile beside it. */ +TEST(fqn_dockerfile_variant_keeps_suffix) { + ASSERT_FQN(cbm_pipeline_fqn_compute("p", "svc/Dockerfile.debug", "GOOS"), + "p.svc.Dockerfile.debug.GOOS"); + ASSERT_FQN(cbm_pipeline_fqn_module("p", "svc/Containerfile.arm64"), "p.svc.Containerfile.arm64"); + ASSERT_FQN(cbm_pipeline_fqn_module("p", "svc/Dockerfile"), "p.svc.Dockerfile"); + ASSERT_FQN(cbm_pipeline_fqn_module("p", "svc/app.py"), "p.svc.app"); + CBMArena a; + cbm_arena_init(&a); + ASSERT_STR_EQ(cbm_fqn_compute(&a, "p", "svc/Dockerfile.debug", "GOOS"), + "p.svc.Dockerfile.debug.GOOS"); + ASSERT_STR_EQ(cbm_fqn_module(&a, "p", "svc/Containerfile.arm64"), "p.svc.Containerfile.arm64"); + ASSERT_STR_EQ(cbm_fqn_module(&a, "p", "svc/app.py"), "p.svc.app"); + cbm_arena_destroy(&a); + PASS(); +} + TEST(fqn_compute_basic_go) { ASSERT_FQN(cbm_pipeline_fqn_compute("myproj", "main.go", "main"), "myproj.main.main"); PASS(); @@ -673,6 +694,7 @@ TEST(project_name_length_capped_issue624) { SUITE(fqn) { /* fqn_compute: basic extensions */ RUN_TEST(fqn_compute_basic_go); + RUN_TEST(fqn_dockerfile_variant_keeps_suffix); RUN_TEST(fqn_compute_basic_py); RUN_TEST(fqn_compute_basic_ts); RUN_TEST(fqn_compute_basic_js); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index ed734f052..f2a69bf7d 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -138,9 +139,12 @@ typedef struct { int modules; /* Module label */ int classes; /* Class label */ int resources; /* Resource label (k8s) */ + int charts; /* Chart label (helm) */ + int variables; /* Variable label */ int imports; /* IMPORTS edges */ int depends_on; /* DEPENDS_ON edges */ int infra_maps; /* INFRA_MAPS edges */ + int routes; /* Route label */ } GpfMetrics; static GpfMetrics gpf_metrics_files(const GpfFile *files, int nfiles) { @@ -154,9 +158,12 @@ static GpfMetrics gpf_metrics_files(const GpfFile *files, int nfiles) { m.modules = gpf_count_label(store, lp.project, "Module"); m.classes = gpf_count_label(store, lp.project, "Class"); m.resources = gpf_count_label(store, lp.project, "Resource"); + m.charts = gpf_count_label(store, lp.project, "Chart"); + m.variables = gpf_count_label(store, lp.project, "Variable"); m.imports = cbm_store_count_edges_by_type(store, lp.project, "IMPORTS"); m.depends_on = cbm_store_count_edges_by_type(store, lp.project, "DEPENDS_ON"); m.infra_maps = cbm_store_count_edges_by_type(store, lp.project, "INFRA_MAPS"); + m.routes = gpf_count_label(store, lp.project, "Route"); } gpf_cleanup(&lp, store); return m; @@ -272,6 +279,53 @@ TEST(probe_hcl_no_spurious_imports) { PASS(); } +/* Only a top-level block (resource, data, module, variable, output, ...) is an + * entity; blocks nested in it (lifecycle, dynamic, content, the resource's own + * argument blocks) are part of it. cloud-foundation-fabric minted 1,342 + * `dynamic` and 205 `content` Classes next to 898 resources. */ +TEST(probe_hcl_nested_blocks_are_not_classes) { + GpfMetrics m = gpf_metrics("main.tf", + "resource \"google_container_cluster\" \"main\" {\n" + " name = \"main\"\n" + " addons_config {\n" + " http_load_balancing { disabled = false }\n" + " }\n" + " dynamic \"node_pool\" {\n" + " for_each = var.pools\n" + " content {\n" + " name = node_pool.value.name\n" + " }\n" + " }\n" + " lifecycle {\n" + " ignore_changes = [node_pool]\n" + " }\n" + "}\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.classes, 1); + PASS(); +} + +/* OpenTofu configuration (.tofu) is indexed like .tf. */ +TEST(probe_tofu_file_indexed) { + GpfMetrics m = gpf_metrics("main.tofu", + "resource \"aws_s3_bucket\" \"logs\" {\n" + " bucket = \"logs\"\n" + "}\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.classes, 1); + PASS(); +} + +/* A variable file's assignments are its content. */ +TEST(probe_tfvars_file_indexed) { + GpfMetrics m = gpf_metrics("prod.tfvars", + "region = \"europe-west1\"\n" + "node_count = 3\n"); + ASSERT_TRUE(m.ok); + ASSERT_TRUE(m.variables >= 2); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * GROUP 2 — Kubernetes manifests (.yaml with apiVersion:) * @@ -351,9 +405,10 @@ TEST(probe_k8s_two_manifests) { PASS(); } -/* K8s: cross-manifest reference via INFRA_MAPS. - * RED: the single-path pipeline does not emit INFRA_MAPS edges for k8s refs; - * pass_k8s only emits Resource nodes (no ref-resolution between manifests). */ +/* K8s: a Service routes to the workload whose pod template carries every + * label of its selector (INFRA_MAPS Service -> workload). The fixture had no + * pod labels and linked only through a "selector value == workload name" + * guess that Kubernetes does not make. */ TEST(probe_k8s_infra_maps_edge) { static const GpfFile files[] = { {"deploy.yaml", @@ -363,6 +418,9 @@ TEST(probe_k8s_infra_maps_edge) { " name: frontend\n" "spec:\n" " template:\n" + " metadata:\n" + " labels:\n" + " app: frontend\n" " spec:\n" " containers:\n" " - name: frontend\n" @@ -378,8 +436,211 @@ TEST(probe_k8s_infra_maps_edge) { }; GpfMetrics m = gpf_metrics_files(files, 2); ASSERT_TRUE(m.ok); - /* RED: inter-manifest INFRA_MAPS edges not modeled by the current k8s pass. */ - ASSERT_TRUE(m.infra_maps >= 1); /* expected RED — no k8s selector → INFRA_MAPS */ + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + +/* A selector is the AND of its pairs: `tier: back` pods are not selected by a + * `{app: web, tier: front}` Service. */ +TEST(probe_k8s_selector_requires_all_pairs) { + static const GpfFile files[] = { + {"front.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web-front\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n" + " tier: front\n"}, + {"back.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web-back\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n" + " tier: back\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " app: web\n" + " tier: front\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 3); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + +/* Only the pod template's labels are selectable; the workload's own + * metadata.labels are not. */ +TEST(probe_k8s_selector_ignores_workload_metadata_labels) { + static const GpfFile files[] = { + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: api\n" + " labels:\n" + " app: web\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: api\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " app: web\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 0); + PASS(); +} + +/* A workload whose NAME equals a selector value is not selected by it. */ +TEST(probe_k8s_selector_no_name_guess) { + static const GpfFile files[] = { + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: frontend\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: fe\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: frontend\n" + "spec:\n" + " selector:\n" + " app: frontend\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 0); + PASS(); +} + +/* `component: web` does not select pods labelled `app: web`. */ +TEST(probe_k8s_selector_key_matters) { + static const GpfFile files[] = { + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: api\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " component: web\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 0); + PASS(); +} + +/* A Service selects pods of its own namespace only. */ +TEST(probe_k8s_selector_namespace_scoped) { + static const GpfFile files[] = { + {"deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + " namespace: team-b\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n"}, + {"service.yaml", + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + " namespace: team-a\n" + "spec:\n" + " selector:\n" + " app: web\n"} + }; + GpfMetrics m = gpf_metrics_files(files, 2); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 0); + PASS(); +} + +/* Selector linking recorded at most 512 manifests; a Service and workload + * indexed after that limit (deckhouse has 2,003 manifest files) were never + * linked. No count of manifests may decide graph content. */ +TEST(probe_k8s_selector_beyond_512_manifests) { + enum { FILLER = 600 }; + GpfFile *files = calloc(FILLER + 2, sizeof(*files)); + char (*names)[32] = calloc(FILLER, sizeof(*names)); + ASSERT_NOT_NULL(files); + ASSERT_NOT_NULL(names); + for (int i = 0; i < FILLER; i++) { + snprintf(names[i], sizeof(names[i]), "a/cm-%03d.yaml", i); + files[i].name = names[i]; + files[i].content = "apiVersion: v1\n" + "kind: ConfigMap\n" + "metadata:\n" + " name: filler\n" + "data:\n" + " k: v\n"; + } + files[FILLER].name = "z/deploy.yaml"; + files[FILLER].content = "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: late\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: late\n"; + files[FILLER + 1].name = "z/service.yaml"; + files[FILLER + 1].content = "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: late\n" + "spec:\n" + " selector:\n" + " app: late\n"; + GpfMetrics m = gpf_metrics_files(files, FILLER + 2); + free(names); + free(files); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.infra_maps, 1); PASS(); } @@ -470,6 +731,2846 @@ TEST(probe_kustomize_with_patches) { PASS(); } +/* A comment between `---` and the mapping (the `# yaml-language-server: + * $schema=...` line editors add) is the document's first named child; the + * manifest behind it is still a manifest. onedr0p/home-ops opens 428 of its + * manifests this way and got a Resource for 26 of 390 objects. */ +TEST(probe_k8s_manifest_after_leading_comment) { + GpfMetrics m = gpf_metrics("helmrelease.yaml", + "---\n" + "# yaml-language-server: $schema=https://example.com/helmrelease.json\n" + "apiVersion: helm.toolkit.fluxcd.io/v2\n" + "kind: HelmRelease\n" + "metadata:\n" + " name: echo\n" + "spec:\n" + " interval: 1h\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 1); + PASS(); +} + +/* A comment between a key and its block (`metadata: # owned by ...`) sits + * ahead of the value; the object still has its name. */ +TEST(probe_k8s_manifest_metadata_after_key_line_comment) { + GpfMetrics m = gpf_metrics("deploy.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata: # owned by the platform team\n" + " name: web\n" + "spec:\n" + " replicas: 1\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 1); + PASS(); +} + +/* ── Identity: every document is an object (IDENTITY.md) ──────────── */ + +/* The QN and properties of the Resource named `name` after indexing `files`; + * false when there is no such node or it is not unique. */ +static bool gpf_resource(const GpfFile *files, int nfiles, const char *name, char *qn, + size_t qn_sz, char *props, size_t props_sz) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + bool found = false; + int hits = 0; + cbm_node_t *nodes = NULL; + int count = 0; + if (store && + cbm_store_find_nodes_by_label(store, lp.project, "Resource", &nodes, &count) == + CBM_STORE_OK) { + for (int i = 0; i < count; i++) { + if (nodes[i].name && strcmp(nodes[i].name, name) == 0) { + hits++; + const char *q = nodes[i].qualified_name ? nodes[i].qualified_name : ""; + const char *sep = strstr(q, "::"); + snprintf(qn, qn_sz, "%s", sep ? sep : q); /* the part after the file base */ + snprintf(props, props_sz, "%s", + nodes[i].properties_json ? nodes[i].properties_json : "{}"); + } + } + cbm_store_free_nodes(nodes, count); + found = hits == 1; + } + gpf_cleanup(&lp, store); + return found; +} + +/* A multi-document manifest is a list of objects: kubernetes-sigs and + * microservices-demo ship a Deployment and its Service in one file, and + * 3,139 objects of the 17 infra corpora sat behind a file's first `---`. */ +TEST(probe_k8s_every_document_is_an_object) { + GpfMetrics m = gpf_metrics("app.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "---\n" + "apiVersion: v1\n" + "kind: ConfigMap\n" + "metadata:\n" + " name: web-config\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 3); + PASS(); +} + +/* ...and the Service in the second document selects the Deployment in the + * first. */ +TEST(probe_k8s_selector_across_documents) { + GpfMetrics m = gpf_metrics("app.yaml", + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: web\n" + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " selector:\n" + " app: web\n"); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.resources, 2); + ASSERT_EQ(m.infra_maps, 1); + PASS(); +} + +/* ── Secret hygiene: config values that are secret material stay out ── */ + +/* Does any node name / QN / properties or edge properties contain `needle`? */ +static int gpf_graph_mentions(const GpfFile *files, int nfiles, const char *needle) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int hits = -1; + if (store) { + hits = 0; + static const char *const sql[] = { + "SELECT COUNT(*) FROM nodes WHERE project=?1 AND (instr(name,?2) OR " + "instr(qualified_name,?2) OR instr(properties,?2))", + "SELECT COUNT(*) FROM edges WHERE project=?1 AND instr(properties,?2)"}; + for (size_t q = 0; q < sizeof(sql) / sizeof(sql[0]); q++) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql[q], -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 2, needle, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + hits += sqlite3_column_int(st, 0); + } + } + sqlite3_finalize(st); + } + } + gpf_cleanup(&lp, store); + return hits; +} + +/* A Secret's stringData is secret material, whatever its keys are called. */ +TEST(probe_secret_stringdata_mints_no_route) { + static const GpfFile files[] = {{"alerting-secret.yaml", + "apiVersion: v1\n" + "kind: Secret\n" + "metadata:\n" + " name: alerting\n" + "stringData:\n" + " webhook_url: https://hooks.example.com/T0/B0/s3cr3tpath\n"}}; + ASSERT_EQ(gpf_graph_mentions(files, 1, "s3cr3tpath"), 0); + PASS(); +} + +/* An endpoint keeps its path; the credentials and query riding on it stay + * out of the graph. */ +TEST(probe_endpoint_url_loses_credentials) { + static const GpfFile files[] = {{"subscription.yaml", + "subscription:\n" + " push_endpoint: https://svc:hunter2@svc.example.com/push" + "?token=abc123\n"}}; + GpfMetrics m = gpf_metrics_files(files, 1); + ASSERT_TRUE(m.ok); + ASSERT_EQ(m.routes, 1); + ASSERT_EQ(gpf_graph_mentions(files, 1, "hunter2"), 0); + ASSERT_EQ(gpf_graph_mentions(files, 1, "abc123"), 0); + ASSERT_TRUE(gpf_graph_mentions(files, 1, "https://svc.example.com/push") > 0); + PASS(); +} + +/* .tfvars assigns values -- a variable file is where secrets live; the + * declaration in the .tf is what the graph keeps. */ +TEST(probe_tfvars_values_mint_no_route) { + static const GpfFile files[] = { + {"prod.tfvars", "push_endpoint = \"https://svc.example.com/tfvars-only-path\"\n"}}; + ASSERT_EQ(gpf_graph_mentions(files, 1, "tfvars-only-path"), 0); + PASS(); +} + +/* ── Object references (Phase 1, PHASE1.md) ────────────────────────── */ + +/* Edges of `type` between two Resources whose properties carry `prop` + * (e.g. "\"ref\":\"volume\""); -1 when indexing failed. */ +static int gpf_resource_edges(const GpfFile *files, int nfiles, const char *type, + const char *prop) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int n = -1; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id " + "JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 AND e.type=?2 " + "AND s.label='Resource' AND t.label='Resource' AND instr(e.properties,?3)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 2, type, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 3, prop, -1, SQLITE_STATIC); + n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return n; +} + +static const char k_pod_refs_app[] = + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: web\n" + "spec:\n" + " template:\n" + " spec:\n" + " serviceAccountName: web-sa\n" + " imagePullSecrets:\n" + " - name: registry-creds\n" + " volumes:\n" + " - name: cfg\n" + " configMap:\n" + " name: web-config\n" + " - name: data\n" + " persistentVolumeClaim:\n" + " claimName: web-data\n" + " containers:\n" + " - name: web\n" + " env:\n" + " - name: DB_PASSWORD\n" + " valueFrom:\n" + " secretKeyRef:\n" + " name: web-db\n" + " key: password\n" + " envFrom:\n" + " - configMapRef:\n" + " name: web-env\n" + "---\n" + "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: web-sa\n" + "---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: registry-creds\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: web-config\n" + "---\n" + "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: web-data\n" + "---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: web-db\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: web-env\n"; + +/* A workload's pod spec names the objects it runs with: its service + * account, pull secret, mounted ConfigMap and claim, env sources. */ +TEST(probe_k8s_pod_spec_references) { + static const GpfFile files[] = {{"app.yaml", k_pod_refs_app}}; + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"serviceAccountName\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"imagePullSecrets\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"volume\""), 2); + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"env\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 1, "USAGE", "\"ref\":\"envFrom\""), 1); + PASS(); +} + +/* RBAC: a binding uses the role it names and the service accounts it + * grants; an Ingress routes to its backend Service; an HPA scales its + * target; a webhook configuration calls its Service. */ +TEST(probe_k8s_rbac_ingress_hpa_webhook_references) { + static const GpfFile files[] = { + {"rbac.yaml", + "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: reader\n" + "---\n" + "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: web-sa\n namespace: prod\n" + "---\n" + "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\n" + "metadata:\n name: web-reader\n" + "roleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: reader\n" + "subjects:\n - kind: ServiceAccount\n name: web-sa\n namespace: prod\n"}, + {"web.yaml", + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n namespace: prod\n" + "---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: prod\n" + "---\n" + "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: web\n" + " namespace: prod\nspec:\n rules:\n - host: web.example.com\n http:\n" + " paths:\n - path: /\n pathType: Prefix\n" + " backend:\n service:\n name: web\n" + " port:\n number: 80\n" + "---\n" + "apiVersion: autoscaling/v2\nkind: HorizontalPodAutoscaler\nmetadata:\n name: web\n" + " namespace: prod\nspec:\n scaleTargetRef:\n apiVersion: apps/v1\n" + " kind: Deployment\n name: web\n" + "---\n" + "apiVersion: admissionregistration.k8s.io/v1\nkind: ValidatingWebhookConfiguration\n" + "metadata:\n name: web-hook\nwebhooks:\n - name: validate.web.example.com\n" + " clientConfig:\n service:\n name: web\n namespace: prod\n"}}; + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"roleRef\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"subject\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "INFRA_MAPS", "\"kind\":\"ingress\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"scaleTarget\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "INFRA_MAPS", "\"kind\":\"webhook\""), 1); + PASS(); +} + +/* A custom resource is an instance of the CRD that declares its group and + * kind; a reference to an object that is not in the repo draws nothing. */ +TEST(probe_k8s_cr_uses_its_crd_and_unresolved_draws_nothing) { + static const GpfFile files[] = { + {"crd.yaml", + "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\n" + "metadata:\n name: widgets.example.com\nspec:\n group: example.com\n" + " names:\n kind: Widget\n plural: widgets\n scope: Namespaced\n"}, + {"widget.yaml", + "apiVersion: example.com/v1\nkind: Widget\nmetadata:\n name: w1\n" + "---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: lonely\nspec:\n" + " template:\n spec:\n serviceAccountName: not-in-repo\n"}}; + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"crd\""), 1); + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"serviceAccountName\""), 0); + PASS(); +} + +/* References resolve inside the bundle: a ConfigMap of the same name in + * another directory (another app's copy) is not the one mounted. */ +TEST(probe_k8s_reference_stays_in_bundle) { + static const GpfFile files[] = { + {"a/app.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\nspec:\n template:\n" + " spec:\n volumes:\n - name: cfg\n configMap:\n" + " name: web-config\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: web-config\n"}, + {"b/cm.yaml", "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: web-config\n"}}; + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"volume\""), 1); + PASS(); +} + +/* A name is one object per kind and namespace: two declarations of it in + * one bundle are alternatives (kubermatic keeps canal_v3.21 ... v3.31 side + * by side; one is applied). A reference means the declaration in its own + * file when there is one. */ +TEST(probe_k8s_reference_prefers_its_own_file) { + static const char v1[] = + "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: canal\nspec:\n template:\n" + " spec:\n volumes:\n - name: cfg\n configMap:\n" + " name: canal-config\n" + "---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: canal-config\n"; + static const GpfFile files[] = {{"canal/canal_v1.yaml", v1}, {"canal/canal_v2.yaml", v1}}; + ASSERT_EQ(gpf_resource_edges(files, 2, "USAGE", "\"ref\":\"volume\""), 2); + PASS(); +} + +/* ── Go operator layer (P1.2) ───────────────────────────────────────── */ + +/* Edges of `type` whose properties carry `prop`, between nodes of + * `src_label` and `tgt_label`; -1 when indexing failed. */ +static int gpf_typed_edges(const GpfFile *files, int nfiles, const char *type, + const char *src_label, const char *tgt_label, const char *prop) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int n = -1; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id " + "JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 AND e.type=?2 " + "AND s.label=?3 AND t.label=?4 AND instr(e.properties,?5)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 2, type, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 3, src_label, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 4, tgt_label, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 5, prop, -1, SQLITE_STATIC); + n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return n; +} + +static const GpfFile k_operator_repo[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"api/v1/doc.go", "// +groupName=example.com\npackage v1\n"}, + {"api/v1/widget_types.go", "package v1\n\ntype Widget struct {\n\tName string\n}\n\n" + "type Gadget struct {\n\tName string\n}\n"}, + {"config/crd/widgets.yaml", + "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\n" + "metadata:\n name: widgets.example.com\nspec:\n group: example.com\n" + " names:\n kind: Widget\n plural: widgets\n scope: Namespaced\n"}, + {"internal/controller/widget.go", "package controller\n\n" + "import (\n\texamplev1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "func SetupWithManager(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).\n" + "\t\tFor(&examplev1.Widget{}).\n" + "\t\tOwns(&examplev1.Gadget{}).\n" + "\t\tComplete(nil)\n" + "}\n"}, + {"config/embed.go", + "package config\n\nimport \"embed\"\n\n//go:embed crd/*\nvar CRDs embed.FS\n"}}; + +/* A controller reconciles the kind it is built For, and the kinds it Owns. */ +TEST(probe_go_controller_handles_its_kinds) { + ASSERT_EQ( + gpf_typed_edges(k_operator_repo, 6, "HANDLES", "Function", "Struct", "\"role\":\"for\""), + 1); + ASSERT_EQ( + gpf_typed_edges(k_operator_repo, 6, "HANDLES", "Function", "Struct", "\"role\":\"owns\""), + 1); + PASS(); +} + +/* The CRD of group example.com / kind Widget is backed by `type Widget` + * in the package marked +groupName=example.com; a Go file embeds the CRD + * files its //go:embed names. */ +TEST(probe_go_crd_type_and_embed) { + ASSERT_EQ( + gpf_typed_edges(k_operator_repo, 6, "USAGE", "Resource", "Struct", "\"ref\":\"crd-type\""), + 1); + ASSERT_EQ(gpf_typed_edges(k_operator_repo, 6, "REFERENCES_FILE", "File", "File", + "\"via\":\"go:embed\""), + 1); + PASS(); +} + +/* The kind a controller reconciles is the type of the package it imports: + * kubermatic has `Cluster` structs and a method `Cluster` elsewhere, and a + * name-only lookup handed `For(&kubermaticv1.Cluster{})` to the method. */ +TEST(probe_go_controller_kind_is_the_imported_type) { + static const GpfFile files[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"api/v1/types.go", "package v1\n\ntype Cluster struct {\n\tName string\n}\n"}, + {"internal/legacy/types.go", "package legacy\n\ntype Cluster struct {\n\tID int\n}\n"}, + {"pkg/resources/creds.go", + "package resources\n\ntype Creds struct{}\n\nfunc (c Creds) Cluster() string {\n" + "\treturn \"\"\n}\n"}, + {"internal/controller/cluster.go", + "package controller\n\n" + "import (\n\tkubermaticv1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "func Add(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).For(&kubermaticv1.Cluster{}).Complete(nil)\n" + "}\n"}}; + ASSERT_EQ(gpf_typed_edges(files, 5, "HANDLES", "Function", "Struct", "\"role\":\"for\""), 1); + ASSERT_EQ(gpf_typed_edges(files, 5, "HANDLES", "Function", "Method", "\"role\":\"for\""), 0); + PASS(); +} + +/* Edges of `type` into nodes defined in `tgt_file`; -1 when indexing failed. */ +static int gpf_edges_into_file(const GpfFile *files, int nfiles, const char *type, + const char *tgt_file) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int n = -1; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type=?2 AND t.file_path=?3"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 2, type, -1, SQLITE_STATIC); + sqlite3_bind_text(st, 3, tgt_file, -1, SQLITE_STATIC); + n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return n; +} + +/* An import path names the directory its go.mod roots: kubermatic's + * `k8c.io/kubermatic/sdk/v2/apis/...` lives in `sdk/apis/...`, which the + * path does not end with, while an unrelated `apis/v1` does. */ +TEST(probe_go_controller_import_follows_go_mod) { + static const GpfFile files[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"sdk/go.mod", "module example.com/proj/sdk/v2\n\ngo 1.22\n"}, + {"sdk/apis/v1/types.go", "package v1\n\ntype Cluster struct {\n\tName string\n}\n"}, + {"apis/v1/types.go", "package v1\n\ntype Cluster struct {\n\tID int\n}\n"}, + {"pkg/controller/cluster.go", + "package controller\n\n" + "import (\n\tkubermaticv1 \"example.com/proj/sdk/v2/apis/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "func Add(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).For(&kubermaticv1.Cluster{}).Complete(nil)\n" + "}\n"}}; + ASSERT_EQ(gpf_edges_into_file(files, 5, "HANDLES", "sdk/apis/v1/types.go"), 1); + ASSERT_EQ(gpf_edges_into_file(files, 5, "HANDLES", "apis/v1/types.go"), 0); + PASS(); +} + +/* A comment line may sit between a chain's '.' and the call (kubermatic's + * seed-sync: `}).` / `// watch all seeds` / `For(&kubermaticv1.Seed{}, ...`), + * and a call written inside a comment builds nothing. */ +TEST(probe_go_controller_chain_with_comment_lines) { + static const GpfFile files[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"api/v1/widget.go", "package v1\n\ntype Widget struct {\n\tName string\n}\n"}, + {"api/v1/gadget.go", "package v1\n\ntype Gadget struct {\n\tName string\n}\n"}, + {"internal/controller/widget.go", + "package controller\n\n" + "import (\n\tv1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "func Add(mgr ctrl.Manager) error {\n" + "\t// gadgets would be b.Owns(&v1.Gadget{}) (not yet)\n" + "\treturn ctrl.NewControllerManagedBy(mgr).\n" + "\t\tNamed(\"widget\").\n" + "\t\t// watch all widgets\n" + "\t\tFor(&v1.Widget{}).\n" + "\t\tComplete(nil)\n" + "}\n"}}; + ASSERT_EQ(gpf_edges_into_file(files, 4, "HANDLES", "api/v1/widget.go"), 1); + ASSERT_EQ(gpf_edges_into_file(files, 4, "HANDLES", "api/v1/gadget.go"), 0); + PASS(); +} + +/* Go method QNs carry no receiver, so every reconciler's SetupWithManager in + * one package is one node: each definition still handles its own kind. */ +TEST(probe_go_controller_methods_sharing_a_node) { + static const GpfFile files[] = { + {"go.mod", "module example.com/proj\n\ngo 1.22\n"}, + {"api/v1/types.go", "package v1\n\ntype Widget struct {\n\tName string\n}\n\n" + "type Gadget struct {\n\tName string\n}\n"}, + {"internal/controller/widget.go", + "package controller\n\n" + "import (\n\tv1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "type WidgetReconciler struct{}\n\n" + "func (r *WidgetReconciler) SetupWithManager(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).For(&v1.Widget{}).Complete(r)\n" + "}\n"}, + {"internal/controller/gadget.go", + "package controller\n\n" + "import (\n\tv1 \"example.com/proj/api/v1\"\n" + "\tctrl \"sigs.k8s.io/controller-runtime\"\n)\n\n" + "type GadgetReconciler struct{}\n\n" + "func (r *GadgetReconciler) SetupWithManager(mgr ctrl.Manager) error {\n" + "\treturn ctrl.NewControllerManagedBy(mgr).For(&v1.Gadget{}).Complete(r)\n" + "}\n"}}; + ASSERT_EQ(gpf_edges_into_file(files, 4, "HANDLES", "api/v1/types.go"), 2); + PASS(); +} + +/* ── Curated CRD references + deploy scopes (P1.3) ──────────────────── */ + +/* A Flux bootstrap: the flux-system Kustomization deploys ./clusters/prod, + * whose Kustomizations deploy ./infra and ./apps/prod. Each is its own + * kustomize root, so the references between them -- a Kustomization's + * sourceRef to the flux-system GitRepository, a HelmRelease's dependsOn in + * another namespace, a Certificate's ClusterIssuer -- meet only in the + * deploy scope the root reaches. */ +TEST(probe_flux_refs_meet_in_the_deploy_scope) { + static const GpfFile files[] = { + {"clusters/prod/flux-system/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - gotk-sync.yaml\n"}, + {"clusters/prod/flux-system/gotk-sync.yaml", + "apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\n" + "metadata:\n name: flux-system\n namespace: flux-system\n" + "spec:\n url: ssh://git@example.com/fleet\n---\n" + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: flux-system\n namespace: flux-system\n" + "spec:\n path: ./clusters/prod\n sourceRef:\n kind: GitRepository\n" + " name: flux-system\n"}, + {"clusters/prod/apps.yaml", + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: infra\n namespace: flux-system\n" + "spec:\n path: ./infra\n sourceRef:\n kind: GitRepository\n" + " name: flux-system\n---\n" + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: apps\n namespace: flux-system\n" + "spec:\n dependsOn:\n - name: infra\n path: ./apps/prod\n" + " sourceRef:\n kind: GitRepository\n name: flux-system\n"}, + {"infra/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - cert-manager.yaml\n"}, + {"infra/cert-manager.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\n" + "metadata:\n name: cert-manager\n namespace: cert-manager\n" + "spec:\n interval: 1h\n---\n" + "apiVersion: cert-manager.io/v1\nkind: ClusterIssuer\n" + "metadata:\n name: letsencrypt\nspec:\n selfSigned: {}\n"}, + {"apps/prod/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "namespace: podinfo\nresources:\n - ../base/podinfo\n"}, + {"apps/base/podinfo/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - release.yaml\n - repo.yaml\n"}, + {"apps/base/podinfo/release.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\n" + "metadata:\n name: podinfo\nspec:\n chart:\n spec:\n chart: podinfo\n" + " sourceRef:\n kind: HelmRepository\n name: podinfo\n" + " dependsOn:\n - name: cert-manager\n namespace: cert-manager\n" + " valuesFrom:\n - kind: ConfigMap\n name: podinfo-values\n---\n" + "apiVersion: cert-manager.io/v1\nkind: Certificate\n" + "metadata:\n name: podinfo-tls\nspec:\n secretName: podinfo-tls\n" + " issuerRef:\n kind: ClusterIssuer\n name: letsencrypt\n"}, + {"apps/base/podinfo/repo.yaml", + "apiVersion: source.toolkit.fluxcd.io/v1\nkind: HelmRepository\n" + "metadata:\n name: podinfo\nspec:\n url: https://example.com/charts\n---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: podinfo-values\n" + "data:\n replicaCount: \"2\"\n"}}; + /* flux-system -> its GitRepository, infra and apps -> it, podinfo -> HelmRepository */ + ASSERT_EQ(gpf_typed_edges(files, 9, "USAGE", "Resource", "Resource", "\"ref\":\"sourceRef\""), + 4); + /* apps -> infra; podinfo -> cert-manager in another namespace and root */ + ASSERT_EQ(gpf_typed_edges(files, 9, "DEPENDS_ON", "Resource", "Resource", + "\"via\":\"dependsOn\""), + 2); + ASSERT_EQ(gpf_typed_edges(files, 9, "USAGE", "Resource", "Resource", "\"ref\":\"issuerRef\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 9, "USAGE", "Resource", "Resource", + "\"ref\":\"valuesFrom\""), + 1); + PASS(); +} + +/* Cluster API: a Cluster names its infrastructure, control plane and + * class; machines name their cluster, bootstrap config and templates. The + * source kind counts in its API group: a CloudNativePG `Cluster` with the + * same field is not a Cluster API one. */ +TEST(probe_capi_refs_follow_the_api_group) { + static const char tmpl[] = + "apiVersion: cluster.x-k8s.io/v1beta2\nkind: Cluster\nmetadata:\n name: c1\n" + "spec:\n infrastructureRef:\n kind: DockerCluster\n name: c1\n" + " controlPlaneRef:\n kind: KubeadmControlPlane\n name: c1-cp\n" + " topology:\n classRef:\n name: quick\n---\n" + "apiVersion: infrastructure.cluster.x-k8s.io/v1beta2\nkind: DockerCluster\n" + "metadata:\n name: c1\n---\n" + "apiVersion: controlplane.cluster.x-k8s.io/v1beta2\nkind: KubeadmControlPlane\n" + "metadata:\n name: c1-cp\nspec:\n machineTemplate:\n spec:\n" + " infrastructureRef:\n kind: DockerMachineTemplate\n name: cp-infra\n---\n" + "apiVersion: infrastructure.cluster.x-k8s.io/v1beta2\nkind: DockerMachineTemplate\n" + "metadata:\n name: cp-infra\n---\n" + "apiVersion: cluster.x-k8s.io/v1beta2\nkind: MachineDeployment\nmetadata:\n name: md\n" + "spec:\n clusterName: c1\n template:\n spec:\n bootstrap:\n" + " configRef:\n kind: KubeadmConfigTemplate\n name: md-boot\n" + " infrastructureRef:\n kind: DockerMachineTemplate\n name: md-infra\n---\n" + "apiVersion: bootstrap.cluster.x-k8s.io/v1beta2\nkind: KubeadmConfigTemplate\n" + "metadata:\n name: md-boot\n---\n" + "apiVersion: infrastructure.cluster.x-k8s.io/v1beta2\nkind: DockerMachineTemplate\n" + "metadata:\n name: md-infra\n---\n" + "apiVersion: cluster.x-k8s.io/v1beta2\nkind: ClusterClass\nmetadata:\n name: quick\n" + "spec:\n infrastructure:\n templateRef:\n kind: DockerClusterTemplate\n" + " name: quick-cluster\n workers:\n machineDeployments:\n - class: w\n" + " bootstrap:\n templateRef:\n kind: KubeadmConfigTemplate\n" + " name: md-boot\n---\n" + "apiVersion: infrastructure.cluster.x-k8s.io/v1beta2\nkind: DockerClusterTemplate\n" + "metadata:\n name: quick-cluster\n---\n" + "apiVersion: postgresql.cnpg.io/v1\nkind: Cluster\nmetadata:\n name: pg\n" + "spec:\n infrastructureRef:\n kind: DockerCluster\n name: c1\n"; + static const GpfFile files[] = {{"templates/cluster.yaml", tmpl}}; + /* Cluster, KubeadmControlPlane and MachineDeployment -- not the cnpg Cluster */ + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"infrastructureRef\""), + 3); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"controlPlaneRef\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"topologyClass\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"clusterName\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"bootstrap\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"template\""), + 2); + PASS(); +} + +/* cert-manager and Kubermatic: a Certificate's issuer (Issuer when the + * kind is omitted) and Secret, an Ingress's cluster-issuer annotation, an + * issuer's CA and DNS-solver Secrets (`*SecretRef` of any provider), a KKP + * Seed's kubeconfig and a user cluster's provider credentials. */ +TEST(probe_cert_manager_and_kkp_refs) { + static const char doc[] = + "apiVersion: cert-manager.io/v1\nkind: Issuer\nmetadata:\n name: ca\n" + "spec:\n ca:\n secretName: ca-key\n---\n" + "apiVersion: cert-manager.io/v1\nkind: ClusterIssuer\nmetadata:\n name: letsencrypt\n" + "spec:\n acme:\n solvers:\n - dns01:\n cloudflare:\n" + " apiTokenSecretRef:\n name: cf-token\n key: token\n" + "---\n" + "apiVersion: cert-manager.io/v1\nkind: Certificate\nmetadata:\n name: web\n" + "spec:\n secretName: web-tls\n issuerRef:\n name: ca\n---\n" + "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: web\n" + " annotations:\n cert-manager.io/cluster-issuer: letsencrypt\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: ca-key\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: cf-token\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: web-tls\n---\n" + "apiVersion: kubermatic.k8c.io/v1\nkind: Seed\nmetadata:\n name: europe\n" + " namespace: kubermatic\nspec:\n kubeconfig:\n name: seed-kubeconfig\n" + " namespace: kubermatic\n---\n" + "apiVersion: kubermatic.k8c.io/v1\nkind: Cluster\nmetadata:\n name: user1\n" + "spec:\n cloud:\n aws:\n credentialsReference:\n name: aws-creds\n" + " namespace: kubermatic\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: seed-kubeconfig\n" + " namespace: kubermatic\n---\n" + "apiVersion: v1\nkind: Secret\nmetadata:\n name: aws-creds\n namespace: kubermatic\n"; + static const GpfFile files[] = {{"pki/certs.yaml", doc}}; + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"issuerRef\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"secretName\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"issuer\""), 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"secretRef\""), + 2); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"kubeconfig\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"credentials\""), + 1); + PASS(); +} + +/* Argo: an Application's project, and a Rollout -- a workload with a pod + * template -- names its Services, analysis template and service account. */ +TEST(probe_argo_application_and_rollout_refs) { + static const char doc[] = + "apiVersion: argoproj.io/v1alpha1\nkind: AppProject\nmetadata:\n name: team-a\n" + " namespace: argocd\n---\n" + "apiVersion: argoproj.io/v1alpha1\nkind: Application\nmetadata:\n name: web\n" + " namespace: argocd\nspec:\n project: team-a\n source:\n path: apps/web\n" + " destination:\n namespace: web\n---\n" + "apiVersion: argoproj.io/v1alpha1\nkind: Rollout\nmetadata:\n name: web\n" + "spec:\n strategy:\n canary:\n canaryService: web-canary\n" + " stableService: web-stable\n analysis:\n templates:\n" + " - templateName: success-rate\n template:\n spec:\n" + " serviceAccountName: web\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: web-canary\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: web-stable\n---\n" + "apiVersion: argoproj.io/v1alpha1\nkind: AnalysisTemplate\nmetadata:\n" + " name: success-rate\n---\n" + "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: web\n"; + static const GpfFile files[] = {{"apps/web/rollout.yaml", doc}}; + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"project\""), 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"rolloutService\""), + 2); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"analysis\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"serviceAccountName\""), + 1); + PASS(); +} + +/* ── Exposed routes + policy selectors (P1.4) ───────────────────────── */ + +/* A PodDisruptionBudget's selector and a NetworkPolicy's podSelector pick + * workloads by their pod labels; an empty podSelector (every pod of the + * namespace) and a selector with matchExpressions draw nothing. */ +TEST(probe_k8s_pdb_and_networkpolicy_select_workloads) { + static const char doc[] = + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\nspec:\n template:\n" + " metadata:\n labels:\n app: web\n---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\nspec:\n template:\n" + " metadata:\n labels:\n app: api\n---\n" + "apiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: web\nspec:\n" + " minAvailable: 1\n selector:\n matchLabels:\n app: web\n---\n" + "apiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: narrowed\nspec:\n" + " selector:\n matchLabels:\n app: web\n matchExpressions:\n" + " - key: track\n operator: In\n values: [canary]\n---\n" + "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: api\nspec:\n" + " podSelector:\n matchLabels:\n app: api\n---\n" + "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: deny-all\n" + "spec:\n podSelector: {}\n"; + static const GpfFile files[] = {{"k8s/app.yaml", doc}}; + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", "\"ref\":\"selector\""), + 1); + ASSERT_EQ(gpf_typed_edges(files, 1, "USAGE", "Resource", "Resource", + "\"ref\":\"podSelector\""), + 1); + PASS(); +} + +/* Versioned manifests side by side (kubermatic addons/canal/canal_v3.2x): + * each declares the same Deployment; a selector picks the copy in its own + * file, not every version's. */ +TEST(probe_k8s_selectors_pick_their_own_version) { + static const char v1[] = + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: kube-controllers\nspec:\n" + " template:\n metadata:\n labels:\n app: kube-controllers\n---\n" + "apiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n" + " name: kube-controllers\nspec:\n selector:\n matchLabels:\n" + " app: kube-controllers\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: metrics\nspec:\n selector:\n" + " app: kube-controllers\n"; + static const GpfFile files[] = {{"addons/canal/canal_v1.yaml", v1}, + {"addons/canal/canal_v2.yaml", v1}}; + ASSERT_EQ(gpf_typed_edges(files, 2, "USAGE", "Resource", "Resource", "\"ref\":\"selector\""), + 2); + ASSERT_EQ(gpf_typed_edges(files, 2, "INFRA_MAPS", "Resource", "Resource", + "\"kind\":\"selector\""), + 2); + PASS(); +} + +/* An Ingress rule's host and paths and an HTTPRoute's hostnames and path + * matches are routes the cluster exposes: the object DEFINES each, and the + * route maps to the Service it is sent to. */ +TEST(probe_k8s_ingress_and_httproute_expose_routes) { + static const char doc[] = + "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: shop\nspec:\n" + " rules:\n - host: shop.example.com\n http:\n paths:\n" + " - path: /api\n pathType: Prefix\n backend:\n" + " service:\n name: api\n port:\n" + " number: 80\n" + " - path: /\n pathType: Prefix\n backend:\n" + " service:\n name: web\n port:\n" + " number: 80\n---\n" + "apiVersion: gateway.networking.k8s.io/v1\nkind: HTTPRoute\nmetadata:\n name: api\n" + "spec:\n hostnames:\n - api.example.com\n rules:\n - matches:\n" + " - path:\n type: PathPrefix\n value: /v1\n" + " backendRefs:\n - name: api\n port: 80\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: api\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n"; + static const GpfFile files[] = {{"k8s/edge.yaml", doc}}; + ASSERT_EQ(gpf_typed_edges(files, 1, "DEFINES", "Resource", "Route", "{"), 3); + ASSERT_EQ(gpf_typed_edges(files, 1, "INFRA_MAPS", "Route", "Resource", "\"kind\":\"ingress\""), + 2); + ASSERT_EQ( + gpf_typed_edges(files, 1, "INFRA_MAPS", "Route", "Resource", "\"kind\":\"httproute\""), 1); + PASS(); +} + +/* ── MCP surface: infra filters + topology (P1.5) ───────────────────── */ + +/* Run MCP `tool` on the indexed fixture; `args_fmt` holds one %s, the + * project. The response is the caller's to free; NULL when indexing failed. */ +static char *gpf_tool_on(const GpfFile *files, int nfiles, const char *tool, + const char *args_fmt) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *resp = NULL; + if (store && lp.srv) { + char args[2048]; + snprintf(args, sizeof(args), args_fmt, lp.project); + resp = cbm_mcp_handle_tool(lp.srv, tool, args); + } + gpf_cleanup(&lp, store); + return resp; +} + +static const GpfFile k_two_teams[] = { + {"k8s/a.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: team-a\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n namespace: team-a\n"}, + {"k8s/b.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\n namespace: team-b\n"}}; + +/* search_graph narrows Resources by kind and by effective namespace, in the + * structural and the BM25 query mode alike. */ +TEST(probe_mcp_search_filters_kind_and_namespace) { + char *r = gpf_tool_on(k_two_teams, 2, "search_graph", + "{\"project\":\"%s\",\"label\":\"Resource\",\"kind\":\"Deployment\"," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "Deployment/team-a/web")); + ASSERT_NOT_NULL(strstr(r, "Deployment/team-b/api")); + ASSERT_NULL(strstr(r, "Service/team-a/web")); + free(r); + r = gpf_tool_on(k_two_teams, 2, "search_graph", + "{\"project\":\"%s\",\"label\":\"Resource\",\"namespace\":\"team-a\"," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "Deployment/team-a/web")); + ASSERT_NOT_NULL(strstr(r, "Service/team-a/web")); + ASSERT_NULL(strstr(r, "Deployment/team-b/api")); + free(r); + r = gpf_tool_on(k_two_teams, 2, "search_graph", + "{\"project\":\"%s\",\"query\":\"web\",\"kind\":\"Service\"," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "Service/team-a/web")); + ASSERT_NULL(strstr(r, "Deployment/team-a/web")); + free(r); + PASS(); +} + +/* trace_path mode=topology walks infra edges: from a Service out to its + * workload and the ConfigMap the workload mounts, and in to the exposed + * route and the Ingress that declares it. The call graph modes do not. */ +TEST(probe_mcp_trace_topology) { + static const char doc[] = + "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: shop\nspec:\n" + " rules:\n - host: shop.example.com\n http:\n paths:\n" + " - path: /api\n pathType: Prefix\n backend:\n" + " service:\n name: api\n port:\n" + " number: 80\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: api\nspec:\n selector:\n" + " app: api\n---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\nspec:\n template:\n" + " metadata:\n labels:\n app: api\n spec:\n volumes:\n" + " - name: cfg\n configMap:\n name: api-config\n---\n" + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: api-config\n"; + static const GpfFile files[] = {{"k8s/shop.yaml", doc}}; + char *r = gpf_tool_on(files, 1, "trace_path", + "{\"function_name\":\"Service/api\",\"project\":\"%s\"," + "\"direction\":\"outbound\",\"mode\":\"topology\",\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "Deployment/-/api")); + ASSERT_NOT_NULL(strstr(r, "ConfigMap/-/api-config")); + free(r); + r = gpf_tool_on(files, 1, "trace_path", + "{\"function_name\":\"Service/api\",\"project\":\"%s\"," + "\"direction\":\"inbound\",\"mode\":\"topology\",\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "__route__ingress__shop.example")); /* qn groups split at a dot */ + ASSERT_NOT_NULL(strstr(r, "Ingress/-/shop")); + free(r); + r = gpf_tool_on(files, 1, "trace_path", + "{\"function_name\":\"Service/api\",\"project\":\"%s\"," + "\"direction\":\"outbound\",\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(strstr(r, "Deployment/-/api")); + free(r); + PASS(); +} + +/* trace_path mode=topology also follows DEPLOYS: from a Flux Kustomization + * out to the kustomization its path applies. */ +TEST(probe_mcp_trace_topology_follows_deploys) { + static const GpfFile files[] = { + {"clusters/sync.yaml", + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\nmetadata:\n" + " name: sync\n namespace: flux-system\nspec:\n path: ./apps\n"}, + {"apps/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - cm.yaml\n"}, + {"apps/cm.yaml", "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: settings\n"}}; + char *r = gpf_tool_on(files, 3, "trace_path", + "{\"function_name\":\"Kustomization/sync\",\"project\":\"%s\"," + "\"direction\":\"outbound\",\"mode\":\"topology\",\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, ".apps.kustomization\"")); /* the kustomization's Resource */ + free(r); + PASS(); +} + +/* trace_path mode=run walks what a Service runs: out to its workload, the + * Image the workload runs, the Dockerfile that builds the image, the Go main + * its ENTRYPOINT starts -- and the environment the container is given. + * mode=topology stops before the program. */ +TEST(probe_mcp_trace_run) { + static const GpfFile files[] = { + {"go.mod", "module example.com/app\n\ngo 1.22\n"}, + {"cmd/server/main.go", "package main\n\nfunc main() {}\n"}, + {"svc/Dockerfile", "FROM golang:1.22 AS build\nWORKDIR /src\nCOPY . .\n" + "RUN CGO_ENABLED=0 go build -o /out/server ./cmd/server\n" + "FROM gcr.io/distroless/static\nCOPY --from=build /out/server /server\n" + "ENTRYPOINT [\"/server\"]\n"}, + {"skaffold.yaml", "apiVersion: skaffold/v4beta6\nkind: Config\nbuild:\n artifacts:\n" + " - image: acme/server\n context: .\n docker:\n" + " dockerfile: svc/Dockerfile\n"}, + {"k8s/app.yaml", + "apiVersion: v1\nkind: Service\nmetadata:\n name: server\nspec:\n selector:\n" + " app: server\n---\n" + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: server\nspec:\n" + " template:\n metadata:\n labels:\n app: server\n spec:\n" + " containers:\n - name: server\n image: acme/server:1.0\n" + " env:\n - name: LISTEN_PORT\n value: \"8080\"\n"}}; + char *r = gpf_tool_on(files, 5, "trace_path", + "{\"function_name\":\"Service/server\",\"project\":\"%s\"," + "\"direction\":\"outbound\",\"mode\":\"run\",\"depth\":5," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + /* qualified names are grouped by prefix, split at a dot */ + ASSERT_NOT_NULL(strstr(r, "Deployment/-/server")); + ASSERT_NOT_NULL(strstr(r, "\"__image__docker\",\"rows\":[[\"io/acme/server\"")); + ASSERT_NOT_NULL(strstr(r, ".svc\",\"rows\":[[\"Dockerfile\"")); /* its Module */ + ASSERT_NOT_NULL(strstr(r, ".cmd.server\",\"rows\":[[\"main\"")); /* what it starts */ + ASSERT_NOT_NULL(strstr(r, "Deployment/-/server#env/server/LISTEN_PORT")); + free(r); + r = gpf_tool_on(files, 5, "trace_path", + "{\"function_name\":\"Service/server\",\"project\":\"%s\"," + "\"direction\":\"outbound\",\"mode\":\"topology\",\"depth\":5," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(strstr(r, ".cmd.server\",\"rows\":[[\"main\"")); + free(r); + PASS(); +} + +/* trace_path mode=deploy walks deploy chains: in from a chart to the shell + * function that installs it (DEPLOYS), the script calling the function + * (CALLS) and the CI job running the script (SPAWNS). The call graph mode + * stops at the chart. */ +TEST(probe_mcp_trace_deploy) { + static const GpfFile files[] = { + {"charts/web/Chart.yaml", "apiVersion: v2\nname: web-app\nversion: 0.1.0\n"}, + {"hack/deploy.sh", "#!/bin/sh\n" + "cd \"$(dirname \"$0\")/..\"\n" + "install_web() {\n" + " helm upgrade --install web charts/web\n" + "}\n" + "install_web\n"}, + {".github/workflows/release.yml", "on: [push]\n" + "jobs:\n" + " ship:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: ./hack/deploy.sh\n"}}; + char *r = gpf_tool_on(files, 3, "trace_path", + "{\"function_name\":\"web-app\",\"project\":\"%s\"," + "\"direction\":\"inbound\",\"mode\":\"deploy\",\"depth\":4," + "\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(strstr(r, "install_web")); + ASSERT_NOT_NULL(strstr(r, ".hack\"")); /* the script's Module: hack.deploy */ + ASSERT_NOT_NULL(strstr(r, ".github.workflows\"")); /* the workflow's Module: release */ + ASSERT_NOT_NULL(strstr(r, "\"release\"")); + free(r); + r = gpf_tool_on(files, 3, "trace_path", + "{\"function_name\":\"web-app\",\"project\":\"%s\"," + "\"direction\":\"inbound\",\"format\":\"json\"}"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(strstr(r, "install_web")); + free(r); + PASS(); +} + +/* ── DEPLOYS (Phase 3) ──────────────────────────────────────────────── */ + +/* A Flux Kustomization deploys what its path applies: the outermost + * kustomization below it (its Kustomization node) and a manifest outside any + * (its File); a HelmRelease its chart -- the repository chart of a Git + * source, else the external chart of that name; an Argo Application whose + * path is a chart dir, that Chart. */ +TEST(probe_gitops_objects_deploy_their_units) { + static const GpfFile files[] = { + {"clusters/prod/flux-system/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - gotk-sync.yaml\n"}, + {"clusters/prod/flux-system/gotk-sync.yaml", + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: flux-system\n namespace: flux-system\n" + "spec:\n path: ./clusters/prod\n"}, + {"clusters/prod/apps.yaml", + "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\n" + "metadata:\n name: apps\n namespace: flux-system\nspec:\n path: ./apps\n"}, + {"apps/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + "resources:\n - release.yaml\n"}, + {"apps/release.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: web\n" + "spec:\n chart:\n spec:\n chart: ./charts/web\n sourceRef:\n" + " kind: GitRepository\n name: flux-system\n---\n" + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: redis\n" + "spec:\n chart:\n spec:\n chart: redis\n sourceRef:\n" + " kind: HelmRepository\n name: bitnami\n"}, + {"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 0.1.0\n"}, + {"argo/web.yaml", + "apiVersion: argoproj.io/v1alpha1\nkind: Application\nmetadata:\n name: web\n" + "spec:\n source:\n path: charts/web\n"}}; + ASSERT_EQ(gpf_typed_edges(files, 7, "DEPLOYS", "Resource", "Resource", "\"via\":\"flux\""), 2); + ASSERT_EQ(gpf_typed_edges(files, 7, "DEPLOYS", "Resource", "File", "\"via\":\"flux\""), 1); + ASSERT_EQ( + gpf_typed_edges(files, 7, "DEPLOYS", "Resource", "Chart", "\"via\":\"helmrelease\""), 2); + ASSERT_EQ(gpf_typed_edges(files, 7, "DEPLOYS", "Resource", "Chart", "\"via\":\"argo\""), 1); + PASS(); +} + +/* Every DEPLOYS edge as "source -> target (via)", sorted, one per line: a + * function by name, top-level code as "@"; a File or Kustomization by + * its path, a Chart by name ("(ext)": the external one). NULL on failure; + * the caller frees. */ +static char *gpf_deploys(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT " + "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END " + "|| ' -> ' || CASE WHEN t.label='Chart' THEN 'chart:' || t.name || " + "CASE WHEN instr(t.qualified_name,'__helm_dep__') THEN '(ext)' ELSE '' END " + "ELSE t.label || ':' || t.file_path END " + "|| ' (' || json_extract(e.properties,'$.via') || ')' AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type='DEPLOYS' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +#define GPF_ASSERT_DEPLOYS(files, n, want) \ + do { \ + char *got_ = gpf_deploys(files, n); \ + if (!got_ || strcmp(got_, want) != 0) { \ + fprintf(stderr, "DEPLOYS got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \ + free(got_); \ + FAIL("DEPLOYS edge set"); \ + } \ + free(got_); \ + } while (0) + +/* Every SPAWNS edge a script or recipe draws (api=shell) as "source -> + * program", sorted, one per line; top-level code as "@". */ +static char *gpf_spawns(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT DISTINCT " + "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END " + "|| ' -> ' || t.name AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type='SPAWNS' AND t.label='Process' " + "AND json_extract(e.properties,'$.api')='shell' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +#define GPF_ASSERT_SPAWNS(files, n, want) \ + do { \ + char *got_ = gpf_spawns(files, n); \ + if (!got_ || strcmp(got_, want) != 0) { \ + fprintf(stderr, "SPAWNS got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \ + free(got_); \ + FAIL("SPAWNS edge set"); \ + } \ + free(got_); \ + } while (0) + +/* Every SPAWNS and DEPLOYS edge as "source -> target (TYPE api|via[ + * ci[/job]])", sorted, one per line: top-level code as "@"; a script + * as "script:", a Process as "proc:", a Chart as "chart:", + * a Make rule as "rule::". */ +static char *gpf_runs(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT DISTINCT " + "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END " + "|| ' -> ' || CASE WHEN t.label IN ('Module','File') THEN 'script:' || t.file_path " + "WHEN t.label='Process' THEN 'proc:' || t.name WHEN t.label='Chart' THEN " + "'chart:' || t.name WHEN t.label='Function' THEN 'rule:' || t.file_path || ':' || " + "t.name ELSE t.label || ':' || t.file_path END " + "|| ' (' || e.type || ' ' || coalesce(json_extract(e.properties,'$.api'), " + "json_extract(e.properties,'$.via')) || coalesce(' ' || " + "json_extract(e.properties,'$.ci') || coalesce('/' || " + "json_extract(e.properties,'$.job'), ''), '') || ')' AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type IN ('SPAWNS','DEPLOYS') ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +#define GPF_ASSERT_RUNS(files, n, want) \ + do { \ + char *got_ = gpf_runs(files, n); \ + if (!got_ || strcmp(got_, want) != 0) { \ + fprintf(stderr, "runs got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \ + free(got_); \ + FAIL("SPAWNS / DEPLOYS edge set"); \ + } \ + free(got_); \ + } while (0) + +/* Every DEPENDS_ON edge into an Image as "@ -> (via)", + * sorted, one per line. */ +static char *gpf_images(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT '@' || s.file_path " + "|| ' -> ' || t.name || ' (' || json_extract(e.properties,'$.via') || ')' AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type='DEPENDS_ON' AND t.label='Image' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +#define GPF_ASSERT_IMAGES(files, n, want) \ + do { \ + char *got_ = gpf_images(files, n); \ + if (!got_ || strcmp(got_, want) != 0) { \ + fprintf(stderr, "images got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \ + free(got_); \ + FAIL("Image DEPENDS_ON edge set"); \ + } \ + free(got_); \ + } while (0) + +/* A Dockerfile DEPENDS_ON each image it builds on: FROM and COPY --from, + * with ARG defaults expanded, under the image's repository (docker.io and + * library/ filled in, a registry host kept, tag and digest dropped). A stage + * (by name or index), scratch and an ARG without a value are no image. */ +TEST(probe_dockerfile_bases_are_images) { + static const GpfFile files[] = { + {"svc/Dockerfile", "ARG GO=1.22\nARG BASE\n" + "FROM golang:${GO} AS build\n" + "RUN go build -o /out/app ./cmd/app\n" + "FROM ${BASE} AS unknown\n" + "FROM build AS test\n" + "FROM gcr.io/distroless/static@sha256:0123 AS run\n" + "COPY --from=build /out/app /app\n" + "COPY --from=0 /out/app /app2\n" + "COPY --from=docker.io/bitnami/kubectl:1.30 /opt/kubectl /kubectl\n" + "FROM scratch\n" + "COPY --from=registry.example.com:5000/team/tools:2 /bin/x /x\n"}}; + GPF_ASSERT_IMAGES(files, 1, + "@svc/Dockerfile -> docker.io/bitnami/kubectl (copy-from)\n" + "@svc/Dockerfile -> docker.io/library/golang (from)\n" + "@svc/Dockerfile -> gcr.io/distroless/static (from)\n" + "@svc/Dockerfile -> registry.example.com:5000/team/tools (copy-from)"); + PASS(); +} + +/* A build file named for its variant is a Dockerfile of its own (its own + * Module, beside the plain Dockerfile): Dockerfile.debug, Dockerfile-test, + * Containerfile, pipeline.Dockerfile. A template of one (Dockerfile.j2) and a + * BuildKit ignore file (Dockerfile.dockerignore) are not. */ +TEST(probe_dockerfile_variants_are_dockerfiles) { + static const GpfFile files[] = { + {"svc/Dockerfile", "FROM alpine:3.20\n"}, + {"svc/Dockerfile.debug", "FROM debian:12\n"}, + {"svc/Dockerfile-test", "FROM ubuntu:24.04\n"}, + {"svc/Dockerfile.dockerignore", "FROM busybox\n"}, + {"Containerfile", "FROM quay.io/podman/stable\n"}, + {"tools/pipeline.Dockerfile", "FROM node:20\n"}, + {"tools/Dockerfile.j2", "FROM {{ base_image }}\n"}}; + GPF_ASSERT_IMAGES(files, 7, + "@Containerfile -> quay.io/podman/stable (from)\n" + "@svc/Dockerfile -> docker.io/library/alpine (from)\n" + "@svc/Dockerfile-test -> docker.io/library/ubuntu (from)\n" + "@svc/Dockerfile.debug -> docker.io/library/debian (from)\n" + "@tools/pipeline.Dockerfile -> docker.io/library/node (from)"); + PASS(); +} + +/* A Dockerfile SPAWNS what its last stage runs: the Go main of a binary a + * stage built and the image copied in, an interpreter script or a file + * copied in (also a CMD file under an inherited ENTRYPOINT), else the + * program's Process. sh -c and arguments-only CMDs are no edge. */ +TEST(probe_dockerfile_entrypoint_runs) { + static const GpfFile files[] = { + {"go.mod", "module example.com/app\n\ngo 1.22\n"}, + {"cmd/server/main.go", "package main\n\nfunc main() {}\n"}, + {"svc/Dockerfile", "FROM golang:1.22 AS build\nWORKDIR /src\nCOPY . .\n" + "RUN CGO_ENABLED=0 go build -o /out/server ./cmd/server\n" + "FROM gcr.io/distroless/static\nCOPY --from=build /out/server /server\n" + "ENTRYPOINT [\"/server\", \"--port\", \"8080\"]\n"}, + {"py/app.py", "print('x')\n"}, + {"py/Dockerfile", "FROM python:3.12\nWORKDIR /app\nCOPY . .\nCMD [\"python\", \"app.py\"]\n"}, + {"node/index.js", "console.log(1);\n"}, + {"node/Dockerfile", + "FROM gcr.io/distroless/nodejs22\nWORKDIR /app\nCOPY index.js .\nCMD [\"index.js\"]\n"}, + {"tool/run.sh", "#!/bin/sh\necho run\n"}, + {"tool/Dockerfile", "FROM alpine:3.20\nCOPY run.sh /usr/local/bin/run.sh\n" + "ENTRYPOINT [\"/usr/local/bin/run.sh\"]\n"}, + {"web/Dockerfile", "FROM nginx:1.27\nCMD [\"nginx\", \"-g\", \"daemon off;\"]\n"}, + {"sh/Dockerfile", "FROM alpine:3.20\nCMD sh -c \"echo hi\"\n"}, + {"args/Dockerfile", "FROM gcr.io/distroless/python3\nCMD [\"--flag\"]\n"}}; + GPF_ASSERT_RUNS(files, 12, + "@node/Dockerfile -> script:node/index.js (SPAWNS dockerfile)\n" + "@py/Dockerfile -> script:py/app.py (SPAWNS dockerfile)\n" + "@svc/Dockerfile -> rule:cmd/server/main.go:main (SPAWNS dockerfile)\n" + "@tool/Dockerfile -> script:tool/run.sh (SPAWNS dockerfile)\n" + "@web/Dockerfile -> proc:nginx (SPAWNS dockerfile)"); + PASS(); +} + +/* Every USAGE {ref:image} edge as " -> ", sorted. */ +static char *gpf_image_users(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT s.name || ' -> ' || " + "t.name AS x FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id WHERE e.project=?1 AND e.type='USAGE' AND t.label='Image' AND " + "json_extract(e.properties,'$.ref')='image' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* An object running a container USAGE {ref:image} the Image of its literal + * image reference -- containers, initContainers, any pod template (CronJob, + * a custom resource). A shell substitution or a template placeholder is no + * image. */ +TEST(probe_k8s_container_images_are_images) { + static const GpfFile files[] = { + {"app.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\nspec:\n template:\n" + " spec:\n initContainers:\n - name: init\n" + " image: busybox@sha256:0123\n containers:\n - name: web\n" + " image: nginx:1.27\n - name: side\n image: $(SIDE_IMAGE)\n" + "---\napiVersion: batch/v1\nkind: CronJob\nmetadata:\n name: nightly\nspec:\n" + " jobTemplate:\n spec:\n template:\n spec:\n containers:\n" + " - name: job\n image: ghcr.io/acme/job:2\n" + "---\napiVersion: example.com/v1\nkind: Widget\nmetadata:\n name: w\nspec:\n" + " template:\n spec:\n containers:\n - name: w\n" + " image: quay.io/acme/widget:3\n - name: placeholder\n" + " image: \n"}}; + char *got = gpf_image_users(files, 1); + static const char want[] = "CronJob/nightly -> ghcr.io/acme/job\n" + "Deployment/web -> docker.io/library/busybox\n" + "Deployment/web -> docker.io/library/nginx\n" + "Widget/w -> quay.io/acme/widget"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "image users got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("USAGE {ref:image} edge set"); + } + free(got); + PASS(); +} + +/* Every DEPENDS_ON {via:build} edge as " -> (tool)". */ +static char *gpf_builds(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT s.name || ' -> ' || " + "t.file_path || ' (' || json_extract(e.properties,'$.tool') || ')' AS x FROM edges e " + "JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id WHERE " + "e.project=?1 AND e.type='DEPENDS_ON' AND s.label='Image' AND " + "json_extract(e.properties,'$.via')='build' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* An image a build configuration builds DEPENDS_ON {via:build} its + * Dockerfile: skaffold artifacts (top level and profiles; docker.dockerfile + * relative to the context; a ko artifact is none), compose services with + * build and image, variables from the project's .env. A service without an + * image or a build, or a build without a Dockerfile, is none. */ +TEST(probe_build_configs_build_images_from_dockerfiles) { + static const GpfFile files[] = { + {"src/web/Dockerfile", "FROM nginx:1.27\n"}, + {"src/api/Dockerfile.prod", "FROM golang:1.22\n"}, + {"skaffold.yaml", + "apiVersion: skaffold/v4beta6\nkind: Config\nbuild:\n artifacts:\n" + " - image: web\n context: src/web\n" + " - image: api\n context: src/api\n docker:\n" + " dockerfile: Dockerfile.prod\n" + " - image: koapp\n ko:\n main: ./cmd\n" + "profiles:\n - name: dev\n build:\n artifacts:\n" + " - image: dev-web\n context: src/web\n"}, + {".env", "REG=ghcr.io/acme\n# comment\nexport UNUSED='x'\n"}, + {"compose.yaml", "services:\n frontend:\n image: ${REG}/frontend:${TAG:-1}\n" + " build:\n context: ./src/web\n" + " worker:\n build: ./src/api\n" + " db:\n image: postgres:16\n" + " missing:\n image: acme/missing\n build: ./nope\n" + " unknown:\n image: ${NOPE}/x\n build: ./src/web\n"}}; + char *got = gpf_builds(files, 5); + static const char want[] = "docker.io/library/api -> src/api/Dockerfile.prod (skaffold)\n" + "docker.io/library/dev-web -> src/web/Dockerfile (skaffold)\n" + "docker.io/library/web -> src/web/Dockerfile (skaffold)\n" + "ghcr.io/acme/frontend -> src/web/Dockerfile (compose)"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "builds got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("DEPENDS_ON {via:build} edge set"); + } + free(got); + PASS(); +} + +/* Every edge from a compose service as " -> (TYPE + * via|ref)", sorted: an Image by repository, a Module by path. */ +static char *gpf_compose(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT s.name || ' -> ' || " + "CASE t.label WHEN 'Image' THEN 'Image:' || t.name WHEN 'Module' THEN 'Module:' || " + "t.file_path ELSE t.name END || ' (' || e.type || ' ' || " + "coalesce(json_extract(e.properties,'$.via'), json_extract(e.properties,'$.ref')) || " + "')' AS x FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id WHERE e.project=?1 AND s.label='Resource' AND " + "json_extract(s.properties,'$.ecosystem')='compose' ORDER BY x)"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A compose service is a Resource {ecosystem:compose}: it DEPENDS_ON the + * services its depends_on (a list or a mapping) and links name, USAGE + * {ref:image} its image and DEPENDS_ON {via:build} the Dockerfile its build + * names (a context, or context + dockerfile). */ +TEST(probe_compose_services_are_resources) { + static const GpfFile files[] = { + {"web/Dockerfile", "FROM nginx:1.27\n"}, + {"worker/Dockerfile", "FROM python:3.12\n"}, + {"compose.yaml", + "services:\n web:\n image: acme/web:1\n build: ./web\n" + " depends_on:\n - db\n - cache\n links:\n - cache:c\n" + " db:\n image: postgres:16\n" + " cache:\n image: redis:7\n depends_on:\n db:\n" + " condition: service_healthy\n" + " worker:\n build:\n context: .\n dockerfile: worker/Dockerfile\n"}}; + char *got = gpf_compose(files, 3); + static const char want[] = + "Service/cache -> Image:docker.io/library/redis (USAGE image)\n" + "Service/cache -> Service/db (DEPENDS_ON depends_on)\n" + "Service/db -> Image:docker.io/library/postgres (USAGE image)\n" + "Service/web -> Image:docker.io/acme/web (USAGE image)\n" + "Service/web -> Module:web/Dockerfile (DEPENDS_ON build)\n" + "Service/web -> Service/cache (DEPENDS_ON depends_on)\n" + "Service/web -> Service/db (DEPENDS_ON depends_on)\n" + "Service/worker -> Module:worker/Dockerfile (DEPENDS_ON build)"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "compose got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("compose service edge set"); + } + free(got); + PASS(); +} + +/* A container build command (docker / podman [buildx] build, in a script or + * a Make recipe) builds each tagged image from the Dockerfile -f names + * (against the cwd), else /Dockerfile. A tag of unknown value keeps + * its repository ($(TAG), $VERSION; a registry port is no tag); an unknown + * context, or another docker command, is no build. */ +TEST(probe_shell_docker_builds_build_images) { + static const GpfFile files[] = { + {"Dockerfile", "FROM alpine:3.20\n"}, + {"svc/Dockerfile", "FROM golang:1.22\n"}, + {"Makefile", "DOCKER_REPO ?= quay.io/acme\nTAG ?= $(shell git describe --tags)\n\n" + "image:\n\tdocker build --label a=b -t $(DOCKER_REPO)/app:$(TAG) .\n"}, + {"build.sh", + "#!/bin/sh\ncd \"$(dirname \"$0\")\"\n" + "docker buildx build --platform linux/amd64 --tag ghcr.io/acme/svc:1 -f svc/Dockerfile .\n" + "podman build -t \"localhost:5000/team/x:$VERSION\" svc\n" + "docker build -t unknown-ctx $CTX\ndocker run -t nope .\n"}}; + char *got = gpf_builds(files, 4); + static const char want[] = "ghcr.io/acme/svc -> svc/Dockerfile (docker-build)\n" + "localhost:5000/team/x -> svc/Dockerfile (docker-build)\n" + "quay.io/acme/app -> Dockerfile (docker-build)"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "builds got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("docker build DEPENDS_ON {via:build} edge set"); + } + free(got); + PASS(); +} + +/* The integer `sql` (?1 = the project) yields on the indexed files; -1 on + * failure. */ +static int gpf_query_int(const GpfFile *files, int nfiles, const char *sql) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + int out = -1; + if (store) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = sqlite3_column_int(st, 0); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A repository built from several Dockerfiles, one per component tag (otel's + * demo:-), is no single image: each component built from + * one Dockerfile is an Image of its own (`:*-`), and a + * reference whose tag names a component uses it -- whatever its version. A + * tag naming none, a component built from two Dockerfiles, and a repository + * built from one Dockerfile stay on the repository Image. */ +TEST(probe_image_hub_splits_by_component) { + static const GpfFile files[] = { + {"svc/a/Dockerfile", "FROM alpine:3.20\n"}, + {"svc/b/Dockerfile", "FROM alpine:3.20\n"}, + {"svc/c/Dockerfile", "FROM alpine:3.20\n"}, + {"solo/Dockerfile", "FROM alpine:3.20\n"}, + {"tool/Dockerfile", "FROM alpine:3.20\n"}, + {"tool/Dockerfile.test", "FROM alpine:3.20\n"}, + {"w1/Dockerfile", "FROM alpine:3.20\n"}, + {"w2/Dockerfile.alt", "FROM alpine:3.20\n"}, + {"v/Dockerfile", "FROM alpine:3.20\n"}, + {"p/Dockerfile", "FROM alpine:3.20\n"}, + {"q/Dockerfile", "FROM alpine:3.20\n"}, + {".env", "V=1.0\n"}, + {"compose.yaml", + "services:\n a:\n image: acme/demo:${V}-a\n build: ./svc/a\n" + " b:\n image: acme/demo:${V}-b\n build: ./svc/b\n" + " c:\n image: acme/demo:${V}-c\n build: ./svc/c\n" + " solo:\n image: acme/solo:1\n build: ./solo\n" + " tool:\n image: acme/tool\n build: ./tool\n" + " tooltest:\n image: acme/tool:test\n build:\n context: ./tool\n" + " dockerfile: Dockerfile.test\n" + " w1:\n image: acme/x:1-w\n build: ./w1\n" + " w2:\n image: acme/x:1-w\n build:\n context: ./w2\n" + " dockerfile: Dockerfile.alt\n" + " v:\n image: acme/x:1-v\n build: ./v\n" + " p:\n image: acme/y:1-p\n build: ./p\n" + " q:\n image: acme/y:1-q\n build: ./q\n"}, + {"k8s.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: b\nspec:\n template:\n" + " spec:\n containers:\n - name: b\n image: acme/demo:0.9-b\n" + "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: z\nspec:\n" + " template:\n spec:\n containers:\n - name: z\n" + " image: acme/demo:0.9-zzz\n" + "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: s\nspec:\n" + " template:\n spec:\n containers:\n - name: s\n" + " image: acme/solo:2\n"}}; + enum { NF = sizeof(files) / sizeof(files[0]) }; + char *got = gpf_builds(files, NF); + static const char want_builds[] = "docker.io/acme/demo:*-a -> svc/a/Dockerfile (compose)\n" + "docker.io/acme/demo:*-b -> svc/b/Dockerfile (compose)\n" + "docker.io/acme/demo:*-c -> svc/c/Dockerfile (compose)\n" + "docker.io/acme/solo -> solo/Dockerfile (compose)\n" + "docker.io/acme/tool -> tool/Dockerfile (compose)\n" + "docker.io/acme/tool:test -> tool/Dockerfile.test (compose)\n" + "docker.io/acme/x -> w1/Dockerfile (compose)\n" + "docker.io/acme/x -> w2/Dockerfile.alt (compose)\n" + "docker.io/acme/x:*-v -> v/Dockerfile (compose)\n" + "docker.io/acme/y:*-p -> p/Dockerfile (compose)\n" + "docker.io/acme/y:*-q -> q/Dockerfile (compose)"; + if (!got || strcmp(got, want_builds) != 0) { + fprintf(stderr, "builds got:\n%s\nwant:\n%s\n", got ? got : "(null)", want_builds); + free(got); + FAIL("component Images' build edges"); + } + free(got); + got = gpf_image_users(files, NF); + static const char want_users[] = "Deployment/b -> docker.io/acme/demo:*-b\n" + "Deployment/s -> docker.io/acme/solo\n" + "Deployment/z -> docker.io/acme/demo\n" + "Service/a -> docker.io/acme/demo:*-a\n" + "Service/b -> docker.io/acme/demo:*-b\n" + "Service/c -> docker.io/acme/demo:*-c\n" + "Service/p -> docker.io/acme/y:*-p\n" + "Service/q -> docker.io/acme/y:*-q\n" + "Service/solo -> docker.io/acme/solo\n" + "Service/tool -> docker.io/acme/tool\n" + "Service/tooltest -> docker.io/acme/tool:test\n" + "Service/v -> docker.io/acme/x:*-v\n" + "Service/w1 -> docker.io/acme/x\n" + "Service/w2 -> docker.io/acme/x"; + if (!got || strcmp(got, want_users) != 0) { + fprintf(stderr, "image users got:\n%s\nwant:\n%s\n", got ? got : "(null)", want_users); + free(got); + FAIL("references on component Images"); + } + free(got); + /* acme/y's repository Image lost every edge to its components: none left */ + int orphans = gpf_query_int(files, NF, + "SELECT COUNT(*) FROM nodes n WHERE n.project=?1 AND " + "n.label='Image' AND NOT EXISTS (SELECT 1 FROM edges e WHERE " + "e.source_id=n.id OR e.target_id=n.id)"); + if (orphans != 0) { + fprintf(stderr, "Images without an edge: %d\n", orphans); + FAIL("a repository Image left without edges"); + } + PASS(); +} + +/* Every CONFIGURES {strategy:env_set} edge as + * " -> / ()"; then the joins a scoped + * EnvVar must not have: "#PORT=" and + * "#joined=". */ +static char *gpf_env_setters(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT CASE s.label WHEN " + "'Module' THEN s.file_path ELSE s.name END || ' -> ' || " + "coalesce(json_extract(t.properties,'$.container'),'-') || '/' || t.name || ' (' || " + "json_extract(e.properties,'$.source') || ')' AS x FROM edges e JOIN nodes s ON " + "s.id=e.source_id JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 AND " + "e.type='CONFIGURES' AND json_extract(e.properties,'$.strategy')='env_set' ORDER BY " + "x)), '') || char(10) || '#PORT=' || (SELECT count(*) FROM nodes WHERE project=?1 " + "AND label='EnvVar' AND name='PORT') || char(10) || '#joined=' || (SELECT count(*) " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND (s.label='EnvVar' OR (substr(t.qualified_name,1,7)='__env__' " + "AND json_extract(e.properties,'$.strategy')='env_set')))"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* The environment a container is given: each name a K8s container's env + * (any pod template; value or valueFrom), a compose service's environment + * (map or list) or the last Dockerfile stage's ENV (with the stages it is + * built FROM, a comment inside the continuation skipped) sets is an EnvVar + * of that container -- two containers' PORT are two nodes, neither joined to + * the PORT a program reads. No value reaches the graph. */ +TEST(probe_container_env_sets_scoped_envvars) { + static const GpfFile files[] = { + {"app.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\nspec:\n template:\n" + " spec:\n initContainers:\n - name: init\n image: busybox\n" + " env:\n - name: MODE\n value: v-init-mode\n" + " containers:\n - name: web\n image: nginx\n env:\n" + " - name: PORT\n value: \"v-8080\"\n" + " - name: DB_PASS\n valueFrom:\n secretKeyRef:\n" + " name: db\n key: pass\n" + " - name: POD_IP\n valueFrom:\n fieldRef:\n" + " fieldPath: status.podIP\n" + "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: api\nspec:\n" + " template:\n spec:\n containers:\n - name: api\n" + " image: acme/api\n env:\n - name: PORT\n" + " value: \"v-9090\"\n"}, + {"compose.yaml", "services:\n web:\n image: acme/web\n environment:\n" + " LOG_LEVEL: v-debug\n API_KEY: v-s3cr3t\n" + " worker:\n image: acme/worker\n environment:\n" + " - QUEUE=v-jobs\n - VERBOSE\n"}, + {"Dockerfile", "FROM golang:1.22 AS build\nENV CGO_ENABLED=0\n" + "FROM alpine:3.20 AS base\nENV TZ=v-utc\n" + "FROM base\nENV APP_HOME=/v-app \\\n # where it lives\n" + " GREETING=\"v-hello world\"\n"}, + {"main.go", "package main\n\nimport \"os\"\n\nfunc main() { _ = os.Getenv(\"PORT\") }\n"}}; + char *got = gpf_env_setters(files, 4); + static const char want[] = "Deployment/api -> api/PORT (value)\n" + "Deployment/web -> init/MODE (value)\n" + "Deployment/web -> web/DB_PASS (secretKeyRef)\n" + "Deployment/web -> web/POD_IP (fieldRef)\n" + "Deployment/web -> web/PORT (value)\n" + "Dockerfile -> -/APP_HOME (dockerfile)\n" + "Dockerfile -> -/GREETING (dockerfile)\n" + "Dockerfile -> -/TZ (dockerfile)\n" + "Service/web -> web/API_KEY (compose)\n" + "Service/web -> web/LOG_LEVEL (compose)\n" + "Service/worker -> worker/QUEUE (compose)\n" + "Service/worker -> worker/VERBOSE (compose)\n" + "#PORT=3\n#joined=0"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "env setters got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("env_set edge set"); + } + free(got); + static const char *const values[] = {"v-init-mode", "v-8080", "v-9090", "v-debug", + "v-s3cr3t", "v-jobs", "v-utc", "v-hello"}; + for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) { + int hits = gpf_graph_mentions(files, 4, values[i]); + if (hits != 0) { + fprintf(stderr, "value %s: %d graph mentions\n", values[i], hits); + FAIL("an env value reached the graph"); + } + } + PASS(); +} + +#define GPF_CM(name) "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " name "\n" +#define GPF_KUST "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n" + +/* A HelmRelease whose chartRef names an OCIRepository deploys the chart that + * repository fetches: the external chart of the oci:// URL's last segment. + * No edge for a URL with a variable, nor for a chartRef naming nothing. */ +TEST(probe_helmrelease_chartref_deploys_its_oci_chart) { + static const GpfFile files[] = { + {"apps/web/kustomization.yaml", + GPF_KUST "namespace: web\nresources:\n - ocirepository.yaml\n - helmrelease.yaml\n"}, + {"apps/web/ocirepository.yaml", + "apiVersion: source.toolkit.fluxcd.io/v1\nkind: OCIRepository\nmetadata:\n" + " name: web-chart\nspec:\n interval: 15m\n ref:\n tag: 1.2.3\n" + " url: oci://ghcr.io/acme/charts/web-app\n"}, + {"apps/web/helmrelease.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: web\n" + "spec:\n chartRef:\n kind: OCIRepository\n name: web-chart\n values:\n" + " fullnameOverride: '{{ .Release.Name }}'\n"}, + {"apps/db/kustomization.yaml", + GPF_KUST "namespace: db\nresources:\n - ocirepository.yaml\n - helmrelease.yaml\n"}, + {"apps/db/ocirepository.yaml", + "apiVersion: source.toolkit.fluxcd.io/v1\nkind: OCIRepository\nmetadata:\n" + " name: db-chart\nspec:\n url: oci://ghcr.io/acme/charts/${CHART}\n"}, + {"apps/db/helmrelease.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: db\n" + "spec:\n chartRef:\n kind: OCIRepository\n name: db-chart\n"}, + {"apps/x/helmrelease.yaml", + "apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: x\n" + "spec:\n chartRef:\n kind: OCIRepository\n name: missing\n"}}; + GPF_ASSERT_DEPLOYS(files, 7, "HelmRelease/web -> chart:web-app(ext) (helmrelease)"); + PASS(); +} + +/* kubectl apply -f deploys a manifest, a directory's manifests (-R: below + * it too; not a file holding no objects), a glob's; -k and a kustomize build + * piped into kubectl a Kustomization. The source is the function running the + * command, top-level code its file; paths resolve against the cd'd root. */ +TEST(probe_shell_kubectl_deploys_what_it_applies) { + static const GpfFile files[] = { + {"manifests/app.yaml", GPF_CM("app")}, + {"manifests/svc.yaml", GPF_CM("svc")}, + {"manifests/notes.yaml", "notes:\n - not an object\n"}, + {"manifests/nested/cm.yaml", GPF_CM("cm")}, + {"crds/a.yaml", GPF_CM("a")}, + {"crds/b.yaml", GPF_CM("b")}, + {"overlays/prod/kustomization.yaml", GPF_KUST}, + {"overlays/dev/kustomization.yaml", GPF_KUST}, + {"hack/deploy.sh", "#!/usr/bin/env bash\n" + "cd \"$(dirname \"$0\")/..\"\n\n" + "apply_all() {\n" + " kubectl apply -f manifests/app.yaml\n" + " kubectl --namespace \"$NS\" apply -f manifests\n" + " kubectl apply -Rf manifests/nested\n" + "}\n\n" + "kubectl apply -f 'crds/*.yaml'\n" + "kubectl apply -k overlays/prod\n" + "kustomize build overlays/dev | sed s/a/b/ | kubectl apply -f -\n" + "cat < File:crds/a.yaml (kubectl)\n" + "@hack/deploy.sh -> File:crds/b.yaml (kubectl)\n" + "@hack/deploy.sh -> Resource:overlays/dev/kustomization.yaml (kustomize)\n" + "@hack/deploy.sh -> Resource:overlays/prod/kustomization.yaml (kubectl)\n" + "apply_all -> File:manifests/app.yaml (kubectl)\n" + "apply_all -> File:manifests/nested/cm.yaml (kubectl)\n" + "apply_all -> File:manifests/svc.yaml (kubectl)"); + PASS(); +} + +/* helm install / upgrade deploys its chart: a chart dir (through options, + * a wrapper, $(git rev-parse --show-toplevel)), else the external chart of + * repo/name or oci://.../name; a helm template piped into kubectl too. No + * edge from an unknown word that may shift the chart's position, nor from + * a variable an if/else leaves undecided. */ +TEST(probe_shell_helm_deploys_its_chart) { + static const GpfFile files[] = { + {"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 0.1.0\n"}, + {"charts/web/values-prod.yaml", "replicas: 2\n"}, + {"charts/api/Chart.yaml", "apiVersion: v2\nname: api\nversion: 0.1.0\n"}, + {"scripts/install.sh", + "#!/bin/sh\n" + "ROOT=\"$(git rev-parse --show-toplevel)\"\n" + "NS=${NAMESPACE:-web}\n" + "install_web() {\n" + " retry 3 helm upgrade --install --namespace \"$NS\" --create-namespace --atomic " + "web \"$ROOT/charts/web/\"\n" + "}\n" + "install_cache() {\n" + " helm install cache bitnami/redis --version 1.2.3\n" + " helm install queue oci://registry.example.com/charts/nats\n" + "}\n" + "render_web() {\n" + " helm template web charts/web -f charts/web/values-prod.yaml | kubectl apply -f -\n" + "}\n" + "split_args() {\n" + " helm upgrade $ARGS charts/api\n" + "}\n" + "branchy() {\n" + " CHART=charts/web\n" + " if [ -n \"$PROD\" ]; then CHART=charts/api; fi\n" + " helm upgrade --install x \"$CHART\"\n" + "}\n"}}; + GPF_ASSERT_DEPLOYS(files, 4, + "install_cache -> chart:nats(ext) (helm)\n" + "install_cache -> chart:redis(ext) (helm)\n" + "install_web -> chart:web (helm)\n" + "render_web -> chart:web (helm)"); + PASS(); +} + +/* Constant propagation: SCRIPT_ROOT from $(cd "$(dirname ...)" && pwd -P), + * pushd / popd, an option whose value is unknown, an array holding the + * command; a subshell's cd does not leak; with no directory tracked a path + * resolves against the script's dir or the root -- only when one of the two + * has it. */ +TEST(probe_shell_paths_follow_the_script) { + static const GpfFile files[] = { + {"apps/web/deployment.yaml", GPF_CM("deployment")}, + {"apps/web/service.yaml", GPF_CM("service")}, + {"apps/web/sub/cm.yaml", GPF_CM("cm")}, + {"apps/web/sub/extra.yaml", GPF_CM("extra")}, + {"apps/web/config.yaml", GPF_CM("config-here")}, + {"config.yaml", GPF_CM("config-root")}, + {"apps/web/deploy.sh", + "#!/usr/bin/env bash\n" + "SCRIPT_ROOT=$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd -P)\n" + "context=\"${KUBECTL_CONTEXT:-gke}\"\n" + "if ! kubectl config get-contexts \"${context}\"; then\n" + " context=\"other\"\n" + "fi\n" + "pushd \"${SCRIPT_ROOT}\" >/dev/null\n" + "kubectl --context=\"${context}\" apply -f .\n" + "popd >/dev/null\n" + "( cd sub && kubectl apply -f cm.yaml )\n" + "kubectl apply -f extra.yaml\n" + "ambiguous() { kubectl apply -f config.yaml; }\n" + "unique() { kubectl apply -f deployment.yaml; }\n" + "deploy() {\n" + " local kubectl=(\n" + " kubectl\n" + " --context=\"${context}\"\n" + " )\n" + " \"${kubectl[@]}\" apply -f service.yaml\n" + "}\n"}}; + GPF_ASSERT_DEPLOYS(files, 7, + "@apps/web/deploy.sh -> File:apps/web/config.yaml (kubectl)\n" + "@apps/web/deploy.sh -> File:apps/web/deployment.yaml (kubectl)\n" + "@apps/web/deploy.sh -> File:apps/web/service.yaml (kubectl)\n" + "@apps/web/deploy.sh -> File:apps/web/sub/cm.yaml (kubectl)\n" + "deploy -> File:apps/web/service.yaml (kubectl)\n" + "unique -> File:apps/web/deployment.yaml (kubectl)"); + PASS(); +} + +/* A program a script starts by a path SPAWNS the Process of its name: from + * a value, or a literal last segment after an unknown prefix; inside $( ) + * and < <( ) too; past a wrapper. Not a script of the repository, not + * `command -v X` (a lookup, not a run). */ +TEST(probe_shell_programs_by_path_spawn) { + static const GpfFile files[] = { + {"hack/lib.sh", "#!/bin/sh\necho lib\n"}, + {"hack/run.sh", + "#!/usr/bin/env bash\n" + "cd \"$(dirname \"$0\")/..\"\n" + "./_build/kubermatic-installer deploy kubermatic-master --config x.yaml\n" + "\"${INSTALL_DIR}/kubermatic-installer\" deploy seed-mla\n" + "./hack/lib.sh\n" + "if command -v bin/checker >/dev/null; then echo ok; fi\n" + "version=$(bin/werf version)\n" + "retry 3 /usr/local/bin/helm version\n" + "while read -r l; do echo \"$l\"; done < <(./bin/lister --all)\n" + "run() { bin/tool --flag; }\n"}}; + GPF_ASSERT_SPAWNS(files, 2, + "@hack/run.sh -> helm\n" + "@hack/run.sh -> kubermatic-installer\n" + "@hack/run.sh -> lister\n" + "@hack/run.sh -> werf\n" + "run -> tool"); + PASS(); +} + +/* Make recipes start programs by path as well: through a variable whose + * prefix is unknown ($(GOBIN)/controller-gen); in a rule tree-sitter-make + * leaves inside an ERROR (an indented, nested ifeq before it), with a + * variable assigned after the conditionals; and line by line where the + * grammar runs one recipe_line over several lines (a quote opened in + * "Don't" and closed lines later) -- there a repository script stays out. */ +TEST(probe_make_programs_by_path_spawn) { + static const GpfFile files[] = { + {"a/tools/validate.sh", "#!/bin/sh\necho ok\n"}, + {"a/Makefile", "check:\n" + "\t@docker run --rm img -f $(file)\n" + "\t@./tools/validate.sh\n" + "\t@./_build/tool run\n" + "\t@echo \"Don't forget\"\n" + "\tdocker run --rm img -c \"/app/check.sh 2>/dev/null | sed '/Checking all " + "the pages/ d; /^Maybe wrong/d' | sort -u\"\n"}, + {"b/Makefile", "LOCALBIN ?= $(shell pwd)/bin\n\n" + "envs:\n" + " ifeq ($(A),)\n" + " \t\texport A=1\n" + " endif\n" + " ifeq ($(B),)\n" + "\t\tifeq ($(C),)\n" + "\t\t\texport B := $(C)\n" + "\t\telse\n" + "\t\t\texport B := x\n" + "\t\tendif\n" + " \tendif\n\n" + "\texport R := $(B)\n\n" + "TOOL ?= $(LOCALBIN)/lint-tool\n\n" + "build: envs\n" + "\tbin/werf build\n" + "\t$(TOOL) run\n"}, + {"c/Makefile", "GOBIN ?= $(shell go env GOPATH)/bin\n" + "CONTROLLER_GEN = $(GOBIN)/controller-gen\n\n" + "generate:\n" + "\t$(CONTROLLER_GEN) object paths=./...\n\n" + "probe:\n" + "\t@command -v bin/checker >/dev/null || echo missing\n"}}; + GPF_ASSERT_SPAWNS(files, 4, + "build -> lint-tool\n" + "build -> werf\n" + "check -> tool\n" + "generate -> controller-gen"); + PASS(); +} + +/* A script of the repository a command runs SPAWNS that script (its Module): + * by its path, through a shell (bash -e FILE), past `env A="$X/y"`; not + * itself. CI runs commands too: a GitHub workflow's run: steps from the + * workspace (`${{ github.workspace }}`, env literals, working-directory; + * an unknown `${{ }}` prefix leaves a Process; no pwsh step, no step of a + * Windows job), a composite action's from its directory + * (`${{ github.action_path }}`), a decorated Prow job's container command + * exec'd from the checkout (`bash -c TEXT` as shell; undecorated: the + * directory unknown, so a Process). Their edges carry the CI and the job. */ +TEST(probe_ci_and_scripts_run_scripts) { + static const GpfFile files[] = { + {"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 0.1.0\n"}, + {"hack/lib/helper.sh", "#!/bin/sh\necho helper\n"}, + {"hack/lint.sh", "#!/bin/sh\necho lint\n"}, + {"hack/ci/run-e2e.sh", "#!/bin/sh\necho e2e\n"}, + {"hack/ci/lint.sh", "#!/bin/sh\necho lint\n"}, + {"ops/apply.sh", "#!/bin/sh\necho apply\n"}, + {"hack/test.sh", "#!/usr/bin/env bash\n" + "bash -e \"$(dirname \"$0\")/lib/helper.sh\"\n" + "env OUT=\"$TMPDIR/x.rpm\" ./hack/lint.sh\n"}, + {"hack/e2e.sh", "#!/bin/sh\n" + "./hack/test.sh\n" + "[ -n \"$AGAIN\" ] || AGAIN=1 \"$0\"\n"}, + {"Makefile", "test:\n" + "\t./hack/test.sh\n" + "\t/bin/bash hack/lint.sh\n"}, + {".github/workflows/ci.yml", + "name: ci\n" + "on: [push]\n" + "env:\n" + " CHART_DIR: charts/web\n" + "jobs:\n" + " test:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - uses: actions/checkout@v4\n" + " - run: ./hack/test.sh\n" + " - name: multi\n" + " run: |\n" + " sh hack/lint.sh --fix\n" + " ${{ github.workspace }}/hack/e2e.sh\n" + " helm upgrade --install web \"$CHART_DIR\"\n" + " ./bin/tool --x\n" + " - working-directory: ops\n" + " run: ./apply.sh\n" + " - run: \"${{ matrix.dir }}/hack/other.sh\"\n" + " - shell: pwsh\n" + " run: ./hack/ci/lint.sh\n" + " win:\n" + " runs-on: windows-latest\n" + " steps:\n" + " - run: ./hack/ci/run-e2e.sh\n"}, + {".github/actions/setup/setup.sh", "#!/bin/sh\necho setup\n"}, + {".github/actions/setup/action.yml", "name: setup\n" + "runs:\n" + " using: composite\n" + " steps:\n" + " - shell: bash\n" + " run: ${{ github.action_path }}/setup.sh\n"}, + {".prow/jobs.yaml", "presubmits:\n" + " - name: pre-e2e\n" + " decorate: true\n" + " spec:\n" + " containers:\n" + " - image: x\n" + " command:\n" + " - \"./hack/ci/run-e2e.sh\"\n" + " - name: pre-lint\n" + " decorate: true\n" + " spec:\n" + " containers:\n" + " - command: [\"bash\", \"-c\", \"./hack/ci/lint.sh\"]\n" + " - name: pre-raw\n" + " spec:\n" + " containers:\n" + " - command: [\"./hack/ci/run-e2e.sh\"]\n"}}; + GPF_ASSERT_RUNS( + files, 13, + "@.github/actions/setup/action.yml -> script:.github/actions/setup/setup.sh " + "(SPAWNS shell github-actions)\n" + "@.github/workflows/ci.yml -> chart:web (DEPLOYS helm github-actions/test)\n" + "@.github/workflows/ci.yml -> proc:other.sh (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> proc:tool (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> script:hack/e2e.sh (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> script:hack/lint.sh (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> script:hack/test.sh (SPAWNS shell github-actions/test)\n" + "@.github/workflows/ci.yml -> script:ops/apply.sh (SPAWNS shell github-actions/test)\n" + "@.prow/jobs.yaml -> proc:run-e2e.sh (SPAWNS exec prow/pre-raw)\n" + "@.prow/jobs.yaml -> script:hack/ci/lint.sh (SPAWNS shell prow/pre-lint)\n" + "@.prow/jobs.yaml -> script:hack/ci/run-e2e.sh (SPAWNS exec prow/pre-e2e)\n" + "@hack/e2e.sh -> script:hack/test.sh (SPAWNS shell)\n" + "@hack/test.sh -> script:hack/lib/helper.sh (SPAWNS shell)\n" + "@hack/test.sh -> script:hack/lint.sh (SPAWNS shell)\n" + "test -> proc:bash (SPAWNS shell)\n" + "test -> script:hack/lint.sh (SPAWNS shell)\n" + "test -> script:hack/test.sh (SPAWNS shell)"); + PASS(); +} + +/* The Make model of a project: every Function of a Makefile / .mk as + * "fn : -[ +variants]" and every IMPORTS edge of one as + * "import ->