trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-10-22 06:09:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(infra): a script's curl / wget request HTTP_CALLS its Route

Browse Source 
Shell scripts, Make recipes and CI steps download tools and call services
with curl and wget; the graph had no edge for any of them, so neither what
a repository fetches nor the scripts that call its own services were
visible.

Such a command now HTTP_CALLS the Route of each URL it states, with
{via: curl | wget, method, url_path, host, line}:
- the method from the flags: -X / --request / --method, a body (-d, -F,
  --data*, --json, --form*, wget --post-data / --post-file) POST, -T PUT,
  -I HEAD, -G back to GET;
- the URL from constant words (VER=1.2.3 ... v${VER}), unknown parts `{}`
  as in a Route path -- also in an unquoted word tree-sitter-bash splits
  around its expansions; a URL whose host is not literal, or the value of a
  proxy / referer option, is no request;
- never the user info, the query or the fragment: a token in a URL does not
  reach the graph;
- the Route: path-only (__route__POST__/v1/items) when the host is a
  Kubernetes Service of the repository in a service DNS form, where that
  service's handler meets it as a code client's request does; any other
  host -- a download, localhost, a compose service -- keeps it
  (__route__GET__https://dl.k8s.io/release/{}/bin/linux/amd64/kubectl,
  props {method, host, external}), and route_nodes makes no path-only twin
  of it.

Incremental indexing now drops a Route without a file that no edge touches
any more -- its last caller's file changed or went. A full index never
holds one; before, the Route of a deleted code client stayed behind too.

Measured on 17 infrastructure repositories: 92 edges, every one correct by
hand; an independent reading of the scripts and Makefiles finds 82
requests, 81 of them in the graph (the last is lost in a file
tree-sitter-bash cannot parse past `[ "$v" \< "1.33" ]`). No property of
any other node or edge changes.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel
2026-10-14 00:12:55 +02:00
parent bd928b3eba
commit 3b93db06b2
10 changed files with 572 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -247,7 +247,7 @@ The install script placed beside the binary is **reported, not deleted** — uni
- `CALL_REFERENCE` — a callable is used at a supported reference site (for example, a direct value argument) and resolves to one exact target
- `USAGE` — an identifier is used, but a unique callable target is not proven (including ambiguous or complex expressions)
- `IMPORTS`, `DEFINES`, `IMPLEMENTS`, `INHERITS`
- `HTTP_CALLS`, `ASYNC_CALLS` (cross-service)
- `HTTP_CALLS`, `ASYNC_CALLS` (cross-service); a shell script, Make recipe or CI step running `curl` / `wget` on a URL it states (constant words resolved, unknown parts `{}`) to its `Route` with the method its flags give: a Kubernetes Service of the repository by the path, as code clients do, any other host (a download, localhost) with the host kept — the edge keeps `via`, `method`, `url_path`, `host`, never the user info or the query
- `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, a shell command or Make recipe running a program by path, ...); the target is a `Process` node named after the program, or `<dynamic>` when the program is not a literal. When a shell script, Make recipe, GitHub Actions step or Prow job runs a script of the repository (`./hack/x.sh`, `bash hack/x.sh`), the target is that script's `Module` (else its `File`); `make TARGET` (also `$(MAKE)`, `-C`, `-f`) targets the rule of that target, followed through included makefiles; edges from CI carry `ci` and `job`. A Dockerfile's last stage `ENTRYPOINT` (else `CMD`) spawns what the container runs (`api: dockerfile`): the repository script or source copied into the image, the Go `main` of a binary a stage builds with `go build -o`, else a `Process`
- `INFRA_MAPS` — a Kubernetes Service to each workload whose pod labels carry every `key: value` of the Service's selector, within one namespace
- `CONNECTS_TO` — a Kubernetes workload or compose service to the Service its containers' environment addresses (`PRODUCT_CATALOG_SERVICE_ADDR: productcatalogservice:3550`, a URL's host, `<svc>.<ns>.svc...`, a bare name on an `*_ADDR` / `*_HOST` / `*_URL` / ... key; compose: the project's `.env` interpolated): a Kubernetes Service of the same bundle and namespace, or a compose service of the same file. The edge keeps `via: env`, `key` and `container` — never the value
src/graph_buffer/graph_buffer.c
+43 -5
View File
@@ -1550,10 +1550,10 @@ static bool gbuf_node_touched(const cbm_gbuf_t *gb, int64_t id, const char *cons
return false;
}
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) {
if (!gb || !label || !edge_types || !edge_types[0]) {
return CBM_NOT_FOUND;
}
/* Delete the `label` nodes no edge of `types` (NULL-terminated) touches --
* with `fileless`, only those carrying no file; returns how many. */
static int gbuf_delete_untouched(cbm_gbuf_t *gb, const char *label, const char *const *types,
bool fileless) {
node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label);
if (!arr || arr->count == 0) {
return 0;
@@ -1568,7 +1568,8 @@ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *
for (int i = 0; i < arr->count; i++) {
const char *qn = arr->items[i]->qualified_name;
cbm_gbuf_node_t *node = qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL;
if (node && !gbuf_node_touched(gb, node->id, edge_types)) {
if (node && !(fileless && node->file_path && node->file_path[0]) &&
!gbuf_node_touched(gb, node->id, types)) {
victims[n++] = node;
}
}
@@ -1604,6 +1605,43 @@ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *
return n;
}
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) {
if (!gb || !label || !edge_types || !edge_types[0]) {
return CBM_NOT_FOUND;
}
return gbuf_delete_untouched(gb, label, edge_types, false);
}
typedef struct {
const char **items;
uint32_t count;
} gb_type_list_t;
static void collect_edge_type(const char *key, void *value, void *ud) {
(void)value;
gb_type_list_t *list = ud;
list->items[list->count++] = key;
}
int cbm_gbuf_delete_edgeless(cbm_gbuf_t *gb, const char *label) {
if (!gb || !label) {
return CBM_NOT_FOUND;
}
/* Edges are indexed by (node, type) only: "any edge" is an edge of any
* type the buffer holds (NULL-terminated by the calloc). */
gb_type_list_t list = {
cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX,
((size_t)cbm_ht_count(gb->edges_by_type) + 1) * sizeof(*list.items)),
0};
if (!list.items) {
return CBM_NOT_FOUND;
}
cbm_ht_foreach(gb->edges_by_type, collect_edge_type, &list);
int n = gbuf_delete_untouched(gb, label, list.items, true);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, list.items);
return n;
}
int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project) {
if (!gb || !db_path || !project) {
return CBM_NOT_FOUND;
src/graph_buffer/graph_buffer.h
+7
View File
@@ -124,6 +124,13 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path);
* is gone, as a full index never makes them. */
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types);
/* Delete every `label` node that carries no file and that no edge of any
* type touches; returns how many. For a label made by edges of many kinds
* (Route: HTTP_CALLS, ASYNC_CALLS, HANDLES, ...): incremental indexing drops
* the file-less one whose last edge went with its files, as a full index
* never makes it. */
int cbm_gbuf_delete_edgeless(cbm_gbuf_t *gb, const char *label);
/* Bulk-load all nodes and edges for a project from an existing SQLite DB
* into this graph buffer. Returns 0 on success. */
int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project);
src/pipeline/pass_route_nodes.c
+11
View File
@@ -78,6 +78,17 @@ static void route_edge_visitor(const cbm_gbuf_edge_t *edge, void *userdata) {
if (strcmp(edge->type, "HTTP_CALLS") != 0 && strcmp(edge->type, "ASYNC_CALLS") != 0) {
return;
}
/* A script's request (pass_shell.c) names its host and already ends at
* its Route, which keeps the host for one outside the repository: a
* path-only Route made from its url_path would meet a handler it never
* calls. A code client's edge names none and refreshes its Route here. */
char host_buf[CBM_SZ_256];
if (json_extract(edge->properties_json, "host", host_buf, sizeof(host_buf))) {
const cbm_gbuf_node_t *target = cbm_gbuf_find_by_id(ctx->gb, edge->target_id);
if (target && target->label && strcmp(target->label, "Route") == 0) {
return;
}
}
/* Extract url_path from properties */
char url_buf[CBM_SZ_512];
src/pipeline/pass_shell.c
+280
View File
@@ -60,6 +60,7 @@
#include "cbm.h"
#include "lang_specs.h"
#include <ctype.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
@@ -378,6 +379,7 @@ typedef struct {
bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */
bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */
bool build; /* the image `ext` is built from the Dockerfile `path` */
bool http; /* HTTP_CALLS the URL `ext` (scheme://host/path) with method `path` */
} sh_hit_t;
typedef struct {
@@ -2040,6 +2042,187 @@ static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) {
sh_hit_push(s, cmd, (sh_hit_t){.via = s->exec ? "exec" : "shell", .ext = name, .spawn = true});
}
/* ── HTTP requests (F3) ─────────────────────────────────────────────── */
/* `url` (http / https) as scheme://host[:port]/path -- the user info, the
* query and the fragment dropped, so no credential or token reaches the
* graph; NULL when it is not one, or its host is unknown. */
static const char *sh_url_norm(CBMArena *a, const char *url) {
const char *scheme_end = strstr(url, "://");
if (!scheme_end || (strncmp(url, "http://", strlen("http://")) != 0 &&
strncmp(url, "https://", strlen("https://")) != 0)) {
return NULL;
}
const char *auth = scheme_end + strlen("://");
size_t al = strcspn(auth, "/?#");
const char *at = NULL;
for (size_t i = 0; i < al; i++) {
if (auth[i] == '@') {
at = auth + i;
}
}
const char *host = at ? at + SKIP_ONE : auth;
size_t hl = (size_t)(auth + al - host);
const char *h = hl ? cbm_arena_strndup(a, host, hl) : NULL;
if (!h || strpbrk(h, "$`{}\"' \t\\")) {
return NULL;
}
const char *path = auth + al;
size_t pl = strcspn(path, "?#");
return cbm_arena_sprintf(a, "%.*s%.*s%.*s", (int)(auth - url), url, (int)hl, host,
pl ? (int)pl : SKIP_ONE, pl ? path : "/");
}
/* Skip one expansion at p (`${...}`, `$(...)`, `$NAME`, a backquoted
* command); returns the byte after it. */
static const char *sh_url_skip_expansion(const char *p) {
if (*p == '`') {
const char *e = strchr(p + SKIP_ONE, '`');
return e ? e + SKIP_ONE : p + strlen(p);
}
char open = p[SKIP_ONE];
if (open == '{' || open == '(') {
char close = open == '{' ? '}' : ')';
int depth = 0;
for (const char *q = p + SKIP_ONE; *q; q++) {
depth += *q == open;
depth -= *q == close;
if (depth == 0) {
return q + SKIP_ONE;
}
}
return p + strlen(p);
}
const char *q = p + SKIP_ONE;
while (isalnum((unsigned char)*q) || *q == '_') {
q++;
}
return q;
}
/* An http(s) URL word the script states with a literal host but variable
* parts after it ("https://github.com/x/releases/download/${VER}/x.tgz"),
* from its source text as the shell reads the word -- up to an unquoted
* blank or operator, as tree-sitter-bash may split an unquoted word with
* expansions into several arguments: quotes dropped, each expansion of the
* path a `{}` as in a Route path; then sh_url_norm. NULL when the host is
* not literal. */
static const char *sh_url_template(sh_t *s, const sh_argv_t *a, int i) {
sh_str_t out = {0};
char quote = 0;
for (const char *p = s->src + a->span[PAIR_LEN * i]; *p;) {
if (!quote && (strchr(" \t\r\n|;&<>)", *p) || (*p == '\\' && p[SKIP_ONE] == '\n'))) {
break;
}
if ((*p == '"' || *p == '\'') && (!quote || quote == *p)) {
quote = quote ? 0 : *p;
p++;
} else if ((*p == '$' || *p == '`') && quote != '\'') {
sh_str_add(s->a, &out, "{}", PAIR_LEN);
p = sh_url_skip_expansion(p);
} else {
sh_str_add(s->a, &out, p, SKIP_ONE);
p++;
}
}
const char *raw = sh_str_done(&out);
return raw ? sh_url_norm(s->a, raw) : NULL; /* a `{}` host is no literal one */
}
static bool sh_ieq(const char *a, const char *b) {
for (; *a && *b; a++, b++) {
if (toupper((unsigned char)*a) != toupper((unsigned char)*b)) {
return false;
}
}
return *a == *b;
}
/* Options whose value is a URL that is not the request: a proxy, a referer. */
static bool sh_http_url_option(const char *o) {
static const char *const OPTS[] = {"-x", "--proxy", "--preproxy", "-e",
"--referer", "--doh-url", "--proxy1.0", NULL};
return sh_in(o, OPTS);
}
/* The method curl / wget sends: -X / --request / --method; a body (-d, -F,
* --data*, --json, --form*, --post-data / --post-file) POST, -T PUT, -I HEAD,
* -G back to GET; default GET. */
static const char *sh_http_method(const sh_argv_t *a, int from) {
static const char *const METHODS[] = {"GET", "POST", "PUT", "PATCH",
"DELETE", "HEAD", "OPTIONS", NULL};
const char *method = "GET";
bool get = false;
for (int i = from; i < a->n; i++) {
const char *w = a->v[i];
if (!w) {
continue;
}
const char *named = NULL;
if (sh_eq(w, "-X") || sh_eq(w, "--request") || sh_eq(w, "--method")) {
named = i + SKIP_ONE < a->n ? a->v[i + SKIP_ONE] : NULL;
} else if (sh_starts(w, "-X") && w[PAIR_LEN]) {
named = w + PAIR_LEN;
} else if (sh_starts(w, "--request=") || sh_starts(w, "--method=")) {
named = strchr(w, '=') + SKIP_ONE;
}
for (int m = 0; named && METHODS[m]; m++) {
if (sh_ieq(named, METHODS[m])) {
return METHODS[m];
}
}
if (sh_eq(w, "-G") || sh_eq(w, "--get")) {
get = true;
} else if (sh_eq(w, "-I") || sh_eq(w, "--head")) {
method = "HEAD";
} else if (sh_eq(w, "-T") || sh_starts(w, "--upload-file")) {
method = "PUT";
} else if (sh_eq(w, "-d") || sh_eq(w, "-F") || sh_starts(w, "--data") ||
sh_starts(w, "--json") || sh_starts(w, "--form") ||
sh_starts(w, "--post-data") || sh_starts(w, "--post-file")) {
method = strcmp(method, "GET") == 0 ? "POST" : method;
}
}
return get && strcmp(method, "POST") == 0 ? "GET" : method;
}
/* curl / wget with a URL the script states (constant words): an HTTP
* request to it, from the command's function. */
static void sh_http(sh_t *s, TSNode cmd, const sh_argv_t *a) {
int p = sh_program(s, a);
const char *prog = p >= 0 && a->v[p] ? sh_basename(a->v[p]) : NULL;
if (!prog || (strcmp(prog, "curl") != 0 && strcmp(prog, "wget") != 0)) {
return;
}
const char *via = strcmp(prog, "curl") == 0 ? "curl" : "wget";
const char *method = sh_http_method(a, p + SKIP_ONE);
for (int i = p + SKIP_ONE; i < a->n; i++) {
/* arguments that touch in the source are one shell word, which
* tree-sitter-bash may split around an expansion */
int j = i;
while (j + SKIP_ONE < a->n &&
a->span[PAIR_LEN * (j + SKIP_ONE)] == a->span[PAIR_LEN * j + SKIP_ONE]) {
j++;
}
bool url_option =
i > p + SKIP_ONE && a->v[i - SKIP_ONE] && sh_http_url_option(a->v[i - SKIP_ONE]);
sh_str_t word = {0};
for (int k = i; k <= j; k++) {
sh_str_add(s->a, &word, a->v[k], a->v[k] ? strlen(a->v[k]) : 0);
}
const char *w = sh_str_done(&word);
/* a `$` left in an evaluated word is an expansion the grammar took
* for text (`$OS-$ARCH.tgz`): no value, so the template */
const char *url = url_option ? NULL
: w && !strpbrk(w, "$`") ? sh_url_norm(s->a, w)
: sh_url_template(s, a, i);
if (url) {
sh_hit_push(s, cmd, (sh_hit_t){.via = via, .ext = url, .path = method, .http = true});
}
i = j;
}
}
/* ── Statements ────────────────────────────────────────────────────── */
static void sh_walk(sh_t *s, TSNode n);
@@ -2344,6 +2527,7 @@ static void sh_command(sh_t *s, TSNode cmd) {
sh_deploy_command(s, cmd, &a, NULL, &out);
sh_spawn(s, cmd, &a);
sh_build(s, cmd, &a);
sh_http(s, cmd, &a);
}
sh_argv_free(&a);
}
@@ -2392,6 +2576,7 @@ static void sh_pipe_run(sh_t *s, TSNode n, sh_src_t *src) {
}
sh_spawn(s, cmd, &a);
sh_build(s, cmd, &a);
sh_http(s, cmd, &a);
sh_argv_free(&a);
sh_redirects(s, el);
}
@@ -3479,11 +3664,106 @@ static int sh_link_build(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, CBMHashTabl
return SKIP_ONE;
}
/* A host that is a service of the repository: one DNS label or a
* `<svc>.<ns>.svc...` name whose first label names a Kubernetes Service of
* it. Its Route is the path alone, where that service's handler of the path
* meets it, as code clients do; any other host -- a download, a
* port-forwarded localhost -- stays in the Route, so it never meets a
* handler it does not reach. (Not a compose service: the Docker pass makes
* those after this one in a full run, while a repair holds them already;
* the QN tells them apart, as a repair's proxies carry no properties.) */
static bool sh_http_service(cbm_pipeline_ctx_t *ctx, const char *host) {
const char *first_dot = strchr(host, '.');
const char *svc = first_dot ? strchr(first_dot + SKIP_ONE, '.') : NULL;
bool dns_form = !first_dot || (svc && sh_starts(svc, ".svc") &&
(svc[strlen(".svc")] == '\0' || svc[strlen(".svc")] == '.'));
if (!dns_form || strcmp(host, "localhost") == 0) {
return false;
}
char name[CBM_SZ_256];
snprintf(name, sizeof(name), "Service/%.*s",
(int)(first_dot ? (size_t)(first_dot - host) : strlen(host)), host);
const cbm_gbuf_node_t **nodes = NULL;
int n = 0;
if (cbm_gbuf_find_by_name(ctx->gbuf, name, &nodes, &n) != 0) {
return false;
}
for (int i = 0; i < n; i++) {
if (nodes[i]->label && strcmp(nodes[i]->label, "Resource") == 0 &&
nodes[i]->qualified_name && !strstr(nodes[i]->qualified_name, "::compose/")) {
return true;
}
}
return false;
}
/* An HTTP request a script makes: HTTP_CALLS {via: curl | wget, method,
* url_path, host} to its Route (sh_http_service). */
static int sh_link_http(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x) {
const char *auth = strstr(x->ext, "://");
if (!auth || src <= 0) {
return 0;
}
auth += strlen("://");
size_t al = strcspn(auth, "/");
char host[CBM_SZ_256];
size_t hl = auth[0] == '[' ? strcspn(auth, "]") + SKIP_ONE : strcspn(auth, ":/");
if (hl == 0 || hl > al || hl >= sizeof(host)) {
return 0;
}
for (size_t i = 0; i < hl; i++) {
host[i] = (char)tolower((unsigned char)auth[i]);
}
host[hl] = '\0';
const char *path = auth + al;
char cpath[CBM_SZ_2K];
cbm_route_canon_path(path, cpath, sizeof(cpath));
bool local = sh_http_service(ctx, host);
char qn[CBM_ROUTE_QN_SIZE];
snprintf(qn, sizeof(qn), "__route__%s__%.*s%s", x->path, local ? 0 : (int)(auth + al - x->ext),
x->ext, cpath);
char authority[CBM_SZ_256]; /* the host with its port */
if (al >= sizeof(authority)) {
return 0;
}
for (size_t i = 0; i < al; i++) {
authority[i] = (char)tolower((unsigned char)auth[i]);
}
authority[al] = '\0';
char ehost[CBM_SZ_512];
char epath[CBM_SZ_4K];
cbm_json_escape(ehost, (int)sizeof(ehost), authority);
cbm_json_escape(epath, (int)sizeof(epath), path);
char rprops[CBM_SZ_1K];
if (local) {
snprintf(rprops, sizeof(rprops), "{\"method\":\"%s\"}", x->path);
} else {
snprintf(rprops, sizeof(rprops), "{\"method\":\"%s\",\"host\":\"%s\",\"external\":true}",
x->path, ehost);
}
int64_t route =
cbm_gbuf_upsert_node(ctx->gbuf, "Route", local ? path : x->ext, qn, "", 0, 0, rprops);
if (route <= 0 || route == src) {
return 0;
}
char ci[CBM_SZ_1K];
sh_ci_props(x, ci, sizeof(ci));
char props[CBM_SZ_4K + CBM_SZ_1K];
snprintf(props, sizeof(props),
"{\"via\":\"%s\",\"method\":\"%s\",\"url_path\":\"%s\",\"host\":\"%s\"%s,\"line\":%d}",
x->via, x->path, epath, ehost, ci, x->line);
cbm_gbuf_insert_edge(ctx->gbuf, src, route, "HTTP_CALLS", props);
return SKIP_ONE;
}
static int sh_link_hit(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, int64_t src,
CBMHashTable *modules) {
if (x->spawn) {
return src > 0 ? sh_link_spawn(ctx, src, x, modules) : 0;
}
if (x->http) {
return sh_link_http(ctx, src, x);
}
int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file);
if (src <= 0 || tgt <= 0 || src == tgt) {
return 0;
src/pipeline/pipeline_delta.c
+33 -12
View File
@@ -252,6 +252,38 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const *
return 0;
}
/* Run a node DELETE binding ?1 = project, ?2 = label, ?3.. = the first
* `type_count` of `types`; the number of nodes deleted, or -1. */
static int delta_delete_nodes(sqlite3 *db, const char *sql, const char *project, const char *label,
const char *const *types, int type_count) {
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
return CBM_NOT_FOUND;
}
sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT);
for (int i = 0; i < type_count; i++) {
sqlite3_bind_text(stmt, i + 3, types[i], CBM_NOT_FOUND, SQLITE_TRANSIENT);
}
int rc = sqlite3_step(stmt);
sqlite3_finalize(stmt);
return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND;
}
int cbm_delta_drop_edgeless(cbm_store_t *store, const char *project, const char *label) {
sqlite3 *db = cbm_store_get_db(store);
if (!db) {
return CBM_NOT_FOUND;
}
/* Each NOT EXISTS is served by an (id, type) index's id prefix. */
static const char SQL[] =
"DELETE FROM nodes WHERE project = ?1 AND label = ?2"
" AND (file_path IS NULL OR file_path = '')"
" AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.target_id = nodes.id)"
" AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.source_id = nodes.id)";
return delta_delete_nodes(db, SQL, project, label, NULL, 0);
}
int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label,
const char *const *edge_types) {
sqlite3 *db = cbm_store_get_db(store);
@@ -274,18 +306,7 @@ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const c
if (t == 0 || len <= 0 || (size_t)len >= sizeof(sql)) {
return CBM_NOT_FOUND;
}
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
return CBM_NOT_FOUND;
}
sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT);
for (int i = 0; i < t; i++) {
sqlite3_bind_text(stmt, i + 3, edge_types[i], CBM_NOT_FOUND, SQLITE_TRANSIENT);
}
int rc = sqlite3_step(stmt);
sqlite3_finalize(stmt);
return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND;
return delta_delete_nodes(db, sql, project, label, edge_types, t);
}
/* The first column of `sql` (?1 = the project) as arena strings in *out. */
src/pipeline/pipeline_incremental.c
+8 -3
View File
@@ -3351,9 +3351,11 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
cbm_log_info("delta.patch_done", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
/* The proxy buffer holds no edges, so only the patched staging graph can
* tell a Process node no SPAWNS reaches any more (its last spawner was
* purged), or an Image nothing builds on, builds or runs. */
* purged), an Image nothing builds on, builds or runs, or a file-less
* Route no caller or handler is left on. */
if (cbm_delta_drop_unreferenced(staging, project, "Process", SH_PROCESS_EDGES) < 0 ||
cbm_delta_drop_unreferenced(staging, project, "Image", DK_IMAGE_EDGES) < 0) {
cbm_delta_drop_unreferenced(staging, project, "Image", DK_IMAGE_EDGES) < 0 ||
cbm_delta_drop_edgeless(staging, project, "Route") < 0) {
goto out;
}
@@ -3986,9 +3988,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
* pass_shell.c), an Image only through its DEPENDS_ON / USAGE edges
* (pass_docker.c); neither carries a file: when the last file naming
* one is gone, nothing else removes it -- a full index never makes
* it. This buffer holds the whole graph. */
* it. Nor a file-less Route (a call site's, pass_calls.c,
* pass_shell.c) without its last caller and handler. This buffer
* holds the whole graph. */
(void)cbm_gbuf_delete_unreferenced(existing, "Process", SH_PROCESS_EDGES);
(void)cbm_gbuf_delete_unreferenced(existing, "Image", DK_IMAGE_EDGES);
(void)cbm_gbuf_delete_edgeless(existing, "Route");
phase_rc = run_postpasses(&ctx, changed_files, ci, project, true);
}
src/pipeline/pipeline_internal.h
+8 -2
View File
@@ -1493,8 +1493,10 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr
* image reference says its container (USAGE {ref:image, container}) and
* a Dockerfile's Module its image's files (image_context, image_files,
* image_ignore). A workload or compose service CONNECTS_TO the Service
* its environment addresses (extract_k8s.c, pass_docker.c). An index
* built before holds none of them, so it rebuilds once on upgrade. */
* its environment addresses (extract_k8s.c, pass_docker.c). A script's
* curl / wget HTTP_CALLS the Route of the URL it states (pass_shell.c).
* An index built before holds none of them, so it rebuilds once on
* upgrade. */
enum { CBM_SEMANTIC_INDEX_VERSION = 9 };
typedef struct {
@@ -1591,6 +1593,10 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const *
* on failure. */
int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label,
const char *const *edge_types);
/* Delete the project's `label` nodes that carry no file and that no edge of
* any type touches (a Route whose last caller and handler were purged);
* returns how many, or -1 on failure. */
int cbm_delta_drop_edgeless(cbm_store_t *store, const char *project, const char *label);
/* An Image's build edges (Image -DEPENDS_ON {via:build}-> Dockerfile,
* pass_docker.c) belong to the build configuration declaring them, not to
* either end: every run recomputes them whole from all build files, the
tests/test_grammar_probe_f.c
+103
View File
@@ -2581,6 +2581,108 @@ TEST(probe_env_address_connects_to_its_service) {
PASS();
}
/* Every script HTTP_CALLS edge as "<source file> <route QN>", then a line
* "edgeless routes <n>": the Route nodes no edge touches. */
static char *gpf_script_http(const GpfFile *files, int nfiles) {
GpfProj lp;
cbm_store_t *store = gpf_index_files(&lp, files, nfiles);
char *out = NULL;
if (store) {
sqlite3_stmt *st = NULL;
const char *sql =
"SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || "
"t.qualified_name AS x FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
"t.id=e.target_id WHERE e.project=?1 AND e.type='HTTP_CALLS' AND "
"json_extract(e.properties,'$.via') IN ('curl','wget') ORDER BY x)), '') || char(10) "
"|| 'edgeless routes ' || (SELECT COUNT(*) FROM nodes r WHERE r.project=?1 AND "
"r.label='Route' AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.target_id=r.id OR "
"e.source_id=r.id))";
if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) {
sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC);
if (sqlite3_step(st) == SQLITE_ROW) {
out = strdup((const char *)sqlite3_column_text(st, 0));
}
}
sqlite3_finalize(st);
}
gpf_cleanup(&lp, store);
return out;
}
/* A script's curl / wget of a URL it states HTTP_CALLS the Route of it, with
* the method its flags give (-X, a body POST, -T PUT, -I HEAD, -G GET,
* wget --post-data POST): a Kubernetes Service of the repository by the path
* alone (`api`, `api.default.svc.cluster.local`), where that service's
* handler meets it; any other host -- a download, a domain under the
* service's name, localhost even beside a Service named so -- with the host
* kept, and no path-only Route made for it. Constant words resolve
* (`v${VER}`), unknown parts are `{}`, also in an unquoted word the grammar
* splits; never a proxy's or referer's URL, and never the user info or the
* query (a token) in the graph. */
TEST(probe_script_http_request_calls_its_route) {
static const GpfFile files[] = {
{"k8s/api.yaml", "apiVersion: v1\nkind: Service\nmetadata:\n name: api\n"},
{"k8s/localhost.yaml", "apiVersion: v1\nkind: Service\nmetadata:\n name: localhost\n"},
{"dl.sh",
"#!/bin/bash\nVER=1.2.3\n"
"curl -fsSL https://github.com/x/y/releases/download/v1/tool.tgz -o tool.tgz\n"
"curl -X POST -d '{}' \"http://api:8080/v1/items?token=s3cr3t-token\"\n"
"wget --post-data=x https://user:pw-secret@dl.example.com/a/b\n"
"curl -I https://example.com/head\n"
"curl -x http://proxy.local:3128 https://example.org/target\n"
"curl -LO \"https://dl.example.com/v${VER}/x-${VER}.tgz\"\n"
"curl \"https://dl.example.com/r/${UNKNOWN}/y.tgz\"\n"
"curl http://localhost:9090/metrics\n"
"curl -G -d q=1 https://example.com/search\n"
"curl --request DELETE https://example.com/item/1\n"
"curl -fsSLO https://dl.example.com/u/$OS-$ARCH.tgz\n"
"curl -LO https://dl.example.com/j/`uname`.tgz\n"
"curl -LO https://dl.example.com/q/${VER}$OS.tgz\n"
"curl https://api.example.com/v1/ping\n"
"curl http://api.default.svc.cluster.local:8080/health\n"
"curl \"https://${HOST}/api/x\"\n"},
{"Makefile", "TOOL_VERSION=0.1.1\nARCH=amd64\n"
"install:\n\tcurl -sSL https://get.example.com/install.sh | sh\n"
"bin/tool:\n\tcurl -sLo \"bin/tool\" https://dl.example.com/t/v$(TOOL_VERSION)/"
"tool_v$(TOOL_VERSION)_$(OS)_$(ARCH)\n"}};
int n = (int)(sizeof(files) / sizeof(files[0]));
char *got = gpf_script_http(files, n);
static const char want[] =
"Makefile __route__GET__https://dl.example.com/t/v0.1.1/tool_v0.1.1_{}_amd64\n"
"Makefile __route__GET__https://get.example.com/install.sh\n"
"dl.sh __route__DELETE__https://example.com/item/1\n"
"dl.sh __route__GET__/health\n"
"dl.sh __route__GET__http://localhost:9090/metrics\n"
"dl.sh __route__GET__https://api.example.com/v1/ping\n"
"dl.sh __route__GET__https://dl.example.com/j/{}.tgz\n"
"dl.sh __route__GET__https://dl.example.com/q/{}{}.tgz\n"
"dl.sh __route__GET__https://dl.example.com/r/{}/y.tgz\n"
"dl.sh __route__GET__https://dl.example.com/u/{}-{}.tgz\n"
"dl.sh __route__GET__https://dl.example.com/v1.2.3/x-1.2.3.tgz\n"
"dl.sh __route__GET__https://example.com/search\n"
"dl.sh __route__GET__https://example.org/target\n"
"dl.sh __route__GET__https://github.com/x/y/releases/download/v1/tool.tgz\n"
"dl.sh __route__HEAD__https://example.com/head\n"
"dl.sh __route__POST__/v1/items\n"
"dl.sh __route__POST__https://dl.example.com/a/b\n"
"edgeless routes 0";
if (!got || strcmp(got, want) != 0) {
fprintf(stderr, "script http got:\n%s\nwant:\n%s\n", got ? got : "(null)", want);
free(got);
FAIL("script HTTP_CALLS edge set");
}
free(got);
static const char *const values[] = {"s3cr3t-token", "pw-secret", "proxy.local"};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
int hits = gpf_graph_mentions(files, n, values[i]);
if (hits != 0) {
fprintf(stderr, "value %s: %d graph mentions\n", values[i], hits);
FAIL("a credential, token or proxy reached the graph");
}
}
PASS();
}
/* A variable a container sets joins the readers of its name inside the files
* its own image is built from: the K8s container's image (two containers,
* two images), built by compose from a Dockerfile; the stages the last one
@@ -4913,6 +5015,7 @@ SUITE(grammar_probe_f) {
RUN_TEST(probe_container_env_sets_scoped_envvars);
RUN_TEST(probe_container_env_joins_its_own_image_readers);
RUN_TEST(probe_env_address_connects_to_its_service);
RUN_TEST(probe_script_http_request_calls_its_route);
RUN_TEST(probe_image_hub_splits_by_component);
RUN_TEST(probe_gitops_objects_deploy_their_units);
RUN_TEST(probe_shell_kubectl_deploys_what_it_applies);
tests/test_pipeline.c
+78
View File
@@ -3587,6 +3587,49 @@ TEST(incremental_deleted_spawner_drops_its_process) {
PASS();
}
/* Deleting the only script requesting a URL leaves no Route behind -- a full
* index never makes one without an edge; the Route a kept script requests
* stays. `legacy`: through the full-load route, else closure repair. Returns
* the Route count after the deletion, or -1; *route: the route taken. */
static int incremental_caller_delete(bool legacy, cbm_incremental_route_t *route) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_routegc_XXXXXX");
if (!cbm_mkdtemp(tmp)) {
return -1;
}
write_temp_file(tmp, "hack/a.sh", "#!/bin/sh\ncurl -fsSL https://dl.example.com/a.tgz\n");
write_temp_file(tmp, "hack/b.sh", "#!/bin/sh\nwget https://dl.example.com/b.tgz\n");
char db_path[512];
snprintf(db_path, sizeof(db_path), "%s/routegc.db", tmp);
static const char routes[] = "SELECT COUNT(*) FROM nodes WHERE label='Route'";
int full = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, routes) : -1;
char path[512];
snprintf(path, sizeof(path), "%s/hack/a.sh", tmp);
(void)remove(path);
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
if (legacy) {
cbm_pipeline_incremental_test_force_legacy_partial_once();
}
#endif
int after = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, routes) : -1;
*route = cbm_pipeline_incremental_test_last_route();
th_rmtree(tmp);
return full == 2 ? after : -1;
}
TEST(incremental_deleted_caller_drops_its_route) {
cbm_incremental_route_t route = CBM_INCREMENTAL_ROUTE_NONE;
int left = incremental_caller_delete(false, &route);
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
ASSERT_EQ(left, 1);
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
left = incremental_caller_delete(true, &route);
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_LEGACY_PARTIAL);
ASSERT_EQ(left, 1);
#endif
PASS();
}
/* Deleting the only Dockerfile naming an image leaves no Image node behind;
* the image another Dockerfile still names stays. `legacy`: through the
* full-load route, else the closure-repair route. Returns the Image count
@@ -4857,6 +4900,39 @@ TEST(incremental_env_join_repair_keeps_unchanged_pairs) {
PASS();
}
/* A script's request to a compose service keeps the host in its Route on a
* repair as in a full run: the repair holds the compose services already
* (a full run makes them after the scripts), yet only a Kubernetes Service
* of the repository makes a Route path-only. */
TEST(incremental_script_request_to_compose_service_keeps_host) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_composeurl_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
closure_probe_repo(tmp);
write_temp_file(tmp, "compose.yaml", "services:\n web:\n image: acme/web:1\n");
write_temp_file(tmp, "hack/ping.sh", "#!/bin/sh\ncurl http://web:3000/x\n");
char db[512];
snprintf(db, sizeof(db), "%s/composeurl.db", tmp);
static const char kept[] = "SELECT COUNT(*) FROM nodes WHERE label='Route' AND "
"qualified_name='__route__GET__http://web:3000/y'";
static const char bare[] = "SELECT COUNT(*) FROM nodes WHERE label='Route' AND "
"qualified_name LIKE '__route__GET__/%'";
int full_rc = stem_index(tmp, db);
write_temp_file(tmp, "hack/ping.sh", "#!/bin/sh\ncurl http://web:3000/y\n");
int inc_rc = stem_index(tmp, db);
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
int inc_kept = k8s_inc_count(db, kept);
int inc_bare = k8s_inc_count(db, bare);
th_rmtree(tmp);
ASSERT_EQ(full_rc, 0);
ASSERT_EQ(inc_rc, 0);
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
ASSERT_EQ(inc_kept, 1);
ASSERT_EQ(inc_bare, 0);
PASS();
}
TEST(pipeline_closure_repair_budget_declines_to_full) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_closure_budget_XXXXXX");
@@ -23400,6 +23476,8 @@ SUITE(pipeline_semantic_manifest_repro) {
RUN_TEST(incremental_k8s_edits_reach_other_files);
RUN_TEST(incremental_shell_deploy_edit_keeps_its_edges);
RUN_TEST(incremental_deleted_spawner_drops_its_process);
RUN_TEST(incremental_deleted_caller_drops_its_route);
RUN_TEST(incremental_script_request_to_compose_service_keeps_host);
RUN_TEST(incremental_deleted_dockerfile_drops_its_image);
RUN_TEST(incremental_dockerfile_edit_keeps_build_edges);
RUN_TEST(incremental_compose_edit_replaces_build_edges);