diff --git a/README.md b/README.md index e64318752..e2bf25918 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,7 @@ The install script placed beside the binary is **reported, not deleted** — uni - `CALL_REFERENCE` — a callable is used at a supported reference site (for example, a direct value argument) and resolves to one exact target - `USAGE` — an identifier is used, but a unique callable target is not proven (including ambiguous or complex expressions) - `IMPORTS`, `DEFINES`, `IMPLEMENTS`, `INHERITS` -- `HTTP_CALLS`, `ASYNC_CALLS` (cross-service) +- `HTTP_CALLS`, `ASYNC_CALLS` (cross-service); a shell script, Make recipe or CI step running `curl` / `wget` on a URL it states (constant words resolved, unknown parts `{}`) to its `Route` with the method its flags give: a Kubernetes Service of the repository by the path, as code clients do, any other host (a download, localhost) with the host kept — the edge keeps `via`, `method`, `url_path`, `host`, never the user info or the query - `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, a shell command or Make recipe running a program by path, ...); the target is a `Process` node named after the program, or `` when the program is not a literal. When a shell script, Make recipe, GitHub Actions step or Prow job runs a script of the repository (`./hack/x.sh`, `bash hack/x.sh`), the target is that script's `Module` (else its `File`); `make TARGET` (also `$(MAKE)`, `-C`, `-f`) targets the rule of that target, followed through included makefiles; edges from CI carry `ci` and `job`. A Dockerfile's last stage `ENTRYPOINT` (else `CMD`) spawns what the container runs (`api: dockerfile`): the repository script or source copied into the image, the Go `main` of a binary a stage builds with `go build -o`, else a `Process` - `INFRA_MAPS` — a Kubernetes Service to each workload whose pod labels carry every `key: value` of the Service's selector, within one namespace - `CONNECTS_TO` — a Kubernetes workload or compose service to the Service its containers' environment addresses (`PRODUCT_CATALOG_SERVICE_ADDR: productcatalogservice:3550`, a URL's host, `..svc...`, a bare name on an `*_ADDR` / `*_HOST` / `*_URL` / ... key; compose: the project's `.env` interpolated): a Kubernetes Service of the same bundle and namespace, or a compose service of the same file. The edge keeps `via: env`, `key` and `container` — never the value diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 7065814b5..670717ac1 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -1550,10 +1550,10 @@ static bool gbuf_node_touched(const cbm_gbuf_t *gb, int64_t id, const char *cons return false; } -int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) { - if (!gb || !label || !edge_types || !edge_types[0]) { - return CBM_NOT_FOUND; - } +/* Delete the `label` nodes no edge of `types` (NULL-terminated) touches -- + * with `fileless`, only those carrying no file; returns how many. */ +static int gbuf_delete_untouched(cbm_gbuf_t *gb, const char *label, const char *const *types, + bool fileless) { node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label); if (!arr || arr->count == 0) { return 0; @@ -1568,7 +1568,8 @@ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char * for (int i = 0; i < arr->count; i++) { const char *qn = arr->items[i]->qualified_name; cbm_gbuf_node_t *node = qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL; - if (node && !gbuf_node_touched(gb, node->id, edge_types)) { + if (node && !(fileless && node->file_path && node->file_path[0]) && + !gbuf_node_touched(gb, node->id, types)) { victims[n++] = node; } } @@ -1604,6 +1605,43 @@ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char * return n; } +int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) { + if (!gb || !label || !edge_types || !edge_types[0]) { + return CBM_NOT_FOUND; + } + return gbuf_delete_untouched(gb, label, edge_types, false); +} + +typedef struct { + const char **items; + uint32_t count; +} gb_type_list_t; + +static void collect_edge_type(const char *key, void *value, void *ud) { + (void)value; + gb_type_list_t *list = ud; + list->items[list->count++] = key; +} + +int cbm_gbuf_delete_edgeless(cbm_gbuf_t *gb, const char *label) { + if (!gb || !label) { + return CBM_NOT_FOUND; + } + /* Edges are indexed by (node, type) only: "any edge" is an edge of any + * type the buffer holds (NULL-terminated by the calloc). */ + gb_type_list_t list = { + cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, + ((size_t)cbm_ht_count(gb->edges_by_type) + 1) * sizeof(*list.items)), + 0}; + if (!list.items) { + return CBM_NOT_FOUND; + } + cbm_ht_foreach(gb->edges_by_type, collect_edge_type, &list); + int n = gbuf_delete_untouched(gb, label, list.items, true); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, list.items); + return n; +} + int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project) { if (!gb || !db_path || !project) { return CBM_NOT_FOUND; diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 83470f5ab..a3b5b3cd0 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -124,6 +124,13 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path); * is gone, as a full index never makes them. */ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types); +/* Delete every `label` node that carries no file and that no edge of any + * type touches; returns how many. For a label made by edges of many kinds + * (Route: HTTP_CALLS, ASYNC_CALLS, HANDLES, ...): incremental indexing drops + * the file-less one whose last edge went with its files, as a full index + * never makes it. */ +int cbm_gbuf_delete_edgeless(cbm_gbuf_t *gb, const char *label); + /* Bulk-load all nodes and edges for a project from an existing SQLite DB * into this graph buffer. Returns 0 on success. */ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project); diff --git a/src/pipeline/pass_route_nodes.c b/src/pipeline/pass_route_nodes.c index 3bc472f6a..9f0c57ddb 100644 --- a/src/pipeline/pass_route_nodes.c +++ b/src/pipeline/pass_route_nodes.c @@ -78,6 +78,17 @@ static void route_edge_visitor(const cbm_gbuf_edge_t *edge, void *userdata) { if (strcmp(edge->type, "HTTP_CALLS") != 0 && strcmp(edge->type, "ASYNC_CALLS") != 0) { return; } + /* A script's request (pass_shell.c) names its host and already ends at + * its Route, which keeps the host for one outside the repository: a + * path-only Route made from its url_path would meet a handler it never + * calls. A code client's edge names none and refreshes its Route here. */ + char host_buf[CBM_SZ_256]; + if (json_extract(edge->properties_json, "host", host_buf, sizeof(host_buf))) { + const cbm_gbuf_node_t *target = cbm_gbuf_find_by_id(ctx->gb, edge->target_id); + if (target && target->label && strcmp(target->label, "Route") == 0) { + return; + } + } /* Extract url_path from properties */ char url_buf[CBM_SZ_512]; diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index cce391672..979b3b172 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -60,6 +60,7 @@ #include "cbm.h" #include "lang_specs.h" +#include #include #include #include @@ -378,6 +379,7 @@ typedef struct { bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */ bool build; /* the image `ext` is built from the Dockerfile `path` */ + bool http; /* HTTP_CALLS the URL `ext` (scheme://host/path) with method `path` */ } sh_hit_t; typedef struct { @@ -2040,6 +2042,187 @@ static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) { sh_hit_push(s, cmd, (sh_hit_t){.via = s->exec ? "exec" : "shell", .ext = name, .spawn = true}); } +/* ── HTTP requests (F3) ─────────────────────────────────────────────── */ + +/* `url` (http / https) as scheme://host[:port]/path -- the user info, the + * query and the fragment dropped, so no credential or token reaches the + * graph; NULL when it is not one, or its host is unknown. */ +static const char *sh_url_norm(CBMArena *a, const char *url) { + const char *scheme_end = strstr(url, "://"); + if (!scheme_end || (strncmp(url, "http://", strlen("http://")) != 0 && + strncmp(url, "https://", strlen("https://")) != 0)) { + return NULL; + } + const char *auth = scheme_end + strlen("://"); + size_t al = strcspn(auth, "/?#"); + const char *at = NULL; + for (size_t i = 0; i < al; i++) { + if (auth[i] == '@') { + at = auth + i; + } + } + const char *host = at ? at + SKIP_ONE : auth; + size_t hl = (size_t)(auth + al - host); + const char *h = hl ? cbm_arena_strndup(a, host, hl) : NULL; + if (!h || strpbrk(h, "$`{}\"' \t\\")) { + return NULL; + } + const char *path = auth + al; + size_t pl = strcspn(path, "?#"); + return cbm_arena_sprintf(a, "%.*s%.*s%.*s", (int)(auth - url), url, (int)hl, host, + pl ? (int)pl : SKIP_ONE, pl ? path : "/"); +} + +/* Skip one expansion at p (`${...}`, `$(...)`, `$NAME`, a backquoted + * command); returns the byte after it. */ +static const char *sh_url_skip_expansion(const char *p) { + if (*p == '`') { + const char *e = strchr(p + SKIP_ONE, '`'); + return e ? e + SKIP_ONE : p + strlen(p); + } + char open = p[SKIP_ONE]; + if (open == '{' || open == '(') { + char close = open == '{' ? '}' : ')'; + int depth = 0; + for (const char *q = p + SKIP_ONE; *q; q++) { + depth += *q == open; + depth -= *q == close; + if (depth == 0) { + return q + SKIP_ONE; + } + } + return p + strlen(p); + } + const char *q = p + SKIP_ONE; + while (isalnum((unsigned char)*q) || *q == '_') { + q++; + } + return q; +} + +/* An http(s) URL word the script states with a literal host but variable + * parts after it ("https://github.com/x/releases/download/${VER}/x.tgz"), + * from its source text as the shell reads the word -- up to an unquoted + * blank or operator, as tree-sitter-bash may split an unquoted word with + * expansions into several arguments: quotes dropped, each expansion of the + * path a `{}` as in a Route path; then sh_url_norm. NULL when the host is + * not literal. */ +static const char *sh_url_template(sh_t *s, const sh_argv_t *a, int i) { + sh_str_t out = {0}; + char quote = 0; + for (const char *p = s->src + a->span[PAIR_LEN * i]; *p;) { + if (!quote && (strchr(" \t\r\n|;&<>)", *p) || (*p == '\\' && p[SKIP_ONE] == '\n'))) { + break; + } + if ((*p == '"' || *p == '\'') && (!quote || quote == *p)) { + quote = quote ? 0 : *p; + p++; + } else if ((*p == '$' || *p == '`') && quote != '\'') { + sh_str_add(s->a, &out, "{}", PAIR_LEN); + p = sh_url_skip_expansion(p); + } else { + sh_str_add(s->a, &out, p, SKIP_ONE); + p++; + } + } + const char *raw = sh_str_done(&out); + return raw ? sh_url_norm(s->a, raw) : NULL; /* a `{}` host is no literal one */ +} + +static bool sh_ieq(const char *a, const char *b) { + for (; *a && *b; a++, b++) { + if (toupper((unsigned char)*a) != toupper((unsigned char)*b)) { + return false; + } + } + return *a == *b; +} + +/* Options whose value is a URL that is not the request: a proxy, a referer. */ +static bool sh_http_url_option(const char *o) { + static const char *const OPTS[] = {"-x", "--proxy", "--preproxy", "-e", + "--referer", "--doh-url", "--proxy1.0", NULL}; + return sh_in(o, OPTS); +} + +/* The method curl / wget sends: -X / --request / --method; a body (-d, -F, + * --data*, --json, --form*, --post-data / --post-file) POST, -T PUT, -I HEAD, + * -G back to GET; default GET. */ +static const char *sh_http_method(const sh_argv_t *a, int from) { + static const char *const METHODS[] = {"GET", "POST", "PUT", "PATCH", + "DELETE", "HEAD", "OPTIONS", NULL}; + const char *method = "GET"; + bool get = false; + for (int i = from; i < a->n; i++) { + const char *w = a->v[i]; + if (!w) { + continue; + } + const char *named = NULL; + if (sh_eq(w, "-X") || sh_eq(w, "--request") || sh_eq(w, "--method")) { + named = i + SKIP_ONE < a->n ? a->v[i + SKIP_ONE] : NULL; + } else if (sh_starts(w, "-X") && w[PAIR_LEN]) { + named = w + PAIR_LEN; + } else if (sh_starts(w, "--request=") || sh_starts(w, "--method=")) { + named = strchr(w, '=') + SKIP_ONE; + } + for (int m = 0; named && METHODS[m]; m++) { + if (sh_ieq(named, METHODS[m])) { + return METHODS[m]; + } + } + if (sh_eq(w, "-G") || sh_eq(w, "--get")) { + get = true; + } else if (sh_eq(w, "-I") || sh_eq(w, "--head")) { + method = "HEAD"; + } else if (sh_eq(w, "-T") || sh_starts(w, "--upload-file")) { + method = "PUT"; + } else if (sh_eq(w, "-d") || sh_eq(w, "-F") || sh_starts(w, "--data") || + sh_starts(w, "--json") || sh_starts(w, "--form") || + sh_starts(w, "--post-data") || sh_starts(w, "--post-file")) { + method = strcmp(method, "GET") == 0 ? "POST" : method; + } + } + return get && strcmp(method, "POST") == 0 ? "GET" : method; +} + +/* curl / wget with a URL the script states (constant words): an HTTP + * request to it, from the command's function. */ +static void sh_http(sh_t *s, TSNode cmd, const sh_argv_t *a) { + int p = sh_program(s, a); + const char *prog = p >= 0 && a->v[p] ? sh_basename(a->v[p]) : NULL; + if (!prog || (strcmp(prog, "curl") != 0 && strcmp(prog, "wget") != 0)) { + return; + } + const char *via = strcmp(prog, "curl") == 0 ? "curl" : "wget"; + const char *method = sh_http_method(a, p + SKIP_ONE); + for (int i = p + SKIP_ONE; i < a->n; i++) { + /* arguments that touch in the source are one shell word, which + * tree-sitter-bash may split around an expansion */ + int j = i; + while (j + SKIP_ONE < a->n && + a->span[PAIR_LEN * (j + SKIP_ONE)] == a->span[PAIR_LEN * j + SKIP_ONE]) { + j++; + } + bool url_option = + i > p + SKIP_ONE && a->v[i - SKIP_ONE] && sh_http_url_option(a->v[i - SKIP_ONE]); + sh_str_t word = {0}; + for (int k = i; k <= j; k++) { + sh_str_add(s->a, &word, a->v[k], a->v[k] ? strlen(a->v[k]) : 0); + } + const char *w = sh_str_done(&word); + /* a `$` left in an evaluated word is an expansion the grammar took + * for text (`$OS-$ARCH.tgz`): no value, so the template */ + const char *url = url_option ? NULL + : w && !strpbrk(w, "$`") ? sh_url_norm(s->a, w) + : sh_url_template(s, a, i); + if (url) { + sh_hit_push(s, cmd, (sh_hit_t){.via = via, .ext = url, .path = method, .http = true}); + } + i = j; + } +} + /* ── Statements ────────────────────────────────────────────────────── */ static void sh_walk(sh_t *s, TSNode n); @@ -2344,6 +2527,7 @@ static void sh_command(sh_t *s, TSNode cmd) { sh_deploy_command(s, cmd, &a, NULL, &out); sh_spawn(s, cmd, &a); sh_build(s, cmd, &a); + sh_http(s, cmd, &a); } sh_argv_free(&a); } @@ -2392,6 +2576,7 @@ static void sh_pipe_run(sh_t *s, TSNode n, sh_src_t *src) { } sh_spawn(s, cmd, &a); sh_build(s, cmd, &a); + sh_http(s, cmd, &a); sh_argv_free(&a); sh_redirects(s, el); } @@ -3479,11 +3664,106 @@ static int sh_link_build(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, CBMHashTabl return SKIP_ONE; } +/* A host that is a service of the repository: one DNS label or a + * `..svc...` name whose first label names a Kubernetes Service of + * it. Its Route is the path alone, where that service's handler of the path + * meets it, as code clients do; any other host -- a download, a + * port-forwarded localhost -- stays in the Route, so it never meets a + * handler it does not reach. (Not a compose service: the Docker pass makes + * those after this one in a full run, while a repair holds them already; + * the QN tells them apart, as a repair's proxies carry no properties.) */ +static bool sh_http_service(cbm_pipeline_ctx_t *ctx, const char *host) { + const char *first_dot = strchr(host, '.'); + const char *svc = first_dot ? strchr(first_dot + SKIP_ONE, '.') : NULL; + bool dns_form = !first_dot || (svc && sh_starts(svc, ".svc") && + (svc[strlen(".svc")] == '\0' || svc[strlen(".svc")] == '.')); + if (!dns_form || strcmp(host, "localhost") == 0) { + return false; + } + char name[CBM_SZ_256]; + snprintf(name, sizeof(name), "Service/%.*s", + (int)(first_dot ? (size_t)(first_dot - host) : strlen(host)), host); + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_name(ctx->gbuf, name, &nodes, &n) != 0) { + return false; + } + for (int i = 0; i < n; i++) { + if (nodes[i]->label && strcmp(nodes[i]->label, "Resource") == 0 && + nodes[i]->qualified_name && !strstr(nodes[i]->qualified_name, "::compose/")) { + return true; + } + } + return false; +} + +/* An HTTP request a script makes: HTTP_CALLS {via: curl | wget, method, + * url_path, host} to its Route (sh_http_service). */ +static int sh_link_http(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x) { + const char *auth = strstr(x->ext, "://"); + if (!auth || src <= 0) { + return 0; + } + auth += strlen("://"); + size_t al = strcspn(auth, "/"); + char host[CBM_SZ_256]; + size_t hl = auth[0] == '[' ? strcspn(auth, "]") + SKIP_ONE : strcspn(auth, ":/"); + if (hl == 0 || hl > al || hl >= sizeof(host)) { + return 0; + } + for (size_t i = 0; i < hl; i++) { + host[i] = (char)tolower((unsigned char)auth[i]); + } + host[hl] = '\0'; + const char *path = auth + al; + char cpath[CBM_SZ_2K]; + cbm_route_canon_path(path, cpath, sizeof(cpath)); + bool local = sh_http_service(ctx, host); + char qn[CBM_ROUTE_QN_SIZE]; + snprintf(qn, sizeof(qn), "__route__%s__%.*s%s", x->path, local ? 0 : (int)(auth + al - x->ext), + x->ext, cpath); + char authority[CBM_SZ_256]; /* the host with its port */ + if (al >= sizeof(authority)) { + return 0; + } + for (size_t i = 0; i < al; i++) { + authority[i] = (char)tolower((unsigned char)auth[i]); + } + authority[al] = '\0'; + char ehost[CBM_SZ_512]; + char epath[CBM_SZ_4K]; + cbm_json_escape(ehost, (int)sizeof(ehost), authority); + cbm_json_escape(epath, (int)sizeof(epath), path); + char rprops[CBM_SZ_1K]; + if (local) { + snprintf(rprops, sizeof(rprops), "{\"method\":\"%s\"}", x->path); + } else { + snprintf(rprops, sizeof(rprops), "{\"method\":\"%s\",\"host\":\"%s\",\"external\":true}", + x->path, ehost); + } + int64_t route = + cbm_gbuf_upsert_node(ctx->gbuf, "Route", local ? path : x->ext, qn, "", 0, 0, rprops); + if (route <= 0 || route == src) { + return 0; + } + char ci[CBM_SZ_1K]; + sh_ci_props(x, ci, sizeof(ci)); + char props[CBM_SZ_4K + CBM_SZ_1K]; + snprintf(props, sizeof(props), + "{\"via\":\"%s\",\"method\":\"%s\",\"url_path\":\"%s\",\"host\":\"%s\"%s,\"line\":%d}", + x->via, x->path, epath, ehost, ci, x->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, route, "HTTP_CALLS", props); + return SKIP_ONE; +} + static int sh_link_hit(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, int64_t src, CBMHashTable *modules) { if (x->spawn) { return src > 0 ? sh_link_spawn(ctx, src, x, modules) : 0; } + if (x->http) { + return sh_link_http(ctx, src, x); + } int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file); if (src <= 0 || tgt <= 0 || src == tgt) { return 0; diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index 365ae9430..895cded93 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -252,6 +252,38 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const * return 0; } +/* Run a node DELETE binding ?1 = project, ?2 = label, ?3.. = the first + * `type_count` of `types`; the number of nodes deleted, or -1. */ +static int delta_delete_nodes(sqlite3 *db, const char *sql, const char *project, const char *label, + const char *const *types, int type_count) { + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + return CBM_NOT_FOUND; + } + sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT); + for (int i = 0; i < type_count; i++) { + sqlite3_bind_text(stmt, i + 3, types[i], CBM_NOT_FOUND, SQLITE_TRANSIENT); + } + int rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND; +} + +int cbm_delta_drop_edgeless(cbm_store_t *store, const char *project, const char *label) { + sqlite3 *db = cbm_store_get_db(store); + if (!db) { + return CBM_NOT_FOUND; + } + /* Each NOT EXISTS is served by an (id, type) index's id prefix. */ + static const char SQL[] = + "DELETE FROM nodes WHERE project = ?1 AND label = ?2" + " AND (file_path IS NULL OR file_path = '')" + " AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.target_id = nodes.id)" + " AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.source_id = nodes.id)"; + return delta_delete_nodes(db, SQL, project, label, NULL, 0); +} + int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label, const char *const *edge_types) { sqlite3 *db = cbm_store_get_db(store); @@ -274,18 +306,7 @@ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const c if (t == 0 || len <= 0 || (size_t)len >= sizeof(sql)) { return CBM_NOT_FOUND; } - sqlite3_stmt *stmt = NULL; - if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { - return CBM_NOT_FOUND; - } - sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); - sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT); - for (int i = 0; i < t; i++) { - sqlite3_bind_text(stmt, i + 3, edge_types[i], CBM_NOT_FOUND, SQLITE_TRANSIENT); - } - int rc = sqlite3_step(stmt); - sqlite3_finalize(stmt); - return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND; + return delta_delete_nodes(db, sql, project, label, edge_types, t); } /* The first column of `sql` (?1 = the project) as arena strings in *out. */ diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 33e2ce6f7..f1de3b960 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -3351,9 +3351,11 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char cbm_log_info("delta.patch_done", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); /* The proxy buffer holds no edges, so only the patched staging graph can * tell a Process node no SPAWNS reaches any more (its last spawner was - * purged), or an Image nothing builds on, builds or runs. */ + * purged), an Image nothing builds on, builds or runs, or a file-less + * Route no caller or handler is left on. */ if (cbm_delta_drop_unreferenced(staging, project, "Process", SH_PROCESS_EDGES) < 0 || - cbm_delta_drop_unreferenced(staging, project, "Image", DK_IMAGE_EDGES) < 0) { + cbm_delta_drop_unreferenced(staging, project, "Image", DK_IMAGE_EDGES) < 0 || + cbm_delta_drop_edgeless(staging, project, "Route") < 0) { goto out; } @@ -3986,9 +3988,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * pass_shell.c), an Image only through its DEPENDS_ON / USAGE edges * (pass_docker.c); neither carries a file: when the last file naming * one is gone, nothing else removes it -- a full index never makes - * it. This buffer holds the whole graph. */ + * it. Nor a file-less Route (a call site's, pass_calls.c, + * pass_shell.c) without its last caller and handler. This buffer + * holds the whole graph. */ (void)cbm_gbuf_delete_unreferenced(existing, "Process", SH_PROCESS_EDGES); (void)cbm_gbuf_delete_unreferenced(existing, "Image", DK_IMAGE_EDGES); + (void)cbm_gbuf_delete_edgeless(existing, "Route"); phase_rc = run_postpasses(&ctx, changed_files, ci, project, true); } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index b956e4293..d327026c5 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1493,8 +1493,10 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr * image reference says its container (USAGE {ref:image, container}) and * a Dockerfile's Module its image's files (image_context, image_files, * image_ignore). A workload or compose service CONNECTS_TO the Service - * its environment addresses (extract_k8s.c, pass_docker.c). An index - * built before holds none of them, so it rebuilds once on upgrade. */ + * its environment addresses (extract_k8s.c, pass_docker.c). A script's + * curl / wget HTTP_CALLS the Route of the URL it states (pass_shell.c). + * An index built before holds none of them, so it rebuilds once on + * upgrade. */ enum { CBM_SEMANTIC_INDEX_VERSION = 9 }; typedef struct { @@ -1591,6 +1593,10 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const * * on failure. */ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label, const char *const *edge_types); +/* Delete the project's `label` nodes that carry no file and that no edge of + * any type touches (a Route whose last caller and handler were purged); + * returns how many, or -1 on failure. */ +int cbm_delta_drop_edgeless(cbm_store_t *store, const char *project, const char *label); /* An Image's build edges (Image -DEPENDS_ON {via:build}-> Dockerfile, * pass_docker.c) belong to the build configuration declaring them, not to * either end: every run recomputes them whole from all build files, the diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 02873d0bf..1056b3b66 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -2581,6 +2581,108 @@ TEST(probe_env_address_connects_to_its_service) { PASS(); } +/* Every script HTTP_CALLS edge as " ", then a line + * "edgeless routes ": the Route nodes no edge touches. */ +static char *gpf_script_http(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || " + "t.qualified_name AS x FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id WHERE e.project=?1 AND e.type='HTTP_CALLS' AND " + "json_extract(e.properties,'$.via') IN ('curl','wget') ORDER BY x)), '') || char(10) " + "|| 'edgeless routes ' || (SELECT COUNT(*) FROM nodes r WHERE r.project=?1 AND " + "r.label='Route' AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.target_id=r.id OR " + "e.source_id=r.id))"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A script's curl / wget of a URL it states HTTP_CALLS the Route of it, with + * the method its flags give (-X, a body POST, -T PUT, -I HEAD, -G GET, + * wget --post-data POST): a Kubernetes Service of the repository by the path + * alone (`api`, `api.default.svc.cluster.local`), where that service's + * handler meets it; any other host -- a download, a domain under the + * service's name, localhost even beside a Service named so -- with the host + * kept, and no path-only Route made for it. Constant words resolve + * (`v${VER}`), unknown parts are `{}`, also in an unquoted word the grammar + * splits; never a proxy's or referer's URL, and never the user info or the + * query (a token) in the graph. */ +TEST(probe_script_http_request_calls_its_route) { + static const GpfFile files[] = { + {"k8s/api.yaml", "apiVersion: v1\nkind: Service\nmetadata:\n name: api\n"}, + {"k8s/localhost.yaml", "apiVersion: v1\nkind: Service\nmetadata:\n name: localhost\n"}, + {"dl.sh", + "#!/bin/bash\nVER=1.2.3\n" + "curl -fsSL https://github.com/x/y/releases/download/v1/tool.tgz -o tool.tgz\n" + "curl -X POST -d '{}' \"http://api:8080/v1/items?token=s3cr3t-token\"\n" + "wget --post-data=x https://user:pw-secret@dl.example.com/a/b\n" + "curl -I https://example.com/head\n" + "curl -x http://proxy.local:3128 https://example.org/target\n" + "curl -LO \"https://dl.example.com/v${VER}/x-${VER}.tgz\"\n" + "curl \"https://dl.example.com/r/${UNKNOWN}/y.tgz\"\n" + "curl http://localhost:9090/metrics\n" + "curl -G -d q=1 https://example.com/search\n" + "curl --request DELETE https://example.com/item/1\n" + "curl -fsSLO https://dl.example.com/u/$OS-$ARCH.tgz\n" + "curl -LO https://dl.example.com/j/`uname`.tgz\n" + "curl -LO https://dl.example.com/q/${VER}$OS.tgz\n" + "curl https://api.example.com/v1/ping\n" + "curl http://api.default.svc.cluster.local:8080/health\n" + "curl \"https://${HOST}/api/x\"\n"}, + {"Makefile", "TOOL_VERSION=0.1.1\nARCH=amd64\n" + "install:\n\tcurl -sSL https://get.example.com/install.sh | sh\n" + "bin/tool:\n\tcurl -sLo \"bin/tool\" https://dl.example.com/t/v$(TOOL_VERSION)/" + "tool_v$(TOOL_VERSION)_$(OS)_$(ARCH)\n"}}; + int n = (int)(sizeof(files) / sizeof(files[0])); + char *got = gpf_script_http(files, n); + static const char want[] = + "Makefile __route__GET__https://dl.example.com/t/v0.1.1/tool_v0.1.1_{}_amd64\n" + "Makefile __route__GET__https://get.example.com/install.sh\n" + "dl.sh __route__DELETE__https://example.com/item/1\n" + "dl.sh __route__GET__/health\n" + "dl.sh __route__GET__http://localhost:9090/metrics\n" + "dl.sh __route__GET__https://api.example.com/v1/ping\n" + "dl.sh __route__GET__https://dl.example.com/j/{}.tgz\n" + "dl.sh __route__GET__https://dl.example.com/q/{}{}.tgz\n" + "dl.sh __route__GET__https://dl.example.com/r/{}/y.tgz\n" + "dl.sh __route__GET__https://dl.example.com/u/{}-{}.tgz\n" + "dl.sh __route__GET__https://dl.example.com/v1.2.3/x-1.2.3.tgz\n" + "dl.sh __route__GET__https://example.com/search\n" + "dl.sh __route__GET__https://example.org/target\n" + "dl.sh __route__GET__https://github.com/x/y/releases/download/v1/tool.tgz\n" + "dl.sh __route__HEAD__https://example.com/head\n" + "dl.sh __route__POST__/v1/items\n" + "dl.sh __route__POST__https://dl.example.com/a/b\n" + "edgeless routes 0"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "script http got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("script HTTP_CALLS edge set"); + } + free(got); + static const char *const values[] = {"s3cr3t-token", "pw-secret", "proxy.local"}; + for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) { + int hits = gpf_graph_mentions(files, n, values[i]); + if (hits != 0) { + fprintf(stderr, "value %s: %d graph mentions\n", values[i], hits); + FAIL("a credential, token or proxy reached the graph"); + } + } + PASS(); +} + /* A variable a container sets joins the readers of its name inside the files * its own image is built from: the K8s container's image (two containers, * two images), built by compose from a Dockerfile; the stages the last one @@ -4913,6 +5015,7 @@ SUITE(grammar_probe_f) { RUN_TEST(probe_container_env_sets_scoped_envvars); RUN_TEST(probe_container_env_joins_its_own_image_readers); RUN_TEST(probe_env_address_connects_to_its_service); + RUN_TEST(probe_script_http_request_calls_its_route); RUN_TEST(probe_image_hub_splits_by_component); RUN_TEST(probe_gitops_objects_deploy_their_units); RUN_TEST(probe_shell_kubectl_deploys_what_it_applies); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a99731d76..7099db3ca 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3587,6 +3587,49 @@ TEST(incremental_deleted_spawner_drops_its_process) { PASS(); } +/* Deleting the only script requesting a URL leaves no Route behind -- a full + * index never makes one without an edge; the Route a kept script requests + * stays. `legacy`: through the full-load route, else closure repair. Returns + * the Route count after the deletion, or -1; *route: the route taken. */ +static int incremental_caller_delete(bool legacy, cbm_incremental_route_t *route) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_routegc_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + return -1; + } + write_temp_file(tmp, "hack/a.sh", "#!/bin/sh\ncurl -fsSL https://dl.example.com/a.tgz\n"); + write_temp_file(tmp, "hack/b.sh", "#!/bin/sh\nwget https://dl.example.com/b.tgz\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/routegc.db", tmp); + static const char routes[] = "SELECT COUNT(*) FROM nodes WHERE label='Route'"; + int full = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, routes) : -1; + char path[512]; + snprintf(path, sizeof(path), "%s/hack/a.sh", tmp); + (void)remove(path); +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API + if (legacy) { + cbm_pipeline_incremental_test_force_legacy_partial_once(); + } +#endif + int after = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, routes) : -1; + *route = cbm_pipeline_incremental_test_last_route(); + th_rmtree(tmp); + return full == 2 ? after : -1; +} + +TEST(incremental_deleted_caller_drops_its_route) { + cbm_incremental_route_t route = CBM_INCREMENTAL_ROUTE_NONE; + int left = incremental_caller_delete(false, &route); + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_EQ(left, 1); +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API + left = incremental_caller_delete(true, &route); + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_LEGACY_PARTIAL); + ASSERT_EQ(left, 1); +#endif + PASS(); +} + /* Deleting the only Dockerfile naming an image leaves no Image node behind; * the image another Dockerfile still names stays. `legacy`: through the * full-load route, else the closure-repair route. Returns the Image count @@ -4857,6 +4900,39 @@ TEST(incremental_env_join_repair_keeps_unchanged_pairs) { PASS(); } +/* A script's request to a compose service keeps the host in its Route on a + * repair as in a full run: the repair holds the compose services already + * (a full run makes them after the scripts), yet only a Kubernetes Service + * of the repository makes a Route path-only. */ +TEST(incremental_script_request_to_compose_service_keeps_host) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_composeurl_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + closure_probe_repo(tmp); + write_temp_file(tmp, "compose.yaml", "services:\n web:\n image: acme/web:1\n"); + write_temp_file(tmp, "hack/ping.sh", "#!/bin/sh\ncurl http://web:3000/x\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/composeurl.db", tmp); + static const char kept[] = "SELECT COUNT(*) FROM nodes WHERE label='Route' AND " + "qualified_name='__route__GET__http://web:3000/y'"; + static const char bare[] = "SELECT COUNT(*) FROM nodes WHERE label='Route' AND " + "qualified_name LIKE '__route__GET__/%'"; + int full_rc = stem_index(tmp, db); + write_temp_file(tmp, "hack/ping.sh", "#!/bin/sh\ncurl http://web:3000/y\n"); + int inc_rc = stem_index(tmp, db); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + int inc_kept = k8s_inc_count(db, kept); + int inc_bare = k8s_inc_count(db, bare); + th_rmtree(tmp); + + ASSERT_EQ(full_rc, 0); + ASSERT_EQ(inc_rc, 0); + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_EQ(inc_kept, 1); + ASSERT_EQ(inc_bare, 0); + PASS(); +} + TEST(pipeline_closure_repair_budget_declines_to_full) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_closure_budget_XXXXXX"); @@ -23400,6 +23476,8 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(incremental_k8s_edits_reach_other_files); RUN_TEST(incremental_shell_deploy_edit_keeps_its_edges); RUN_TEST(incremental_deleted_spawner_drops_its_process); + RUN_TEST(incremental_deleted_caller_drops_its_route); + RUN_TEST(incremental_script_request_to_compose_service_keeps_host); RUN_TEST(incremental_deleted_dockerfile_drops_its_image); RUN_TEST(incremental_dockerfile_edit_keeps_build_edges); RUN_TEST(incremental_compose_edit_replaces_build_edges);