mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-05 08:23:53 +08:00
Martin Vogel
3b93db06b2
Shell scripts, Make recipes and CI steps download tools and call services
with curl and wget; the graph had no edge for any of them, so neither what
a repository fetches nor the scripts that call its own services were
visible.
Such a command now HTTP_CALLS the Route of each URL it states, with
{via: curl | wget, method, url_path, host, line}:
- the method from the flags: -X / --request / --method, a body (-d, -F,
--data*, --json, --form*, wget --post-data / --post-file) POST, -T PUT,
-I HEAD, -G back to GET;
- the URL from constant words (VER=1.2.3 ... v${VER}), unknown parts `{}`
as in a Route path -- also in an unquoted word tree-sitter-bash splits
around its expansions; a URL whose host is not literal, or the value of a
proxy / referer option, is no request;
- never the user info, the query or the fragment: a token in a URL does not
reach the graph;
- the Route: path-only (__route__POST__/v1/items) when the host is a
Kubernetes Service of the repository in a service DNS form, where that
service's handler meets it as a code client's request does; any other
host -- a download, localhost, a compose service -- keeps it
(__route__GET__https://dl.k8s.io/release/{}/bin/linux/amd64/kubectl,
props {method, host, external}), and route_nodes makes no path-only twin
of it.
Incremental indexing now drops a Route without a file that no edge touches
any more -- its last caller's file changed or went. A full index never
holds one; before, the Route of a deleted code client stayed behind too.
Measured on 17 infrastructure repositories: 92 edges, every one correct by
hand; an independent reading of the scripts and Makefiles finds 82
requests, 81 of them in the graph (the last is lost in a file
tree-sitter-bash cannot parse past `[ "$v" \< "1.33" ]`). No property of
any other node or edge changes.
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
233 lines
11 KiB
C
233 lines
11 KiB
C
/*
|
|
* graph_buffer.h — In-memory graph buffer for pipeline indexing.
|
|
*
|
|
* Holds all nodes and edges in RAM during indexing, then dumps to SQLite.
|
|
* Provides O(1) node lookup by qualified name and edge dedup by key.
|
|
*
|
|
* Depends on: foundation (hash_table, dyn_array), store (data structs)
|
|
*/
|
|
#ifndef CBM_GRAPH_BUFFER_H
|
|
#define CBM_GRAPH_BUFFER_H
|
|
|
|
#include <stdbool.h>
|
|
#include <stdint.h>
|
|
#include <stdatomic.h>
|
|
|
|
/* ── Opaque handle ──────────────────────────────────────────────── */
|
|
|
|
typedef struct cbm_gbuf cbm_gbuf_t;
|
|
|
|
/* Forward declare store for dump path */
|
|
typedef struct cbm_store cbm_store_t;
|
|
|
|
/* ── Node / Edge structs (owned by the buffer) ───────────────────── */
|
|
|
|
typedef struct {
|
|
int64_t id; /* temp ID (sequential from 1) */
|
|
char *label; /* heap-owned */
|
|
char *name; /* heap-owned */
|
|
char *qualified_name; /* heap-owned */
|
|
char *file_path; /* heap-owned */
|
|
int start_line;
|
|
int end_line;
|
|
char *properties_json; /* heap-owned JSON string, "{}" default */
|
|
} cbm_gbuf_node_t;
|
|
|
|
typedef struct {
|
|
int64_t id; /* temp ID */
|
|
int64_t source_id; /* temp node ID */
|
|
int64_t target_id; /* temp node ID */
|
|
char *type; /* heap-owned */
|
|
char *properties_json; /* heap-owned JSON string, "{}" default */
|
|
} cbm_gbuf_edge_t;
|
|
|
|
/* ── Lifecycle ──────────────────────────────────────────────────── */
|
|
|
|
/* Create a new graph buffer for a project. */
|
|
cbm_gbuf_t *cbm_gbuf_new(const char *project, const char *root_path);
|
|
|
|
/* Create a graph buffer with a shared atomic ID source.
|
|
* IDs are allocated via atomic_fetch_add on *id_source.
|
|
* Used for parallel extraction where multiple gbufs need unique IDs.
|
|
* If id_source is NULL, behaves like cbm_gbuf_new(). */
|
|
/* A worker buffer: shared ids, no secondary indexes (by label / name / edge
|
|
* type). It is filled by one worker and merged into the main buffer; the
|
|
* finders that need those indexes return nothing on it. */
|
|
cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path,
|
|
_Atomic int64_t *id_source);
|
|
|
|
cbm_gbuf_t *cbm_gbuf_new_shared_ids(const char *project, const char *root_path,
|
|
_Atomic int64_t *id_source);
|
|
|
|
/* Free the graph buffer and all owned data. NULL-safe. */
|
|
void cbm_gbuf_free(cbm_gbuf_t *gb);
|
|
|
|
/* Merge all nodes and edges from src into dst.
|
|
* Nodes are merged by QN: on collision, src wins (updates dst node fields).
|
|
* New nodes are inserted with their original IDs (from shared ID source).
|
|
* Edges are remapped for any QN-colliding nodes, then inserted with dedup.
|
|
* After merge, src can be safely freed (all data is copied).
|
|
* Returns 0 on success, -1 on error. */
|
|
int cbm_gbuf_merge(cbm_gbuf_t *dst, cbm_gbuf_t *src);
|
|
|
|
/* ── Node operations ─────────────────────────────────────────────── */
|
|
|
|
/* Upsert a node by qualified name. Returns the temp ID.
|
|
* All string fields are copied (buffer owns the copies).
|
|
* Returns 0 on error. */
|
|
int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name,
|
|
const char *qualified_name, const char *file_path, int start_line,
|
|
int end_line, const char *properties_json);
|
|
|
|
/* Find a node by qualified name. Returns NULL if not found. */
|
|
const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn);
|
|
|
|
/* Find a node by temp ID. Returns NULL if not found. */
|
|
const cbm_gbuf_node_t *cbm_gbuf_find_by_id(const cbm_gbuf_t *gb, int64_t id);
|
|
|
|
/* Find nodes by label. Sets *out and *count. Caller does NOT free.
|
|
* Returns 0 on success, -1 on error. */
|
|
int cbm_gbuf_find_by_label(const cbm_gbuf_t *gb, const char *label, const cbm_gbuf_node_t ***out,
|
|
int *count);
|
|
|
|
/* Find nodes by name (exact). Sets *out and *count. Caller does NOT free. */
|
|
int cbm_gbuf_find_by_name(const cbm_gbuf_t *gb, const char *name, const cbm_gbuf_node_t ***out,
|
|
int *count);
|
|
|
|
/* Count total nodes in buffer. */
|
|
int cbm_gbuf_node_count(const cbm_gbuf_t *gb);
|
|
|
|
/* Get the next ID that would be assigned. Used to initialize shared atomic counters. */
|
|
int64_t cbm_gbuf_next_id(const cbm_gbuf_t *gb);
|
|
|
|
/* Set the next ID counter. Pins it: the delta preseed sets it to the DB's next id. */
|
|
void cbm_gbuf_set_next_id(cbm_gbuf_t *gb, int64_t next_id);
|
|
|
|
/* Raise the next ID counter to at least next_id; never lowers it, so no id
|
|
* already handed out comes back. Used to sync after merging worker gbufs. */
|
|
void cbm_gbuf_raise_next_id(cbm_gbuf_t *gb, int64_t next_id);
|
|
|
|
/* Number of edges whose id an earlier edge already carries (0 = ids unique). */
|
|
int cbm_gbuf_duplicate_edge_ids(const cbm_gbuf_t *gb);
|
|
|
|
/* Delete all nodes with a label. Cascade-deletes referencing edges. */
|
|
int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label);
|
|
|
|
/* Delete all nodes for a given file path. Cascade-deletes referencing edges.
|
|
* Used by incremental indexing to remove stale nodes before re-extraction. */
|
|
int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path);
|
|
|
|
/* Delete every `label` node that no edge of `edge_types` (NULL-terminated)
|
|
* touches, as source or target (with any edges it still has); returns how
|
|
* many. For nodes that exist only through such edges and carry no file
|
|
* (Process, Image): incremental indexing drops them when their last referrer
|
|
* is gone, as a full index never makes them. */
|
|
int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types);
|
|
|
|
/* Delete every `label` node that carries no file and that no edge of any
|
|
* type touches; returns how many. For a label made by edges of many kinds
|
|
* (Route: HTTP_CALLS, ASYNC_CALLS, HANDLES, ...): incremental indexing drops
|
|
* the file-less one whose last edge went with its files, as a full index
|
|
* never makes it. */
|
|
int cbm_gbuf_delete_edgeless(cbm_gbuf_t *gb, const char *label);
|
|
|
|
/* Bulk-load all nodes and edges for a project from an existing SQLite DB
|
|
* into this graph buffer. Returns 0 on success. */
|
|
int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project);
|
|
|
|
/* Iterate all live nodes (not deleted from QN index). */
|
|
typedef void (*cbm_gbuf_node_visitor_fn)(const cbm_gbuf_node_t *node, void *userdata);
|
|
void cbm_gbuf_foreach_node(const cbm_gbuf_t *gb, cbm_gbuf_node_visitor_fn fn, void *userdata);
|
|
|
|
/* Iterate all edges. */
|
|
typedef void (*cbm_gbuf_edge_visitor_fn)(const cbm_gbuf_edge_t *edge, void *userdata);
|
|
void cbm_gbuf_foreach_edge(const cbm_gbuf_t *gb, cbm_gbuf_edge_visitor_fn fn, void *userdata);
|
|
|
|
/* ── Edge operations ─────────────────────────────────────────────── */
|
|
|
|
/* Insert an edge. Deduplicates by (source_id, target_id, type).
|
|
* On duplicate, merges properties (later wins). Returns edge temp ID.
|
|
* Returns 0 on error. */
|
|
int64_t cbm_gbuf_insert_edge(cbm_gbuf_t *gb, int64_t source_id, int64_t target_id, const char *type,
|
|
const char *properties_json);
|
|
|
|
/* Find edges from source_id with given type.
|
|
* Sets *out and *count. Caller does NOT free. */
|
|
int cbm_gbuf_find_edges_by_source_type(const cbm_gbuf_t *gb, int64_t source_id, const char *type,
|
|
const cbm_gbuf_edge_t ***out, int *count);
|
|
|
|
/* Find edges to target_id with given type. */
|
|
int cbm_gbuf_find_edges_by_target_type(const cbm_gbuf_t *gb, int64_t target_id, const char *type,
|
|
const cbm_gbuf_edge_t ***out, int *count);
|
|
|
|
/* Find all edges of a given type. */
|
|
int cbm_gbuf_find_edges_by_type(const cbm_gbuf_t *gb, const char *type,
|
|
const cbm_gbuf_edge_t ***out, int *count);
|
|
|
|
/* Count total edges. */
|
|
int cbm_gbuf_edge_count(const cbm_gbuf_t *gb);
|
|
|
|
/* Count edges of a given type. */
|
|
int cbm_gbuf_edge_count_by_type(const cbm_gbuf_t *gb, const char *type);
|
|
|
|
/* Delete all edges of a type. */
|
|
int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type);
|
|
|
|
/* Delete the edges of `type` whose source node has label `label`. */
|
|
int cbm_gbuf_delete_edges_from_label(cbm_gbuf_t *gb, const char *type, const char *label);
|
|
|
|
/* Delete edges[0..n) (pointers found in this buffer; the array is sorted in
|
|
* place). */
|
|
int cbm_gbuf_delete_edges(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **edges, int n);
|
|
|
|
/* ── Vector storage (for semantic embeddings) ───────────────────── */
|
|
|
|
/* Store an int8-quantized vector for a node. The vector data is copied.
|
|
* Called by pass_semantic_edges after computing RI vectors.
|
|
* Vectors are carried through to cbm_write_db during the dump phase. */
|
|
int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector, int vector_len);
|
|
|
|
/* Store an enriched token vector for query-time lookup.
|
|
* Called by pass_semantic_edges after corpus finalization.
|
|
* Token string and vector data are copied. */
|
|
int cbm_gbuf_store_token_vector(cbm_gbuf_t *gb, const char *token, const uint8_t *vector,
|
|
int vector_len, float idf);
|
|
|
|
/* ── Dump to SQLite ──────────────────────────────────────────────── */
|
|
|
|
/* Dump the entire buffer to a SQLite file using the direct page writer.
|
|
* Assigns sequential final IDs and remaps edge references.
|
|
* Returns 0 on success, -1 on error. */
|
|
int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path);
|
|
|
|
/* Flush the buffer to an existing store via the store API.
|
|
* Deletes existing project data first. Returns 0 on success. */
|
|
int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store);
|
|
|
|
/* Merge the buffer into an existing store WITHOUT deleting existing data.
|
|
* Upserts nodes, inserts edges. Used for incremental indexing.
|
|
* Returns 0 on success. */
|
|
int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store);
|
|
|
|
/* Replace a node's properties_json with a buffer-owned copy of json (NULL
|
|
* maps to "{}"). The buffer allocates and frees every string it owns through
|
|
* the memory core; passes that rewrite properties MUST go through here rather
|
|
* than freeing node->properties_json themselves. Returns 0, or -1 on OOM
|
|
* (the old value is kept). */
|
|
int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json);
|
|
|
|
/* Write the definition spans gathered from same-QN collisions into each
|
|
* surviving node's `variants` property. The dump, flush and merge entry
|
|
* points do it themselves; call it directly only to read a node's
|
|
* properties_json from the buffer after collisions. */
|
|
void cbm_gbuf_settle_variants(cbm_gbuf_t *gb);
|
|
|
|
/* Visit every definition span node `n` stands for: the spans gathered from
|
|
* same-QN collisions (not yet written, or already in its `variants`), else
|
|
* its own. For passes that place a source line inside a definition. */
|
|
typedef void (*cbm_gbuf_span_fn)(const char *file, int start, int end, void *ud);
|
|
void cbm_gbuf_foreach_span(const cbm_gbuf_t *gb, const cbm_gbuf_node_t *n, cbm_gbuf_span_fn fn,
|
|
void *ud);
|
|
|
|
#endif /* CBM_GRAPH_BUFFER_H */
|