Merge pull request #1723 from liuchong/feat/index-resource-discovery

feat(index): add opt-in discovery resource limits
This commit is contained in:
Martin Vogel
2026-09-20 17:17:47 +02:00
committed by GitHub
25 changed files with 1371 additions and 44 deletions
+2
View File
@@ -325,6 +325,7 @@ FOUNDATION_SRCS = \
src/foundation/profile.c \
src/foundation/dump_verify.c \
src/foundation/limits.c \
src/foundation/index_policy.c \
src/foundation/subprocess.c \
src/foundation/sha256.c \
src/foundation/secure_random.c \
@@ -617,6 +618,7 @@ TEST_FOUNDATION_SRCS = \
tests/test_str_intern.c \
tests/test_log.c \
tests/test_str_util.c \
tests/test_index_policy.c \
tests/test_workspace.c \
tests/test_platform.c \
tests/test_diagnostics.c \
+6
View File
@@ -733,9 +733,15 @@ codebase-memory-mcp config set auto_index true # auto-index on session
codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index
codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true)
codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true)
codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit
codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit
codebase-memory-mcp config reset auto_index # reset to default
```
The two `index_max_*` settings default to `off`. Exceeding one fails the complete
index attempt rather than publishing a partial graph; an existing serving index
is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md).
### Environment Variables
| Variable | Default | Description |
+10
View File
@@ -89,6 +89,8 @@ Current keys:
| `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. |
| `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). |
| `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. |
| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |
| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. |
> **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the
> watcher *subsystem* starts at all (the background poll thread). `auto_watch` is
@@ -116,6 +118,14 @@ Current keys:
> `auto_index` still runs, and `index_repository` stays available for manual
> reindexing.
The two `index_max_*` settings are independent and disabled by default. They
apply to explicit indexing, automatic indexing, and watcher re-indexing, but not
to `cross-repo-intelligence`, which does not scan repository source files.
Equality is allowed; exceeding either setting fails the complete index request
and preserves any previously serving database. See
[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and
error-response details.
## 3. UI Settings
The optional built-in graph UI stores its settings in:
+68
View File
@@ -0,0 +1,68 @@
# Index resource limits
Index resource limits are optional operator controls for repositories whose
discovery breadth is not known in advance. They are disabled by default so
existing large-repository workloads retain their current behavior.
## Discovery settings
| Key | Default | Accepted value | Protects |
|---|---:|---:|---|
| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count |
| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes |
Set or reset them with the normal configuration command:
```bash
codebase-memory-mcp config set index_max_files 250000
codebase-memory-mcp config set index_max_source_mb 16384
codebase-memory-mcp config reset index_max_files
```
Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero,
negative values, suffixes, trailing characters, and values outside the stated
ranges are rejected without changing the stored value.
## Counting and failure semantics
`index_max_files` counts a file only after it passes directory pruning, ignore
rules, filename and suffix filters, language detection, and the existing
per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same
accepted set.
Equality is allowed. The first file or byte that makes an observed value greater
than its limit stops discovery. CBM discards the partial file list and does not
publish a partial graph as a complete index.
For an explicit MCP request the error payload contains:
```json
{
"status": "error",
"code": "resource_limit_exceeded",
"stage": "discovery",
"resource": "files",
"observed": 250001,
"limit": 250000,
"unit": "files",
"retryable": true,
"serving_index_preserved": true,
"message": "Index discovery exceeded index_max_files"
}
```
The previous database remains available because publication occurs only after a
complete discovery and successful staged build. If no previous database exists,
`serving_index_preserved` is false.
## Trust and compatibility
Limits are read from the CLI-managed `_config.db`; they are not MCP request
arguments. A supervised parent replaces any caller-supplied internal policy
before spawning its worker, and the worker rejects a missing or incomplete
parent policy.
These settings do not replace or increase `auto_index_limit`, change the 512 MiB
single-file cap, alter workspace-root authorization, or affect
`cross-repo-intelligence`. With both settings `off`, discovery follows the
existing unbounded path.
+10
View File
@@ -105,6 +105,16 @@ _cbm_test_runtime_daemon() {
CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2"
}
# The supervisor resolves one resource policy per index and hands it to the
# worker in argv; a worker that finds no complete policy refuses to start rather
# than index unbounded. A shell test that spawns `cli --index-worker` itself
# stands in for the supervisor and owes the worker the same object. Mirrors
# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and
# nothing else.
cbm_test_index_worker_policy_json() {
printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}'
}
cbm_test_runtime_cleanup() {
local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT"
local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0
+54 -2
View File
@@ -7404,6 +7404,25 @@ bool cbm_config_watcher_enabled(cbm_config_t *cfg) {
return cbm_config_get_bool(cfg, CBM_CONFIG_WATCHER_ENABLED, true);
}
bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
char *error, size_t error_size) {
if (!cfg || !policy) {
if (error && error_size > 0) {
(void)snprintf(error, error_size, "index resource configuration is unavailable");
}
return false;
}
cbm_index_policy_init(policy);
for (size_t index = 0; index < cbm_index_policy_key_count(); index++) {
const char *key = cbm_index_policy_key_at(index);
const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key));
if (!cbm_index_policy_set(policy, key, value, error, error_size)) {
return false;
}
}
return true;
}
/* ── Config CLI subcommand ────────────────────────────────────── */
/* THE config-key table. list, get, help, and key validation all read this one
@@ -7428,6 +7447,8 @@ static const config_key_def_t CONFIG_KEYS[] = {
{CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"},
{CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"},
{CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"},
{CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"},
{CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"},
};
/* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing
@@ -7453,6 +7474,31 @@ static bool config_key_is_ui(const char *key) {
return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0);
}
static bool config_key_is_index_policy(const char *key) {
return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 ||
strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0);
}
static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) {
cbm_index_resource_policy_t candidate;
cbm_index_policy_init(&candidate);
char error[CLI_BUF_256];
if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) {
(void)fprintf(stderr, "error: %s\n", error);
return CLI_ERR;
}
int rc = cbm_config_set(config, key, value);
if (rc != 0) {
/* The caller suppresses its own message for policy keys because this
* helper names the precise reason. That is only true if the helper
* speaks on every failure it can return: a validated value whose write
* then fails -- a locked or read-only _config.db -- used to exit
* non-zero having printed nothing at all. */
(void)fprintf(stderr, "error: failed to set %s\n", key);
}
return rc;
}
static void config_ui_read(const char *key, char *out, size_t out_sz) {
cbm_ui_config_t ui;
cbm_ui_config_load(&ui);
@@ -7590,10 +7636,16 @@ int cbm_cmd_config(int argc, char **argv) {
rc = CLI_TRUE;
}
} else {
if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) {
int set_rc =
config_key_is_index_policy(argv[CLI_SKIP_ONE])
? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN])
: cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
if (set_rc == 0) {
printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
} else {
(void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) {
(void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
}
rc = CLI_TRUE;
}
}
+7
View File
@@ -13,6 +13,8 @@
#include <stddef.h>
#include <stdint.h>
#include "foundation/index_policy.h"
typedef struct cbm_mcp_server cbm_mcp_server_t;
/* ── Version ──────────────────────────────────────────────────── */
@@ -420,6 +422,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value);
/* Delete a config key. Returns 0 on success. */
int cbm_config_delete(cbm_config_t *cfg, const char *key);
/* Load and validate the operator-controlled discovery policy. Invalid stored
* values fail closed instead of silently disabling a guard. */
bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
char *error, size_t error_size);
/* Well-known config keys */
#define CBM_CONFIG_AUTO_INDEX "auto_index"
#define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit"
+29 -6
View File
@@ -208,7 +208,8 @@ static bool application_unique_recovery_file(char out[APPLICATION_PATH_CAP], con
static bool application_update_reap(cbm_daemon_application_t *application, bool wait,
uint32_t timeout_ms);
static void *application_job_thread(void *opaque);
static char *application_auto_index_args(const char *root_path);
static char *application_auto_index_args(cbm_daemon_application_t *application,
const char *root_path);
static cbm_daemon_application_job_t *application_job_subscribe_locked(
cbm_daemon_application_t *application, const char *project_key, const char *root_path,
const char *args_json, application_job_subscribe_status_t *status_out);
@@ -1455,7 +1456,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t
application_refresh_watch_locked(session);
continue;
}
char *args = application_auto_index_args(root_path);
char *args = application_auto_index_args(application, root_path);
if (!args) {
continue;
}
@@ -2005,7 +2006,27 @@ static bool application_update_reap(cbm_daemon_application_t *application, bool
}
}
static char *application_auto_index_args(const char *root_path) {
static bool application_index_args_add_policy(cbm_daemon_application_t *application,
yyjson_mut_doc *document, yyjson_mut_val *root) {
cbm_config_t *owned_config = NULL;
cbm_config_t *config = application ? application->config : NULL;
if (!config) {
owned_config = cbm_config_open(cbm_resolve_cache_dir());
config = owned_config;
}
cbm_index_resource_policy_t policy;
char error[CBM_SZ_256] = {0};
bool loaded = cbm_config_load_index_policy(config, &policy, error, sizeof(error));
cbm_config_close(owned_config);
if (!loaded) {
cbm_log_error("daemon.index.policy", "error", error);
return false;
}
return cbm_mcp_index_policy_add_to_args(document, root, &policy);
}
static char *application_auto_index_args(cbm_daemon_application_t *application,
const char *root_path) {
yyjson_mut_doc *document = yyjson_mut_doc_new(NULL);
yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL;
if (!document || !root) {
@@ -2013,7 +2034,8 @@ static char *application_auto_index_args(const char *root_path) {
return NULL;
}
yyjson_mut_doc_set_root(document, root);
char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path)
char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
application_index_args_add_policy(application, document, root)
? yyjson_mut_write(document, 0, NULL)
: NULL;
yyjson_mut_doc_free(document);
@@ -2060,7 +2082,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio
files);
}
bool args_required = auto_index_candidate && within_auto_index_limit;
char *args = args_required ? application_auto_index_args(root_path) : NULL;
char *args = args_required ? application_auto_index_args(application, root_path) : NULL;
application_jobs_reap_completed(application);
cbm_mutex_lock(&application->mutex);
if (application->stopping || application_request_cancelled_locked(session)) {
@@ -3458,7 +3480,8 @@ static int application_background_index(cbm_daemon_application_t *application,
return -1;
}
yyjson_mut_doc_set_root(document, root);
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root);
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) &&
application_index_args_add_policy(application, document, root);
char *default_project = cbm_project_name_from_path(canonical_root);
bool custom_project =
project_name[0] && (!default_project || strcmp(default_project, project_name) != 0);
+49 -2
View File
@@ -13,6 +13,7 @@
#include "foundation/constants.h"
#include "foundation/compat_fs.h"
#include "foundation/limits.h"
#include "foundation/workspace.h"
#include "foundation/platform.h"
#ifdef _WIN32
@@ -414,6 +415,10 @@ typedef struct {
int capacity;
int max_files;
uint64_t deadline_ms;
const cbm_index_resource_policy_t *resource_policy;
cbm_index_resource_violation_t *resource_violation;
uint64_t source_files;
uint64_t source_bytes;
bool count_only;
bool collect_excluded;
bool limit_exceeded;
@@ -435,6 +440,21 @@ typedef struct {
int ignored_total;
} file_list_t;
static void file_list_resource_violation(file_list_t *fl, cbm_index_resource_t resource,
uint64_t observed, uint64_t limit) {
if (!fl || fl->limit_exceeded || fl->failed) {
return;
}
fl->limit_exceeded = true;
if (fl->resource_violation) {
*fl->resource_violation = (cbm_index_resource_violation_t){
.resource = resource,
.observed = observed,
.limit = limit,
};
}
}
static bool file_list_should_stop(file_list_t *fl) {
if (!fl) {
return true;
@@ -504,6 +524,28 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path,
fl->limit_exceeded = true;
return;
}
uint64_t source_size = size > 0 ? (uint64_t)size : 0;
bool resource_accepted = fl->resource_policy && source_size <= (uint64_t)cbm_max_file_bytes();
if (resource_accepted) {
if (fl->resource_policy->max_files.enabled &&
fl->source_files >= fl->resource_policy->max_files.value) {
file_list_resource_violation(fl, CBM_INDEX_RESOURCE_FILES, fl->source_files + 1U,
fl->resource_policy->max_files.value);
return;
}
if (fl->resource_policy->max_source_bytes.enabled) {
uint64_t limit = fl->resource_policy->max_source_bytes.value;
if (source_size > limit || fl->source_bytes > limit - source_size) {
uint64_t observed = source_size > UINT64_MAX - fl->source_bytes
? UINT64_MAX
: fl->source_bytes + source_size;
file_list_resource_violation(fl, CBM_INDEX_RESOURCE_SOURCE_BYTES, observed, limit);
return;
}
fl->source_bytes += source_size;
}
fl->source_files++;
}
if (fl->count_only) {
fl->count++;
return;
@@ -1200,6 +1242,9 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc
if (ignored_total_out) {
*ignored_total_out = 0;
}
if (opts && opts->resource_violation) {
*opts->resource_violation = (cbm_index_resource_violation_t){0};
}
if (!repo_path || !out || !count || (count_only && max_files < 0)) {
return CBM_DISCOVER_ERROR;
}
@@ -1274,6 +1319,8 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc
file_list_t fl = {
.max_files = count_only ? max_files : -1,
.deadline_ms = count_only ? deadline_ms : 0,
.resource_policy = opts ? opts->resource_policy : NULL,
.resource_violation = opts ? opts->resource_violation : NULL,
.count_only = count_only,
.collect_excluded = !count_only && excluded_out != NULL,
.collect_ignored = !count_only && ignored_out != NULL,
@@ -1296,11 +1343,11 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc
}
return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_OK;
}
if (fl.failed) {
if (fl.failed || fl.limit_exceeded) {
cbm_discover_free(fl.files, fl.count);
cbm_discover_free_excluded(fl.excluded, fl.excluded_count);
cbm_discover_free_ignored(fl.ignored, fl.ignored_count);
return CBM_DISCOVER_ERROR;
return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_ERROR;
}
*out = fl.files;
+6 -3
View File
@@ -17,6 +17,7 @@
/* Use the existing CBMLanguage enum from extraction layer */
#include "cbm.h"
#include "foundation/index_policy.h"
/* ── Language detection ──────────────────────────────────────────── */
@@ -143,9 +144,11 @@ typedef struct {
} cbm_file_info_t;
typedef struct {
cbm_index_mode_t mode; /* CBM_MODE_FULL or CBM_MODE_FAST */
const char *ignore_file; /* path to .cbmignore file, or NULL */
int64_t max_file_size; /* 0 = no limit */
cbm_index_mode_t mode; /* discovery filtering mode */
const char *ignore_file; /* .cbmignore path, or NULL */
int64_t max_file_size; /* 0 = no per-file limit */
const cbm_index_resource_policy_t *resource_policy; /* NULL = no resource limits */
cbm_index_resource_violation_t *resource_violation; /* optional exact diagnostic */
} cbm_discover_opts_t;
typedef enum {
+153
View File
@@ -0,0 +1,153 @@
#include "foundation/index_policy.h"
#include <stdio.h>
#include <string.h>
static const char *const INDEX_POLICY_KEYS[] = {
CBM_INDEX_CONFIG_MAX_FILES,
CBM_INDEX_CONFIG_MAX_SOURCE_MB,
};
static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) {
if (error && error_size > 0) {
(void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key,
(unsigned long long)maximum);
}
}
static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) {
if (!value || !value[0] || !parsed) {
return false;
}
uint64_t result = 0;
for (const unsigned char *cursor = (const unsigned char *)value; *cursor; cursor++) {
if (*cursor < '0' || *cursor > '9') {
return false;
}
uint64_t digit = (uint64_t)(*cursor - '0');
if (result > (maximum - digit) / 10U) {
return false;
}
result = result * 10U + digit;
}
if (result == 0) {
return false;
}
*parsed = result;
return true;
}
void cbm_index_policy_init(cbm_index_resource_policy_t *policy) {
if (policy) {
*policy = (cbm_index_resource_policy_t){0};
}
}
bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) {
return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled);
}
size_t cbm_index_policy_key_count(void) {
return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]);
}
const char *cbm_index_policy_key_at(size_t index) {
return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL;
}
const char *cbm_index_policy_default_value(const char *key) {
for (size_t index = 0; index < cbm_index_policy_key_count(); index++) {
if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) {
return "off";
}
}
return NULL;
}
bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value,
char *error, size_t error_size) {
if (error && error_size > 0) {
error[0] = '\0';
}
if (!policy || !key || !value) {
set_error(error, error_size, key ? key : "index resource limit", 0);
return false;
}
cbm_index_limit_u64_t *target = NULL;
uint64_t maximum = 0;
uint64_t multiplier = 1;
if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) {
target = &policy->max_files;
maximum = CBM_INDEX_MAX_FILES_VALUE;
} else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) {
target = &policy->max_source_bytes;
maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE;
multiplier = CBM_INDEX_MIB_BYTES;
} else {
set_error(error, error_size, key, 0);
return false;
}
cbm_index_limit_u64_t candidate = {0};
if (strcmp(value, "off") != 0) {
uint64_t parsed = 0;
if (!parse_bounded_uint64(value, maximum, &parsed)) {
set_error(error, error_size, key, maximum);
return false;
}
candidate.enabled = true;
candidate.value = parsed * multiplier;
}
*target = candidate;
return true;
}
bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out,
size_t out_size) {
if (!policy || !key || !out || out_size == 0) {
return false;
}
const cbm_index_limit_u64_t *limit = NULL;
uint64_t divisor = 1;
if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) {
limit = &policy->max_files;
} else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) {
limit = &policy->max_source_bytes;
divisor = CBM_INDEX_MIB_BYTES;
} else {
return false;
}
int length = limit->enabled
? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor))
: snprintf(out, out_size, "off");
return length >= 0 && (size_t)length < out_size;
}
const char *cbm_index_resource_name(cbm_index_resource_t resource) {
switch (resource) {
case CBM_INDEX_RESOURCE_FILES:
return "files";
case CBM_INDEX_RESOURCE_SOURCE_BYTES:
return "source_bytes";
case CBM_INDEX_RESOURCE_NONE:
default:
return "unknown";
}
}
const char *cbm_index_resource_unit(cbm_index_resource_t resource) {
return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes";
}
const char *cbm_index_resource_config_key(cbm_index_resource_t resource) {
switch (resource) {
case CBM_INDEX_RESOURCE_FILES:
return CBM_INDEX_CONFIG_MAX_FILES;
case CBM_INDEX_RESOURCE_SOURCE_BYTES:
return CBM_INDEX_CONFIG_MAX_SOURCE_MB;
case CBM_INDEX_RESOURCE_NONE:
default:
return "index_resource_limit";
}
}
+53
View File
@@ -0,0 +1,53 @@
#ifndef CBM_INDEX_POLICY_H
#define CBM_INDEX_POLICY_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#define CBM_INDEX_CONFIG_MAX_FILES "index_max_files"
#define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb"
#define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000)
#define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576)
#define CBM_INDEX_MIB_BYTES UINT64_C(1048576)
typedef struct {
bool enabled;
uint64_t value;
} cbm_index_limit_u64_t;
typedef struct {
cbm_index_limit_u64_t max_files;
cbm_index_limit_u64_t max_source_bytes;
} cbm_index_resource_policy_t;
typedef enum {
CBM_INDEX_RESOURCE_NONE = 0,
CBM_INDEX_RESOURCE_FILES,
CBM_INDEX_RESOURCE_SOURCE_BYTES,
} cbm_index_resource_t;
typedef struct {
cbm_index_resource_t resource;
uint64_t observed;
uint64_t limit;
} cbm_index_resource_violation_t;
void cbm_index_policy_init(cbm_index_resource_policy_t *policy);
bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy);
size_t cbm_index_policy_key_count(void);
const char *cbm_index_policy_key_at(size_t index);
const char *cbm_index_policy_default_value(const char *key);
bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value,
char *error, size_t error_size);
bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out,
size_t out_size);
const char *cbm_index_resource_name(cbm_index_resource_t resource);
const char *cbm_index_resource_unit(cbm_index_resource_t resource);
const char *cbm_index_resource_config_key(cbm_index_resource_t resource);
#endif
+128 -7
View File
@@ -10600,6 +10600,63 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition(
return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE;
}
static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy,
char *error, size_t error_size) {
yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL;
yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL;
yyjson_val *encoded =
root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL;
if (!encoded || !yyjson_is_obj(encoded) ||
yyjson_obj_size(encoded) != cbm_index_policy_key_count()) {
yyjson_doc_free(doc);
(void)snprintf(error, error_size, "missing or incomplete trusted worker policy");
return false;
}
cbm_index_policy_init(policy);
bool valid = true;
for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) {
const char *key = cbm_index_policy_key_at(index);
yyjson_val *value = yyjson_obj_get(encoded, key);
valid = value && yyjson_is_str(value) &&
cbm_index_policy_set(policy, key, yyjson_get_str(value), error, error_size);
}
yyjson_doc_free(doc);
if (!valid && error && error_size > 0 && error[0] == '\0') {
(void)snprintf(error, error_size, "invalid trusted worker policy");
}
return valid;
}
static bool load_index_policy(cbm_mcp_server_t *srv, const char *args,
cbm_index_resource_policy_t *policy, char *error, size_t error_size) {
if (cbm_index_worker_active()) {
return index_policy_from_worker_args(args, policy, error, error_size);
}
cbm_config_t *owned_config = NULL;
cbm_config_t *config = srv ? srv->config : NULL;
if (!config) {
owned_config = cbm_config_open(cbm_resolve_cache_dir());
config = owned_config;
}
bool loaded = cbm_config_load_index_policy(config, policy, error, error_size);
cbm_config_close(owned_config);
return loaded;
}
bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root,
const cbm_index_resource_policy_t *policy) {
yyjson_mut_val *encoded = yyjson_mut_obj(doc);
bool valid = encoded != NULL;
for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) {
const char *key = cbm_index_policy_key_at(index);
char value[CBM_SZ_64];
valid = cbm_index_policy_format(policy, key, value, sizeof(value)) &&
yyjson_mut_obj_add_strcpy(doc, encoded, key, value);
}
return valid && yyjson_mut_obj_add_val(doc, root, "_cbm_index_policy", encoded);
}
/* Run index_repository in a supervised worker subprocess with skip-and-continue
* (Stage 3c). Returns the response string (caller frees):
* - the worker's own response on a clean first run (the common path);
@@ -10865,10 +10922,20 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p
if (!root_path || !root_path[0]) {
return NULL;
}
cbm_index_resource_policy_t policy;
char policy_error[CBM_SZ_256] = {0};
if (!load_index_policy(srv, NULL, &policy, policy_error, sizeof(policy_error))) {
cbm_log_error("index.policy", "error", policy_error);
return NULL;
}
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
yyjson_mut_val *root = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, root);
yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path);
if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) ||
!cbm_mcp_index_policy_add_to_args(doc, root, &policy)) {
yyjson_mut_doc_free(doc);
return NULL;
}
char *args = yy_doc_to_str(doc);
yyjson_mut_doc_free(doc);
if (!args) {
@@ -10917,8 +10984,9 @@ static bool resolve_session_repo_path(cbm_mcp_server_t *srv, char **repo_path) {
/* Preserve every index option while replacing all caller-supplied repo_path
* keys with the one canonical path that was actually authorized. */
static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path) {
if (!args || !canonical_repo_path) {
static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path,
const cbm_index_resource_policy_t *policy) {
if (!args || !canonical_repo_path || !policy) {
return NULL;
}
yyjson_doc *source = yyjson_read(args, strlen(args), 0);
@@ -10935,8 +11003,14 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r
yyjson_mut_doc_free(copy);
return NULL;
}
(void)yyjson_mut_obj_remove_key(copy_root, "repo_path");
if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path)) {
while (yyjson_mut_obj_get(copy_root, "repo_path")) {
(void)yyjson_mut_obj_remove_key(copy_root, "repo_path");
}
while (yyjson_mut_obj_get(copy_root, "_cbm_index_policy")) {
(void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_policy");
}
if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) ||
!cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) {
yyjson_mut_doc_free(copy);
return NULL;
}
@@ -10975,6 +11049,19 @@ static char *resolved_repo_path_from_project_arg(const char *args) {
return root_path;
}
static bool project_db_is_servable(const char *project, const char *db_path) {
cbm_store_t *store = db_path && db_path[0] ? cbm_store_open_path_query(db_path) : NULL;
if (!store) {
return false;
}
cbm_project_t stored_project = {0};
bool servable = cbm_store_get_project(store, project, &stored_project) == CBM_STORE_OK &&
stored_project.root_path && stored_project.root_path[0];
cbm_project_free_fields(&stored_project);
cbm_store_close(store);
return servable;
}
static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");
char *mode_str = cbm_mcp_get_string_arg(args, "mode");
@@ -11029,10 +11116,19 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
return result;
}
cbm_index_resource_policy_t resource_policy;
char policy_error[CBM_SZ_256] = {0};
if (!load_index_policy(srv, args, &resource_policy, policy_error, sizeof(policy_error))) {
free(mode_str);
free(name_override);
free(repo_path);
return cbm_mcp_text_result(policy_error, true);
}
/* A daemon session delegates the one physical write to its shared job
* registry only after path canonicalization and workspace authorization. */
if (srv->index_executor) {
char *worker_args = index_args_with_repo_path(args, repo_path);
char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy);
char *coordinated =
worker_args ? srv->index_executor(srv->index_executor_context, repo_path, worker_args)
: NULL;
@@ -11064,7 +11160,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
* installs the same guard before running the in-process pipeline. A marked
* host fails closed if preparation or worker startup cannot complete. */
if (cbm_index_supervisor_should_wrap()) {
char *worker_args = index_args_with_repo_path(args, repo_path);
char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy);
if (!worker_args) {
free(mutation_project);
free(repo_path);
@@ -11136,11 +11232,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
}
free(name_override);
cbm_pipeline_set_persistence(p, persistence);
cbm_pipeline_set_resource_policy(p, &resource_policy);
char *project_name = heap_strdup(cbm_pipeline_project_name(p));
/* Bootstrap from artifact if no local DB exists */
try_artifact_bootstrap(project_name, repo_path);
char serving_db_path[CBM_SZ_1K];
project_db_path(project_name, serving_db_path, sizeof(serving_db_path));
bool serving_index_was_servable = project_db_is_servable(project_name, serving_db_path);
/* Close cached store — pipeline will delete + recreate the .db file */
if (srv->owns_store && srv->store) {
@@ -11172,6 +11272,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
cbm_file_error_t *file_errors = NULL;
int file_error_count = 0;
cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count);
cbm_index_resource_violation_t resource_violation = {0};
cbm_pipeline_get_resource_violation(p, &resource_violation);
cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */
@@ -11269,6 +11371,25 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
"The validated staging database could not be published. Check "
"free disk space and permissions on the cache directory; the "
"previous index may have been rolled back.");
} else if (rc == CBM_PIPELINE_RESOURCE_LIMIT &&
resource_violation.resource != CBM_INDEX_RESOURCE_NONE) {
const char *config_key = cbm_index_resource_config_key(resource_violation.resource);
char message[CBM_SZ_256];
(void)snprintf(message, sizeof(message), "Index discovery exceeded %s", config_key);
yyjson_mut_obj_add_str(doc, root, "status", "error");
yyjson_mut_obj_add_str(doc, root, "code", "resource_limit_exceeded");
yyjson_mut_obj_add_str(doc, root, "stage", "discovery");
yyjson_mut_obj_add_str(doc, root, "resource",
cbm_index_resource_name(resource_violation.resource));
yyjson_mut_obj_add_uint(doc, root, "observed", resource_violation.observed);
yyjson_mut_obj_add_uint(doc, root, "limit", resource_violation.limit);
yyjson_mut_obj_add_str(doc, root, "unit",
cbm_index_resource_unit(resource_violation.resource));
yyjson_mut_obj_add_bool(doc, root, "retryable", true);
yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved",
serving_index_was_servable &&
project_db_is_servable(project_name, serving_db_path));
yyjson_mut_obj_add_strcpy(doc, root, "message", message);
} else {
yyjson_mut_obj_add_str(doc, root, "status", "error");
yyjson_mut_obj_add_str(doc, root, "hint",
+8
View File
@@ -1,10 +1,13 @@
#ifndef CBM_MCP_INTERNAL_H
#define CBM_MCP_INTERNAL_H
#include "foundation/index_policy.h"
#include "mcp/mcp.h"
#include "pipeline/pipeline.h" /* cbm_changed_hunk_t */
#include "store/store.h" /* cbm_node_t */
#include <yyjson/yyjson.h>
/* White-box fault injection for deterministic cross-platform quarantine
* safety tests. This header is internal and is not part of the MCP API. */
typedef bool (*cbm_mcp_quarantine_test_hook_fn)(void *context, const char *step);
@@ -36,6 +39,11 @@ bool cbm_mcp_server_release_pristine_memory_store(cbm_mcp_server_t *srv);
* On success replaces and frees *response_io; on failure it is unchanged. */
bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *notice);
/* Encode the complete trusted policy on an internal worker request. Callers
* must remove any untrusted field with the same name before invoking this. */
bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root,
const cbm_index_resource_policy_t *policy);
enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 };
/* Count indexable files with the pipeline's native full-mode discovery policy,
+39 -8
View File
@@ -190,6 +190,8 @@ struct cbm_pipeline {
atomic_int cancelled_storage;
atomic_int *cancelled;
bool persistence; /* write .codebase-memory/graph.db.zst after indexing */
cbm_index_resource_policy_t resource_policy;
cbm_index_resource_violation_t resource_violation;
/* Indexing state (set during run) */
cbm_gbuf_t *gbuf;
@@ -373,6 +375,20 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) {
}
}
void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p,
const cbm_index_resource_policy_t *policy) {
if (p && policy) {
p->resource_policy = *policy;
}
}
void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p,
cbm_index_resource_violation_t *violation) {
if (violation) {
*violation = p ? p->resource_violation : (cbm_index_resource_violation_t){0};
}
}
bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) {
if (!p || !name || !name[0]) {
return false;
@@ -482,6 +498,14 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) {
return p ? p->repo_path : NULL;
}
const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) {
return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL;
}
cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) {
return p ? &p->resource_violation : NULL;
}
atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) {
return p ? p->cancelled : NULL;
}
@@ -2437,15 +2461,17 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
cbm_pipeline_persist_test_run_before_final_manifest();
#endif
if (cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode,
&manifest, &manifest_count) != 0) {
int manifest_rc =
cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &manifest, &manifest_count);
if (manifest_rc != 0) {
cbm_log_error("pipeline.err", "phase", "semantic_manifest");
/* db_path and db_dir are this function's strdups; the success tail and
* the publish-failure return release them, and these two aborts must
* too -- LSan caught exactly these paths leaking both strings. */
free(db_dir);
free(db_path);
return CBM_PIPELINE_ABORT_PRESERVE_DB;
return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT
: CBM_PIPELINE_ABORT_PRESERVE_DB;
}
if (!cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest,
manifest_count)) {
@@ -2702,10 +2728,14 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
/* Phase 1: Discover files */
CBM_PROF_START(t_discover);
p->resource_violation = (cbm_index_resource_violation_t){0};
cbm_discover_opts_t opts = {
.mode = p->requested_mode,
.ignore_file = NULL,
.max_file_size = 0,
.resource_policy =
cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL,
.resource_violation = &p->resource_violation,
};
cbm_file_info_t *files = NULL;
int file_count = 0;
@@ -2730,7 +2760,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms",
itoa_buf((int)elapsed_ms(t0)));
if (rc != 0 || check_cancel(p)) {
rc = CBM_NOT_FOUND;
rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND;
goto cleanup;
}
@@ -2738,14 +2768,15 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
* bytes drive exact no-op comparison and are checked against a fresh
* rediscovery immediately before any replacement is published. */
rc = mode_promoted
? cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode,
&baseline_manifest, &baseline_count)
? cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &baseline_manifest,
&baseline_count)
: cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files,
file_count, p->excluded_dirs, p->excluded_count,
&p->git_ctx, p->userconfig, &baseline_manifest,
&baseline_count);
if (rc != 0) {
rc = CBM_PIPELINE_ABORT_PRESERVE_DB;
rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT
: CBM_PIPELINE_ABORT_PRESERVE_DB;
goto cleanup;
}
@@ -2789,7 +2820,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
cbm_log_info("pipeline.rediscover", "requested_mode", pipeline_mode_name(p->requested_mode),
"effective_mode", pipeline_mode_name(p->mode), "files", itoa_buf(file_count));
if (rc != 0 || check_cancel(p)) {
rc = CBM_NOT_FOUND;
rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND;
goto cleanup;
}
}
+9
View File
@@ -53,6 +53,13 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, cbm
* When enabled, the pipeline writes a compressed artifact after indexing. */
void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled);
/* Apply a validated discovery resource policy. The value is copied. */
void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy);
/* Copy the exact discovery violation from the most recent run. */
void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p,
cbm_index_resource_violation_t *violation);
/* Free a pipeline and all its internal state. NULL-safe. */
void cbm_pipeline_free(cbm_pipeline_t *p);
@@ -71,6 +78,8 @@ void cbm_pipeline_free(cbm_pipeline_t *p);
* written, the previous DB is intact (#1997 #832). Distinct from the cancel
* sentinel so callers can name the cause instead of "pipeline failed". */
#define CBM_PIPELINE_ABORT_OVER_BUDGET (-5)
/* Opt-in discovery/resource policy breach: fail the attempt, keep serving DB. */
#define CBM_PIPELINE_RESOURCE_LIMIT (-6)
int cbm_pipeline_run(cbm_pipeline_t *p);
/* Request cancellation of a running pipeline (thread-safe). */
+16 -12
View File
@@ -601,17 +601,20 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left
return equal;
}
int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode,
int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project,
cbm_file_hash_t **out, int *out_count) {
if (!project || !repo_path || !out || !out_count) {
const char *repo_path = cbm_pipeline_repo_path(p);
if (!p || !project || !repo_path || !out || !out_count) {
return CBM_NOT_FOUND;
}
*out = NULL;
*out_count = 0;
cbm_discover_opts_t opts = {
.mode = (cbm_index_mode_t)mode,
.mode = (cbm_index_mode_t)cbm_pipeline_get_mode(p),
.ignore_file = NULL,
.max_file_size = 0,
.resource_policy = cbm_pipeline_resource_policy(p),
.resource_violation = cbm_pipeline_resource_violation(p),
};
cbm_file_info_t *fresh_files = NULL;
int fresh_file_count = 0;
@@ -2264,13 +2267,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
cbm_pipeline_persist_test_run_before_final_manifest();
#endif
if (cbm_pipeline_build_fresh_semantic_manifest(project, cbm_pipeline_repo_path(p),
cbm_pipeline_get_mode(p), &manifest,
&manifest_count) != 0 ||
!cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest,
manifest_count)) {
int manifest_rc =
cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count);
if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal(
baseline_manifest, baseline_count, manifest, manifest_count)) {
cbm_log_warn("delta.abort", "reason", "semantic_inputs_changed");
result = CBM_PIPELINE_ABORT_PRESERVE_DB;
result = manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT
: CBM_PIPELINE_ABORT_PRESERVE_DB;
goto out;
}
@@ -2856,8 +2859,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
cbm_pipeline_persist_test_run_before_final_manifest();
#endif
int manifest_rc = cbm_pipeline_build_fresh_semantic_manifest(
project, cbm_pipeline_repo_path(p), cbm_pipeline_get_mode(p), &manifest, &manifest_count);
int manifest_rc =
cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count);
if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal(
baseline_manifest, baseline_count, manifest, manifest_count)) {
cbm_log_warn("incremental.abort", "reason", "semantic_inputs_changed");
@@ -2867,7 +2870,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
free_mode_skipped(mode_skipped, mode_skipped_count);
free(saved_adr);
cbm_gbuf_free(existing);
return CBM_PIPELINE_ABORT_PRESERVE_DB;
return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT
: CBM_PIPELINE_ABORT_PRESERVE_DB;
}
/* Step 7: atomically publish the complete staged generation. */
+3 -1
View File
@@ -781,7 +781,7 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left
const cbm_file_hash_t *right, int right_count);
/* Re-run discovery and hash its exact semantic inputs. Used at the publication
* boundary so late additions/deletions cannot escape a frozen file list. */
int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode,
int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project,
cbm_file_hash_t **out, int *out_count);
/* Compatibility contract persisted in coverage metadata. Increment when a
@@ -874,6 +874,8 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row
/* Pipeline accessors for incremental use */
const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p);
const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p);
cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p);
atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p);
/* Record committed graph size (#334 gate axis) from the incremental path,
* which cannot see the opaque cbm_pipeline struct. Call before the dump. */
+85 -1
View File
@@ -1348,6 +1348,7 @@ TEST(daemon_application_prune_clears_logical_watch_for_reregistration) {
}
enum { APP_FAKE_MAX_ATTEMPTS = 16 };
enum { APP_FAKE_ARGS_CAP = 2048 };
typedef struct {
atomic_int starts;
@@ -1368,6 +1369,11 @@ typedef struct {
char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP];
char quarantine_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP];
char quarantine_seen[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP];
char args_json[APP_FAKE_MAX_ATTEMPTS][APP_FAKE_ARGS_CAP];
/* `starts` is incremented before the slot is filled, so waiting on it says
* a worker began, not that its arguments are readable. Each slot publishes
* itself instead, which is the edge a reader on another thread needs. */
atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS];
size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS];
} app_fake_worker_context_t;
@@ -1432,6 +1438,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem
atomic_init(&worker->cancelled, false);
worker->result.exit_code = -1;
if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) {
(void)snprintf(context->args_json[worker->attempt], APP_FAKE_ARGS_CAP, "%s",
args_json ? args_json : "");
atomic_store(&context->args_captured[worker->attempt], true);
context->memory_budgets[worker->attempt] = memory_budget_bytes;
if (marker_file) {
(void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s",
@@ -1456,6 +1465,33 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem
return 0;
}
static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected);
static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt,
const char *max_files, const char *max_source_mb) {
if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) {
return false;
}
if (!app_wait_for_atomic_bool(&context->args_captured[attempt], true)) {
return false;
}
const char *args = context->args_json[attempt];
yyjson_doc *document = yyjson_read(args, strlen(args), 0);
yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL;
yyjson_val *policy =
root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL;
yyjson_val *files =
policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_FILES) : NULL;
yyjson_val *bytes = policy && yyjson_is_obj(policy)
? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB)
: NULL;
bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) &&
strcmp(yyjson_get_str(files), max_files) == 0 &&
strcmp(yyjson_get_str(bytes), max_source_mb) == 0;
yyjson_doc_free(document);
return equal;
}
static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque,
cbm_daemon_application_worker_t handle,
const cbm_index_worker_result_t **result_out) {
@@ -2009,7 +2045,9 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {
cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL;
bool config_ready = stored_config &&
cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 &&
cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0;
cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 &&
cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 &&
cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0;
char canonical_root[APP_TEST_PATH_CAP] = {0};
bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root));
char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL;
@@ -2054,6 +2092,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {
bool first_owned = first_initialized && project &&
app_wait_for_subscribers(application, project, 1) &&
app_wait_for_atomic_int(&fake.starts, 1);
bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4");
bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root,
CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL);
bool coalesced = first_owned && second_initialized && project &&
@@ -2115,6 +2154,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {
ASSERT_TRUE(restricted_started_nothing);
ASSERT_TRUE(first_initialized);
ASSERT_TRUE(first_owned);
ASSERT_TRUE(auto_policy_propagated);
ASSERT_TRUE(second_initialized);
ASSERT_TRUE(coalesced);
ASSERT_TRUE(restricted_disconnect_kept_job);
@@ -2389,6 +2429,46 @@ TEST(daemon_application_sensitive_root_blocks_watch_but_preserves_controls) {
PASS();
}
TEST(daemon_application_programmatic_index_injects_resource_policy) {
char *root = th_mktempdir("cbm_app_policy_root");
char *cache = th_mktempdir("cbm_app_policy_cache");
ASSERT_NOT_NULL(root);
ASSERT_NOT_NULL(cache);
cbm_config_t *stored_config = cbm_config_open(cache);
bool configured = stored_config &&
cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 &&
cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0;
app_fake_worker_context_t fake;
app_fake_worker_context_init(&fake);
atomic_store(&fake.allow_completion, true);
cbm_daemon_application_worker_ops_t worker_ops = {
.context = &fake,
.start = app_fake_worker_start,
.poll = app_fake_worker_poll,
.cancel = app_fake_worker_cancel,
.log_path = app_fake_worker_log_path,
.destroy = app_fake_worker_destroy,
};
cbm_daemon_application_config_t config = {
.config = stored_config,
.worker_ops = &worker_ops,
};
cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL;
int index_rc =
application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1;
bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6");
bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS);
cbm_daemon_application_free(application);
cbm_config_close(stored_config);
th_cleanup(root);
th_cleanup(cache);
ASSERT_TRUE(configured);
ASSERT_TRUE(policy_propagated);
ASSERT_TRUE(stopped);
PASS();
}
TEST(daemon_application_auto_index_honors_tracked_file_limit) {
app_env_backup_t cache_environment;
bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR");
@@ -2464,6 +2544,9 @@ TEST(daemon_application_auto_index_honors_tracked_file_limit) {
PASS();
}
/* Native discovery never interprets a repository path as command text. Valid
* non-Git paths containing shell metacharacters remain countable literally. */
/* Native discovery never interprets a repository path as command text. Valid
* non-Git paths containing shell metacharacters remain countable literally. */
TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path) {
@@ -5570,6 +5653,7 @@ SUITE(daemon_application) {
RUN_TEST(daemon_application_sensitive_root_blocks_auto_index_but_preserves_controls);
RUN_TEST(daemon_application_sensitive_root_blocks_watch_but_preserves_controls);
RUN_TEST(daemon_application_index_args_compare_repo_path_separator_equivalently);
RUN_TEST(daemon_application_programmatic_index_injects_resource_policy);
RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit);
RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path);
RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots);
+161
View File
@@ -475,6 +475,163 @@ TEST(discover_bounded_count_matches_shebang_discovery) {
PASS();
}
TEST(discover_resource_policy_off_matches_legacy_discovery) {
char *base = th_mktempdir("cbm_disc_policy_off");
ASSERT(base != NULL);
th_write_file(TH_PATH(base, "src/first.c"), "int first;\n");
th_write_file(TH_PATH(base, "src/second.py"), "second = 2\n");
th_write_file(TH_PATH(base, "src/ignored.png"), "not source\n");
cbm_file_info_t *legacy_files = NULL;
int legacy_count = 0;
cbm_discover_opts_t legacy_opts = {.mode = CBM_MODE_FULL};
ASSERT_EQ(cbm_discover(base, &legacy_opts, &legacy_files, &legacy_count), CBM_DISCOVER_OK);
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
cbm_index_resource_violation_t violation = {0};
cbm_file_info_t *policy_files = NULL;
int policy_count = 0;
cbm_discover_opts_t policy_opts = {
.mode = CBM_MODE_FULL,
.resource_policy = &policy,
.resource_violation = &violation,
};
cbm_discover_status_t status = cbm_discover(base, &policy_opts, &policy_files, &policy_count);
ASSERT_EQ(status, CBM_DISCOVER_OK);
ASSERT_EQ(policy_count, legacy_count);
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE);
for (int index = 0; index < legacy_count; index++) {
ASSERT_STR_EQ(policy_files[index].rel_path, legacy_files[index].rel_path);
ASSERT_EQ(policy_files[index].size, legacy_files[index].size);
}
cbm_discover_free(legacy_files, legacy_count);
cbm_discover_free(policy_files, policy_count);
th_cleanup(base);
PASS();
}
TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources) {
char *base = th_mktempdir("cbm_disc_policy_files");
ASSERT(base != NULL);
th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n");
th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n");
th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n");
th_write_file(TH_PATH(base, "unsupported.png"), "not source\n");
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1};
cbm_index_resource_violation_t violation = {0};
cbm_discover_opts_t opts = {
.mode = CBM_MODE_FULL,
.resource_policy = &policy,
.resource_violation = &violation,
};
cbm_file_info_t *files = NULL;
int count = 0;
ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK);
ASSERT_EQ(count, 1);
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE);
cbm_discover_free(files, count);
th_write_file(TH_PATH(base, "second.py"), "second = 2\n");
files = NULL;
count = 99;
ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED);
ASSERT(files == NULL);
ASSERT_EQ(count, 0);
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES);
ASSERT_EQ(violation.observed, 2);
ASSERT_EQ(violation.limit, 1);
th_cleanup(base);
PASS();
}
TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte) {
char *base = th_mktempdir("cbm_disc_policy_bytes");
ASSERT(base != NULL);
th_write_file(TH_PATH(base, "exact.c"), "1234567");
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
policy.max_source_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 7};
cbm_index_resource_violation_t violation = {0};
cbm_discover_opts_t opts = {
.mode = CBM_MODE_FULL,
.resource_policy = &policy,
.resource_violation = &violation,
};
cbm_file_info_t *files = NULL;
int count = 0;
ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK);
ASSERT_EQ(count, 1);
cbm_discover_free(files, count);
th_write_file(TH_PATH(base, "plus.py"), "x");
files = NULL;
count = 99;
ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED);
ASSERT(files == NULL);
ASSERT_EQ(count, 0);
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_SOURCE_BYTES);
ASSERT_EQ(violation.observed, 8);
ASSERT_EQ(violation.limit, 7);
th_cleanup(base);
PASS();
}
TEST(discover_resource_file_budget_excludes_existing_oversized_skip) {
char *base = th_mktempdir("cbm_disc_policy_oversized");
ASSERT(base != NULL);
th_write_file(TH_PATH(base, "accepted.c"), "x");
th_write_file(TH_PATH(base, "oversized.py"), "123");
const char *saved_limit = getenv("CBM_MAX_FILE_BYTES");
char *saved_limit_copy = saved_limit ? strdup(saved_limit) : NULL;
cbm_setenv("CBM_MAX_FILE_BYTES", "2", 1);
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1};
cbm_index_resource_violation_t violation = {0};
cbm_discover_opts_t opts = {
.mode = CBM_MODE_FULL,
.resource_policy = &policy,
.resource_violation = &violation,
};
cbm_file_info_t *files = NULL;
int count = 0;
cbm_discover_status_t initial_status = cbm_discover(base, &opts, &files, &count);
bool oversized_did_not_consume_budget = initial_status == CBM_DISCOVER_OK && count == 2 &&
violation.resource == CBM_INDEX_RESOURCE_NONE;
cbm_discover_free(files, count);
th_write_file(TH_PATH(base, "second.c"), "y");
files = NULL;
count = 0;
cbm_discover_status_t exceeded_status = cbm_discover(base, &opts, &files, &count);
bool accepted_sources_exceeded = exceeded_status == CBM_DISCOVER_LIMIT_EXCEEDED &&
files == NULL && count == 0 &&
violation.resource == CBM_INDEX_RESOURCE_FILES &&
violation.observed == 2 && violation.limit == 1;
if (saved_limit_copy) {
cbm_setenv("CBM_MAX_FILE_BYTES", saved_limit_copy, 1);
} else {
cbm_unsetenv("CBM_MAX_FILE_BYTES");
}
free(saved_limit_copy);
th_cleanup(base);
ASSERT_TRUE(oversized_did_not_consume_budget);
ASSERT_TRUE(accepted_sources_exceeded);
PASS();
}
TEST(discover_skips_git_dir) {
char *base = th_mktempdir("cbm_disc_git");
ASSERT(base != NULL);
@@ -1931,6 +2088,10 @@ SUITE(discover) {
RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact);
RUN_TEST(discover_bounded_count_fails_closed_after_deadline);
RUN_TEST(discover_bounded_count_matches_shebang_discovery);
RUN_TEST(discover_resource_policy_off_matches_legacy_discovery);
RUN_TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources);
RUN_TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte);
RUN_TEST(discover_resource_file_budget_excludes_existing_oversized_skip);
RUN_TEST(discover_skips_git_dir);
RUN_TEST(discover_with_gitignore);
RUN_TEST(discover_with_global_xdg_ignore);
+403
View File
@@ -0,0 +1,403 @@
#include "test_framework.h"
#include "test_helpers.h"
#include "cli/cli.h"
#include "foundation/compat.h"
#include "foundation/index_policy.h"
#include "mcp/index_supervisor.h"
#include "mcp/mcp.h"
#include "store/store.h"
#include <sqlite3.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
TEST(index_policy_defaults_are_disabled) {
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
ASSERT_FALSE(policy.max_files.enabled);
ASSERT_FALSE(policy.max_source_bytes.enabled);
ASSERT_FALSE(cbm_index_policy_enabled(&policy));
ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off");
ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off");
PASS();
}
TEST(index_policy_file_limit_accepts_off_and_exact_range) {
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
char error[256];
ASSERT_TRUE(
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "1", error, sizeof(error)));
ASSERT_TRUE(policy.max_files.enabled);
ASSERT_EQ(policy.max_files.value, 1);
ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "10000000", error,
sizeof(error)));
ASSERT_EQ(policy.max_files.value, 10000000);
ASSERT_TRUE(
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "off", error, sizeof(error)));
ASSERT_FALSE(policy.max_files.enabled);
PASS();
}
TEST(index_policy_source_limit_converts_mib_without_overflow) {
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
char error[256];
ASSERT_TRUE(
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1", error, sizeof(error)));
ASSERT_TRUE(policy.max_source_bytes.enabled);
ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1024) * UINT64_C(1024));
ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1048576", error,
sizeof(error)));
ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1048576) * UINT64_C(1024) * UINT64_C(1024));
PASS();
}
TEST(index_policy_invalid_value_is_rejected_atomically) {
static const char *const invalid[] = {"", "0", "-1", "1MB",
"1 ", "+1", "10000001", "18446744073709551616"};
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
char error[256];
ASSERT_TRUE(
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error)));
for (size_t index = 0; index < sizeof(invalid) / sizeof(invalid[0]); index++) {
cbm_index_resource_policy_t before = policy;
ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, invalid[index],
error, sizeof(error)));
ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0);
ASSERT_TRUE(error[0] != '\0');
}
ASSERT_FALSE(cbm_index_policy_set(&policy, "index_unknown_limit", "1", error, sizeof(error)));
PASS();
}
TEST(index_policy_format_round_trips_public_values) {
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
char error[256];
char value[64];
ASSERT_TRUE(
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "321", error, sizeof(error)));
ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_FILES, value, sizeof(value)));
ASSERT_STR_EQ(value, "321");
ASSERT_TRUE(
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "7", error, sizeof(error)));
ASSERT_TRUE(
cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value)));
ASSERT_STR_EQ(value, "7");
PASS();
}
TEST(index_policy_violation_metadata_is_stable) {
ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FILES), "files");
ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes");
ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files");
ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes");
ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES),
CBM_INDEX_CONFIG_MAX_FILES);
ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES),
CBM_INDEX_CONFIG_MAX_SOURCE_MB);
PASS();
}
TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) {
char *cache = th_mktempdir("cbm_index_policy_config");
ASSERT_NOT_NULL(cache);
cbm_config_t *config = cbm_config_open(cache);
ASSERT_NOT_NULL(config);
cbm_index_resource_policy_t policy;
char error[256];
ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error)));
ASSERT_FALSE(cbm_index_policy_enabled(&policy));
ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0);
ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0);
ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error)));
ASSERT_EQ(policy.max_files.value, 9);
ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES);
ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0);
ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error)));
ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_MAX_FILES) != NULL);
cbm_config_close(config);
th_cleanup(cache);
PASS();
}
TEST(index_policy_cli_lists_both_operator_keys) {
bool files_found = false;
bool bytes_found = false;
for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) {
const char *key = cbm_cli_config_key_at_for_testing(index);
files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0);
bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0);
}
ASSERT_TRUE(files_found);
ASSERT_TRUE(bytes_found);
PASS();
}
TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) {
char *cache = th_mktempdir("cbm_index_policy_cli");
ASSERT_NOT_NULL(cache);
const char *saved_cache = getenv("CBM_CACHE_DIR");
char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL;
(void)cbm_setenv("CBM_CACHE_DIR", cache, 1);
char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"};
char *set_invalid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "0"};
char *reset[] = {"reset", CBM_INDEX_CONFIG_MAX_FILES};
int valid_rc = cbm_cmd_config(3, set_valid);
cbm_config_t *config = cbm_config_open(cache);
const char *stored =
config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing";
bool valid_stored = strcmp(stored, "7") == 0;
int invalid_rc = cbm_cmd_config(3, set_invalid);
stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing";
bool invalid_preserved = strcmp(stored, "7") == 0;
int reset_rc = cbm_cmd_config(2, reset);
stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "off") : "missing";
bool reset_to_default = strcmp(stored, "off") == 0;
cbm_config_close(config);
if (saved_cache_copy) {
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1);
} else {
(void)cbm_unsetenv("CBM_CACHE_DIR");
}
free(saved_cache_copy);
th_cleanup(cache);
ASSERT_EQ(valid_rc, 0);
ASSERT_TRUE(valid_stored);
ASSERT_TRUE(invalid_rc != 0);
ASSERT_TRUE(invalid_preserved);
ASSERT_EQ(reset_rc, 0);
ASSERT_TRUE(reset_to_default);
PASS();
}
/* Capture stderr across one cbm_cmd_config invocation, mirroring the stdout
* idiom in test_cli.c: tmpfile+dup2+rewind is what works on the MinGW leg. */
static int index_policy_config_stderr(int argc, char **argv, char *out, size_t cap) {
out[0] = '\0';
FILE *capture = tmpfile();
int saved = capture ? dup(fileno(stderr)) : -1;
if (!capture || saved < 0) {
if (capture) {
(void)fclose(capture);
}
if (saved >= 0) {
(void)close(saved);
}
return -1000;
}
(void)fflush(stderr);
if (dup2(fileno(capture), fileno(stderr)) < 0) {
(void)fclose(capture);
(void)close(saved);
return -1000;
}
int rc = cbm_cmd_config(argc, argv);
(void)fflush(stderr);
(void)dup2(saved, fileno(stderr));
(void)close(saved);
rewind(capture);
size_t got = fread(out, 1, cap - 1, capture);
out[got] = '\0';
(void)fclose(capture);
return rc;
}
/* `config set` suppresses its own generic message for a policy key, trusting
* the policy writer to name the precise reason. That trust held only for a
* value the writer rejects. A value it accepts whose write then fails — a
* _config.db that cannot be written — exited non-zero having printed nothing
* at all, which is a CLI that failed in silence.
*
* The unwritable database here is a `config` that is a view: CREATE TABLE IF
* NOT EXISTS finds the name taken and succeeds, so the command gets past open
* with a healthy handle and fails at the INSERT. That is the exact path with
* no message, and it needs no file permissions, so it behaves the same on
* every leg and under a root CI container. */
TEST(index_policy_cli_set_reports_a_failed_write) {
char *cache = th_mktempdir("cbm_index_policy_cli_write");
ASSERT_NOT_NULL(cache);
const char *saved_cache = getenv("CBM_CACHE_DIR");
char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL;
(void)cbm_setenv("CBM_CACHE_DIR", cache, 1);
char db_path[1024];
(void)snprintf(db_path, sizeof(db_path), "%s/_config.db", cache);
sqlite3 *db = NULL;
bool fixture_ready =
sqlite3_open(db_path, &db) == SQLITE_OK &&
sqlite3_exec(db, "CREATE VIEW config(key, value) AS SELECT 'pinned', 'pinned'", NULL, NULL,
NULL) == SQLITE_OK;
if (db) {
(void)sqlite3_close(db);
}
char captured[1024];
char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"};
int rc = index_policy_config_stderr(3, set_valid, captured, sizeof(captured));
if (saved_cache_copy) {
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1);
} else {
(void)cbm_unsetenv("CBM_CACHE_DIR");
}
free(saved_cache_copy);
th_cleanup(cache);
ASSERT_TRUE(fixture_ready);
ASSERT_TRUE(rc != 0);
ASSERT_TRUE(strstr(captured, CBM_INDEX_CONFIG_MAX_FILES) != NULL);
PASS();
}
TEST(index_policy_worker_rejects_missing_parent_policy) {
char *repo = th_mktempdir("cbm_index_policy_worker");
ASSERT_NOT_NULL(repo);
char args[1024];
(void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"mode\":\"fast\"}", repo);
cbm_mcp_server_t *server = cbm_mcp_server_new(NULL);
cbm_index_set_worker_role(true, NULL);
char *response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL;
cbm_index_set_worker_role(false, NULL);
bool rejected = response && strstr(response, "missing or incomplete trusted worker policy");
free(response);
cbm_mcp_server_free(server);
th_cleanup(repo);
ASSERT_TRUE(rejected);
PASS();
}
TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) {
char *repo = th_mktempdir("cbm_index_policy_mcp_repo");
char *cache = th_mktempdir("cbm_index_policy_mcp_cache");
ASSERT_NOT_NULL(repo);
ASSERT_NOT_NULL(cache);
ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first(void) { return 1; }\n"), 0);
const char *saved_cache = getenv("CBM_CACHE_DIR");
char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL;
(void)cbm_setenv("CBM_CACHE_DIR", cache, 1);
cbm_config_t *config = cbm_config_open(cache);
cbm_mcp_server_t *server = cbm_mcp_server_new(NULL);
if (server && config) {
cbm_mcp_server_set_config(server, config);
}
char args[2048];
(void)snprintf(args, sizeof(args),
"{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\","
"\"mode\":\"fast\"}",
repo);
char *first_response =
server && config ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL;
bool first_indexed = first_response && strstr(first_response, "\\\"status\\\":\\\"indexed\\\"");
free(first_response);
char db_path[2048];
(void)snprintf(db_path, sizeof(db_path), "%s/ResourcePolicyFixture.db", cache);
cbm_store_t *before_store = cbm_store_open_path_query(db_path);
cbm_project_t before_project = {0};
bool before_read = before_store && cbm_store_get_project(before_store, "ResourcePolicyFixture",
&before_project) == CBM_STORE_OK;
char *indexed_at_before =
before_read && before_project.indexed_at ? strdup(before_project.indexed_at) : NULL;
cbm_project_free_fields(&before_project);
cbm_store_close(before_store);
bool configured =
config && cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0 &&
th_write_file(TH_PATH(repo, "second.py"), "def second():\n return 2\n") == 0;
(void)snprintf(args, sizeof(args),
"{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\","
"\"mode\":\"fast\",\"_cbm_index_policy\":{"
"\"index_max_files\":\"off\",\"index_max_source_mb\":\"off\"}}",
repo);
char *limited_response =
configured && server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL;
bool contract_ok = limited_response && strstr(limited_response, "resource_limit_exceeded") &&
strstr(limited_response, "\\\"stage\\\":\\\"discovery\\\"") &&
strstr(limited_response, "\\\"resource\\\":\\\"files\\\"") &&
strstr(limited_response, "\\\"observed\\\":2") &&
strstr(limited_response, "\\\"limit\\\":1") &&
strstr(limited_response, "\\\"unit\\\":\\\"files\\\"") &&
strstr(limited_response, "\\\"retryable\\\":true") &&
strstr(limited_response, "\\\"serving_index_preserved\\\":true");
free(limited_response);
cbm_store_t *after_store = cbm_store_open_path_query(db_path);
cbm_project_t after_project = {0};
bool after_read = after_store && cbm_store_get_project(after_store, "ResourcePolicyFixture",
&after_project) == CBM_STORE_OK;
bool generation_preserved = indexed_at_before && after_read && after_project.indexed_at &&
strcmp(indexed_at_before, after_project.indexed_at) == 0;
cbm_project_free_fields(&after_project);
cbm_store_close(after_store);
free(indexed_at_before);
(void)snprintf(args, sizeof(args),
"{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\","
"\"mode\":\"cross-repo-intelligence\"}",
repo);
char *cross_response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL;
bool cross_repo_unaffected =
cross_response && strstr(cross_response, "resource_limit_exceeded") == NULL;
free(cross_response);
cbm_mcp_server_free(server);
cbm_config_close(config);
(void)cbm_unlink(db_path);
char sidecar[2100];
(void)snprintf(sidecar, sizeof(sidecar), "%s-wal", db_path);
(void)cbm_unlink(sidecar);
(void)snprintf(sidecar, sizeof(sidecar), "%s-shm", db_path);
(void)cbm_unlink(sidecar);
th_cleanup(repo);
th_cleanup(cache);
if (saved_cache_copy) {
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1);
} else {
(void)cbm_unsetenv("CBM_CACHE_DIR");
}
free(saved_cache_copy);
ASSERT_TRUE(first_indexed);
ASSERT_TRUE(configured);
ASSERT_TRUE(contract_ok);
ASSERT_TRUE(generation_preserved);
ASSERT_TRUE(cross_repo_unaffected);
PASS();
}
SUITE(index_policy) {
RUN_TEST(index_policy_defaults_are_disabled);
RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range);
RUN_TEST(index_policy_source_limit_converts_mib_without_overflow);
RUN_TEST(index_policy_invalid_value_is_rejected_atomically);
RUN_TEST(index_policy_format_round_trips_public_values);
RUN_TEST(index_policy_violation_metadata_is_stable);
RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption);
RUN_TEST(index_policy_cli_lists_both_operator_keys);
RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite);
RUN_TEST(index_policy_cli_set_reports_a_failed_write);
RUN_TEST(index_policy_worker_rejects_missing_parent_policy);
RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index);
}
+2
View File
@@ -776,6 +776,7 @@ extern void suite_dyn_array(void);
extern void suite_str_intern(void);
extern void suite_log(void);
extern void suite_str_util(void);
extern void suite_index_policy(void);
extern void suite_workspace(void);
extern void suite_platform(void);
extern void suite_diagnostics(void);
@@ -1054,6 +1055,7 @@ int main(int argc, char **argv) {
RUN_SELECTED_SUITE(str_intern);
RUN_SELECTED_SUITE(log);
RUN_SELECTED_SUITE(str_util);
RUN_SELECTED_SUITE(index_policy);
RUN_SELECTED_SUITE(workspace);
RUN_SELECTED_SUITE(platform);
RUN_SELECTED_SUITE(diagnostics);
+68
View File
@@ -151,6 +151,34 @@ TEST(pipeline_run_null) {
PASS();
}
TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing) {
char *repo = th_mktempdir("cbm_pipeline_discovery_limit");
ASSERT_NOT_NULL(repo);
ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0);
ASSERT_EQ(th_write_file(TH_PATH(repo, "second.py"), "second = 2\n"), 0);
char db_path[512];
snprintf(db_path, sizeof(db_path), "%s/index.db", repo);
cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(pipeline);
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1};
cbm_pipeline_set_resource_policy(pipeline, &policy);
ASSERT_EQ(cbm_pipeline_run(pipeline), CBM_PIPELINE_RESOURCE_LIMIT);
cbm_index_resource_violation_t violation = {0};
cbm_pipeline_get_resource_violation(pipeline, &violation);
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES);
ASSERT_EQ(violation.observed, 2);
ASSERT_EQ(violation.limit, 1);
ASSERT_FALSE(cbm_file_exists(db_path));
cbm_pipeline_free(pipeline);
th_cleanup(repo);
PASS();
}
/* ── Focused: file-backed store persistence ─────────────────────── */
TEST(store_file_persistence) {
@@ -3412,6 +3440,44 @@ static void mutate_semantic_input_before_final_manifest(void *userdata) {
mutation->write_rc = th_write_file(mutation->path, mutation->replacement);
}
TEST(pipeline_late_source_limit_preserves_previous_generation) {
char *repo = th_mktempdir("cbm_pipeline_late_limit");
ASSERT_NOT_NULL(repo);
ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0);
char db_path[512];
char late_path[512];
snprintf(db_path, sizeof(db_path), "%s/index.db", repo);
snprintf(late_path, sizeof(late_path), "%s/late.py", repo);
manifest_race_mutation_t mutation = {
.path = late_path,
.replacement = "late = 2\n",
.write_rc = -1,
};
cbm_pipeline_incremental_test_before_final_manifest_once(
mutate_semantic_input_before_final_manifest, &mutation);
cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(pipeline);
cbm_index_resource_policy_t policy;
cbm_index_policy_init(&policy);
policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1};
cbm_pipeline_set_resource_policy(pipeline, &policy);
int rc = cbm_pipeline_run(pipeline);
cbm_index_resource_violation_t violation = {0};
cbm_pipeline_get_resource_violation(pipeline, &violation);
cbm_pipeline_free(pipeline);
cbm_pipeline_incremental_test_reset_faults();
ASSERT_EQ(mutation.write_rc, 0);
ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT);
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES);
ASSERT_EQ(violation.observed, 2);
ASSERT_EQ(violation.limit, 1);
ASSERT_FALSE(cbm_file_exists(db_path));
th_cleanup(repo);
PASS();
}
/* A graph and its exact manifest are one generation. If source bytes change
* after extraction but before publication, the mixed generation must be
* rejected and the previous live DB preserved. */
@@ -14726,6 +14792,8 @@ SUITE(pipeline) {
RUN_TEST(pipeline_cancel);
RUN_TEST(pipeline_cancel_null);
RUN_TEST(pipeline_run_null);
RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing);
RUN_TEST(pipeline_late_source_limit_preserves_previous_generation);
/* Extraction back-pressure */
RUN_TEST(pipeline_backpressure_futile_nap_disengages);
RUN_TEST(pipeline_over_budget_after_futility_fails_whole_and_preserves_db);
+1 -1
View File
@@ -48,7 +48,7 @@ trap cleanup EXIT
missing="${tmpdir}/repository-does-not-exist"
response="${tmpdir}/worker.response"
args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\"}"
args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\",$(cbm_test_index_worker_policy_json)}"
if ! CBM_CACHE_DIR="${tmpdir}/cache-worker" \
"${BINARY}" cli --index-worker \
+1 -1
View File
@@ -109,7 +109,7 @@ SH
chmod +x "${tmpdir}/wrapper.sh"
CBM_BINARY="${BINARY}" BUILD_FINGERPRINT="${BUILD_FINGERPRINT}" TMPDIR_PATH="${tmpdir}" \
ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\"}" \
ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\",$(cbm_test_index_worker_policy_json)}" \
CBM_TEST_HANG_ON=hang_me \
CBM_TEST_WORKER_DESCENDANT_PID_FILE="${tmpdir}/descendant.pid" \
"${tmpdir}/wrapper.sh" &