mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-10-02 04:54:47 +08:00
Merge pull request #1723 from liuchong/feat/index-resource-discovery
feat(index): add opt-in discovery resource limits
This commit is contained in:
@@ -325,6 +325,7 @@ FOUNDATION_SRCS = \
|
||||
src/foundation/profile.c \
|
||||
src/foundation/dump_verify.c \
|
||||
src/foundation/limits.c \
|
||||
src/foundation/index_policy.c \
|
||||
src/foundation/subprocess.c \
|
||||
src/foundation/sha256.c \
|
||||
src/foundation/secure_random.c \
|
||||
@@ -617,6 +618,7 @@ TEST_FOUNDATION_SRCS = \
|
||||
tests/test_str_intern.c \
|
||||
tests/test_log.c \
|
||||
tests/test_str_util.c \
|
||||
tests/test_index_policy.c \
|
||||
tests/test_workspace.c \
|
||||
tests/test_platform.c \
|
||||
tests/test_diagnostics.c \
|
||||
|
||||
@@ -733,9 +733,15 @@ codebase-memory-mcp config set auto_index true # auto-index on session
|
||||
codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index
|
||||
codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true)
|
||||
codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true)
|
||||
codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit
|
||||
codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit
|
||||
codebase-memory-mcp config reset auto_index # reset to default
|
||||
```
|
||||
|
||||
The two `index_max_*` settings default to `off`. Exceeding one fails the complete
|
||||
index attempt rather than publishing a partial graph; an existing serving index
|
||||
is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md).
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|
||||
@@ -89,6 +89,8 @@ Current keys:
|
||||
| `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. |
|
||||
| `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). |
|
||||
| `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. |
|
||||
| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |
|
||||
| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. |
|
||||
|
||||
> **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the
|
||||
> watcher *subsystem* starts at all (the background poll thread). `auto_watch` is
|
||||
@@ -116,6 +118,14 @@ Current keys:
|
||||
> `auto_index` still runs, and `index_repository` stays available for manual
|
||||
> reindexing.
|
||||
|
||||
The two `index_max_*` settings are independent and disabled by default. They
|
||||
apply to explicit indexing, automatic indexing, and watcher re-indexing, but not
|
||||
to `cross-repo-intelligence`, which does not scan repository source files.
|
||||
Equality is allowed; exceeding either setting fails the complete index request
|
||||
and preserves any previously serving database. See
|
||||
[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and
|
||||
error-response details.
|
||||
|
||||
## 3. UI Settings
|
||||
|
||||
The optional built-in graph UI stores its settings in:
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# Index resource limits
|
||||
|
||||
Index resource limits are optional operator controls for repositories whose
|
||||
discovery breadth is not known in advance. They are disabled by default so
|
||||
existing large-repository workloads retain their current behavior.
|
||||
|
||||
## Discovery settings
|
||||
|
||||
| Key | Default | Accepted value | Protects |
|
||||
|---|---:|---:|---|
|
||||
| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count |
|
||||
| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes |
|
||||
|
||||
Set or reset them with the normal configuration command:
|
||||
|
||||
```bash
|
||||
codebase-memory-mcp config set index_max_files 250000
|
||||
codebase-memory-mcp config set index_max_source_mb 16384
|
||||
codebase-memory-mcp config reset index_max_files
|
||||
```
|
||||
|
||||
Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero,
|
||||
negative values, suffixes, trailing characters, and values outside the stated
|
||||
ranges are rejected without changing the stored value.
|
||||
|
||||
## Counting and failure semantics
|
||||
|
||||
`index_max_files` counts a file only after it passes directory pruning, ignore
|
||||
rules, filename and suffix filters, language detection, and the existing
|
||||
per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same
|
||||
accepted set.
|
||||
|
||||
Equality is allowed. The first file or byte that makes an observed value greater
|
||||
than its limit stops discovery. CBM discards the partial file list and does not
|
||||
publish a partial graph as a complete index.
|
||||
|
||||
For an explicit MCP request the error payload contains:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "error",
|
||||
"code": "resource_limit_exceeded",
|
||||
"stage": "discovery",
|
||||
"resource": "files",
|
||||
"observed": 250001,
|
||||
"limit": 250000,
|
||||
"unit": "files",
|
||||
"retryable": true,
|
||||
"serving_index_preserved": true,
|
||||
"message": "Index discovery exceeded index_max_files"
|
||||
}
|
||||
```
|
||||
|
||||
The previous database remains available because publication occurs only after a
|
||||
complete discovery and successful staged build. If no previous database exists,
|
||||
`serving_index_preserved` is false.
|
||||
|
||||
## Trust and compatibility
|
||||
|
||||
Limits are read from the CLI-managed `_config.db`; they are not MCP request
|
||||
arguments. A supervised parent replaces any caller-supplied internal policy
|
||||
before spawning its worker, and the worker rejects a missing or incomplete
|
||||
parent policy.
|
||||
|
||||
These settings do not replace or increase `auto_index_limit`, change the 512 MiB
|
||||
single-file cap, alter workspace-root authorization, or affect
|
||||
`cross-repo-intelligence`. With both settings `off`, discovery follows the
|
||||
existing unbounded path.
|
||||
@@ -105,6 +105,16 @@ _cbm_test_runtime_daemon() {
|
||||
CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2"
|
||||
}
|
||||
|
||||
# The supervisor resolves one resource policy per index and hands it to the
|
||||
# worker in argv; a worker that finds no complete policy refuses to start rather
|
||||
# than index unbounded. A shell test that spawns `cli --index-worker` itself
|
||||
# stands in for the supervisor and owes the worker the same object. Mirrors
|
||||
# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and
|
||||
# nothing else.
|
||||
cbm_test_index_worker_policy_json() {
|
||||
printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}'
|
||||
}
|
||||
|
||||
cbm_test_runtime_cleanup() {
|
||||
local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT"
|
||||
local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0
|
||||
|
||||
+54
-2
@@ -7404,6 +7404,25 @@ bool cbm_config_watcher_enabled(cbm_config_t *cfg) {
|
||||
return cbm_config_get_bool(cfg, CBM_CONFIG_WATCHER_ENABLED, true);
|
||||
}
|
||||
|
||||
bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
|
||||
char *error, size_t error_size) {
|
||||
if (!cfg || !policy) {
|
||||
if (error && error_size > 0) {
|
||||
(void)snprintf(error, error_size, "index resource configuration is unavailable");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
cbm_index_policy_init(policy);
|
||||
for (size_t index = 0; index < cbm_index_policy_key_count(); index++) {
|
||||
const char *key = cbm_index_policy_key_at(index);
|
||||
const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key));
|
||||
if (!cbm_index_policy_set(policy, key, value, error, error_size)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* ── Config CLI subcommand ────────────────────────────────────── */
|
||||
|
||||
/* THE config-key table. list, get, help, and key validation all read this one
|
||||
@@ -7428,6 +7447,8 @@ static const config_key_def_t CONFIG_KEYS[] = {
|
||||
{CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"},
|
||||
{CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"},
|
||||
{CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"},
|
||||
{CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"},
|
||||
{CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"},
|
||||
};
|
||||
|
||||
/* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing
|
||||
@@ -7453,6 +7474,31 @@ static bool config_key_is_ui(const char *key) {
|
||||
return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0);
|
||||
}
|
||||
|
||||
static bool config_key_is_index_policy(const char *key) {
|
||||
return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 ||
|
||||
strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0);
|
||||
}
|
||||
|
||||
static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) {
|
||||
cbm_index_resource_policy_t candidate;
|
||||
cbm_index_policy_init(&candidate);
|
||||
char error[CLI_BUF_256];
|
||||
if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) {
|
||||
(void)fprintf(stderr, "error: %s\n", error);
|
||||
return CLI_ERR;
|
||||
}
|
||||
int rc = cbm_config_set(config, key, value);
|
||||
if (rc != 0) {
|
||||
/* The caller suppresses its own message for policy keys because this
|
||||
* helper names the precise reason. That is only true if the helper
|
||||
* speaks on every failure it can return: a validated value whose write
|
||||
* then fails -- a locked or read-only _config.db -- used to exit
|
||||
* non-zero having printed nothing at all. */
|
||||
(void)fprintf(stderr, "error: failed to set %s\n", key);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
static void config_ui_read(const char *key, char *out, size_t out_sz) {
|
||||
cbm_ui_config_t ui;
|
||||
cbm_ui_config_load(&ui);
|
||||
@@ -7590,10 +7636,16 @@ int cbm_cmd_config(int argc, char **argv) {
|
||||
rc = CLI_TRUE;
|
||||
}
|
||||
} else {
|
||||
if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) {
|
||||
int set_rc =
|
||||
config_key_is_index_policy(argv[CLI_SKIP_ONE])
|
||||
? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN])
|
||||
: cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
|
||||
if (set_rc == 0) {
|
||||
printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
|
||||
} else {
|
||||
(void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
|
||||
if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) {
|
||||
(void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
|
||||
}
|
||||
rc = CLI_TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "foundation/index_policy.h"
|
||||
|
||||
typedef struct cbm_mcp_server cbm_mcp_server_t;
|
||||
|
||||
/* ── Version ──────────────────────────────────────────────────── */
|
||||
@@ -420,6 +422,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value);
|
||||
/* Delete a config key. Returns 0 on success. */
|
||||
int cbm_config_delete(cbm_config_t *cfg, const char *key);
|
||||
|
||||
/* Load and validate the operator-controlled discovery policy. Invalid stored
|
||||
* values fail closed instead of silently disabling a guard. */
|
||||
bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
|
||||
char *error, size_t error_size);
|
||||
|
||||
/* Well-known config keys */
|
||||
#define CBM_CONFIG_AUTO_INDEX "auto_index"
|
||||
#define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit"
|
||||
|
||||
@@ -208,7 +208,8 @@ static bool application_unique_recovery_file(char out[APPLICATION_PATH_CAP], con
|
||||
static bool application_update_reap(cbm_daemon_application_t *application, bool wait,
|
||||
uint32_t timeout_ms);
|
||||
static void *application_job_thread(void *opaque);
|
||||
static char *application_auto_index_args(const char *root_path);
|
||||
static char *application_auto_index_args(cbm_daemon_application_t *application,
|
||||
const char *root_path);
|
||||
static cbm_daemon_application_job_t *application_job_subscribe_locked(
|
||||
cbm_daemon_application_t *application, const char *project_key, const char *root_path,
|
||||
const char *args_json, application_job_subscribe_status_t *status_out);
|
||||
@@ -1455,7 +1456,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t
|
||||
application_refresh_watch_locked(session);
|
||||
continue;
|
||||
}
|
||||
char *args = application_auto_index_args(root_path);
|
||||
char *args = application_auto_index_args(application, root_path);
|
||||
if (!args) {
|
||||
continue;
|
||||
}
|
||||
@@ -2005,7 +2006,27 @@ static bool application_update_reap(cbm_daemon_application_t *application, bool
|
||||
}
|
||||
}
|
||||
|
||||
static char *application_auto_index_args(const char *root_path) {
|
||||
static bool application_index_args_add_policy(cbm_daemon_application_t *application,
|
||||
yyjson_mut_doc *document, yyjson_mut_val *root) {
|
||||
cbm_config_t *owned_config = NULL;
|
||||
cbm_config_t *config = application ? application->config : NULL;
|
||||
if (!config) {
|
||||
owned_config = cbm_config_open(cbm_resolve_cache_dir());
|
||||
config = owned_config;
|
||||
}
|
||||
cbm_index_resource_policy_t policy;
|
||||
char error[CBM_SZ_256] = {0};
|
||||
bool loaded = cbm_config_load_index_policy(config, &policy, error, sizeof(error));
|
||||
cbm_config_close(owned_config);
|
||||
if (!loaded) {
|
||||
cbm_log_error("daemon.index.policy", "error", error);
|
||||
return false;
|
||||
}
|
||||
return cbm_mcp_index_policy_add_to_args(document, root, &policy);
|
||||
}
|
||||
|
||||
static char *application_auto_index_args(cbm_daemon_application_t *application,
|
||||
const char *root_path) {
|
||||
yyjson_mut_doc *document = yyjson_mut_doc_new(NULL);
|
||||
yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL;
|
||||
if (!document || !root) {
|
||||
@@ -2013,7 +2034,8 @@ static char *application_auto_index_args(const char *root_path) {
|
||||
return NULL;
|
||||
}
|
||||
yyjson_mut_doc_set_root(document, root);
|
||||
char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path)
|
||||
char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
|
||||
application_index_args_add_policy(application, document, root)
|
||||
? yyjson_mut_write(document, 0, NULL)
|
||||
: NULL;
|
||||
yyjson_mut_doc_free(document);
|
||||
@@ -2060,7 +2082,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio
|
||||
files);
|
||||
}
|
||||
bool args_required = auto_index_candidate && within_auto_index_limit;
|
||||
char *args = args_required ? application_auto_index_args(root_path) : NULL;
|
||||
char *args = args_required ? application_auto_index_args(application, root_path) : NULL;
|
||||
application_jobs_reap_completed(application);
|
||||
cbm_mutex_lock(&application->mutex);
|
||||
if (application->stopping || application_request_cancelled_locked(session)) {
|
||||
@@ -3458,7 +3480,8 @@ static int application_background_index(cbm_daemon_application_t *application,
|
||||
return -1;
|
||||
}
|
||||
yyjson_mut_doc_set_root(document, root);
|
||||
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root);
|
||||
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) &&
|
||||
application_index_args_add_policy(application, document, root);
|
||||
char *default_project = cbm_project_name_from_path(canonical_root);
|
||||
bool custom_project =
|
||||
project_name[0] && (!default_project || strcmp(default_project, project_name) != 0);
|
||||
|
||||
+49
-2
@@ -13,6 +13,7 @@
|
||||
|
||||
#include "foundation/constants.h"
|
||||
#include "foundation/compat_fs.h"
|
||||
#include "foundation/limits.h"
|
||||
#include "foundation/workspace.h"
|
||||
#include "foundation/platform.h"
|
||||
#ifdef _WIN32
|
||||
@@ -414,6 +415,10 @@ typedef struct {
|
||||
int capacity;
|
||||
int max_files;
|
||||
uint64_t deadline_ms;
|
||||
const cbm_index_resource_policy_t *resource_policy;
|
||||
cbm_index_resource_violation_t *resource_violation;
|
||||
uint64_t source_files;
|
||||
uint64_t source_bytes;
|
||||
bool count_only;
|
||||
bool collect_excluded;
|
||||
bool limit_exceeded;
|
||||
@@ -435,6 +440,21 @@ typedef struct {
|
||||
int ignored_total;
|
||||
} file_list_t;
|
||||
|
||||
static void file_list_resource_violation(file_list_t *fl, cbm_index_resource_t resource,
|
||||
uint64_t observed, uint64_t limit) {
|
||||
if (!fl || fl->limit_exceeded || fl->failed) {
|
||||
return;
|
||||
}
|
||||
fl->limit_exceeded = true;
|
||||
if (fl->resource_violation) {
|
||||
*fl->resource_violation = (cbm_index_resource_violation_t){
|
||||
.resource = resource,
|
||||
.observed = observed,
|
||||
.limit = limit,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
static bool file_list_should_stop(file_list_t *fl) {
|
||||
if (!fl) {
|
||||
return true;
|
||||
@@ -504,6 +524,28 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path,
|
||||
fl->limit_exceeded = true;
|
||||
return;
|
||||
}
|
||||
uint64_t source_size = size > 0 ? (uint64_t)size : 0;
|
||||
bool resource_accepted = fl->resource_policy && source_size <= (uint64_t)cbm_max_file_bytes();
|
||||
if (resource_accepted) {
|
||||
if (fl->resource_policy->max_files.enabled &&
|
||||
fl->source_files >= fl->resource_policy->max_files.value) {
|
||||
file_list_resource_violation(fl, CBM_INDEX_RESOURCE_FILES, fl->source_files + 1U,
|
||||
fl->resource_policy->max_files.value);
|
||||
return;
|
||||
}
|
||||
if (fl->resource_policy->max_source_bytes.enabled) {
|
||||
uint64_t limit = fl->resource_policy->max_source_bytes.value;
|
||||
if (source_size > limit || fl->source_bytes > limit - source_size) {
|
||||
uint64_t observed = source_size > UINT64_MAX - fl->source_bytes
|
||||
? UINT64_MAX
|
||||
: fl->source_bytes + source_size;
|
||||
file_list_resource_violation(fl, CBM_INDEX_RESOURCE_SOURCE_BYTES, observed, limit);
|
||||
return;
|
||||
}
|
||||
fl->source_bytes += source_size;
|
||||
}
|
||||
fl->source_files++;
|
||||
}
|
||||
if (fl->count_only) {
|
||||
fl->count++;
|
||||
return;
|
||||
@@ -1200,6 +1242,9 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc
|
||||
if (ignored_total_out) {
|
||||
*ignored_total_out = 0;
|
||||
}
|
||||
if (opts && opts->resource_violation) {
|
||||
*opts->resource_violation = (cbm_index_resource_violation_t){0};
|
||||
}
|
||||
if (!repo_path || !out || !count || (count_only && max_files < 0)) {
|
||||
return CBM_DISCOVER_ERROR;
|
||||
}
|
||||
@@ -1274,6 +1319,8 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc
|
||||
file_list_t fl = {
|
||||
.max_files = count_only ? max_files : -1,
|
||||
.deadline_ms = count_only ? deadline_ms : 0,
|
||||
.resource_policy = opts ? opts->resource_policy : NULL,
|
||||
.resource_violation = opts ? opts->resource_violation : NULL,
|
||||
.count_only = count_only,
|
||||
.collect_excluded = !count_only && excluded_out != NULL,
|
||||
.collect_ignored = !count_only && ignored_out != NULL,
|
||||
@@ -1296,11 +1343,11 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc
|
||||
}
|
||||
return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_OK;
|
||||
}
|
||||
if (fl.failed) {
|
||||
if (fl.failed || fl.limit_exceeded) {
|
||||
cbm_discover_free(fl.files, fl.count);
|
||||
cbm_discover_free_excluded(fl.excluded, fl.excluded_count);
|
||||
cbm_discover_free_ignored(fl.ignored, fl.ignored_count);
|
||||
return CBM_DISCOVER_ERROR;
|
||||
return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_ERROR;
|
||||
}
|
||||
|
||||
*out = fl.files;
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
|
||||
/* Use the existing CBMLanguage enum from extraction layer */
|
||||
#include "cbm.h"
|
||||
#include "foundation/index_policy.h"
|
||||
|
||||
/* ── Language detection ──────────────────────────────────────────── */
|
||||
|
||||
@@ -143,9 +144,11 @@ typedef struct {
|
||||
} cbm_file_info_t;
|
||||
|
||||
typedef struct {
|
||||
cbm_index_mode_t mode; /* CBM_MODE_FULL or CBM_MODE_FAST */
|
||||
const char *ignore_file; /* path to .cbmignore file, or NULL */
|
||||
int64_t max_file_size; /* 0 = no limit */
|
||||
cbm_index_mode_t mode; /* discovery filtering mode */
|
||||
const char *ignore_file; /* .cbmignore path, or NULL */
|
||||
int64_t max_file_size; /* 0 = no per-file limit */
|
||||
const cbm_index_resource_policy_t *resource_policy; /* NULL = no resource limits */
|
||||
cbm_index_resource_violation_t *resource_violation; /* optional exact diagnostic */
|
||||
} cbm_discover_opts_t;
|
||||
|
||||
typedef enum {
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
#include "foundation/index_policy.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
static const char *const INDEX_POLICY_KEYS[] = {
|
||||
CBM_INDEX_CONFIG_MAX_FILES,
|
||||
CBM_INDEX_CONFIG_MAX_SOURCE_MB,
|
||||
};
|
||||
|
||||
static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) {
|
||||
if (error && error_size > 0) {
|
||||
(void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key,
|
||||
(unsigned long long)maximum);
|
||||
}
|
||||
}
|
||||
|
||||
static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) {
|
||||
if (!value || !value[0] || !parsed) {
|
||||
return false;
|
||||
}
|
||||
uint64_t result = 0;
|
||||
for (const unsigned char *cursor = (const unsigned char *)value; *cursor; cursor++) {
|
||||
if (*cursor < '0' || *cursor > '9') {
|
||||
return false;
|
||||
}
|
||||
uint64_t digit = (uint64_t)(*cursor - '0');
|
||||
if (result > (maximum - digit) / 10U) {
|
||||
return false;
|
||||
}
|
||||
result = result * 10U + digit;
|
||||
}
|
||||
if (result == 0) {
|
||||
return false;
|
||||
}
|
||||
*parsed = result;
|
||||
return true;
|
||||
}
|
||||
|
||||
void cbm_index_policy_init(cbm_index_resource_policy_t *policy) {
|
||||
if (policy) {
|
||||
*policy = (cbm_index_resource_policy_t){0};
|
||||
}
|
||||
}
|
||||
|
||||
bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) {
|
||||
return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled);
|
||||
}
|
||||
|
||||
size_t cbm_index_policy_key_count(void) {
|
||||
return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]);
|
||||
}
|
||||
|
||||
const char *cbm_index_policy_key_at(size_t index) {
|
||||
return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL;
|
||||
}
|
||||
|
||||
const char *cbm_index_policy_default_value(const char *key) {
|
||||
for (size_t index = 0; index < cbm_index_policy_key_count(); index++) {
|
||||
if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) {
|
||||
return "off";
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value,
|
||||
char *error, size_t error_size) {
|
||||
if (error && error_size > 0) {
|
||||
error[0] = '\0';
|
||||
}
|
||||
if (!policy || !key || !value) {
|
||||
set_error(error, error_size, key ? key : "index resource limit", 0);
|
||||
return false;
|
||||
}
|
||||
|
||||
cbm_index_limit_u64_t *target = NULL;
|
||||
uint64_t maximum = 0;
|
||||
uint64_t multiplier = 1;
|
||||
if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) {
|
||||
target = &policy->max_files;
|
||||
maximum = CBM_INDEX_MAX_FILES_VALUE;
|
||||
} else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) {
|
||||
target = &policy->max_source_bytes;
|
||||
maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE;
|
||||
multiplier = CBM_INDEX_MIB_BYTES;
|
||||
} else {
|
||||
set_error(error, error_size, key, 0);
|
||||
return false;
|
||||
}
|
||||
|
||||
cbm_index_limit_u64_t candidate = {0};
|
||||
if (strcmp(value, "off") != 0) {
|
||||
uint64_t parsed = 0;
|
||||
if (!parse_bounded_uint64(value, maximum, &parsed)) {
|
||||
set_error(error, error_size, key, maximum);
|
||||
return false;
|
||||
}
|
||||
candidate.enabled = true;
|
||||
candidate.value = parsed * multiplier;
|
||||
}
|
||||
*target = candidate;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out,
|
||||
size_t out_size) {
|
||||
if (!policy || !key || !out || out_size == 0) {
|
||||
return false;
|
||||
}
|
||||
const cbm_index_limit_u64_t *limit = NULL;
|
||||
uint64_t divisor = 1;
|
||||
if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) {
|
||||
limit = &policy->max_files;
|
||||
} else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) {
|
||||
limit = &policy->max_source_bytes;
|
||||
divisor = CBM_INDEX_MIB_BYTES;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
int length = limit->enabled
|
||||
? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor))
|
||||
: snprintf(out, out_size, "off");
|
||||
return length >= 0 && (size_t)length < out_size;
|
||||
}
|
||||
|
||||
const char *cbm_index_resource_name(cbm_index_resource_t resource) {
|
||||
switch (resource) {
|
||||
case CBM_INDEX_RESOURCE_FILES:
|
||||
return "files";
|
||||
case CBM_INDEX_RESOURCE_SOURCE_BYTES:
|
||||
return "source_bytes";
|
||||
case CBM_INDEX_RESOURCE_NONE:
|
||||
default:
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
const char *cbm_index_resource_unit(cbm_index_resource_t resource) {
|
||||
return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes";
|
||||
}
|
||||
|
||||
const char *cbm_index_resource_config_key(cbm_index_resource_t resource) {
|
||||
switch (resource) {
|
||||
case CBM_INDEX_RESOURCE_FILES:
|
||||
return CBM_INDEX_CONFIG_MAX_FILES;
|
||||
case CBM_INDEX_RESOURCE_SOURCE_BYTES:
|
||||
return CBM_INDEX_CONFIG_MAX_SOURCE_MB;
|
||||
case CBM_INDEX_RESOURCE_NONE:
|
||||
default:
|
||||
return "index_resource_limit";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
#ifndef CBM_INDEX_POLICY_H
|
||||
#define CBM_INDEX_POLICY_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define CBM_INDEX_CONFIG_MAX_FILES "index_max_files"
|
||||
#define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb"
|
||||
|
||||
#define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000)
|
||||
#define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576)
|
||||
#define CBM_INDEX_MIB_BYTES UINT64_C(1048576)
|
||||
|
||||
typedef struct {
|
||||
bool enabled;
|
||||
uint64_t value;
|
||||
} cbm_index_limit_u64_t;
|
||||
|
||||
typedef struct {
|
||||
cbm_index_limit_u64_t max_files;
|
||||
cbm_index_limit_u64_t max_source_bytes;
|
||||
} cbm_index_resource_policy_t;
|
||||
|
||||
typedef enum {
|
||||
CBM_INDEX_RESOURCE_NONE = 0,
|
||||
CBM_INDEX_RESOURCE_FILES,
|
||||
CBM_INDEX_RESOURCE_SOURCE_BYTES,
|
||||
} cbm_index_resource_t;
|
||||
|
||||
typedef struct {
|
||||
cbm_index_resource_t resource;
|
||||
uint64_t observed;
|
||||
uint64_t limit;
|
||||
} cbm_index_resource_violation_t;
|
||||
|
||||
void cbm_index_policy_init(cbm_index_resource_policy_t *policy);
|
||||
bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy);
|
||||
|
||||
size_t cbm_index_policy_key_count(void);
|
||||
const char *cbm_index_policy_key_at(size_t index);
|
||||
const char *cbm_index_policy_default_value(const char *key);
|
||||
|
||||
bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value,
|
||||
char *error, size_t error_size);
|
||||
bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out,
|
||||
size_t out_size);
|
||||
|
||||
const char *cbm_index_resource_name(cbm_index_resource_t resource);
|
||||
const char *cbm_index_resource_unit(cbm_index_resource_t resource);
|
||||
const char *cbm_index_resource_config_key(cbm_index_resource_t resource);
|
||||
|
||||
#endif
|
||||
+128
-7
@@ -10600,6 +10600,63 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition(
|
||||
return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE;
|
||||
}
|
||||
|
||||
static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy,
|
||||
char *error, size_t error_size) {
|
||||
yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL;
|
||||
yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL;
|
||||
yyjson_val *encoded =
|
||||
root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL;
|
||||
if (!encoded || !yyjson_is_obj(encoded) ||
|
||||
yyjson_obj_size(encoded) != cbm_index_policy_key_count()) {
|
||||
yyjson_doc_free(doc);
|
||||
(void)snprintf(error, error_size, "missing or incomplete trusted worker policy");
|
||||
return false;
|
||||
}
|
||||
|
||||
cbm_index_policy_init(policy);
|
||||
bool valid = true;
|
||||
for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) {
|
||||
const char *key = cbm_index_policy_key_at(index);
|
||||
yyjson_val *value = yyjson_obj_get(encoded, key);
|
||||
valid = value && yyjson_is_str(value) &&
|
||||
cbm_index_policy_set(policy, key, yyjson_get_str(value), error, error_size);
|
||||
}
|
||||
yyjson_doc_free(doc);
|
||||
if (!valid && error && error_size > 0 && error[0] == '\0') {
|
||||
(void)snprintf(error, error_size, "invalid trusted worker policy");
|
||||
}
|
||||
return valid;
|
||||
}
|
||||
|
||||
static bool load_index_policy(cbm_mcp_server_t *srv, const char *args,
|
||||
cbm_index_resource_policy_t *policy, char *error, size_t error_size) {
|
||||
if (cbm_index_worker_active()) {
|
||||
return index_policy_from_worker_args(args, policy, error, error_size);
|
||||
}
|
||||
cbm_config_t *owned_config = NULL;
|
||||
cbm_config_t *config = srv ? srv->config : NULL;
|
||||
if (!config) {
|
||||
owned_config = cbm_config_open(cbm_resolve_cache_dir());
|
||||
config = owned_config;
|
||||
}
|
||||
bool loaded = cbm_config_load_index_policy(config, policy, error, error_size);
|
||||
cbm_config_close(owned_config);
|
||||
return loaded;
|
||||
}
|
||||
|
||||
bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root,
|
||||
const cbm_index_resource_policy_t *policy) {
|
||||
yyjson_mut_val *encoded = yyjson_mut_obj(doc);
|
||||
bool valid = encoded != NULL;
|
||||
for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) {
|
||||
const char *key = cbm_index_policy_key_at(index);
|
||||
char value[CBM_SZ_64];
|
||||
valid = cbm_index_policy_format(policy, key, value, sizeof(value)) &&
|
||||
yyjson_mut_obj_add_strcpy(doc, encoded, key, value);
|
||||
}
|
||||
return valid && yyjson_mut_obj_add_val(doc, root, "_cbm_index_policy", encoded);
|
||||
}
|
||||
|
||||
/* Run index_repository in a supervised worker subprocess with skip-and-continue
|
||||
* (Stage 3c). Returns the response string (caller frees):
|
||||
* - the worker's own response on a clean first run (the common path);
|
||||
@@ -10865,10 +10922,20 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p
|
||||
if (!root_path || !root_path[0]) {
|
||||
return NULL;
|
||||
}
|
||||
cbm_index_resource_policy_t policy;
|
||||
char policy_error[CBM_SZ_256] = {0};
|
||||
if (!load_index_policy(srv, NULL, &policy, policy_error, sizeof(policy_error))) {
|
||||
cbm_log_error("index.policy", "error", policy_error);
|
||||
return NULL;
|
||||
}
|
||||
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
|
||||
yyjson_mut_val *root = yyjson_mut_obj(doc);
|
||||
yyjson_mut_doc_set_root(doc, root);
|
||||
yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path);
|
||||
if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) ||
|
||||
!cbm_mcp_index_policy_add_to_args(doc, root, &policy)) {
|
||||
yyjson_mut_doc_free(doc);
|
||||
return NULL;
|
||||
}
|
||||
char *args = yy_doc_to_str(doc);
|
||||
yyjson_mut_doc_free(doc);
|
||||
if (!args) {
|
||||
@@ -10917,8 +10984,9 @@ static bool resolve_session_repo_path(cbm_mcp_server_t *srv, char **repo_path) {
|
||||
|
||||
/* Preserve every index option while replacing all caller-supplied repo_path
|
||||
* keys with the one canonical path that was actually authorized. */
|
||||
static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path) {
|
||||
if (!args || !canonical_repo_path) {
|
||||
static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path,
|
||||
const cbm_index_resource_policy_t *policy) {
|
||||
if (!args || !canonical_repo_path || !policy) {
|
||||
return NULL;
|
||||
}
|
||||
yyjson_doc *source = yyjson_read(args, strlen(args), 0);
|
||||
@@ -10935,8 +11003,14 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r
|
||||
yyjson_mut_doc_free(copy);
|
||||
return NULL;
|
||||
}
|
||||
(void)yyjson_mut_obj_remove_key(copy_root, "repo_path");
|
||||
if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path)) {
|
||||
while (yyjson_mut_obj_get(copy_root, "repo_path")) {
|
||||
(void)yyjson_mut_obj_remove_key(copy_root, "repo_path");
|
||||
}
|
||||
while (yyjson_mut_obj_get(copy_root, "_cbm_index_policy")) {
|
||||
(void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_policy");
|
||||
}
|
||||
if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) ||
|
||||
!cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) {
|
||||
yyjson_mut_doc_free(copy);
|
||||
return NULL;
|
||||
}
|
||||
@@ -10975,6 +11049,19 @@ static char *resolved_repo_path_from_project_arg(const char *args) {
|
||||
return root_path;
|
||||
}
|
||||
|
||||
static bool project_db_is_servable(const char *project, const char *db_path) {
|
||||
cbm_store_t *store = db_path && db_path[0] ? cbm_store_open_path_query(db_path) : NULL;
|
||||
if (!store) {
|
||||
return false;
|
||||
}
|
||||
cbm_project_t stored_project = {0};
|
||||
bool servable = cbm_store_get_project(store, project, &stored_project) == CBM_STORE_OK &&
|
||||
stored_project.root_path && stored_project.root_path[0];
|
||||
cbm_project_free_fields(&stored_project);
|
||||
cbm_store_close(store);
|
||||
return servable;
|
||||
}
|
||||
|
||||
static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
|
||||
char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");
|
||||
char *mode_str = cbm_mcp_get_string_arg(args, "mode");
|
||||
@@ -11029,10 +11116,19 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
|
||||
return result;
|
||||
}
|
||||
|
||||
cbm_index_resource_policy_t resource_policy;
|
||||
char policy_error[CBM_SZ_256] = {0};
|
||||
if (!load_index_policy(srv, args, &resource_policy, policy_error, sizeof(policy_error))) {
|
||||
free(mode_str);
|
||||
free(name_override);
|
||||
free(repo_path);
|
||||
return cbm_mcp_text_result(policy_error, true);
|
||||
}
|
||||
|
||||
/* A daemon session delegates the one physical write to its shared job
|
||||
* registry only after path canonicalization and workspace authorization. */
|
||||
if (srv->index_executor) {
|
||||
char *worker_args = index_args_with_repo_path(args, repo_path);
|
||||
char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy);
|
||||
char *coordinated =
|
||||
worker_args ? srv->index_executor(srv->index_executor_context, repo_path, worker_args)
|
||||
: NULL;
|
||||
@@ -11064,7 +11160,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
|
||||
* installs the same guard before running the in-process pipeline. A marked
|
||||
* host fails closed if preparation or worker startup cannot complete. */
|
||||
if (cbm_index_supervisor_should_wrap()) {
|
||||
char *worker_args = index_args_with_repo_path(args, repo_path);
|
||||
char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy);
|
||||
if (!worker_args) {
|
||||
free(mutation_project);
|
||||
free(repo_path);
|
||||
@@ -11136,11 +11232,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
|
||||
}
|
||||
free(name_override);
|
||||
cbm_pipeline_set_persistence(p, persistence);
|
||||
cbm_pipeline_set_resource_policy(p, &resource_policy);
|
||||
|
||||
char *project_name = heap_strdup(cbm_pipeline_project_name(p));
|
||||
|
||||
/* Bootstrap from artifact if no local DB exists */
|
||||
try_artifact_bootstrap(project_name, repo_path);
|
||||
char serving_db_path[CBM_SZ_1K];
|
||||
project_db_path(project_name, serving_db_path, sizeof(serving_db_path));
|
||||
bool serving_index_was_servable = project_db_is_servable(project_name, serving_db_path);
|
||||
|
||||
/* Close cached store — pipeline will delete + recreate the .db file */
|
||||
if (srv->owns_store && srv->store) {
|
||||
@@ -11172,6 +11272,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
|
||||
cbm_file_error_t *file_errors = NULL;
|
||||
int file_error_count = 0;
|
||||
cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count);
|
||||
cbm_index_resource_violation_t resource_violation = {0};
|
||||
cbm_pipeline_get_resource_violation(p, &resource_violation);
|
||||
|
||||
cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */
|
||||
|
||||
@@ -11269,6 +11371,25 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
|
||||
"The validated staging database could not be published. Check "
|
||||
"free disk space and permissions on the cache directory; the "
|
||||
"previous index may have been rolled back.");
|
||||
} else if (rc == CBM_PIPELINE_RESOURCE_LIMIT &&
|
||||
resource_violation.resource != CBM_INDEX_RESOURCE_NONE) {
|
||||
const char *config_key = cbm_index_resource_config_key(resource_violation.resource);
|
||||
char message[CBM_SZ_256];
|
||||
(void)snprintf(message, sizeof(message), "Index discovery exceeded %s", config_key);
|
||||
yyjson_mut_obj_add_str(doc, root, "status", "error");
|
||||
yyjson_mut_obj_add_str(doc, root, "code", "resource_limit_exceeded");
|
||||
yyjson_mut_obj_add_str(doc, root, "stage", "discovery");
|
||||
yyjson_mut_obj_add_str(doc, root, "resource",
|
||||
cbm_index_resource_name(resource_violation.resource));
|
||||
yyjson_mut_obj_add_uint(doc, root, "observed", resource_violation.observed);
|
||||
yyjson_mut_obj_add_uint(doc, root, "limit", resource_violation.limit);
|
||||
yyjson_mut_obj_add_str(doc, root, "unit",
|
||||
cbm_index_resource_unit(resource_violation.resource));
|
||||
yyjson_mut_obj_add_bool(doc, root, "retryable", true);
|
||||
yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved",
|
||||
serving_index_was_servable &&
|
||||
project_db_is_servable(project_name, serving_db_path));
|
||||
yyjson_mut_obj_add_strcpy(doc, root, "message", message);
|
||||
} else {
|
||||
yyjson_mut_obj_add_str(doc, root, "status", "error");
|
||||
yyjson_mut_obj_add_str(doc, root, "hint",
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
#ifndef CBM_MCP_INTERNAL_H
|
||||
#define CBM_MCP_INTERNAL_H
|
||||
|
||||
#include "foundation/index_policy.h"
|
||||
#include "mcp/mcp.h"
|
||||
#include "pipeline/pipeline.h" /* cbm_changed_hunk_t */
|
||||
#include "store/store.h" /* cbm_node_t */
|
||||
|
||||
#include <yyjson/yyjson.h>
|
||||
|
||||
/* White-box fault injection for deterministic cross-platform quarantine
|
||||
* safety tests. This header is internal and is not part of the MCP API. */
|
||||
typedef bool (*cbm_mcp_quarantine_test_hook_fn)(void *context, const char *step);
|
||||
@@ -36,6 +39,11 @@ bool cbm_mcp_server_release_pristine_memory_store(cbm_mcp_server_t *srv);
|
||||
* On success replaces and frees *response_io; on failure it is unchanged. */
|
||||
bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *notice);
|
||||
|
||||
/* Encode the complete trusted policy on an internal worker request. Callers
|
||||
* must remove any untrusted field with the same name before invoking this. */
|
||||
bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root,
|
||||
const cbm_index_resource_policy_t *policy);
|
||||
|
||||
enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 };
|
||||
|
||||
/* Count indexable files with the pipeline's native full-mode discovery policy,
|
||||
|
||||
+39
-8
@@ -190,6 +190,8 @@ struct cbm_pipeline {
|
||||
atomic_int cancelled_storage;
|
||||
atomic_int *cancelled;
|
||||
bool persistence; /* write .codebase-memory/graph.db.zst after indexing */
|
||||
cbm_index_resource_policy_t resource_policy;
|
||||
cbm_index_resource_violation_t resource_violation;
|
||||
|
||||
/* Indexing state (set during run) */
|
||||
cbm_gbuf_t *gbuf;
|
||||
@@ -373,6 +375,20 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) {
|
||||
}
|
||||
}
|
||||
|
||||
void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p,
|
||||
const cbm_index_resource_policy_t *policy) {
|
||||
if (p && policy) {
|
||||
p->resource_policy = *policy;
|
||||
}
|
||||
}
|
||||
|
||||
void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p,
|
||||
cbm_index_resource_violation_t *violation) {
|
||||
if (violation) {
|
||||
*violation = p ? p->resource_violation : (cbm_index_resource_violation_t){0};
|
||||
}
|
||||
}
|
||||
|
||||
bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) {
|
||||
if (!p || !name || !name[0]) {
|
||||
return false;
|
||||
@@ -482,6 +498,14 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) {
|
||||
return p ? p->repo_path : NULL;
|
||||
}
|
||||
|
||||
const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) {
|
||||
return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL;
|
||||
}
|
||||
|
||||
cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) {
|
||||
return p ? &p->resource_violation : NULL;
|
||||
}
|
||||
|
||||
atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) {
|
||||
return p ? p->cancelled : NULL;
|
||||
}
|
||||
@@ -2437,15 +2461,17 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas
|
||||
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
|
||||
cbm_pipeline_persist_test_run_before_final_manifest();
|
||||
#endif
|
||||
if (cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode,
|
||||
&manifest, &manifest_count) != 0) {
|
||||
int manifest_rc =
|
||||
cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &manifest, &manifest_count);
|
||||
if (manifest_rc != 0) {
|
||||
cbm_log_error("pipeline.err", "phase", "semantic_manifest");
|
||||
/* db_path and db_dir are this function's strdups; the success tail and
|
||||
* the publish-failure return release them, and these two aborts must
|
||||
* too -- LSan caught exactly these paths leaking both strings. */
|
||||
free(db_dir);
|
||||
free(db_path);
|
||||
return CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT
|
||||
: CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
}
|
||||
if (!cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest,
|
||||
manifest_count)) {
|
||||
@@ -2702,10 +2728,14 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
|
||||
|
||||
/* Phase 1: Discover files */
|
||||
CBM_PROF_START(t_discover);
|
||||
p->resource_violation = (cbm_index_resource_violation_t){0};
|
||||
cbm_discover_opts_t opts = {
|
||||
.mode = p->requested_mode,
|
||||
.ignore_file = NULL,
|
||||
.max_file_size = 0,
|
||||
.resource_policy =
|
||||
cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL,
|
||||
.resource_violation = &p->resource_violation,
|
||||
};
|
||||
cbm_file_info_t *files = NULL;
|
||||
int file_count = 0;
|
||||
@@ -2730,7 +2760,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
|
||||
cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms",
|
||||
itoa_buf((int)elapsed_ms(t0)));
|
||||
if (rc != 0 || check_cancel(p)) {
|
||||
rc = CBM_NOT_FOUND;
|
||||
rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
@@ -2738,14 +2768,15 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
|
||||
* bytes drive exact no-op comparison and are checked against a fresh
|
||||
* rediscovery immediately before any replacement is published. */
|
||||
rc = mode_promoted
|
||||
? cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode,
|
||||
&baseline_manifest, &baseline_count)
|
||||
? cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &baseline_manifest,
|
||||
&baseline_count)
|
||||
: cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files,
|
||||
file_count, p->excluded_dirs, p->excluded_count,
|
||||
&p->git_ctx, p->userconfig, &baseline_manifest,
|
||||
&baseline_count);
|
||||
if (rc != 0) {
|
||||
rc = CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT
|
||||
: CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
@@ -2789,7 +2820,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
|
||||
cbm_log_info("pipeline.rediscover", "requested_mode", pipeline_mode_name(p->requested_mode),
|
||||
"effective_mode", pipeline_mode_name(p->mode), "files", itoa_buf(file_count));
|
||||
if (rc != 0 || check_cancel(p)) {
|
||||
rc = CBM_NOT_FOUND;
|
||||
rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +53,13 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, cbm
|
||||
* When enabled, the pipeline writes a compressed artifact after indexing. */
|
||||
void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled);
|
||||
|
||||
/* Apply a validated discovery resource policy. The value is copied. */
|
||||
void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy);
|
||||
|
||||
/* Copy the exact discovery violation from the most recent run. */
|
||||
void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p,
|
||||
cbm_index_resource_violation_t *violation);
|
||||
|
||||
/* Free a pipeline and all its internal state. NULL-safe. */
|
||||
void cbm_pipeline_free(cbm_pipeline_t *p);
|
||||
|
||||
@@ -71,6 +78,8 @@ void cbm_pipeline_free(cbm_pipeline_t *p);
|
||||
* written, the previous DB is intact (#1997 #832). Distinct from the cancel
|
||||
* sentinel so callers can name the cause instead of "pipeline failed". */
|
||||
#define CBM_PIPELINE_ABORT_OVER_BUDGET (-5)
|
||||
/* Opt-in discovery/resource policy breach: fail the attempt, keep serving DB. */
|
||||
#define CBM_PIPELINE_RESOURCE_LIMIT (-6)
|
||||
int cbm_pipeline_run(cbm_pipeline_t *p);
|
||||
|
||||
/* Request cancellation of a running pipeline (thread-safe). */
|
||||
|
||||
@@ -601,17 +601,20 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left
|
||||
return equal;
|
||||
}
|
||||
|
||||
int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode,
|
||||
int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project,
|
||||
cbm_file_hash_t **out, int *out_count) {
|
||||
if (!project || !repo_path || !out || !out_count) {
|
||||
const char *repo_path = cbm_pipeline_repo_path(p);
|
||||
if (!p || !project || !repo_path || !out || !out_count) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
*out = NULL;
|
||||
*out_count = 0;
|
||||
cbm_discover_opts_t opts = {
|
||||
.mode = (cbm_index_mode_t)mode,
|
||||
.mode = (cbm_index_mode_t)cbm_pipeline_get_mode(p),
|
||||
.ignore_file = NULL,
|
||||
.max_file_size = 0,
|
||||
.resource_policy = cbm_pipeline_resource_policy(p),
|
||||
.resource_violation = cbm_pipeline_resource_violation(p),
|
||||
};
|
||||
cbm_file_info_t *fresh_files = NULL;
|
||||
int fresh_file_count = 0;
|
||||
@@ -2264,13 +2267,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
|
||||
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
|
||||
cbm_pipeline_persist_test_run_before_final_manifest();
|
||||
#endif
|
||||
if (cbm_pipeline_build_fresh_semantic_manifest(project, cbm_pipeline_repo_path(p),
|
||||
cbm_pipeline_get_mode(p), &manifest,
|
||||
&manifest_count) != 0 ||
|
||||
!cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest,
|
||||
manifest_count)) {
|
||||
int manifest_rc =
|
||||
cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count);
|
||||
if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal(
|
||||
baseline_manifest, baseline_count, manifest, manifest_count)) {
|
||||
cbm_log_warn("delta.abort", "reason", "semantic_inputs_changed");
|
||||
result = CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
result = manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT
|
||||
: CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
goto out;
|
||||
}
|
||||
|
||||
@@ -2856,8 +2859,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
|
||||
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
|
||||
cbm_pipeline_persist_test_run_before_final_manifest();
|
||||
#endif
|
||||
int manifest_rc = cbm_pipeline_build_fresh_semantic_manifest(
|
||||
project, cbm_pipeline_repo_path(p), cbm_pipeline_get_mode(p), &manifest, &manifest_count);
|
||||
int manifest_rc =
|
||||
cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count);
|
||||
if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal(
|
||||
baseline_manifest, baseline_count, manifest, manifest_count)) {
|
||||
cbm_log_warn("incremental.abort", "reason", "semantic_inputs_changed");
|
||||
@@ -2867,7 +2870,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
|
||||
free_mode_skipped(mode_skipped, mode_skipped_count);
|
||||
free(saved_adr);
|
||||
cbm_gbuf_free(existing);
|
||||
return CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT
|
||||
: CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
}
|
||||
|
||||
/* Step 7: atomically publish the complete staged generation. */
|
||||
|
||||
@@ -781,7 +781,7 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left
|
||||
const cbm_file_hash_t *right, int right_count);
|
||||
/* Re-run discovery and hash its exact semantic inputs. Used at the publication
|
||||
* boundary so late additions/deletions cannot escape a frozen file list. */
|
||||
int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode,
|
||||
int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project,
|
||||
cbm_file_hash_t **out, int *out_count);
|
||||
|
||||
/* Compatibility contract persisted in coverage metadata. Increment when a
|
||||
@@ -874,6 +874,8 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row
|
||||
|
||||
/* Pipeline accessors for incremental use */
|
||||
const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p);
|
||||
const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p);
|
||||
cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p);
|
||||
atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p);
|
||||
/* Record committed graph size (#334 gate axis) from the incremental path,
|
||||
* which cannot see the opaque cbm_pipeline struct. Call before the dump. */
|
||||
|
||||
@@ -1348,6 +1348,7 @@ TEST(daemon_application_prune_clears_logical_watch_for_reregistration) {
|
||||
}
|
||||
|
||||
enum { APP_FAKE_MAX_ATTEMPTS = 16 };
|
||||
enum { APP_FAKE_ARGS_CAP = 2048 };
|
||||
|
||||
typedef struct {
|
||||
atomic_int starts;
|
||||
@@ -1368,6 +1369,11 @@ typedef struct {
|
||||
char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP];
|
||||
char quarantine_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP];
|
||||
char quarantine_seen[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP];
|
||||
char args_json[APP_FAKE_MAX_ATTEMPTS][APP_FAKE_ARGS_CAP];
|
||||
/* `starts` is incremented before the slot is filled, so waiting on it says
|
||||
* a worker began, not that its arguments are readable. Each slot publishes
|
||||
* itself instead, which is the edge a reader on another thread needs. */
|
||||
atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS];
|
||||
size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS];
|
||||
} app_fake_worker_context_t;
|
||||
|
||||
@@ -1432,6 +1438,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem
|
||||
atomic_init(&worker->cancelled, false);
|
||||
worker->result.exit_code = -1;
|
||||
if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) {
|
||||
(void)snprintf(context->args_json[worker->attempt], APP_FAKE_ARGS_CAP, "%s",
|
||||
args_json ? args_json : "");
|
||||
atomic_store(&context->args_captured[worker->attempt], true);
|
||||
context->memory_budgets[worker->attempt] = memory_budget_bytes;
|
||||
if (marker_file) {
|
||||
(void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s",
|
||||
@@ -1456,6 +1465,33 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected);
|
||||
|
||||
static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt,
|
||||
const char *max_files, const char *max_source_mb) {
|
||||
if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) {
|
||||
return false;
|
||||
}
|
||||
if (!app_wait_for_atomic_bool(&context->args_captured[attempt], true)) {
|
||||
return false;
|
||||
}
|
||||
const char *args = context->args_json[attempt];
|
||||
yyjson_doc *document = yyjson_read(args, strlen(args), 0);
|
||||
yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL;
|
||||
yyjson_val *policy =
|
||||
root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL;
|
||||
yyjson_val *files =
|
||||
policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_FILES) : NULL;
|
||||
yyjson_val *bytes = policy && yyjson_is_obj(policy)
|
||||
? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB)
|
||||
: NULL;
|
||||
bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) &&
|
||||
strcmp(yyjson_get_str(files), max_files) == 0 &&
|
||||
strcmp(yyjson_get_str(bytes), max_source_mb) == 0;
|
||||
yyjson_doc_free(document);
|
||||
return equal;
|
||||
}
|
||||
|
||||
static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque,
|
||||
cbm_daemon_application_worker_t handle,
|
||||
const cbm_index_worker_result_t **result_out) {
|
||||
@@ -2009,7 +2045,9 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {
|
||||
cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL;
|
||||
bool config_ready = stored_config &&
|
||||
cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 &&
|
||||
cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0;
|
||||
cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 &&
|
||||
cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 &&
|
||||
cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0;
|
||||
char canonical_root[APP_TEST_PATH_CAP] = {0};
|
||||
bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root));
|
||||
char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL;
|
||||
@@ -2054,6 +2092,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {
|
||||
bool first_owned = first_initialized && project &&
|
||||
app_wait_for_subscribers(application, project, 1) &&
|
||||
app_wait_for_atomic_int(&fake.starts, 1);
|
||||
bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4");
|
||||
bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root,
|
||||
CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL);
|
||||
bool coalesced = first_owned && second_initialized && project &&
|
||||
@@ -2115,6 +2154,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {
|
||||
ASSERT_TRUE(restricted_started_nothing);
|
||||
ASSERT_TRUE(first_initialized);
|
||||
ASSERT_TRUE(first_owned);
|
||||
ASSERT_TRUE(auto_policy_propagated);
|
||||
ASSERT_TRUE(second_initialized);
|
||||
ASSERT_TRUE(coalesced);
|
||||
ASSERT_TRUE(restricted_disconnect_kept_job);
|
||||
@@ -2389,6 +2429,46 @@ TEST(daemon_application_sensitive_root_blocks_watch_but_preserves_controls) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(daemon_application_programmatic_index_injects_resource_policy) {
|
||||
char *root = th_mktempdir("cbm_app_policy_root");
|
||||
char *cache = th_mktempdir("cbm_app_policy_cache");
|
||||
ASSERT_NOT_NULL(root);
|
||||
ASSERT_NOT_NULL(cache);
|
||||
cbm_config_t *stored_config = cbm_config_open(cache);
|
||||
bool configured = stored_config &&
|
||||
cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 &&
|
||||
cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0;
|
||||
|
||||
app_fake_worker_context_t fake;
|
||||
app_fake_worker_context_init(&fake);
|
||||
atomic_store(&fake.allow_completion, true);
|
||||
cbm_daemon_application_worker_ops_t worker_ops = {
|
||||
.context = &fake,
|
||||
.start = app_fake_worker_start,
|
||||
.poll = app_fake_worker_poll,
|
||||
.cancel = app_fake_worker_cancel,
|
||||
.log_path = app_fake_worker_log_path,
|
||||
.destroy = app_fake_worker_destroy,
|
||||
};
|
||||
cbm_daemon_application_config_t config = {
|
||||
.config = stored_config,
|
||||
.worker_ops = &worker_ops,
|
||||
};
|
||||
cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL;
|
||||
int index_rc =
|
||||
application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1;
|
||||
bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6");
|
||||
bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS);
|
||||
|
||||
cbm_daemon_application_free(application);
|
||||
cbm_config_close(stored_config);
|
||||
th_cleanup(root);
|
||||
th_cleanup(cache);
|
||||
ASSERT_TRUE(configured);
|
||||
ASSERT_TRUE(policy_propagated);
|
||||
ASSERT_TRUE(stopped);
|
||||
PASS();
|
||||
}
|
||||
TEST(daemon_application_auto_index_honors_tracked_file_limit) {
|
||||
app_env_backup_t cache_environment;
|
||||
bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR");
|
||||
@@ -2464,6 +2544,9 @@ TEST(daemon_application_auto_index_honors_tracked_file_limit) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Native discovery never interprets a repository path as command text. Valid
|
||||
* non-Git paths containing shell metacharacters remain countable literally. */
|
||||
|
||||
/* Native discovery never interprets a repository path as command text. Valid
|
||||
* non-Git paths containing shell metacharacters remain countable literally. */
|
||||
TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path) {
|
||||
@@ -5570,6 +5653,7 @@ SUITE(daemon_application) {
|
||||
RUN_TEST(daemon_application_sensitive_root_blocks_auto_index_but_preserves_controls);
|
||||
RUN_TEST(daemon_application_sensitive_root_blocks_watch_but_preserves_controls);
|
||||
RUN_TEST(daemon_application_index_args_compare_repo_path_separator_equivalently);
|
||||
RUN_TEST(daemon_application_programmatic_index_injects_resource_policy);
|
||||
RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit);
|
||||
RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path);
|
||||
RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots);
|
||||
|
||||
@@ -475,6 +475,163 @@ TEST(discover_bounded_count_matches_shebang_discovery) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(discover_resource_policy_off_matches_legacy_discovery) {
|
||||
char *base = th_mktempdir("cbm_disc_policy_off");
|
||||
ASSERT(base != NULL);
|
||||
th_write_file(TH_PATH(base, "src/first.c"), "int first;\n");
|
||||
th_write_file(TH_PATH(base, "src/second.py"), "second = 2\n");
|
||||
th_write_file(TH_PATH(base, "src/ignored.png"), "not source\n");
|
||||
|
||||
cbm_file_info_t *legacy_files = NULL;
|
||||
int legacy_count = 0;
|
||||
cbm_discover_opts_t legacy_opts = {.mode = CBM_MODE_FULL};
|
||||
ASSERT_EQ(cbm_discover(base, &legacy_opts, &legacy_files, &legacy_count), CBM_DISCOVER_OK);
|
||||
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
cbm_index_resource_violation_t violation = {0};
|
||||
cbm_file_info_t *policy_files = NULL;
|
||||
int policy_count = 0;
|
||||
cbm_discover_opts_t policy_opts = {
|
||||
.mode = CBM_MODE_FULL,
|
||||
.resource_policy = &policy,
|
||||
.resource_violation = &violation,
|
||||
};
|
||||
cbm_discover_status_t status = cbm_discover(base, &policy_opts, &policy_files, &policy_count);
|
||||
|
||||
ASSERT_EQ(status, CBM_DISCOVER_OK);
|
||||
ASSERT_EQ(policy_count, legacy_count);
|
||||
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE);
|
||||
for (int index = 0; index < legacy_count; index++) {
|
||||
ASSERT_STR_EQ(policy_files[index].rel_path, legacy_files[index].rel_path);
|
||||
ASSERT_EQ(policy_files[index].size, legacy_files[index].size);
|
||||
}
|
||||
|
||||
cbm_discover_free(legacy_files, legacy_count);
|
||||
cbm_discover_free(policy_files, policy_count);
|
||||
th_cleanup(base);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources) {
|
||||
char *base = th_mktempdir("cbm_disc_policy_files");
|
||||
ASSERT(base != NULL);
|
||||
th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n");
|
||||
th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n");
|
||||
th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n");
|
||||
th_write_file(TH_PATH(base, "unsupported.png"), "not source\n");
|
||||
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1};
|
||||
cbm_index_resource_violation_t violation = {0};
|
||||
cbm_discover_opts_t opts = {
|
||||
.mode = CBM_MODE_FULL,
|
||||
.resource_policy = &policy,
|
||||
.resource_violation = &violation,
|
||||
};
|
||||
cbm_file_info_t *files = NULL;
|
||||
int count = 0;
|
||||
ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK);
|
||||
ASSERT_EQ(count, 1);
|
||||
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE);
|
||||
cbm_discover_free(files, count);
|
||||
|
||||
th_write_file(TH_PATH(base, "second.py"), "second = 2\n");
|
||||
files = NULL;
|
||||
count = 99;
|
||||
ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED);
|
||||
ASSERT(files == NULL);
|
||||
ASSERT_EQ(count, 0);
|
||||
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES);
|
||||
ASSERT_EQ(violation.observed, 2);
|
||||
ASSERT_EQ(violation.limit, 1);
|
||||
|
||||
th_cleanup(base);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte) {
|
||||
char *base = th_mktempdir("cbm_disc_policy_bytes");
|
||||
ASSERT(base != NULL);
|
||||
th_write_file(TH_PATH(base, "exact.c"), "1234567");
|
||||
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
policy.max_source_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 7};
|
||||
cbm_index_resource_violation_t violation = {0};
|
||||
cbm_discover_opts_t opts = {
|
||||
.mode = CBM_MODE_FULL,
|
||||
.resource_policy = &policy,
|
||||
.resource_violation = &violation,
|
||||
};
|
||||
cbm_file_info_t *files = NULL;
|
||||
int count = 0;
|
||||
ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK);
|
||||
ASSERT_EQ(count, 1);
|
||||
cbm_discover_free(files, count);
|
||||
|
||||
th_write_file(TH_PATH(base, "plus.py"), "x");
|
||||
files = NULL;
|
||||
count = 99;
|
||||
ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED);
|
||||
ASSERT(files == NULL);
|
||||
ASSERT_EQ(count, 0);
|
||||
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_SOURCE_BYTES);
|
||||
ASSERT_EQ(violation.observed, 8);
|
||||
ASSERT_EQ(violation.limit, 7);
|
||||
|
||||
th_cleanup(base);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(discover_resource_file_budget_excludes_existing_oversized_skip) {
|
||||
char *base = th_mktempdir("cbm_disc_policy_oversized");
|
||||
ASSERT(base != NULL);
|
||||
th_write_file(TH_PATH(base, "accepted.c"), "x");
|
||||
th_write_file(TH_PATH(base, "oversized.py"), "123");
|
||||
const char *saved_limit = getenv("CBM_MAX_FILE_BYTES");
|
||||
char *saved_limit_copy = saved_limit ? strdup(saved_limit) : NULL;
|
||||
cbm_setenv("CBM_MAX_FILE_BYTES", "2", 1);
|
||||
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1};
|
||||
cbm_index_resource_violation_t violation = {0};
|
||||
cbm_discover_opts_t opts = {
|
||||
.mode = CBM_MODE_FULL,
|
||||
.resource_policy = &policy,
|
||||
.resource_violation = &violation,
|
||||
};
|
||||
cbm_file_info_t *files = NULL;
|
||||
int count = 0;
|
||||
cbm_discover_status_t initial_status = cbm_discover(base, &opts, &files, &count);
|
||||
bool oversized_did_not_consume_budget = initial_status == CBM_DISCOVER_OK && count == 2 &&
|
||||
violation.resource == CBM_INDEX_RESOURCE_NONE;
|
||||
cbm_discover_free(files, count);
|
||||
|
||||
th_write_file(TH_PATH(base, "second.c"), "y");
|
||||
files = NULL;
|
||||
count = 0;
|
||||
cbm_discover_status_t exceeded_status = cbm_discover(base, &opts, &files, &count);
|
||||
bool accepted_sources_exceeded = exceeded_status == CBM_DISCOVER_LIMIT_EXCEEDED &&
|
||||
files == NULL && count == 0 &&
|
||||
violation.resource == CBM_INDEX_RESOURCE_FILES &&
|
||||
violation.observed == 2 && violation.limit == 1;
|
||||
|
||||
if (saved_limit_copy) {
|
||||
cbm_setenv("CBM_MAX_FILE_BYTES", saved_limit_copy, 1);
|
||||
} else {
|
||||
cbm_unsetenv("CBM_MAX_FILE_BYTES");
|
||||
}
|
||||
free(saved_limit_copy);
|
||||
th_cleanup(base);
|
||||
|
||||
ASSERT_TRUE(oversized_did_not_consume_budget);
|
||||
ASSERT_TRUE(accepted_sources_exceeded);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(discover_skips_git_dir) {
|
||||
char *base = th_mktempdir("cbm_disc_git");
|
||||
ASSERT(base != NULL);
|
||||
@@ -1931,6 +2088,10 @@ SUITE(discover) {
|
||||
RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact);
|
||||
RUN_TEST(discover_bounded_count_fails_closed_after_deadline);
|
||||
RUN_TEST(discover_bounded_count_matches_shebang_discovery);
|
||||
RUN_TEST(discover_resource_policy_off_matches_legacy_discovery);
|
||||
RUN_TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources);
|
||||
RUN_TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte);
|
||||
RUN_TEST(discover_resource_file_budget_excludes_existing_oversized_skip);
|
||||
RUN_TEST(discover_skips_git_dir);
|
||||
RUN_TEST(discover_with_gitignore);
|
||||
RUN_TEST(discover_with_global_xdg_ignore);
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
#include "test_framework.h"
|
||||
#include "test_helpers.h"
|
||||
|
||||
#include "cli/cli.h"
|
||||
#include "foundation/compat.h"
|
||||
#include "foundation/index_policy.h"
|
||||
#include "mcp/index_supervisor.h"
|
||||
#include "mcp/mcp.h"
|
||||
#include "store/store.h"
|
||||
|
||||
#include <sqlite3.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
TEST(index_policy_defaults_are_disabled) {
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
|
||||
ASSERT_FALSE(policy.max_files.enabled);
|
||||
ASSERT_FALSE(policy.max_source_bytes.enabled);
|
||||
ASSERT_FALSE(cbm_index_policy_enabled(&policy));
|
||||
ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off");
|
||||
ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off");
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_file_limit_accepts_off_and_exact_range) {
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
char error[256];
|
||||
|
||||
ASSERT_TRUE(
|
||||
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "1", error, sizeof(error)));
|
||||
ASSERT_TRUE(policy.max_files.enabled);
|
||||
ASSERT_EQ(policy.max_files.value, 1);
|
||||
ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "10000000", error,
|
||||
sizeof(error)));
|
||||
ASSERT_EQ(policy.max_files.value, 10000000);
|
||||
ASSERT_TRUE(
|
||||
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "off", error, sizeof(error)));
|
||||
ASSERT_FALSE(policy.max_files.enabled);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_source_limit_converts_mib_without_overflow) {
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
char error[256];
|
||||
|
||||
ASSERT_TRUE(
|
||||
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1", error, sizeof(error)));
|
||||
ASSERT_TRUE(policy.max_source_bytes.enabled);
|
||||
ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1024) * UINT64_C(1024));
|
||||
ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1048576", error,
|
||||
sizeof(error)));
|
||||
ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1048576) * UINT64_C(1024) * UINT64_C(1024));
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_invalid_value_is_rejected_atomically) {
|
||||
static const char *const invalid[] = {"", "0", "-1", "1MB",
|
||||
"1 ", "+1", "10000001", "18446744073709551616"};
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
char error[256];
|
||||
ASSERT_TRUE(
|
||||
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error)));
|
||||
|
||||
for (size_t index = 0; index < sizeof(invalid) / sizeof(invalid[0]); index++) {
|
||||
cbm_index_resource_policy_t before = policy;
|
||||
ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, invalid[index],
|
||||
error, sizeof(error)));
|
||||
ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0);
|
||||
ASSERT_TRUE(error[0] != '\0');
|
||||
}
|
||||
ASSERT_FALSE(cbm_index_policy_set(&policy, "index_unknown_limit", "1", error, sizeof(error)));
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_format_round_trips_public_values) {
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
char error[256];
|
||||
char value[64];
|
||||
|
||||
ASSERT_TRUE(
|
||||
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "321", error, sizeof(error)));
|
||||
ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_FILES, value, sizeof(value)));
|
||||
ASSERT_STR_EQ(value, "321");
|
||||
ASSERT_TRUE(
|
||||
cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "7", error, sizeof(error)));
|
||||
ASSERT_TRUE(
|
||||
cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value)));
|
||||
ASSERT_STR_EQ(value, "7");
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_violation_metadata_is_stable) {
|
||||
ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FILES), "files");
|
||||
ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes");
|
||||
ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files");
|
||||
ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes");
|
||||
ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES),
|
||||
CBM_INDEX_CONFIG_MAX_FILES);
|
||||
ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES),
|
||||
CBM_INDEX_CONFIG_MAX_SOURCE_MB);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) {
|
||||
char *cache = th_mktempdir("cbm_index_policy_config");
|
||||
ASSERT_NOT_NULL(cache);
|
||||
cbm_config_t *config = cbm_config_open(cache);
|
||||
ASSERT_NOT_NULL(config);
|
||||
cbm_index_resource_policy_t policy;
|
||||
char error[256];
|
||||
|
||||
ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error)));
|
||||
ASSERT_FALSE(cbm_index_policy_enabled(&policy));
|
||||
ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0);
|
||||
ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0);
|
||||
ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error)));
|
||||
ASSERT_EQ(policy.max_files.value, 9);
|
||||
ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES);
|
||||
|
||||
ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0);
|
||||
ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error)));
|
||||
ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_MAX_FILES) != NULL);
|
||||
|
||||
cbm_config_close(config);
|
||||
th_cleanup(cache);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_cli_lists_both_operator_keys) {
|
||||
bool files_found = false;
|
||||
bool bytes_found = false;
|
||||
for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) {
|
||||
const char *key = cbm_cli_config_key_at_for_testing(index);
|
||||
files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0);
|
||||
bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0);
|
||||
}
|
||||
ASSERT_TRUE(files_found);
|
||||
ASSERT_TRUE(bytes_found);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) {
|
||||
char *cache = th_mktempdir("cbm_index_policy_cli");
|
||||
ASSERT_NOT_NULL(cache);
|
||||
const char *saved_cache = getenv("CBM_CACHE_DIR");
|
||||
char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL;
|
||||
(void)cbm_setenv("CBM_CACHE_DIR", cache, 1);
|
||||
|
||||
char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"};
|
||||
char *set_invalid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "0"};
|
||||
char *reset[] = {"reset", CBM_INDEX_CONFIG_MAX_FILES};
|
||||
int valid_rc = cbm_cmd_config(3, set_valid);
|
||||
cbm_config_t *config = cbm_config_open(cache);
|
||||
const char *stored =
|
||||
config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing";
|
||||
bool valid_stored = strcmp(stored, "7") == 0;
|
||||
int invalid_rc = cbm_cmd_config(3, set_invalid);
|
||||
stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing";
|
||||
bool invalid_preserved = strcmp(stored, "7") == 0;
|
||||
int reset_rc = cbm_cmd_config(2, reset);
|
||||
stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "off") : "missing";
|
||||
bool reset_to_default = strcmp(stored, "off") == 0;
|
||||
cbm_config_close(config);
|
||||
|
||||
if (saved_cache_copy) {
|
||||
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1);
|
||||
} else {
|
||||
(void)cbm_unsetenv("CBM_CACHE_DIR");
|
||||
}
|
||||
free(saved_cache_copy);
|
||||
th_cleanup(cache);
|
||||
|
||||
ASSERT_EQ(valid_rc, 0);
|
||||
ASSERT_TRUE(valid_stored);
|
||||
ASSERT_TRUE(invalid_rc != 0);
|
||||
ASSERT_TRUE(invalid_preserved);
|
||||
ASSERT_EQ(reset_rc, 0);
|
||||
ASSERT_TRUE(reset_to_default);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Capture stderr across one cbm_cmd_config invocation, mirroring the stdout
|
||||
* idiom in test_cli.c: tmpfile+dup2+rewind is what works on the MinGW leg. */
|
||||
static int index_policy_config_stderr(int argc, char **argv, char *out, size_t cap) {
|
||||
out[0] = '\0';
|
||||
FILE *capture = tmpfile();
|
||||
int saved = capture ? dup(fileno(stderr)) : -1;
|
||||
if (!capture || saved < 0) {
|
||||
if (capture) {
|
||||
(void)fclose(capture);
|
||||
}
|
||||
if (saved >= 0) {
|
||||
(void)close(saved);
|
||||
}
|
||||
return -1000;
|
||||
}
|
||||
(void)fflush(stderr);
|
||||
if (dup2(fileno(capture), fileno(stderr)) < 0) {
|
||||
(void)fclose(capture);
|
||||
(void)close(saved);
|
||||
return -1000;
|
||||
}
|
||||
int rc = cbm_cmd_config(argc, argv);
|
||||
(void)fflush(stderr);
|
||||
(void)dup2(saved, fileno(stderr));
|
||||
(void)close(saved);
|
||||
rewind(capture);
|
||||
size_t got = fread(out, 1, cap - 1, capture);
|
||||
out[got] = '\0';
|
||||
(void)fclose(capture);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* `config set` suppresses its own generic message for a policy key, trusting
|
||||
* the policy writer to name the precise reason. That trust held only for a
|
||||
* value the writer rejects. A value it accepts whose write then fails — a
|
||||
* _config.db that cannot be written — exited non-zero having printed nothing
|
||||
* at all, which is a CLI that failed in silence.
|
||||
*
|
||||
* The unwritable database here is a `config` that is a view: CREATE TABLE IF
|
||||
* NOT EXISTS finds the name taken and succeeds, so the command gets past open
|
||||
* with a healthy handle and fails at the INSERT. That is the exact path with
|
||||
* no message, and it needs no file permissions, so it behaves the same on
|
||||
* every leg and under a root CI container. */
|
||||
TEST(index_policy_cli_set_reports_a_failed_write) {
|
||||
char *cache = th_mktempdir("cbm_index_policy_cli_write");
|
||||
ASSERT_NOT_NULL(cache);
|
||||
const char *saved_cache = getenv("CBM_CACHE_DIR");
|
||||
char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL;
|
||||
(void)cbm_setenv("CBM_CACHE_DIR", cache, 1);
|
||||
|
||||
char db_path[1024];
|
||||
(void)snprintf(db_path, sizeof(db_path), "%s/_config.db", cache);
|
||||
sqlite3 *db = NULL;
|
||||
bool fixture_ready =
|
||||
sqlite3_open(db_path, &db) == SQLITE_OK &&
|
||||
sqlite3_exec(db, "CREATE VIEW config(key, value) AS SELECT 'pinned', 'pinned'", NULL, NULL,
|
||||
NULL) == SQLITE_OK;
|
||||
if (db) {
|
||||
(void)sqlite3_close(db);
|
||||
}
|
||||
|
||||
char captured[1024];
|
||||
char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"};
|
||||
int rc = index_policy_config_stderr(3, set_valid, captured, sizeof(captured));
|
||||
|
||||
if (saved_cache_copy) {
|
||||
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1);
|
||||
} else {
|
||||
(void)cbm_unsetenv("CBM_CACHE_DIR");
|
||||
}
|
||||
free(saved_cache_copy);
|
||||
th_cleanup(cache);
|
||||
|
||||
ASSERT_TRUE(fixture_ready);
|
||||
ASSERT_TRUE(rc != 0);
|
||||
ASSERT_TRUE(strstr(captured, CBM_INDEX_CONFIG_MAX_FILES) != NULL);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_worker_rejects_missing_parent_policy) {
|
||||
char *repo = th_mktempdir("cbm_index_policy_worker");
|
||||
ASSERT_NOT_NULL(repo);
|
||||
char args[1024];
|
||||
(void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"mode\":\"fast\"}", repo);
|
||||
cbm_mcp_server_t *server = cbm_mcp_server_new(NULL);
|
||||
|
||||
cbm_index_set_worker_role(true, NULL);
|
||||
char *response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL;
|
||||
cbm_index_set_worker_role(false, NULL);
|
||||
bool rejected = response && strstr(response, "missing or incomplete trusted worker policy");
|
||||
|
||||
free(response);
|
||||
cbm_mcp_server_free(server);
|
||||
th_cleanup(repo);
|
||||
ASSERT_TRUE(rejected);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) {
|
||||
char *repo = th_mktempdir("cbm_index_policy_mcp_repo");
|
||||
char *cache = th_mktempdir("cbm_index_policy_mcp_cache");
|
||||
ASSERT_NOT_NULL(repo);
|
||||
ASSERT_NOT_NULL(cache);
|
||||
ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first(void) { return 1; }\n"), 0);
|
||||
|
||||
const char *saved_cache = getenv("CBM_CACHE_DIR");
|
||||
char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL;
|
||||
(void)cbm_setenv("CBM_CACHE_DIR", cache, 1);
|
||||
cbm_config_t *config = cbm_config_open(cache);
|
||||
cbm_mcp_server_t *server = cbm_mcp_server_new(NULL);
|
||||
if (server && config) {
|
||||
cbm_mcp_server_set_config(server, config);
|
||||
}
|
||||
|
||||
char args[2048];
|
||||
(void)snprintf(args, sizeof(args),
|
||||
"{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\","
|
||||
"\"mode\":\"fast\"}",
|
||||
repo);
|
||||
char *first_response =
|
||||
server && config ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL;
|
||||
bool first_indexed = first_response && strstr(first_response, "\\\"status\\\":\\\"indexed\\\"");
|
||||
free(first_response);
|
||||
|
||||
char db_path[2048];
|
||||
(void)snprintf(db_path, sizeof(db_path), "%s/ResourcePolicyFixture.db", cache);
|
||||
cbm_store_t *before_store = cbm_store_open_path_query(db_path);
|
||||
cbm_project_t before_project = {0};
|
||||
bool before_read = before_store && cbm_store_get_project(before_store, "ResourcePolicyFixture",
|
||||
&before_project) == CBM_STORE_OK;
|
||||
char *indexed_at_before =
|
||||
before_read && before_project.indexed_at ? strdup(before_project.indexed_at) : NULL;
|
||||
cbm_project_free_fields(&before_project);
|
||||
cbm_store_close(before_store);
|
||||
|
||||
bool configured =
|
||||
config && cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0 &&
|
||||
th_write_file(TH_PATH(repo, "second.py"), "def second():\n return 2\n") == 0;
|
||||
(void)snprintf(args, sizeof(args),
|
||||
"{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\","
|
||||
"\"mode\":\"fast\",\"_cbm_index_policy\":{"
|
||||
"\"index_max_files\":\"off\",\"index_max_source_mb\":\"off\"}}",
|
||||
repo);
|
||||
char *limited_response =
|
||||
configured && server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL;
|
||||
bool contract_ok = limited_response && strstr(limited_response, "resource_limit_exceeded") &&
|
||||
strstr(limited_response, "\\\"stage\\\":\\\"discovery\\\"") &&
|
||||
strstr(limited_response, "\\\"resource\\\":\\\"files\\\"") &&
|
||||
strstr(limited_response, "\\\"observed\\\":2") &&
|
||||
strstr(limited_response, "\\\"limit\\\":1") &&
|
||||
strstr(limited_response, "\\\"unit\\\":\\\"files\\\"") &&
|
||||
strstr(limited_response, "\\\"retryable\\\":true") &&
|
||||
strstr(limited_response, "\\\"serving_index_preserved\\\":true");
|
||||
free(limited_response);
|
||||
|
||||
cbm_store_t *after_store = cbm_store_open_path_query(db_path);
|
||||
cbm_project_t after_project = {0};
|
||||
bool after_read = after_store && cbm_store_get_project(after_store, "ResourcePolicyFixture",
|
||||
&after_project) == CBM_STORE_OK;
|
||||
bool generation_preserved = indexed_at_before && after_read && after_project.indexed_at &&
|
||||
strcmp(indexed_at_before, after_project.indexed_at) == 0;
|
||||
cbm_project_free_fields(&after_project);
|
||||
cbm_store_close(after_store);
|
||||
free(indexed_at_before);
|
||||
|
||||
(void)snprintf(args, sizeof(args),
|
||||
"{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\","
|
||||
"\"mode\":\"cross-repo-intelligence\"}",
|
||||
repo);
|
||||
char *cross_response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL;
|
||||
bool cross_repo_unaffected =
|
||||
cross_response && strstr(cross_response, "resource_limit_exceeded") == NULL;
|
||||
free(cross_response);
|
||||
|
||||
cbm_mcp_server_free(server);
|
||||
cbm_config_close(config);
|
||||
(void)cbm_unlink(db_path);
|
||||
char sidecar[2100];
|
||||
(void)snprintf(sidecar, sizeof(sidecar), "%s-wal", db_path);
|
||||
(void)cbm_unlink(sidecar);
|
||||
(void)snprintf(sidecar, sizeof(sidecar), "%s-shm", db_path);
|
||||
(void)cbm_unlink(sidecar);
|
||||
th_cleanup(repo);
|
||||
th_cleanup(cache);
|
||||
if (saved_cache_copy) {
|
||||
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1);
|
||||
} else {
|
||||
(void)cbm_unsetenv("CBM_CACHE_DIR");
|
||||
}
|
||||
free(saved_cache_copy);
|
||||
|
||||
ASSERT_TRUE(first_indexed);
|
||||
ASSERT_TRUE(configured);
|
||||
ASSERT_TRUE(contract_ok);
|
||||
ASSERT_TRUE(generation_preserved);
|
||||
ASSERT_TRUE(cross_repo_unaffected);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(index_policy) {
|
||||
RUN_TEST(index_policy_defaults_are_disabled);
|
||||
RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range);
|
||||
RUN_TEST(index_policy_source_limit_converts_mib_without_overflow);
|
||||
RUN_TEST(index_policy_invalid_value_is_rejected_atomically);
|
||||
RUN_TEST(index_policy_format_round_trips_public_values);
|
||||
RUN_TEST(index_policy_violation_metadata_is_stable);
|
||||
RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption);
|
||||
RUN_TEST(index_policy_cli_lists_both_operator_keys);
|
||||
RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite);
|
||||
RUN_TEST(index_policy_cli_set_reports_a_failed_write);
|
||||
RUN_TEST(index_policy_worker_rejects_missing_parent_policy);
|
||||
RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index);
|
||||
}
|
||||
@@ -776,6 +776,7 @@ extern void suite_dyn_array(void);
|
||||
extern void suite_str_intern(void);
|
||||
extern void suite_log(void);
|
||||
extern void suite_str_util(void);
|
||||
extern void suite_index_policy(void);
|
||||
extern void suite_workspace(void);
|
||||
extern void suite_platform(void);
|
||||
extern void suite_diagnostics(void);
|
||||
@@ -1054,6 +1055,7 @@ int main(int argc, char **argv) {
|
||||
RUN_SELECTED_SUITE(str_intern);
|
||||
RUN_SELECTED_SUITE(log);
|
||||
RUN_SELECTED_SUITE(str_util);
|
||||
RUN_SELECTED_SUITE(index_policy);
|
||||
RUN_SELECTED_SUITE(workspace);
|
||||
RUN_SELECTED_SUITE(platform);
|
||||
RUN_SELECTED_SUITE(diagnostics);
|
||||
|
||||
@@ -151,6 +151,34 @@ TEST(pipeline_run_null) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing) {
|
||||
char *repo = th_mktempdir("cbm_pipeline_discovery_limit");
|
||||
ASSERT_NOT_NULL(repo);
|
||||
ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0);
|
||||
ASSERT_EQ(th_write_file(TH_PATH(repo, "second.py"), "second = 2\n"), 0);
|
||||
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/index.db", repo);
|
||||
cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(pipeline);
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1};
|
||||
cbm_pipeline_set_resource_policy(pipeline, &policy);
|
||||
|
||||
ASSERT_EQ(cbm_pipeline_run(pipeline), CBM_PIPELINE_RESOURCE_LIMIT);
|
||||
cbm_index_resource_violation_t violation = {0};
|
||||
cbm_pipeline_get_resource_violation(pipeline, &violation);
|
||||
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES);
|
||||
ASSERT_EQ(violation.observed, 2);
|
||||
ASSERT_EQ(violation.limit, 1);
|
||||
ASSERT_FALSE(cbm_file_exists(db_path));
|
||||
|
||||
cbm_pipeline_free(pipeline);
|
||||
th_cleanup(repo);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Focused: file-backed store persistence ─────────────────────── */
|
||||
|
||||
TEST(store_file_persistence) {
|
||||
@@ -3412,6 +3440,44 @@ static void mutate_semantic_input_before_final_manifest(void *userdata) {
|
||||
mutation->write_rc = th_write_file(mutation->path, mutation->replacement);
|
||||
}
|
||||
|
||||
TEST(pipeline_late_source_limit_preserves_previous_generation) {
|
||||
char *repo = th_mktempdir("cbm_pipeline_late_limit");
|
||||
ASSERT_NOT_NULL(repo);
|
||||
ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0);
|
||||
char db_path[512];
|
||||
char late_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/index.db", repo);
|
||||
snprintf(late_path, sizeof(late_path), "%s/late.py", repo);
|
||||
|
||||
manifest_race_mutation_t mutation = {
|
||||
.path = late_path,
|
||||
.replacement = "late = 2\n",
|
||||
.write_rc = -1,
|
||||
};
|
||||
cbm_pipeline_incremental_test_before_final_manifest_once(
|
||||
mutate_semantic_input_before_final_manifest, &mutation);
|
||||
cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(pipeline);
|
||||
cbm_index_resource_policy_t policy;
|
||||
cbm_index_policy_init(&policy);
|
||||
policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1};
|
||||
cbm_pipeline_set_resource_policy(pipeline, &policy);
|
||||
int rc = cbm_pipeline_run(pipeline);
|
||||
cbm_index_resource_violation_t violation = {0};
|
||||
cbm_pipeline_get_resource_violation(pipeline, &violation);
|
||||
cbm_pipeline_free(pipeline);
|
||||
cbm_pipeline_incremental_test_reset_faults();
|
||||
|
||||
ASSERT_EQ(mutation.write_rc, 0);
|
||||
ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT);
|
||||
ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES);
|
||||
ASSERT_EQ(violation.observed, 2);
|
||||
ASSERT_EQ(violation.limit, 1);
|
||||
ASSERT_FALSE(cbm_file_exists(db_path));
|
||||
th_cleanup(repo);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A graph and its exact manifest are one generation. If source bytes change
|
||||
* after extraction but before publication, the mixed generation must be
|
||||
* rejected and the previous live DB preserved. */
|
||||
@@ -14726,6 +14792,8 @@ SUITE(pipeline) {
|
||||
RUN_TEST(pipeline_cancel);
|
||||
RUN_TEST(pipeline_cancel_null);
|
||||
RUN_TEST(pipeline_run_null);
|
||||
RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing);
|
||||
RUN_TEST(pipeline_late_source_limit_preserves_previous_generation);
|
||||
/* Extraction back-pressure */
|
||||
RUN_TEST(pipeline_backpressure_futile_nap_disengages);
|
||||
RUN_TEST(pipeline_over_budget_after_futility_fails_whole_and_preserves_db);
|
||||
|
||||
@@ -48,7 +48,7 @@ trap cleanup EXIT
|
||||
|
||||
missing="${tmpdir}/repository-does-not-exist"
|
||||
response="${tmpdir}/worker.response"
|
||||
args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\"}"
|
||||
args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\",$(cbm_test_index_worker_policy_json)}"
|
||||
|
||||
if ! CBM_CACHE_DIR="${tmpdir}/cache-worker" \
|
||||
"${BINARY}" cli --index-worker \
|
||||
|
||||
@@ -109,7 +109,7 @@ SH
|
||||
chmod +x "${tmpdir}/wrapper.sh"
|
||||
|
||||
CBM_BINARY="${BINARY}" BUILD_FINGERPRINT="${BUILD_FINGERPRINT}" TMPDIR_PATH="${tmpdir}" \
|
||||
ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\"}" \
|
||||
ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\",$(cbm_test_index_worker_policy_json)}" \
|
||||
CBM_TEST_HANG_ON=hang_me \
|
||||
CBM_TEST_WORKER_DESCENDANT_PID_FILE="${tmpdir}/descendant.pid" \
|
||||
"${tmpdir}/wrapper.sh" &
|
||||
|
||||
Reference in New Issue
Block a user