diff --git a/scripts/autotune.py b/scripts/autotune.py new file mode 100644 index 000000000..ec17f81a8 --- /dev/null +++ b/scripts/autotune.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +""" +autotune.py — Auto-tune codebase-memory-mcp ranking parameters. + +Usage: + python3 scripts/autotune.py [--binary PATH] [--timeout SECS] [--clone] + [--repo-url NAME=URL ...] + +Sends JSON-RPC directly to the binary via stdin/stdout (no MCP client library). +For each experiment: resets config to defaults, applies overrides, queries +codebase://architecture for each repo, scores results against the expected top-10 +ground truth, and reports the best-scoring configuration. + +Config changes are GLOBAL (stored in the binary's SQLite config DB). The script +resets all tunable keys to defaults on exit — including after errors — via atexit. + +Repo discovery order (for each repo): + 1. candidate_paths checked in order (primary system paths first) + 2. If --clone and a URL is known (via --repo-url or clone_url), clone to the + last candidate path (adjacent to this script file) + 3. If no URL available, print a hint and return None + +Examples: + python3 scripts/autotune.py + python3 scripts/autotune.py --timeout 120 # for first-time indexing + python3 scripts/autotune.py --clone --repo-url rtk=https://github.com/user/rtk + python3 scripts/autotune.py --binary /usr/local/bin/codebase-memory-mcp +""" +from __future__ import annotations + +import argparse +import atexit +import json +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# Directory containing this script — used as the fallback clone target root. +_SCRIPT_DIR = Path(__file__).parent + + +# ── Repo definitions ────────────────────────────────────────────────────────── +# Each Repo lists: candidate_paths to check in order, expected top-10 ground +# truth names, and an optional clone_url (may be None for private repos). +# Users can supply clone URLs at runtime with --repo-url name=https://... + +@dataclass +class Repo: + name: str + expected: list[str] + candidate_paths: list[Path] + clone_url: str | None = None # None = private / URL unknown + + +REPOS: list[Repo] = [ + Repo( + name="codebase-memory-mcp", + expected=[ + "cbm_arena_alloc", # in-degree 21, core allocator + "cbm_store_close", # in-degree 19 + "cbm_store_upsert_node", # in-degree 18 + "cbm_gbuf_insert_edge", # in-degree 18 + "cbm_node_text", # in-degree 18 + "cbm_arena_strdup", # in-degree 18 + "cbm_pagerank_compute_with_config", # PageRank entry point + "cbm_mcp_server_handle", # MCP entry point + "cbm_pipeline_check_cancel", # pipeline control + "build_key_functions_sql", # architecture SQL builder + ], + candidate_paths=[ + Path.home() / ".claude/codebase-memory-mcp", # primary (developer) + Path.home() / "codebase-memory-mcp", # alternate home location + _SCRIPT_DIR / "codebase-memory-mcp", # adjacent to script (clone target) + ], + clone_url=None, # supply via --repo-url codebase-memory-mcp=https://... + ), + Repo( + name="autorun", + expected=[ + "session_state", # 375 callers — hot path + "check_blocked_commands", # 170 callers — command engine + "command_matches_pattern", # 145 callers + "_not_in_pipe", # 106 callers + "get_tmux_utilities", # 96 callers + "is_premature_stop", # 64 callers + "normalize_hook_payload", # 60 callers + "validate_hook_response", # core hook + "SessionStateManager", # key class + "AutorunApp", # main class + ], + candidate_paths=[ + Path.home() / ".claude/autorun", # primary (developer) + Path.home() / "autorun", # alternate + _SCRIPT_DIR / "autorun", # adjacent to script (clone target) + ], + clone_url=None, # supply via --repo-url autorun=https://... + ), + Repo( + name="rtk", + expected=[ + "tokenize", # 115 callers — central lexer + "resolved_command", # 77 callers + "status", # 68 callers (hook_check.rs) + "strip_ansi", # high combined degree + "check_for_hook", # main hook dispatch + "check_for_hook_inner", # hook logic + "try_route_native_command", # routing + "auto_detect_filter", # pipe detection + "estimate_tokens", # token tracking + "make_filters", # filter config + # EXCLUDED: args() — test helper with 300 callers, not production code + ], + candidate_paths=[ + Path.home() / "source/rtk", # primary (developer) + Path.home() / "rtk", # alternate + _SCRIPT_DIR / "rtk", # adjacent to script (clone target) + ], + clone_url=None, # supply via --repo-url rtk=https://... + ), +] + + +# ── Config defaults ─────────────────────────────────────────────────────────── +# Reset before each experiment AND on script exit (atexit), preventing config leaks. + +DEFAULTS: dict[str, str] = { + "edge_weight_calls": "1.0", + "edge_weight_usage": "0.7", + "edge_weight_defines": "0.1", + "edge_weight_tests": "0.05", + "edge_weight_imports": "0.3", + "key_functions_count": "25", + "key_functions_exclude": "", + "pagerank_max_iter": "20", +} + + +# ── Experiment definitions ──────────────────────────────────────────────────── + +@dataclass +class Experiment: + label: str + overrides: dict[str, str] = field(default_factory=dict) + notes: str = "" + + +EXPERIMENTS: list[Experiment] = [ + Experiment("baseline_25", + {"key_functions_count": "25"}, + "Default config, just raise count from 10 to 25"), + Experiment("exclude_ui", + {"key_functions_count": "25", + "key_functions_exclude": "graph-ui/**,tools/**,scripts/**"}, + "Filter TypeScript UI and tooling — exposes C core functions"), + Experiment("calls_boost", + {"key_functions_count": "25", + "edge_weight_calls": "2.0", + "edge_weight_usage": "0.3"}, + "Boost direct call edges, dampen type-reference edges"), + Experiment("usage_dampen", + {"key_functions_count": "25", + "edge_weight_usage": "0.3", + "edge_weight_defines": "0.05"}, + "Dampen usage and define weights"), + Experiment("tests_kill", + {"key_functions_count": "25", + "edge_weight_tests": "0.01", + "edge_weight_usage": "0.3"}, + "Suppress test-file influence on production rankings"), + Experiment("calls_boost_excl", + {"key_functions_count": "25", + "edge_weight_calls": "2.0", + "edge_weight_usage": "0.3", + "key_functions_exclude": "graph-ui/**,tools/**,scripts/**"}, + "Combined: boost calls + exclude UI"), + Experiment("more_iters", + {"key_functions_count": "25", + "pagerank_max_iter": "100"}, + "More PageRank iterations for convergence on large graphs"), +] + + +# ── Repo discovery ──────────────────────────────────────────────────────────── + +def _resolve_repo(repo: Repo, clone: bool, + extra_urls: dict[str, str]) -> Path | None: + """Return the first existing candidate path, or clone if requested. + + Resolution order: + 1. Check candidate_paths in order — first existing dir wins. + 2. If none found and --clone is set: clone using extra_urls[name] or + repo.clone_url into the last candidate path (script-adjacent dir). + 3. If no URL available, print a hint and return None. + """ + for path in repo.candidate_paths: + if path.is_dir(): + return path + + clone_url = extra_urls.get(repo.name) or repo.clone_url + if not clone_url: + print(f" [info] '{repo.name}' not found at any candidate path.") + print(f" Tried: {[str(p) for p in repo.candidate_paths]}") + print(f" Supply a URL with: --repo-url {repo.name}=https://github.com/user/{repo.name}") + if not clone: + print(f" Or pass --clone to auto-clone once a URL is set.") + return None + + if not clone: + print(f" [info] '{repo.name}' not found. Pass --clone to auto-clone from {clone_url}") + return None + + target = repo.candidate_paths[-1] # script-adjacent dir as clone target + print(f" [clone] {repo.name} -> {target} (from {clone_url})") + target.parent.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + ["git", "clone", "--depth=1", clone_url, str(target)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f" [error] clone failed: {result.stderr.strip()}", file=sys.stderr) + return None + return target + + +# ── JSON-RPC helpers ────────────────────────────────────────────────────────── + +def _jsonrpc(req_id: int, method: str, params: dict[str, Any] | None = None) -> str: + msg: dict[str, Any] = {"jsonrpc": "2.0", "id": req_id, "method": method} + if params: + msg["params"] = params + return json.dumps(msg) + + +def _send_batch(binary: str, messages: list[str], timeout: int) -> dict[int, Any]: + """Send newline-delimited JSON-RPC to the binary via stdin, parse stdout responses.""" + payload = "\n".join(messages) + "\n" + try: + proc = subprocess.run( + [binary], + input=payload.encode(), + capture_output=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + print(f" [warn] binary timed out after {timeout}s — " + "raise --timeout for first-time indexing", file=sys.stderr) + return {} + except FileNotFoundError: + print(f" [error] binary not found: {binary}", file=sys.stderr) + sys.exit(1) + + responses: dict[int, Any] = {} + for line in proc.stdout.decode(errors="replace").splitlines(): + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + if "id" in r: + responses[r["id"]] = r + except json.JSONDecodeError: + pass + return responses + + +def query_architecture(binary: str, repo_root: str, timeout: int, + retries: int = 2) -> list[dict[str, Any]]: + """Query codebase://architecture, return key_functions list. + + Retries on empty results: the binary may still be indexing on first call. + """ + init = _jsonrpc(1, "initialize", { + "protocolVersion": "2024-11-05", + "capabilities": {"resources": {}}, + "clientInfo": {"name": "autotune", "version": "1.0"}, + "rootUri": f"file://{repo_root}", + }) + read = _jsonrpc(2, "resources/read", {"uri": "codebase://architecture"}) + + for attempt in range(retries + 1): + responses = _send_batch(binary, [init, read], timeout) + r2 = responses.get(2, {}) + contents = r2.get("result", {}).get("contents", []) + if contents: + try: + data = json.loads(contents[0].get("text", "{}")) + kf = data.get("key_functions", []) + if kf: + return kf + except (json.JSONDecodeError, KeyError): + pass + if attempt < retries: + wait = 3 * (attempt + 1) + print(f" [retry {attempt + 1}/{retries}] empty results — " + f"waiting {wait}s (repo may still be indexing)...") + time.sleep(wait) + + return [] + + +def set_config(binary: str, key: str, value: str, timeout: int = 10) -> None: + """Set a config value via binary CLI: `binary config set key value`.""" + try: + subprocess.run( + [binary, "config", "set", key, value], + capture_output=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + print(f" [warn] config set {key!r} timed out", file=sys.stderr) + + +def reset_to_defaults(binary: str) -> None: + """Reset all tunable config keys to baseline defaults. + + Called before each experiment and registered with atexit so no stale config + persists after a crash or KeyboardInterrupt. + """ + for k, v in DEFAULTS.items(): + set_config(binary, k, v) + + +# ── Scoring ─────────────────────────────────────────────────────────────────── + +def score_result(key_functions: list[dict[str, Any]], expected: list[str]) -> int: + """Count how many expected names appear in key_functions (case-insensitive).""" + names: set[str] = set() + for kf in key_functions: + name = kf.get("name", "") + if name: + names.add(name.lower()) + qn = kf.get("qualified_name", "") + if qn: + # Qualified names encode full paths; take the last segment + names.add(qn.split(".")[-1].lower()) + return sum(1 for e in expected if e.lower() in names) + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main() -> None: + parser = argparse.ArgumentParser( + description="Auto-tune codebase-memory-mcp ranking via JSON-RPC.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "Examples:\n" + " python3 scripts/autotune.py\n" + " python3 scripts/autotune.py --timeout 120 # for first-time indexing\n" + " python3 scripts/autotune.py --clone --repo-url rtk=https://github.com/user/rtk\n" + " python3 scripts/autotune.py --binary /usr/local/bin/codebase-memory-mcp\n" + "\n" + "NOTE: Config changes are global (stored in the binary's SQLite DB).\n" + " Stop any running codebase-memory-mcp MCP server before running autotune,\n" + " or accept that the server will use whatever config autotune is currently testing.\n" + " All config is reset to defaults on exit (including Ctrl-C).\n" + ), + ) + parser.add_argument( + "--binary", + default=str(Path.home() / ".local/bin/codebase-memory-mcp"), + help="Path to binary (default: ~/.local/bin/codebase-memory-mcp)", + ) + parser.add_argument( + "--timeout", + type=int, + default=60, + help="Seconds before JSON-RPC times out (default: 60; raise for first-time indexing)", + ) + parser.add_argument( + "--clone", + action="store_true", + help="Auto-clone missing repos (requires --repo-url or clone_url set in REPOS)", + ) + parser.add_argument( + "--repo-url", + action="append", + default=[], + metavar="NAME=URL", + help="Clone URL for a repo, e.g. --repo-url rtk=https://github.com/user/rtk " + "(can be repeated for multiple repos)", + ) + args = parser.parse_args() + binary = args.binary + + # Parse --repo-url NAME=URL pairs into a dict + extra_urls: dict[str, str] = {} + for item in args.repo_url: + if "=" in item: + name, url = item.split("=", 1) + extra_urls[name.strip()] = url.strip() + else: + print(f"[warn] --repo-url {item!r} ignored: expected NAME=URL format", + file=sys.stderr) + + if not Path(binary).is_file(): + print(f"Error: binary not found: {binary}", file=sys.stderr) + print("Build with: env -i HOME=$HOME PATH=$PATH make -f Makefile.cbm cbm", + file=sys.stderr) + sys.exit(1) + + # Resolve repos before experiments (discovery/cloning happens once) + resolved: list[tuple[Repo, Path]] = [] + for repo in REPOS: + path = _resolve_repo(repo, args.clone, extra_urls) + if path is not None: + resolved.append((repo, path)) + + if not resolved: + print("Error: no repos found. Use --clone with --repo-url, or place repos at " + "the candidate paths listed above.", file=sys.stderr) + sys.exit(1) + + # Always reset config on exit — even after Ctrl-C or crash + atexit.register(reset_to_defaults, binary) + + total_expected = sum(len(repo.expected) for repo, _ in resolved) + print(f"Binary: {binary}") + print(f"Repos: {[(repo.name, str(path)) for repo, path in resolved]}") + print(f"Timeout: {args.timeout}s per query") + print(f"Max score: {total_expected} ({len(resolved)} repos x ~10 each)\n") + + best_experiment: Experiment | None = None + best_score = -1 + all_results: list[tuple[str, int]] = [] + + for exp in EXPERIMENTS: + print(f"\n=== {exp.label} ===") + if exp.notes: + print(f" ({exp.notes})") + reset_to_defaults(binary) + for k, v in exp.overrides.items(): + set_config(binary, k, v) + print(f" config set {k} = {v!r}") + + total_score = 0 + for repo, repo_path in resolved: + kf = query_architecture(binary, str(repo_path), args.timeout) + if not kf: + print(f" [warn] {repo.name}: no key_functions returned — " + "ensure repo is indexed: codebase-memory-mcp index ") + continue + score = score_result(kf, repo.expected) + total_score += score + top5 = [kf_item.get("name") or kf_item.get("qualified_name", "?") + for kf_item in kf[:5]] + print(f" {repo.name}: {score}/{len(repo.expected)} top-5: {top5}") + + print(f" TOTAL: {total_score}/{total_expected}") + all_results.append((exp.label, total_score)) + if total_score > best_score: + best_score = total_score + best_experiment = exp + + print("\n" + "=" * 60) + if best_experiment is None: + print("No experiments produced results. Ensure repos are indexed.") + print("Index a repo: codebase-memory-mcp index ") + return + + print(f"BEST: {best_experiment.label} score={best_score}/{total_expected}") + if best_experiment.notes: + print(f" ({best_experiment.notes})") + print("\nApply permanently:") + for k, v in best_experiment.overrides.items(): + print(f" codebase-memory-mcp config set {k} {v!r}") + + print("\nAll results (best first):") + for label, score in sorted(all_results, key=lambda x: x[1], reverse=True): + marker = " <" if label == best_experiment.label else "" + print(f" {score:3d}/{total_expected} {label}{marker}") + + +if __name__ == "__main__": + main() diff --git a/src/cli/cli.c b/src/cli/cli.c index 3e8cd8e7c..55abe0d4c 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1835,40 +1835,195 @@ int cbm_config_delete(cbm_config_t *cfg, const char *key) { /* ── Config registry ──────────────────────────────────────────── */ const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = { - /* Indexing */ - {"auto_index", "true", "CBM_AUTO_INDEX", "Indexing", "Auto-index session project on startup"}, - {"auto_index_limit", "50000", "CBM_AUTO_INDEX_LIMIT", "Indexing", "Max files for auto-indexing (skip larger repos)"}, - {"reindex_on_startup", "false", "CBM_REINDEX_ON_STARTUP", "Indexing", "Re-index stale projects on restart"}, - {"reindex_stale_seconds","0", NULL, "Indexing", "Max DB age in seconds before stale (0=disabled)"}, - /* Search */ - {"search_limit", "50", NULL, "Search", "Default max results for search_code_graph"}, - {"trace_max_results", "25", NULL, "Search", "Default max nodes per direction in trace_call_path"}, - {"query_max_output_bytes","32768",NULL, "Search", "Max output bytes for query_graph (0=unlimited)"}, - {"snippet_max_lines", "200", NULL, "Search", "Max source lines in get_code_snippet (0=unlimited)"}, - {"key_functions_exclude","", "CBM_KEY_FUNCTIONS_EXCLUDE","Search", "Comma-separated globs to exclude from key_functions"}, - /* Tools */ - {"tool_mode", "streamlined","CBM_TOOL_MODE", "Tools", "Tool visibility: streamlined (3 tools) or classic (15)"}, - /* PageRank */ - {"pagerank_max_iter", "20", NULL, "PageRank", "Max power iterations for PageRank convergence"}, - {"rank_scope", "project",NULL,"PageRank", "PageRank scope: project or global"}, - {"edge_weight_calls", "1.0", NULL, "PageRank", "Edge weight: direct function/method calls"}, - {"edge_weight_usage", "0.7", NULL, "PageRank", "Edge weight: type refs, attribute access, isinstance"}, - {"edge_weight_defines_method","0.5", NULL, "PageRank", "Edge weight: class defines method (structural)"}, - {"edge_weight_imports", "0.3", NULL, "PageRank", "Edge weight: module imports"}, - {"edge_weight_decorates", "0.2", NULL, "PageRank", "Edge weight: decorator applied to function"}, - {"edge_weight_writes", "0.15", NULL, "PageRank", "Edge weight: function writes to variable/file"}, - {"edge_weight_defines", "0.1", NULL, "PageRank", "Edge weight: module defines symbol (structural noise)"}, - {"edge_weight_configures", "0.1", NULL, "PageRank", "Edge weight: config file links"}, - {"edge_weight_tests", "0.05", NULL, "PageRank", "Edge weight: test→production (dampened to avoid inflation)"}, - {"edge_weight_http_calls", "0.5", NULL, "PageRank", "Edge weight: cross-service HTTP calls"}, - {"edge_weight_async_calls", "0.8", NULL, "PageRank", "Edge weight: async function calls"}, - {"edge_weight_default", "0.1", NULL, "PageRank", "Edge weight: fallback for unrecognized edge types"}, - {"edge_weight_member_of", "0.5", NULL, "PageRank", "Edge weight: rank flow from method to parent class via MEMBER_OF (0=disabled)"}, - /* Dependencies */ - {"auto_index_deps", "true", NULL, "Dependencies", "Auto-index installed packages (from package.json, Cargo.toml, etc.)"}, - {"auto_dep_limit", "20", NULL, "Dependencies", "Max packages to index (e.g. 20 = top 20 deps like numpy, express)"}, - {"dep_max_files", "1000", NULL, "Dependencies", "Max source files per package (large packages truncated, 0=unlimited)"}, - {NULL, NULL, NULL, NULL, NULL} /* sentinel */ + /* ── Indexing ── */ + {"auto_index", "true", "CBM_AUTO_INDEX", "Indexing", + "Auto-index session project on startup", + "true|false", + "Enable to always have fresh data; disable for manual control or CI environments."}, + {"auto_index_limit", "50000", "CBM_AUTO_INDEX_LIMIT", "Indexing", + "Max files before auto-index is skipped (0=no limit, index everything)", + "0-10000000", + "Protects against accidentally indexing huge monorepos. Raise for large codebases. " + "Set 0 to disable the limit and always index regardless of repo size."}, + {"reindex_on_startup", "false", "CBM_REINDEX_ON_STARTUP", "Indexing", + "Re-index stale projects when server starts", + "true|false", + "Enable for always-fresh indexes (adds startup latency). Prefer reindex_stale_seconds for scheduled refresh."}, + {"reindex_stale_seconds", "0", NULL, "Indexing", + "Re-index if DB is older than N seconds (0=disabled)", + "0-2592000", + "0=disabled. 3600=hourly, 86400=daily, 604800=weekly. Runs on startup if stale."}, + /* ── Search ── */ + {"search_limit", "50", NULL, "Search", + "Default max results for search_code_graph", + "1-100000", + "Higher = more results but more tokens. Overridden by limit param per-query. " + "50 is good for exploration; 200+ for exhaustive analysis."}, + {"trace_max_results", "25", NULL, "Search", + "Default max nodes per direction in trace_call_path", + "1-10000", + "Controls how far call chains are traced. 25 covers typical call depth; raise to 100+ for deep dependency tracing."}, + {"query_max_output_bytes", "32768", NULL, "Search", + "Max response bytes for query_graph (0=unlimited)", + "0-104857600", + "32KB default prevents huge responses. Set 0 for unlimited Cypher results. Raise for bulk analysis queries."}, + {"snippet_max_lines", "200", NULL, "Search", + "Max source lines returned by get_code (0=unlimited)", + "0-1000000", + "200 lines covers most functions. Set 0 for unlimited to get full file contents."}, + {"key_functions_exclude", "", "CBM_KEY_FUNCTIONS_EXCLUDE", "Search", + "Comma-separated glob patterns to exclude from architecture key functions", + "glob patterns, e.g. graph-ui/**,tests/**", + "Use to remove UI, generated code, or test helpers from the architecture view. " + "Example: 'graph-ui/**,tools/**,scripts/**,tests/**'."}, + {"key_functions_count", "25", NULL, "Search", + "Max key functions returned in codebase://architecture and search context", + "1-10000", + "The architecture resource ranks every symbol by PageRank importance and returns the top N. " + "Use 25 for most projects. Raise to 50-100 for large multi-language codebases where " + "important functions may not appear in the first 25. Lower to 10 when tokens are limited."}, + /* ── Tools ── */ + {"tool_mode", "streamlined", "CBM_TOOL_MODE", "Tools", + "Which set of tools the MCP server exposes: 3 combined tools or all 15 individual tools", + "streamlined|classic", + "'streamlined' (default): exposes search_code_graph (search+Cypher), trace_call_path, get_code. " + "'classic': exposes all 15 individual tools including index_repository, query_graph, get_architecture, " + "list_projects, detect_changes, manage_adr, etc. " + "You can also enable individual classic tools without switching modes: " + "config set tool_index_repository true"}, + /* ── PageRank ── */ + {"pagerank_max_iter", "20", NULL, "PageRank", + "Max iterations for PageRank algorithm before stopping (more = more accurate convergence)", + "1-10000", + "PageRank is an iterative algorithm — each iteration refines importance scores. " + "20 iterations converges in ~5ms for 16K-node codebases. Typical convergence is 10-15 iters. " + "Raise to 50-100 for very large codebases (>100K nodes). " + "Diminishing returns above convergence — set too high wastes CPU at reindex time."}, + {"rank_scope", "project", NULL, "PageRank", + "Whether PageRank importance is computed per-project or across all indexed projects", + "project|full", + "'project' (default): each project's symbols are scored independently — scores are " + "comparable within a project but not across projects. " + "'full': scores all projects in one global computation — enables cross-project comparison " + "but is slower and dependency scores mix with your project's scores."}, + {"edge_weight_calls", "1.0", NULL, "PageRank", + "How much importance flows along direct function/method call edges (CALLS)", + "0.0-100.0", + "PageRank works like Google PageRank: importance flows along edges. Higher weight = more " + "importance flows when one function calls another. 1.0 is the anchor — all other weights " + "are relative to it. Increase to 2.0 for call-heavy C/Rust codebases. " + "Decrease to 0.5 for event-driven systems where direct calls aren't the primary coupling."}, + {"edge_weight_usage", "0.7", NULL, "PageRank", + "How much importance flows along type-reference edges: type annotations, attribute access, isinstance (USAGE)", + "0.0-100.0", + "USAGE edges are created when code references a type (e.g. 'x: MyClass', 'isinstance(x, Foo)'). " + "These are dense in TypeScript/Python and can inflate UI utilities over core functions. " + "Reduce to 0.2-0.3 if type annotations are dominating your architecture results."}, + {"edge_weight_defines_method", "0.5", NULL, "PageRank", + "How much importance flows from a class to each method it defines (DEFINES_METHOD)", + "0.0-100.0", + "Every class has one DEFINES_METHOD edge per method. Higher = classes with many methods rank " + "higher relative to standalone functions. Lower to 0.1 to treat functions and class methods equally."}, + {"edge_weight_imports", "0.3", NULL, "PageRank", + "How much importance flows along module import edges (IMPORTS)", + "0.0-100.0", + "Created when file A imports file/module B. Higher promotes widely-imported utility modules " + "(e.g. a shared 'utils.py' imported by 50 files). Raise to 0.6-0.8 to emphasize shared infrastructure; " + "keep low if star-imports create many spurious edges."}, + {"edge_weight_decorates", "0.2", NULL, "PageRank", + "How much importance flows from a decorator to the function it decorates (DECORATES)", + "0.0-100.0", + "Created when @decorator is applied to a function. Raise to 0.5+ in Python web frameworks " + "where @route, @cached, @requires_auth are semantically important architectural markers."}, + {"edge_weight_writes", "0.15", NULL, "PageRank", + "How much importance flows when a function writes to a variable or file (WRITES)", + "0.0-100.0", + "Tracks side effects: function writes to a shared variable or file. Raise for ETL or " + "data-pipeline codebases where write targets (databases, output files) are the primary output."}, + {"edge_weight_defines", "0.1", NULL, "PageRank", + "How much importance flows from a file/module to each symbol it defines (DEFINES — structural)", + "0.0-100.0", + "Every function has exactly one DEFINES edge from its containing file. This is purely structural " + "bookkeeping — keep very low (0.01-0.1). Raising this inflates ALL symbols in a file equally, " + "which is rarely what you want."}, + {"edge_weight_configures", "0.1", NULL, "PageRank", + "How much importance flows from config files to the code they configure (CONFIGURES)", + "0.0-100.0", + "Created when a config file references a code symbol (e.g. a YAML file referencing a handler " + "class). Raise to 0.3+ for infrastructure projects where config -> code coupling is important."}, + {"edge_weight_tests", "0.05", NULL, "PageRank", + "How much importance flows from test code to the production function it tests (TESTS)", + "0.0-100.0", + "Intentionally very low so test files don't inflate production function rankings. A function " + "with 100 tests would otherwise rank at the top of every project. Raise only if you want " + "heavily-tested functions to rank higher (useful for spotting critical code paths)."}, + {"edge_weight_http_calls", "0.5", NULL, "PageRank", + "How much importance flows along cross-service HTTP call edges (HTTP_CALLS)", + "0.0-100.0", + "Created when code makes an HTTP call to another service endpoint. Raise to 1.0-2.0 for " + "microservice architectures where HTTP calls ARE the primary coupling between components " + "and you want service entry points to appear prominently in architecture results."}, + {"edge_weight_async_calls", "0.8", NULL, "PageRank", + "How much importance flows along async function call edges (ASYNC_CALLS)", + "0.0-100.0", + "Like edge_weight_calls but for async/await call patterns. Slightly lower than sync calls " + "by default. Reduce to 0.3 for heavily async Node.js or Python asyncio codebases where " + "awaited spans are dense and create noise in the rankings."}, + {"edge_weight_default", "0.1", NULL, "PageRank", + "Fallback importance weight for edge types not listed above", + "0.0-100.0", + "Safety net for any edge types added in future without explicit weights. " + "Rarely affects results. Keep low."}, + {"edge_weight_member_of", "0.5", NULL, "PageRank", + "How much importance flows from a method back up to its parent class (MEMBER_OF — reverse structural)", + "0.0-100.0", + "Set to 0 to disable (method importance stays in the method, not the class). " + "Higher values propagate method-level importance up to the parent class — " + "raise to 0.8 to make heavily-called classes rank higher than individual methods."}, + /* ── Watcher ── */ + {"watcher_poll_base_ms", "5000", NULL, "Watcher", + "Base file-watcher poll interval in milliseconds", + "100-3600000", + "5 seconds by default. Lower for faster change detection (100ms for dev loops); " + "raise for large repos to reduce CPU overhead. Actual interval scales with file count."}, + {"watcher_poll_max_ms", "60000", NULL, "Watcher", + "Maximum file-watcher poll interval in milliseconds (cap for large repos)", + "100-3600000", + "60 seconds for repos with 50K+ files. Lower to 10000 for faster detection in large repos " + "if CPU allows. Formula: min(base + file_count/500 * 1000, max)."}, + /* ── Architecture ── */ + {"arch_hotspot_limit", "25", NULL, "Architecture", + "Max hotspot functions shown in the classic get_architecture tool's hotspots section", + "1-10000", + "Hotspots are functions ranked by how many times they are directly called (calls_in count). " + "They identify the most-invoked code — good candidates for optimization and risk assessment. " + "25 is enough for orientation; raise to 100 for exhaustive call-density analysis. " + "Only applies to the classic 'get_architecture' tool (tool_mode=classic)."}, + /* ── Degree / Sort ── */ + {"degree_mode", "weighted", NULL, "Degree", + "What 'degree' means for min_degree/max_degree filters and sort_by=degree ranking", + "weighted|unweighted|calls_only", + "Degree = how connected a symbol is. 'weighted' multiplies each connection by its edge type weight " + "(e.g. a direct call counts 1.0x, a test call counts 0.05x) — best overall signal. " + "'unweighted' = raw connection count regardless of type. " + "'calls_only' = only count direct function call connections — best for finding the most-called functions."}, + /* ── Dependencies ── */ + {"auto_index_deps", "true", NULL, "Dependencies", + "Auto-index installed packages from package.json, Cargo.toml, go.mod, etc.", + "true|false", + "Enable to trace calls into dependencies (e.g. find all callers of a library function). " + "Disable for faster indexing when cross-package search is not needed."}, + {"auto_dep_limit", "20", NULL, "Dependencies", + "Max number of packages to auto-index", + "0-10000", + "20 covers the most-used imports. Raise to 100+ for comprehensive dependency analysis. " + "0 = unlimited (may be very slow for large dependency trees)."}, + {"dep_max_files", "1000", NULL, "Dependencies", + "Max source files per dependency package (0=unlimited)", + "0-1000000", + "Caps indexing of large packages (TensorFlow, LLVM). 1000 covers most packages. " + "Set 0 for unlimited if you need complete large-package analysis."}, + {NULL, NULL, NULL, NULL, NULL, NULL, NULL} /* sentinel */ }; /* Get config value with env var override priority: env > db > default. @@ -1915,12 +2070,18 @@ int cbm_cmd_config(int argc, char **argv) { last_cat = e->category; } if (e->env_var) { - printf(" %-28s default=%-8s %s [env: %s]\n", - e->key, e->default_val, e->description, e->env_var); + printf(" %-30s default=%-14s [env: %s]\n", + e->key, e->default_val, e->env_var); } else { - printf(" %-28s default=%-8s %s\n", - e->key, e->default_val, e->description); + printf(" %-30s default=%-14s\n", + e->key, e->default_val); } + if (e->range || e->description) + printf(" [%-20s] %s\n", + e->range ? e->range : "any", + e->description ? e->description : ""); + if (e->guidance) + printf(" %s\n\n", e->guidance); } return 0; } @@ -1963,7 +2124,13 @@ int cbm_cmd_config(int argc, char **argv) { /* Check if DB value differs from default */ const char *db_val = cbm_config_get(cfg, e->key, NULL); if (!source[0] && db_val) source = " (set)"; - printf(" %-28s = %-12s%s\n", e->key, val, source); + printf(" %-30s = %-14s%s\n", e->key, val, source); + if (e->range || e->description) + printf(" [%-20s] %s\n", + e->range ? e->range : "any", + e->description ? e->description : ""); + if (e->guidance) + printf(" %s\n\n", e->guidance); } } else if (strcmp(argv[0], "get") == 0) { if (argc < 2) { @@ -1972,13 +2139,21 @@ int cbm_cmd_config(int argc, char **argv) { } else { /* Find default from registry */ const char *def = ""; + const cbm_config_entry_t *found_entry = NULL; for (int i = 0; CBM_CONFIG_REGISTRY[i].key; i++) { if (strcmp(CBM_CONFIG_REGISTRY[i].key, argv[1]) == 0) { def = CBM_CONFIG_REGISTRY[i].default_val; + found_entry = &CBM_CONFIG_REGISTRY[i]; break; } } printf("%s\n", cbm_config_get_effective(cfg, argv[1], def)); + if (found_entry) { + if (found_entry->range) + printf("range: %s\n", found_entry->range); + if (found_entry->guidance) + printf("guidance: %s\n", found_entry->guidance); + } } } else if (strcmp(argv[0], "set") == 0) { if (argc < 3) { diff --git a/src/cli/cli.h b/src/cli/cli.h index 6d494dd4e..82b35fc6b 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -242,6 +242,8 @@ typedef struct { const char *env_var; /* env var override name, NULL if none */ const char *category; /* display category for config list */ const char *description; /* one-line description */ + const char *range; /* broadest feasible range/valid values */ + const char *guidance; /* actionable: why change it, effect on output */ } cbm_config_entry_t; /* All known config keys. Defined in cli.c. NULL-terminated. */ diff --git a/src/main.c b/src/main.c index 57010e401..abfcb016b 100644 --- a/src/main.c +++ b/src/main.c @@ -59,10 +59,17 @@ static void signal_handler(int sig) { /* ── Watcher background thread ──────────────────────────────────── */ +typedef struct { + cbm_watcher_t *w; + int base_ms; + int max_ms; +} watcher_thread_args_t; + +static watcher_thread_args_t g_watcher_args; /* lifetime: static, no free needed */ + static void *watcher_thread(void *arg) { - cbm_watcher_t *w = arg; -#define WATCHER_BASE_INTERVAL_MS 5000 - cbm_watcher_run(w, WATCHER_BASE_INTERVAL_MS); + watcher_thread_args_t *a = arg; + cbm_watcher_run(a->w, a->base_ms, a->max_ms); return NULL; } @@ -265,7 +272,14 @@ int main(int argc, char **argv) { bool watcher_started = false; if (g_watcher) { - if (cbm_thread_create(&watcher_tid, 0, watcher_thread, g_watcher) == 0) { + g_watcher_args.w = g_watcher; + g_watcher_args.base_ms = runtime_config + ? cbm_config_get_int(runtime_config, "watcher_poll_base_ms", 5000) + : 5000; + g_watcher_args.max_ms = runtime_config + ? cbm_config_get_int(runtime_config, "watcher_poll_max_ms", 60000) + : 60000; + if (cbm_thread_create(&watcher_tid, 0, watcher_thread, &g_watcher_args) == 0) { watcher_started = true; } } diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index bc5489c05..76b7f8316 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -85,6 +85,8 @@ static void add_pagerank_val(yyjson_mut_doc *doc, yyjson_mut_val *obj, double v) /* Config key: comma-separated glob patterns to exclude from key_functions. * Set via: config set key_functions_exclude "scripts/,tools/,tests/" */ #define CBM_CONFIG_KEY_FUNCTIONS_EXCLUDE "key_functions_exclude" +#define CBM_CONFIG_KEY_FUNCTIONS_COUNT "key_functions_count" +#define CBM_CONFIG_ARCH_HOTSPOT_LIMIT "arch_hotspot_limit" /* Directory permissions: rwxr-xr-x */ #define ADR_DIR_PERMS 0755 @@ -290,9 +292,10 @@ static const tool_def_t TOOLS[] = { "Response includes has_more and pagination_hint when more pages exist." "\"},\"offset\":{\"type\":\"integer\",\"default\":0,\"description\":\"Skip N results " "for pagination. Check pagination_hint in response for next page offset.\"}," - "\"sort_by\":{\"type\":\"string\",\"enum\":[\"relevance\",\"name\",\"degree\"]," + "\"sort_by\":{\"type\":\"string\",\"enum\":[\"relevance\",\"name\",\"degree\",\"calls\",\"linkrank\"]," "\"description\":\"Sort order: relevance (PageRank structural importance, default), " - "name (alphabetical), degree (most connected).\"}," + "name (alphabetical), degree (most connected by edge weight), " + "calls (most direct function calls in+out), linkrank (link-based rank score).\"}," "\"mode\":{\"type\":\"string\",\"enum\":[\"full\",\"summary\"],\"default\":\"full\"," "\"description\":\"full=individual results (default), summary=aggregate counts by label and " "file. Use summary first to understand scope, then full with filters to drill down." @@ -440,7 +443,7 @@ static const tool_def_t STREAMLINED_TOOLS[] = { "patterns. When provided, other filter params are ignored. Add LIMIT.\"}," "\"label\":{\"type\":\"string\"},\"name_pattern\":{\"type\":\"string\"}," "\"qn_pattern\":{\"type\":\"string\"},\"file_pattern\":{\"type\":\"string\"}," - "\"sort_by\":{\"type\":\"string\",\"enum\":[\"relevance\",\"name\",\"degree\"]}," + "\"sort_by\":{\"type\":\"string\",\"enum\":[\"relevance\",\"name\",\"degree\",\"calls\",\"linkrank\"]}," "\"mode\":{\"type\":\"string\",\"enum\":[\"full\",\"summary\"]}," "\"compact\":{\"type\":\"boolean\"},\"include_dependencies\":{\"type\":\"boolean\"}," "\"limit\":{\"type\":\"integer\"},\"offset\":{\"type\":\"integer\"}," @@ -679,7 +682,7 @@ static void free_string_array(char **arr) { /* Forward declarations for functions defined after first use */ static void notify_resources_updated(cbm_mcp_server_t *srv); -static char *build_key_functions_sql(const char *exclude_csv, const char **exclude_arr); +static char *build_key_functions_sql(const char *exclude_csv, const char **exclude_arr, int limit); char *cbm_glob_to_like(const char *pattern); /* store.c */ struct cbm_mcp_server { @@ -1600,7 +1603,7 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { char errbuf[256]; snprintf(errbuf, sizeof(errbuf), "{\"error\":\"invalid sort_by '%s'\"," - "\"hint\":\"Valid values: relevance, name, degree\"}", sort_by); + "\"hint\":\"Valid values: relevance, name, degree, calls, linkrank\"}", sort_by); free(label); free(name_pattern); free(qn_pattern); free(file_pattern); free(relationship); free(sort_by); free(pe.value); return cbm_mcp_text_result(errbuf, true); @@ -1649,6 +1652,9 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { params.file_pattern = file_pattern; params.relationship = relationship; params.sort_by = sort_by; + params.degree_mode = srv->config + ? cbm_config_get(srv->config, "degree_mode", NULL) + : NULL; params.limit = effective_limit; params.offset = offset; params.min_degree = min_degree; @@ -2177,7 +2183,10 @@ static char *handle_get_architecture(cbm_mcp_server_t *srv, const char *args) { const char *excl_csv = srv->config ? cbm_config_get(srv->config, CBM_CONFIG_KEY_FUNCTIONS_EXCLUDE, "") : ""; - char *kf_sql_heap = build_key_functions_sql(excl_csv, (const char **)excl_arr); + int kf_limit = srv->config + ? cbm_config_get_int(srv->config, CBM_CONFIG_KEY_FUNCTIONS_COUNT, 25) + : 25; + char *kf_sql_heap = build_key_functions_sql(excl_csv, (const char **)excl_arr, kf_limit); free_string_array(excl_arr); const char *kf_sql = kf_sql_heap; sqlite3_stmt *kf_stmt = NULL; @@ -4276,7 +4285,7 @@ static void build_resource_schema(yyjson_mut_doc *doc, yyjson_mut_val *root, * exclude_arr: NULL-terminated array from tool param, or NULL. * Returns a heap-allocated SQL string. Caller must free. */ static char *build_key_functions_sql(const char *exclude_csv, - const char **exclude_arr) { + const char **exclude_arr, int limit) { char sql[4096]; int pos = 0; pos += snprintf(sql + pos, sizeof(sql) - pos, @@ -4314,7 +4323,8 @@ static char *build_key_functions_sql(const char *exclude_csv, } } - snprintf(sql + pos, sizeof(sql) - pos, "ORDER BY pr.rank DESC LIMIT 10"); + snprintf(sql + pos, sizeof(sql) - pos, "ORDER BY pr.rank DESC LIMIT %d", + limit > 0 ? limit : 25); return heap_strdup(sql); } @@ -4340,7 +4350,10 @@ static void build_resource_architecture(yyjson_mut_doc *doc, yyjson_mut_val *roo const char *excl_csv = srv->config ? cbm_config_get(srv->config, CBM_CONFIG_KEY_FUNCTIONS_EXCLUDE, "") : ""; - char *sql = build_key_functions_sql(excl_csv, NULL); + int kf_limit = srv->config + ? cbm_config_get_int(srv->config, CBM_CONFIG_KEY_FUNCTIONS_COUNT, 25) + : 25; + char *sql = build_key_functions_sql(excl_csv, NULL, kf_limit); sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) { sqlite3_bind_text(stmt, 1, proj, -1, SQLITE_TRANSIENT); diff --git a/src/store/store.c b/src/store/store.c index 12e42dc7c..58dd1d96e 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -1794,14 +1794,33 @@ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_sear sqlite3_finalize(check); } } + /* Choose degree columns based on degree_mode param. + * degree_mode: "weighted"→weighted_in/out, "calls_only"→calls_in/out, + * NULL/"unweighted"→total_in/out (default). Only applies when has_degree_table. */ + const char *in_expr = "COALESCE(nd.total_in, 0)"; + const char *out_expr = "COALESCE(nd.total_out, 0)"; + if (has_degree_table && params->degree_mode) { + if (strcmp(params->degree_mode, "weighted") == 0) { + in_expr = "COALESCE(nd.weighted_in, 0)"; + out_expr = "COALESCE(nd.weighted_out, 0)"; + } else if (strcmp(params->degree_mode, "calls_only") == 0) { + in_expr = "COALESCE(nd.calls_in, 0)"; + out_expr = "COALESCE(nd.calls_out, 0)"; + } + } + char sel_with_pr_deg[512]; + char sel_deg_only[512]; + snprintf(sel_with_pr_deg, sizeof(sel_with_pr_deg), + "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " + "n.file_path, n.start_line, n.end_line, n.properties, " + "%s AS in_deg, %s AS out_deg, COALESCE(pr.rank, 0.0) AS pr_rank ", in_expr, out_expr); + snprintf(sel_deg_only, sizeof(sel_deg_only), + "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " + "n.file_path, n.start_line, n.end_line, n.properties, " + "%s AS in_deg, %s AS out_deg ", in_expr, out_expr); const char *select_cols; if (use_pagerank && has_degree_table) { - select_cols = - "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " - "n.file_path, n.start_line, n.end_line, n.properties, " - "COALESCE(nd.total_in, 0) AS in_deg, " - "COALESCE(nd.total_out, 0) AS out_deg, " - "COALESCE(pr.rank, 0.0) AS pr_rank "; + select_cols = sel_with_pr_deg; } else if (use_pagerank) { select_cols = "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " @@ -1810,11 +1829,7 @@ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_sear "(SELECT COUNT(*) FROM edges e WHERE e.source_id = n.id) AS out_deg, " "COALESCE(pr.rank, 0.0) AS pr_rank "; } else if (has_degree_table) { - select_cols = - "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " - "n.file_path, n.start_line, n.end_line, n.properties, " - "COALESCE(nd.total_in, 0) AS in_deg, " - "COALESCE(nd.total_out, 0) AS out_deg "; + select_cols = sel_deg_only; } else { select_cols = "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " @@ -2029,6 +2044,29 @@ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_sear snprintf(order_limit, sizeof(order_limit), " ORDER BY (in_deg + out_deg) DESC, %s, %s LIMIT %d OFFSET %d", name_col, id_col, limit, offset); + } else if (params->sort_by && strcmp(params->sort_by, "calls") == 0) { + if (has_degree_table) { + snprintf(order_limit, sizeof(order_limit), + " ORDER BY COALESCE(nd.calls_in + nd.calls_out, 0) DESC, %s, %s" + " LIMIT %d OFFSET %d", + name_col, id_col, limit, offset); + } else { + /* Fallback: no precomputed calls data — use total degree */ + snprintf(order_limit, sizeof(order_limit), + " ORDER BY (in_deg + out_deg) DESC, %s, %s LIMIT %d OFFSET %d", + name_col, id_col, limit, offset); + } + } else if (params->sort_by && strcmp(params->sort_by, "linkrank") == 0) { + if (has_degree_table) { + snprintf(order_limit, sizeof(order_limit), + " ORDER BY COALESCE(nd.linkrank_in, 0) DESC, %s, %s LIMIT %d OFFSET %d", + name_col, id_col, limit, offset); + } else { + /* Fallback: no precomputed linkrank — use total degree */ + snprintf(order_limit, sizeof(order_limit), + " ORDER BY (in_deg + out_deg) DESC, %s, %s LIMIT %d OFFSET %d", + name_col, id_col, limit, offset); + } } else { /* name sort (explicit or fallback) */ if (params->project_pattern) { @@ -2879,7 +2917,7 @@ static int arch_routes(cbm_store_t *s, const char *project, cbm_architecture_inf return CBM_STORE_OK; } -enum { CBM_ARCH_HOTSPOT_DEFAULT_LIMIT = 10 }; +enum { CBM_ARCH_HOTSPOT_DEFAULT_LIMIT = 25 }; static int arch_hotspots(cbm_store_t *s, const char *project, cbm_architecture_info_t *out, int limit) { @@ -3919,7 +3957,8 @@ static bool want_aspect(const char **aspects, int aspect_count, const char *name } int cbm_store_get_architecture(cbm_store_t *s, const char *project, const char **aspects, - int aspect_count, cbm_architecture_info_t *out) { + int aspect_count, cbm_architecture_info_t *out, + int hotspot_limit) { memset(out, 0, sizeof(*out)); int rc; @@ -3948,7 +3987,8 @@ int cbm_store_get_architecture(cbm_store_t *s, const char *project, const char * } } if (want_aspect(aspects, aspect_count, "hotspots")) { - rc = arch_hotspots(s, project, out, CBM_ARCH_HOTSPOT_DEFAULT_LIMIT); + rc = arch_hotspots(s, project, out, + hotspot_limit > 0 ? hotspot_limit : CBM_ARCH_HOTSPOT_DEFAULT_LIMIT); if (rc != CBM_STORE_OK) { return rc; } diff --git a/src/store/store.h b/src/store/store.h index 7df6dd1e6..99ed9608f 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -110,11 +110,12 @@ typedef struct { const char *direction; /* "inbound" / "outbound" / "any", NULL = any */ int min_degree; /* -1 = no filter (default), 0+ = minimum */ int max_degree; /* -1 = no filter (default), 0+ = maximum */ - int limit; /* 0 = default (10) */ + int limit; /* 0 = unlimited */ int offset; bool exclude_entry_points; bool include_connected; - const char *sort_by; /* "relevance" / "name" / "degree", NULL = relevance */ + const char *sort_by; /* "relevance" / "name" / "degree" / "calls" / "linkrank", NULL = relevance */ + const char *degree_mode; /* "weighted" / "unweighted" / "calls_only", NULL = unweighted */ bool case_sensitive; const char **exclude_labels; /* NULL-terminated array, or NULL */ const char **exclude_paths; /* NULL-terminated array of glob patterns to exclude by file_path */ @@ -495,7 +496,8 @@ typedef struct { } cbm_architecture_info_t; int cbm_store_get_architecture(cbm_store_t *s, const char *project, const char **aspects, - int aspect_count, cbm_architecture_info_t *out); + int aspect_count, cbm_architecture_info_t *out, + int hotspot_limit); void cbm_store_architecture_free(cbm_architecture_info_t *out); /* ── ADR (Architecture Decision Record) ────────────────────────── */ diff --git a/src/watcher/watcher.c b/src/watcher/watcher.c index 54da362de..fd5f6655a 100644 --- a/src/watcher/watcher.c +++ b/src/watcher/watcher.c @@ -49,6 +49,8 @@ struct cbm_watcher { void *user_data; CBMHashTable *projects; /* name → project_state_t* */ atomic_int stopped; + int poll_base_ms; /* 0 = use POLL_BASE_MS default */ + int poll_max_ms; /* 0 = use POLL_MAX_MS default */ }; /* ── Constants ─────────────────────────────────────────────────── */ @@ -76,10 +78,12 @@ static int64_t now_ns(void) { /* ── Adaptive interval ──────────────────────────────────────────── */ -int cbm_watcher_poll_interval_ms(int file_count) { - int ms = POLL_BASE_MS + ((file_count / POLL_FILE_STEP) * 1000); - if (ms > POLL_MAX_MS) { - ms = POLL_MAX_MS; +int cbm_watcher_poll_interval_ms(int file_count, int base_ms, int max_ms) { + if (base_ms <= 0) base_ms = POLL_BASE_MS; + if (max_ms <= 0) max_ms = POLL_MAX_MS; + int ms = base_ms + ((file_count / POLL_FILE_STEP) * 1000); + if (ms > max_ms) { + ms = max_ms; } return ms; } @@ -269,7 +273,7 @@ int cbm_watcher_watch_count(const cbm_watcher_t *w) { /* ── Single poll cycle ──────────────────────────────────────────── */ /* Init baseline for a project: check if git, get HEAD, count files */ -static void init_baseline(project_state_t *s) { +static void init_baseline(project_state_t *s, const cbm_watcher_t *w) { struct stat st; if (stat(s->root_path, &st) != 0) { cbm_log_warn("watcher.root_gone", "project", s->project_name, "path", s->root_path); @@ -284,7 +288,7 @@ static void init_baseline(project_state_t *s) { if (s->is_git) { git_head(s->root_path, s->last_head, sizeof(s->last_head)); s->file_count = git_file_count(s->root_path); - s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count); + s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count, w->poll_base_ms, w->poll_max_ms); cbm_log_info("watcher.baseline", "project", s->project_name, "strategy", "git", "files", s->file_count > 0 ? "yes" : "0"); } else { @@ -333,7 +337,7 @@ static void poll_project(const char *key, void *val, void *ud) { /* Initialize baseline on first poll */ if (!s->baseline_done) { - init_baseline(s); + init_baseline(s, ctx->w); return; } @@ -364,7 +368,7 @@ static void poll_project(const char *key, void *val, void *ud) { git_head(s->root_path, s->last_head, sizeof(s->last_head)); /* Refresh file count for interval */ s->file_count = git_file_count(s->root_path); - s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count); + s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count, ctx->w->poll_base_ms, ctx->w->poll_max_ms); } else { cbm_log_warn("watcher.index.err", "project", s->project_name); } @@ -395,13 +399,13 @@ void cbm_watcher_stop(cbm_watcher_t *w) { } } -int cbm_watcher_run(cbm_watcher_t *w, int base_interval_ms) { +int cbm_watcher_run(cbm_watcher_t *w, int base_ms, int max_ms) { if (!w) { return -1; } - if (base_interval_ms <= 0) { - base_interval_ms = POLL_BASE_MS; - } + int base_interval_ms = (base_ms > 0) ? base_ms : POLL_BASE_MS; + w->poll_base_ms = base_interval_ms; + w->poll_max_ms = (max_ms > 0) ? max_ms : POLL_MAX_MS; cbm_log_info("watcher.start", "interval_ms", base_interval_ms > 999 ? "multi-sec" : "fast"); diff --git a/src/watcher/watcher.h b/src/watcher/watcher.h index 259210977..242dde772 100644 --- a/src/watcher/watcher.h +++ b/src/watcher/watcher.h @@ -54,9 +54,10 @@ void cbm_watcher_touch(cbm_watcher_t *w, const char *project_name); * Returns the number of projects that were reindexed. */ int cbm_watcher_poll_once(cbm_watcher_t *w); -/* Run the blocking poll loop. Polls every base_interval_ms until - * cbm_watcher_stop() is called. Returns 0 on clean shutdown. */ -int cbm_watcher_run(cbm_watcher_t *w, int base_interval_ms); +/* Run the blocking poll loop. Polls every base_ms until cbm_watcher_stop() is called. + * max_ms caps the adaptive interval for large repos. 0 = use defaults (5000/60000). + * Returns 0 on clean shutdown. */ +int cbm_watcher_run(cbm_watcher_t *w, int base_ms, int max_ms); /* Request the run loop to stop (thread-safe). */ void cbm_watcher_stop(cbm_watcher_t *w); @@ -66,7 +67,8 @@ void cbm_watcher_stop(cbm_watcher_t *w); /* Return the number of projects in the watch list. */ int cbm_watcher_watch_count(const cbm_watcher_t *w); -/* Return the adaptive poll interval (ms) for a given file count. */ -int cbm_watcher_poll_interval_ms(int file_count); +/* Return the adaptive poll interval (ms) for a given file count. + * base_ms/max_ms: 0 = use defaults (POLL_BASE_MS=5000, POLL_MAX_MS=60000). */ +int cbm_watcher_poll_interval_ms(int file_count, int base_ms, int max_ms); #endif /* CBM_WATCHER_H */ diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c index 32663f3a2..64cb6b5a5 100644 --- a/tests/test_store_arch.c +++ b/tests/test_store_arch.c @@ -141,7 +141,7 @@ static cbm_store_t *setup_arch_test_store(void) { TEST(arch_get_all) { cbm_store_t *s = setup_arch_test_store(); cbm_architecture_info_t info; - ASSERT_EQ(cbm_store_get_architecture(s, "test", NULL, 0, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", NULL, 0, &info, 0), CBM_STORE_OK); ASSERT_TRUE(info.language_count > 0); ASSERT_TRUE(info.package_count > 0); @@ -160,7 +160,7 @@ TEST(arch_entry_points_exclude_tests) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"entry_points"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); for (int i = 0; i < info.entry_point_count; i++) { ASSERT_TRUE(strstr(info.entry_points[i].file, "test") == NULL); @@ -177,7 +177,7 @@ TEST(arch_hotspots_exclude_tests) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"hotspots"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); for (int i = 0; i < info.hotspot_count; i++) { ASSERT_TRUE(strstr(info.hotspots[i].name, "Test") == NULL); @@ -192,7 +192,7 @@ TEST(arch_specific_aspects) { cbm_store_t *s = setup_arch_test_store(); cbm_architecture_info_t info; const char *aspects[] = {"languages", "hotspots"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 2, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 2, &info, 0), CBM_STORE_OK); ASSERT_TRUE(info.language_count > 0); ASSERT_TRUE(info.hotspot_count > 0); @@ -213,7 +213,7 @@ TEST(arch_empty_project) { cbm_architecture_info_t info; const char *aspects[] = {"all"}; - ASSERT_EQ(cbm_store_get_architecture(s, "empty", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "empty", aspects, 1, &info, 0), CBM_STORE_OK); /* All should be empty but no errors */ cbm_store_architecture_free(&info); @@ -226,7 +226,7 @@ TEST(arch_languages) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"languages"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); /* Check Go=3, Python=1, JavaScript=1 */ int go_count = 0, py_count = 0, js_count = 0; @@ -252,7 +252,7 @@ TEST(arch_routes) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"routes"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); ASSERT_EQ(info.route_count, 1); ASSERT_STR_EQ(info.routes[0].method, "POST"); @@ -269,7 +269,7 @@ TEST(arch_hotspots) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"hotspots"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); ASSERT_TRUE(info.hotspot_count > 0); /* ProcessOrder should be a hotspot (called by HandleRequest) */ @@ -293,7 +293,7 @@ TEST(arch_boundaries) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"boundaries"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); ASSERT_TRUE(info.boundary_count > 0); /* server → handler and handler → service should be present */ @@ -319,7 +319,7 @@ TEST(arch_layers) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"layers"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); ASSERT_TRUE(info.layer_count > 0); /* Handler package has routes, should be "api" */ @@ -339,7 +339,7 @@ TEST(arch_file_tree) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"file_tree"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); ASSERT_TRUE(info.file_tree_count > 0); /* Check that entries have valid types */ @@ -358,7 +358,7 @@ TEST(arch_clusters) { cbm_architecture_info_t info; memset(&info, 0, sizeof(info)); const char *aspects[] = {"clusters"}; - ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info), CBM_STORE_OK); + ASSERT_EQ(cbm_store_get_architecture(s, "test", aspects, 1, &info, 0), CBM_STORE_OK); /* With 5 functions and 4 edges, Louvain should find at least 1 cluster */ if (info.cluster_count == 0) { diff --git a/tests/test_watcher.c b/tests/test_watcher.c index 7a3d8a364..bd0659356 100644 --- a/tests/test_watcher.c +++ b/tests/test_watcher.c @@ -20,36 +20,36 @@ TEST(poll_interval_base) { /* 0 files → 5s base */ - int ms = cbm_watcher_poll_interval_ms(0); + int ms = cbm_watcher_poll_interval_ms(0, 0, 0); ASSERT_EQ(ms, 5000); PASS(); } TEST(poll_interval_scaling) { /* 1000 files → 5000 + 2*1000 = 7000ms */ - int ms = cbm_watcher_poll_interval_ms(1000); + int ms = cbm_watcher_poll_interval_ms(1000, 0, 0); ASSERT_EQ(ms, 7000); /* 5000 files → 5000 + 10*1000 = 15000ms */ - ms = cbm_watcher_poll_interval_ms(5000); + ms = cbm_watcher_poll_interval_ms(5000, 0, 0); ASSERT_EQ(ms, 15000); PASS(); } TEST(poll_interval_cap) { /* 100K files → capped at 60s */ - int ms = cbm_watcher_poll_interval_ms(100000); + int ms = cbm_watcher_poll_interval_ms(100000, 0, 0); ASSERT_EQ(ms, 60000); PASS(); } TEST(poll_interval_small) { /* 499 files → 5000 + 0*1000 = 5000ms (integer division) */ - int ms = cbm_watcher_poll_interval_ms(499); + int ms = cbm_watcher_poll_interval_ms(499, 0, 0); ASSERT_EQ(ms, 5000); /* 500 files → 5000 + 1*1000 = 6000ms */ - ms = cbm_watcher_poll_interval_ms(500); + ms = cbm_watcher_poll_interval_ms(500, 0, 0); ASSERT_EQ(ms, 6000); PASS(); } @@ -215,7 +215,7 @@ TEST(watcher_stop_flag) { cbm_watcher_stop(w); /* Run should return immediately */ - int rc = cbm_watcher_run(w, 1000); + int rc = cbm_watcher_run(w, 1000, 0); ASSERT_EQ(rc, 0); cbm_watcher_free(w); @@ -580,7 +580,7 @@ TEST(watcher_poll_interval_full_table) { }; int n = (int)(sizeof(tests) / sizeof(tests[0])); for (int i = 0; i < n; i++) { - int got = cbm_watcher_poll_interval_ms(tests[i].files); + int got = cbm_watcher_poll_interval_ms(tests[i].files, 0, 0); if (got != tests[i].expected_ms) { fprintf(stderr, "FAIL pollInterval(%d) = %d, want %d\n", tests[i].files, got, tests[i].expected_ms);