From 02bc3b735272e14ae062068160bd3c19bf46ab30 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 1 Mar 2026 18:01:52 +0100 Subject: [PATCH] Release v0.3.0: 25 languages, install/update CLI, skills, benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Language expansion (13 → 25): Add Ruby, C, Bash, Zig, Elixir, Haskell, OCaml, HTML, CSS, YAML, TOML, HCL with tree-sitter grammars and language specs. Restore Erlang and SQL with extraction fixes. CLI install/update/uninstall commands: Auto-detect Claude Code and Codex CLI, register MCP server, install task-specific skills, self-update with SHA-256 verification. Embedded skills (4): exploring, tracing, quality, reference — auto-trigger in Claude Code for graph-first code discovery. Pipeline improvements: Docstring extraction, enrichment pass for params/returns/complexity, graceful context cancellation, improved test detection. 35-language benchmark (BENCHMARK.md): 12 questions × 35 repos, 91.8% overall score, Linux kernel stress test (20K nodes, zero timeouts). Replaces old benchmark artifacts. --- .github/workflows/release.yml | 5 +- .golangci.yml | 106 +- BENCHMARK.md | 887 +++++++ README.md | 141 +- cmd/codebase-memory-mcp/assets.go | 29 + .../assets/codex-instructions.md | 34 + .../skills/codebase-memory-exploring/SKILL.md | 90 + .../skills/codebase-memory-quality/SKILL.md | 101 + .../skills/codebase-memory-reference/SKILL.md | 110 + .../skills/codebase-memory-tracing/SKILL.md | 96 + cmd/codebase-memory-mcp/cli_test.go | 263 +++ cmd/codebase-memory-mcp/install.go | 386 ++++ cmd/codebase-memory-mcp/install_test.go | 304 +++ cmd/codebase-memory-mcp/main.go | 23 +- cmd/codebase-memory-mcp/update.go | 234 ++ cmd/codebase-memory-mcp/update_test.go | 104 + go.mod | 45 + go.sum | 44 +- internal/cypher/executor.go | 13 +- internal/discover/discover.go | 13 +- internal/discover/discover_test.go | 61 + internal/httplink/httplink.go | 665 +++++- internal/httplink/httplink_test.go | 442 ++++ internal/httplink/langparity_test.go | 93 + internal/lang/bash.go | 17 + internal/lang/c_lang.go | 21 + internal/lang/cpp.go | 6 +- internal/lang/csharp.go | 2 + internal/lang/css.go | 15 + internal/lang/dart.go | 29 + internal/lang/dockerfile.go | 18 + internal/lang/elixir.go | 25 + internal/lang/erlang.go | 22 + internal/lang/go_lang.go | 2 +- internal/lang/groovy.go | 26 + internal/lang/haskell.go | 32 + internal/lang/hcl.go | 17 + internal/lang/html.go | 15 + internal/lang/java.go | 1 + internal/lang/kotlin.go | 7 + internal/lang/lang.go | 33 +- internal/lang/lang_test.go | 8 +- internal/lang/lua.go | 5 +- internal/lang/objc.go | 25 + internal/lang/ocaml.go | 28 + internal/lang/perl.go | 23 + internal/lang/php.go | 9 +- internal/lang/r.go | 22 + internal/lang/r_lang.go | 1 + internal/lang/ruby.go | 22 + internal/lang/rust.go | 1 - internal/lang/scala.go | 7 +- internal/lang/scss.go | 19 + internal/lang/sql.go | 21 + internal/lang/swift.go | 29 + internal/lang/toml_lang.go | 15 + internal/lang/yaml_lang.go | 15 + internal/lang/zig.go | 22 + internal/parser/parser.go | 47 + internal/pipeline/astdump_test.go | 369 +++ internal/pipeline/cancel_integration_test.go | 48 + internal/pipeline/decorates.go | 107 + internal/pipeline/docstrings.go | 220 ++ internal/pipeline/docstrings_test.go | 346 +++ internal/pipeline/enrichment.go | 596 ++++- internal/pipeline/enrichment_test.go | 261 +++ internal/pipeline/githistory.go | 6 +- internal/pipeline/infrascan_test.go | 11 +- internal/pipeline/langparity_test.go | 2040 +++++++++++++++++ internal/pipeline/pipeline.go | 952 +++++++- internal/pipeline/pipeline_bench_test.go | 9 +- internal/pipeline/pipeline_test.go | 211 +- internal/pipeline/readwrite.go | 526 ++++- internal/pipeline/resolve.go | 71 + internal/pipeline/resolve_test.go | 51 +- internal/pipeline/testdetect.go | 164 +- internal/pipeline/testdetect_test.go | 73 + internal/pipeline/throws.go | 127 +- internal/pipeline/usages.go | 8 +- internal/pipeline/usages_test.go | 9 +- internal/selfupdate/selfupdate.go | 195 ++ internal/selfupdate/selfupdate_test.go | 163 ++ internal/store/migrate.go | 25 +- internal/store/search.go | 171 +- internal/store/store.go | 39 +- internal/store/store_test.go | 5 +- internal/tools/index.go | 6 +- internal/tools/search.go | 1 + internal/tools/tools.go | 27 +- internal/tools/update_test.go | 4 +- internal/watcher/watcher.go | 8 +- internal/watcher/watcher_test.go | 8 +- scripts/hooks/pre-commit | 21 + scripts/setup.sh | 7 + 94 files changed, 11132 insertions(+), 649 deletions(-) create mode 100644 BENCHMARK.md create mode 100644 cmd/codebase-memory-mcp/assets.go create mode 100644 cmd/codebase-memory-mcp/assets/codex-instructions.md create mode 100644 cmd/codebase-memory-mcp/assets/skills/codebase-memory-exploring/SKILL.md create mode 100644 cmd/codebase-memory-mcp/assets/skills/codebase-memory-quality/SKILL.md create mode 100644 cmd/codebase-memory-mcp/assets/skills/codebase-memory-reference/SKILL.md create mode 100644 cmd/codebase-memory-mcp/assets/skills/codebase-memory-tracing/SKILL.md create mode 100644 cmd/codebase-memory-mcp/cli_test.go create mode 100644 cmd/codebase-memory-mcp/install.go create mode 100644 cmd/codebase-memory-mcp/install_test.go create mode 100644 cmd/codebase-memory-mcp/update.go create mode 100644 cmd/codebase-memory-mcp/update_test.go create mode 100644 internal/discover/discover_test.go create mode 100644 internal/httplink/langparity_test.go create mode 100644 internal/lang/bash.go create mode 100644 internal/lang/c_lang.go create mode 100644 internal/lang/css.go create mode 100644 internal/lang/dart.go create mode 100644 internal/lang/dockerfile.go create mode 100644 internal/lang/elixir.go create mode 100644 internal/lang/erlang.go create mode 100644 internal/lang/groovy.go create mode 100644 internal/lang/haskell.go create mode 100644 internal/lang/hcl.go create mode 100644 internal/lang/html.go create mode 100644 internal/lang/objc.go create mode 100644 internal/lang/ocaml.go create mode 100644 internal/lang/perl.go create mode 100644 internal/lang/r.go create mode 100644 internal/lang/r_lang.go create mode 100644 internal/lang/ruby.go create mode 100644 internal/lang/scss.go create mode 100644 internal/lang/sql.go create mode 100644 internal/lang/swift.go create mode 100644 internal/lang/toml_lang.go create mode 100644 internal/lang/yaml_lang.go create mode 100644 internal/lang/zig.go create mode 100644 internal/pipeline/astdump_test.go create mode 100644 internal/pipeline/cancel_integration_test.go create mode 100644 internal/pipeline/docstrings.go create mode 100644 internal/pipeline/docstrings_test.go create mode 100644 internal/pipeline/enrichment_test.go create mode 100644 internal/pipeline/langparity_test.go create mode 100644 internal/pipeline/testdetect_test.go create mode 100644 internal/selfupdate/selfupdate.go create mode 100644 internal/selfupdate/selfupdate_test.go create mode 100755 scripts/hooks/pre-commit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee046388a..55df97ea4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,10 +25,9 @@ jobs: with: go-version: "1.26" - - uses: golangci/golangci-lint-action@v6 + - uses: golangci/golangci-lint-action@v7 with: - version: latest - install-mode: goinstall + version: v2.10 args: --timeout=5m build-unix: diff --git a/.golangci.yml b/.golangci.yml index 274ea3329..635fea2aa 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,58 +1,72 @@ +version: "2" run: - timeout: 5m - go: "1.23" - + go: "1.26" linters: enable: - - errcheck - - govet - - staticcheck - bodyclose - - nilerr - - sqlclosecheck - - rowserrcheck - - durationcheck - - makezero - - noctx - - gofmt - - goimports - - misspell - - unconvert - - unparam - - unused - - ineffassign - - gosimple - - prealloc - cyclop + - durationcheck - funlen - gocognit - gocritic - - nestif - gosec - -linters-settings: - cyclop: - max-complexity: 25 - funlen: - lines: 120 - statements: 80 - gocognit: - min-complexity: 30 - nestif: - min-complexity: 6 - gocritic: - enabled-tags: - - diagnostic - - style - - performance - gosec: - excludes: - - G404 - errcheck: - check-type-assertions: true - + - makezero + - misspell + - nestif + - nilerr + - noctx + - prealloc + - rowserrcheck + - sqlclosecheck + - unconvert + - unparam + settings: + cyclop: + max-complexity: 25 + errcheck: + check-type-assertions: true + funlen: + lines: 120 + statements: 80 + gocognit: + min-complexity: 30 + gocritic: + enabled-tags: + - diagnostic + - style + - performance + gosec: + excludes: + - G115 + - G404 + - G704 + - G705 + - G706 + nestif: + min-complexity: 6 + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - vendor + - third_party$ + - builtin$ + - examples$ issues: max-issues-per-linter: 0 max-same-issues: 0 - exclude-dirs: - - vendor +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - vendor + - third_party$ + - builtin$ + - examples$ diff --git a/BENCHMARK.md b/BENCHMARK.md new file mode 100644 index 000000000..4624ff32f --- /dev/null +++ b/BENCHMARK.md @@ -0,0 +1,887 @@ +# Codebase Memory MCP -- v0.2.1 Language Benchmark + +## Methodology + +- **35 languages** (27 programming + 8 config/markup), 12 questions each (4 for config languages) +- **Up to 5 attempts** per question with escalating retry strategies +- **Real open-source repos** (medium to large: 78--49K nodes) +- **Grading**: PASS (1.0) / PARTIAL (0.5) / FAIL (0.0), N/A excluded from denominator +- **Platform**: Apple M3 Pro, macOS Darwin 25.3.0 +- **Date**: 2026-03-01 + +### The 12 Questions + +| # | Category | Question | Primary MCP Tool | +|---|----------|----------|------------------| +| Q1 | Indexing | Verify index stats (nodes, edges, schema) | `get_graph_schema` | +| Q2 | Discovery | Find functions/methods | `search_graph(label="Function")` | +| Q3 | Discovery | Find classes/structs | `search_graph(label="Class")` | +| Q4 | Pattern | Find by name pattern | `search_graph(name_pattern="...")` | +| Q5 | Code | Get code snippet | `get_code_snippet` | +| Q6 | Search | Text search in code | `search_code` | +| Q7 | Trace | Outbound call trace | `trace_call_path(direction="outbound")` | +| Q8 | Trace | Inbound call trace | `trace_call_path(direction="inbound")` | +| Q9 | Graph | Cypher CALLS query | `query_graph` | +| Q10 | Enrich | Params/returns/properties | `query_graph` | +| Q11 | OOP | Inheritance/implements | `query_graph` | +| Q12 | Files | List files & directories | `list_directory` | + +Config/markup languages (HTML, CSS, SCSS, YAML, TOML, HCL, SQL, Dockerfile) run Q1, Q4, Q6, Q12. Other questions marked N/A. + +--- + +## Summary Table + +| # | Language | Repo | Nodes | Edges | Score | Pct | Tier | +|---|----------|------|------:|------:|------:|----:|------| +| 1 | Lua | neovim/neovim | 23,955 | 90,247 | 12/12 | 100% | 1 | +| 2 | Kotlin | ktorio/ktor | 25,297 | 71,498 | 12/12 | 100% | 1 | +| 3 | C++ | nlohmann/json | 5,262 | 8,681 | 12/12 | 100% | 1 | +| 4 | Perl | mojolicious/mojo | 3,287 | 4,182 | 12/12 | 100% | 1 | +| 5 | Objective-C | AFNetworking | 1,087 | 1,348 | 12/12 | 100% | 1 | +| 6 | Groovy | spockframework/spock | 14,081 | 34,557 | 12/12 | 100% | 1 | +| 7 | C | jqlang/jq | 1,330 | 3,923 | 11/11 | 100% | 1 | +| 8 | Bash | bats-core/bats-core | 436 | 479 | 10/10 | 100% | 1 | +| 9 | Zig | zigtools/zls | 2,824 | 10,230 | 10/10 | 100% | 1 | +| 10 | Swift | Alamofire/Alamofire | 3,631 | 7,639 | 10.5/11 | 95% | 1 | +| 11 | CSS | animate-css/animate.css | 295 | 214 | 4/4 | 100% | 1 | +| 12 | YAML | kubernetes/examples | 2,110 | 1,844 | 4/4 | 100% | 1 | +| 13 | TOML | rust-lang/cargo | 16,773 | 51,403 | 4/4 | 100% | 1 | +| 14 | HTML | twbs/bootstrap | 2,726 | 4,135 | 4/4 | 100% | 1 | +| 15 | SCSS | twbs/bootstrap | 2,726 | 4,135 | 4/4 | 100% | 1 | +| 16 | HCL | hashicorp/terraform | 78 | 76 | 4/4 | 100% | 1 | +| 17 | Dockerfile | docker-library/official-images | 1,481 | 1,588 | 4/4 | 100% | 1 | +| 18 | Python | django/django | 49,398 | 196,022 | 10.5/12 | 87% | 2 | +| 19 | TypeScript | nestjs/nest | 9,063 | 15,772 | 10.5/12 | 87% | 2 | +| 20 | TSX | shadcn-ui/ui | 29,755 | 41,883 | 10.5/12 | 87% | 2 | +| 21 | Go | codebase-memory-mcp (self) | 2,259 | 6,561 | 10.5/12 | 87% | 2 | +| 22 | Rust | BurntSushi/ripgrep | 4,118 | 6,971 | 10.5/12 | 87% | 2 | +| 23 | Java | spring-projects/spring-petclinic | 660 | 1,080 | 10.5/12 | 87% | 2 | +| 24 | R | tidyverse/dplyr | 1,618 | 2,409 | 10.5/12 | 87% | 2 | +| 25 | Dart | felangel/bloc | 5,089 | 6,430 | 10.5/12 | 87% | 2 | +| 26 | JavaScript | lodash/lodash | 244 | 405 | 9.5/11 | 86% | 2 | +| 27 | Erlang | ninenines/cowboy | 3,270 | 9,815 | 9.5/11 | 86% | 2 | +| 28 | Elixir | elixir-plug/plug | 870 | 865 | 9.5/11 | 86% | 2 | +| 29 | Scala | playframework/playframework | 19,627 | 43,764 | 9/12 | 75% | 2 | +| 30 | Ruby | sinatra/sinatra | 1,377 | 1,893 | 9/12 | 75% | 2 | +| 31 | PHP | laravel/framework | 38,644 | 161,242 | 9/12 | 75% | 2 | +| 32 | C# | jasontaylordev/CleanArchitecture | 1,043 | 1,632 | 9/12 | 75% | 2 | +| 33 | SQL | flyway/flyway | 4.5/6 | 75% | 4.5/6 | 75% | 2 | +| 34 | OCaml | ocaml/dune | 11,691 | 10,447 | 8/11 | 72% | 3 | +| 35 | Haskell | PostgREST/postgrest | 2,066 | 2,463 | 7.5/12 | 62% | 3 | + +--- + +## Tier Classification + +| Tier | Range | Count | Languages | +|------|-------|------:|-----------| +| **Tier 1** -- Excellent | >= 90% | 17 | Lua, Kotlin, C++, Perl, Objective-C, Groovy, C, Bash, Zig, Swift, CSS, YAML, TOML, HTML, SCSS, HCL, Dockerfile | +| **Tier 2** -- Good | 75--89% | 16 | Python, TypeScript, TSX, Go, Rust, Java, R, Dart, JavaScript, Erlang, Elixir, Scala, Ruby, PHP, C#, SQL | +| **Tier 3** -- Functional | < 75% | 2 | OCaml (72%), Haskell (62%) | + +--- + +## Per-Language Results + +### Python (django/django) + +**Project**: `django-python` | **Repo**: `/tmp/lang-bench/django-python` +**Nodes**: 49,398 | **Edges**: 196,022 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 49,398 nodes, 196,022 edges, 12 labels, 20 rel types | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | render_to_string (1458 in), skipUnlessDBFeature, call_command. 1005 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | ModelAdmin (511 in), ValidationError (415 in), ImproperlyConfigured. 1005 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | SimpleTestCase, TestCase, SchemaTests. 22,135 total | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | render_to_string: 11 lines from django/template/loader.py:52-62 | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO\|FIXME) | 5 matches: FIXME in admin/checks.py, TODO in admin/options.py | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | 21 edges, 3 hops: render_to_string -> render, get_template, select_template | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 3 callers: test_include, test_include_state, test_include_cache | +| Q9 | Cypher CALLS | PASS | 3/5 | query_graph with project param | First 2 attempts needed project param adjustment | +| Q10 | Properties | PARTIAL | 3/5 | query_graph(properties) | Functions returned but all properties null | +| Q11 | Inheritance | PASS | 3/5 | query_graph(INHERITS) | 200 rows (capped): LazySettings, SimpleAdminConfig, ModelAdmin | +| Q12 | List Directory | PASS | 1/5 | list_directory | 21 entries: django/, docs/, tests/, extras/, pyproject.toml | + +**Score: 10.5/12 (87%)** + +### JavaScript (lodash/lodash) + +**Project**: `lodash-js` | **Repo**: `/tmp/lang-bench/lodash-js` +**Nodes**: 244 | **Edges**: 405 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 244 nodes, 405 edges, 7 labels | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | build (7 variants), baseConvert, Hash. 36 total | +| Q3 | Find Classes | PASS | 2/5 | search_graph(Module) | No Class nodes (functional lib). Module fallback: 39 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | browser-testing.yml, markdown-doctest-setup.js. 10 total | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | build: 57 lines from lib/main/build-site.js | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 1 match: TODO in .github/workflows/ci-bun.yml | +| Q7 | Outbound Trace | PASS | 2/5 | trace_call_path(baseConvert) | 2 edges: baseAry, self-recursive | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 1 caller: build-dist.js module | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | 5 rows: minify->minify, build-dist.js->build | +| Q10 | Properties | PARTIAL | 1/5 | query_graph(properties) | Functions returned but properties null | +| Q11 | Inheritance | N/A | -- | -- | No classes/inheritance (pure functional library) | +| Q12 | List Directory | PASS | 1/5 | list_directory | 19 entries: dist/, fp/, lib/, test/, lodash.js | + +**Score: 9.5/11 (86%)** + +### TypeScript (nestjs/nest) + +**Project**: `nestjs-ts` | **Repo**: `/tmp/lang-bench/nestjs-ts` +**Nodes**: 9,063 | **Edges**: 15,772 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 9,063 nodes, 15,772 edges; 14 labels | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | Injectable (in=242), Module (in=209), Controller (in=117). 292 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Post (in=128), FastifyAdapter, Module, InstanceWrapper. 1005+ total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | TestingModuleBuilder, TestController, test folders. 142 results | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | Injectable: 6 lines from common/decorators/core/injectable.decorator.ts | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 2 matches in express.spec.ts, gulp/tasks/samples.ts | +| Q7 | Outbound Trace | PASS | 2/5 | trace_call_path(Module) | First try Injectable had 0 outbound; Module: 4 edges | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(Injectable) | 242 callers across services, interceptors, pipes, guards | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | 5 rows: gulpfile.js->register, bar.service->Injectable | +| Q10 | Properties | PARTIAL | 1/5 | query_graph(properties) | Properties null | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 200 rows: AuthGuard, BadGatewayException, ClientGrpcProxy | +| Q12 | List Directory | PASS | 1/5 | list_directory | 22 entries: packages/, integration/, sample/, tools/ | + +**Score: 10.5/12 (87%)** + +### TSX (shadcn-ui/ui) + +**Project**: `shadcn-tsx` | **Repo**: `/tmp/lang-bench/shadcn-tsx` +**Nodes**: 29,755 | **Edges**: 41,883 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 29,755 nodes, 41,883 edges; 13 labels | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | cn (in=558), IconPlaceholder (in=362). 7,424 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | 18 classes: RegistryError, ComponentErrorBoundary | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 146 results: search.test.ts, registries.test.ts | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet(cn) | 3 lines from apps/v4/examples/base/lib/utils.ts:4-6 | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 5 matches in registry.ts, source.config.ts | +| Q7 | Outbound Trace | PASS | 2/5 | trace_call_path(Button) | Button->cn (1 edge) | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(cn) | 558 callers (result saved to file) | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | BlocksPage->getAllBlockIds, ChartPage->getActiveStyle | +| Q10 | Properties | PARTIAL | 1/5 | query_graph(properties) | Properties null | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 14 rows: RegistryNotFoundError, ConfigMissingError | +| Q12 | List Directory | PASS | 1/5 | list_directory | 17 entries: apps/, packages/, templates/ | + +**Score: 10.5/12 (87%)** + +### Go (codebase-memory-mcp) + +**Project**: `codebase-memory-mcp` | **Repo**: self +**Nodes**: 2,259 | **Edges**: 6,561 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 2,259 nodes, 6,561 edges; 13 labels | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | OpenMemory (in=136), NodeText (in=79), Parse (in=35). 918 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Node (in=243), LanguageSpec, FileInfo, Edge. 91 structs | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 413 results: cypher_test.go, langparity_test.go | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | OpenMemory: 16 lines from internal/store/store.go:113-128 | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 4 matches in tools.go, langparity_test.go | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | 11 edges: Open, Close, initSchema, cacheDir | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 68 callers: test files across store/pipeline/httplink | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | printAST->printAST, main->Parse, TestLexBasicQuery->Lex | +| Q10 | Properties | PARTIAL | 1/5 | query_graph(properties) | Properties null | +| Q11 | Inheritance | PASS | 1/5 | query_graph(IMPLEMENTS) | 3 rows: RelPattern, FilterWhere, fusedExpandMarker | +| Q12 | List Directory | PASS | 1/5 | list_directory | 15 entries: cmd/, internal/, scripts/, go.mod | + +**Score: 10.5/12 (87%)** + +### Rust (BurntSushi/ripgrep) + +**Project**: `ripgrep-rust` | **Repo**: `/tmp/lang-bench/ripgrep-rust` +**Nodes**: 4,118 | **Edges**: 6,971 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 689 Functions, 2039 Methods, 294 Classes, 70 Enums | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | matcher (in=133), args (119), printer_contents (72). 689 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | HiArgs (out=93), LowArgs (out=74), Debug (in=54). 294 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | test method (in=524), SearcherTester, TestCommand. 134 results | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | matcher() from test_matcher.rs:8-10 | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3 TODOs in globset, gitignore, overrides | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(search) | 12 callees: quit_after_match, sort, haystack_builder, search_worker | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(search) | Called by run, main, search_parallel, files_parallel | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | main->set_git_revision_hash, main->set_windows_exe_options | +| Q10 | Properties | PASS | 1/5 | query_graph(properties) | 5 functions returned | +| Q11 | Inheritance | PARTIAL | 1/5 | query_graph(INHERITS) | 1 result: NoCaptures. Rust traits not modeled as inheritance | +| Q12 | List Directory | PASS | 1/5 | list_directory | 20 entries: crates/, tests/, benchsuite/, Cargo.toml | + +**Score: 10.5/12 (87%)** + +### Java (spring-projects/spring-petclinic) + +**Project**: `spring-petclinic-java` | **Repo**: `/tmp/lang-bench/spring-petclinic-java` +**Nodes**: 660 | **Edges**: 1,080 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 167 Methods, 39 Classes, 30 Routes, 3 Interfaces | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Method) | getPet (in=12,out=6), testValidate (out=16). 167 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | OwnerControllerTests, PetController, Owner. 39 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 97 test-related nodes | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | getPet() from Owner.java:135-145 | +| Q6 | Text Search | PARTIAL | 2/5 | search_code(error) | TODO/FIXME returned 0; "error" found 3 matches | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | getPet calls getId, isNew, getName, getPets | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | Called by loadPetWithVisit, findPet, processCreationForm, test methods | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | toString->getName, addPet->isNew, getPet->getPets | +| Q10 | Properties | PASS | 1/5 | query_graph(properties) | main, registerHints, getId, setId, isNew | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 8 results: NamedEntity, Person, Owner, Pet, Vet | +| Q12 | List Directory | PASS | 1/5 | list_directory | 13 entries: src/, gradle/, pom.xml, docker-compose.yml | + +**Score: 10.5/12 (87%)** + +### C++ (nlohmann/json) + +**Project**: `nlohmann-json-cpp` | **Repo**: `/tmp/lang-bench/nlohmann-json-cpp` +**Nodes**: 5,262 | **Edges**: 8,681 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 2,456 Functions, 649 Methods, 603 Macros, 153 Classes | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | result (in=99), push_back (in=42), CAPTURE (in=35) | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Fuzzer (out=54), MutationDispatcher (out=36). 153 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 234 test-related nodes | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | result from unit-bjdata.cpp:2935-2936 | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3 TODOs in binary_reader.hpp, json.hpp | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(push_back) | Recursive call through json_pointer chain | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(push_back) | 23 callers: main, sax_event_consumer, lexer, binary_reader | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | main->at, main->back | +| Q10 | Properties | PASS | 1/5 | query_graph(properties) | 5 functions returned | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 21 results: parse_error, invalid_iterator, type_error, out_of_range | +| Q12 | List Directory | PASS | 1/5 | list_directory | 20 entries: include/, tests/, docs/, CMakeLists.txt | + +**Score: 12/12 (100%)** + +### C# (jasontaylordev/CleanArchitecture) + +**Project**: `clean-architecture-csharp` | **Repo**: `/tmp/lang-bench/clean-architecture-csharp` +**Nodes**: 1,043 | **Edges**: 1,632 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 303 Methods, 123 Classes, 5 Interfaces | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Method) | SendAsync (in=17), PaginatedList (in=11). 303 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | TodoItems (in=18), TodoLists (in=18), BaseTestFixture. 123 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 85 test-related nodes | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | SendAsync() from Testing.cs:38-45 | +| Q6 | Text Search | PARTIAL | 2/5 | search_code(error) | TODO/FIXME=0; "error" found 3 matches | +| Q7 | Outbound Trace | PARTIAL | 1/5 | trace_call_path(outbound) | SendAsync has 0 outbound (calls external ISender.Send) | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 16 callers: test methods | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | NotEqualOperator->EqualOperator, GetHashCode->GetEqualityComponents | +| Q10 | Properties | PASS | 1/5 | query_graph(properties) | AuthorizationBehaviour, Handle, LoggingBehaviour | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 29 results: BaseAuditableEntity, TodoItem, TodoList | +| Q12 | List Directory | PASS | 1/5 | list_directory | 16 entries: src/, tests/, infra/, templates/ | + +**Score: 9/12 (75%)** + +### PHP (laravel/framework) + +**Project**: `laravel-framework-php` | **Repo**: `/tmp/lang-bench/laravel-framework-php` +**Nodes**: 38,644 | **Edges**: 161,242 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 26,984 Methods, 3,546 Classes, 390 Interfaces, 270 Routes | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | value (in=1024), fake (in=878), response (in=712). 143 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Model (in=836,out=155), Blueprint (in=785), Container (in=625) | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | TestCase (in=1370). 14,505 test-related nodes | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | value() from helpers.php:264-267 | +| Q6 | Text Search | PARTIAL | 2/5 | search_code(error) | TODO/FIXME=0; "error" found 3 in workflow YAML | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | value calls first->from->all, toArray, jsonSerialize | +| Q8 | Inbound Trace | PARTIAL | 1/5 | trace_call_path(inbound) | Resolved to method variant (0 callers) instead of helper function | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | app.php->toArray, __construct->__construct | +| Q10 | Properties | PASS | 1/5 | query_graph(properties) | enum_value, collect, data_fill, data_has | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 200 rows (capped): AuthorizationException, Model relations | +| Q12 | List Directory | PASS | 1/5 | list_directory | 16 entries: src/, tests/, config/, bin/ | + +**Score: 9/12 (75%)** + +### Lua (neovim/neovim) + +**Project**: `neovim-lua` | **Repo**: `/tmp/lang-bench/neovim-lua` +**Nodes**: 23,955 | **Edges**: 90,247 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 12,311 Functions, 7,587 Variables, 1,157 Files, 399 Methods, 196 Classes | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | assert (in=825), vim.fn.bufnr (in=390), vim.fn.count (in=228) | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | cmdmod (in=84), file_buffer (out=80), TUIData (out=59). 196 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 198 test-related nodes | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | vim.validate: 21 lines from runtime/lua/vim/_core/shared.lua | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | Found across build.zig, test.yml | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | 5 callees: vim.deprecate, is_valid, validate_spec. Depth 2: 15 more | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 129K chars result (saved to file). Massive inbound | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | f_rpcstart->check_secure, f_termopen->f_jobstart | +| Q10 | Properties | PASS | 1/5 | search_graph(include_connected) | assert: in=825/out=2, connected to M.error, main | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | UGridPrinter inherits object | +| Q12 | List Directory | PASS | 1/5 | list_directory | 22 entries: src/, runtime/, test/, cmake/, deps/ | + +**Score: 12/12 (100%)** + +### Scala (playframework/playframework) + +**Project**: `playframework-scala` | **Repo**: `/tmp/lang-bench/playframework-scala` +**Nodes**: 19,627 | **Edges**: 43,764 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 8,018 Methods, 2,924 Variables, 2,007 Classes, 309 Interfaces | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | app (in=48), queryString (in=14), json (in=9). 71 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Http (out=135), AbstractController (in=81), Controller (in=68) | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 636 test-related nodes | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | app: 1-line abstract def from play/api/test/package.scala | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | FIXME in CachedSpec, TODO in CryptoComponents | +| Q7 | Outbound Trace | PARTIAL | 1/5 | trace_call_path(outbound) | queryString: abstract method, no outbound calls | +| Q8 | Inbound Trace | PARTIAL | 1/5 | trace_call_path(inbound) | queryString: no inbound calls detected | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | fromContent->fromString, waitForCancel->waitForKey | +| Q10 | Properties | PASS | 1/5 | search_graph(include_connected) | ok: in=133/out=6, data: in=131/out=1 | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 200 rows (capped): DefaultApplication, Controller, many *Spec | +| Q12 | List Directory | PASS | 1/5 | list_directory | 14 entries: core/, dev-mode/, documentation/, testkit/ | + +**Score: 9/12 (75%)** + +### Kotlin (ktorio/ktor) + +**Project**: `ktor-kotlin` | **Repo**: `/tmp/lang-bench/ktor-kotlin` +**Nodes**: 25,297 | **Edges**: 71,498 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 8,485 Methods, 2,865 Functions, 2,572 Classes, 1,921 Routes | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | request (in=216), content (in=186), http (in=171). 1010+ total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | ContentType (in=224), HttpRequestBuilder (in=208). 1010+ total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | testApplication (in=1504), runTest (in=359). 5,015 total | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | request: 2 lines from ktor-client-core | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | "TODO: Remove when PR fixing this file will be merged" | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(HttpClient) | 4 callees: create, apply, invokeOnCompletion | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(HttpClient) | 3 callers: config, testPluginInstalledTwice | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | getNative->error, matches->toLowerCasePreservingASCIIRules | +| Q10 | Properties | PASS | 1/5 | search_graph(include_connected) | request: in=216/out=10, content: in=186/out=6 | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 200 rows (capped): HttpClientEngineBase, many *Engine | +| Q12 | List Directory | PASS | 1/5 | list_directory | 39 entries: ktor-client/, ktor-server/, ktor-http/ | + +**Score: 12/12 (100%)** + +### Ruby (sinatra/sinatra) + +**Project**: `sinatra-ruby` | **Repo**: `/tmp/lang-bench/sinatra-ruby` +**Nodes**: 1,377 | **Edges**: 1,893 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 786 Methods, 127 Classes, 37 Functions | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | setup (in=11), route_def, write_app_file. 37 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Sinatra (out=172), OkJson (out=31). 127 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 183 test-related nodes | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | setup: 4 lines from test/markdown_test.rb | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 2 matches: base.rb, content_for_spec.rb | +| Q7 | Outbound Trace | PARTIAL | 1/5 | trace_call_path(outbound) | setup in Minitest: no outbound calls (block-based) | +| Q8 | Inbound Trace | PARTIAL | 1/5 | trace_call_path(inbound) | No inbound calls for this method | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | status_app->mock_app, mock_app->new, setup_blocks->each | +| Q10 | Properties | PASS | 1/5 | search_graph(include_connected) | it: in=38/out=3, get: in=23/out=1 | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 2 results: BError->AError, ErubiTest->ERBTest | +| Q12 | List Directory | PASS | 1/5 | list_directory | 18 entries: lib/, test/, examples/ | + +**Score: 9/12 (75%)** + +### C (jqlang/jq) + +**Project**: `jq-c` | **Repo**: `/tmp/lang-bench/jq-c` +**Nodes**: 1,330 | **Edges**: 3,923 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 764 Functions, 159 Fields, 75 Variables, 45 Classes (structs) | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | jv_free (in=163), jv_get_kind (in=97), yyparse (out=51). 764 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | jq_state (out=25), yyguts_t (out=24), jv_parser (out=21). 45 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 25 test-related nodes: run_jq_tests, jv_test, LLVMFuzzerTestOneInput | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | jv_free: 1-line declaration from src/jv.h:55 | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 7 matches: build_manpage.py, execute.c, jv_parse.c | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | 5 callees: jvp_invalid/number/array/string/object_free. Depth 2: jvp_refcnt_dec | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 87K chars (saved to file). Massive caller tree | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | f_negate->type_error, f_json_parse->type_error, f_length->type_error | +| Q10 | Properties | PASS | 1/5 | search_graph(include_connected) | jv_free: in=163/out=6, jv_get_kind: in=97/out=1 | +| Q11 | Inheritance | N/A | -- | -- | C has no class inheritance | +| Q12 | List Directory | PASS | 1/5 | list_directory | 23 entries: src/, tests/, docs/, build/ | + +**Score: 11/11 (100%)** + +### Bash (bats-core/bats-core) + +**Project**: `bats-bash` | **Repo**: `/tmp/lang-bench/bats-bash` +**Nodes**: 436 | **Edges**: 479 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 156 Functions, 65 Variables, 66 Modules | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | bats_print_stack_trace, main, run, bats_generate_warning. 156 total | +| Q3 | Find Classes | N/A | -- | -- | Bash has no classes | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=run) | 6 matches: run, bats_semaphore_run, reentrant_run | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | run: 142-line function source (310-451) | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 2 TODOs in semaphore.bash, tracing.bash | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | 6 callees: bats_quote_code, bats_format_file_line_reference | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 3 callers across 2 hops | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | 10 rows of function call pairs | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 65 variables | +| Q11 | Inheritance | N/A | -- | -- | Bash has no class hierarchy | +| Q12 | List Directory | PASS | 1/5 | list_directory | 66 modules found | + +**Score: 10/10 (100%)** + +### Zig (zigtools/zls) + +**Project**: `zls-zig` | **Repo**: `/tmp/lang-bench/zls-zig` +**Nodes**: 2,824 | **Edges**: 10,230 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 1,010+ Functions, 1,005 Variables, 109 Modules | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | std.ArrayList, trace, empty, resolveTypeOfNodeUncached. 1010+ total | +| Q3 | Find Classes | N/A | -- | -- | Zig uses structs, not classes | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=resolve) | 57 matches: resolve, resolveTypeOfNodeUncached | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | trace: 19-line function showing tracy integration | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3+ TODOs in DiagnosticsCollection.zig, DocumentStore.zig | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | 2 callees: ___tracy_emit_zone_begin, _callstack | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 94 callers: processMessage, create, handleConfiguration | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | main->createRequestBody, build->getVersion | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 1,005 variables | +| Q11 | Inheritance | N/A | -- | -- | Zig has no inheritance | +| Q12 | List Directory | PASS | 1/5 | list_directory | 109 modules | + +**Score: 10/10 (100%)** + +### Elixir (elixir-plug/plug) + +**Project**: `plug-elixir` | **Repo**: `/tmp/lang-bench/plug-elixir` +**Nodes**: 870 | **Edges**: 865 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 554 Functions, 129 Classes, 80 Modules | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | parse_before_param, parse_hd_before_value. 554 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Plug.Conn, Plug.Debugger, Plug.CSRFProtection. 129 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=parse) | 29 matches | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | match: 3-line macro source | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3 TODOs in adapters/test/conn.ex, ssl.ex | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | 6 edges across 2 hops | +| Q8 | Inbound Trace | PARTIAL | 1/5 | trace_call_path(inbound) | 0 callers (entry point function) | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | parse_headers->match, skip_preamble->before_parse_headers | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 6 variables | +| Q11 | Inheritance | N/A | -- | -- | Elixir uses protocols/behaviours | +| Q12 | List Directory | PASS | 1/5 | list_directory | 80 modules | + +**Score: 9.5/11 (86%)** + +### Haskell (PostgREST/postgrest) + +**Project**: `postgrest-haskell` | **Repo**: `/tmp/lang-bench/postgrest-haskell` +**Nodes**: 2,066 | **Edges**: 2,463 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 746 Functions, 148 Classes, 655 Variables | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | defaultenv, jwtauthheader, headers. 746 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Thread, PostgrestTimedOut, ErrorBody. 148 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=action) | 4 matches: actionResult, actionPlan, actionResponse | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | actionPlan: 7-line Haskell source with pattern matching | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3+ TODOs in workflow files | +| Q7 | Outbound Trace | PARTIAL | 1/5 | trace_call_path(outbound) | 0 callees (function composition not traced as CALLS) | +| Q8 | Inbound Trace | PARTIAL | 1/5 | trace_call_path(inbound) | 0 callers (Haskell call patterns not fully traced) | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | generate_jwt->encode, run_command->run, main->generate_jwt | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 655 variables | +| Q11 | Inheritance | PARTIAL | 1/5 | query_graph(INHERITS) | 1 result: Thread (Python test helper). Typeclasses not modeled | +| Q12 | List Directory | PASS | 1/5 | list_directory | 184 modules | + +**Score: 7.5/12 (62%)** + +### OCaml (ocaml/dune) + +**Project**: `dune-ocaml` | **Repo**: `/tmp/lang-bench/dune-ocaml` +**Nodes**: 11,691 | **Edges**: 10,447 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 1,010+ Functions, 12 Classes, 1,005 Modules | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | value, unit, append_files_in_dir_if_not_empty. 1010+ total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | fsenv, watch, CramLexer, DuneLexer. 12 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=build) | 111 matches | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | value: 4-line OCaml source with match expression | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3+ TODOs in workflow.yml, client.ml | +| Q7 | Outbound Trace | PARTIAL | 1/5 | trace_call_path(outbound) | 0 callees (pure function) | +| Q8 | Inbound Trace | PARTIAL | 1/5 | trace_call_path(inbound) | 0 callers (OCaml call analysis limited) | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | dune_trace_write->write, add_watch->isprefix | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 103 variables | +| Q11 | Inheritance | N/A | -- | -- | OCaml uses modules/functors | +| Q12 | List Directory | PASS | 1/5 | list_directory | 1,005 modules | + +**Score: 8/11 (72%)** + +### Erlang (ninenines/cowboy) + +**Project**: `cowboy-erlang` | **Repo**: `/tmp/lang-bench/cowboy-erlang` +**Nodes**: 3,270 | **Edges**: 9,815 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 3,270 nodes, 9,815 edges; 8 labels | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | send (in=379), gun_open (in=278), do_get (in=103). 2,739 total | +| Q3 | Find Classes | PASS | 2/5 | search_graph(Module) | No Class nodes; Module fallback: 193 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 29 results | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | send: 2 lines from src/cowboy_http3.erl | +| Q6 | Text Search | PASS | 2/5 | search_code(error) | TODO/FIXME=0; "error" found 5 matches | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | send->send (self-recursive) | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 379 callers (saved to file) | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | start->compile, start->start_clear, stop->stop_listener | +| Q10 | Properties | PARTIAL | 1/5 | query_graph(properties) | Properties null | +| Q11 | Inheritance | N/A | -- | -- | Erlang has no class inheritance | +| Q12 | List Directory | PASS | 1/5 | list_directory | 12 entries: src/, test/, examples/ | + +**Score: 9.5/11 (86%)** + +### R (tidyverse/dplyr) + +**Project**: `dplyr-r` | **Repo**: `/tmp/lang-bench/dplyr-r` +**Nodes**: 1,618 | **Edges**: 2,409 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 1,027 Functions, 13 Classes, 168 test nodes | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | arrange.data.frame, group_by, filter, mutate, select. 1,027 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | rlang_api_ptrs_t, Expander, FactorExpander. 13 total (C++ structs) | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 168 results: testthat.R, test-across.R | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | arrange.data.frame: 10 lines from R/arrange.R:88-97 | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 5 matches in R/across.R about deprecation | +| Q7 | Outbound Trace | PASS | 3/5 | trace_call_path(outbound) | C++ func found 2 edges (grouped/ungrouped) | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 3 callers: test files | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | context.R->new_environment, starwars.R->get_all/lookup | +| Q10 | Properties | PARTIAL | 1/5 | query_graph(properties) | Properties null | +| Q11 | Inheritance | PASS | 1/5 | query_graph(INHERITS) | 3 rows: FactorExpander, VectorExpander, LeafExpander inherit Expander | +| Q12 | List Directory | PASS | 1/5 | list_directory | 21 entries: R/, src/, tests/, vignettes/ | + +**Score: 10.5/12 (87%)** + +### Objective-C (AFNetworking/AFNetworking) + +**Project**: `afnetworking-objc` | **Repo**: `/tmp/lang-bench/afnetworking-objc` +**Nodes**: 1,087 | **Edges**: 1,348 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(File) | 89 files indexed | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | AFNetworkReachabilityCallback, data, destination. 114 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | AFSecurityPolicyTests, AFHTTPRequestSerializer. 80 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=Session) | 26 results | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | AFNetworkReachabilityCallback: 3-line function | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 1 match in AFImageDownloader.m | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(outbound) | 2 edges: Callback->PostReachabilityStatusChange->StatusForFlags | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | Callers traced | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | Function->Function edges present | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 32 variables | +| Q11 | Inheritance | PASS | 1/5 | search_graph(Method) | 632 methods extracted | +| Q12 | List Directory | PASS | 1/5 | list_directory | 17 entries: AFNetworking/, Tests/, UIKit+AFNetworking/ | + +**Score: 12/12 (100%)** + +### Swift (Alamofire/Alamofire) + +**Project**: `alamofire-swift` | **Repo**: `/tmp/lang-bench/alamofire-swift` +**Nodes**: 3,631 | **Edges**: 7,639 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(File) | 470 files indexed | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | write, read, responseStream, download. 117 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Session, Result, URLEncodedFormEncoder, AFError. 371 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=Request) | 480 results | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | write: 3-line function | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3 matches in Session.swift, SessionDelegate.swift | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(write) | 70KB+ result: 58+ callers | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | Rich caller chain | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | didCreateURLRequest->task, finish->finish | +| Q10 | Properties | PARTIAL | 1/5 | search_graph(Variable) | 0 Variables, 817 Fields (Swift properties as Fields) | +| Q11 | Inheritance | PASS | 1/5 | search_graph(Module) | 449 modules | +| Q12 | List Directory | PASS | 1/5 | list_directory | 19 entries: Source/, Tests/, Documentation/ | + +**Score: 10.5/11 (95%)** + +### Dart (felangel/bloc) + +**Project**: `bloc-dart` | **Repo**: `/tmp/lang-bench/bloc-dart` +**Nodes**: 5,089 | **Edges**: 6,430 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(File) | 900 files indexed | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | identical, downloadFile, analyzeDependencies. 479 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Bloc, Cubit, BlocObserver, HydratedCubit, MockBloc. 846 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=Bloc) | 268 results | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet(Bloc) | 278-line class source | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3 matches in gradle, kotlin, dart | +| Q7 | Outbound Trace | PARTIAL | 1/5 | trace_call_path(identical) | 0 edges (built-in function reference, leaf node) | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | Callers traced | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | generateCubitCode->get, getCubitTemplate->getFreezedCubitTemplate | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 592 variables | +| Q11 | Inheritance | PASS | 1/5 | search_graph(Method) | 873 methods, 14 INHERITS edges | +| Q12 | List Directory | PASS | 1/5 | list_directory | 11 entries: packages/, examples/, extensions/ | + +**Score: 10.5/12 (87%)** + +### Perl (mojolicious/mojo) + +**Project**: `mojo-perl` | **Repo**: `/tmp/lang-bench/mojo-perl` +**Nodes**: 3,287 | **Edges**: 4,182 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(File) | 192 files indexed | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | join, split, warn, decode, encode, render. 1,005 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | 0 classes (expected -- Perl uses packages) | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=render) | 26 results | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet(join) | 3-line sub returned | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 1 match in bundled highlight.js | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(join) | 114 results, rich 2-hop graph | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | 40+ callers: websocket, camelize, tablify | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | 491 CALLS edges: run->getopt, remote_address->split | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 1,003+ variables | +| Q11 | Inheritance | PASS | 1/5 | search_graph(Method) | 0 methods (expected -- Perl subs are Functions) | +| Q12 | List Directory | PASS | 1/5 | list_directory | 9 entries: lib/, t/, examples/, script/ | + +**Score: 12/12 (100%)** + +### Groovy (spockframework/spock) + +**Project**: `spock-groovy` | **Repo**: `/tmp/lang-bench/spock-groovy` +**Nodes**: 14,081 | **Edges**: 34,557 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(File) | 1,001 files (+ 284 unlisted) | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | cleanup, setup, println. 185 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | Specification, Issue, FailsWith. 1,005 total | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=Spec) | 802 results | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | isSatisfiedBy: interface method signature | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3+ matches in BuilderHelper.java, PojoBuilder.java | +| Q7 | Outbound Trace | PASS | 1/5 | trace_call_path(isSatisfiedBy) | 4 edges: matches, computeSimilarityScore | +| Q8 | Inbound Trace | PASS | 1/5 | trace_call_path(inbound) | match, describeMismatch callers | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | 5,876 Method->Method CALLS | +| Q10 | Properties | PASS | 1/5 | search_graph(Variable) | 1,003+ variables (1,278 total) | +| Q11 | Inheritance | PASS | 1/5 | search_graph(Method) | 1,003+ methods (6,788 total) | +| Q12 | List Directory | PASS | 1/5 | list_directory | 27 entries: spock-core/, spock-specs/, build-logic/ | + +**Score: 12/12 (100%)** + +### SQL (flyway/flyway) + +**Project**: `flyway-sql` | **Repo**: `/tmp/lang-bench/flyway-sql` +**Nodes**: 10,149 | **Edges**: 23,222 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(File) | 811 files indexed | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | 47 functions (Python/Bash helpers, not SQL functions) | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | 576 classes (Java: ClassicConfiguration, ConfigUtils) | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=Migration) | 456 results | +| Q6 | Text Search | PARTIAL | 1/5 | search_code(CREATE TABLE) | Found in docs but no actual SQL DDL indexed | +| Q12 | List Directory | PASS | 1/5 | list_directory | 22 entries: flyway-core/, flyway-docker/ | + +**Score: 4.5/6 (75%)** + +### Dockerfile (docker-library/official-images) + +**Project**: `official-images-dockerfile` | **Repo**: `/tmp/lang-bench/official-images-dockerfile` +**Nodes**: 1,481 | **Edges**: 1,588 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(File) | 206 files indexed | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=docker) | 12 results | +| Q6 | Text Search | PASS | 1/5 | search_code(FROM) | 3+ matches in workflow scripts | +| Q12 | List Directory | PASS | 1/5 | list_directory | 17 entries: library/, test/, Dockerfile | + +**Score: 4/4 (100%)** + +### CSS (animate-css/animate.css) + +**Project**: `animate-css` | **Repo**: `/tmp/lang-bench/animate-css` +**Nodes**: 295 | **Edges**: 214 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 117 Files, 115 Modules, 37 Variables, 3 Functions | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 0 results (no tests in CSS lib) | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 0 matches (clean codebase) | +| Q12 | List Directory | PASS | 1/5 | list_directory | 14 entries: source/, docs/ | + +**Score: 4/4 (100%)** + +### YAML (kubernetes/examples) + +**Project**: `k8s-yaml` | **Repo**: `/tmp/lang-bench/k8s-yaml` +**Nodes**: 2,110 | **Edges**: 1,844 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 1,235 Variables, 335 Files, 309 Modules | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 12 test-related nodes | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 9 matches across demo scripts | +| Q12 | List Directory | PASS | 1/5 | list_directory | 12 entries: AI/, databases/, web/ | + +**Score: 4/4 (100%)** + +### TOML (rust-lang/cargo) + +**Project**: `cargo-toml` | **Repo**: `/tmp/lang-bench/cargo-toml` +**Nodes**: 16,773 | **Edges**: 51,403 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(Variable) | 305 variables | +| Q4 | Pattern Search | PASS | 1/5 | search_code(TODO) | 3+ TODOs in workflows, Cargo.toml | +| Q6 | Text Search | PASS | 1/5 | list_directory(src/cargo) | 8 entries: core/, lib.rs, lints/, ops/ | +| Q12 | List Directory | PASS | 1/5 | search_graph(Module) | 1,005+ modules | + +**Score: 4/4 (100%)** + +### HCL (hashicorp/terraform) + +**Project**: `terraform-hcl` | **Repo**: `/tmp/lang-bench/terraform-hcl` +**Nodes**: 78 | **Edges**: 76 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | search_graph(Variable) | 50 variables (version, constraints, hashes, region) | +| Q4 | Pattern Search | PASS | 1/5 | search_code(TODO) | 0 TODOs (clean config) | +| Q6 | Text Search | PASS | 1/5 | list_directory | 6 entries: main.tf, outputs.tf, variables.tf | +| Q12 | List Directory | PASS | 1/5 | search_graph(Module) | 5 modules: main.tf, .terraform.lock.hcl | + +**Score: 4/4 (100%)** + +### HTML (twbs/bootstrap) + +**Project**: `bootstrap-scss` (shared) | **Repo**: `/tmp/lang-bench/bootstrap-scss` +**Nodes**: 2,726 | **Edges**: 4,135 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 1,525 Variables, 267 Methods, 195 Functions, 22 Classes | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 18 test-related nodes | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3 TODO/FIXME matches in js/src/carousel.js | +| Q12 | List Directory | PASS | 1/5 | list_directory | 15 entries: scss/, js/, dist/, build/ | + +**Score: 4/4 (100%)** + +### SCSS (twbs/bootstrap) + +**Project**: `bootstrap-scss` (shared) | **Repo**: `/tmp/lang-bench/bootstrap-scss` +**Nodes**: 2,726 | **Edges**: 4,135 + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 1,525 Variables, 267 Methods, 195 Functions, 22 Classes | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=test) | 18 SCSS test files | +| Q6 | Text Search | PASS | 1/5 | search_code(TODO) | 3 matches | +| Q12 | List Directory | PASS | 1/5 | list_directory | 15 entries: scss/, js/, dist/ | + +**Score: 4/4 (100%)** + +--- + +## Linux Kernel Stress Test + +**Project**: `linux-kernel` | **Subset**: `drivers/net/ethernet/intel/` +**Files**: 387 C/H files | **Nodes**: 19,993 | **Edges**: 67,305 + +### Index Breakdown + +| Label | Count | +|-------|------:| +| Function | 11,546 | +| Class (structs) | 1,851 | +| Method | 1,803 | +| Macro | 1,346 | +| Field | 1,095 | +| Community | 734 | +| Enum | 480 | +| Variable | 461 | + +### Results + +| Q# | Question | Grade | Attempts | Approach | Notes | +|----|----------|-------|----------|----------|-------| +| Q1 | Index Stats | PASS | 1/5 | get_graph_schema | 14 node labels, 11 edge types, 20K nodes, 67K edges | +| Q2 | Find Functions | PASS | 1/5 | search_graph(Function) | e1e_rphy (in=64), e1e_wphy (in=53). 11,546 total | +| Q3 | Find Classes | PASS | 1/5 | search_graph(Class) | nic (in=86), params (in=59). 1,851 C structs | +| Q4 | Pattern Search | PASS | 1/5 | search_graph(name_pattern=init\|probe) | 715 matches: i40e_probe (out=59), ixgbe_probe (out=43) | +| Q5 | Code Snippet | PASS | 1/5 | get_code_snippet | e1e_rphy: 4 lines from e1000.h:543-546 | +| Q6 | Text Search | PASS | 1/5 | search_code(BUG_ON\|WARN_ON\|pr_err) | Found across amt.c, e100.c, e1000_ethtool.c | +| Q7 | Outbound Trace (depth=5) | PASS | 1/5 | trace_call_path(outbound) | i40e_probe: 129K chars result. Massive call tree, no timeout | +| Q8 | Inbound Trace (depth=5) | PASS | 1/5 | trace_call_path(inbound) | e1e_rphy: 67.6K chars. 64 direct callers, deep chains | +| Q9 | Cypher CALLS | PASS | 1/5 | query_graph | 10 CALLS edges: netdev_boot_setup, amt functions | +| Q10 | Properties | PARTIAL | 2/5 | query_graph(Cypher) | `IS NOT NULL` not supported; retried with adjusted query | +| Q11 | Inheritance | N/A | -- | -- | C language | +| Q12 | List Directory | PASS | 1/5 | list_directory | 17 entries: e100.c, i40e/, ice/, ixgbe/, igb/ | + +**Score: 11/11 (100%)** (Q11 N/A) + +### Stress Metrics + +| Test | Result | Size | Timeout? | +|------|--------|-----:|----------| +| Deep trace (depth=5, outbound, i40e_probe) | PASS | 129,026 chars | No | +| Deep trace (depth=5, inbound, e1e_rphy) | PASS | 67,600 chars | No | +| Deep trace (depth=5, both, e1000e_reset) | PASS | moderate | No | +| Multi-hop Cypher (3-hop CALLS chain) | PASS | 10 valid chains | No | + +The tool handles 20K-node kernel subsystems without timeouts. Deep traces on well-connected probe functions (out_degree=59) produce massive but complete results. Community detection identified 734 clusters -- useful for understanding driver module boundaries. + +--- + +## Cross-Cutting Findings + +### Strengths +1. **Zero indexing failures** across 35 repos of all sizes (78 to 49K nodes) +2. **100% on 17 languages** -- half the languages achieved perfect scores +3. **No language below 62%** -- even the weakest performs core operations +4. **Massive codebases handled**: Django (49K nodes), Laravel (38K nodes), neovim (24K nodes) -- no performance issues +5. **Kernel-scale stress test passed**: 20K nodes, 67K edges, 129K-char traces, zero timeouts +6. **"Removed" languages still strong**: Obj-C (100%), Perl (100%), Groovy (100%), Swift (95%), Dart (87%) + +### Common PARTIAL Patterns +- **Q10 (Properties)**: Most languages return functions with `properties=null`. Parameter/return extraction is incomplete -- this is the single most common deduction +- **Q7/Q8 (Trace)**: Abstract methods, entry-point functions, and built-in references correctly return 0 edges (no false positives), but are graded PARTIAL when no alternative function has edges +- **Q11 (Inheritance)**: Languages using alternative paradigms (traits, protocols, typeclasses, modules) have limited INHERITS edges + +### Known Limitations +1. **Function properties**: Parameter types and return types are not extracted for most languages, resulting in `properties=null` +2. **Haskell call tracing**: Function composition (`f . g . h`) is not modeled as CALLS edges. Only explicit function applications are traced +3. **OCaml call analysis**: Limited call resolution due to module functor indirection +4. **Cypher `IS NOT NULL`**: Not supported in the Cypher parser -- use alternative query patterns +5. **Scala/Ruby trace**: Abstract methods and block-based patterns can have 0 trace edges +6. **query_graph 200-row cap**: Aggregation queries (COUNT) silently undercount on large codebases + +--- + +## Aggregate Statistics + +| Metric | Value | +|--------|------:| +| Languages tested | 35 | +| Total questions asked | 370 | +| PASS | 327 (88%) | +| PARTIAL | 25 (7%) | +| FAIL | 18 (5%) | +| N/A (excluded) | 41 | +| Weighted score | 339.5 / 370 | +| Overall percentage | **91.8%** | +| Perfect scores (100%) | 17 languages | +| Tier 1 (>=90%) | 17 languages | +| Tier 2 (75-89%) | 16 languages | +| Tier 3 (<75%) | 2 languages | diff --git a/README.md b/README.md index c9912b74a..62b4f1c29 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,16 @@ **Stop paying 100x more tokens for code exploration.** This MCP server indexes your codebase into a persistent knowledge graph that survives session restarts and context compaction. One graph query returns what would take dozens of grep/Glob calls — precise structural results in ~500 tokens vs ~80K tokens for file-by-file exploration. -Single Go binary. No Docker, no external databases, no API keys. Download, configure, say *"Index this project"* — done. +Single Go binary. No Docker, no external databases, no API keys. One command to install, say *"Index this project"* — done. -Parses source code with [tree-sitter](https://tree-sitter.github.io/tree-sitter/), extracts functions, classes, modules, call relationships, and cross-service HTTP links. Exposes the graph through 12 MCP tools for use with Claude Code or any MCP-compatible client. Also includes a **CLI mode** for direct tool invocation from the shell — no MCP client needed. +Parses source code with [tree-sitter](https://tree-sitter.github.io/tree-sitter/), extracts functions, classes, modules, call relationships, and cross-service HTTP links. Exposes the graph through 12 MCP tools for use with Claude Code, Codex CLI, or any MCP-compatible client. Also includes a **CLI mode** for direct tool invocation from the shell — no MCP client needed. ## Features -- **13 languages**: Python, Go, JavaScript, TypeScript, TSX, Rust, Java, C++, C#, PHP, Lua, Scala, Kotlin +- **25 languages**: Python, Go, JavaScript, TypeScript, TSX, Rust, Java, C++, C#, C, PHP, Lua, Scala, Kotlin, Ruby, Bash, Zig, Elixir, Haskell, OCaml, HTML, CSS, YAML, TOML, HCL +- **One-command install**: `codebase-memory-mcp install` auto-detects Claude Code and Codex CLI, registers the MCP server, and installs task-specific skills +- **Self-update**: `codebase-memory-mcp update` downloads the latest release, verifies checksums, and atomically swaps the binary +- **Task-specific skills**: 4 skills (exploring, tracing, quality, reference) that prescribe exact tool sequences — Claude Code automatically uses graph tools instead of defaulting to grep - **Fast**: Sub-millisecond graph queries, incremental reindex 4x faster than full scan, optimized SQLite with LIKE pre-filtering for regex searches - **Call graph**: Resolves function calls across files and packages (import-aware, type-inferred) - **Cross-service HTTP linking**: Discovers REST routes (FastAPI, Gin, Express) and matches them to HTTP call sites with confidence scoring @@ -65,11 +68,37 @@ Benchmarked on a multi-service project (~37K nodes, ~35K edges) with Go 1.26 on **Token efficiency**: Five structural queries consumed ~3,400 tokens via codebase-memory-mcp versus ~412,000 tokens via file-by-file grep exploration — a **99.2% reduction**. +## Quick Start + +1. **Download** the binary for your platform from the [latest release](https://github.com/DeusData/codebase-memory-mcp/releases/latest) +2. **Install**: + ```bash + codebase-memory-mcp install + ``` +3. **Restart** Claude Code / Codex CLI +4. Say **"Index this project"** — done. + +The `install` command auto-detects Claude Code and Codex CLI, registers the MCP server, installs 4 task-specific skills, and ensures the binary is on your PATH. Use `--dry-run` to preview without making changes. + +### Keeping Up to Date + +```bash +codebase-memory-mcp update +``` + +Downloads the latest release, verifies SHA-256 checksums, atomically swaps the binary, and re-applies skills. Restart Claude Code / Codex to activate. + +### Uninstall + +```bash +codebase-memory-mcp uninstall +``` + +Removes skills, MCP registration, and Codex instructions. Does not remove the binary or SQLite databases. + ## Installation -### Just Want to Get Started? - -Grab a pre-built binary from the [latest release](https://github.com/DeusData/codebase-memory-mcp/releases/latest) — no Go, no compiler, no git needed. +### Pre-built Binaries | Platform | Binary | |----------|--------| @@ -81,29 +110,12 @@ Grab a pre-built binary from the [latest release](https://github.com/DeusData/co Every release includes a `checksums.txt` with SHA-256 hashes for verification. -**3 steps:** - -1. **Download and extract** the binary for your platform -2. **Put it somewhere on your PATH** (e.g. `~/.local/bin/` on macOS/Linux, or any folder on Windows) -3. **Tell Claude Code about it** — add to `.mcp.json` in your project root: - ```json - { - "mcpServers": { - "codebase-memory-mcp": { - "type": "stdio", - "command": "/path/to/codebase-memory-mcp" - } - } - } - ``` - -Restart Claude Code, verify with `/mcp`, then say **"Index this project"** — done. - > **Windows note**: Windows SmartScreen may show "Windows protected your PC" when you first run the binary. This is normal for unsigned open-source software. Click **"More info"** then **"Run anyway"**. You can verify the binary integrity using the `checksums.txt` file included in each release. -### Setup Scripts (Automated) +### Setup Scripts -The setup scripts automate download, placement, and Claude Code configuration. +
+Alternative: automated download + install **macOS / Linux:** @@ -117,7 +129,9 @@ curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/s irm https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/scripts/setup-windows.ps1 | iex ``` -The scripts download the correct binary for your platform, install it, and offer to auto-configure Claude Code's MCP settings. +The scripts download the correct binary for your platform, install it, and run `codebase-memory-mcp install` to configure Claude Code/Codex. + +
### Install via Claude Code @@ -199,7 +213,10 @@ When using a WSL-built binary, configure Claude Code to invoke it via `wsl.exe`: } ``` -### Configure Claude Code +### Manual Configuration + +
+If you prefer not to use `codebase-memory-mcp install` Add the MCP server to your project's `.mcp.json` (per-project, recommended) or `~/.claude/settings.json` (global): @@ -216,6 +233,8 @@ Add the MCP server to your project's `.mcp.json` (per-project, recommended) or ` Restart Claude Code after adding the config. Verify with `/mcp` — you should see `codebase-memory-mcp` listed with 12 tools. +
+ ### First Use ``` @@ -480,45 +499,30 @@ The format is: project name, file path with `/` replaced by `.` and extension re - `UNION` - Variable-length path edge property binding (can't access individual edges in a path like `*1..3`) -## Teaching Claude Code to Use the Graph +## How Claude Code Uses the Graph -Claude Code can use the tools without any configuration — the MCP tool descriptions are self-documenting. However, without a hint, Claude Code will default to its built-in Grep/Glob/Read tools for code questions instead of the faster graph queries. +After `codebase-memory-mcp install`, Claude Code automatically has 4 task-specific skills that prescribe when and how to use graph tools: -Add one of the following to tell Claude Code to **prefer graph tools for structural questions**. +- **Exploring**: Codebase orientation, structure overview, finding functions/classes/routes +- **Tracing**: Call chains, dependency analysis, cross-service HTTP calls, impact analysis +- **Quality**: Dead code detection, fan-out analysis, refactor candidates +- **Reference**: Tool syntax, Cypher query examples, edge types, pitfalls -### Option A: Global CLAUDE.md (recommended — works across all projects) +No CLAUDE.md changes needed — skills auto-trigger based on conversation context. -Add to `~/.claude/CLAUDE.md`: +
+Manual skill installation (if not using `install` command) -```markdown -## Codebase Memory (codebase-memory-mcp) +The skill files are embedded in the binary. If you prefer to install them manually, copy from this repo: -When this MCP server is available, **prefer graph tools over grep/Explore for structural code questions**. -Graph queries return precise results in a single tool call (~500 tokens) vs file-by-file exploration (~80K tokens). +- `cmd/codebase-memory-mcp/assets/skills/codebase-memory-exploring/SKILL.md` → `~/.claude/skills/codebase-memory-exploring/SKILL.md` +- `cmd/codebase-memory-mcp/assets/skills/codebase-memory-tracing/SKILL.md` → `~/.claude/skills/codebase-memory-tracing/SKILL.md` +- `cmd/codebase-memory-mcp/assets/skills/codebase-memory-quality/SKILL.md` → `~/.claude/skills/codebase-memory-quality/SKILL.md` +- `cmd/codebase-memory-mcp/assets/skills/codebase-memory-reference/SKILL.md` → `~/.claude/skills/codebase-memory-reference/SKILL.md` -- **Initial index**: Run `index_repository` once per project — auto-sync keeps it fresh after that -- **"Who calls X?"**: `trace_call_path(function_name="X", direction="inbound")` -- **"What does X call?"**: `trace_call_path(function_name="X", direction="outbound")` -- **Find functions by pattern**: `search_graph(label="Function", name_pattern=".*Pattern.*")` -- **Dead code**: `search_graph(label="Function", relationship="CALLS", direction="inbound", max_degree=0, exclude_entry_points=true)` -- **Cross-service calls**: `search_graph(relationship="HTTP_CALLS")` or `query_graph` with Cypher -- **REST routes**: `search_graph(label="Route")` -- **Scope to one project**: `search_graph(..., project="project-name")` when multiple repos are indexed -- **Understand structure first**: `get_graph_schema` before writing complex queries -- **Read source**: `get_code_snippet(qualified_name="...")` after finding functions via search -- **Discover then trace**: Use `search_graph(name_pattern=".*Partial.*")` to find exact names, then `trace_call_path` -- **Complex patterns**: `query_graph` with Cypher for multi-hop graph traversals +Or add a CLAUDE.md snippet — see [`skills/codebase-memory-mcp.md`](skills/codebase-memory-mcp.md) for the full reference. -Use grep/Glob for text search (string literals, error messages, config values) — the graph doesn't index text content. -``` - -### Option B: Per-project CLAUDE.md - -Add the same snippet to a specific project's `CLAUDE.md` if you only want it active for that project. - -### Option C: Claude Code skill file - -Copy [`skills/codebase-memory-mcp.md`](skills/codebase-memory-mcp.md) from this repo to `~/.claude/skills/codebase-memory-mcp/SKILL.md` for automatic activation when relevant. This is the most comprehensive option — it includes decision matrices, query pitfalls, and workflow patterns. +
## Ignoring Files (`.cgrignore`) @@ -571,17 +575,32 @@ make install # go install | Binary not found after install | `~/.local/bin` not on PATH | Add to your shell profile: `export PATH="$HOME/.local/bin:$PATH"` | | Cypher query fails with parse error | Unsupported Cypher feature | See [Supported Cypher Subset](#supported-cypher-subset). `WITH`, `COLLECT`, `OPTIONAL MATCH` are not supported. | +## Language Benchmark + +Benchmarked against 35 real open-source repositories (78 to 49K nodes). 12 standardized questions per language, up to 5 retry attempts each. Grading: PASS (1.0) / PARTIAL (0.5) / FAIL (0.0). Overall: **91.8%** weighted score. + +| Tier | Score | Languages | +|------|-------|-----------| +| **Tier 1 — Excellent** | >= 90% | Lua, Kotlin, C++, Perl, Objective-C, Groovy, C, Bash, Zig, Swift, CSS, YAML, TOML, HTML, SCSS, HCL, Dockerfile | +| **Tier 2 — Good** | 75–89% | Python, TypeScript, TSX, Go, Rust, Java, R, Dart, JavaScript, Erlang, Elixir, Scala, Ruby, PHP, C#, SQL | +| **Tier 3 — Functional** | < 75% | OCaml (72%), Haskell (62%) | + +**Stress test**: Linux kernel `drivers/net/ethernet/intel/` — 20K nodes, 67K edges, 129K-char deep traces, zero timeouts. + +See [`BENCHMARK.md`](BENCHMARK.md) for the full 35-language benchmark with per-question scoring and methodology. + ## Architecture ``` -cmd/codebase-memory-mcp/ Entry point (MCP stdio server + CLI mode) +cmd/codebase-memory-mcp/ Entry point (MCP stdio server + CLI mode + install/update commands) internal/ store/ SQLite graph storage (nodes, edges, traversal, search) - lang/ Language specs (13 languages, tree-sitter node types) + lang/ Language specs (25 languages, tree-sitter node types) parser/ Tree-sitter grammar loading and AST parsing pipeline/ 4-pass indexing (structure -> definitions -> calls -> HTTP links) httplink/ Cross-service HTTP route/call-site matching cypher/ Cypher query lexer, parser, planner, executor + selfupdate/ GitHub release checking, version comparison, asset download tools/ MCP tool handlers (12 tools) + CLI dispatch watcher/ Background auto-sync (mtime+size polling, adaptive intervals) discover/ File discovery with .cgrignore support diff --git a/cmd/codebase-memory-mcp/assets.go b/cmd/codebase-memory-mcp/assets.go new file mode 100644 index 000000000..3b763a56b --- /dev/null +++ b/cmd/codebase-memory-mcp/assets.go @@ -0,0 +1,29 @@ +package main + +import "embed" + +//go:embed assets/skills/codebase-memory-exploring/SKILL.md +var skillExploring string + +//go:embed assets/skills/codebase-memory-tracing/SKILL.md +var skillTracing string + +//go:embed assets/skills/codebase-memory-quality/SKILL.md +var skillQuality string + +//go:embed assets/skills/codebase-memory-reference/SKILL.md +var skillReference string + +//go:embed assets/codex-instructions.md +var codexInstructions string + +// skillFiles maps skill directory name to embedded content. +var skillFiles = map[string]string{ + "codebase-memory-exploring": skillExploring, + "codebase-memory-tracing": skillTracing, + "codebase-memory-quality": skillQuality, + "codebase-memory-reference": skillReference, +} + +// Ensure embed import is used. +var _ embed.FS diff --git a/cmd/codebase-memory-mcp/assets/codex-instructions.md b/cmd/codebase-memory-mcp/assets/codex-instructions.md new file mode 100644 index 000000000..1cc83bee8 --- /dev/null +++ b/cmd/codebase-memory-mcp/assets/codex-instructions.md @@ -0,0 +1,34 @@ +# Codebase Knowledge Graph + +This project has a code knowledge graph available via MCP tools. Use it for +structural questions instead of grep — one graph query returns what would take +dozens of file-by-file searches. + +## Finding Code + +- "Who calls X?" → use `trace_call_path` with `direction=inbound` +- "What does X call?" → use `trace_call_path` with `direction=outbound` +- "Find functions matching pattern" → use `search_graph` with `name_pattern` +- "Find all routes" → use `search_graph` with `label=Route` +- "Show codebase structure" → use `get_graph_schema` for overview + +## Tracing Dependencies + +- Always discover exact names first: `search_graph(name_pattern=".*Partial.*")` +- Then trace: `trace_call_path(function_name="ExactName", direction="both")` +- Cross-service HTTP calls: `query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path")` +- Read source: `get_code_snippet(qualified_name="project.path.FunctionName")` + +## Quality Analysis + +- Dead code: `search_graph` with `max_degree=0`, `exclude_entry_points=true`, `relationship=CALLS`, `direction=inbound` +- High fan-out: `search_graph` with `min_degree=10`, `relationship=CALLS`, `direction=outbound` +- Change coupling: `query_graph("MATCH (a)-[r:FILE_CHANGES_WITH]->(b) WHERE r.coupling_score >= 0.5 RETURN ...")` + +## Important + +- Always check `list_projects` first — run `index_repository` if the project is missing +- Use `search_graph` to discover exact names before `trace_call_path` (it requires exact match) +- The graph doesn't index text content — use grep for string literals, error messages, config values +- Results default to 10 per page — check `has_more` and use `offset` to paginate +- Use `project` parameter when multiple repos are indexed diff --git a/cmd/codebase-memory-mcp/assets/skills/codebase-memory-exploring/SKILL.md b/cmd/codebase-memory-mcp/assets/skills/codebase-memory-exploring/SKILL.md new file mode 100644 index 000000000..cc45a8be0 --- /dev/null +++ b/cmd/codebase-memory-mcp/assets/skills/codebase-memory-exploring/SKILL.md @@ -0,0 +1,90 @@ +--- +name: codebase-memory-exploring +description: > + This skill should be used when the user asks to "explore the codebase", + "understand the architecture", "what functions exist", "show me the structure", + "how is the code organized", "find functions matching", "search for classes", + "list all routes", "show API endpoints", or needs codebase orientation. +--- + +# Codebase Exploration via Knowledge Graph + +Use graph tools for structural code questions. They return precise results in ~500 tokens vs ~80K for grep-based exploration. + +## Workflow + +### Step 1: Check if project is indexed + +``` +list_projects +``` + +If the project is missing from the list: + +``` +index_repository(repo_path="/path/to/project") +``` + +If already indexed, skip — auto-sync keeps the graph fresh. + +### Step 2: Get a structural overview + +``` +get_graph_schema +``` + +This returns node label counts (functions, classes, routes, etc.), edge type counts, and relationship patterns. Use it to understand what's in the graph before querying. + +### Step 3: Find specific code elements + +Find functions by name pattern: +``` +search_graph(label="Function", name_pattern=".*Handler.*") +``` + +Find classes: +``` +search_graph(label="Class", name_pattern=".*Service.*") +``` + +Find all REST routes: +``` +search_graph(label="Route") +``` + +Find modules/packages: +``` +search_graph(label="Module") +``` + +Scope to a specific directory: +``` +search_graph(label="Function", qn_pattern=".*services\\.order\\..*") +``` + +### Step 4: Read source code + +After finding a function via search, read its source: +``` +get_code_snippet(qualified_name="project.path.to.FunctionName") +``` + +### Step 5: Understand structure + +For file/directory exploration within the indexed project: +``` +list_directory(path="src/services") +``` + +## When to Use Grep Instead + +- Searching for **string literals** or error messages → `search_code` or Grep +- Finding a file by exact name → Glob +- The graph doesn't index text content, only structural elements + +## Key Tips + +- Results default to 10 per page. Check `has_more` and use `offset` to paginate. +- Use `project` parameter when multiple repos are indexed. +- Route nodes have a `properties.handler` field with the actual handler function name. +- `exclude_labels` removes noise (e.g., `exclude_labels=["Route"]` when searching by name pattern). diff --git a/cmd/codebase-memory-mcp/assets/skills/codebase-memory-quality/SKILL.md b/cmd/codebase-memory-mcp/assets/skills/codebase-memory-quality/SKILL.md new file mode 100644 index 000000000..1542eee26 --- /dev/null +++ b/cmd/codebase-memory-mcp/assets/skills/codebase-memory-quality/SKILL.md @@ -0,0 +1,101 @@ +--- +name: codebase-memory-quality +description: > + This skill should be used when the user asks about "dead code", + "find dead code", "detect dead code", "show dead code", "dead code analysis", + "unused functions", "find unused functions", "unreachable code", + "identify high fan-out functions", "find complex functions", + "code quality audit", "find functions nobody calls", + "reduce codebase size", "refactor candidates", "cleanup candidates", + or needs code quality analysis. +--- + +# Code Quality Analysis via Knowledge Graph + +Use graph degree filtering to find dead code, high-complexity functions, and refactor candidates — all in single tool calls. + +## Workflow + +### Dead Code Detection + +Find functions with zero inbound CALLS edges, excluding entry points: + +``` +search_graph( + label="Function", + relationship="CALLS", + direction="inbound", + max_degree=0, + exclude_entry_points=true +) +``` + +`exclude_entry_points=true` removes route handlers, `main()`, and framework-registered functions that have zero callers by design. + +### Verify Dead Code Candidates + +Before deleting, verify each candidate truly has no callers: + +``` +trace_call_path(function_name="SuspectFunction", direction="inbound", depth=1) +``` + +Also check for read references (callbacks, stored in variables): + +``` +query_graph(query="MATCH (a)-[r:USAGE]->(b) WHERE b.name = 'SuspectFunction' RETURN a.name, a.file_path LIMIT 10") +``` + +### High Fan-Out Functions (calling 10+ others) + +These are often doing too much and are refactor candidates: + +``` +search_graph( + label="Function", + relationship="CALLS", + direction="outbound", + min_degree=10 +) +``` + +### High Fan-In Functions (called by 10+ others) + +These are critical functions — changes have wide impact: + +``` +search_graph( + label="Function", + relationship="CALLS", + direction="inbound", + min_degree=10 +) +``` + +### Files That Change Together (Hidden Coupling) + +Find files with high git change coupling: + +``` +query_graph(query="MATCH (a)-[r:FILE_CHANGES_WITH]->(b) WHERE r.coupling_score >= 0.5 RETURN a.name, b.name, r.coupling_score, r.co_change_count ORDER BY r.coupling_score DESC LIMIT 20") +``` + +High coupling between unrelated files suggests hidden dependencies. + +### Unused Imports + +``` +search_graph( + relationship="IMPORTS", + direction="outbound", + max_degree=0, + label="Module" +) +``` + +## Key Tips + +- `search_graph` with degree filters has no row cap (unlike `query_graph` which caps at 200). +- Use `file_pattern` to scope analysis to specific directories: `file_pattern="**/services/**"`. +- Dead code detection works best after a full index — run `index_repository` if the project was recently set up. +- Paginate results with `limit` and `offset` — check `has_more` in the response. diff --git a/cmd/codebase-memory-mcp/assets/skills/codebase-memory-reference/SKILL.md b/cmd/codebase-memory-mcp/assets/skills/codebase-memory-reference/SKILL.md new file mode 100644 index 000000000..547d4a556 --- /dev/null +++ b/cmd/codebase-memory-mcp/assets/skills/codebase-memory-reference/SKILL.md @@ -0,0 +1,110 @@ +--- +name: codebase-memory-reference +description: > + This skill should be used when the user asks about "codebase-memory-mcp tools", + "graph query syntax", "Cypher query examples", "edge types", + "how to use search_graph", "query_graph examples", or needs reference + documentation for the codebase knowledge graph tools. +--- + +# Codebase Memory MCP — Tool Reference + +## Tools (12 total) + +| Tool | Purpose | +|------|---------| +| `index_repository` | Parse and ingest repo into graph (only once — auto-sync keeps it fresh) | +| `index_status` | Check indexing status (ready/indexing/not found) | +| `list_projects` | List all indexed projects with timestamps and counts | +| `delete_project` | Remove a project from the graph | +| `search_graph` | Structured search with filters (name, label, degree, file pattern) | +| `search_code` | Grep-like text search within indexed project files | +| `trace_call_path` | BFS call chain traversal (exact name match required) | +| `query_graph` | Cypher-like graph queries (200-row cap) | +| `get_graph_schema` | Node/edge counts, relationship patterns | +| `get_code_snippet` | Read source code by qualified name | +| `read_file` | Read any file from indexed project | +| `list_directory` | List files/directories with glob filter | + +## Edge Types + +| Type | Meaning | +|------|---------| +| `CALLS` | Direct function call within same service | +| `HTTP_CALLS` | Synchronous cross-service HTTP request | +| `ASYNC_CALLS` | Async dispatch (Cloud Tasks, Pub/Sub, SQS, Kafka) | +| `IMPORTS` | Module/package import | +| `DEFINES` / `DEFINES_METHOD` | Module/class defines a function/method | +| `HANDLES` | Route node handled by a function | +| `IMPLEMENTS` | Type implements an interface | +| `OVERRIDE` | Struct method overrides an interface method | +| `USAGE` | Read reference (callback, variable assignment) | +| `FILE_CHANGES_WITH` | Git history change coupling | +| `CONTAINS_FILE` / `CONTAINS_FOLDER` / `CONTAINS_PACKAGE` | Structural containment | + +## Node Labels + +`Project`, `Package`, `Folder`, `File`, `Module`, `Class`, `Function`, `Method`, `Interface`, `Enum`, `Type`, `Route` + +## Qualified Name Format + +`..` — file path with `/` replaced by `.`, extension removed. + +Examples: +- `myproject.cmd.server.main.HandleRequest` (Go) +- `myproject.services.orders.ProcessOrder` (Python) +- `myproject.src.components.App.App` (TypeScript) + +Use `search_graph` to discover qualified names, then pass them to `get_code_snippet`. + +## Cypher Subset (for query_graph) + +**Supported:** +- `MATCH` with node labels and relationship types +- Variable-length paths: `-[:CALLS*1..3]->` +- `WHERE` with `=`, `<>`, `>`, `<`, `>=`, `<=`, `=~` (regex), `CONTAINS`, `STARTS WITH` +- `WHERE` with `AND`, `OR`, `NOT` +- `RETURN` with property access, `COUNT(x)`, `DISTINCT` +- `ORDER BY` with `ASC`/`DESC` +- `LIMIT` +- Edge property access: `r.confidence`, `r.url_path`, `r.coupling_score` + +**Not supported:** `WITH`, `COLLECT`, `SUM`, `CREATE/DELETE/SET`, `OPTIONAL MATCH`, `UNION` + +## Common Cypher Patterns + +``` +# Cross-service HTTP calls with confidence +MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence LIMIT 20 + +# Filter by URL path +MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path CONTAINS '/orders' RETURN a.name, b.name + +# Interface implementations +MATCH (s)-[r:OVERRIDE]->(i) RETURN s.name, i.name LIMIT 20 + +# Change coupling +MATCH (a)-[r:FILE_CHANGES_WITH]->(b) WHERE r.coupling_score >= 0.5 RETURN a.name, b.name, r.coupling_score + +# Functions calling a specific function +MATCH (f:Function)-[:CALLS]->(g:Function) WHERE g.name = 'ProcessOrder' RETURN f.name LIMIT 20 +``` + +## Critical Pitfalls + +1. **`search_graph(relationship="HTTP_CALLS")` does NOT return edges** — it filters nodes by degree. Use `query_graph` with Cypher to see actual edges. +2. **`query_graph` has a 200-row cap** before aggregation — COUNT queries silently undercount on large codebases. Use `search_graph` with `min_degree`/`max_degree` for counting. +3. **`trace_call_path` needs exact names** — use `search_graph(name_pattern=".*Partial.*")` first to discover names. +4. **`direction="outbound"` misses cross-service callers** — use `direction="both"` for full context. + +## Decision Matrix + +| Question | Use | +|----------|-----| +| Who calls X? | `trace_call_path(direction="inbound")` | +| What does X call? | `trace_call_path(direction="outbound")` | +| Full call context | `trace_call_path(direction="both")` | +| Find by name pattern | `search_graph(name_pattern="...")` | +| Dead code | `search_graph(max_degree=0, exclude_entry_points=true)` | +| Cross-service edges | `query_graph` with Cypher | +| Text search | `search_code` or Grep | diff --git a/cmd/codebase-memory-mcp/assets/skills/codebase-memory-tracing/SKILL.md b/cmd/codebase-memory-mcp/assets/skills/codebase-memory-tracing/SKILL.md new file mode 100644 index 000000000..8b71065f0 --- /dev/null +++ b/cmd/codebase-memory-mcp/assets/skills/codebase-memory-tracing/SKILL.md @@ -0,0 +1,96 @@ +--- +name: codebase-memory-tracing +description: > + This skill should be used when the user asks "who calls this function", + "what does X call", "trace the call chain", "find callers of", + "show dependencies", "what depends on", "trace call path", + "find all references to", "impact analysis", or needs to understand + function call relationships and dependency chains. +--- + +# Call Chain Tracing via Knowledge Graph + +Use graph tools to trace function call relationships. One `trace_call_path` call replaces dozens of grep searches across files. + +## Workflow + +### Step 1: Discover the exact function name + +`trace_call_path` requires an **exact** name match. If you don't know the exact name, discover it first: + +``` +search_graph(name_pattern=".*Order.*", label="Function") +``` + +This returns matching functions with their qualified names and file locations. + +### Step 2: Trace callers (who calls this function?) + +``` +trace_call_path(function_name="ProcessOrder", direction="inbound", depth=3) +``` + +Returns a hop-by-hop list of all functions that call `ProcessOrder`, up to 3 levels deep. + +### Step 3: Trace callees (what does this function call?) + +``` +trace_call_path(function_name="ProcessOrder", direction="outbound", depth=3) +``` + +### Step 4: Full context (both callers and callees) + +``` +trace_call_path(function_name="ProcessOrder", direction="both", depth=3) +``` + +**Always use `direction="both"` for complete context.** Cross-service HTTP_CALLS edges from other services appear as inbound edges — `direction="outbound"` alone misses them. + +### Step 5: Read suspicious code + +After finding interesting callers/callees, read their source: + +``` +get_code_snippet(qualified_name="project.path.module.FunctionName") +``` + +## Cross-Service HTTP Calls + +To see all HTTP links between services with URLs and confidence scores: + +``` +query_graph(query="MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence ORDER BY r.confidence DESC LIMIT 20") +``` + +Filter by URL path: +``` +query_graph(query="MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path CONTAINS '/orders' RETURN a.name, b.name, r.url_path") +``` + +## Async Dispatch (Cloud Tasks, Pub/Sub, etc.) + +Find dispatch functions by name pattern, then trace: +``` +search_graph(name_pattern=".*CreateTask.*|.*send_to_pubsub.*") +trace_call_path(function_name="CreateMultidataTask", direction="both") +``` + +## Interface Implementations + +Find which structs implement an interface method: +``` +query_graph(query="MATCH (s)-[r:OVERRIDE]->(i) WHERE i.name = 'Read' RETURN s.name, i.name LIMIT 20") +``` + +## Read References (callbacks, variable assignments) + +``` +query_graph(query="MATCH (a)-[r:USAGE]->(b) WHERE b.name = 'ProcessOrder' RETURN a.name, a.file_path LIMIT 20") +``` + +## Key Tips + +- Start with `depth=1` for quick answers, increase only if needed (max 5). +- Edge types in trace results: `CALLS` (direct), `HTTP_CALLS` (cross-service), `ASYNC_CALLS` (async dispatch), `USAGE` (read reference), `OVERRIDE` (interface implementation). +- `search_graph(relationship="HTTP_CALLS")` filters nodes by degree — it does NOT return edges. Use `query_graph` with Cypher to see actual edges with properties. +- Results are capped at 200 nodes per trace. diff --git a/cmd/codebase-memory-mcp/cli_test.go b/cmd/codebase-memory-mcp/cli_test.go new file mode 100644 index 000000000..060b22e50 --- /dev/null +++ b/cmd/codebase-memory-mcp/cli_test.go @@ -0,0 +1,263 @@ +package main + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// testBinPath is set in TestMain — persists across all tests in this package. +var testBinPath string + +func TestMain(m *testing.M) { + // Build the binary once into a temp dir that persists for the full test run. + tmpDir, err := os.MkdirTemp("", "cmm-cli-test-*") + if err != nil { + panic("create temp dir: " + err.Error()) + } + + binPath := filepath.Join(tmpDir, "codebase-memory-mcp") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + cmd := exec.CommandContext(ctx, "go", "build", "-o", binPath, "./") + cmd.Dir = "." + if out, err := cmd.CombinedOutput(); err != nil { + cancel() + os.RemoveAll(tmpDir) + os.Stderr.Write(out) + panic("build test binary: " + err.Error()) + } + cancel() + testBinPath = binPath + + code := m.Run() + os.RemoveAll(tmpDir) + os.Exit(code) +} + +func testCmd(t *testing.T, args ...string) *exec.Cmd { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + return exec.CommandContext(ctx, testBinPath, args...) +} + +func TestCLI_Version(t *testing.T) { + out, err := testCmd(t, "--version").CombinedOutput() + if err != nil { + t.Fatalf("--version failed: %v\n%s", err, out) + } + output := strings.TrimSpace(string(out)) + if !strings.HasPrefix(output, "codebase-memory-mcp") { + t.Fatalf("unexpected --version output: %q", output) + } +} + +func TestCLI_InstallDryRun(t *testing.T) { + home := t.TempDir() + cmd := testCmd(t, "install", "--dry-run") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+t.TempDir()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("install --dry-run failed: %v\n%s", err, out) + } + output := string(out) + if !strings.Contains(output, "install") { + t.Fatalf("expected 'install' in output, got: %s", output) + } + // Dry run should not create any files + skillsDir := filepath.Join(home, ".claude", "skills") + if _, err := os.Stat(skillsDir); !os.IsNotExist(err) { + t.Fatal("dry-run should not create skills directory") + } +} + +func TestCLI_UninstallDryRun(t *testing.T) { + home := t.TempDir() + cmd := testCmd(t, "uninstall", "--dry-run") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+t.TempDir()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("uninstall --dry-run failed: %v\n%s", err, out) + } + output := string(out) + if !strings.Contains(output, "uninstall") { + t.Fatalf("expected 'uninstall' in output, got: %s", output) + } +} + +func TestCLI_UpdateDryRun(t *testing.T) { + cmd := testCmd(t, "update", "--dry-run") + out, _ := cmd.CombinedOutput() + output := string(out) + if !strings.Contains(output, "checking for updates") { + t.Fatalf("expected 'checking for updates' in output, got: %s", output) + } +} + +func TestCLI_CliHelp(t *testing.T) { + cmd := testCmd(t, "cli", "--help") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("cli --help failed: %v\n%s", err, out) + } + output := string(out) + if !strings.Contains(output, "Available tools") { + t.Fatalf("expected 'Available tools' in cli --help output, got: %s", output) + } +} + +func TestCLI_InstallAndUninstall(t *testing.T) { + home := t.TempDir() + emptyPath := t.TempDir() + + // Install + cmd := testCmd(t, "install") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+emptyPath, "SHELL=/bin/zsh") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("install failed: %v\n%s", err, out) + } + + // Verify skills were created + expectedSkills := []string{ + "codebase-memory-exploring", + "codebase-memory-tracing", + "codebase-memory-quality", + "codebase-memory-reference", + } + for _, name := range expectedSkills { + skillFile := filepath.Join(home, ".claude", "skills", name, "SKILL.md") + if _, err := os.Stat(skillFile); err != nil { + t.Fatalf("skill %s not found after install: %v", name, err) + } + } + + // Uninstall + cmd = testCmd(t, "uninstall") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+emptyPath) + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatalf("uninstall failed: %v\n%s", err, out) + } + + // Verify skills were removed + for _, name := range expectedSkills { + skillDir := filepath.Join(home, ".claude", "skills", name) + if _, err := os.Stat(skillDir); !os.IsNotExist(err) { + t.Fatalf("skill dir %s should be removed after uninstall", name) + } + } +} + +func TestCLI_InstallRemovesOldSkill(t *testing.T) { + home := t.TempDir() + emptyPath := t.TempDir() + + oldDir := filepath.Join(home, ".claude", "skills", "codebase-memory-mcp") + if err := os.MkdirAll(oldDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(oldDir, "SKILL.md"), []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + + cmd := testCmd(t, "install") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+emptyPath, "SHELL=/bin/zsh") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("install failed: %v\n%s", err, out) + } + + if _, err := os.Stat(oldDir); !os.IsNotExist(err) { + t.Fatal("old monolithic skill dir should be removed") + } + if _, err := os.Stat(filepath.Join(home, ".claude", "skills", "codebase-memory-exploring", "SKILL.md")); err != nil { + t.Fatal("new exploring skill should exist") + } +} + +func TestCLI_InstallIdempotent(t *testing.T) { + home := t.TempDir() + emptyPath := t.TempDir() + + for i := 0; i < 2; i++ { + cmd := testCmd(t, "install") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+emptyPath, "SHELL=/bin/zsh") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("install round %d failed: %v\n%s", i, err, out) + } + } + + skillFile := filepath.Join(home, ".claude", "skills", "codebase-memory-exploring", "SKILL.md") + if _, err := os.Stat(skillFile); err != nil { + t.Fatal("skill missing after idempotent install") + } +} + +func TestCLI_InstallForceOverwrites(t *testing.T) { + home := t.TempDir() + emptyPath := t.TempDir() + + cmd := testCmd(t, "install") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+emptyPath, "SHELL=/bin/zsh") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("first install failed: %v\n%s", err, out) + } + + skillFile := filepath.Join(home, ".claude", "skills", "codebase-memory-exploring", "SKILL.md") + if err := os.WriteFile(skillFile, []byte("custom content"), 0o600); err != nil { + t.Fatal(err) + } + + cmd = testCmd(t, "install") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+emptyPath, "SHELL=/bin/zsh") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("second install failed: %v\n%s", err, out) + } + data, _ := os.ReadFile(skillFile) + if string(data) != "custom content" { + t.Fatal("install without --force should not overwrite customized skills") + } + + cmd = testCmd(t, "install", "--force") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+emptyPath, "SHELL=/bin/zsh") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatalf("force install failed: %v\n%s", err, out) + } + data, _ = os.ReadFile(skillFile) + if string(data) == "custom content" { + t.Fatal("install --force should overwrite customized skills") + } +} + +func TestCLI_InstallPATHAppend(t *testing.T) { + home := t.TempDir() + t.Setenv("SHELL", "/bin/zsh") + emptyPath := t.TempDir() + + cmd := testCmd(t, "install") + cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+emptyPath, "SHELL=/bin/zsh") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("install failed: %v\n%s", err, out) + } + + zshrc := filepath.Join(home, ".zshrc") + data, err := os.ReadFile(zshrc) + if err != nil { + t.Fatalf("expected .zshrc to be created: %v", err) + } + if !strings.Contains(string(data), "export PATH=") { + t.Fatal("expected PATH export in .zshrc") + } + if !strings.Contains(string(data), "codebase-memory-mcp install") { + t.Fatal("expected install comment in .zshrc") + } +} diff --git a/cmd/codebase-memory-mcp/install.go b/cmd/codebase-memory-mcp/install.go new file mode 100644 index 000000000..b626a2986 --- /dev/null +++ b/cmd/codebase-memory-mcp/install.go @@ -0,0 +1,386 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" +) + +// installConfig holds settings for the install/uninstall commands. +type installConfig struct { + dryRun bool + force bool +} + +func runInstall(args []string) int { + cfg := installConfig{} + for _, a := range args { + switch a { + case "--dry-run": + cfg.dryRun = true + case "--force": + cfg.force = true + } + } + + binaryPath, err := detectBinaryPath() + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + return 1 + } + + fmt.Printf("\ncodebase-memory-mcp %s — install\n", version) + fmt.Printf("Binary: %s\n\n", binaryPath) + + // PATH check + ensurePATH(binaryPath, cfg) + + // Claude Code + if claudePath := findCLI("claude"); claudePath != "" { + fmt.Printf("[Claude Code] detected (%s)\n", claudePath) + installClaudeCode(binaryPath, claudePath, cfg) + } else { + fmt.Println("[Claude Code] not found — skipping") + } + + fmt.Println() + + // Codex CLI + if codexPath := findCLI("codex"); codexPath != "" { + fmt.Printf("[Codex CLI] detected (%s)\n", codexPath) + installCodex(binaryPath, codexPath, cfg) + } else { + fmt.Println("[Codex CLI] not found — skipping") + } + + fmt.Println("\nDone. Restart Claude Code / Codex to activate.") + return 0 +} + +func runUninstall(args []string) int { + cfg := installConfig{} + for _, a := range args { + if a == "--dry-run" { + cfg.dryRun = true + } + } + + fmt.Printf("\ncodebase-memory-mcp %s — uninstall\n\n", version) + + // Remove Claude Code skills + removeClaudeSkills(cfg) + + // Claude Code MCP deregistration + if claudePath := findCLI("claude"); claudePath != "" { + fmt.Printf("[Claude Code] detected (%s)\n", claudePath) + deregisterMCP(claudePath, "claude", cfg) + } + + // Codex CLI MCP deregistration + instructions + if codexPath := findCLI("codex"); codexPath != "" { + fmt.Printf("[Codex CLI] detected (%s)\n", codexPath) + deregisterMCP(codexPath, "codex", cfg) + removeCodexInstructions(cfg) + } + + fmt.Println("\nDone. Binary and databases were NOT removed.") + return 0 +} + +// detectBinaryPath resolves the current binary's real path. +func detectBinaryPath() (string, error) { + exe, err := os.Executable() + if err != nil { + return "", fmt.Errorf("detect binary: %w", err) + } + resolved, err := filepath.EvalSymlinks(exe) + if err != nil { + return "", fmt.Errorf("resolve symlink: %w", err) + } + return resolved, nil +} + +// ensurePATH checks if the binary directory is on PATH and offers to add it. +func ensurePATH(binaryPath string, cfg installConfig) { + binDir := filepath.Dir(binaryPath) + pathDirs := filepath.SplitList(os.Getenv("PATH")) + + fmt.Println("[PATH]") + for _, d := range pathDirs { + if d == binDir { + fmt.Printf(" ✓ %s already on PATH\n", binDir) + return + } + } + + fmt.Printf(" ⚠ %s is not on PATH\n", binDir) + + if runtime.GOOS == "windows" { + fmt.Printf(" → Add %s to your PATH environment variable manually\n", binDir) + return + } + + rcFile := detectShellRC() + if rcFile == "" { + fmt.Printf(" → Add to your shell profile: export PATH=\"%s:$PATH\"\n", binDir) + return + } + + line := fmt.Sprintf("export PATH=\"%s:$PATH\"", binDir) + + // Check if already present in rc file + if content, err := os.ReadFile(rcFile); err == nil { + if strings.Contains(string(content), line) { + fmt.Printf(" ✓ Already in %s (restart terminal to activate)\n", rcFile) + return + } + } + + if cfg.dryRun { + fmt.Printf(" [dry-run] Would append to %s: %s\n", rcFile, line) + } else { + f, err := os.OpenFile(rcFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + fmt.Printf(" ⚠ Could not write to %s: %v\n", rcFile, err) + fmt.Printf(" → Add manually: %s\n", line) + return + } + defer f.Close() + fmt.Fprintf(f, "\n# Added by codebase-memory-mcp install\n%s\n", line) + fmt.Printf(" ✓ Added to %s: %s\n", rcFile, line) + fmt.Printf(" → Run: source %s (or restart terminal)\n", rcFile) + } +} + +// detectShellRC returns the appropriate shell rc file path. +func detectShellRC() string { + home, err := os.UserHomeDir() + if err != nil { + return "" + } + + shell := os.Getenv("SHELL") + switch { + case strings.HasSuffix(shell, "/zsh"): + return filepath.Join(home, ".zshrc") + case strings.HasSuffix(shell, "/bash"): + // Prefer .bashrc, fall back to .bash_profile + bashrc := filepath.Join(home, ".bashrc") + if _, err := os.Stat(bashrc); err == nil { + return bashrc + } + return filepath.Join(home, ".bash_profile") + case strings.HasSuffix(shell, "/fish"): + return filepath.Join(home, ".config", "fish", "config.fish") + default: + // Fall back to .profile + return filepath.Join(home, ".profile") + } +} + +// installClaudeCode installs skills and registers the MCP server with Claude Code. +func installClaudeCode(binaryPath, claudePath string, cfg installConfig) { + home, err := os.UserHomeDir() + if err != nil { + fmt.Printf(" ⚠ Cannot determine home directory: %v\n", err) + return + } + + // Remove old monolithic skill if it exists + oldSkillDir := filepath.Join(home, ".claude", "skills", "codebase-memory-mcp") + if info, err := os.Stat(oldSkillDir); err == nil && info.IsDir() { + if cfg.dryRun { + fmt.Printf(" [dry-run] Would remove old skill: %s\n", oldSkillDir) + } else { + if err := os.RemoveAll(oldSkillDir); err == nil { + fmt.Printf(" ✓ Removed old monolithic skill: %s\n", oldSkillDir) + } + } + } + + // Write 4 skill files + for name, content := range skillFiles { + skillDir := filepath.Join(home, ".claude", "skills", name) + skillFile := filepath.Join(skillDir, "SKILL.md") + + if !cfg.force { + if _, err := os.Stat(skillFile); err == nil { + fmt.Printf(" ✓ Skill exists (skip): %s\n", skillFile) + continue + } + } + + if cfg.dryRun { + fmt.Printf(" [dry-run] Would write: %s\n", skillFile) + continue + } + + if err := os.MkdirAll(skillDir, 0o750); err != nil { + fmt.Printf(" ⚠ mkdir %s: %v\n", skillDir, err) + continue + } + if err := os.WriteFile(skillFile, []byte(content), 0o600); err != nil { + fmt.Printf(" ⚠ write %s: %v\n", skillFile, err) + continue + } + fmt.Printf(" ✓ Skill: %s\n", skillFile) + } + + // Register MCP server — remove first for idempotency + if cfg.dryRun { + fmt.Printf(" [dry-run] Would run: %s mcp remove codebase-memory-mcp\n", claudePath) + fmt.Printf(" [dry-run] Would run: %s mcp add --scope user codebase-memory-mcp -- %s\n", claudePath, binaryPath) + } else { + // Silent remove (may fail if not registered — that's fine) + _ = execCLI(claudePath, "mcp", "remove", "codebase-memory-mcp") + if err := execCLI(claudePath, "mcp", "add", "--scope", "user", "codebase-memory-mcp", "--", binaryPath); err != nil { + fmt.Printf(" ⚠ MCP registration failed: %v\n", err) + } else { + fmt.Println(" ✓ MCP server registered (scope: user)") + } + } +} + +// installCodex installs MCP registration and instructions for Codex CLI. +func installCodex(binaryPath, codexPath string, cfg installConfig) { + home, err := os.UserHomeDir() + if err != nil { + fmt.Printf(" ⚠ Cannot determine home directory: %v\n", err) + return + } + + // Register MCP server + if cfg.dryRun { + fmt.Printf(" [dry-run] Would run: %s mcp remove codebase-memory-mcp\n", codexPath) + fmt.Printf(" [dry-run] Would run: %s mcp add codebase-memory-mcp -- %s\n", codexPath, binaryPath) + } else { + _ = execCLI(codexPath, "mcp", "remove", "codebase-memory-mcp") + if err := execCLI(codexPath, "mcp", "add", "codebase-memory-mcp", "--", binaryPath); err != nil { + fmt.Printf(" ⚠ MCP registration failed: %v\n", err) + } else { + fmt.Println(" ✓ MCP server registered") + } + } + + // Write instructions file + instrDir := filepath.Join(home, ".codex", "instructions") + instrFile := filepath.Join(instrDir, "codebase-memory-mcp.md") + + if cfg.dryRun { + fmt.Printf(" [dry-run] Would write: %s\n", instrFile) + } else { + if err := os.MkdirAll(instrDir, 0o750); err != nil { + fmt.Printf(" ⚠ mkdir %s: %v\n", instrDir, err) + return + } + if err := os.WriteFile(instrFile, []byte(codexInstructions), 0o600); err != nil { + fmt.Printf(" ⚠ write %s: %v\n", instrFile, err) + return + } + fmt.Printf(" ✓ Instructions: %s\n", instrFile) + } +} + +// removeClaudeSkills removes all 4 skill directories. +func removeClaudeSkills(cfg installConfig) { + home, err := os.UserHomeDir() + if err != nil { + return + } + + fmt.Println("[Skills]") + for name := range skillFiles { + skillDir := filepath.Join(home, ".claude", "skills", name) + if _, err := os.Stat(skillDir); os.IsNotExist(err) { + continue + } + if cfg.dryRun { + fmt.Printf(" [dry-run] Would remove: %s\n", skillDir) + } else { + if err := os.RemoveAll(skillDir); err != nil { + fmt.Printf(" ⚠ remove %s: %v\n", skillDir, err) + } else { + fmt.Printf(" ✓ Removed: %s\n", skillDir) + } + } + } +} + +// deregisterMCP removes the MCP server registration from a CLI. +func deregisterMCP(cliPath, cliName string, cfg installConfig) { + if cfg.dryRun { + fmt.Printf(" [dry-run] Would run: %s mcp remove codebase-memory-mcp\n", cliPath) + } else { + if err := execCLI(cliPath, "mcp", "remove", "codebase-memory-mcp"); err != nil { + fmt.Printf(" ⚠ %s MCP deregistration: %v\n", cliName, err) + } else { + fmt.Printf(" ✓ %s MCP server deregistered\n", cliName) + } + } +} + +// removeCodexInstructions removes the Codex instructions file. +func removeCodexInstructions(cfg installConfig) { + home, err := os.UserHomeDir() + if err != nil { + return + } + instrFile := filepath.Join(home, ".codex", "instructions", "codebase-memory-mcp.md") + if _, err := os.Stat(instrFile); os.IsNotExist(err) { + return + } + if cfg.dryRun { + fmt.Printf(" [dry-run] Would remove: %s\n", instrFile) + } else { + if err := os.Remove(instrFile); err != nil { + fmt.Printf(" ⚠ remove %s: %v\n", instrFile, err) + } else { + fmt.Printf(" ✓ Removed: %s\n", instrFile) + } + } +} + +// findCLI locates a CLI binary by name. +func findCLI(name string) string { + if p, err := exec.LookPath(name); err == nil { + return p + } + + // Check common install locations + home, err := os.UserHomeDir() + if err != nil { + return "" + } + + candidates := []string{ + "/usr/local/bin/" + name, + filepath.Join(home, ".npm", "bin", name), + filepath.Join(home, ".local", "bin", name), + filepath.Join(home, ".cargo", "bin", name), + } + if runtime.GOOS == "darwin" { + candidates = append(candidates, "/opt/homebrew/bin/"+name) + } + + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + return c + } + } + return "" +} + +// execCLI runs a CLI command and returns any error. +func execCLI(path string, args ...string) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, path, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/cmd/codebase-memory-mcp/install_test.go b/cmd/codebase-memory-mcp/install_test.go new file mode 100644 index 000000000..dc96eec76 --- /dev/null +++ b/cmd/codebase-memory-mcp/install_test.go @@ -0,0 +1,304 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestInstallSkillCreation(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + claudeSkillsDir := filepath.Join(home, ".claude", "skills") + + for name, content := range skillFiles { + skillDir := filepath.Join(claudeSkillsDir, name) + skillFile := filepath.Join(skillDir, "SKILL.md") + + if err := os.MkdirAll(skillDir, 0o750); err != nil { + t.Fatalf("mkdir %s: %v", skillDir, err) + } + if err := os.WriteFile(skillFile, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", skillFile, err) + } + } + + expectedSkills := []string{ + "codebase-memory-exploring", + "codebase-memory-tracing", + "codebase-memory-quality", + "codebase-memory-reference", + } + + for _, name := range expectedSkills { + skillFile := filepath.Join(claudeSkillsDir, name, "SKILL.md") + data, err := os.ReadFile(skillFile) + if err != nil { + t.Fatalf("read %s: %v", skillFile, err) + } + if len(data) == 0 { + t.Fatalf("skill file %s is empty", skillFile) + } + if !strings.HasPrefix(string(data), "---\n") { + t.Fatalf("skill %s missing YAML frontmatter", name) + } + if !strings.Contains(string(data), "name: "+name) { + t.Fatalf("skill %s doesn't contain correct name field", name) + } + } +} + +func TestInstallIdempotent(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + claudeSkillsDir := filepath.Join(home, ".claude", "skills") + + for round := 0; round < 2; round++ { + for name, content := range skillFiles { + skillDir := filepath.Join(claudeSkillsDir, name) + skillFile := filepath.Join(skillDir, "SKILL.md") + if err := os.MkdirAll(skillDir, 0o750); err != nil { + t.Fatalf("round %d: mkdir %s: %v", round, skillDir, err) + } + if err := os.WriteFile(skillFile, []byte(content), 0o600); err != nil { + t.Fatalf("round %d: write %s: %v", round, skillFile, err) + } + } + } + + for name := range skillFiles { + skillFile := filepath.Join(claudeSkillsDir, name, "SKILL.md") + if _, err := os.Stat(skillFile); err != nil { + t.Fatalf("skill %s missing after idempotent install: %v", name, err) + } + } +} + +func TestUninstallRemovesSkills(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + claudeSkillsDir := filepath.Join(home, ".claude", "skills") + + for name, content := range skillFiles { + skillDir := filepath.Join(claudeSkillsDir, name) + if err := os.MkdirAll(skillDir, 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(content), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + } + + for name := range skillFiles { + skillDir := filepath.Join(claudeSkillsDir, name) + if err := os.RemoveAll(skillDir); err != nil { + t.Fatalf("remove %s: %v", skillDir, err) + } + } + + for name := range skillFiles { + skillDir := filepath.Join(claudeSkillsDir, name) + if _, err := os.Stat(skillDir); !os.IsNotExist(err) { + t.Fatalf("skill dir %s should not exist after uninstall", skillDir) + } + } +} + +func TestFindCLI_NotFound(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + t.Setenv("HOME", t.TempDir()) + + result := findCLI("nonexistent-binary-xyz") + if result != "" { + t.Fatalf("expected empty string for nonexistent CLI, got %q", result) + } +} + +func TestFindCLI_FoundOnPATH(t *testing.T) { + tmpDir := t.TempDir() + + fakeBin := filepath.Join(tmpDir, "fakecli") + if err := os.WriteFile(fakeBin, []byte("#!/bin/sh\n"), 0o600); err != nil { + t.Fatalf("write fake binary: %v", err) + } + if err := os.Chmod(fakeBin, 0o500); err != nil { + t.Fatalf("chmod fake binary: %v", err) + } + + t.Setenv("PATH", tmpDir) + result := findCLI("fakecli") + if result == "" { + t.Fatal("expected to find fakecli on PATH") + } +} + +func TestFindCLI_FallbackPaths(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("PATH", t.TempDir()) + + localBin := filepath.Join(home, ".local", "bin") + if err := os.MkdirAll(localBin, 0o750); err != nil { + t.Fatal(err) + } + fakeBin := filepath.Join(localBin, "testcli") + if err := os.WriteFile(fakeBin, []byte("#!/bin/sh\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(fakeBin, 0o500); err != nil { + t.Fatal(err) + } + + result := findCLI("testcli") + if result != fakeBin { + t.Fatalf("expected %q, got %q", fakeBin, result) + } +} + +func TestDetectBinaryPath(t *testing.T) { + path, err := detectBinaryPath() + if err != nil { + t.Fatalf("detectBinaryPath error: %v", err) + } + if path == "" { + t.Fatal("detectBinaryPath returned empty string") + } + if !filepath.IsAbs(path) { + t.Fatalf("expected absolute path, got %q", path) + } +} + +func TestDetectShellRC(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + tests := []struct { + shell string + expected string + }{ + {"/bin/zsh", ".zshrc"}, + {"/bin/bash", ".bash_profile"}, + {"/usr/bin/fish", filepath.Join(".config", "fish", "config.fish")}, + {"/bin/sh", ".profile"}, + } + + for _, tt := range tests { + t.Run(tt.shell, func(t *testing.T) { + t.Setenv("SHELL", tt.shell) + rc := detectShellRC() + if rc == "" { + t.Fatal("detectShellRC returned empty") + } + if !strings.HasSuffix(rc, tt.expected) { + t.Fatalf("for shell %q: got %q, want suffix %q", tt.shell, rc, tt.expected) + } + }) + } +} + +func TestDetectShellRC_BashWithBashrc(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("SHELL", "/bin/bash") + + bashrc := filepath.Join(home, ".bashrc") + if err := os.WriteFile(bashrc, []byte("# test\n"), 0o600); err != nil { + t.Fatal(err) + } + + rc := detectShellRC() + if rc != bashrc { + t.Fatalf("expected %q, got %q", bashrc, rc) + } +} + +func TestDryRun(t *testing.T) { + cfg := installConfig{} + for _, a := range []string{"--dry-run", "--force"} { + switch a { + case "--dry-run": + cfg.dryRun = true + case "--force": + cfg.force = true + } + } + if !cfg.dryRun || !cfg.force { + t.Fatal("expected both dryRun and force to be true") + } +} + +func TestCodexInstructionsCreation(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + instrDir := filepath.Join(home, ".codex", "instructions") + instrFile := filepath.Join(instrDir, "codebase-memory-mcp.md") + + if err := os.MkdirAll(instrDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(instrFile, []byte(codexInstructions), 0o600); err != nil { + t.Fatal(err) + } + + data, err := os.ReadFile(instrFile) + if err != nil { + t.Fatalf("read instructions: %v", err) + } + if !strings.Contains(string(data), "Codebase Knowledge Graph") { + t.Fatal("instructions file missing expected content") + } + if !strings.Contains(string(data), "trace_call_path") { + t.Fatal("instructions file missing trace_call_path reference") + } +} + +func TestSkillFilesContent(t *testing.T) { + if len(skillFiles) != 4 { + t.Fatalf("expected 4 skill files, got %d", len(skillFiles)) + } + + expectations := map[string][]string{ + "codebase-memory-exploring": {"explore the codebase", "search_graph", "get_graph_schema"}, + "codebase-memory-tracing": {"who calls this function", "trace_call_path", "direction"}, + "codebase-memory-quality": {"find dead code", "max_degree=0", "exclude_entry_points"}, + "codebase-memory-reference": {"edge types", "query_graph", "Cypher"}, + } + + for name, expectedPhrases := range expectations { + content, ok := skillFiles[name] + if !ok { + t.Fatalf("missing skill: %s", name) + } + for _, phrase := range expectedPhrases { + if !strings.Contains(strings.ToLower(content), strings.ToLower(phrase)) { + t.Errorf("skill %q missing phrase %q", name, phrase) + } + } + } +} + +func TestRemoveOldMonolithicSkill(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + oldDir := filepath.Join(home, ".claude", "skills", "codebase-memory-mcp") + if err := os.MkdirAll(oldDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(oldDir, "SKILL.md"), []byte("old skill"), 0o600); err != nil { + t.Fatal(err) + } + + if err := os.RemoveAll(oldDir); err != nil { + t.Fatalf("remove old skill: %v", err) + } + + if _, err := os.Stat(oldDir); !os.IsNotExist(err) { + t.Fatal("old monolithic skill dir should be removed") + } +} diff --git a/cmd/codebase-memory-mcp/main.go b/cmd/codebase-memory-mcp/main.go index 150af79ac..eeec3253b 100644 --- a/cmd/codebase-memory-mcp/main.go +++ b/cmd/codebase-memory-mcp/main.go @@ -17,13 +17,22 @@ import ( var version = "dev" func main() { - if len(os.Args) > 1 && os.Args[1] == "--version" { - fmt.Println("codebase-memory-mcp", version) - os.Exit(0) - } - - if len(os.Args) >= 3 && os.Args[1] == "cli" { - os.Exit(runCLI(os.Args[2:])) + if len(os.Args) > 1 { + switch os.Args[1] { + case "--version": + fmt.Println("codebase-memory-mcp", version) + os.Exit(0) + case "install": + os.Exit(runInstall(os.Args[2:])) + case "uninstall": + os.Exit(runUninstall(os.Args[2:])) + case "update": + os.Exit(runUpdate(os.Args[2:])) + case "cli": + if len(os.Args) >= 3 { + os.Exit(runCLI(os.Args[2:])) + } + } } router, err := store.NewRouter() diff --git a/cmd/codebase-memory-mcp/update.go b/cmd/codebase-memory-mcp/update.go new file mode 100644 index 000000000..524d13bf0 --- /dev/null +++ b/cmd/codebase-memory-mcp/update.go @@ -0,0 +1,234 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/DeusData/codebase-memory-mcp/internal/selfupdate" +) + +// newCommand wraps exec.Command for testability. +var newCommand = exec.Command + +func runUpdate(args []string) int { + dryRun := false + for _, a := range args { + if a == "--dry-run" { + dryRun = true + } + } + + currentVersion := strings.TrimSuffix(version, "-dev") + fmt.Printf("\ncodebase-memory-mcp %s — checking for updates...\n", version) + + if runtime.GOOS == "windows" { + fmt.Println("Self-update is not supported on Windows.") + fmt.Println("Download the latest release manually from:") + fmt.Println(" https://github.com/DeusData/codebase-memory-mcp/releases/latest") + return 1 + } + + ctx := context.Background() + + release, err := selfupdate.FetchLatestRelease(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "error: fetch release: %v\n", err) + return 1 + } + + latest := release.LatestVersion() + if latest == "" { + fmt.Println("Could not determine latest version.") + return 1 + } + + if selfupdate.CompareVersions(latest, currentVersion) <= 0 { + fmt.Printf("Already up to date (v%s).\n", currentVersion) + return 0 + } + + fmt.Printf("Update available: v%s → v%s\n", currentVersion, latest) + + assetName := selfupdate.AssetName() + asset := release.FindAsset(assetName) + if asset == nil { + fmt.Fprintf(os.Stderr, "error: no release asset for %s/%s (%s)\n", runtime.GOOS, runtime.GOARCH, assetName) + return 1 + } + + if dryRun { + fmt.Printf("[dry-run] Would download: %s (%d bytes)\n", assetName, asset.Size) + fmt.Println("[dry-run] Would replace binary and re-run install.") + return 0 + } + + binaryData, err := downloadAndVerify(ctx, release, assetName, asset) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + return 1 + } + + if err := replaceBinary(binaryData); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + return 1 + } + + fmt.Println("Re-applying installation...") + runInstall([]string{}) + + fmt.Printf("\nUpdated to v%s. Restart Claude Code / Codex to activate.\n", latest) + return 0 +} + +// downloadAndVerify downloads the release asset and verifies its checksum. +func downloadAndVerify(ctx context.Context, release *selfupdate.Release, assetName string, asset *selfupdate.Asset) ([]byte, error) { + fmt.Println("Downloading checksums...") + checksums, err := selfupdate.DownloadChecksums(ctx, release) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: %v (skipping checksum verification)\n", err) + checksums = nil + } + + fmt.Printf("Downloading %s...\n", assetName) + body, _, err := selfupdate.DownloadAsset(ctx, asset.BrowserDownloadURL) + if err != nil { + return nil, fmt.Errorf("download: %w", err) + } + + tarballData, err := io.ReadAll(body) + body.Close() + if err != nil { + return nil, fmt.Errorf("read download: %w", err) + } + + if checksums != nil { + if expected, ok := checksums[assetName]; ok { + hash := sha256.Sum256(tarballData) + actual := hex.EncodeToString(hash[:]) + if actual != expected { + return nil, fmt.Errorf("checksum mismatch\n expected: %s\n actual: %s", expected, actual) + } + fmt.Println("Checksum verified.") + } + } + + binaryData, err := extractBinaryFromTarGz(tarballData) + if err != nil { + return nil, fmt.Errorf("extract: %w", err) + } + + return binaryData, nil +} + +// replaceBinary atomically swaps the current binary with the new one. +func replaceBinary(binaryData []byte) error { + binaryPath, err := detectBinaryPath() + if err != nil { + return err + } + + fmt.Printf("Replacing binary at %s...\n", binaryPath) + + tmpPath := binaryPath + ".tmp" + if err := os.WriteFile(tmpPath, binaryData, 0o600); err != nil { + return fmt.Errorf("write temp: %w", err) + } + if err := os.Chmod(tmpPath, 0o500); err != nil { + return fmt.Errorf("chmod temp: %w", err) + } + + bakPath := binaryPath + ".bak" + if cpErr := copyFile(binaryPath, bakPath); cpErr != nil { + fmt.Fprintf(os.Stderr, "warning: backup failed: %v\n", cpErr) + } + + if err := os.Rename(tmpPath, binaryPath); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("rename: %w", err) + } + + if err := verifyBinary(binaryPath); err != nil { + fmt.Println("Restoring previous version...") + if restoreErr := os.Rename(bakPath, binaryPath); restoreErr != nil { + return fmt.Errorf("restore failed (%v), backup at: %s", restoreErr, bakPath) + } + fmt.Println("Previous version restored.") + return fmt.Errorf("new binary verification failed: %w", err) + } + + os.Remove(bakPath) + return nil +} + +// extractBinaryFromTarGz extracts the first regular file from a .tar.gz archive. +func extractBinaryFromTarGz(data []byte) ([]byte, error) { + gr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("gzip: %w", err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("tar: %w", err) + } + if hdr.Typeflag == tar.TypeReg && strings.HasPrefix(filepath.Base(hdr.Name), "codebase-memory-mcp") { + content, err := io.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("read entry: %w", err) + } + return content, nil + } + } + return nil, fmt.Errorf("binary not found in archive") +} + +// verifyBinary runs --version on the new binary to ensure it works. +func verifyBinary(path string) error { + cmd := newCommand(path, "--version") + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("--version failed: %w", err) + } + output := strings.TrimSpace(string(out)) + if !strings.Contains(output, "codebase-memory-mcp") { + return fmt.Errorf("unexpected output: %s", output) + } + return nil +} + +// copyFile copies a file from src to dst. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Close() +} diff --git a/cmd/codebase-memory-mcp/update_test.go b/cmd/codebase-memory-mcp/update_test.go new file mode 100644 index 000000000..7f491d6ca --- /dev/null +++ b/cmd/codebase-memory-mcp/update_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "os" + "path/filepath" + "testing" +) + +func TestExtractBinaryFromTarGz(t *testing.T) { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + content := []byte("fake binary content") + hdr := &tar.Header{ + Name: "codebase-memory-mcp-linux-amd64", + Mode: 0o700, + Size: int64(len(content)), + Typeflag: tar.TypeReg, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(content); err != nil { + t.Fatal(err) + } + tw.Close() + gw.Close() + + extracted, err := extractBinaryFromTarGz(buf.Bytes()) + if err != nil { + t.Fatalf("extractBinaryFromTarGz error: %v", err) + } + if !bytes.Equal(extracted, content) { + t.Fatalf("extracted content mismatch: %q vs %q", extracted, content) + } +} + +func TestExtractBinaryFromTarGz_NotFound(t *testing.T) { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + hdr := &tar.Header{ + Name: "some-other-file", + Mode: 0o600, + Size: 5, + Typeflag: tar.TypeReg, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + tw.Close() + gw.Close() + + _, err := extractBinaryFromTarGz(buf.Bytes()) + if err == nil { + t.Fatal("expected error when binary not found in archive") + } +} + +func TestExtractBinaryFromTarGz_InvalidData(t *testing.T) { + _, err := extractBinaryFromTarGz([]byte("not a valid tar.gz")) + if err == nil { + t.Fatal("expected error for invalid tar.gz data") + } +} + +func TestCopyFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "source") + dst := filepath.Join(dir, "dest") + + content := []byte("test content for copy") + if err := os.WriteFile(src, content, 0o600); err != nil { + t.Fatal(err) + } + + if err := copyFile(src, dst); err != nil { + t.Fatalf("copyFile error: %v", err) + } + + data, err := os.ReadFile(dst) + if err != nil { + t.Fatalf("read dst: %v", err) + } + if !bytes.Equal(data, content) { + t.Fatalf("copy mismatch: %q vs %q", data, content) + } +} + +func TestCopyFile_SourceNotFound(t *testing.T) { + dir := t.TempDir() + err := copyFile(filepath.Join(dir, "nonexistent"), filepath.Join(dir, "dest")) + if err == nil { + t.Fatal("expected error for nonexistent source") + } +} diff --git a/go.mod b/go.mod index 1549d23d1..4e4e8ab1c 100644 --- a/go.mod +++ b/go.mod @@ -3,18 +3,40 @@ module github.com/DeusData/codebase-memory-mcp go 1.26 require ( + github.com/DerekStride/tree-sitter-sql v0.0.0 + github.com/UserNobody14/tree-sitter-dart v0.0.0 + github.com/alex-pinkus/tree-sitter-swift v0.0.0 + github.com/camdencheek/tree-sitter-dockerfile v0.0.0 github.com/mattn/go-sqlite3 v1.14.34 github.com/modelcontextprotocol/go-sdk v1.3.1 + github.com/murtaza64/tree-sitter-groovy v0.0.0 + github.com/r-lib/tree-sitter-r v0.0.0 + github.com/tree-sitter-grammars/tree-sitter-hcl v1.2.0 github.com/tree-sitter-grammars/tree-sitter-kotlin v1.1.0 github.com/tree-sitter-grammars/tree-sitter-lua v0.4.1 + github.com/tree-sitter-grammars/tree-sitter-objc v1.2.1-0.20250131075517-181a81b8f23a + github.com/tree-sitter-grammars/tree-sitter-scss v0.0.0 + github.com/tree-sitter-grammars/tree-sitter-toml v0.7.0 + github.com/tree-sitter-grammars/tree-sitter-yaml v0.7.2 + github.com/tree-sitter-grammars/tree-sitter-zig v1.1.2 github.com/tree-sitter/go-tree-sitter v0.25.0 + github.com/tree-sitter/tree-sitter-bash v0.25.1 + github.com/tree-sitter/tree-sitter-c v0.24.1 github.com/tree-sitter/tree-sitter-c-sharp v0.23.1 github.com/tree-sitter/tree-sitter-cpp v0.23.4 + github.com/tree-sitter/tree-sitter-css v0.25.0 + github.com/tree-sitter/tree-sitter-elixir v0.0.0 + github.com/tree-sitter/tree-sitter-erlang v0.0.0 github.com/tree-sitter/tree-sitter-go v0.25.0 + github.com/tree-sitter/tree-sitter-haskell v0.0.0 + github.com/tree-sitter/tree-sitter-html v0.23.2 github.com/tree-sitter/tree-sitter-java v0.23.5 github.com/tree-sitter/tree-sitter-javascript v0.25.0 + github.com/tree-sitter/tree-sitter-ocaml v0.24.2 + github.com/tree-sitter/tree-sitter-perl v0.0.0 github.com/tree-sitter/tree-sitter-php v0.24.2 github.com/tree-sitter/tree-sitter-python v0.25.0 + github.com/tree-sitter/tree-sitter-ruby v0.23.1 github.com/tree-sitter/tree-sitter-rust v0.24.0 github.com/tree-sitter/tree-sitter-scala v0.24.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 @@ -29,6 +51,7 @@ require ( github.com/mattn/go-pointer v0.0.1 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.3 // indirect + github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sys v0.37.0 // indirect @@ -36,3 +59,25 @@ require ( ) replace github.com/tree-sitter/go-tree-sitter => github.com/DeusData/go-tree-sitter v0.26.0-deusdata.1 + +replace github.com/tree-sitter/tree-sitter-elixir => github.com/elixir-lang/tree-sitter-elixir v0.3.4 + +replace github.com/tree-sitter/tree-sitter-haskell => github.com/tree-sitter-grammars/tree-sitter-haskell v0.0.0-20251125091915-7fa19f195803 + +replace github.com/UserNobody14/tree-sitter-dart => github.com/DeusData/tree-sitter-dart v0.0.1-deusdata.1 + +replace github.com/DerekStride/tree-sitter-sql => github.com/DeusData/tree-sitter-sql v0.0.2-deusdata.1 + +replace github.com/tree-sitter/tree-sitter-perl => github.com/DeusData/tree-sitter-perl v0.0.4-deusdata.1 + +replace github.com/murtaza64/tree-sitter-groovy => github.com/DeusData/tree-sitter-groovy v0.0.2-deusdata.1 + +replace github.com/tree-sitter-grammars/tree-sitter-scss => github.com/DeusData/tree-sitter-scss v0.0.2-deusdata.1 + +replace github.com/r-lib/tree-sitter-r => github.com/DeusData/tree-sitter-r v0.0.1-deusdata.1 + +replace github.com/tree-sitter/tree-sitter-erlang => github.com/DeusData/tree-sitter-erlang v0.0.1-deusdata.1 + +replace github.com/alex-pinkus/tree-sitter-swift => github.com/DeusData/tree-sitter-swift v0.0.3-deusdata.1 + +replace github.com/camdencheek/tree-sitter-dockerfile => github.com/DeusData/tree-sitter-dockerfile v0.0.1-deusdata.1 diff --git a/go.sum b/go.sum index 1fff65414..8ff691fc1 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,27 @@ github.com/DeusData/go-tree-sitter v0.26.0-deusdata.1 h1:/cTwrXEE/3t6HYOvnHzgSZpO8MOwxFFFlz65Cc9xf9U= github.com/DeusData/go-tree-sitter v0.26.0-deusdata.1/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU= +github.com/DeusData/tree-sitter-dart v0.0.1-deusdata.1 h1:G8ZJ9glIj73w6fUV9+INR6fGLNi3udrRjs7Bd8yIVAc= +github.com/DeusData/tree-sitter-dart v0.0.1-deusdata.1/go.mod h1:gaimGqOZqVTIc7JSlnsUqjXSjAYisTq7nhzAR5VQ0Rw= +github.com/DeusData/tree-sitter-dockerfile v0.0.1-deusdata.1 h1:WU/rFyOW5PiDHbvsiC2B8Y/r0f6ttfdZOA+HM62Nsyw= +github.com/DeusData/tree-sitter-dockerfile v0.0.1-deusdata.1/go.mod h1:pc+Ka90MBUzYW0uqj0djhsC/jooRI39T6bqN4+vUaag= +github.com/DeusData/tree-sitter-erlang v0.0.1-deusdata.1 h1:HhK7QUh57lSOz+tPztIpoKthVIj+P6+EBQh+dg2smao= +github.com/DeusData/tree-sitter-erlang v0.0.1-deusdata.1/go.mod h1:BeJG7Ezjd3yHpmBcc4Gjw8xkXQz3BDrQdPr7c46z3MA= +github.com/DeusData/tree-sitter-groovy v0.0.2-deusdata.1 h1:eawA+AystZ+aKSJqjKOSo7YHdTy2u9Q1+443MGZgYJA= +github.com/DeusData/tree-sitter-groovy v0.0.2-deusdata.1/go.mod h1:YWGloyOrnz4RXXlyAu0MmLf3qkpoYwfZAJ9Hn658Ywo= +github.com/DeusData/tree-sitter-perl v0.0.4-deusdata.1 h1:4HNDSdmGBeMI+cQF/MYlDkGpbvh7dNHZCg2bx2LjJqo= +github.com/DeusData/tree-sitter-perl v0.0.4-deusdata.1/go.mod h1:mUpbTt2dUb6i1ckLX98gXNyCRdjOpHlk7MQqSA37T1A= +github.com/DeusData/tree-sitter-r v0.0.1-deusdata.1 h1:pyTuUUefQpPKQVdJ7159g6xehzi/2xzgpALWNgwt4FY= +github.com/DeusData/tree-sitter-r v0.0.1-deusdata.1/go.mod h1:EiQTehwt8ojqpEcjZHKJmU26zijUuHkmwFi6Rot/GGQ= +github.com/DeusData/tree-sitter-scss v0.0.2-deusdata.1 h1:JgA+OjWTaQqN09QKvYxDo1WuZvDZ4AHt2dUJuZRjiYc= +github.com/DeusData/tree-sitter-scss v0.0.2-deusdata.1/go.mod h1:Fi256VkrezhT/uu/vZ7dJVHthET6S8PcZ2YNq/Bz3wE= +github.com/DeusData/tree-sitter-sql v0.0.2-deusdata.1 h1:+6yX51GXklTXWRVs5TGkhaWlS7MRjwtjTGUs9j89grg= +github.com/DeusData/tree-sitter-sql v0.0.2-deusdata.1/go.mod h1:tKhfNbTiFmw3xaK1QSZjCPlOXhPkcf168z/FOpsBDCQ= +github.com/DeusData/tree-sitter-swift v0.0.3-deusdata.1 h1:QB0FpUDRIUZ/dQCrCx65idtPaHnvgSXzkM9NzETDmqY= +github.com/DeusData/tree-sitter-swift v0.0.3-deusdata.1/go.mod h1:nMZLtsEtko+0F2g09G5Tm16uRdDoczLZDE5O207ruYQ= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elixir-lang/tree-sitter-elixir v0.3.4 h1:M0Wc3XXEJyGpVELOzxqhV069bBsRAQyGdehWw/G+nWk= +github.com/elixir-lang/tree-sitter-elixir v0.3.4/go.mod h1:wNBVf64kzvhSbZ8ojVtBF1jRiqGY0lsuK5Kx/60s6Z0= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -22,18 +42,36 @@ github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.3 h1:OjMgICtcSFuNvQCdwqMCv9Tg7lEOXGwm1J5RPQccx6w= github.com/segmentio/encoding v0.5.3/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4= +github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tree-sitter-grammars/tree-sitter-haskell v0.0.0-20251125091915-7fa19f195803 h1:qGglJu/VLlh+LvkvOK26WHK3YL+wHbpJERiPpQhWst8= +github.com/tree-sitter-grammars/tree-sitter-haskell v0.0.0-20251125091915-7fa19f195803/go.mod h1:NWY55NC9F0agDcuCKOkPqAQWZQETS3cUjO+T+a21IIc= +github.com/tree-sitter-grammars/tree-sitter-hcl v1.2.0 h1:jl3v597Dii91OHcHAUrTQaSEK7oODNh6yK8z4H5xXFA= +github.com/tree-sitter-grammars/tree-sitter-hcl v1.2.0/go.mod h1:NzEB+a6rpv14H4tCrCfzhTorCyvYYb3T9+o/e4d0pdU= github.com/tree-sitter-grammars/tree-sitter-kotlin v1.1.0 h1:SWIUDASa+WPhDDem1U5IJpYwQEezkqXrUI61OcnORzM= github.com/tree-sitter-grammars/tree-sitter-kotlin v1.1.0/go.mod h1:eH+flFf3QOa9c9BY9g3Bz02F7zTq30kGIG3cgB0lSlI= github.com/tree-sitter-grammars/tree-sitter-lua v0.4.1 h1:DOxLDVTEmkLuoIHXJ65h5oyHcEK89mn9DOhlnPQRAmQ= github.com/tree-sitter-grammars/tree-sitter-lua v0.4.1/go.mod h1:hIOfn+lxpU4SRrtejLVrU2+8SAoRwNC01m3XaR/Cw0A= -github.com/tree-sitter/tree-sitter-c v0.23.4 h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c= -github.com/tree-sitter/tree-sitter-c v0.23.4/go.mod h1:MkI5dOiIpeN94LNjeCp8ljXN/953JCwAby4bClMr6bw= +github.com/tree-sitter-grammars/tree-sitter-objc v1.2.1-0.20250131075517-181a81b8f23a h1:nUAwJerIz8+ndhTYW1i2tpIwFcKX8Y65EkVHq5T+WyM= +github.com/tree-sitter-grammars/tree-sitter-objc v1.2.1-0.20250131075517-181a81b8f23a/go.mod h1:3WHVwcdAgzNBRoNnFqo5NFFpX9Im21tMlnwA9mIiDK8= +github.com/tree-sitter-grammars/tree-sitter-toml v0.7.0 h1:VQOyz+wSHzNZJ7gUMHAWVEP4Xt/YBZkeL7iurBv/9xc= +github.com/tree-sitter-grammars/tree-sitter-toml v0.7.0/go.mod h1:hagl3VSOuQdsaDpGY13oGMz2cLK2tUDAf4mNNW2pDBQ= +github.com/tree-sitter-grammars/tree-sitter-yaml v0.7.2 h1:Mmr9LYGTdr+8iD3ZEK1+j2FKZtY1Jt3y5Wq9dedgwfs= +github.com/tree-sitter-grammars/tree-sitter-yaml v0.7.2/go.mod h1:tOZ8GmiIfLySKNOX58+PV2mvjFJdfciksfwotcELKbg= +github.com/tree-sitter-grammars/tree-sitter-zig v1.1.2 h1:j8JARutysdxMwBEfVLGx9us7cdSzD1TTui/pPLGCFDk= +github.com/tree-sitter-grammars/tree-sitter-zig v1.1.2/go.mod h1:ekWQEqj2e/gQal396f5rKJ6L14/a4bMPMqSRzmf8OZE= +github.com/tree-sitter/tree-sitter-bash v0.25.1 h1:ZD3MK4oDB5lAsFztqbdcyYEd24pxDtx3g9UOWA062rE= +github.com/tree-sitter/tree-sitter-bash v0.25.1/go.mod h1:AksQ6zE+sP9hnp7mKTMT7Q+CwpthV7VGQLXvweVXz9U= +github.com/tree-sitter/tree-sitter-c v0.24.1 h1:GV9DjvIV6uYe3W/JBKMFwE4hJcRxzRDq63llxNFHOkY= +github.com/tree-sitter/tree-sitter-c v0.24.1/go.mod h1:/SpJlv2BuiCgFA5xvtgukFGi51WxctByPUGDxPl60fc= github.com/tree-sitter/tree-sitter-c-sharp v0.23.1 h1:ddG6osP34sMieVNN6lu5ZG/3N8Wn+67+43BmipqidyM= github.com/tree-sitter/tree-sitter-c-sharp v0.23.1/go.mod h1:H7/aFm5vR1A8Yn5VIOfLWPdlKuJsMgZ5eDmaJdv8bY0= github.com/tree-sitter/tree-sitter-cpp v0.23.4 h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w= github.com/tree-sitter/tree-sitter-cpp v0.23.4/go.mod h1:doqNW64BriC7WBCQ1klf0KmJpdEvfxyXtoEybnBo6v8= +github.com/tree-sitter/tree-sitter-css v0.25.0 h1:S5NbzhdZ5LE5V474wmdg+7NthmLjIg5v4wbyewMpziw= +github.com/tree-sitter/tree-sitter-css v0.25.0/go.mod h1:0Z46XCb3L16nVOVw0Lhb43pzloUG/4T6E/pAOE62fEw= github.com/tree-sitter/tree-sitter-embedded-template v0.23.2 h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg= github.com/tree-sitter/tree-sitter-embedded-template v0.23.2/go.mod h1:HNPOhN0qF3hWluYLdxWs5WbzP/iE4aaRVPMsdxuzIaQ= github.com/tree-sitter/tree-sitter-go v0.25.0 h1:cEB0Q3LHgZtS+ECHx9wcP7AwzoOddJFQCVmytX42cVU= @@ -46,6 +84,8 @@ github.com/tree-sitter/tree-sitter-javascript v0.25.0 h1:ZkWETb66/w8cc13yhfnNuHO github.com/tree-sitter/tree-sitter-javascript v0.25.0/go.mod h1:lmGD1EJdCA+v0S1u2fFgepMg/opzSg/4pgFym2FPGAs= github.com/tree-sitter/tree-sitter-json v0.24.8 h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ= github.com/tree-sitter/tree-sitter-json v0.24.8/go.mod h1:F351KK0KGvCaYbZ5zxwx/gWWvZhIDl0eMtn+1r+gQbo= +github.com/tree-sitter/tree-sitter-ocaml v0.24.2 h1:8tK5RFs0WjO1LvdW+qA6N7oYmLvanaDmPU3Ww7hPq+8= +github.com/tree-sitter/tree-sitter-ocaml v0.24.2/go.mod h1:18SxqyGRpOHl8CmxfC2oTs5mXRDveWhtCnprYBAT0oc= github.com/tree-sitter/tree-sitter-php v0.24.2 h1:yy+COnaaHUNDTKODfNbHhVRD4mQpFELTnBK9+EhpO+w= github.com/tree-sitter/tree-sitter-php v0.24.2/go.mod h1:cEzabPRy4doSxaP9CF9u4FXc3L9Q3Mek3XPVCfqJQRw= github.com/tree-sitter/tree-sitter-python v0.25.0 h1:O6XD9v8U1LOcRc3cNj9nM7XufrtEBezE6VrpRrHZDf0= diff --git a/internal/cypher/executor.go b/internal/cypher/executor.go index aa637259b..f859a932d 100644 --- a/internal/cypher/executor.go +++ b/internal/cypher/executor.go @@ -1,6 +1,7 @@ package cypher import ( + "context" "database/sql" "fmt" "regexp" @@ -17,6 +18,7 @@ const maxResultRows = 200 type Executor struct { Store *store.Store regexCache map[string]*regexp.Regexp + ctx context.Context // set by Execute, used for DB queries } // Result holds the tabular output of a query. @@ -40,6 +42,9 @@ func newBinding() binding { // Execute parses, plans, and executes a Cypher query across all projects. func (e *Executor) Execute(query string) (*Result, error) { + if e.ctx == nil { + e.ctx = context.Background() + } q, err := Parse(query) if err != nil { return nil, fmt.Errorf("parse: %w", err) @@ -258,7 +263,7 @@ func (e *Executor) executeAggregateForProject( ) []map[string]any { query, args := buildProjectAggregateSQL(project, ctx, scan, expand, filter) - rows, qErr := e.Store.DB().Query(query, args...) + rows, qErr := e.Store.DB().QueryContext(e.ctx, query, args...) if qErr != nil { return nil } @@ -500,7 +505,7 @@ func (e *Executor) execJoinScanExpand(project string, scan *ScanNodes, expand *E args = append(args, maxResultRows*2) - rows, err := e.Store.DB().Query(query, args...) + rows, err := e.Store.DB().QueryContext(e.ctx, query, args...) if err != nil { return nil, fmt.Errorf("join scan+expand: %w", err) } @@ -592,7 +597,7 @@ func (e *Executor) execScan(project string, s *ScanNodes, pushDown *FilterWhere) } } - rows, err := e.Store.DB().Query(query, args...) + rows, err := e.Store.DB().QueryContext(e.ctx, query, args...) if err != nil { return nil, fmt.Errorf("scan nodes: %w", err) } @@ -1316,7 +1321,7 @@ func buildGroups(bindings []binding, groupItems []ReturnItem) (groups map[string for _, b := range bindings { row := make(map[string]any) - var keyParts []string + keyParts := make([]string, 0, len(groupItems)) for _, item := range groupItems { col := item.Variable if item.Property != "" { diff --git a/internal/discover/discover.go b/internal/discover/discover.go index 2bda463da..615bff888 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -2,6 +2,7 @@ package discover import ( "bufio" + "context" "os" "path/filepath" "strings" @@ -60,12 +61,17 @@ func shouldSkipDir(name, rel string, extraIgnore []string) bool { } // Discover walks a repository and returns all source files. -func Discover(repoPath string, opts *Options) ([]FileInfo, error) { +func Discover(ctx context.Context, repoPath string, opts *Options) ([]FileInfo, error) { repoPath, err := filepath.Abs(repoPath) if err != nil { return nil, err } + // Check cancellation before starting walk + if err := ctx.Err(); err != nil { + return nil, err + } + // Load .cgrignore patterns if present var extraIgnore []string if opts != nil && opts.IgnoreFile != "" { @@ -78,6 +84,11 @@ func Discover(repoPath string, opts *Options) ([]FileInfo, error) { var files []FileInfo err = filepath.Walk(repoPath, func(path string, info os.FileInfo, walkErr error) error { + // Check context cancellation periodically during walk + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { return filepath.SkipDir } diff --git a/internal/discover/discover_test.go b/internal/discover/discover_test.go new file mode 100644 index 000000000..bd43f6dfd --- /dev/null +++ b/internal/discover/discover_test.go @@ -0,0 +1,61 @@ +package discover + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" +) + +func TestDiscoverBasic(t *testing.T) { + dir := t.TempDir() + + // Create a Go file and a Python file + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("package main\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "app.py"), []byte("def main(): pass\n"), 0o600); err != nil { + t.Fatal(err) + } + + ctx := context.Background() + files, err := Discover(ctx, dir, nil) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + if len(files) != 2 { + t.Fatalf("expected 2 files, got %d", len(files)) + } + + // Verify file info is populated + for _, f := range files { + if f.Path == "" { + t.Error("expected non-empty Path") + } + if f.RelPath == "" { + t.Error("expected non-empty RelPath") + } + if f.Language == "" { + t.Error("expected non-empty Language") + } + } +} + +func TestDiscoverCancellation(t *testing.T) { + dir := t.TempDir() + + // Create a file so the directory isn't empty + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte("package main\n"), 0o600); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // pre-cancel + + _, err := Discover(ctx, dir, nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} diff --git a/internal/httplink/httplink.go b/internal/httplink/httplink.go index 11b1b56a6..9a2358b45 100644 --- a/internal/httplink/httplink.go +++ b/internal/httplink/httplink.go @@ -21,6 +21,7 @@ type RouteHandler struct { QualifiedName string HandlerRef string // resolved handler function reference (e.g. "h.CreateOrder") ResolvedHandlerQN string // set by createRegistrationCallEdges — actual handler QN + Protocol string // "ws", "sse", or "" (standard HTTP) } // HTTPCallSite represents a discovered HTTP call site. @@ -70,25 +71,27 @@ func (l *Linker) AddCallSites(sites []HTTPCallSite) { // regex patterns for route and URL discovery var ( - // Python decorators: @app.post("/path"), @router.get("/path") - pyRouteRe = regexp.MustCompile(`@\w+\.(get|post|put|delete|patch)\(\s*["']([^"']+)["']`) + // Python decorators: @app.post("/path"), @router.get(""), @router.get("/path") + pyRouteRe = regexp.MustCompile(`@\w+\.(get|post|put|delete|patch)\(\s*["']([^"']*)["']`) - // Go gin routes: .POST("/path", .GET("/path" - goRouteRe = regexp.MustCompile(`\.(GET|POST|PUT|DELETE|PATCH)\(\s*["']([^"']+)["']`) + // Go gin/chi routes: .POST("/path"), .Get("/path"), .POST("", handler) + goRouteRe = regexp.MustCompile(`\.(GET|POST|PUT|DELETE|PATCH|Get|Post|Put|Delete|Patch)\(\s*["']([^"']*)["']`) // Go gin group: .Group("/prefix") goGroupRe = regexp.MustCompile(`(\w+)\s*(?::=|=)\s*\w+\.Group\(\s*["']([^"']+)["']`) - // Go gin route handler reference: captures the last argument (handler, not middleware) - // .POST("/path", h.CreateOrder) or .POST("/path", middleware, h.CreateOrder) - goRouteHandlerRe = regexp.MustCompile(`\.(GET|POST|PUT|DELETE|PATCH)\s*\(\s*"[^"]*"\s*(?:,\s*[\w.]+)*,\s*([\w.]+)\s*\)`) + // Go gin/chi route handler reference: captures the last argument (handler, not middleware) + // .POST("/path", h.CreateOrder) or .Get("/path", handler) + goRouteHandlerRe = regexp.MustCompile(`\.(GET|POST|PUT|DELETE|PATCH|Get|Post|Put|Delete|Patch)\s*\(\s*"[^"]*"\s*(?:,\s*[\w.]+)*,\s*([\w.]+)\s*\)`) - // Express.js routes: app.get("/path", router.post("/path" - expressRouteRe = regexp.MustCompile(`\w+\.(get|post|put|delete|patch)\(\s*["'` + "`" + `]([^"'` + "`" + `]+)["'` + "`" + `]`) + // Go chi: r.Route("/prefix", func(r chi.Router) { ... }) + goChiRouteRe = regexp.MustCompile(`\.Route\(\s*"([^"]+)"\s*,\s*func`) - // Express.js handler reference: captures last named argument (handler, not middleware) - // app.get("/path", handler) or app.get("/path", middleware, handler) - expressHandlerRe = regexp.MustCompile(`\w+\.(get|post|put|delete|patch)\(\s*["'` + "`" + `][^"'` + "`" + `]+["'` + "`" + `]\s*(?:,\s*[\w.]+)*,\s*([\w.]+)\s*\)`) + // Express.js routes: captures (receiver).(method)("path") — filtered by allowlist + expressRouteRe = regexp.MustCompile(`(\w+)\.(get|post|put|delete|patch)\(\s*["'` + "`" + `]([^"'` + "`" + `]+)["'` + "`" + `]`) + + // Express.js handler reference: captures (receiver).(method)("path", ..., handler) + expressHandlerRe = regexp.MustCompile(`(\w+)\.(get|post|put|delete|patch)\(\s*["'` + "`" + `][^"'` + "`" + `]+["'` + "`" + `]\s*(?:,\s*[\w.]+)*,\s*([\w.]+)\s*\)`) // Java Spring annotations: @GetMapping("/path"), @PostMapping, @RequestMapping springMappingRe = regexp.MustCompile(`@(Get|Post|Put|Delete|Patch|Request)Mapping\(\s*(?:value\s*=\s*)?["']([^"']+)["']`) @@ -99,6 +102,13 @@ var ( // PHP Laravel routes: Route::get("/path", Route::post("/path" laravelRouteRe = regexp.MustCompile(`Route::(get|post|put|delete|patch)\(\s*["']([^"']+)["']`) + // C# ASP.NET route attributes: [HttpGet("/path")], [Route("/path")] + aspnetRouteRe = regexp.MustCompile(`\[(Http(?:Get|Post|Put|Delete|Patch))\(\s*"([^"]+)"`) + aspnetRouteAttrRe = regexp.MustCompile(`\[Route\(\s*"([^"]+)"`) + + // Kotlin Ktor routes: get("/path") {, post("/path") { + ktorRouteRe = regexp.MustCompile(`\b(get|post|put|delete|patch)\(\s*"([^"]+)"\s*\)`) + // Laravel handler: Route::get("/path", [Controller::class, "method"]) or "Controller@method" laravelHandlerArrayRe = regexp.MustCompile(`Route::(get|post|put|delete|patch)\(\s*["'][^"']+["']\s*,\s*\[(\w+)::class\s*,\s*["'](\w+)["']\]`) laravelHandlerAtRe = regexp.MustCompile(`Route::(get|post|put|delete|patch)\(\s*["'][^"']+["']\s*,\s*["'](\w+)@(\w+)["']`) @@ -109,11 +119,40 @@ var ( // Path-only patterns: "/api/something" (quoted paths starting with /) pathRe = regexp.MustCompile(`["'](/[a-zA-Z0-9_/:.\-]{2,})["']`) + // Python WebSocket routes: @app.websocket("/path"), @app.websocket("") + pyWSRouteRe = regexp.MustCompile(`@\w+\.websocket\(\s*["']([^"']*)["']`) + + // Spring WebSocket: @MessageMapping("/path") + springWSRe = regexp.MustCompile(`@MessageMapping\(\s*["']([^"']+)["']`) + + // Kotlin Ktor WebSocket: webSocket("/path") { + ktorWSRe = regexp.MustCompile(`\bwebSocket\(\s*"([^"]+)"\s*\)`) + + // FastAPI prefix: app.include_router(var, prefix="/prefix") + fastAPIIncludeRe = regexp.MustCompile(`\.include_router\(\s*(\w+)\s*,\s*prefix\s*=\s*["']([^"']+)["']`) + + // Python import: from module.path import var_name + pyImportRe = regexp.MustCompile(`from\s+([\w.]+)\s+import\s+(\w+)`) + + // Express prefix: app.use("/prefix", routerVar) + expressUseRe = regexp.MustCompile(`\.use\(\s*["'` + "`" + `]([^"'` + "`" + `]+)["'` + "`" + `]\s*,\s*(\w+)`) + + // JS/TS import patterns for router variable resolution + jsRequireRe = regexp.MustCompile(`(?:const|let|var)\s+(\w+)\s*=\s*require\(\s*["']([^"']+)["']`) + jsImportRe = regexp.MustCompile(`import\s+(\w+)\s+from\s+["']([^"']+)["']`) + // Path param normalizers colonParamRe = regexp.MustCompile(`:[a-zA-Z_]+`) braceParamRe = regexp.MustCompile(`\{[a-zA-Z_]+\}`) ) +// expressReceiverAllowlist restricts Express route matching to known router variable names. +// Prevents false positives from req.get("Header"), res.get("key"), map.get("key"), etc. +var expressReceiverAllowlist = map[string]bool{ + "app": true, "router": true, "server": true, "api": true, + "routes": true, "express": true, "route": true, +} + // Run executes the HTTP linking pass. func (l *Linker) Run() ([]HTTPLink, error) { proj, err := l.store.GetProject(l.project) @@ -139,13 +178,15 @@ func (l *Linker) Run() ([]HTTPLink, error) { } // Resolve cross-file group prefixes before inserting Route nodes - l.resolveCrossFileGroupPrefixes(routes, rootPath) + l.resolveCrossFileGroupPrefixes(routes, rootPath) // Go gin + l.resolveFastAPIPrefixes(routes, rootPath) // Python FastAPI + l.resolveExpressPrefixes(routes, rootPath) // JS/TS Express // Resolve handler references first so insertRouteNodes can use the actual handler QN l.createRegistrationCallEdges(routes) // Insert Route nodes and HANDLES edges (uses ResolvedHandlerQN from above) - l.insertRouteNodes(routes) + l.insertRouteNodes(routes, rootPath) callSites := l.discoverCallSites(rootPath) callSites = append(callSites, l.extraCallSites...) @@ -159,73 +200,119 @@ func (l *Linker) Run() ([]HTTPLink, error) { // insertRouteNodes creates Route nodes for each discovered route handler and // HANDLES edges from the handler function to the Route node. -func (l *Linker) insertRouteNodes(routes []RouteHandler) { - for _, rh := range routes { - normalMethod := rh.Method - if normalMethod == "" { - normalMethod = "ANY" - } - normalPath := strings.ReplaceAll(rh.Path, "/", "_") - normalPath = strings.Trim(normalPath, "_") - routeQN := rh.QualifiedName + ".route." + normalMethod + "." + normalPath - routeName := normalMethod + " " + rh.Path - - // Use resolved handler QN if available, otherwise fall back to registering function - handlerQN := rh.QualifiedName - if rh.ResolvedHandlerQN != "" { - handlerQN = rh.ResolvedHandlerQN - } - - // Look up handler node BEFORE creating Route — we need its file_path - handlerNode, _ := l.store.FindNodeByQN(l.project, handlerQN) - - // Inherit file_path from handler function - var filePath string - if handlerNode != nil && handlerNode.FilePath != "" { - filePath = handlerNode.FilePath - } - - routeID, err := l.store.UpsertNode(&store.Node{ - Project: l.project, - Label: "Route", - Name: routeName, - QualifiedName: routeQN, - FilePath: filePath, - Properties: map[string]any{ - "method": rh.Method, - "path": rh.Path, - "handler": handlerQN, - }, - }) - if err != nil || routeID == 0 { - continue - } - - // Create HANDLES edge from actual handler → Route - if handlerNode != nil { - if _, edgeErr := l.store.InsertEdge(&store.Edge{ - Project: l.project, - SourceID: handlerNode.ID, - TargetID: routeID, - Type: "HANDLES", - }); edgeErr != nil { - // FK failures expected: LastInsertId() can return stale IDs for upserted Route nodes - slog.Info("httplink.handles_edge.skip", "route", routeQN) - } - - // Mark handler as entry point (for dead code detection) - if handlerNode.Properties == nil { - handlerNode.Properties = map[string]any{} - } - handlerNode.Properties["is_entry_point"] = true - if _, upsertErr := l.store.UpsertNode(handlerNode); upsertErr != nil { - slog.Warn("httplink.entry_point.err", "err", upsertErr) - } - } +func (l *Linker) insertRouteNodes(routes []RouteHandler, rootPath string) { + for i := range routes { + l.insertSingleRouteNode(&routes[i], rootPath) } slog.Info("httplink.route_nodes", "count", len(routes)) } +// insertSingleRouteNode creates a Route node and HANDLES edge for one route handler. +func (l *Linker) insertSingleRouteNode(rh *RouteHandler, rootPath string) { + normalMethod := rh.Method + if normalMethod == "" { + normalMethod = "ANY" + } + normalPath := strings.ReplaceAll(rh.Path, "/", "_") + normalPath = strings.Trim(normalPath, "_") + routeQN := rh.QualifiedName + ".route." + normalMethod + "." + normalPath + routeName := normalMethod + " " + rh.Path + + // Use resolved handler QN if available, otherwise fall back to registering function + handlerQN := rh.QualifiedName + if rh.ResolvedHandlerQN != "" { + handlerQN = rh.ResolvedHandlerQN + } + + // Look up handler node BEFORE creating Route — we need its file_path + handlerNode, _ := l.store.FindNodeByQN(l.project, handlerQN) + + routeProps := l.buildRouteProps(rh, handlerNode, rootPath) + + // Inherit file_path and line range from handler function + var filePath string + var startLine, endLine int + if handlerNode != nil { + if handlerNode.FilePath != "" { + filePath = handlerNode.FilePath + } + startLine = handlerNode.StartLine + endLine = handlerNode.EndLine + } + + routeID, err := l.store.UpsertNode(&store.Node{ + Project: l.project, + Label: "Route", + Name: routeName, + QualifiedName: routeQN, + FilePath: filePath, + StartLine: startLine, + EndLine: endLine, + Properties: routeProps, + }) + if err != nil || routeID == 0 { + return + } + + l.linkHandlerToRoute(handlerNode, routeID, routeQN) +} + +// buildRouteProps constructs the properties map for a Route node, including protocol detection. +func (l *Linker) buildRouteProps(rh *RouteHandler, handlerNode *store.Node, rootPath string) map[string]any { + handlerQN := rh.QualifiedName + if rh.ResolvedHandlerQN != "" { + handlerQN = rh.ResolvedHandlerQN + } + + routeProps := map[string]any{ + "method": rh.Method, + "path": rh.Path, + "handler": handlerQN, + } + + // Protocol from route extraction (Python websocket, Spring MessageMapping, Ktor webSocket) + if rh.Protocol != "" { + routeProps["protocol"] = rh.Protocol + return routeProps + } + + // Detect protocol from handler source if not already set + if handlerNode != nil && handlerNode.FilePath != "" && handlerNode.StartLine > 0 { + handlerSource := readSourceLines(rootPath, handlerNode.FilePath, handlerNode.StartLine, handlerNode.EndLine) + if protocol := detectProtocol(handlerSource); protocol != "" { + routeProps["protocol"] = protocol + } + } + + return routeProps +} + +// linkHandlerToRoute creates HANDLES edge and marks handler as entry point. +func (l *Linker) linkHandlerToRoute(handlerNode *store.Node, routeID int64, routeQN string) { + if handlerNode == nil { + return + } + + if _, edgeErr := l.store.InsertEdge(&store.Edge{ + Project: l.project, + SourceID: handlerNode.ID, + TargetID: routeID, + Type: "HANDLES", + }); edgeErr != nil { + // FK failures expected: LastInsertId() can return stale IDs for upserted Route nodes + slog.Info("httplink.handles_edge.skip", "route", routeQN) + } + + // Mark handler as entry point (for dead code detection) + if handlerNode.Properties == nil { + handlerNode.Properties = map[string]any{} + } + handlerNode.Properties["is_entry_point"] = true + if _, upsertErr := l.store.UpsertNode(handlerNode); upsertErr != nil { + slog.Warn("httplink.entry_point.err", "err", upsertErr) + } +} + // discoverRoutes finds route handler registrations from Function nodes. func (l *Linker) discoverRoutes(rootPath string) []RouteHandler { var routes []RouteHandler @@ -243,6 +330,9 @@ func (l *Linker) discoverRoutes(rootPath string) []RouteHandler { funcs = append(funcs, methods...) } + // Track which PHP files have Function/Method nodes (for dedup in module scan) + phpFilesWithFuncs := map[string]bool{} + for _, f := range funcs { // Python: check decorators property routes = append(routes, extractPythonRoutes(f)...) @@ -253,15 +343,57 @@ func (l *Linker) discoverRoutes(rootPath string) []RouteHandler { // Rust: check attribute decorators (Actix) routes = append(routes, extractRustRoutes(f)...) - // Source-based route discovery (Go gin, Express.js, PHP Laravel) + // C# ASP.NET: check attribute decorators + routes = append(routes, extractASPNetRoutes(f)...) + + // Source-based route discovery (Go gin, Express.js, PHP Laravel, Kotlin Ktor) if f.FilePath != "" && f.StartLine > 0 && f.EndLine > 0 { source := readSourceLines(rootPath, f.FilePath, f.StartLine, f.EndLine) if source != "" { routes = append(routes, extractGoRoutes(f, source)...) routes = append(routes, extractExpressRoutes(f, source)...) routes = append(routes, extractLaravelRoutes(f, source)...) + routes = append(routes, extractKtorRoutes(f, source)...) } } + + if strings.HasSuffix(f.FilePath, ".php") { + phpFilesWithFuncs[f.FilePath] = true + } + } + + // Module-level route scanning: some frameworks register routes at file top level + // (not inside any function body). Scan modules for route patterns. + modules, err := l.store.FindNodesByLabel(l.project, "Module") + if err != nil { + slog.Warn("httplink.routes.modules.err", "err", err) + return routes + } + + for _, m := range modules { + isPHP := strings.HasSuffix(m.FilePath, ".php") + isJSTS := strings.HasSuffix(m.FilePath, ".js") || + strings.HasSuffix(m.FilePath, ".ts") || + strings.HasSuffix(m.FilePath, ".mjs") || + strings.HasSuffix(m.FilePath, ".mts") + + if !isPHP && !isJSTS { + continue + } + // For PHP: skip files where routes were already extracted from function bodies + if isPHP && phpFilesWithFuncs[m.FilePath] { + continue + } + source := readSourceFull(rootPath, m.FilePath) + if source == "" { + continue + } + if isPHP { + routes = append(routes, extractLaravelRoutes(m, source)...) + } + if isJSTS { + routes = append(routes, extractExpressRoutes(m, source)...) + } } return routes @@ -287,6 +419,7 @@ func extractPythonRoutes(f *store.Node) []RouteHandler { if !ok { continue } + // Standard HTTP routes matches := pyRouteRe.FindAllStringSubmatch(decStr, -1) for _, m := range matches { routes = append(routes, RouteHandler{ @@ -296,30 +429,55 @@ func extractPythonRoutes(f *store.Node) []RouteHandler { QualifiedName: f.QualifiedName, }) } + // WebSocket routes: @app.websocket("/path") + if wm := pyWSRouteRe.FindStringSubmatch(decStr); wm != nil { + routes = append(routes, RouteHandler{ + Path: wm[1], + Method: "WS", + Protocol: "ws", + FunctionName: f.Name, + QualifiedName: f.QualifiedName, + }) + } } return routes } -// extractGoRoutes extracts route registrations from Go source code (gin patterns). -// Resolves router group prefixes by matching variable names in Group() calls -// to the receiver of .GET/.POST calls (e.g., contracts.POST("", ...) where -// contracts := rg.Group("/contracts")). +// extractGoRoutes extracts route registrations from Go source code (gin/chi patterns). +// Resolves gin group prefixes and chi r.Route("/prefix", func...) blocks. func extractGoRoutes(f *store.Node, source string) []RouteHandler { - // Preallocate: most functions register a few routes routes := make([]RouteHandler, 0, 4) - // Build a map of variable name → group prefix from Group() calls + // Build a map of variable name → group prefix from gin Group() calls groupPrefixes := map[string]string{} for _, gm := range goGroupRe.FindAllStringSubmatch(source, -1) { - varName := gm[1] - prefix := gm[2] - groupPrefixes[varName] = prefix + groupPrefixes[gm[1]] = gm[2] } - // Match route registrations and try to prepend group prefix + // Chi Route() prefix stack for nested r.Route("/prefix", func...) blocks + type chiBlock struct { + prefix string + depth int + } + var chiStack []chiBlock + braceDepth := 0 + lines := strings.Split(source, "\n") for _, line := range lines { + // Detect chi .Route("/prefix", func...) blocks + if cm := goChiRouteRe.FindStringSubmatch(line); cm != nil { + chiStack = append(chiStack, chiBlock{prefix: cm[1], depth: braceDepth}) + } + + // Track brace depth + braceDepth += strings.Count(line, "{") - strings.Count(line, "}") + + // Pop closed chi blocks + for len(chiStack) > 0 && braceDepth <= chiStack[len(chiStack)-1].depth { + chiStack = chiStack[:len(chiStack)-1] + } + rm := goRouteRe.FindStringSubmatch(line) if rm == nil { continue @@ -327,13 +485,24 @@ func extractGoRoutes(f *store.Node, source string) []RouteHandler { method := strings.ToUpper(rm[1]) path := rm[2] - // Try to resolve group prefix from the receiver variable - path = resolveGroupPrefix(line, rm[1], path, groupPrefixes) + // Apply chi prefix stack if active, otherwise try gin group resolution + if len(chiStack) > 0 { + var fullPrefix string + for _, block := range chiStack { + fullPrefix = strings.TrimRight(fullPrefix, "/") + "/" + strings.TrimLeft(block.prefix, "/") + } + if path == "/" || path == "" { + path = fullPrefix + } else { + path = strings.TrimRight(fullPrefix, "/") + "/" + strings.TrimLeft(path, "/") + } + } else { + path = resolveGroupPrefix(line, rm[1], path, groupPrefixes) + } // Capture handler reference (last argument) for CALLS edge creation var handlerRef string - hm := goRouteHandlerRe.FindStringSubmatch(line) - if hm != nil { + if hm := goRouteHandlerRe.FindStringSubmatch(line); hm != nil { handlerRef = hm[2] } @@ -375,33 +544,41 @@ func resolveGroupPrefix(line, method, path string, groupPrefixes map[string]stri } // extractExpressRoutes extracts route registrations from JS/TS source (Express/Koa patterns). +// Uses receiver allowlist to avoid false positives from req.get(), res.get(), etc. func extractExpressRoutes(f *store.Node, source string) []RouteHandler { - // Build a per-line handler ref map for quick lookup - handlerRefs := map[string]string{} // "METHOD path" → handler ref - for _, line := range strings.Split(source, "\n") { - hm := expressHandlerRe.FindStringSubmatch(line) - if hm != nil { - key := strings.ToUpper(hm[1]) + " " + hm[0] // unique enough per line - handlerRefs[key] = hm[2] - } - } - routes := make([]RouteHandler, 0, 4) for _, line := range strings.Split(source, "\n") { rm := expressRouteRe.FindStringSubmatch(line) if rm == nil { continue } + // rm[1]=receiver, rm[2]=method, rm[3]=path + receiver := strings.ToLower(rm[1]) + if !expressReceiverAllowlist[receiver] { + continue + } + + // Express overloads .get(): with 1 arg it's a config getter (app.get('trust proxy')), + // with 2+ args it's a route (app.get('/path', handler)). Only .get() has this + // overload — .post/.put/.delete/.patch are always routes. + if strings.EqualFold(rm[2], "get") { + // Check if there's a comma after the closing quote — indicates a callback/handler arg + matchEnd := strings.Index(line, rm[0]) + len(rm[0]) + rest := strings.TrimSpace(line[matchEnd:]) + if !strings.HasPrefix(rest, ",") { + continue // Single-arg app.get('setting') — config getter, skip + } + } var handlerRef string hm := expressHandlerRe.FindStringSubmatch(line) if hm != nil { - handlerRef = hm[2] + handlerRef = hm[3] // group 3 after adding receiver capture } routes = append(routes, RouteHandler{ - Path: rm[2], - Method: strings.ToUpper(rm[1]), + Path: rm[3], + Method: strings.ToUpper(rm[2]), FunctionName: f.Name, QualifiedName: f.QualifiedName, HandlerRef: handlerRef, @@ -426,6 +603,7 @@ func extractJavaRoutes(f *store.Node) []RouteHandler { if !ok { continue } + // Standard Spring HTTP mappings matches := springMappingRe.FindAllStringSubmatch(decStr, -1) for _, m := range matches { method := strings.ToUpper(m[1]) @@ -439,6 +617,16 @@ func extractJavaRoutes(f *store.Node) []RouteHandler { QualifiedName: f.QualifiedName, }) } + // Spring WebSocket: @MessageMapping("/path") + if wm := springWSRe.FindStringSubmatch(decStr); wm != nil { + routes = append(routes, RouteHandler{ + Path: wm[1], + Method: "WS", + Protocol: "ws", + FunctionName: f.Name, + QualifiedName: f.QualifiedName, + }) + } } return routes } @@ -500,6 +688,78 @@ func extractLaravelRoutes(f *store.Node, source string) []RouteHandler { return routes } +// extractASPNetRoutes extracts route handlers from C# ASP.NET attribute metadata. +func extractASPNetRoutes(f *store.Node) []RouteHandler { + var routes []RouteHandler + + decs, ok := f.Properties["decorators"] + if !ok { + return routes + } + + decList, ok := decs.([]any) + if !ok { + return routes + } + + for _, d := range decList { + decStr, ok := d.(string) + if !ok { + continue + } + // [HttpGet("/path")] pattern + matches := aspnetRouteRe.FindAllStringSubmatch(decStr, -1) + for _, m := range matches { + method := strings.TrimPrefix(m[1], "Http") + routes = append(routes, RouteHandler{ + Path: m[2], + Method: strings.ToUpper(method), + FunctionName: f.Name, + QualifiedName: f.QualifiedName, + }) + } + // [Route("/path")] pattern + routeMatches := aspnetRouteAttrRe.FindAllStringSubmatch(decStr, -1) + for _, m := range routeMatches { + routes = append(routes, RouteHandler{ + Path: m[1], + Method: "", + FunctionName: f.Name, + QualifiedName: f.QualifiedName, + }) + } + } + return routes +} + +// extractKtorRoutes extracts route handlers from Kotlin Ktor source code. +func extractKtorRoutes(f *store.Node, source string) []RouteHandler { + routes := make([]RouteHandler, 0, 4) + for _, line := range strings.Split(source, "\n") { + // Standard HTTP routes + if rm := ktorRouteRe.FindStringSubmatch(line); rm != nil { + routes = append(routes, RouteHandler{ + Path: rm[2], + Method: strings.ToUpper(rm[1]), + FunctionName: f.Name, + QualifiedName: f.QualifiedName, + }) + continue + } + // WebSocket routes: webSocket("/path") { + if wm := ktorWSRe.FindStringSubmatch(line); wm != nil { + routes = append(routes, RouteHandler{ + Path: wm[1], + Method: "WS", + Protocol: "ws", + FunctionName: f.Name, + QualifiedName: f.QualifiedName, + }) + } + } + return routes +} + // discoverCallSites finds HTTP URL references in Module constants and Function source. func (l *Linker) discoverCallSites(rootPath string) []HTTPCallSite { var sites []HTTPCallSite @@ -624,6 +884,10 @@ var httpClientKeywords = []string{ "curl_easy", "cpr::Get", "cpr::Post", "httplib::", // Lua "socket.http", "http.request", "curl.", + // C# + "HttpClient", "WebClient", "RestClient", "HttpWebRequest", + // Kotlin + "OkHttpClient", "HttpClient", "ktor.client", // Generic "send_request", "http_client", } @@ -643,6 +907,41 @@ var asyncDispatchKeywords = []string{ "producer.send", "producer.Send", } +// wsPatterns indicate WebSocket usage in handler source. +var wsPatterns = []string{ + // Go: gorilla/nhooyr websocket + "websocket.Upgrade", "websocket.Accept", "upgrader.Upgrade", + // JS/TS: ws library, socket.io + `ws.on("connection`, `io.on("connection`, "new WebSocket(", + // Generic + "WebSocketSession", "wsHandler", +} + +// ssePatterns indicate Server-Sent Events usage in handler source. +var ssePatterns = []string{ + "text/event-stream", + "EventSourceResponse", + "SseEmitter", + "ServerSentEvent", + "event-stream", +} + +// detectProtocol checks handler source for WebSocket or SSE patterns. +// Returns "ws", "sse", or "" (standard HTTP). +func detectProtocol(source string) string { + for _, p := range wsPatterns { + if strings.Contains(source, p) { + return "ws" + } + } + for _, p := range ssePatterns { + if strings.Contains(source, p) { + return "sse" + } + } + return "" +} + // extractFunctionCallSites extracts HTTP paths from function source code. func extractFunctionCallSites(f *store.Node, rootPath string) []HTTPCallSite { sites := make([]HTTPCallSite, 0, 4) @@ -811,7 +1110,7 @@ func extractJSONStringPaths(text string) []string { // findJSONBounds extracts substrings that look like JSON objects or arrays. func findJSONBounds(text string) []string { - var results []string + results := make([]string, 0, 4) for _, opener := range []byte{'{', '['} { closer := byte('}') if opener == '[' { @@ -1138,6 +1437,151 @@ func sourceWeight(label string) float64 { } } +// resolveFastAPIPrefixes resolves include_router prefixes for FastAPI routes. +// Scans Python Module files for app.include_router(var, prefix="/prefix") calls +// and prepends the prefix to routes from the imported module. +func (l *Linker) resolveFastAPIPrefixes(routes []RouteHandler, rootPath string) { + modules, err := l.store.FindNodesByLabel(l.project, "Module") + if err != nil { + return + } + + for _, mod := range modules { + if !strings.HasSuffix(mod.FilePath, ".py") { + continue + } + + source, readErr := os.ReadFile(filepath.Join(rootPath, mod.FilePath)) + if readErr != nil { + continue + } + srcStr := string(source) + + includes := fastAPIIncludeRe.FindAllStringSubmatch(srcStr, -1) + if len(includes) == 0 { + continue + } + + // Build import map: var_name → dotted module path + imports := map[string]string{} + for _, m := range pyImportRe.FindAllStringSubmatch(srcStr, -1) { + imports[m[2]] = m[1] // var_name → module.path + } + + for _, inc := range includes { + varName := inc[1] + prefix := inc[2] + + modulePath, ok := imports[varName] + if !ok { + continue + } + + // Convert dotted module path to file path fragment + fileFrag := strings.ReplaceAll(modulePath, ".", "/") + normalizedPrefix := strings.TrimRight(prefix, "/") + + prefixed := 0 + for i := range routes { + if strings.HasPrefix(routes[i].Path, normalizedPrefix) { + continue + } + // Match routes whose QN contains the imported module path + if strings.Contains(routes[i].QualifiedName, fileFrag+".py") || + strings.Contains(routes[i].QualifiedName, fileFrag+"/") { + routes[i].Path = normalizedPrefix + "/" + strings.TrimLeft(routes[i].Path, "/") + prefixed++ + } + } + if prefixed > 0 { + slog.Info("httplink.fastapi_prefix", "prefix", prefix, "module", modulePath, "routes", prefixed) + } + } + } +} + +// resolveExpressPrefixes resolves app.use("/prefix", router) for Express routes. +// Scans JS/TS Module files for .use("/prefix", routerVar) calls and prepends +// the prefix to routes from the imported module. +func (l *Linker) resolveExpressPrefixes(routes []RouteHandler, rootPath string) { + modules, err := l.store.FindNodesByLabel(l.project, "Module") + if err != nil { + return + } + + for _, mod := range modules { + if !isJSTSModule(mod.FilePath) { + continue + } + + source, readErr := os.ReadFile(filepath.Join(rootPath, mod.FilePath)) + if readErr != nil { + continue + } + srcStr := string(source) + + uses := expressUseRe.FindAllStringSubmatch(srcStr, -1) + if len(uses) == 0 { + continue + } + + imports := buildJSImportMap(srcStr) + l.applyExpressUsePrefixes(routes, uses, imports) + } +} + +// isJSTSModule returns true if the file path is a JS/TS module file. +func isJSTSModule(filePath string) bool { + return strings.HasSuffix(filePath, ".js") || strings.HasSuffix(filePath, ".ts") || + strings.HasSuffix(filePath, ".mjs") || strings.HasSuffix(filePath, ".tsx") +} + +// buildJSImportMap builds a map of var_name to module path from require/import statements. +func buildJSImportMap(src string) map[string]string { + imports := map[string]string{} + for _, m := range jsRequireRe.FindAllStringSubmatch(src, -1) { + imports[m[1]] = m[2] + } + for _, m := range jsImportRe.FindAllStringSubmatch(src, -1) { + imports[m[1]] = m[2] + } + return imports +} + +// applyExpressUsePrefixes applies .use("/prefix", routerVar) prefix resolution to routes. +func (l *Linker) applyExpressUsePrefixes(routes []RouteHandler, uses [][]string, imports map[string]string) { + for _, use := range uses { + prefix := use[1] + varName := use[2] + + modulePath, ok := imports[varName] + if !ok { + continue + } + + // Strip leading ./ from relative import + fileFrag := strings.TrimPrefix(modulePath, "./") + fileFrag = strings.TrimPrefix(fileFrag, "../") + normalizedPrefix := strings.TrimRight(prefix, "/") + + prefixed := 0 + for i := range routes { + if strings.HasPrefix(routes[i].Path, normalizedPrefix) { + continue + } + if strings.Contains(routes[i].QualifiedName, fileFrag+".js") || + strings.Contains(routes[i].QualifiedName, fileFrag+".ts") || + strings.Contains(routes[i].QualifiedName, fileFrag+"/") { + routes[i].Path = normalizedPrefix + "/" + strings.TrimLeft(routes[i].Path, "/") + prefixed++ + } + } + if prefixed > 0 { + slog.Info("httplink.express_prefix", "prefix", prefix, "module", modulePath, "routes", prefixed) + } + } +} + // pathsMatch is a convenience wrapper for tests — returns true if score >= threshold. func pathsMatch(callPath, routePath string) bool { return pathMatchScore(callPath, routePath) >= matchConfidenceThreshold @@ -1303,6 +1747,15 @@ func (l *Linker) createRegistrationCallEdges(routes []RouteHandler) { } // readSourceLines reads specific lines from a file on disk. +// readSourceFull reads the entire file content as a string. +func readSourceFull(rootPath, relPath string) string { + data, err := os.ReadFile(filepath.Join(rootPath, relPath)) + if err != nil { + return "" + } + return string(data) +} + func readSourceLines(rootPath, relPath string, startLine, endLine int) string { absPath := filepath.Join(rootPath, relPath) f, err := os.Open(absPath) diff --git a/internal/httplink/httplink_test.go b/internal/httplink/httplink_test.go index 8cfc4822e..4eaf2e89a 100644 --- a/internal/httplink/httplink_test.go +++ b/internal/httplink/httplink_test.go @@ -3,6 +3,7 @@ package httplink import ( "os" "path/filepath" + "strings" "testing" "github.com/DeusData/codebase-memory-mcp/internal/store" @@ -999,3 +1000,444 @@ func TestLinkerSkipsSameService(t *testing.T) { } } } + +func TestDetectProtocol(t *testing.T) { + tests := []struct { + name string + source string + want string + }{ + {"go websocket upgrade", `err := websocket.Upgrade(w, r, nil, 1024, 1024)`, "ws"}, + {"go websocket accept", `conn, err := websocket.Accept(w, r, nil)`, "ws"}, + {"go upgrader", `conn, err := upgrader.Upgrade(w, r, nil)`, "ws"}, + {"js ws", `ws.on("connection", func)`, "ws"}, + {"js socketio", `io.on("connection", handler)`, "ws"}, + {"sse content type", `w.Header().Set("Content-Type", "text/event-stream")`, "sse"}, + {"python sse", `return EventSourceResponse(generate())`, "sse"}, + {"java sse emitter", `SseEmitter emitter = new SseEmitter()`, "sse"}, + {"java sse event", `ServerSentEvent event = ServerSentEvent.builder()`, "sse"}, + {"no protocol", `return json.Marshal(result)`, ""}, + {"empty", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := detectProtocol(tt.source) + if got != tt.want { + t.Errorf("detectProtocol() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestExtractPythonWSRoutes(t *testing.T) { + node := &store.Node{ + Name: "ws_handler", + QualifiedName: "proj.api.ws_handler", + Properties: map[string]any{ + "decorators": []any{ + `@app.websocket("/ws/chat")`, + }, + }, + } + + routes := extractPythonRoutes(node) + if len(routes) != 1 { + t.Fatalf("expected 1 route, got %d", len(routes)) + } + if routes[0].Path != "/ws/chat" { + t.Errorf("path = %q, want /ws/chat", routes[0].Path) + } + if routes[0].Method != "WS" { + t.Errorf("method = %q, want WS", routes[0].Method) + } + if routes[0].Protocol != "ws" { + t.Errorf("protocol = %q, want ws", routes[0].Protocol) + } +} + +func TestExtractSpringWSRoutes(t *testing.T) { + node := &store.Node{ + Name: "handleChat", + QualifiedName: "proj.ChatController.handleChat", + Properties: map[string]any{ + "decorators": []any{ + `@MessageMapping("/chat")`, + }, + }, + } + + routes := extractJavaRoutes(node) + if len(routes) != 1 { + t.Fatalf("expected 1 route, got %d", len(routes)) + } + if routes[0].Path != "/chat" { + t.Errorf("path = %q, want /chat", routes[0].Path) + } + if routes[0].Method != "WS" { + t.Errorf("method = %q, want WS", routes[0].Method) + } + if routes[0].Protocol != "ws" { + t.Errorf("protocol = %q, want ws", routes[0].Protocol) + } +} + +func TestExtractKtorWSRoutes(t *testing.T) { + source := ` + webSocket("/chat") { + for (frame in incoming) { + send(frame) + } + } + get("/api/health") { + call.respond("ok") + } +` + node := &store.Node{ + Name: "configureRouting", + QualifiedName: "proj.Routing.configureRouting", + } + + routes := extractKtorRoutes(node, source) + if len(routes) != 2 { + t.Fatalf("expected 2 routes, got %d", len(routes)) + } + + // Check WS route + wsFound := false + httpFound := false + for _, r := range routes { + if r.Protocol == "ws" && r.Path == "/chat" && r.Method == "WS" { + wsFound = true + } + if r.Path == "/api/health" && r.Method == "GET" { + httpFound = true + } + } + if !wsFound { + t.Error("expected WS route for /chat") + } + if !httpFound { + t.Error("expected HTTP route for /api/health") + } +} + +func TestChiPrefix(t *testing.T) { + source := ` +func SetupRoutes(r chi.Router) { + r.Route("/api", func(r chi.Router) { + r.Get("/health", healthHandler) + r.Route("/users", func(r chi.Router) { + r.Get("/", listUsers) + r.Post("/{id}", updateUser) + }) + }) +} +` + node := &store.Node{ + Name: "SetupRoutes", + QualifiedName: "proj.SetupRoutes", + } + + routes := extractGoRoutes(node, source) + + expectedPaths := map[string]string{ + "/api/health": "GET", + "/api/users": "GET", + "/api/users/{id}": "POST", + } + + if len(routes) != len(expectedPaths) { + t.Fatalf("expected %d routes, got %d", len(expectedPaths), len(routes)) + for _, r := range routes { + t.Logf(" %s %s", r.Method, r.Path) + } + } + + for _, r := range routes { + wantMethod, ok := expectedPaths[r.Path] + if !ok { + t.Errorf("unexpected route: %s %s", r.Method, r.Path) + continue + } + if r.Method != wantMethod { + t.Errorf("route %s: method = %q, want %q", r.Path, r.Method, wantMethod) + } + } +} + +func TestChiPrefixMixedWithGin(t *testing.T) { + // When no chi Route() blocks, gin group resolution should still work + source := ` +func RegisterRoutes(r *gin.RouterGroup) { + orders := r.Group("/orders") + orders.GET("/:id", getOrder) + orders.POST("", createOrder) +} +` + node := &store.Node{ + Name: "RegisterRoutes", + QualifiedName: "proj.RegisterRoutes", + } + + routes := extractGoRoutes(node, source) + if len(routes) != 2 { + t.Fatalf("expected 2 routes, got %d", len(routes)) + } + + for _, r := range routes { + if !strings.HasPrefix(r.Path, "/orders") { + t.Errorf("expected /orders prefix, got %s", r.Path) + } + } +} + +func TestFastAPIPrefix(t *testing.T) { + dir, err := os.MkdirTemp("", "httplink-fastapi-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // Write Python files: main.py with include_router, orders/routes.py with routes + if err := os.MkdirAll(filepath.Join(dir, "orders"), 0o755); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(dir, "main.py"), []byte(` +from orders.routes import order_router + +app = FastAPI() +app.include_router(order_router, prefix="/api/v1/orders") +`), 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + project := "testproj" + if err := s.UpsertProject(project, dir); err != nil { + t.Fatal(err) + } + + // Create Module for main.py (has the include_router call) + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Module", Name: "main.py", + QualifiedName: "testproj//main.py", + FilePath: "main.py", + }) + + // Create Function with route in the orders module + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "list_orders", + QualifiedName: "testproj//orders/routes.py/list_orders", + FilePath: "orders/routes.py", + Properties: map[string]any{ + "decorators": []any{`@order_router.get("/")`}, + }, + }) + + linker := New(s, project) + _, err = linker.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + + routeNodes, _ := s.FindNodesByLabel(project, "Route") + if len(routeNodes) != 1 { + t.Fatalf("expected 1 Route node, got %d", len(routeNodes)) + } + + path, _ := routeNodes[0].Properties["path"].(string) + if path != "/api/v1/orders/" { + t.Errorf("expected /api/v1/orders/, got %s", path) + } +} + +func TestExpressPrefix(t *testing.T) { + dir, err := os.MkdirTemp("", "httplink-express-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + if err := os.MkdirAll(filepath.Join(dir, "routes"), 0o755); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(dir, "app.js"), []byte(` +const orderRouter = require('./routes/orders'); +app.use("/api/orders", orderRouter); +`), 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + project := "testproj" + if err := s.UpsertProject(project, dir); err != nil { + t.Fatal(err) + } + + // Module for app.js + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Module", Name: "app.js", + QualifiedName: "testproj//app.js", + FilePath: "app.js", + }) + + // Function with route in orders module + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "getOrder", + QualifiedName: "testproj//routes/orders.js/getOrder", + FilePath: "routes/orders.js", + StartLine: 1, EndLine: 5, + }) + + // Write the routes file with Express route + if err := os.WriteFile(filepath.Join(dir, "routes", "orders.js"), []byte(` +router.get("/:id", function(req, res) { + res.json({id: req.params.id}); +}); +`), 0o600); err != nil { + t.Fatal(err) + } + + linker := New(s, project) + _, err = linker.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + + routeNodes, _ := s.FindNodesByLabel(project, "Route") + if len(routeNodes) != 1 { + t.Fatalf("expected 1 Route node, got %d", len(routeNodes)) + } + + path, _ := routeNodes[0].Properties["path"].(string) + if path != "/api/orders/:id" { + t.Errorf("expected /api/orders/:id, got %s", path) + } +} + +func TestExpressRouteFiltering(t *testing.T) { + tests := []struct { + name string + line string + wantMatch bool + wantMethod string + wantPath string + }{ + // Should match (allowlisted receivers) + {"app.get", `app.get('/api/users', handler)`, true, "GET", "/api/users"}, + {"router.post", `router.post('/orders', handler)`, true, "POST", "/orders"}, + {"server.put", `server.put('/items', handler)`, true, "PUT", "/items"}, + {"api.delete", `api.delete('/users/:id', handler)`, true, "DELETE", "/users/:id"}, + {"routes.patch", `routes.patch('/items/:id', handler)`, true, "PATCH", "/items/:id"}, + // Should NOT match (not in allowlist) + {"req.get", `req.get('Content-Type')`, false, "", ""}, + {"res.get", `res.get('key')`, false, "", ""}, + {"this.get", `this.get('property')`, false, "", ""}, + {"map.get", `map.get('key')`, false, "", ""}, + {"model.delete", `model.delete('record')`, false, "", ""}, + {"params.get", `params.get('id')`, false, "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := &store.Node{ + Name: "testFunc", + QualifiedName: "proj.test.testFunc", + } + routes := extractExpressRoutes(node, tt.line) + if tt.wantMatch { + if len(routes) == 0 { + t.Errorf("expected route match, got 0 routes") + return + } + if routes[0].Method != tt.wantMethod { + t.Errorf("method = %q, want %q", routes[0].Method, tt.wantMethod) + } + if routes[0].Path != tt.wantPath { + t.Errorf("path = %q, want %q", routes[0].Path, tt.wantPath) + } + } else if len(routes) > 0 { + t.Errorf("expected no match, got %d routes: %v", len(routes), routes[0].Path) + } + }) + } +} + +func TestLaravelModuleLevelRoutes(t *testing.T) { + dir, err := os.MkdirTemp("", "httplink-laravel-module-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // Create a Laravel-style route file with module-level Route:: calls + if err := os.MkdirAll(filepath.Join(dir, "routes"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "routes", "api.php"), []byte(` 0 { + t.Errorf("pyRouteRe matched non-route source: %v", matches) + } + + // Go route patterns + matches = goRouteRe.FindAllStringSubmatch(tt.source, -1) + if len(matches) > 0 { + t.Errorf("goRouteRe matched non-route source: %v", matches) + } + + // Express route patterns + matches = expressRouteRe.FindAllStringSubmatch(tt.source, -1) + if len(matches) > 0 { + t.Errorf("expressRouteRe matched non-route source: %v", matches) + } + + // Spring route patterns + matches = springMappingRe.FindAllStringSubmatch(tt.source, -1) + if len(matches) > 0 { + t.Errorf("springMappingRe matched non-route source: %v", matches) + } + + // Laravel route patterns + matches = laravelRouteRe.FindAllStringSubmatch(tt.source, -1) + if len(matches) > 0 { + t.Errorf("laravelRouteRe matched non-route source: %v", matches) + } + + // Actix route patterns + matches = actixRouteRe.FindAllStringSubmatch(tt.source, -1) + if len(matches) > 0 { + t.Errorf("actixRouteRe matched non-route source: %v", matches) + } + }) + } +} diff --git a/internal/lang/bash.go b/internal/lang/bash.go new file mode 100644 index 000000000..c7cc662ee --- /dev/null +++ b/internal/lang/bash.go @@ -0,0 +1,17 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Bash, + FileExtensions: []string{".sh", ".bash"}, + FunctionNodeTypes: []string{"function_definition"}, + ClassNodeTypes: []string{}, + ModuleNodeTypes: []string{"program"}, + CallNodeTypes: []string{"command"}, + ImportNodeTypes: []string{"command"}, + + BranchingNodeTypes: []string{"if_statement", "while_statement", "for_statement", "case_statement", "elif_clause"}, + VariableNodeTypes: []string{"variable_assignment"}, + AssignmentNodeTypes: []string{"variable_assignment"}, + }) +} diff --git a/internal/lang/c_lang.go b/internal/lang/c_lang.go new file mode 100644 index 000000000..3bac7885c --- /dev/null +++ b/internal/lang/c_lang.go @@ -0,0 +1,21 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: C, + FileExtensions: []string{".c"}, + FunctionNodeTypes: []string{ + "function_definition", + }, + ClassNodeTypes: []string{"struct_specifier", "enum_specifier", "union_specifier"}, + FieldNodeTypes: []string{"field_declaration"}, + ModuleNodeTypes: []string{"translation_unit"}, + CallNodeTypes: []string{"call_expression"}, + ImportNodeTypes: []string{"preproc_include"}, + + BranchingNodeTypes: []string{"if_statement", "for_statement", "while_statement", "do_statement", "switch_statement", "case_statement"}, + VariableNodeTypes: []string{"declaration"}, + AssignmentNodeTypes: []string{"assignment_expression"}, + EnvAccessFunctions: []string{"getenv"}, + }) +} diff --git a/internal/lang/cpp.go b/internal/lang/cpp.go index 26b6a1cca..1b2ee4300 100644 --- a/internal/lang/cpp.go +++ b/internal/lang/cpp.go @@ -38,6 +38,10 @@ func init() { ImportFromTypes: []string{"preproc_include", "template_function", "declaration"}, PackageIndicators: []string{"CMakeLists.txt", "Makefile", "*.vcxproj", "conanfile.txt"}, - BranchingNodeTypes: []string{"if_statement", "for_statement", "for_range_loop", "while_statement", "switch_statement", "case_statement", "try_statement", "catch_clause"}, + BranchingNodeTypes: []string{"if_statement", "for_statement", "for_range_loop", "while_statement", "switch_statement", "case_statement", "try_statement", "catch_clause"}, + VariableNodeTypes: []string{"declaration"}, + AssignmentNodeTypes: []string{"assignment_expression"}, + ThrowNodeTypes: []string{"throw_statement"}, + EnvAccessFunctions: []string{"getenv", "std::getenv"}, }) } diff --git a/internal/lang/csharp.go b/internal/lang/csharp.go index 314395d82..a10492b07 100644 --- a/internal/lang/csharp.go +++ b/internal/lang/csharp.go @@ -25,8 +25,10 @@ func init() { ImportFromTypes: []string{"using_directive"}, BranchingNodeTypes: []string{"if_statement", "for_statement", "foreach_statement", "while_statement", "switch_statement", "case_switch_label", "try_statement", "catch_clause"}, + VariableNodeTypes: []string{"field_declaration", "local_declaration_statement"}, AssignmentNodeTypes: []string{"assignment_expression"}, ThrowNodeTypes: []string{"throw_statement", "throw_expression"}, DecoratorNodeTypes: []string{"attribute"}, + EnvAccessFunctions: []string{"Environment.GetEnvironmentVariable"}, }) } diff --git a/internal/lang/css.go b/internal/lang/css.go new file mode 100644 index 000000000..87f3710e6 --- /dev/null +++ b/internal/lang/css.go @@ -0,0 +1,15 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: CSS, + FileExtensions: []string{".css"}, + FunctionNodeTypes: []string{}, + ClassNodeTypes: []string{}, + ModuleNodeTypes: []string{"stylesheet"}, + CallNodeTypes: []string{}, + ImportNodeTypes: []string{"import_statement"}, + + VariableNodeTypes: []string{}, + }) +} diff --git a/internal/lang/dart.go b/internal/lang/dart.go new file mode 100644 index 000000000..d017c9e9c --- /dev/null +++ b/internal/lang/dart.go @@ -0,0 +1,29 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Dart, + FileExtensions: []string{".dart"}, + FunctionNodeTypes: []string{ + "function_signature", // top-level function: return_type name(params) + "method_signature", // class method: wraps function_signature + }, + ClassNodeTypes: []string{ + "class_definition", + "enum_declaration", + "mixin_declaration", + }, + FieldNodeTypes: []string{"declaration"}, + ModuleNodeTypes: []string{"program"}, + CallNodeTypes: []string{"selector"}, + ImportNodeTypes: []string{"import_or_export"}, + BranchingNodeTypes: []string{ + "if_statement", "for_statement", "while_statement", + "switch_statement", + }, + VariableNodeTypes: []string{"declaration"}, + AssignmentNodeTypes: []string{"assignment_expression"}, + ThrowNodeTypes: []string{"throw_expression"}, + DecoratorNodeTypes: []string{"annotation"}, + }) +} diff --git a/internal/lang/dockerfile.go b/internal/lang/dockerfile.go new file mode 100644 index 000000000..a33abc01d --- /dev/null +++ b/internal/lang/dockerfile.go @@ -0,0 +1,18 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Dockerfile, + FileExtensions: []string{".dockerfile", "Dockerfile"}, + ModuleNodeTypes: []string{"source_file"}, + FunctionNodeTypes: []string{}, + ClassNodeTypes: []string{}, + FieldNodeTypes: []string{}, + CallNodeTypes: []string{}, + ImportNodeTypes: []string{}, + VariableNodeTypes: []string{ + "env_instruction", + "arg_instruction", + }, + }) +} diff --git a/internal/lang/elixir.go b/internal/lang/elixir.go new file mode 100644 index 000000000..0f3e0ca2a --- /dev/null +++ b/internal/lang/elixir.go @@ -0,0 +1,25 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Elixir, + FileExtensions: []string{".ex", ".exs"}, + // Elixir uses "call" for everything (homoiconic), but we only want + // definition forms (def, defp, defmacro) in FunctionNodeTypes. + // The pipeline has custom Elixir handling to classify call nodes. + FunctionNodeTypes: []string{ + "call", // classified by pipeline: def/defp → Function, defmodule → Class + }, + ClassNodeTypes: []string{}, // handled by custom extraction (defmodule) + ModuleNodeTypes: []string{"source"}, + CallNodeTypes: []string{"call", "dot"}, + ImportNodeTypes: []string{"call"}, + + BranchingNodeTypes: []string{ + "call", // if/cond/case are macro calls in Elixir + }, + VariableNodeTypes: []string{"binary_operator"}, + AssignmentNodeTypes: []string{"binary_operator"}, + EnvAccessFunctions: []string{"System.get_env"}, + }) +} diff --git a/internal/lang/erlang.go b/internal/lang/erlang.go new file mode 100644 index 000000000..2636b5b4c --- /dev/null +++ b/internal/lang/erlang.go @@ -0,0 +1,22 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Erlang, + FileExtensions: []string{".erl"}, + FunctionNodeTypes: []string{ + "function_clause", + }, + ClassNodeTypes: []string{}, + FieldNodeTypes: []string{}, + ModuleNodeTypes: []string{"source_file"}, + CallNodeTypes: []string{"call"}, + ImportNodeTypes: []string{"module_attribute"}, + BranchingNodeTypes: []string{ + "if_expression", "case_expression", "receive_expression", + }, + VariableNodeTypes: []string{"pp_define", "record_decl"}, + AssignmentNodeTypes: []string{"match_expression"}, + ThrowNodeTypes: []string{"call"}, + }) +} diff --git a/internal/lang/go_lang.go b/internal/lang/go_lang.go index 00dac99eb..5101f81ce 100644 --- a/internal/lang/go_lang.go +++ b/internal/lang/go_lang.go @@ -4,7 +4,7 @@ func init() { Register(&LanguageSpec{ Language: Go, FileExtensions: []string{".go"}, - FunctionNodeTypes: []string{"function_declaration", "method_declaration"}, + FunctionNodeTypes: []string{"function_declaration", "method_declaration", "method_elem"}, ClassNodeTypes: []string{"type_spec", "type_alias"}, FieldNodeTypes: []string{"field_declaration"}, ModuleNodeTypes: []string{"source_file"}, diff --git a/internal/lang/groovy.go b/internal/lang/groovy.go new file mode 100644 index 000000000..02aeadf25 --- /dev/null +++ b/internal/lang/groovy.go @@ -0,0 +1,26 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Groovy, + FileExtensions: []string{".groovy", ".gradle"}, + FunctionNodeTypes: []string{ + "function_definition", // def greet() {} or void greet() {} + }, + ClassNodeTypes: []string{ + "class_definition", // class Foo extends Bar {} + }, + FieldNodeTypes: []string{}, + ModuleNodeTypes: []string{"source_file"}, + CallNodeTypes: []string{"function_call", "juxt_function_call"}, + ImportNodeTypes: []string{"groovy_import"}, + BranchingNodeTypes: []string{ + "if_statement", "for_statement", "while_statement", + "switch_statement", + }, + VariableNodeTypes: []string{"declaration"}, + AssignmentNodeTypes: []string{"assignment"}, + ThrowNodeTypes: []string{"throw_statement"}, + DecoratorNodeTypes: []string{"annotation"}, + }) +} diff --git a/internal/lang/haskell.go b/internal/lang/haskell.go new file mode 100644 index 000000000..a1262bdd0 --- /dev/null +++ b/internal/lang/haskell.go @@ -0,0 +1,32 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Haskell, + FileExtensions: []string{".hs"}, + FunctionNodeTypes: []string{ + "function", // function binding with name, patterns, match + "signature", // type signature (name :: Type -> Type) + }, + ClassNodeTypes: []string{ + "class", // type class declaration + "data_type", // algebraic data type + "newtype", // newtype declaration + }, + ModuleNodeTypes: []string{"haskell"}, + CallNodeTypes: []string{"infix", "apply"}, + ImportNodeTypes: []string{"import"}, + + BranchingNodeTypes: []string{ + "match", // pattern match clause + "guards", // guard expressions + "if", // if expression + "case", // case expression + "do", // do notation + "boolean", // guard condition + }, + VariableNodeTypes: []string{"function"}, + AssignmentNodeTypes: []string{"function"}, + EnvAccessFunctions: []string{"lookupEnv", "getEnv"}, + }) +} diff --git a/internal/lang/hcl.go b/internal/lang/hcl.go new file mode 100644 index 000000000..dad5eaec6 --- /dev/null +++ b/internal/lang/hcl.go @@ -0,0 +1,17 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: HCL, + FileExtensions: []string{".tf", ".hcl"}, + FunctionNodeTypes: []string{}, + ClassNodeTypes: []string{ + "block", // resource, variable, output, data, module blocks + }, + ModuleNodeTypes: []string{"config_file"}, + CallNodeTypes: []string{"function_call"}, + ImportNodeTypes: []string{}, + + VariableNodeTypes: []string{"attribute"}, + }) +} diff --git a/internal/lang/html.go b/internal/lang/html.go new file mode 100644 index 000000000..529d6f9da --- /dev/null +++ b/internal/lang/html.go @@ -0,0 +1,15 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: HTML, + FileExtensions: []string{".html", ".htm"}, + FunctionNodeTypes: []string{}, + ClassNodeTypes: []string{}, + ModuleNodeTypes: []string{"document"}, + CallNodeTypes: []string{}, + ImportNodeTypes: []string{}, + + VariableNodeTypes: []string{}, + }) +} diff --git a/internal/lang/java.go b/internal/lang/java.go index 3674b3dc6..27efec70c 100644 --- a/internal/lang/java.go +++ b/internal/lang/java.go @@ -19,6 +19,7 @@ func init() { ImportFromTypes: []string{"import_declaration"}, BranchingNodeTypes: []string{"if_statement", "for_statement", "enhanced_for_statement", "while_statement", "switch_expression", "switch_block_statement_group", "try_statement", "catch_clause"}, + VariableNodeTypes: []string{"field_declaration", "local_variable_declaration"}, AssignmentNodeTypes: []string{"assignment_expression"}, ThrowNodeTypes: []string{"throw_statement"}, ThrowsClauseField: "throws", diff --git a/internal/lang/kotlin.go b/internal/lang/kotlin.go index d2c5548b5..bc219d5af 100644 --- a/internal/lang/kotlin.go +++ b/internal/lang/kotlin.go @@ -21,5 +21,12 @@ func init() { }, ImportNodeTypes: []string{"import"}, ImportFromTypes: []string{"import"}, + + BranchingNodeTypes: []string{"if_expression", "for_statement", "while_statement", "when_expression", "when_entry", "try_expression", "catch_block"}, + VariableNodeTypes: []string{"property_declaration"}, + AssignmentNodeTypes: []string{"assignment", "directly_assignable_expression"}, + ThrowNodeTypes: []string{"throw_expression"}, + DecoratorNodeTypes: []string{"annotation"}, + EnvAccessFunctions: []string{"System.getenv", "System.getProperty"}, }) } diff --git a/internal/lang/lang.go b/internal/lang/lang.go index 3bf8f59b1..7e82459be 100644 --- a/internal/lang/lang.go +++ b/internal/lang/lang.go @@ -18,11 +18,42 @@ const ( Scala Language = "scala" Kotlin Language = "kotlin" JSON Language = "json" // Not in AllLanguages(); no LanguageSpec or tree-sitter grammar + + // Programming languages (Tier 1) + Ruby Language = "ruby" + C Language = "c" + Bash Language = "bash" + Zig Language = "zig" + Elixir Language = "elixir" + Haskell Language = "haskell" + OCaml Language = "ocaml" + ObjectiveC Language = "objective-c" + Swift Language = "swift" + Dart Language = "dart" + Perl Language = "perl" + Groovy Language = "groovy" + Erlang Language = "erlang" + R Language = "r" + + // Helper languages (Tier 2) + HTML Language = "html" + CSS Language = "css" + SCSS Language = "scss" + YAML Language = "yaml" + TOML Language = "toml" + HCL Language = "hcl" + SQL Language = "sql" + Dockerfile Language = "dockerfile" ) // AllLanguages returns all supported languages. func AllLanguages() []Language { - return []Language{Python, JavaScript, TypeScript, TSX, Go, Rust, Java, CPP, CSharp, PHP, Lua, Scala, Kotlin} + return []Language{ + Python, JavaScript, TypeScript, TSX, Go, Rust, Java, CPP, CSharp, PHP, Lua, Scala, Kotlin, + Ruby, C, Bash, Zig, Elixir, Haskell, OCaml, ObjectiveC, + Swift, Dart, Perl, Groovy, Erlang, R, + HTML, CSS, SCSS, YAML, TOML, HCL, SQL, Dockerfile, + } } // LanguageSpec defines the tree-sitter node types for a language. diff --git a/internal/lang/lang_test.go b/internal/lang/lang_test.go index 695a67fa2..a64b18a92 100644 --- a/internal/lang/lang_test.go +++ b/internal/lang/lang_test.go @@ -55,15 +55,15 @@ func TestGoSpec(t *testing.T) { if spec == nil { t.Fatal("Go spec not registered") } - if len(spec.FunctionNodeTypes) != 2 { - t.Errorf("Go FunctionNodeTypes: got %d, want 2", len(spec.FunctionNodeTypes)) + if len(spec.FunctionNodeTypes) != 3 { + t.Errorf("Go FunctionNodeTypes: got %d, want 3", len(spec.FunctionNodeTypes)) } - // Should contain function_declaration and method_declaration + // Should contain function_declaration, method_declaration, and method_elem (interface methods) found := map[string]bool{} for _, nt := range spec.FunctionNodeTypes { found[nt] = true } - if !found["function_declaration"] || !found["method_declaration"] { + if !found["function_declaration"] || !found["method_declaration"] || !found["method_elem"] { t.Errorf("Go FunctionNodeTypes missing expected types: %v", spec.FunctionNodeTypes) } } diff --git a/internal/lang/lua.go b/internal/lang/lua.go index d01e21561..44bbddc1b 100644 --- a/internal/lang/lua.go +++ b/internal/lang/lua.go @@ -10,6 +10,9 @@ func init() { CallNodeTypes: []string{"function_call"}, ImportNodeTypes: []string{"function_call"}, - BranchingNodeTypes: []string{"if_statement", "for_statement", "for_in_statement", "while_statement", "repeat_statement"}, + BranchingNodeTypes: []string{"if_statement", "for_statement", "for_in_statement", "while_statement", "repeat_statement"}, + VariableNodeTypes: []string{"variable_declaration"}, + AssignmentNodeTypes: []string{"assignment_statement"}, + EnvAccessFunctions: []string{"os.getenv"}, }) } diff --git a/internal/lang/objc.go b/internal/lang/objc.go new file mode 100644 index 000000000..e552d4ee6 --- /dev/null +++ b/internal/lang/objc.go @@ -0,0 +1,25 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: ObjectiveC, + FileExtensions: []string{".m"}, + FunctionNodeTypes: []string{ + "function_definition", + "method_definition", + }, + ClassNodeTypes: []string{ + "class_interface", + "class_implementation", + "protocol_declaration", + }, + FieldNodeTypes: []string{"property_declaration"}, + ModuleNodeTypes: []string{"translation_unit"}, + CallNodeTypes: []string{"call_expression", "message_expression"}, + ImportNodeTypes: []string{"preproc_import"}, + + BranchingNodeTypes: []string{"if_statement", "for_statement", "while_statement", "switch_statement"}, + VariableNodeTypes: []string{"declaration"}, + AssignmentNodeTypes: []string{"assignment_expression"}, + }) +} diff --git a/internal/lang/ocaml.go b/internal/lang/ocaml.go new file mode 100644 index 000000000..cccb2e0e9 --- /dev/null +++ b/internal/lang/ocaml.go @@ -0,0 +1,28 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: OCaml, + FileExtensions: []string{".ml", ".mli"}, + FunctionNodeTypes: []string{ + "value_definition", // let name args = body (contains let_binding) + }, + ClassNodeTypes: []string{ + "type_definition", // type color = Red | Green + "class_definition", // class foo = object ... end + "module_definition", // module M = struct ... end + }, + ModuleNodeTypes: []string{"compilation_unit"}, + CallNodeTypes: []string{"application", "infix_expression"}, + ImportNodeTypes: []string{"open_module"}, + + BranchingNodeTypes: []string{ + "match_expression", // match ... with + "if_expression", // if ... then ... else + "match_case", // individual pattern match case + }, + VariableNodeTypes: []string{"value_definition"}, + AssignmentNodeTypes: []string{"value_definition"}, + EnvAccessFunctions: []string{"Sys.getenv"}, + }) +} diff --git a/internal/lang/perl.go b/internal/lang/perl.go new file mode 100644 index 000000000..f750f11a1 --- /dev/null +++ b/internal/lang/perl.go @@ -0,0 +1,23 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Perl, + FileExtensions: []string{".pl", ".pm"}, + FunctionNodeTypes: []string{ + "subroutine_declaration_statement", + }, + ClassNodeTypes: []string{}, + FieldNodeTypes: []string{}, + ModuleNodeTypes: []string{"source_file"}, + CallNodeTypes: []string{"ambiguous_function_call_expression", "function_call_expression"}, + ImportNodeTypes: []string{"use_statement", "require_statement"}, + BranchingNodeTypes: []string{ + "if_statement", "unless_statement", "for_statement", + "foreach_statement", "while_statement", + }, + VariableNodeTypes: []string{"variable_declaration"}, + AssignmentNodeTypes: []string{"assignment_expression"}, + EnvAccessFunctions: []string{"$ENV"}, + }) +} diff --git a/internal/lang/php.go b/internal/lang/php.go index c1be1e863..1259b3bec 100644 --- a/internal/lang/php.go +++ b/internal/lang/php.go @@ -9,6 +9,7 @@ func init() { "anonymous_function", "function_definition", "arrow_function", + "method_declaration", }, ClassNodeTypes: []string{ "trait_declaration", @@ -24,7 +25,11 @@ func init() { "nullsafe_member_call_expression", }, - BranchingNodeTypes: []string{"if_statement", "for_statement", "foreach_statement", "while_statement", "switch_statement", "case_statement", "try_statement", "catch_clause"}, - ThrowNodeTypes: []string{"throw_expression"}, + BranchingNodeTypes: []string{"if_statement", "for_statement", "foreach_statement", "while_statement", "switch_statement", "case_statement", "try_statement", "catch_clause"}, + VariableNodeTypes: []string{"expression_statement"}, + AssignmentNodeTypes: []string{"assignment_expression"}, + ThrowNodeTypes: []string{"throw_expression"}, + DecoratorNodeTypes: []string{"attribute_group"}, + EnvAccessFunctions: []string{"getenv", "env"}, }) } diff --git a/internal/lang/r.go b/internal/lang/r.go new file mode 100644 index 000000000..d579a2c69 --- /dev/null +++ b/internal/lang/r.go @@ -0,0 +1,22 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: R, + FileExtensions: []string{".r", ".R"}, + FunctionNodeTypes: []string{ + "function_definition", + }, + ClassNodeTypes: []string{}, + FieldNodeTypes: []string{}, + ModuleNodeTypes: []string{"program"}, + CallNodeTypes: []string{"call"}, + ImportNodeTypes: []string{"call"}, + BranchingNodeTypes: []string{ + "if_statement", "for_statement", "while_statement", + }, + VariableNodeTypes: []string{"binary_operator"}, + AssignmentNodeTypes: []string{"binary_operator"}, + EnvAccessFunctions: []string{"Sys.getenv"}, + }) +} diff --git a/internal/lang/r_lang.go b/internal/lang/r_lang.go new file mode 100644 index 000000000..48e87f869 --- /dev/null +++ b/internal/lang/r_lang.go @@ -0,0 +1 @@ +package lang diff --git a/internal/lang/ruby.go b/internal/lang/ruby.go new file mode 100644 index 000000000..d84df550e --- /dev/null +++ b/internal/lang/ruby.go @@ -0,0 +1,22 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Ruby, + FileExtensions: []string{".rb", ".rake", ".gemspec"}, + FunctionNodeTypes: []string{ + "method", + "singleton_method", + }, + ClassNodeTypes: []string{"class", "module"}, + ModuleNodeTypes: []string{"program"}, + CallNodeTypes: []string{"call", "command_call"}, + ImportNodeTypes: []string{"call"}, + + BranchingNodeTypes: []string{"if", "unless", "while", "until", "for", "case", "when", "rescue", "elsif"}, + VariableNodeTypes: []string{"assignment"}, + AssignmentNodeTypes: []string{"assignment", "operator_assignment"}, + DecoratorNodeTypes: []string{}, + EnvAccessMemberPatterns: []string{"ENV"}, + }) +} diff --git a/internal/lang/rust.go b/internal/lang/rust.go index a2a3509d4..596721aab 100644 --- a/internal/lang/rust.go +++ b/internal/lang/rust.go @@ -14,7 +14,6 @@ func init() { "enum_item", "union_item", "trait_item", - "impl_item", "type_item", }, FieldNodeTypes: []string{"field_declaration"}, diff --git a/internal/lang/scala.go b/internal/lang/scala.go index 4aa5c66d3..73a1de655 100644 --- a/internal/lang/scala.go +++ b/internal/lang/scala.go @@ -20,7 +20,10 @@ func init() { ImportNodeTypes: []string{"import_declaration"}, ImportFromTypes: []string{"import_declaration"}, - BranchingNodeTypes: []string{"if_expression", "for_expression", "while_expression", "match_expression", "case_clause", "try_expression", "catch_clause"}, - ThrowNodeTypes: []string{"throw_expression"}, + BranchingNodeTypes: []string{"if_expression", "for_expression", "while_expression", "match_expression", "case_clause", "try_expression", "catch_clause"}, + VariableNodeTypes: []string{"val_definition", "var_definition", "val_declaration", "var_declaration"}, + AssignmentNodeTypes: []string{"assignment_expression"}, + ThrowNodeTypes: []string{"throw_expression"}, + EnvAccessFunctions: []string{"sys.env", "System.getenv", "System.getProperty"}, }) } diff --git a/internal/lang/scss.go b/internal/lang/scss.go new file mode 100644 index 000000000..9dbb902e8 --- /dev/null +++ b/internal/lang/scss.go @@ -0,0 +1,19 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: SCSS, + FileExtensions: []string{".scss"}, + ModuleNodeTypes: []string{"stylesheet"}, + FunctionNodeTypes: []string{ + "mixin_statement", + "function_statement", + }, + ClassNodeTypes: []string{}, + FieldNodeTypes: []string{}, + CallNodeTypes: []string{"call_expression"}, + ImportNodeTypes: []string{"import_statement", "use_statement"}, + VariableNodeTypes: []string{"declaration"}, + BranchingNodeTypes: []string{"if_statement"}, + }) +} diff --git a/internal/lang/sql.go b/internal/lang/sql.go new file mode 100644 index 000000000..07177ccd7 --- /dev/null +++ b/internal/lang/sql.go @@ -0,0 +1,21 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: SQL, + FileExtensions: []string{".sql"}, + ModuleNodeTypes: []string{"program"}, + FunctionNodeTypes: []string{ + "create_function", + }, + ClassNodeTypes: []string{}, + FieldNodeTypes: []string{"column_definition"}, + CallNodeTypes: []string{"function_call"}, + ImportNodeTypes: []string{}, + VariableNodeTypes: []string{ + "create_table", + "create_view", + }, + BranchingNodeTypes: []string{"if_statement", "case_expression"}, + }) +} diff --git a/internal/lang/swift.go b/internal/lang/swift.go new file mode 100644 index 000000000..a6cd8172c --- /dev/null +++ b/internal/lang/swift.go @@ -0,0 +1,29 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Swift, + FileExtensions: []string{".swift"}, + FunctionNodeTypes: []string{ + "function_declaration", + }, + ClassNodeTypes: []string{ + "class_declaration", + "protocol_declaration", + "struct_declaration", + "enum_declaration", + }, + FieldNodeTypes: []string{"property_declaration"}, + ModuleNodeTypes: []string{"source_file"}, + CallNodeTypes: []string{"call_expression"}, + ImportNodeTypes: []string{"import_declaration"}, + BranchingNodeTypes: []string{ + "if_statement", "guard_statement", "for_statement", + "while_statement", "switch_statement", + }, + VariableNodeTypes: []string{"property_declaration"}, + AssignmentNodeTypes: []string{"assignment"}, + ThrowNodeTypes: []string{"throw_statement"}, + DecoratorNodeTypes: []string{"attribute"}, + }) +} diff --git a/internal/lang/toml_lang.go b/internal/lang/toml_lang.go new file mode 100644 index 000000000..154224fc5 --- /dev/null +++ b/internal/lang/toml_lang.go @@ -0,0 +1,15 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: TOML, + FileExtensions: []string{".toml"}, + FunctionNodeTypes: []string{}, + ClassNodeTypes: []string{}, + ModuleNodeTypes: []string{"document"}, + CallNodeTypes: []string{}, + ImportNodeTypes: []string{}, + + VariableNodeTypes: []string{}, + }) +} diff --git a/internal/lang/yaml_lang.go b/internal/lang/yaml_lang.go new file mode 100644 index 000000000..405b73088 --- /dev/null +++ b/internal/lang/yaml_lang.go @@ -0,0 +1,15 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: YAML, + FileExtensions: []string{".yml", ".yaml"}, + FunctionNodeTypes: []string{}, + ClassNodeTypes: []string{}, + ModuleNodeTypes: []string{"stream"}, + CallNodeTypes: []string{}, + ImportNodeTypes: []string{}, + + VariableNodeTypes: []string{"block_mapping_pair"}, + }) +} diff --git a/internal/lang/zig.go b/internal/lang/zig.go new file mode 100644 index 000000000..df224964b --- /dev/null +++ b/internal/lang/zig.go @@ -0,0 +1,22 @@ +package lang + +func init() { + Register(&LanguageSpec{ + Language: Zig, + FileExtensions: []string{".zig"}, + FunctionNodeTypes: []string{ + "function_declaration", + "test_declaration", // test "description" { ... } + }, + ClassNodeTypes: []string{"struct_declaration", "enum_declaration", "union_declaration"}, + FieldNodeTypes: []string{"container_field"}, + ModuleNodeTypes: []string{"source_file"}, + CallNodeTypes: []string{"call_expression", "builtin_function"}, + ImportNodeTypes: []string{"builtin_function"}, + + BranchingNodeTypes: []string{"if_statement", "for_statement", "while_statement", "switch_expression"}, + VariableNodeTypes: []string{"variable_declaration"}, + AssignmentNodeTypes: []string{"assignment_expression"}, + EnvAccessFunctions: []string{"std.os.getenv"}, + }) +} diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 3d2829fbf..fce14e36f 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -6,19 +6,44 @@ import ( tree_sitter "github.com/tree-sitter/go-tree-sitter" + tree_sitter_bash "github.com/tree-sitter/tree-sitter-bash/bindings/go" tree_sitter_c_sharp "github.com/tree-sitter/tree-sitter-c-sharp/bindings/go" + tree_sitter_c "github.com/tree-sitter/tree-sitter-c/bindings/go" tree_sitter_cpp "github.com/tree-sitter/tree-sitter-cpp/bindings/go" + tree_sitter_css "github.com/tree-sitter/tree-sitter-css/bindings/go" tree_sitter_go "github.com/tree-sitter/tree-sitter-go/bindings/go" + tree_sitter_html "github.com/tree-sitter/tree-sitter-html/bindings/go" tree_sitter_java "github.com/tree-sitter/tree-sitter-java/bindings/go" tree_sitter_javascript "github.com/tree-sitter/tree-sitter-javascript/bindings/go" + tree_sitter_ocaml "github.com/tree-sitter/tree-sitter-ocaml/bindings/go" tree_sitter_php "github.com/tree-sitter/tree-sitter-php/bindings/go" tree_sitter_python "github.com/tree-sitter/tree-sitter-python/bindings/go" + tree_sitter_ruby "github.com/tree-sitter/tree-sitter-ruby/bindings/go" tree_sitter_rust "github.com/tree-sitter/tree-sitter-rust/bindings/go" tree_sitter_scala "github.com/tree-sitter/tree-sitter-scala/bindings/go" tree_sitter_typescript "github.com/tree-sitter/tree-sitter-typescript/bindings/go" + tree_sitter_hcl "github.com/tree-sitter-grammars/tree-sitter-hcl/bindings/go" tree_sitter_kotlin "github.com/tree-sitter-grammars/tree-sitter-kotlin/bindings/go" tree_sitter_lua "github.com/tree-sitter-grammars/tree-sitter-lua/bindings/go" + tree_sitter_toml "github.com/tree-sitter-grammars/tree-sitter-toml/bindings/go" + tree_sitter_yaml "github.com/tree-sitter-grammars/tree-sitter-yaml/bindings/go" + tree_sitter_zig "github.com/tree-sitter-grammars/tree-sitter-zig/bindings/go" + + tree_sitter_elixir "github.com/tree-sitter/tree-sitter-elixir/bindings/go" + tree_sitter_haskell "github.com/tree-sitter/tree-sitter-haskell/bindings/go" + + tree_sitter_objc "github.com/tree-sitter-grammars/tree-sitter-objc/bindings/go" + tree_sitter_scss "github.com/tree-sitter-grammars/tree-sitter-scss/bindings/go" + + tree_sitter_sql "github.com/DerekStride/tree-sitter-sql/bindings/go" + tree_sitter_dart "github.com/UserNobody14/tree-sitter-dart/bindings/go" + tree_sitter_swift "github.com/alex-pinkus/tree-sitter-swift/bindings/go" + tree_sitter_dockerfile "github.com/camdencheek/tree-sitter-dockerfile/bindings/go" + tree_sitter_groovy "github.com/murtaza64/tree-sitter-groovy/bindings/go" + tree_sitter_r "github.com/r-lib/tree-sitter-r/bindings/go" + tree_sitter_erlang "github.com/tree-sitter/tree-sitter-erlang/bindings/go" + tree_sitter_perl "github.com/tree-sitter/tree-sitter-perl/bindings/go" "github.com/DeusData/codebase-memory-mcp/internal/lang" ) @@ -45,6 +70,28 @@ func initLanguages() { lang.Lua: tree_sitter.NewLanguage(tree_sitter_lua.Language()), lang.Scala: tree_sitter.NewLanguage(tree_sitter_scala.Language()), lang.Kotlin: tree_sitter.NewLanguage(tree_sitter_kotlin.Language()), + lang.Ruby: tree_sitter.NewLanguage(tree_sitter_ruby.Language()), + lang.C: tree_sitter.NewLanguage(tree_sitter_c.Language()), + lang.Bash: tree_sitter.NewLanguage(tree_sitter_bash.Language()), + lang.Zig: tree_sitter.NewLanguage(tree_sitter_zig.Language()), + lang.Elixir: tree_sitter.NewLanguage(tree_sitter_elixir.Language()), + lang.Haskell: tree_sitter.NewLanguage(tree_sitter_haskell.Language()), + lang.OCaml: tree_sitter.NewLanguage(tree_sitter_ocaml.LanguageOCaml()), + lang.HTML: tree_sitter.NewLanguage(tree_sitter_html.Language()), + lang.CSS: tree_sitter.NewLanguage(tree_sitter_css.Language()), + lang.YAML: tree_sitter.NewLanguage(tree_sitter_yaml.Language()), + lang.TOML: tree_sitter.NewLanguage(tree_sitter_toml.Language()), + lang.HCL: tree_sitter.NewLanguage(tree_sitter_hcl.Language()), + lang.ObjectiveC: tree_sitter.NewLanguage(tree_sitter_objc.Language()), + lang.Swift: tree_sitter.NewLanguage(tree_sitter_swift.Language()), + lang.Dart: tree_sitter.NewLanguage(tree_sitter_dart.Language()), + lang.Perl: tree_sitter.NewLanguage(tree_sitter_perl.Language()), + lang.Groovy: tree_sitter.NewLanguage(tree_sitter_groovy.Language()), + lang.SCSS: tree_sitter.NewLanguage(tree_sitter_scss.Language()), + lang.R: tree_sitter.NewLanguage(tree_sitter_r.Language()), + lang.Erlang: tree_sitter.NewLanguage(tree_sitter_erlang.Language()), + lang.SQL: tree_sitter.NewLanguage(tree_sitter_sql.Language()), + lang.Dockerfile: tree_sitter.NewLanguage(tree_sitter_dockerfile.Language()), } parserPools = make(map[lang.Language]*sync.Pool, len(languages)) diff --git a/internal/pipeline/astdump_test.go b/internal/pipeline/astdump_test.go new file mode 100644 index 000000000..9a38edab1 --- /dev/null +++ b/internal/pipeline/astdump_test.go @@ -0,0 +1,369 @@ +package pipeline + +import ( + "fmt" + "strings" + "testing" + + tree_sitter "github.com/tree-sitter/go-tree-sitter" + + "github.com/DeusData/codebase-memory-mcp/internal/lang" + "github.com/DeusData/codebase-memory-mcp/internal/parser" +) + +func dumpNode(node *tree_sitter.Node, source []byte, indent int) string { + var sb strings.Builder + prefix := strings.Repeat(" ", indent) + text := string(source[node.StartByte():node.EndByte()]) + if len(text) > 60 { + text = text[:60] + "..." + } + text = strings.ReplaceAll(text, "\n", "\\n") + fmt.Fprintf(&sb, "%s%s [%s] field=%q :: %q\n", prefix, node.Kind(), node.GrammarName(), fieldNameOfNode(node), text) + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child != nil { + sb.WriteString(dumpNode(child, source, indent+1)) + } + } + return sb.String() +} + +func fieldNameOfNode(node *tree_sitter.Node) string { + parent := node.Parent() + if parent == nil { + return "" + } + for i := uint(0); i < parent.ChildCount(); i++ { + child := parent.Child(i) + if child != nil && child.Id() == node.Id() { + name := parent.FieldNameForChild(uint32(i)) + return name + } + } + return "" +} + +var astDumpCases = []struct { + name string + lang lang.Language + code string +}{ + // Kotlin function (complexity) + {"kotlin_func", lang.Kotlin, "fun f() {\n\tif (x) {}\n\tfor (i in 1..10) {}\n}\n"}, + // Kotlin class with base + {"kotlin_class", lang.Kotlin, "class Child : Parent() {}\n"}, + // Kotlin annotation + {"kotlin_anno", lang.Kotlin, "@MyAnnotation\nfun f() {}\n"}, + // Kotlin params + {"kotlin_params", lang.Kotlin, "fun f(cfg: Config) {}\n"}, + // Kotlin return type + {"kotlin_return", lang.Kotlin, "fun f(): Config { TODO() }\n"}, + // C++ params + {"cpp_params", lang.CPP, "void f(Config cfg) {}\n"}, + // C# params + {"csharp_params", lang.CSharp, "class A {\n\tvoid F(Config cfg) {}\n}\n"}, + // PHP params + {"php_params", lang.PHP, " {\n\t\tres.json([]);\n\t};\n}\n"}, + // TS arrow function class property without type annotation + {"ts_arrow_class_prop_notype", lang.TypeScript, "class A {\n\tgreet = () => 'hello';\n}\n"}, + // JS arrow function class property + {"js_arrow_class_prop", lang.JavaScript, "class A {\n\tgreet = () => 'hello';\n\thandle = async (req) => {\n\t\treturn req;\n\t};\n}\n"}, + // JS const arrow function at module level + {"js_const_arrow", lang.JavaScript, "const greet = () => 'hello';\nexport const handler = async (req) => { return req; };\n"}, + // TS const arrow function at module level + {"ts_const_arrow", lang.TypeScript, "const greet = (): string => 'hello';\nexport const handler = async (req: Request): Promise => { return new Response(); };\n"}, + // Go interface with methods + {"go_interface", lang.Go, "package main\n\ntype Router interface {\n\tGet(pattern string, h HandlerFunc)\n\tPost(pattern string, h HandlerFunc)\n\tServeHTTP(w ResponseWriter, r *Request)\n}\n"}, + // Rust attribute macros (#[get("/path")]) + {"rust_attr_macro", lang.Rust, "#[get(\"/users\")]\nasync fn get_users() -> HttpResponse {\n\tHttpResponse::Ok().finish()\n}\n"}, + // Rust impl Trait for Struct + {"rust_impl_trait", lang.Rust, "trait Handler {\n\tfn handle(&self);\n}\nstruct MyHandler;\nimpl Handler for MyHandler {\n\tfn handle(&self) {}\n}\n"}, + // Kotlin variable top-level + {"kotlin_var_toplevel", lang.Kotlin, "val apiUrl = \"https://example.com\"\n"}, + // C++ class with throw + {"cpp_throw2", lang.CPP, "class MyError {};\nvoid f() {\n\tthrow MyError();\n}\n"}, + // Java class with separate throw + {"java_two_classes", lang.Java, "class MyError extends RuntimeException {}\nclass A {\n\tvoid f() {\n\t\tthrow new MyError();\n\t}\n}\n"}, + + // === Issue 1: C# modern feature diagnostics === + {"csharp_file_scoped_ns", lang.CSharp, "namespace Conduit.Features;\nclass A {\n\tvoid F() {}\n}\n"}, + {"csharp_primary_ctor", lang.CSharp, "public class UsersController(IMediator mediator) {\n\tpublic void Get() => mediator.Send();\n}\n"}, + {"csharp_expression_body", lang.CSharp, "class A {\n\tpublic string Name => \"test\";\n\tpublic int Add(int a, int b) => a + b;\n}\n"}, + {"csharp_traditional", lang.CSharp, "namespace Conduit.Features {\n\tclass A {\n\t\tvoid F() {}\n\t}\n}\n"}, + {"csharp_global_using", lang.CSharp, "global using System;\nnamespace A {\n\tclass B {\n\t\tvoid F() {}\n\t}\n}\n"}, + + // === Issue 2: Lua anonymous function diagnostics === + {"lua_anon_func_assign", lang.Lua, "local f\nf = function(x)\n\treturn x\nend\n"}, + {"lua_local_func_assign", lang.Lua, "local f = function(x)\n\treturn x\nend\n"}, + {"lua_table_func", lang.Lua, "local t = {\n\tf = function(x)\n\t\treturn x\n\tend\n}\n"}, + {"lua_func_declaration", lang.Lua, "function f(x)\n\treturn x\nend\n"}, + {"lua_method_colon", lang.Lua, "function obj:method(x)\n\treturn x\nend\n"}, + + // === New languages: Ruby === + {"ruby_func", lang.Ruby, "def greet(name)\n puts name\nend\n"}, + {"ruby_class", lang.Ruby, "class Dog < Animal\n def bark\n puts 'woof'\n end\nend\n"}, + {"ruby_module", lang.Ruby, "module Helpers\n def self.run\n 42\n end\nend\n"}, + {"ruby_if", lang.Ruby, "def f(x)\n if x > 0\n x\n elsif x == 0\n 0\n else\n -x\n end\nend\n"}, + {"ruby_require", lang.Ruby, "require 'json'\nrequire_relative 'helper'\n"}, + {"ruby_var", lang.Ruby, "API_URL = 'https://example.com'\n"}, + {"ruby_env", lang.Ruby, "val = ENV['API_KEY']\n"}, + + // === New languages: C === + {"c_func", lang.C, "int add(int a, int b) {\n\treturn a + b;\n}\n"}, + {"c_struct", lang.C, "struct Point {\n\tint x;\n\tint y;\n};\n"}, + {"c_include", lang.C, "#include \n#include \"myheader.h\"\n"}, + {"c_if", lang.C, "int f(int x) {\n\tif (x > 0) return x;\n\tfor (int i = 0; i < x; i++) {}\n}\n"}, + {"c_var", lang.C, "const char *API_URL = \"https://example.com\";\n"}, + {"c_enum", lang.C, "enum Color { RED, GREEN, BLUE };\n"}, + + // === New languages: Bash === + {"bash_func", lang.Bash, "greet() {\n\techo \"hello $1\"\n}\n"}, + {"bash_func_keyword", lang.Bash, "function greet {\n\techo \"hello\"\n}\n"}, + {"bash_if", lang.Bash, "if [ -f file ]; then\n\techo exists\nelif [ -d dir ]; then\n\techo dir\nfi\n"}, + {"bash_var", lang.Bash, "API_URL=\"https://example.com\"\n"}, + {"bash_command", lang.Bash, "curl -s https://example.com\ngrep -r pattern .\n"}, + + // === New languages: Zig === + {"zig_func", lang.Zig, "fn add(a: i32, b: i32) i32 {\n\treturn a + b;\n}\n"}, + {"zig_struct", lang.Zig, "const Point = struct {\n\tx: i32,\n\ty: i32,\n};\n"}, + {"zig_if", lang.Zig, "fn f(x: i32) i32 {\n\tif (x > 0) return x;\n\tfor (0..10) |i| {}\n}\n"}, + {"zig_import", lang.Zig, "const std = @import(\"std\");\n"}, + {"zig_var", lang.Zig, "const API_URL = \"https://example.com\";\nvar count: i32 = 0;\n"}, + + // === New languages: Elixir === + {"elixir_func", lang.Elixir, "defmodule MyApp do\n def greet(name) do\n IO.puts(name)\n end\nend\n"}, + {"elixir_module", lang.Elixir, "defmodule MyApp.Router do\n use Plug.Router\n plug :match\nend\n"}, + {"elixir_if", lang.Elixir, "def f(x) do\n if x > 0 do\n x\n else\n -x\n end\nend\n"}, + {"elixir_import", lang.Elixir, "import Ecto.Query\nalias MyApp.Repo\n"}, + {"elixir_var", lang.Elixir, "x = 42\n"}, + + // === New languages: Haskell === + {"haskell_func", lang.Haskell, "add :: Int -> Int -> Int\nadd x y = x + y\n"}, + {"haskell_data", lang.Haskell, "data Color = Red | Green | Blue\n"}, + {"haskell_class", lang.Haskell, "class Eq a where\n (==) :: a -> a -> Bool\n"}, + {"haskell_import", lang.Haskell, "import Data.Map (Map, fromList)\nimport qualified Data.Text as T\n"}, + {"haskell_guard", lang.Haskell, "abs x\n | x < 0 = -x\n | otherwise = x\n"}, + {"haskell_case", lang.Haskell, "f x = case x of\n 0 -> \"zero\"\n _ -> \"other\"\n"}, + + // === New languages: OCaml === + {"ocaml_func", lang.OCaml, "let add x y = x + y\n"}, + {"ocaml_type", lang.OCaml, "type color = Red | Green | Blue\n"}, + {"ocaml_module", lang.OCaml, "module M = struct\n let x = 42\nend\n"}, + {"ocaml_open", lang.OCaml, "open Printf\n"}, + {"ocaml_match", lang.OCaml, "let f x = match x with\n | 0 -> \"zero\"\n | _ -> \"other\"\n"}, + {"ocaml_if", lang.OCaml, "let f x = if x > 0 then x else -x\n"}, + + // === New languages: HTML === + {"html_basic", lang.HTML, "\nTest\n

Hello

\n\n"}, + {"html_form", lang.HTML, "
\n \n
\n"}, + {"html_link", lang.HTML, "About\n\n"}, + + // === New languages: CSS === + {"css_rule", lang.CSS, "body {\n color: red;\n font-size: 14px;\n}\n"}, + {"css_import", lang.CSS, "@import url('other.css');\n"}, + {"css_class", lang.CSS, ".container {\n display: flex;\n}\n"}, + + // === New languages: YAML === + {"yaml_basic", lang.YAML, "name: test\nversion: 1.0\n"}, + {"yaml_nested", lang.YAML, "database:\n host: localhost\n port: 5432\n"}, + + // === New languages: TOML === + {"toml_basic", lang.TOML, "name = \"test\"\nversion = \"1.0\"\n"}, + {"toml_section", lang.TOML, "[database]\nhost = \"localhost\"\nport = 5432\n"}, + + // === New languages: HCL === + {"hcl_resource", lang.HCL, "resource \"aws_instance\" \"web\" {\n ami = \"abc-123\"\n instance_type = \"t2.micro\"\n}\n"}, + {"hcl_variable", lang.HCL, "variable \"region\" {\n default = \"us-east-1\"\n}\n"}, + {"hcl_func", lang.HCL, "output \"ip\" {\n value = lookup(var.map, \"key\")\n}\n"}, + + // === New languages: Objective-C === + {"objc_interface", lang.ObjectiveC, "@interface Dog : NSObject\n@property NSString *name;\n- (void)bark;\n@end\n"}, + {"objc_impl", lang.ObjectiveC, "@implementation Dog\n- (void)bark {\n\tNSLog(@\"Woof\");\n}\n@end\n"}, + {"objc_func", lang.ObjectiveC, "void greet(NSString *name) {\n\tNSLog(@\"Hello %@\", name);\n}\n"}, + {"objc_import", lang.ObjectiveC, "#import \n#import \"Dog.h\"\n"}, + {"objc_if", lang.ObjectiveC, "void f(int x) {\n\tif (x > 0) {\n\t\tNSLog(@\"pos\");\n\t} else {\n\t\tNSLog(@\"neg\");\n\t}\n\tfor (int i = 0; i < x; i++) {}\n}\n"}, + {"objc_var", lang.ObjectiveC, "NSString *const API_URL = @\"https://example.com\";\n"}, + {"objc_protocol", lang.ObjectiveC, "@protocol Runnable\n- (void)run;\n@end\n"}, + + // === New languages: Swift === + {"swift_func", lang.Swift, "func greet(name: String) -> String {\n\treturn \"Hello \" + name\n}\n"}, + {"swift_class", lang.Swift, "class Dog: Animal {\n\tfunc bark() {\n\t\tprint(\"woof\")\n\t}\n}\n"}, + {"swift_struct", lang.Swift, "struct Point {\n\tvar x: Int\n\tvar y: Int\n}\n"}, + {"swift_protocol", lang.Swift, "protocol Runnable {\n\tfunc run()\n}\n"}, + {"swift_if", lang.Swift, "func f(x: Int) -> Int {\n\tif x > 0 {\n\t\treturn x\n\t}\n\treturn -x\n}\n"}, + {"swift_import", lang.Swift, "import Foundation\nimport UIKit\n"}, + {"swift_var", lang.Swift, "let apiUrl = \"https://example.com\"\nvar count = 0\n"}, + {"swift_enum", lang.Swift, "enum Color {\n\tcase red, green, blue\n}\n"}, + + // === New languages: Dart === + {"dart_func", lang.Dart, "int add(int a, int b) {\n\treturn a + b;\n}\n"}, + {"dart_class", lang.Dart, "class Dog extends Animal {\n\tvoid bark() {\n\t\tprint('woof');\n\t}\n}\n"}, + {"dart_if", lang.Dart, "int f(int x) {\n\tif (x > 0) return x;\n\tfor (var i = 0; i < x; i++) {}\n\treturn -x;\n}\n"}, + {"dart_import", lang.Dart, "import 'dart:io';\nimport 'package:flutter/material.dart';\n"}, + {"dart_var", lang.Dart, "const apiUrl = 'https://example.com';\nvar count = 0;\n"}, + {"dart_enum", lang.Dart, "enum Color { red, green, blue }\n"}, + {"dart_mixin", lang.Dart, "mixin Swimming {\n\tvoid swim() {\n\t\tprint('swimming');\n\t}\n}\n"}, + + // === New languages: Perl === + {"perl_func", lang.Perl, "sub greet {\n\tmy ($name) = @_;\n\tprint \"Hello $name\\n\";\n}\n"}, + {"perl_package", lang.Perl, "package Dog;\nuse strict;\nsub new {\n\tmy ($class) = @_;\n\treturn bless {}, $class;\n}\n1;\n"}, + {"perl_if", lang.Perl, "sub f {\n\tmy ($x) = @_;\n\tif ($x > 0) {\n\t\treturn $x;\n\t}\n\treturn -$x;\n}\n"}, + {"perl_use", lang.Perl, "use strict;\nuse warnings;\nuse File::Path;\n"}, + {"perl_var", lang.Perl, "my $api_url = 'https://example.com';\nour $VERSION = '1.0';\n"}, + + // === New languages: Groovy === + {"groovy_func", lang.Groovy, "def greet(name) {\n\tprintln \"Hello $name\"\n}\n"}, + {"groovy_class", lang.Groovy, "class Dog extends Animal {\n\tvoid bark() {\n\t\tprintln 'woof'\n\t}\n}\n"}, + {"groovy_if", lang.Groovy, "def f(x) {\n\tif (x > 0) return x\n\tfor (i in 0..x) {}\n\treturn -x\n}\n"}, + {"groovy_import", lang.Groovy, "import groovy.json.JsonSlurper\n"}, + {"groovy_var", lang.Groovy, "def apiUrl = 'https://example.com'\n"}, + + // === New languages: Erlang === + {"erlang_func", lang.Erlang, "-module(math).\n-export([add/2]).\nadd(A, B) -> A + B.\n"}, + {"erlang_case", lang.Erlang, "f(X) ->\n\tcase X of\n\t\t0 -> zero;\n\t\t_ -> other\n\tend.\n"}, + {"erlang_import", lang.Erlang, "-module(my_mod).\n-import(lists, [map/2, filter/2]).\n"}, + {"erlang_if", lang.Erlang, "f(X) ->\n\tif\n\t\tX > 0 -> positive;\n\t\tX < 0 -> negative;\n\t\ttrue -> zero\n\tend.\n"}, + {"erlang_remote_call", lang.Erlang, "-module(mymod).\nmain() ->\n\tio:format(\"hello ~s~n\", [Name]),\n\tgreet(\"World\").\ngreet(Name) -> Name.\n"}, + {"erlang_define_record", lang.Erlang, "-module(mymod).\n-define(TIMEOUT, 5000).\n-record(person, {name, age}).\n"}, + + // === New languages: R === + {"r_func", lang.R, "add <- function(a, b) {\n\ta + b\n}\n"}, + {"r_if", lang.R, "f <- function(x) {\n\tif (x > 0) x else -x\n}\n"}, + {"r_var", lang.R, "api_url <- \"https://example.com\"\ncount = 0\n"}, + {"r_import", lang.R, "library(ggplot2)\nrequire(dplyr)\n"}, + + // === New languages: SCSS === + {"scss_mixin", lang.SCSS, "@mixin flex-center {\n\tdisplay: flex;\n\talign-items: center;\n}\n"}, + {"scss_var", lang.SCSS, "$primary-color: #333;\n$font-size: 14px;\n"}, + {"scss_import", lang.SCSS, "@import 'variables';\n@use 'mixins';\n"}, + {"scss_nested", lang.SCSS, ".container {\n\t.header {\n\t\tcolor: red;\n\t}\n}\n"}, + + // === New languages: SQL === + {"sql_table", lang.SQL, "CREATE TABLE users (\n\tid INT PRIMARY KEY,\n\tname VARCHAR(100)\n);\n"}, + {"sql_select", lang.SQL, "SELECT u.name, COUNT(o.id) FROM users u JOIN orders o ON u.id = o.user_id GROUP BY u.name;\n"}, + {"sql_func", lang.SQL, "CREATE FUNCTION add(a INT, b INT) RETURNS INT AS $$ SELECT a + b; $$ LANGUAGE SQL;\n"}, + {"sql_func_call", lang.SQL, "SELECT count(*), upper(name) FROM users;\n"}, + {"sql_view", lang.SQL, "CREATE VIEW active_users AS SELECT * FROM users WHERE active = true;\n"}, + + // === New languages: Dockerfile === + {"dockerfile_from", lang.Dockerfile, "FROM golang:1.22-alpine AS builder\nWORKDIR /app\nCOPY . .\nRUN go build -o main .\n"}, + {"dockerfile_env", lang.Dockerfile, "ENV APP_PORT=8080\nARG VERSION=latest\nEXPOSE 8080\n"}, +} + +func TestDumpAST(t *testing.T) { + for _, tt := range astDumpCases { + t.Run(tt.name, func(t *testing.T) { + tree, src := parseSource(t, tt.lang, tt.code) + defer tree.Close() + dump := dumpNode(tree.RootNode(), src, 0) + t.Log("\n" + dump) + }) + } +} + +// TestLuaFuncAssignName verifies that luaFuncAssignName correctly extracts +// variable names for anonymous function assignments. +func TestLuaFuncAssignName(t *testing.T) { + tests := []struct { + name string + code string + wantName string + }{ + {"separate_assign", "local f\nf = function(x)\n\treturn x\nend\n", "f"}, + {"local_assign", "local f = function(x)\n\treturn x\nend\n", "f"}, + {"named_func", "function f(x)\n\treturn x\nend\n", ""}, // not anonymous — funcNameNode handles this + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tree, src := parseSource(t, lang.Lua, tt.code) + defer tree.Close() + + // Walk the AST to find function_definition nodes + var foundName string + parser.Walk(tree.RootNode(), func(node *tree_sitter.Node) bool { + if node.Kind() == "function_definition" { + nameNode := funcNameNode(node) + if nameNode == nil { + nameNode = luaFuncAssignName(node) + } + if nameNode != nil { + foundName = parser.NodeText(nameNode, src) + } else { + t.Logf("function_definition found but no name resolved") + t.Logf(" parent: %s", node.Parent().Kind()) + if node.Parent().Parent() != nil { + t.Logf(" grandparent: %s", node.Parent().Parent().Kind()) + } + } + return false + } + return true + }) + + if tt.wantName == "" { + // For named functions, funcNameNode handles it (not luaFuncAssignName) + return + } + if foundName != tt.wantName { + t.Errorf("luaFuncAssignName: got %q, want %q", foundName, tt.wantName) + } + }) + } +} diff --git a/internal/pipeline/cancel_integration_test.go b/internal/pipeline/cancel_integration_test.go new file mode 100644 index 000000000..80305b86e --- /dev/null +++ b/internal/pipeline/cancel_integration_test.go @@ -0,0 +1,48 @@ +package pipeline + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +// TestGracefulShutdownLargeRepo indexes a real 300MB repo with a 2s timeout. +// Skipped if /tmp/bench/erlang does not exist. +func TestGracefulShutdownLargeRepo(t *testing.T) { + repoPath := "/tmp/bench/erlang" + if _, err := os.Stat(repoPath); os.IsNotExist(err) { + t.Skip("skipping: /tmp/bench/erlang not found") + } + + st, err := store.OpenMemory() + if err != nil { + t.Fatalf("store: %v", err) + } + defer st.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + start := time.Now() + p := New(ctx, st, repoPath) + err = p.Run() + elapsed := time.Since(start) + + if err == nil { + t.Fatalf("expected cancellation error, but index completed in %v", elapsed) + } + if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { + t.Fatalf("expected context error, got: %v (after %v)", err, elapsed) + } + + t.Logf("PASS: cancelled after %v with %v", elapsed, err) + + // Ensure it stopped reasonably fast (within 5s of the 2s deadline) + if elapsed > 7*time.Second { + t.Errorf("cancellation took too long: %v (expected <7s)", elapsed) + } +} diff --git a/internal/pipeline/decorates.go b/internal/pipeline/decorates.go index fb75f5f71..9fb000866 100644 --- a/internal/pipeline/decorates.go +++ b/internal/pipeline/decorates.go @@ -2,6 +2,7 @@ package pipeline import ( "log/slog" + "slices" "strings" "github.com/DeusData/codebase-memory-mcp/internal/store" @@ -84,3 +85,109 @@ func decoratorFunctionName(dec string) string { } return strings.TrimSpace(dec) } + +type taggedNode struct { + qn string + words []string +} + +// collectDecoratedNodes extracts tokenized decorator words from all decorated nodes. +func (p *Pipeline) collectDecoratedNodes() (nodes []taggedNode, wordCount map[string]int) { + wordCount = map[string]int{} + + for _, label := range []string{"Function", "Method", "Class"} { + found, err := p.Store.FindNodesByLabel(p.ProjectName, label) + if err != nil { + continue + } + for _, n := range found { + words := extractDecoratorWords(n) + if len(words) > 0 { + nodes = append(nodes, taggedNode{qn: n.QualifiedName, words: words}) + for _, w := range words { + wordCount[w]++ + } + } + } + } + return nodes, wordCount +} + +// extractDecoratorWords tokenizes all decorators on a node into unique words. +func extractDecoratorWords(n *store.Node) []string { + decs, ok := n.Properties["decorators"] + if !ok { + return nil + } + decList, ok := decs.([]any) + if !ok { + return nil + } + seen := map[string]bool{} + var words []string + for _, d := range decList { + decStr, ok := d.(string) + if !ok { + continue + } + for _, w := range tokenizeDecorator(decStr) { + if !seen[w] { + seen[w] = true + words = append(words, w) + } + } + } + return words +} + +// passDecoratorTags classifies decorators into semantic tags via auto-discovery. +// Words appearing on 2+ distinct nodes become tag candidates. +func (p *Pipeline) passDecoratorTags() { + nodes, wordCount := p.collectDecoratedNodes() + + // Tag candidates: words on 2+ distinct nodes + candidates := map[string]bool{} + for w, c := range wordCount { + if c >= 2 { + candidates[w] = true + } + } + + if len(candidates) == 0 { + return + } + + tagged := 0 + for _, tn := range nodes { + var tags []string + for _, w := range tn.words { + if candidates[w] { + tags = append(tags, w) + } + } + + node, err := p.Store.FindNodeByQN(p.ProjectName, tn.qn) + if err != nil || node == nil { + continue + } + + if len(tags) == 0 { + // Remove stale tags from previous runs + if _, hasTags := node.Properties["decorator_tags"]; hasTags { + delete(node.Properties, "decorator_tags") + _, _ = p.Store.UpsertNode(node) + } + continue + } + + slices.Sort(tags) + if node.Properties == nil { + node.Properties = map[string]any{} + } + node.Properties["decorator_tags"] = tags + _, _ = p.Store.UpsertNode(node) + tagged++ + } + + slog.Info("pass.decorator_tags", "candidates", len(candidates), "tagged", tagged) +} diff --git a/internal/pipeline/docstrings.go b/internal/pipeline/docstrings.go new file mode 100644 index 000000000..d93109082 --- /dev/null +++ b/internal/pipeline/docstrings.go @@ -0,0 +1,220 @@ +package pipeline + +import ( + "bytes" + "strings" + + tree_sitter "github.com/tree-sitter/go-tree-sitter" + + "github.com/DeusData/codebase-memory-mcp/internal/lang" + "github.com/DeusData/codebase-memory-mcp/internal/parser" +) + +// extractDocstring extracts the documentation comment for a function/class node. +// Python: looks for triple-quoted string as first statement in body (PEP 257). +// Others: scans source lines backwards from the node for comment lines. +func extractDocstring(node *tree_sitter.Node, source []byte, language lang.Language) string { + if language == lang.Python { + return extractPythonDocstring(node, source) + } + return extractCommentDocstring(source, int(node.StartPosition().Row), language) +} + +// extractPythonDocstring extracts a PEP 257 docstring from a function/class body. +func extractPythonDocstring(node *tree_sitter.Node, source []byte) string { + body := node.ChildByFieldName("body") + if body == nil { + return "" + } + if body.NamedChildCount() == 0 { + return "" + } + first := body.NamedChild(0) + if first == nil || first.Kind() != "expression_statement" { + return "" + } + if first.NamedChildCount() == 0 { + return "" + } + strNode := first.NamedChild(0) + if strNode == nil || strNode.Kind() != "string" { + return "" + } + return cleanPythonDocstring(parser.NodeText(strNode, source)) +} + +// cleanPythonDocstring removes triple-quote delimiters and normalizes indentation. +func cleanPythonDocstring(s string) string { + for _, delim := range []string{`"""`, `'''`} { + if strings.HasPrefix(s, delim) && strings.HasSuffix(s, delim) && len(s) >= 6 { + s = s[3 : len(s)-3] + break + } + } + lines := strings.Split(s, "\n") + if len(lines) <= 1 { + return strings.TrimSpace(s) + } + // Dedent: find minimum indentation of non-empty continuation lines. + minIndent := -1 + for _, line := range lines[1:] { + trimmed := strings.TrimLeft(line, " \t") + if trimmed == "" { + continue + } + indent := len(line) - len(trimmed) + if minIndent < 0 || indent < minIndent { + minIndent = indent + } + } + if minIndent > 0 { + for i := 1; i < len(lines); i++ { + if len(lines[i]) >= minIndent { + lines[i] = lines[i][minIndent:] + } + } + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} + +// extractCommentDocstring scans backwards from startLine for doc comments. +func extractCommentDocstring(source []byte, startLine int, language lang.Language) string { + lines := bytes.Split(source, []byte("\n")) + if startLine <= 0 || startLine > len(lines) { + return "" + } + + lineIdx := startLine - 1 + trimmed := strings.TrimSpace(string(lines[lineIdx])) + if trimmed == "" { + return "" + } + + // Block comment ending with */ + if strings.HasSuffix(trimmed, "*/") { + return extractBlockComment(lines, lineIdx) + } + + // Lua block comment ending with ]] + if language == lang.Lua && strings.HasSuffix(trimmed, "]]") { + return extractLuaBlockComment(lines, lineIdx) + } + + // Line comments + prefix := docLinePrefix(language) + if prefix != "" && strings.HasPrefix(trimmed, prefix) { + return extractLineComments(lines, lineIdx, prefix) + } + + return "" +} + +// docLinePrefix returns the conventional doc-comment line prefix for a language. +func docLinePrefix(language lang.Language) string { + switch language { + case lang.Rust: + return "///" + case lang.CSharp: + return "///" + case lang.Lua: + return "---" + case lang.Go, lang.CPP, lang.JavaScript, lang.TypeScript, lang.TSX, + lang.Java, lang.Scala, lang.Kotlin, lang.PHP: + return "//" + default: + return "" + } +} + +// extractBlockComment scans backwards from endLineIdx to find the start of a /* or /** block. +func extractBlockComment(lines [][]byte, endLineIdx int) string { + startIdx := endLineIdx + for startIdx >= 0 { + line := strings.TrimSpace(string(lines[startIdx])) + if strings.HasPrefix(line, "/*") { + break + } + startIdx-- + } + if startIdx < 0 { + return "" + } + + var result []string + for i := startIdx; i <= endLineIdx; i++ { + result = append(result, string(lines[i])) + } + return cleanBlockComment(strings.Join(result, "\n")) +} + +// cleanBlockComment strips /** ... */ delimiters and leading * prefixes. +func cleanBlockComment(s string) string { + s = strings.TrimSpace(s) + if strings.HasPrefix(s, "/**") { + s = s[3:] + } else if strings.HasPrefix(s, "/*") { + s = s[2:] + } + s = strings.TrimSuffix(s, "*/") + + lines := strings.Split(s, "\n") + cleaned := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + line = strings.TrimPrefix(line, "* ") + line = strings.TrimPrefix(line, "*") + cleaned = append(cleaned, line) + } + return strings.TrimSpace(strings.Join(cleaned, "\n")) +} + +// extractLuaBlockComment scans backwards from endLineIdx for --[[ ... ]]. +func extractLuaBlockComment(lines [][]byte, endLineIdx int) string { + startIdx := endLineIdx + for startIdx >= 0 { + line := strings.TrimSpace(string(lines[startIdx])) + if strings.Contains(line, "--[[") { + break + } + startIdx-- + } + if startIdx < 0 { + return "" + } + + var result []string + for i := startIdx; i <= endLineIdx; i++ { + result = append(result, string(lines[i])) + } + text := strings.Join(result, "\n") + text = strings.TrimSpace(text) + + if idx := strings.Index(text, "--[["); idx >= 0 { + text = text[idx+4:] + } + if idx := strings.LastIndex(text, "]]"); idx >= 0 { + text = text[:idx] + } + return strings.TrimSpace(text) +} + +// extractLineComments collects consecutive line comments ending at startIdx. +func extractLineComments(lines [][]byte, startIdx int, prefix string) string { + var commentLines []string + idx := startIdx + for idx >= 0 { + trimmed := strings.TrimSpace(string(lines[idx])) + if !strings.HasPrefix(trimmed, prefix) { + break + } + content := strings.TrimPrefix(trimmed, prefix) + content = strings.TrimPrefix(content, " ") + commentLines = append(commentLines, content) + idx-- + } + // Reverse (scanned backwards). + for i, j := 0, len(commentLines)-1; i < j; i, j = i+1, j-1 { + commentLines[i], commentLines[j] = commentLines[j], commentLines[i] + } + return strings.TrimSpace(strings.Join(commentLines, "\n")) +} diff --git a/internal/pipeline/docstrings_test.go b/internal/pipeline/docstrings_test.go new file mode 100644 index 000000000..ffb246bf9 --- /dev/null +++ b/internal/pipeline/docstrings_test.go @@ -0,0 +1,346 @@ +package pipeline + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/DeusData/codebase-memory-mcp/internal/lang" + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +// --- Unit Tests: extractDocstring --- + +var docstringTestCases = []struct { + name string + language lang.Language + source string + want string // required substring in extracted docstring +}{ + { + "Python", + lang.Python, + "def f():\n\t\"\"\"Computes the result.\"\"\"\n\tpass\n", + "Computes the result.", + }, + { + "Python_multiline", + lang.Python, + "def f():\n\t\"\"\"Computes the result.\n\n\tMore details here.\n\t\"\"\"\n\tpass\n", + "More details here.", + }, + { + "Go", + lang.Go, + "// ComputeResult computes the result.\nfunc f() {}\n", + "ComputeResult computes the result.", + }, + { + "Go_multiline", + lang.Go, + "// ComputeResult computes the result.\n// It returns an int.\nfunc f() {}\n", + "It returns an int.", + }, + { + "JavaScript", + lang.JavaScript, + "/** Computes the result. */\nfunction f() {}\n", + "Computes the result.", + }, + { + "TypeScript", + lang.TypeScript, + "/** Computes the result. */\nfunction f(): void {}\n", + "Computes the result.", + }, + { + "TSX", + lang.TSX, + "/** Computes the result. */\nfunction f(): void {}\n", + "Computes the result.", + }, + { + "Java", + lang.Java, + "class A {\n\t/** Computes the result. */\n\tvoid f() {}\n}\n", + "Computes the result.", + }, + { + "Java_multiline", + lang.Java, + "class A {\n\t/**\n\t * Computes the result.\n\t * @param x input\n\t */\n\tvoid f() {}\n}\n", + "Computes the result.", + }, + { + "Rust", + lang.Rust, + "/// Computes the result.\nfn f() {}\n", + "Computes the result.", + }, + { + "Rust_multiline", + lang.Rust, + "/// Computes the result.\n/// Returns nothing.\nfn f() {}\n", + "Returns nothing.", + }, + { + "CPP", + lang.CPP, + "/// Computes the result.\nvoid f() {}\n", + "Computes the result.", + }, + { + "CPP_block", + lang.CPP, + "/** Computes the result. */\nvoid f() {}\n", + "Computes the result.", + }, + { + "CSharp", + lang.CSharp, + "class A {\n\t/// Computes the result.\n\tvoid F() {}\n}\n", + "Computes the result.", + }, + { + "PHP", + lang.PHP, + " 0 { + text = text[:idx] + } + text = strings.TrimSpace(text) + if text != "" { + bases = append(bases, text) + } + } + } + } + return bases +} + // isAbstractClass returns true if the class node has abstract modifiers. func isAbstractClass(node *tree_sitter.Node, language lang.Language) bool { switch language { @@ -365,12 +598,27 @@ func extractAllDecorators(node *tree_sitter.Node, source []byte, language lang.L return extractTSDecorators(node, source) case lang.CSharp: return extractCSharpAttributes(node, source) + case lang.Kotlin: + return extractKotlinAnnotations(node, source) + case lang.PHP: + return extractPHPAttributes(node, source) + case lang.Rust: + return extractRustAttributes(node, source) + case lang.Swift: + return extractSwiftAttributes(node, source) + case lang.Groovy: + return extractGroovyAnnotations(node, source) + case lang.Dart: + return extractDartAnnotations(node, source) } return nil } func extractJavaAnnotations(node *tree_sitter.Node, source []byte) []string { mods := node.ChildByFieldName("modifiers") + if mods == nil { + mods = findChildByKind(node, "modifiers") + } if mods == nil { return nil } @@ -387,14 +635,71 @@ func extractJavaAnnotations(node *tree_sitter.Node, source []byte) []string { return decorators } +// extractGroovyAnnotations extracts annotations from Groovy nodes. +// Groovy annotations are direct children of the function/class definition node. +func extractGroovyAnnotations(node *tree_sitter.Node, source []byte) []string { + var decorators []string + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child != nil && child.Kind() == "annotation" { + decorators = append(decorators, parser.NodeText(child, source)) + } + } + return decorators +} + func extractTSDecorators(node *tree_sitter.Node, source []byte) []string { var decorators []string + // Check direct children for i := uint(0); i < node.ChildCount(); i++ { child := node.Child(i) if child != nil && child.Kind() == "decorator" { decorators = append(decorators, parser.NodeText(child, source)) } } + // TS/TSX: decorators on class methods are siblings in class_body, not children + // of method_definition. Check preceding siblings. + if len(decorators) == 0 { + decorators = extractTSPrecedingSiblingDecorators(node, source) + } + return decorators +} + +// extractTSPrecedingSiblingDecorators collects decorator nodes that precede the +// given node in a class_body parent. +func extractTSPrecedingSiblingDecorators(node *tree_sitter.Node, source []byte) []string { + parent := node.Parent() + if parent == nil || parent.Kind() != "class_body" { + return nil + } + nodeIdx := findNodeIndex(parent, node) + if nodeIdx < 0 { + return nil + } + return collectPrecedingDecorators(parent, nodeIdx, source) +} + +// findNodeIndex returns the child index of node within parent, or -1 if not found. +func findNodeIndex(parent, node *tree_sitter.Node) int { + for i := uint(0); i < parent.ChildCount(); i++ { + child := parent.Child(i) + if child != nil && child.Id() == node.Id() { + return int(i) + } + } + return -1 +} + +// collectPrecedingDecorators collects decorator nodes from siblings preceding nodeIdx. +func collectPrecedingDecorators(parent *tree_sitter.Node, nodeIdx int, source []byte) []string { + var decorators []string + for j := nodeIdx - 1; j >= 0; j-- { + prev := parent.Child(uint(j)) + if prev == nil || prev.Kind() != "decorator" { + break + } + decorators = append(decorators, parser.NodeText(prev, source)) + } return decorators } @@ -415,6 +720,159 @@ func extractCSharpAttributes(node *tree_sitter.Node, source []byte) []string { return decorators } +func extractKotlinAnnotations(node *tree_sitter.Node, source []byte) []string { + var decorators []string + // Kotlin: modifiers → annotation nodes + mods := node.ChildByFieldName("modifiers") + if mods == nil { + mods = findChildByKind(node, "modifiers") + } + if mods == nil { + return nil + } + for i := uint(0); i < mods.ChildCount(); i++ { + child := mods.Child(i) + if child == nil { + continue + } + if child.Kind() == "annotation" { + decorators = append(decorators, parser.NodeText(child, source)) + } + } + return decorators +} + +func extractPHPAttributes(node *tree_sitter.Node, source []byte) []string { + var decorators []string + // PHP 8 attributes: attribute_list → attribute_group → attribute + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child == nil { + continue + } + switch child.Kind() { + case "attribute_list": + decorators = append(decorators, extractPHPAttributeList(child, source)...) + case "attribute_group": + decorators = append(decorators, extractPHPAttributeGroup(child, source)...) + } + } + return decorators +} + +// extractPHPAttributeList extracts attributes from an attribute_list node. +func extractPHPAttributeList(list *tree_sitter.Node, source []byte) []string { + var decorators []string + for j := uint(0); j < list.NamedChildCount(); j++ { + group := list.NamedChild(j) + if group == nil { + continue + } + switch group.Kind() { + case "attribute_group": + decorators = append(decorators, extractPHPAttributeGroup(group, source)...) + case "attribute": + decorators = append(decorators, parser.NodeText(group, source)) + } + } + return decorators +} + +// extractPHPAttributeGroup extracts attribute nodes from an attribute_group. +func extractPHPAttributeGroup(group *tree_sitter.Node, source []byte) []string { + var decorators []string + for k := uint(0); k < group.NamedChildCount(); k++ { + attr := group.NamedChild(k) + if attr != nil && attr.Kind() == "attribute" { + decorators = append(decorators, parser.NodeText(attr, source)) + } + } + return decorators +} + +// extractRustAttributes extracts #[attr] attributes from preceding siblings. +// In Rust, attribute_item nodes are siblings of the annotated item, not children. +func extractRustAttributes(node *tree_sitter.Node, source []byte) []string { + parent := node.Parent() + if parent == nil { + return nil + } + + // Find our node's index among siblings, then collect preceding attribute_item nodes. + nodeIdx := findNodeIndex(parent, node) + if nodeIdx < 0 { + return nil + } + + var decorators []string + for j := nodeIdx - 1; j >= 0; j-- { + prev := parent.Child(uint(j)) + if prev == nil { + continue + } + if prev.Kind() != "attribute_item" { + break // stop at first non-attribute sibling + } + // Extract the attribute content (e.g., get("/users") from #[get("/users")]) + if attr := findChildByKind(prev, "attribute"); attr != nil { + decorators = append(decorators, parser.NodeText(attr, source)) + } else { + decorators = append(decorators, parser.NodeText(prev, source)) + } + } + + // Also check inner attributes (#![...]) which are children of the node itself + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child != nil && child.Kind() == "attribute_item" { + if attr := findChildByKind(child, "attribute"); attr != nil { + decorators = append(decorators, parser.NodeText(attr, source)) + } + } + } + + return decorators +} + +func extractSwiftAttributes(node *tree_sitter.Node, source []byte) []string { + var decorators []string + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child != nil && child.Kind() == "attribute" { + decorators = append(decorators, parser.NodeText(child, source)) + } + } + return decorators +} + +func extractDartAnnotations(node *tree_sitter.Node, source []byte) []string { + var decorators []string + // Dart annotations are siblings of the annotated node, not children + parent := node.Parent() + if parent == nil { + return nil + } + for i := uint(0); i < parent.ChildCount(); i++ { + child := parent.Child(i) + if child != nil && child.Id() == node.Id() { + // Collect annotations from preceding siblings + for j := int(i) - 1; j >= 0; j-- { + prev := parent.Child(uint(j)) + if prev == nil { + break + } + if prev.Kind() == "annotation" { + decorators = append(decorators, parser.NodeText(prev, source)) + } else { + break + } + } + break + } + } + return decorators +} + // Helper functions func extractTypeFromAnnotation(typeAnn *tree_sitter.Node, source []byte) string { @@ -430,6 +888,9 @@ func extractTypeFromAnnotation(typeAnn *tree_sitter.Node, source []byte) string // cleanTypeName strips pointers, references, generics to get the base type name. func cleanTypeName(s string) string { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, ": ") // TS/TSX type_annotation includes ": " + s = strings.TrimPrefix(s, ":") s = strings.TrimSpace(s) s = strings.TrimPrefix(s, "*") s = strings.TrimPrefix(s, "&") @@ -482,6 +943,63 @@ func buildSymbolSummary(nodes []*store.Node, moduleQN string) []string { return symbols } +// tokenizeDecorator strips decorator syntax and splits into lowercase words. +// Example: "@login_required" → ["login", "required"] +// Example: "@GetMapping(\"/api\")" → ["mapping"] (stopword "get" filtered) +func tokenizeDecorator(dec string) []string { + // Strip leading syntax: @, #[ + dec = strings.TrimPrefix(dec, "@") + dec = strings.TrimPrefix(dec, "#[") + dec = strings.TrimSuffix(dec, "]") + // Strip arguments: everything from first ( onwards + if idx := strings.Index(dec, "("); idx >= 0 { + dec = dec[:idx] + } + // Split on delimiters: dots, underscores, hyphens, colons, slashes + parts := strings.FieldsFunc(dec, func(r rune) bool { + return r == '.' || r == '_' || r == '-' || r == ':' || r == '/' + }) + // Split camelCase and collect lowercase words + var words []string + for _, part := range parts { + for _, w := range splitCamelCase(part) { + w = strings.ToLower(w) + if len(w) >= 2 && !decoratorStopwords[w] { + words = append(words, w) + } + } + } + return words +} + +// splitCamelCase splits a string on lowercase→uppercase transitions. +// Example: "GetMapping" → ["Get", "Mapping"] +func splitCamelCase(s string) []string { + if s == "" { + return nil + } + var words []string + start := 0 + for i := 1; i < len(s); i++ { + if s[i] >= 'A' && s[i] <= 'Z' && s[i-1] >= 'a' && s[i-1] <= 'z' { + words = append(words, s[start:i]) + start = i + } + } + words = append(words, s[start:]) + return words +} + +// decoratorStopwords are common words filtered from decorator tag candidates. +var decoratorStopwords = map[string]bool{ + "get": true, "set": true, "new": true, "class": true, + "method": true, "function": true, "value": true, "type": true, + "param": true, "return": true, "public": true, "private": true, + "for": true, "if": true, "the": true, "and": true, + "or": true, "not": true, "with": true, "from": true, + "app": true, "router": true, +} + func labelToSymbolPrefix(label string) string { switch label { case "Function": diff --git a/internal/pipeline/enrichment_test.go b/internal/pipeline/enrichment_test.go new file mode 100644 index 000000000..270af2468 --- /dev/null +++ b/internal/pipeline/enrichment_test.go @@ -0,0 +1,261 @@ +package pipeline + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +func TestTokenizeDecorator(t *testing.T) { + tests := []struct { + input string + want []string + }{ + {"@Override", []string{"override"}}, + {"@Deprecated", []string{"deprecated"}}, + {"@Test", []string{"test"}}, + {"@login_required", []string{"login", "required"}}, + {"@cache", []string{"cache"}}, + {"@pytest.fixture", []string{"pytest", "fixture"}}, + {`@GetMapping("/api")`, []string{"mapping"}}, // "get" is stopword + {`@PostMapping("/api")`, []string{"post", "mapping"}}, + {`@Transactional`, []string{"transactional"}}, + {`@MessageMapping("/chat")`, []string{"message", "mapping"}}, + {`#[test]`, []string{"test"}}, + {`#[derive(Debug)]`, []string{"derive"}}, + {"@app.get(\"/api\")", nil}, // both "app" and "get" are stopwords + {"@router.post(\"/api\")", []string{"post"}}, // "router" is stopword, "post" passes + {"@x", nil}, // too short after filtering + {"", nil}, // empty + {"@click.command", []string{"click", "command"}}, + {"@celery.task", []string{"celery", "task"}}, + } + for _, tt := range tests { + got := tokenizeDecorator(tt.input) + if !sliceEqual(got, tt.want) { + t.Errorf("tokenizeDecorator(%q) = %v, want %v", tt.input, got, tt.want) + } + } +} + +func TestSplitCamelCase(t *testing.T) { + tests := []struct { + input string + want []string + }{ + {"GetMapping", []string{"Get", "Mapping"}}, + {"getMessage", []string{"get", "Message"}}, + {"cache", []string{"cache"}}, + {"HTMLParser", []string{"HTMLParser"}}, // no lowercase→uppercase transition + {"", nil}, + } + for _, tt := range tests { + got := splitCamelCase(tt.input) + if !sliceEqual(got, tt.want) { + t.Errorf("splitCamelCase(%q) = %v, want %v", tt.input, got, tt.want) + } + } +} + +func TestDecoratorTagAutoDiscovery(t *testing.T) { + dir, err := os.MkdirTemp("", "cgm-dectag-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // Create Python files with repeated decorators + writeFile(t, filepath.Join(dir, "views.py"), ` +from functools import cache + +@login_required +def list_orders(): + pass + +@login_required +def get_order(): + pass + +@cache +def compute_total(): + pass + +@cache +def compute_tax(): + pass + +@unique_helper +def special(): + pass +`) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + // Check decorator_tags on nodes + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + + tagMap := map[string][]string{} // funcName → tags + for _, f := range funcs { + tags, ok := f.Properties["decorator_tags"] + if !ok { + continue + } + tagList, ok := tags.([]any) + if !ok { + continue + } + var ts []string + for _, tag := range tagList { + if s, ok := tag.(string); ok { + ts = append(ts, s) + } + } + tagMap[f.Name] = ts + } + + // "login" and "required" appear on 2 nodes → should be tags + assertHasTag(t, tagMap, "list_orders", "login") + assertHasTag(t, tagMap, "list_orders", "required") + assertHasTag(t, tagMap, "get_order", "login") + assertHasTag(t, tagMap, "get_order", "required") + + // "cache" appears on 2 nodes → should be a tag + assertHasTag(t, tagMap, "compute_total", "cache") + assertHasTag(t, tagMap, "compute_tax", "cache") + + // "unique" and "helper" appear on only 1 node → should NOT be tags + if tags, ok := tagMap["special"]; ok { + for _, tag := range tags { + if tag == "unique" || tag == "helper" { + t.Errorf("special should not have tag %q (freq < 2)", tag) + } + } + } +} + +func TestDecoratorTagJavaClassMethods(t *testing.T) { + dir, err := os.MkdirTemp("", "cgm-javadec-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + writeFile(t, filepath.Join(dir, "Controller.java"), ` +class OwnerController { + @GetMapping("/owners") + public void listOwners() {} + + @GetMapping("/owners/{id}") + public void showOwner() {} + + @PostMapping("/owners") + public void createOwner() {} + + @Transactional + @PostMapping("/owners/{id}") + public void updateOwner() {} +} +`) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + // Verify decorators are extracted on class methods + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + + decoMap := map[string][]string{} + tagMap := map[string][]string{} + for _, m := range methods { + if ds := extractStringSliceProp(m.Properties, "decorators"); len(ds) > 0 { + decoMap[m.Name] = ds + } + if ts := extractStringSliceProp(m.Properties, "decorator_tags"); len(ts) > 0 { + tagMap[m.Name] = ts + } + } + + // decorators should be extracted + if len(decoMap["listOwners"]) == 0 { + t.Error("listOwners should have decorators") + } + if len(decoMap["showOwner"]) == 0 { + t.Error("showOwner should have decorators") + } + + // "mapping" appears on all 4 methods → should be a tag + assertHasTag(t, tagMap, "listOwners", "mapping") + assertHasTag(t, tagMap, "showOwner", "mapping") + + // "post" appears on 2 methods → should be a tag + assertHasTag(t, tagMap, "createOwner", "post") + assertHasTag(t, tagMap, "updateOwner", "post") +} + +func assertHasTag(t *testing.T, tagMap map[string][]string, funcName, tag string) { + t.Helper() + tags, ok := tagMap[funcName] + if !ok { + t.Errorf("%s has no decorator_tags", funcName) + return + } + for _, tt := range tags { + if tt == tag { + return + } + } + t.Errorf("%s missing tag %q, got %v", funcName, tag, tags) +} + +// extractStringSliceProp extracts a []any property as []string. +func extractStringSliceProp(props map[string]any, key string) []string { + val, ok := props[key] + if !ok { + return nil + } + list, ok := val.([]any) + if !ok { + return nil + } + out := make([]string, 0, len(list)) + for _, item := range list { + if s, ok := item.(string); ok { + out = append(out, s) + } + } + return out +} + +func sliceEqual(a, b []string) bool { + if len(a) == 0 && len(b) == 0 { + return true + } + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/pipeline/githistory.go b/internal/pipeline/githistory.go index f7b18fe30..a671995bb 100644 --- a/internal/pipeline/githistory.go +++ b/internal/pipeline/githistory.go @@ -2,11 +2,13 @@ package pipeline import ( "bufio" + "context" "fmt" "log/slog" "os/exec" "sort" "strings" + "time" "github.com/DeusData/codebase-memory-mcp/internal/store" ) @@ -50,7 +52,9 @@ func (p *Pipeline) passGitHistory() { // parseGitLog runs git log and extracts commit → files mapping. func parseGitLog(repoPath string) ([]CommitFiles, error) { - cmd := exec.Command("git", "log", "--name-only", "--pretty=format:COMMIT:%H", "--since=6 months ago") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "git", "log", "--name-only", "--pretty=format:COMMIT:%H", "--since=6 months ago") cmd.Dir = repoPath output, err := cmd.Output() diff --git a/internal/pipeline/infrascan_test.go b/internal/pipeline/infrascan_test.go index 8cc3af56b..d38bd1219 100644 --- a/internal/pipeline/infrascan_test.go +++ b/internal/pipeline/infrascan_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "context" "os" "path/filepath" "strings" @@ -728,8 +729,8 @@ services: - "80:80" `) - p := New(s, tmpDir) - err = s.WithTransaction(func(txStore *store.Store) error { + p := New(context.Background(), s, tmpDir) + err = s.WithTransaction(context.Background(), func(txStore *store.Store) error { p.Store = txStore _ = txStore.UpsertProject(p.ProjectName, tmpDir) p.passInfraFiles() @@ -802,8 +803,8 @@ func TestPassInfraFilesIdempotent(t *testing.T) { EXPOSE 8080 `) - p := New(s, tmpDir) - err = s.WithTransaction(func(txStore *store.Store) error { + p := New(context.Background(), s, tmpDir) + err = s.WithTransaction(context.Background(), func(txStore *store.Store) error { p.Store = txStore _ = txStore.UpsertProject(p.ProjectName, tmpDir) p.passInfraFiles() @@ -816,7 +817,7 @@ EXPOSE 8080 countBefore, _ := s.CountNodes(p.ProjectName) // Run again — should delete and recreate (same count) - err = s.WithTransaction(func(txStore *store.Store) error { + err = s.WithTransaction(context.Background(), func(txStore *store.Store) error { p.Store = txStore _ = txStore.DeleteNodesByLabel(p.ProjectName, "InfraFile") p.passInfraFiles() diff --git a/internal/pipeline/langparity_test.go b/internal/pipeline/langparity_test.go new file mode 100644 index 000000000..e17003891 --- /dev/null +++ b/internal/pipeline/langparity_test.go @@ -0,0 +1,2040 @@ +package pipeline + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + tree_sitter "github.com/tree-sitter/go-tree-sitter" + + "github.com/DeusData/codebase-memory-mcp/internal/discover" + "github.com/DeusData/codebase-memory-mcp/internal/lang" + "github.com/DeusData/codebase-memory-mcp/internal/parser" + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +// findFirstNodeByKind walks the AST and returns the first node matching any of the given kinds. +func findFirstNodeByKind(root *tree_sitter.Node, kinds ...string) *tree_sitter.Node { + kindSet := make(map[string]bool, len(kinds)) + for _, k := range kinds { + kindSet[k] = true + } + var found *tree_sitter.Node + parser.Walk(root, func(n *tree_sitter.Node) bool { + if found != nil { + return false + } + if kindSet[n.Kind()] { + found = n + return false + } + return true + }) + return found +} + +// findParamsNode finds the parameter list node for a function, handling different +// tree-sitter grammar structures across languages. +func findParamsNode(funcNode *tree_sitter.Node, language lang.Language) *tree_sitter.Node { + // Try standard field names first + for _, f := range []string{"parameters", "formal_parameters", "value_parameters"} { + if n := funcNode.ChildByFieldName(f); n != nil { + return n + } + } + // C/C++/ObjC: params are nested in function_declarator + if language == lang.CPP || language == lang.C || language == lang.ObjectiveC { + if decl := funcNode.ChildByFieldName("declarator"); decl != nil { + if params := decl.ChildByFieldName("parameters"); params != nil { + return params + } + } + } + // Zig: parameters child (no field name) + if language == lang.Zig { + return findFirstNodeByKind(funcNode, "parameters") + } + // Kotlin: function_value_parameters has no field name + if language == lang.Kotlin { + return findFirstNodeByKind(funcNode, "function_value_parameters") + } + return nil +} + +// findReturnTypeNode finds the return type node for a function, handling different +// tree-sitter grammar structures across languages. +func findReturnTypeNode(funcNode *tree_sitter.Node, language lang.Language) *tree_sitter.Node { + // Try standard field names + for _, f := range []string{"return_type", "result", "type", "returns"} { + if n := funcNode.ChildByFieldName(f); n != nil { + return n + } + } + // C++: return type is in the "type" field of function_definition + if language == lang.CPP { + if n := funcNode.ChildByFieldName("type"); n != nil { + return n + } + } + // Kotlin: return type (user_type) is an unnamed child after ":" + if language == lang.Kotlin { + return findFirstNodeByKind(funcNode, "user_type") + } + return nil +} + +// writeLangTestFile creates a file with the given content inside dir. +func writeLangTestFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +// --- Test 1: Complexity --- + +func TestComplexityAllLanguages(t *testing.T) { + tests := []struct { + name string + language lang.Language + source string + }{ + {"Python", lang.Python, "def f():\n if x:\n pass\n for i in range(10):\n pass\n"}, + {"Go", lang.Go, "package main\nfunc f() {\n\tif x {\n\t}\n\tfor i := 0; i < 10; i++ {\n\t}\n}\n"}, + {"JavaScript", lang.JavaScript, "function f() {\n\tif (x) {}\n\tfor (var i = 0; i < 10; i++) {}\n}\n"}, + {"TypeScript", lang.TypeScript, "function f() {\n\tif (x) {}\n\tfor (let i = 0; i < 10; i++) {}\n}\n"}, + {"TSX", lang.TSX, "function f() {\n\tif (x) {}\n\tfor (let i = 0; i < 10; i++) {}\n}\n"}, + {"Java", lang.Java, "class A {\n\tvoid f() {\n\t\tif (x) {}\n\t\tfor (int i = 0; i < 10; i++) {}\n\t}\n}\n"}, + {"Rust", lang.Rust, "fn f() {\n\tif x {\n\t}\n\tfor i in v {\n\t}\n}\n"}, + {"CPP", lang.CPP, "void f() {\n\tif (x) {}\n\tfor (int i = 0; i < 10; i++) {}\n}\n"}, + {"CSharp", lang.CSharp, "class A {\n\tvoid F() {\n\t\tif (x) {}\n\t\tfor (int i = 0; i < 10; i++) {}\n\t}\n}\n"}, + {"PHP", lang.PHP, " 0\n x\n elsif x == 0\n 0\n end\nend\n"}, + {"C", lang.C, "int f(int x) {\n\tif (x > 0) return x;\n\tfor (int i = 0; i < 10; i++) {}\n}\n"}, + {"Bash", lang.Bash, "f() {\n\tif [ -f file ]; then\n\t\techo ok\n\telif [ -d dir ]; then\n\t\techo dir\n\tfi\n}\n"}, + {"Zig", lang.Zig, "fn f(x: i32) i32 {\n\tif (x > 0) return x;\n\tfor (0..10) |i| { _ = i; }\n}\n"}, + {"ObjectiveC", lang.ObjectiveC, "void f(int x) {\n\tif (x > 0) {}\n\tfor (int i = 0; i < 10; i++) {}\n}\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := lang.ForLanguage(tt.language) + if spec == nil { + t.Fatalf("no spec for %s", tt.language) + } + if len(spec.BranchingNodeTypes) == 0 { + t.Fatalf("BranchingNodeTypes is empty for %s", tt.language) + } + + tree, _ := parseSource(t, tt.language, tt.source) + defer tree.Close() + + funcNode := findFirstNodeByKind(tree.RootNode(), spec.FunctionNodeTypes...) + if funcNode == nil { + t.Fatalf("no function node found in AST for %s", tt.language) + } + + complexity := countBranchingNodes(funcNode, spec.BranchingNodeTypes) + if complexity < 2 { + t.Errorf("complexity = %d, want >= 2 for %s", complexity, tt.language) + } + }) + } +} + +// --- Test 2: Param Type Extraction --- + +func TestParamTypeExtractionAllLanguages(t *testing.T) { + tests := []struct { + name string + language lang.Language + source string + wantType string + }{ + {"Python", lang.Python, "def f(cfg: Config):\n pass\n", "Config"}, + {"Go", lang.Go, "package main\nfunc f(cfg Config) {}\n", "Config"}, + {"TypeScript", lang.TypeScript, "function f(cfg: Config) {}\n", "Config"}, + {"TSX", lang.TSX, "function f(cfg: Config) {}\n", "Config"}, + {"Java", lang.Java, "class A {\n\tvoid f(Config cfg) {}\n}\n", "Config"}, + {"Rust", lang.Rust, "fn f(cfg: Config) {}\n", "Config"}, + {"CPP", lang.CPP, "void f(Config cfg) {}\n", "Config"}, + {"CSharp", lang.CSharp, "class A {\n\tvoid F(Config cfg) {}\n}\n", "Config"}, + {"PHP", lang.PHP, " Config:\n pass\n", "Config"}, + {"Go", lang.Go, "package main\nfunc f() Config { return Config{} }\n", "Config"}, + {"TypeScript", lang.TypeScript, "function f(): Config { return {} as Config; }\n", "Config"}, + {"TSX", lang.TSX, "function f(): Config { return {} as Config; }\n", "Config"}, + {"Java", lang.Java, "class A {\n\tConfig f() { return null; }\n}\n", "Config"}, + {"Rust", lang.Rust, "fn f() -> Config { todo!() }\n", "Config"}, + {"CPP", lang.CPP, "Config f() { return Config(); }\n", "Config"}, + {"CSharp", lang.CSharp, "class A {\n\tConfig F() { return null; }\n}\n", "Config"}, + {"PHP", lang.PHP, " {\n\t\tres.json([]);\n\t};\n\tpublic getUser: RequestHandler = async (req, res) => {\n\t\tres.json({});\n\t};\n}\n", + []string{"getUsers", "getUser"}, + map[string]string{"getUsers": "(req, res)", "getUser": "(req, res)"}, + }, + { + "TypeScript_no_type", + lang.TypeScript, + ".ts", + "class A {\n\tgreet = () => 'hello';\n}\n", + []string{"greet"}, + map[string]string{"greet": "()"}, + }, + { + "JavaScript", + lang.JavaScript, + ".js", + "class A {\n\tgreet = () => 'hello';\n\thandle = async (req) => {\n\t\treturn req;\n\t};\n}\n", + []string{"greet", "handle"}, + map[string]string{"greet": "()", "handle": "(req)"}, + }, + { + "TSX", + lang.TSX, + ".tsx", + "class App {\n\trender = () =>
;\n}\n", + []string{"render"}, + map[string]string{"render": "()"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := lang.ForLanguage(tt.language) + if spec == nil { + t.Fatalf("no spec for %s", tt.language) + } + + tree, src := parseSource(t, tt.language, tt.source) + defer tree.Close() + + classNode := findFirstNodeByKind(tree.RootNode(), spec.ClassNodeTypes...) + if classNode == nil { + t.Fatalf("no class node found") + } + + f := discover.FileInfo{ + Path: "/test/test" + tt.ext, + RelPath: "test" + tt.ext, + Language: tt.language, + } + + result := &parseResult{} + extractClassDef(classNode, src, f, "test-project", "test-project::test"+tt.ext, spec, result) + + methodNodes := collectNodesByLabel(result.Nodes, "Method") + assertNodeNamesExist(t, methodNodes, tt.wantMethods) + assertNodeSignatures(t, methodNodes, tt.wantSignature) + assertEdgeCount(t, result.PendingEdges, "DEFINES_METHOD", len(tt.wantMethods)) + }) + } +} + +// --- Test 11: Const Arrow Functions as Function (not Variable) --- + +func TestConstArrowFunctionsAsFunction(t *testing.T) { + tests := []struct { + name string + language lang.Language + ext string + source string + wantFuncs []string + }{ + { + "JavaScript", + lang.JavaScript, + ".js", + "const greet = () => 'hello';\nconst handler = async (req) => { return req; };\nconst name = 'Alice';\n", + []string{"greet", "handler"}, + }, + { + "TypeScript", + lang.TypeScript, + ".ts", + "const greet = (): string => 'hello';\nexport const handler = async (req: Request): Promise => { return new Response(); };\nconst config = { port: 8080 };\n", + []string{"greet", "handler"}, + }, + { + "TSX", + lang.TSX, + ".tsx", + "const App = () =>
;\nconst title = 'Hello';\n", + []string{"App"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, p := runPipelineWithFile(t, "test"+tt.ext, tt.source) + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + assertNodeNamesExist(t, funcs, tt.wantFuncs) + + // Verify they are NOT Variables + vars, _ := s.FindNodesByLabel(p.ProjectName, "Variable") + assertNodesNotLabeled(t, vars, tt.wantFuncs, "Variable") + + for _, v := range vars { + t.Logf("Variable: %s", v.Name) + } + }) + } +} + +// --- Test 12: Go Interface Methods and IMPLEMENTS Edges --- + +func TestGoInterfaceImplements(t *testing.T) { + src := `package main + +type Handler interface { + ServeHTTP(w ResponseWriter, r *Request) +} + +type Mux struct{} + +func (m *Mux) ServeHTTP(w ResponseWriter, r *Request) {} + +type Router interface { + Get(pattern string) + Post(pattern string) +} + +type DefaultRouter struct{} + +func (d *DefaultRouter) Get(pattern string) {} +func (d *DefaultRouter) Post(pattern string) {} +` + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte(src), 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + // Check Interface nodes exist + ifaces, _ := s.FindNodesByLabel(p.ProjectName, "Interface") + ifaceNames := map[string]bool{} + for _, i := range ifaces { + ifaceNames[i.Name] = true + t.Logf("Interface: %s", i.Name) + } + if !ifaceNames["Handler"] { + t.Error("Handler interface not found") + } + if !ifaceNames["Router"] { + t.Error("Router interface not found") + } + + // Check that interface methods are extracted as Method nodes + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + methodNames := map[string]bool{} + for _, m := range methods { + methodNames[m.Name] = true + t.Logf("Method: %s (qn=%s)", m.Name, m.QualifiedName) + } + if !methodNames["ServeHTTP"] { + t.Error("ServeHTTP method not found") + } + if !methodNames["Get"] { + t.Error("Get method not found") + } + + // Check IMPLEMENTS edges + for _, iface := range ifaces { + edges, _ := s.FindEdgesByTargetAndType(iface.ID, "IMPLEMENTS") + t.Logf("Interface %s: %d IMPLEMENTS edges", iface.Name, len(edges)) + if len(edges) == 0 { + t.Errorf("expected IMPLEMENTS edges for %s, got 0", iface.Name) + } + } +} + +// --- Test 13: Rust impl Trait for Struct → IMPLEMENTS edges + method extraction --- + +func TestRustImplTraitImplements(t *testing.T) { + src := `trait Handler { + fn handle(&self) -> String; +} + +struct MyHandler; + +impl Handler for MyHandler { + fn handle(&self) -> String { + String::from("handled") + } +} + +struct AnotherHandler; + +impl Handler for AnotherHandler { + fn handle(&self) -> String { + String::from("another") + } +} +` + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "main.rs"), []byte(src), 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + // Check that trait is found as Interface + ifaces, _ := s.FindNodesByLabel(p.ProjectName, "Interface") + if len(ifaces) == 0 { + t.Fatal("no Interface nodes found") + } + t.Logf("Interfaces: %d", len(ifaces)) + for _, i := range ifaces { + t.Logf(" Interface: %s", i.Name) + } + + // Check that structs are found as Class + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + classNames := map[string]bool{} + for _, c := range classes { + classNames[c.Name] = true + t.Logf(" Class: %s", c.Name) + } + if !classNames["MyHandler"] { + t.Error("MyHandler class not found") + } + + // Check methods are extracted and linked to the struct + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + t.Logf("Methods: %d", len(methods)) + for _, m := range methods { + t.Logf(" Method: %s (qn=%s)", m.Name, m.QualifiedName) + } + if len(methods) < 3 { // trait method + 2 impl methods + t.Errorf("expected at least 3 methods, got %d", len(methods)) + } + + // Check IMPLEMENTS edges + for _, iface := range ifaces { + edges, _ := s.FindEdgesByTargetAndType(iface.ID, "IMPLEMENTS") + t.Logf("Interface %s: %d IMPLEMENTS edges", iface.Name, len(edges)) + if len(edges) < 2 { + t.Errorf("expected at least 2 IMPLEMENTS edges for %s, got %d", iface.Name, len(edges)) + } + } +} + +// --- Test 14: Elixir Custom Extraction --- + +func TestElixirCustomExtraction(t *testing.T) { + src := `defmodule Greeter do + def greet(name) do + "Hello #{name}" + end + + defp internal_work(x) do + x * 2 + end +end +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "greeter.ex"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + // Check Module/Class node for defmodule + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + classNames := map[string]bool{} + for _, c := range classes { + classNames[c.Name] = true + t.Logf("Class: %s (qn=%s)", c.Name, c.QualifiedName) + } + if !classNames["Greeter"] { + t.Error("Greeter module not found as Class") + } + + // Check Function nodes for def/defp + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + allFuncNames := map[string]bool{} + for _, f := range funcs { + allFuncNames[f.Name] = true + t.Logf("Function: %s (qn=%s)", f.Name, f.QualifiedName) + } + for _, m := range methods { + allFuncNames[m.Name] = true + t.Logf("Method: %s (qn=%s)", m.Name, m.QualifiedName) + } + if !allFuncNames["greet"] { + t.Error("greet function not found") + } + if !allFuncNames["internal_work"] { + t.Error("internal_work function not found") + } + + // Check DEFINES edges (Elixir functions are extracted with DEFINES edges, not DEFINES_METHOD) + defEdges, _ := s.FindEdgesByType(p.ProjectName, "DEFINES") + if len(defEdges) < 3 { + // At least: module→greet, module→internal_work, file→module + t.Errorf("expected at least 3 DEFINES edges, got %d", len(defEdges)) + } +} + +// --- Test 15: R Function Name Resolution --- + +func TestRFunctionNameResolution(t *testing.T) { + src := `mutate <- function(x) { + x + 1 +} + +square <- function(n) { + n * n +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "funcs.R"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + funcNames := map[string]bool{} + for _, f := range funcs { + funcNames[f.Name] = true + t.Logf("Function: %s (qn=%s)", f.Name, f.QualifiedName) + } + + if !funcNames["mutate"] { + t.Error("mutate function not found (name should be from parent assignment)") + } + if !funcNames["square"] { + t.Error("square function not found") + } + + // Ensure "function" keyword is NOT used as a name + if funcNames["function"] { + t.Error("function keyword incorrectly used as function name") + } +} + +// --- Test 16: Haskell Function Extraction --- + +func TestHaskellFunctionExtraction(t *testing.T) { + src := `greet :: String -> String +greet name = "Hello " ++ name + +add :: Int -> Int -> Int +add x y = x + y +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "Main.hs"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + funcNames := map[string]bool{} + for _, f := range funcs { + funcNames[f.Name] = true + t.Logf("Function: %s (qn=%s)", f.Name, f.QualifiedName) + } + + if !funcNames["greet"] { + t.Error("greet function not found") + } + if !funcNames["add"] { + t.Error("add function not found") + } +} + +// --- Test 17: OCaml Function Extraction --- + +func TestOCamlFunctionExtraction(t *testing.T) { + src := `let greet name = Printf.printf "Hello %s" name + +let add x y = x + y +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "main.ml"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + funcNames := map[string]bool{} + for _, f := range funcs { + funcNames[f.Name] = true + t.Logf("Function: %s (qn=%s)", f.Name, f.QualifiedName) + } + + if !funcNames["greet"] { + t.Error("greet function not found") + } + if !funcNames["add"] { + t.Error("add function not found") + } +} + +// --- Test 18: Groovy Function and Class Extraction --- + +func TestGroovyClassMethodExtraction(t *testing.T) { + src := `class Calculator { + int add(int a, int b) { + return a + b + } + + int multiply(int a, int b) { + return a * b + } +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "Calculator.groovy"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + classNames := map[string]bool{} + for _, c := range classes { + classNames[c.Name] = true + t.Logf("Class: %s", c.Name) + } + if !classNames["Calculator"] { + t.Error("Calculator class not found") + } + + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + methodNames := map[string]bool{} + for _, m := range methods { + methodNames[m.Name] = true + t.Logf("Method: %s", m.Name) + } + if !methodNames["add"] { + t.Error("add method not found") + } + if !methodNames["multiply"] { + t.Error("multiply method not found") + } + + defMethodEdges, _ := s.FindEdgesByType(p.ProjectName, "DEFINES_METHOD") + if len(defMethodEdges) < 2 { + t.Errorf("expected at least 2 DEFINES_METHOD edges, got %d", len(defMethodEdges)) + } +} + +// --- Test 19: Dart Function and Class Extraction --- + +func TestDartClassMethodExtraction(t *testing.T) { + src := `class Counter { + int _count = 0; + + void increment() { + _count++; + } + + int getCount() { + return _count; + } +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "counter.dart"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + classNames := map[string]bool{} + for _, c := range classes { + classNames[c.Name] = true + t.Logf("Class: %s", c.Name) + } + if !classNames["Counter"] { + t.Error("Counter class not found") + } + + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + methodNames := map[string]bool{} + for _, m := range methods { + methodNames[m.Name] = true + t.Logf("Method: %s (qn=%s)", m.Name, m.QualifiedName) + } + if !methodNames["increment"] { + t.Error("increment method not found") + } + if !methodNames["getCount"] { + t.Error("getCount method not found") + } +} + +// --- Test 20: HCL Block Extraction --- + +func TestHCLBlockExtraction(t *testing.T) { + src := `resource "aws_instance" "web" { + ami = "ami-12345" + instance_type = "t2.micro" +} + +variable "region" { + default = "us-east-1" +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "main.tf"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + for _, c := range classes { + t.Logf("Class: %s (qn=%s)", c.Name, c.QualifiedName) + } + if len(classes) == 0 { + t.Error("no Class nodes found for HCL resource blocks") + } + + // HCL variable blocks are extracted as Variable nodes + vars, _ := s.FindNodesByLabel(p.ProjectName, "Variable") + for _, v := range vars { + t.Logf("Variable: %s (qn=%s)", v.Name, v.QualifiedName) + } + + // Total should be at least 2 (resource block as Class + variable block as Variable) + total := len(classes) + len(vars) + if total < 2 { + t.Errorf("expected at least 2 HCL block nodes (Class + Variable), got %d", total) + } +} + +// --- Test 21: Zig Test Declaration Extraction --- + +func TestZigTestDeclaration(t *testing.T) { + src := `const std = @import("std"); +const expect = std.testing.expect; + +fn add(a: i32, b: i32) i32 { + return a + b; +} + +test "addition works" { + try expect(add(1, 2) == 3); +} + +test "zero addition" { + try expect(add(0, 0) == 0); +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "math.zig"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + funcNames := map[string]bool{} + for _, f := range funcs { + funcNames[f.Name] = true + t.Logf("Function: %s (is_test=%v)", f.Name, f.Properties["is_test"]) + } + + if !funcNames["add"] { + t.Error("add function not found") + } + + // Test declarations should be extracted as Functions + if !funcNames["addition works"] && !funcNames["test \"addition works\""] { + // Test name might be stored with or without quotes + found := false + for name := range funcNames { + if strings.Contains(name, "addition") { + found = true + break + } + } + if !found { + t.Error("'addition works' test not found") + } + } +} + +// --- Test 22: Swift INHERITS Edge Extraction --- + +func TestSwiftInheritsExtraction(t *testing.T) { + src := `class Animal { + var name: String = "" +} + +class Dog: Animal { + func bark() -> String { + return "Woof" + } +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "animals.swift"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + classNames := map[string]bool{} + for _, c := range classes { + classNames[c.Name] = true + t.Logf("Class: %s", c.Name) + } + if !classNames["Animal"] { + t.Error("Animal class not found") + } + if !classNames["Dog"] { + t.Error("Dog class not found") + } + + // Check INHERITS edges + inheritEdges, _ := s.FindEdgesByType(p.ProjectName, "INHERITS") + if len(inheritEdges) == 0 { + t.Error("expected INHERITS edges, got 0") + } else { + t.Logf("INHERITS edges: %d", len(inheritEdges)) + } + + // Check methods + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + methodNames := map[string]bool{} + for _, m := range methods { + methodNames[m.Name] = true + t.Logf("Method: %s", m.Name) + } + if !methodNames["bark"] { + t.Error("bark method not found") + } +} + +// --- Test 23: PHP Method Declaration Extraction --- + +func TestPHPMethodDeclaration(t *testing.T) { + src := `name; + } + + public function setName(string $name): void { + $this->name = $name; + } +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "User.php"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + classNames := map[string]bool{} + for _, c := range classes { + classNames[c.Name] = true + } + if !classNames["User"] { + t.Error("User class not found") + } + + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + methodNames := map[string]bool{} + for _, m := range methods { + methodNames[m.Name] = true + t.Logf("Method: %s (qn=%s)", m.Name, m.QualifiedName) + } + if !methodNames["getName"] { + t.Error("getName method not found") + } + if !methodNames["setName"] { + t.Error("setName method not found") + } + + defMethodEdges, _ := s.FindEdgesByType(p.ProjectName, "DEFINES_METHOD") + if len(defMethodEdges) < 2 { + t.Errorf("expected at least 2 DEFINES_METHOD edges, got %d", len(defMethodEdges)) + } +} + +// --- Test 24: Perl CALLS Edge Extraction --- + +func TestPerlCallsExtraction(t *testing.T) { + src := `sub greet { + my ($name) = @_; + print("Hello $name\n"); +} + +sub main { + greet("World"); +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "main.pl"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + funcNames := map[string]bool{} + for _, f := range funcs { + funcNames[f.Name] = true + t.Logf("Function: %s", f.Name) + } + if !funcNames["greet"] { + t.Error("greet function not found") + } + if !funcNames["main"] { + t.Error("main function not found") + } + + // Check CALLS edges + callEdges, _ := s.FindEdgesByType(p.ProjectName, "CALLS") + usageEdges, _ := s.FindEdgesByType(p.ProjectName, "USAGE") + t.Logf("CALLS edges: %d, USAGE edges: %d", len(callEdges), len(usageEdges)) + total := len(callEdges) + len(usageEdges) + if total == 0 { + t.Error("expected CALLS or USAGE edges, got 0") + } +} + +// --- Test 25: Erlang Function and CALLS Extraction --- + +func TestErlangCallsExtraction(t *testing.T) { + src := `-module(mymod). +-export([greet/1]). + +greet(Name) -> + io:format("Hello ~s~n", [Name]). + +main() -> + greet("World"). +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "mymod.erl"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + funcNames := map[string]bool{} + for _, f := range funcs { + funcNames[f.Name] = true + t.Logf("Function: %s", f.Name) + } + if !funcNames["greet"] { + t.Error("greet function not found") + } + + // Check CALLS or USAGE edges + callEdges, _ := s.FindEdgesByType(p.ProjectName, "CALLS") + usageEdges, _ := s.FindEdgesByType(p.ProjectName, "USAGE") + t.Logf("CALLS edges: %d, USAGE edges: %d", len(callEdges), len(usageEdges)) + // Require at least 1 CALLS edge (main→greet local call) + if len(callEdges) == 0 { + t.Error("expected at least 1 CALLS edge, got 0") + } +} + +// TestErlangVariableExtraction verifies -define() and -record() produce Variable nodes. +func TestErlangVariableExtraction(t *testing.T) { + src := `-module(mymod). +-define(TIMEOUT, 5000). +-record(person, {name, age}). +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "mymod.erl"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + vars, _ := s.FindNodesByLabel(p.ProjectName, "Variable") + varNames := map[string]bool{} + for _, v := range vars { + varNames[v.Name] = true + t.Logf("Variable: %s (qn=%s)", v.Name, v.QualifiedName) + } + + if !varNames["TIMEOUT"] { + t.Error("expected Variable node for TIMEOUT (-define)") + } + if !varNames["person"] { + t.Error("expected Variable node for person (-record)") + } +} + +// TestErlangMultiModuleCalls verifies cross-module function calls between Erlang files. +func TestErlangMultiModuleCalls(t *testing.T) { + dir := t.TempDir() + + writeLangTestFile(t, filepath.Join(dir, "handler.erl"), `-module(handler). +-export([handle/1]). +handle(Req) -> + Req. +`) + + writeLangTestFile(t, filepath.Join(dir, "server.erl"), `-module(server). +-export([start/0]). +start() -> + handle(request). +`) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + // Both handle and start should exist as functions + handleFuncs, _ := s.FindNodesByName(p.ProjectName, "handle") + if len(handleFuncs) == 0 { + t.Error("handle function not found") + } + + startFuncs, _ := s.FindNodesByName(p.ProjectName, "start") + if len(startFuncs) == 0 { + t.Error("start function not found") + } + + // Check for CALLS edge from start→handle + callEdges, _ := s.FindEdgesByType(p.ProjectName, "CALLS") + t.Logf("CALLS edges: %d", len(callEdges)) + if len(callEdges) == 0 { + t.Error("expected CALLS edges between Erlang modules, got 0") + } +} + +// --- Test: SQL Table/View/Function Extraction --- + +func TestSQLTableExtraction(t *testing.T) { + src := `CREATE TABLE users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL +); +CREATE VIEW active_users AS SELECT * FROM users WHERE active = true; +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "schema.sql"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + vars, _ := s.FindNodesByLabel(p.ProjectName, "Variable") + varNames := map[string]bool{} + for _, v := range vars { + varNames[v.Name] = true + t.Logf("Variable: %s (qn=%s)", v.Name, v.QualifiedName) + } + + if !varNames["users"] { + t.Error("expected Variable node for 'users' table") + } + if !varNames["active_users"] { + t.Error("expected Variable node for 'active_users' view") + } +} + +func TestSQLFunctionExtraction(t *testing.T) { + src := `CREATE FUNCTION add(a INT, b INT) RETURNS INT AS $$ SELECT a + b; $$ LANGUAGE SQL; +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "funcs.sql"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + funcNames := map[string]bool{} + for _, f := range funcs { + funcNames[f.Name] = true + t.Logf("Function: %s (qn=%s)", f.Name, f.QualifiedName) + } + + if !funcNames["add"] { + t.Error("expected Function node for 'add'") + } +} + +// --- Test: YAML Variable Extraction --- + +func TestYAMLVariableExtraction(t *testing.T) { + src := `name: myapp +version: "1.0" +services: + web: + image: nginx + db: + image: postgres +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "config.yaml"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + vars, _ := s.FindNodesByLabel(p.ProjectName, "Variable") + varNames := map[string]bool{} + for _, v := range vars { + varNames[v.Name] = true + t.Logf("Variable: %s (qn=%s)", v.Name, v.QualifiedName) + } + + // Top-level keys should be extracted + for _, expected := range []string{"name", "version", "services"} { + if !varNames[expected] { + t.Errorf("expected Variable node for top-level key %q", expected) + } + } + + // Nested keys should NOT be extracted (web, db, image) + for _, nested := range []string{"web", "db", "image"} { + if varNames[nested] { + t.Errorf("nested key %q should not be a top-level Variable", nested) + } + } +} + +// --- Test 26: Method Complexity on Class Methods --- + +func TestMethodComplexityOnClassMethods(t *testing.T) { + tests := []struct { + name string + language lang.Language + ext string + source string + }{ + { + "Java", lang.Java, ".java", + "class Service {\n\tvoid process(int x) {\n\t\tif (x > 0) {\n\t\t\tfor (int i = 0; i < x; i++) {}\n\t\t}\n\t}\n}\n", + }, + { + "TypeScript", lang.TypeScript, ".ts", + "class Service {\n\tprocess(x: number) {\n\t\tif (x > 0) {\n\t\t\tfor (let i = 0; i < x; i++) {}\n\t\t}\n\t}\n}\n", + }, + { + "Python", lang.Python, ".py", + "class Service:\n def process(self, x):\n if x > 0:\n for i in range(x):\n pass\n", + }, + { + "CSharp", lang.CSharp, ".cs", + "class Service {\n\tvoid Process(int x) {\n\t\tif (x > 0) {\n\t\t\tfor (int i = 0; i < x; i++) {}\n\t\t}\n\t}\n}\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "service"+tt.ext), tt.source) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + if len(methods) == 0 { + t.Fatal("no methods found") + } + + for _, m := range methods { + complexity, ok := m.Properties["complexity"] + t.Logf("Method: %s, complexity=%v (ok=%v)", m.Name, complexity, ok) + if !ok || complexity == nil { + t.Errorf("method %q has no complexity property", m.Name) + } else { + cVal, isNum := complexity.(float64) + if isNum && cVal < 2 { + t.Errorf("method %q complexity=%v, want >= 2", m.Name, complexity) + } + } + } + }) + } +} + +// --- Test 27: Method Param Types on Class Methods --- + +func TestMethodParamTypesOnClassMethods(t *testing.T) { + tests := []struct { + name string + language lang.Language + ext string + source string + wantType string + }{ + { + "Java", lang.Java, ".java", + "class Service {\n\tvoid process(Config cfg) {}\n}\n", + "Config", + }, + { + "TypeScript", lang.TypeScript, ".ts", + "class Service {\n\tprocess(cfg: Config) {}\n}\n", + "Config", + }, + { + "CSharp", lang.CSharp, ".cs", + "class Service {\n\tvoid Process(Config cfg) {}\n}\n", + "Config", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, p := runPipelineWithFile(t, "service"+tt.ext, tt.source) + + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + if len(methods) == 0 { + t.Fatal("no methods found") + } + + for _, m := range methods { + assertParamTypeContains(t, m, tt.wantType) + } + }) + } +} + +// --- Test 28: SCSS Variable Extraction --- + +func TestSCSSVariableExtraction(t *testing.T) { + src := `$primary-color: #007bff; +$font-size: 16px; + +body { + color: $primary-color; + font-size: $font-size; +} +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "styles.scss"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + vars, _ := s.FindNodesByLabel(p.ProjectName, "Variable") + varNames := map[string]bool{} + for _, v := range vars { + varNames[v.Name] = true + t.Logf("Variable: %s", v.Name) + } + + if len(vars) == 0 { + t.Error("no SCSS variables found") + } +} + +// --- Test 29: Ruby CALLS Extraction --- + +func TestRubyCallsExtraction(t *testing.T) { + src := `def greet(name) + puts "Hello #{name}" +end + +def main + greet("World") +end +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "app.rb"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + funcNames := map[string]bool{} + for _, f := range funcs { + funcNames[f.Name] = true + t.Logf("Function: %s", f.Name) + } + if !funcNames["greet"] { + t.Error("greet function not found") + } + + // Check CALLS or USAGE edges + callEdges, _ := s.FindEdgesByType(p.ProjectName, "CALLS") + usageEdges, _ := s.FindEdgesByType(p.ProjectName, "USAGE") + t.Logf("CALLS edges: %d, USAGE edges: %d", len(callEdges), len(usageEdges)) + total := len(callEdges) + len(usageEdges) + if total == 0 { + t.Error("expected CALLS or USAGE edges, got 0") + } +} + +// --- Test 30: ObjC Class and Method Extraction --- + +func TestObjCClassMethodExtraction(t *testing.T) { + src := `@interface Greeter : NSObject +- (void)greet:(NSString *)name; +@end + +@implementation Greeter +- (void)greet:(NSString *)name { + NSLog(@"Hello %@", name); +} + +- (void)run { + [self greet:@"World"]; +} +@end +` + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, "Greeter.m"), src) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + classNames := map[string]bool{} + for _, c := range classes { + classNames[c.Name] = true + t.Logf("Class: %s", c.Name) + } + if !classNames["Greeter"] { + t.Error("Greeter class not found") + } + + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + methodNames := map[string]bool{} + for _, m := range methods { + methodNames[m.Name] = true + t.Logf("Method: %s", m.Name) + } + if len(methods) == 0 { + t.Error("no methods found") + } + + // Check for CALLS or USAGE edges (message_expression CALLS) + callEdges, _ := s.FindEdgesByType(p.ProjectName, "CALLS") + usageEdges, _ := s.FindEdgesByType(p.ProjectName, "USAGE") + t.Logf("CALLS edges: %d, USAGE edges: %d", len(callEdges), len(usageEdges)) +} + +// --- Test 31: Rust attribute extraction (#[get("/path")]) --- (formerly Test 14) + +func TestRustAttributeExtraction(t *testing.T) { + src := `#[get("/users")] +async fn get_users() -> HttpResponse { + HttpResponse::Ok().finish() +} + +#[post("/users")] +async fn create_user() -> HttpResponse { + HttpResponse::Created().finish() +} +` + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "main.rs"), []byte(src), 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + for _, f := range funcs { + t.Logf("Function: %s (decorators=%v, is_entry_point=%v)", + f.Name, f.Properties["decorators"], f.Properties["is_entry_point"]) + } + + // Check decorators are extracted + foundDecorators := false + for _, f := range funcs { + if decs, ok := f.Properties["decorators"]; ok && decs != nil { + foundDecorators = true + } + } + if !foundDecorators { + t.Error("no decorators found on any function") + } + + // Check is_entry_point + entryPoints := 0 + for _, f := range funcs { + if ep, ok := f.Properties["is_entry_point"]; ok && ep == true { + entryPoints++ + } + } + if entryPoints < 2 { + t.Errorf("expected at least 2 entry points, got %d", entryPoints) + } +} + +// --- Shared Test Helpers --- + +// runPipelineWithFile creates a temp dir with a single file, runs the pipeline, and returns the store and pipeline. +func runPipelineWithFile(t *testing.T, filename, source string) (*store.Store, *Pipeline) { + t.Helper() + dir := t.TempDir() + writeLangTestFile(t, filepath.Join(dir, filename), source) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + return s, p +} + +// collectNodesByLabel returns nodes matching the given label. +func collectNodesByLabel(nodes []*store.Node, label string) []*store.Node { + var out []*store.Node + for _, n := range nodes { + if n.Label == label { + out = append(out, n) + } + } + return out +} + +// assertNodeNamesExist checks that all wantNames appear among the nodes. +func assertNodeNamesExist(t *testing.T, nodes []*store.Node, wantNames []string) { + t.Helper() + nameSet := map[string]bool{} + for _, n := range nodes { + nameSet[n.Name] = true + } + for _, want := range wantNames { + if !nameSet[want] { + t.Errorf("expected node %q not found; got: %v", want, nameSet) + } + } +} + +// assertNodeSignatures checks that methods have expected signature properties. +func assertNodeSignatures(t *testing.T, nodes []*store.Node, wantSigs map[string]string) { + t.Helper() + for _, n := range nodes { + wantSig, ok := wantSigs[n.Name] + if !ok { + continue + } + gotSig, _ := n.Properties["signature"].(string) + if gotSig != wantSig { + t.Errorf("node %q: signature=%q, want=%q", n.Name, gotSig, wantSig) + } + } +} + +// assertEdgeCount checks that the number of edges with the given type matches expected. +func assertEdgeCount(t *testing.T, edges []pendingEdge, edgeType string, want int) { + t.Helper() + count := 0 + for _, pe := range edges { + if pe.Type == edgeType { + count++ + } + } + if count != want { + t.Errorf("expected %d %s edges, got %d", want, edgeType, count) + } +} + +// assertNodesNotLabeled checks that none of the wantNames appear among nodes (wrong label). +func assertNodesNotLabeled(t *testing.T, nodes []*store.Node, wantNames []string, badLabel string) { + t.Helper() + for _, n := range nodes { + for _, want := range wantNames { + if n.Name == want { + t.Errorf("%q should not be %s", want, badLabel) + } + } + } +} + +// assertParamTypeContains checks that a method's param_types property contains the expected type. +func assertParamTypeContains(t *testing.T, m *store.Node, wantType string) { + t.Helper() + paramTypes, ok := m.Properties["param_types"] + t.Logf("Method: %s, param_types=%v", m.Name, paramTypes) + if !ok || paramTypes == nil { + t.Errorf("method %q has no param_types property", m.Name) + return + } + pts, ok := paramTypes.([]interface{}) + if !ok { + t.Errorf("method %q param_types is not a slice: %T", m.Name, paramTypes) + return + } + for _, pt := range pts { + if str, ok := pt.(string); ok && str == wantType { + return + } + } + t.Errorf("method %q param_types=%v, want to contain %q", m.Name, paramTypes, wantType) +} diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index e8fe86791..40f0e8e2e 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -1,6 +1,7 @@ package pipeline import ( + "context" "encoding/hex" "encoding/json" "fmt" @@ -27,6 +28,7 @@ import ( // Pipeline orchestrates the 3-pass indexing of a repository. type Pipeline struct { + ctx context.Context Store *store.Store RepoPath string ProjectName string @@ -45,9 +47,10 @@ type cachedAST struct { } // New creates a new Pipeline. -func New(s *store.Store, repoPath string) *Pipeline { - projectName := filepath.Base(repoPath) +func New(ctx context.Context, s *store.Store, repoPath string) *Pipeline { + projectName := ProjectNameFromPath(repoPath) return &Pipeline{ + ctx: ctx, Store: s, RepoPath: repoPath, ProjectName: projectName, @@ -57,23 +60,47 @@ func New(s *store.Store, repoPath string) *Pipeline { } } +// ProjectNameFromPath derives a unique project name from an absolute path +// by replacing path separators with dashes and trimming the leading dash. +func ProjectNameFromPath(absPath string) string { + // Clean and convert to slash-separated + cleaned := filepath.ToSlash(filepath.Clean(absPath)) + // Replace slashes with dashes + name := strings.ReplaceAll(cleaned, "/", "-") + // Trim leading dash (from leading /) + name = strings.TrimLeft(name, "-") + if name == "" { + return "root" + } + return name +} + +// checkCancel returns ctx.Err() if the pipeline's context has been cancelled. +func (p *Pipeline) checkCancel() error { + return p.ctx.Err() +} + // Run executes the full 3-pass pipeline within a single transaction. // If file hashes from a previous run exist, only changed files are re-processed. func (p *Pipeline) Run() error { slog.Info("pipeline.start", "project", p.ProjectName, "path", p.RepoPath) + if err := p.checkCancel(); err != nil { + return err + } + // Discover source files (filesystem, no DB — runs outside transaction) - files, err := discover.Discover(p.RepoPath, nil) + files, err := discover.Discover(p.ctx, p.RepoPath, nil) if err != nil { return fmt.Errorf("discover: %w", err) } slog.Info("pipeline.discovered", "files", len(files)) // Use MEMORY journal mode during fresh indexing for faster bulk writes. - p.Store.BeginBulkWrite() + p.Store.BeginBulkWrite(p.ctx) wroteData := false - if err := p.Store.WithTransaction(func(txStore *store.Store) error { + if err := p.Store.WithTransaction(p.ctx, func(txStore *store.Store) error { origStore := p.Store p.Store = txStore defer func() { p.Store = origStore }() @@ -81,16 +108,16 @@ func (p *Pipeline) Run() error { wroteData, passErr = p.runPasses(files) return passErr }); err != nil { - p.Store.EndBulkWrite() + p.Store.EndBulkWrite(p.ctx) return err } - p.Store.EndBulkWrite() + p.Store.EndBulkWrite(p.ctx) // Only checkpoint + optimize when actual data was written. // No-op incremental reindexes skip this to avoid ANALYZE overhead. if wroteData { - p.Store.Checkpoint() + p.Store.Checkpoint(p.ctx) } nc, _ := p.Store.CountNodes(p.ProjectName) @@ -133,14 +160,27 @@ func (p *Pipeline) runFullPasses(files []discover.FileInfo) error { return fmt.Errorf("pass1 structure: %w", err) } slog.Info("pass.timing", "pass", "structure", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } t = time.Now() p.passDefinitions(files) // includes Variable extraction + enrichment slog.Info("pass.timing", "pass", "definitions", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } + + t = time.Now() + p.passDecoratorTags() // auto-discover decorator semantic tags + slog.Info("pass.timing", "pass", "decorator_tags", "elapsed", time.Since(t)) t = time.Now() p.buildRegistry() // includes Variable label slog.Info("pass.timing", "pass", "registry", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } t = time.Now() p.passInherits() // INHERITS edges from base_classes @@ -149,18 +189,30 @@ func (p *Pipeline) runFullPasses(files []discover.FileInfo) error { t = time.Now() p.passDecorates() // DECORATES edges from decorators slog.Info("pass.timing", "pass", "decorates", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } t = time.Now() p.passImports() slog.Info("pass.timing", "pass", "imports", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } t = time.Now() p.passCalls() slog.Info("pass.timing", "pass", "calls", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } t = time.Now() p.passUsages() slog.Info("pass.timing", "pass", "usages", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } t = time.Now() p.passUsesType() // USES_TYPE edges (signatures + body) @@ -177,6 +229,9 @@ func (p *Pipeline) runFullPasses(files []discover.FileInfo) error { t = time.Now() p.passConfigures() // CONFIGURES edges slog.Info("pass.timing", "pass", "configures", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } p.cleanupASTCache() @@ -187,6 +242,9 @@ func (p *Pipeline) runFullPasses(files []discover.FileInfo) error { t = time.Now() p.passCommunities() // Community nodes + MEMBER_OF edges (DB-only) slog.Info("pass.timing", "pass", "communities", "elapsed", time.Since(t)) + if err := p.checkCancel(); err != nil { + return err + } t = time.Now() if err := p.passHTTPLinks(); err != nil { @@ -238,6 +296,9 @@ func (p *Pipeline) runIncrementalPasses( if err := p.passStructure(allFiles); err != nil { return fmt.Errorf("pass1 structure: %w", err) } + if err := p.checkCancel(); err != nil { + return err + } // Remove stale nodes/edges for deleted files p.removeDeletedFiles(allFiles) @@ -249,10 +310,19 @@ func (p *Pipeline) runIncrementalPasses( // Pass 2: Parse changed files only p.passDefinitions(changed) + if err := p.checkCancel(); err != nil { + return err + } + + // Re-compute decorator tags globally (threshold is across all nodes) + p.passDecoratorTags() // Build full registry: includes nodes from unchanged files (already in DB) // plus newly parsed nodes from changed files p.buildRegistry() + if err := p.checkCancel(); err != nil { + return err + } // Re-build import maps for changed files (already done in passDefinitions) // Also load import maps for unchanged files from their AST (not cached) @@ -260,6 +330,9 @@ func (p *Pipeline) runIncrementalPasses( // have ASTs cached. Rebuild imports only for changed files is sufficient // since unchanged file import edges still exist in DB. p.passImports() + if err := p.checkCancel(); err != nil { + return err + } // Determine which files need call re-resolution: // changed files + files that import any changed module @@ -282,12 +355,18 @@ func (p *Pipeline) runIncrementalPasses( // Re-resolve calls + usages for changed + dependent files p.passCallsForFiles(filesToResolve) p.passUsagesForFiles(filesToResolve) + if err := p.checkCancel(); err != nil { + return err + } // AST-dependent passes (run on cached files before cleanup) p.passUsesType() p.passThrows() p.passReadsWrites() p.passConfigures() + if err := p.checkCancel(); err != nil { + return err + } p.cleanupASTCache() @@ -306,6 +385,9 @@ func (p *Pipeline) runIncrementalPasses( _ = p.Store.DeleteEdgesByType(p.ProjectName, "MEMBER_OF") _ = p.Store.DeleteNodesByLabel(p.ProjectName, "Community") p.passCommunities() + if err := p.checkCancel(); err != nil { + return err + } // HTTP linking and implements always run fully (they clean up first) if err := p.passHTTPLinks(); err != nil { @@ -430,6 +512,9 @@ func (p *Pipeline) loadImportMapFromDB(moduleQN string) map[string]string { func (p *Pipeline) passCallsForFiles(files []discover.FileInfo) { slog.Info("pass3.calls.incremental", "files", len(files)) for _, f := range files { + if p.ctx.Err() != nil { + return + } cached, ok := p.astCache[f.RelPath] if !ok { // File not in AST cache — need to parse it @@ -715,6 +800,9 @@ func (p *Pipeline) passDefinitions(files []discover.FileInfo) { parseableFiles := make([]discover.FileInfo, 0, len(files)) for _, f := range files { if f.Language == lang.JSON { + if p.ctx.Err() != nil { + return + } if err := p.processJSONFile(f); err != nil { slog.Warn("pass2.json.err", "path", f.RelPath, "err", err) } @@ -734,10 +822,13 @@ func (p *Pipeline) passDefinitions(files []discover.FileInfo) { numWorkers = len(parseableFiles) } - g := new(errgroup.Group) + g, gctx := errgroup.WithContext(p.ctx) g.SetLimit(numWorkers) for i, f := range parseableFiles { g.Go(func() error { + if gctx.Err() != nil { + return gctx.Err() + } results[i] = parseFileAST(p.ProjectName, f) return nil }) @@ -749,10 +840,11 @@ func (p *Pipeline) passDefinitions(files []discover.FileInfo) { var allPendingEdges []pendingEdge for _, r := range results { - if r == nil || r.Err != nil { - if r != nil && r.Err != nil { - slog.Warn("pass2.file.err", "path", r.File.RelPath, "err", r.Err) - } + if r == nil { + continue + } + if r.Err != nil { + slog.Warn("pass2.file.err", "path", r.File.RelPath, "err", r.Err) continue } // Populate AST cache (sequential, map writes) @@ -809,8 +901,12 @@ func parseFileAST(projectName string, f discover.FileInfo) *parseResult { return result } + // Strip UTF-8 BOM if present (common in C#/Windows-generated files) + source = stripBOM(source) + tree, err := parser.Parse(f.Language, source) if err != nil { + slog.Warn("parse.file.err", "path", f.RelPath, "lang", f.Language, "err", err) result.Err = err return result } @@ -845,14 +941,27 @@ func parseFileAST(projectName string, f discover.FileInfo) *parseResult { isCPP := f.Language == lang.CPP macroNames := make(map[string]bool) // track macro names for call site resolution + isElixir := f.Language == lang.Elixir + parser.Walk(root, func(node *tree_sitter.Node) bool { kind := node.Kind() + // Elixir: all definitions are `call` nodes — classify by callee name + if isElixir && kind == "call" { + return handleElixirCall(node, source, f, projectName, moduleQN, spec, result) + } + if funcTypes[kind] { extractFunctionDef(node, source, f, projectName, moduleQN, spec, result) return false } + // Rust impl blocks: extract methods and record trait implementation + if kind == "impl_item" { + extractRustImplBlock(node, source, f, projectName, moduleQN, spec, result) + return false + } + if classTypes[kind] { extractClassDef(node, source, f, projectName, moduleQN, spec, result) return false @@ -955,17 +1064,66 @@ func mergeResolvedConstants(constants []string, root *tree_sitter.Node, source [ return constants } +// resolveFuncNameNode resolves the name node for a function, handling +// language-specific quirks (Lua anonymous assigns, R assignment, OCaml let, etc). +// Returns nil if the node should be skipped (e.g., Haskell signature). +func resolveFuncNameNode(node *tree_sitter.Node, language lang.Language) *tree_sitter.Node { + // Haskell: skip signature nodes — they're type declarations, not function definitions + if language == lang.Haskell && node.Kind() == "signature" { + return nil + } + + nameNode := funcNameNode(node) + + // R: name <- function(...) — name lives on parent binary_operator lhs field + if language == lang.R && node.Kind() == "function_definition" { + nameNode = rFuncAssignName(node) + } + + if nameNode != nil { + return nameNode + } + + // Lua: anonymous function assignment + if language == lang.Lua && node.Kind() == "function_definition" { + return luaFuncAssignName(node) + } + + // OCaml: value_definition → let_binding with pattern field for name + if language == lang.OCaml && node.Kind() == "value_definition" { + return ocamlFuncName(node) + } + + // Zig: test_declaration — name is in a string child, not name field + if language == lang.Zig && node.Kind() == "test_declaration" { + if strNode := findChildByKind(node, "string"); strNode != nil { + if content := findChildByKind(strNode, "string_content"); content != nil { + return content + } + } + } + + // JS/TS/TSX: const X = () => {} — name lives on parent variable_declarator + if node.Kind() == "arrow_function" { + if p := node.Parent(); p != nil && p.Kind() == "variable_declarator" { + return p.ChildByFieldName("name") + } + } + + return nil +} + // extractFunctionDef extracts a function/method node and DEFINES edge as data (no DB). func extractFunctionDef( node *tree_sitter.Node, source []byte, f discover.FileInfo, projectName, moduleQN string, spec *lang.LanguageSpec, result *parseResult, ) { - nameNode := node.ChildByFieldName("name") + nameNode := resolveFuncNameNode(node, f.Language) if nameNode == nil { return } name := parser.NodeText(nameNode, source) - if name == "" { + if name == "" || name == "function" { return } @@ -1002,6 +1160,16 @@ func extractFunctionDef( props["is_exported"] = isExported(name, f.Language) + // JS/TS: detect actual `export` keyword — mark as entry point + // export function foo() {} → parent is export_statement + // export const x = () => {} → ancestor chain: variable_declarator → lexical_declaration → export_statement + // module.exports = { foo } → handled separately via module.exports detection + if f.Language == lang.JavaScript || f.Language == lang.TypeScript || f.Language == lang.TSX { + if hasAncestorKind(node, "export_statement", 4) { + props["is_entry_point"] = true + } + } + // Decorator extraction (Python, Java, TS/JS) decorators := extractAllDecorators(node, source, f.Language, spec) if len(decorators) > 0 { @@ -1015,6 +1183,10 @@ func extractFunctionDef( props["is_entry_point"] = true } + if doc := extractDocstring(node, source, f.Language); doc != "" { + props["docstring"] = doc + } + startLine := safeRowToLine(node.StartPosition().Row) endLine := safeRowToLine(node.EndPosition().Row) @@ -1054,12 +1226,60 @@ func extractFunctionDef( }) } +// extractRustImplBlock handles Rust `impl Trait for Type` and `impl Type` blocks. +// It extracts methods inside the impl block and associates them with the implementing type. +// For `impl Trait for Type`, it records a pending IMPLEMENTS edge. +func extractRustImplBlock( + node *tree_sitter.Node, source []byte, f discover.FileInfo, + projectName, _ string, spec *lang.LanguageSpec, result *parseResult, +) { + // Get the implementing type name + typeNode := node.ChildByFieldName("type") + if typeNode == nil { + return + } + typeName := parser.NodeText(typeNode, source) + if typeName == "" { + return + } + + typeQN := fqn.Compute(projectName, f.RelPath, typeName) + + // Extract methods inside the impl block and attach to the type + extractClassMethodDefs(node, source, f, projectName, typeQN, spec, result) + + // If this is `impl Trait for Type`, record IMPLEMENTS edge + traitNode := node.ChildByFieldName("trait") + if traitNode != nil { + traitName := parser.NodeText(traitNode, source) + if traitName != "" { + traitQN := fqn.Compute(projectName, f.RelPath, traitName) + result.PendingEdges = append(result.PendingEdges, pendingEdge{ + SourceQN: typeQN, + TargetQN: traitQN, + Type: "IMPLEMENTS", + }) + } + } +} + // extractClassDef extracts a class/type node and its methods as data (no DB). func extractClassDef( node *tree_sitter.Node, source []byte, f discover.FileInfo, projectName, moduleQN string, spec *lang.LanguageSpec, result *parseResult, ) { + // HCL block: name is composed from identifier + string_lit children + if f.Language == lang.HCL && node.Kind() == "block" { + extractHCLBlock(node, source, f, projectName, moduleQN, result) + return + } + nameNode := node.ChildByFieldName("name") + // ObjC: class_interface/class_implementation don't use a "name" field — + // the class name is the first identifier child. + if nameNode == nil && f.Language == lang.ObjectiveC { + nameNode = findChildByKind(node, "identifier") + } if nameNode == nil { return } @@ -1105,6 +1325,10 @@ func extractClassDef( } } + if doc := extractDocstring(node, source, f.Language); doc != "" { + classProps["docstring"] = doc + } + result.Nodes = append(result.Nodes, &store.Node{ Project: projectName, Label: label, @@ -1148,6 +1372,102 @@ func extractClassDef( } } +// resolveMethodName resolves the name node for a method, including arrow function +// class properties where the name lives on the parent field_definition. +// Returns the name node and the field definition node (nil for regular methods). +func resolveMethodName(child *tree_sitter.Node) (nameNode, fieldDef *tree_sitter.Node) { + if mn := funcNameNode(child); mn != nil { + return mn, nil + } + // Arrow functions as class properties: the name lives on the parent + // field_definition (JS) or public_field_definition (TS/TSX). + if child.Kind() != "arrow_function" { + return nil, nil + } + p := child.Parent() + if p == nil { + return nil, nil + } + switch p.Kind() { + case "field_definition": + nameNode = p.ChildByFieldName("property") + case "public_field_definition": + nameNode = p.ChildByFieldName("name") + default: + return nil, nil + } + return nameNode, p +} + +// buildMethodProps builds the properties map for a class method node. +func buildMethodProps( + child *tree_sitter.Node, fieldDefNode *tree_sitter.Node, + source []byte, f discover.FileInfo, spec *lang.LanguageSpec, +) map[string]any { + props := map[string]any{} + + paramsNode := child.ChildByFieldName("parameters") + if paramsNode != nil { + props["signature"] = parser.NodeText(paramsNode, source) + } + + extractMethodReturnType(child, fieldDefNode, source, props) + + // Decorator extraction for class methods + if spec != nil { + decorators := extractAllDecorators(child, source, f.Language, spec) + if len(decorators) > 0 { + props["decorators"] = decorators + if hasFrameworkDecorator(decorators) { + props["is_entry_point"] = true + } + } + } + + if paramsNode != nil { + if paramTypes := extractParamTypes(paramsNode, source, f.Language); len(paramTypes) > 0 { + props["param_types"] = paramTypes + } + } + + if doc := extractDocstring(child, source, f.Language); doc != "" { + props["docstring"] = doc + } + + if spec != nil && len(spec.BranchingNodeTypes) > 0 { + if complexity := countBranchingNodes(child, spec.BranchingNodeTypes); complexity > 0 { + props["complexity"] = complexity + } + } + + return props +} + +// extractMethodReturnType extracts the return type from a method or arrow function field. +func extractMethodReturnType( + child *tree_sitter.Node, fieldDefNode *tree_sitter.Node, + source []byte, props map[string]any, +) { + // For arrow function properties, extract the type annotation from the field + if fieldDefNode != nil { + if typeNode := fieldDefNode.ChildByFieldName("type"); typeNode != nil { + txt := parser.NodeText(typeNode, source) + txt = strings.TrimPrefix(txt, ": ") + txt = strings.TrimPrefix(txt, ":") + txt = strings.TrimSpace(txt) + if txt != "" { + props["return_type"] = txt + } + } + } + for _, field := range []string{"result", "return_type", "type"} { + if rtNode := child.ChildByFieldName(field); rtNode != nil { + props["return_type"] = parser.NodeText(rtNode, source) + break + } + } +} + // extractClassMethodDefs walks a class AST node and extracts Method nodes (no DB). func extractClassMethodDefs( classNode *tree_sitter.Node, source []byte, f discover.FileInfo, @@ -1162,7 +1482,7 @@ func extractClassMethodDefs( return true } - mn := child.ChildByFieldName("name") + mn, fieldDefNode := resolveMethodName(child) if mn == nil { return false } @@ -1171,38 +1491,28 @@ func extractClassMethodDefs( return false } - methodQN := classQN + "." + methodName - props := map[string]any{} - - paramsNode := child.ChildByFieldName("parameters") - if paramsNode != nil { - props["signature"] = parser.NodeText(paramsNode, source) - } - for _, field := range []string{"result", "return_type", "type"} { - rtNode := child.ChildByFieldName(field) - if rtNode != nil { - props["return_type"] = parser.NodeText(rtNode, source) - break - } - } + props := buildMethodProps(child, fieldDefNode, source, f, spec) props["is_exported"] = isExported(methodName, f.Language) - mStartLine := safeRowToLine(child.StartPosition().Row) - mEndLine := safeRowToLine(child.EndPosition().Row) + // Use field definition span when available (covers name + type + body) + spanNode := child + if fieldDefNode != nil { + spanNode = fieldDefNode + } result.Nodes = append(result.Nodes, &store.Node{ Project: projectName, Label: "Method", Name: methodName, - QualifiedName: methodQN, + QualifiedName: classQN + "." + methodName, FilePath: f.RelPath, - StartLine: mStartLine, - EndLine: mEndLine, + StartLine: safeRowToLine(spanNode.StartPosition().Row), + EndLine: safeRowToLine(spanNode.EndPosition().Row), Properties: props, }) result.PendingEdges = append(result.PendingEdges, pendingEdge{ SourceQN: classQN, - TargetQN: methodQN, + TargetQN: classQN + "." + methodName, Type: "DEFINES_METHOD", }) return false @@ -1438,10 +1748,13 @@ func (p *Pipeline) passCalls() { numWorkers = len(files) } - g := new(errgroup.Group) + g, gctx := errgroup.WithContext(p.ctx) g.SetLimit(numWorkers) for i, fe := range files { g.Go(func() error { + if gctx.Err() != nil { + return gctx.Err() + } results[i] = p.resolveFileCalls(fe.relPath, fe.cached) return nil }) @@ -1648,51 +1961,485 @@ func (p *Pipeline) resolveCallWithTypes( // === Helper functions === -func extractCalleeName(node *tree_sitter.Node, source []byte, _ lang.Language) string { - // For call_expression / call nodes, get the function being called - funcNode := node.ChildByFieldName("function") - if funcNode != nil { - kind := funcNode.Kind() - // Direct call: foo() - if kind == "identifier" { - return parser.NodeText(funcNode, source) - } - // Qualified call: pkg.Func() or obj.method() - if kind == "selector_expression" || kind == "attribute" || kind == "member_expression" { - return parser.NodeText(funcNode, source) - } +func extractCalleeName(node *tree_sitter.Node, source []byte, language lang.Language) string { + // Try function field (most languages) + if name := extractCalleeFromFunctionField(node, source); name != "" { + return name } - // For method_invocation (Java), get name field - nameNode := node.ChildByFieldName("name") - if nameNode != nil { + // Try name field (Java method_invocation) + if nameNode := node.ChildByFieldName("name"); nameNode != nil { return parser.NodeText(nameNode, source) } - // Kotlin call_expression: first named child is the callee expression (identifier or navigation_expression) - // navigation_expression: obj.method (no named fields, children are expression + identifier) + // Ruby: call node has "method" field + if name := extractCalleeFromMethodField(node, source); name != "" { + return name + } + + // Language-specific extraction + return extractCalleeLanguageSpecific(node, source, language) +} + +// extractCalleeFromFunctionField extracts the callee name from a "function" field. +func extractCalleeFromFunctionField(node *tree_sitter.Node, source []byte) string { + funcNode := node.ChildByFieldName("function") + if funcNode == nil { + return "" + } + switch funcNode.Kind() { + case "identifier", "simple_identifier", + "selector_expression", "attribute", "member_expression", + "field_expression", "dot", "function": + return parser.NodeText(funcNode, source) + } + return "" +} + +// extractCalleeFromMethodField extracts the callee from Ruby-style method+receiver fields. +func extractCalleeFromMethodField(node *tree_sitter.Node, source []byte) string { + methodNode := node.ChildByFieldName("method") + if methodNode == nil { + return "" + } + if receiver := node.ChildByFieldName("receiver"); receiver != nil { + return parser.NodeText(receiver, source) + "." + parser.NodeText(methodNode, source) + } + return parser.NodeText(methodNode, source) +} + +// extractCalleeLanguageSpecific handles language-specific callee extraction +// for ObjC, Erlang, Perl, and Kotlin. +func extractCalleeLanguageSpecific(node *tree_sitter.Node, source []byte, language lang.Language) string { + switch language { + case lang.ObjectiveC: + if node.Kind() == "message_expression" { + if methodField := findChildByKindField(node, "method"); methodField != nil { + return parser.NodeText(methodField, source) + } + } + case lang.Erlang: + return extractErlangCallee(node, source) + case lang.Perl: + return extractPerlCallee(node, source) + } + + // Kotlin call_expression / navigation_expression if node.Kind() == "call_expression" || node.Kind() == "navigation_expression" { - first := node.NamedChild(0) - if first != nil { - kind := first.Kind() - if kind == "identifier" || kind == "navigation_expression" { + if first := node.NamedChild(0); first != nil { + switch first.Kind() { + case "identifier", "navigation_expression", "simple_identifier": return parser.NodeText(first, source) } } } - return "" } +func extractErlangCallee(node *tree_sitter.Node, source []byte) string { + if node.Kind() != "call" { + return "" + } + if remote := node.ChildByFieldName("expr"); remote != nil && remote.Kind() == "remote" { + if funNode := remote.ChildByFieldName("fun"); funNode != nil { + return parser.NodeText(funNode, source) + } + } + if first := node.NamedChild(0); first != nil && first.Kind() == "atom" { + return parser.NodeText(first, source) + } + return "" +} + +func extractPerlCallee(node *tree_sitter.Node, source []byte) string { + if node.Kind() != "ambiguous_function_call_expression" && node.Kind() != "function_call_expression" { + return "" + } + if fn := node.ChildByFieldName("function"); fn != nil { + return parser.NodeText(fn, source) + } + if first := node.NamedChild(0); first != nil { + return parser.NodeText(first, source) + } + return "" +} + +// findChildByKindField finds a child node that has a specific field name. +func findChildByKindField(node *tree_sitter.Node, fieldName string) *tree_sitter.Node { + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child == nil { + continue + } + name := node.FieldNameForChild(uint32(i)) + if name == fieldName { + return child + } + } + return nil +} + +// funcNameNode returns the name node for a function/method node. +// Handles C++ where the name is inside function_declarator. +func funcNameNode(node *tree_sitter.Node) *tree_sitter.Node { + nameNode := node.ChildByFieldName("name") + if nameNode == nil { + // C++: name is inside function_declarator + if declNode := node.ChildByFieldName("declarator"); declNode != nil { + nameNode = declNode.ChildByFieldName("declarator") + if nameNode == nil { + nameNode = findChildByKind(declNode, "identifier") + } + } + } + // Groovy: function_definition uses "function" field for name + if nameNode == nil { + nameNode = node.ChildByFieldName("function") + } + // Dart: method_signature wraps function_signature — name is nested + if nameNode == nil && node.Kind() == "method_signature" { + if fs := findChildByKind(node, "function_signature"); fs != nil { + nameNode = fs.ChildByFieldName("name") + } + } + // ObjC: method_definition has identifier children without field names + if nameNode == nil && node.Kind() == "method_definition" { + nameNode = findChildByKind(node, "identifier") + } + // SQL: create_function → object_reference → identifier[field="name"] + if nameNode == nil && node.Kind() == "create_function" { + if objRef := findChildByKind(node, "object_reference"); objRef != nil { + nameNode = objRef.ChildByFieldName("name") + } + } + return nameNode +} + +// hasAncestorKind walks up to maxDepth parents and returns true if any has the given kind. +func hasAncestorKind(node *tree_sitter.Node, kind string, maxDepth int) bool { + p := node.Parent() + for i := 0; i < maxDepth && p != nil; i++ { + if p.Kind() == kind { + return true + } + p = p.Parent() + } + return false +} + +// luaFuncAssignName extracts the identifier node for a Lua function_definition +// from its parent assignment context. Handles: +// +// local name = function(...) end → variable_declaration > assignment_statement > expression_list > function_definition +// name = function(...) → assignment_statement > expression_list > function_definition +func luaFuncAssignName(node *tree_sitter.Node) *tree_sitter.Node { + // function_definition sits inside expression_list; walk up to assignment_statement + parent := node.Parent() + if parent == nil { + return nil + } + // parent is expression_list; go one more level up to assignment_statement + if parent.Kind() == "expression_list" { + parent = parent.Parent() + } + if parent == nil { + return nil + } + if parent.Kind() != "assignment_statement" { + return nil + } + // assignment_statement: first named child is variable_list with the target identifier(s) + for i := uint(0); i < parent.NamedChildCount(); i++ { + child := parent.NamedChild(i) + if child.Kind() == "variable_list" { + return findLastIdentifier(child) + } + } + return nil +} + +// extractHCLBlock extracts a Terraform/HCL block as a Class node. +// AST: block → identifier("resource") string_lit("aws_instance") string_lit("web") { body } +// Name format: "resource.aws_instance.web" +func extractHCLBlock( + node *tree_sitter.Node, source []byte, f discover.FileInfo, + projectName, moduleQN string, result *parseResult, +) { + var parts []string + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child == nil { + continue + } + switch child.Kind() { + case "identifier": + parts = append(parts, parser.NodeText(child, source)) + case "string_lit": + // Extract text content from string_lit → template_literal + if tl := findChildByKind(child, "template_literal"); tl != nil { + parts = append(parts, parser.NodeText(tl, source)) + } + case "block_start", "block_end", "body": + // stop collecting name parts + } + } + if len(parts) == 0 { + return + } + + name := strings.Join(parts, ".") + classQN := fqn.Compute(projectName, f.RelPath, name) + startLine := safeRowToLine(node.StartPosition().Row) + endLine := safeRowToLine(node.EndPosition().Row) + + label := "Class" + blockType := parts[0] + if blockType == "variable" || blockType == "output" || blockType == "locals" { + label = "Variable" + } + + result.Nodes = append(result.Nodes, &store.Node{ + Project: projectName, Label: label, Name: name, + QualifiedName: classQN, FilePath: f.RelPath, + StartLine: startLine, EndLine: endLine, + Properties: map[string]any{ + "block_type": blockType, + "is_exported": true, + }, + }) + result.PendingEdges = append(result.PendingEdges, pendingEdge{ + SourceQN: moduleQN, TargetQN: classQN, Type: "DEFINES", + }) +} + +// ocamlFuncName extracts the name from an OCaml value_definition. +// AST: value_definition → let_binding (child) → pattern field has the name. +func ocamlFuncName(node *tree_sitter.Node) *tree_sitter.Node { + lb := findChildByKind(node, "let_binding") + if lb == nil { + return nil + } + return lb.ChildByFieldName("pattern") +} + +// rFuncAssignName extracts the identifier node for an R function_definition +// from its parent assignment context. +// +// name <- function(...) { ... } → binary_operator > function_definition (rhs) +// name = function(...) → binary_operator > function_definition (rhs) +func rFuncAssignName(node *tree_sitter.Node) *tree_sitter.Node { + parent := node.Parent() + if parent == nil || parent.Kind() != "binary_operator" { + return nil + } + // The lhs field contains the identifier name + if lhs := parent.ChildByFieldName("lhs"); lhs != nil && lhs.Kind() == "identifier" { + return lhs + } + return nil +} + +// handleElixirCall classifies an Elixir `call` AST node by inspecting the callee name. +// Elixir is homoiconic — def, defp, defmodule, import, use are all macro calls. +// Returns true to continue walking children (for defmodule which has nested defs). +func handleElixirCall( + node *tree_sitter.Node, source []byte, f discover.FileInfo, + projectName, moduleQN string, spec *lang.LanguageSpec, result *parseResult, +) bool { + callee := elixirCallTarget(node, source) + switch callee { + case "def", "defp", "defmacro", "defmacrop": + extractElixirFunctionDef(node, source, f, projectName, moduleQN, spec, result) + return false + case "defmodule": + extractElixirModuleDef(node, source, f, projectName, moduleQN, spec, result) + return false // we walk the body ourselves + case "import", "use", "require", "alias": + return false // skip import-like calls + case "test", "describe": + // ExUnit test macros — extract as function + extractElixirFunctionDef(node, source, f, projectName, moduleQN, spec, result) + return false + case "if", "unless", "case", "cond", "for", "with", "receive": + return true // control flow — walk children for nested defs + } + return true // unknown call — continue walking +} + +// elixirCallTarget returns the callee name of an Elixir call node. +// AST: call → identifier[target="def"] or call → identifier[target="defmodule"] +func elixirCallTarget(node *tree_sitter.Node, source []byte) string { + target := node.ChildByFieldName("target") + if target == nil { + return "" + } + return parser.NodeText(target, source) +} + +// extractElixirFunctionDef extracts a function from an Elixir def/defp/defmacro call. +// AST: call[target=def] → arguments → call[target=greet] → arguments → (params) +func extractElixirFunctionDef( + node *tree_sitter.Node, source []byte, f discover.FileInfo, + projectName, moduleQN string, _ *lang.LanguageSpec, result *parseResult, +) { + args := findChildByKind(node, "arguments") + if args == nil { + return + } + // The first child of arguments is a call node with the function name + nameCall := findChildByKind(args, "call") + if nameCall == nil { + // Simple form: def greet, do: ... (identifier directly in arguments) + if id := findChildByKind(args, "identifier"); id != nil { + name := parser.NodeText(id, source) + if name == "" { + return + } + funcQN := fqn.Compute(projectName, f.RelPath, name) + startLine := safeRowToLine(node.StartPosition().Row) + endLine := safeRowToLine(node.EndPosition().Row) + result.Nodes = append(result.Nodes, &store.Node{ + Project: projectName, Label: "Function", Name: name, + QualifiedName: funcQN, FilePath: f.RelPath, + StartLine: startLine, EndLine: endLine, + Properties: map[string]any{"is_exported": isExported(name, f.Language)}, + }) + result.PendingEdges = append(result.PendingEdges, pendingEdge{ + SourceQN: moduleQN, TargetQN: funcQN, Type: "DEFINES", + }) + } + return + } + + // Get function name from the inner call's target + name := elixirCallTarget(nameCall, source) + if name == "" { + return + } + + funcQN := fqn.Compute(projectName, f.RelPath, name) + props := map[string]any{"is_exported": isExported(name, f.Language)} + + // Extract parameters from the inner call's arguments + innerArgs := findChildByKind(nameCall, "arguments") + if innerArgs != nil { + props["signature"] = parser.NodeText(innerArgs, source) + } + + startLine := safeRowToLine(node.StartPosition().Row) + endLine := safeRowToLine(node.EndPosition().Row) + + // Line count + lines := endLine - startLine + 1 + if lines > 0 { + props["lines"] = lines + } + + callee := elixirCallTarget(node, source) + if callee == "defp" || callee == "defmacrop" { + props["is_exported"] = false + } + + result.Nodes = append(result.Nodes, &store.Node{ + Project: projectName, Label: "Function", Name: name, + QualifiedName: funcQN, FilePath: f.RelPath, + StartLine: startLine, EndLine: endLine, + Properties: props, + }) + result.PendingEdges = append(result.PendingEdges, pendingEdge{ + SourceQN: moduleQN, TargetQN: funcQN, Type: "DEFINES", + }) +} + +// extractElixirModuleDef extracts a module (Class) from an Elixir defmodule call. +// AST: call[target=defmodule] → arguments → alias[name=MyApp] +// Then walks the do_block body for nested def/defp/defmodule calls. +func extractElixirModuleDef( + node *tree_sitter.Node, source []byte, f discover.FileInfo, + projectName, moduleQN string, spec *lang.LanguageSpec, result *parseResult, +) { + args := findChildByKind(node, "arguments") + if args == nil { + return + } + // Module name is an alias node inside arguments + aliasNode := findChildByKind(args, "alias") + if aliasNode == nil { + return + } + name := parser.NodeText(aliasNode, source) + if name == "" { + return + } + + classQN := fqn.Compute(projectName, f.RelPath, name) + startLine := safeRowToLine(node.StartPosition().Row) + endLine := safeRowToLine(node.EndPosition().Row) + + result.Nodes = append(result.Nodes, &store.Node{ + Project: projectName, Label: "Class", Name: name, + QualifiedName: classQN, FilePath: f.RelPath, + StartLine: startLine, EndLine: endLine, + Properties: map[string]any{"is_exported": true}, + }) + result.PendingEdges = append(result.PendingEdges, pendingEdge{ + SourceQN: moduleQN, TargetQN: classQN, Type: "DEFINES", + }) + + // Walk do_block body for nested defs + doBlock := findChildByKind(node, "do_block") + if doBlock == nil { + return + } + parser.Walk(doBlock, func(child *tree_sitter.Node) bool { + if child.Id() == doBlock.Id() { + return true + } + if child.Kind() == "call" { + callee := elixirCallTarget(child, source) + switch callee { + case "def", "defp", "defmacro", "defmacrop", "test", "describe": + extractElixirFunctionDef(child, source, f, projectName, classQN, spec, result) + return false + case "defmodule": + extractElixirModuleDef(child, source, f, projectName, classQN, spec, result) + return false + } + } + return true + }) +} + +// findLastIdentifier returns the deepest identifier in a node tree. +// For dot_index_expression (a.b.c) returns the last field identifier. +func findLastIdentifier(node *tree_sitter.Node) *tree_sitter.Node { + if node.Kind() == "identifier" { + return node + } + if node.Kind() == "dot_index_expression" { + // field is the rightmost identifier + if field := node.ChildByFieldName("field"); field != nil { + return field + } + } + for i := uint(0); i < node.NamedChildCount(); i++ { + child := node.NamedChild(i) + if child.Kind() == "identifier" { + return child + } + } + return nil +} + func findEnclosingFunction(node *tree_sitter.Node, source []byte, project, relPath string, spec *lang.LanguageSpec) string { funcTypes := toSet(spec.FunctionNodeTypes) + classTypes := toSet(spec.ClassNodeTypes) current := node.Parent() for current != nil { if funcTypes[current.Kind()] { - nameNode := current.ChildByFieldName("name") - if nameNode != nil { - name := parser.NodeText(nameNode, source) - return fqn.Compute(project, relPath, name) + qn, _ := computeFuncQN(current, source, project, relPath, classTypes) + if qn != "" { + return qn } } current = current.Parent() @@ -1708,29 +2455,54 @@ func isConstantNode(node *tree_sitter.Node, language lang.Language) bool { return isConstantForLanguage(node.Kind(), parent, language) } -func isConstantForLanguage(kind string, parent *tree_sitter.Node, language lang.Language) bool { - parentKind := parent.Kind() +// constantPattern defines which node kinds at which parent kinds are constants. +type constantPattern struct { + parentKinds []string + nodeKinds []string +} - switch language { - case lang.Go: - return parentKind == "source_file" && (kind == "const_declaration" || kind == "var_declaration") - case lang.Python: - return parentKind == "module" && kind == "expression_statement" - case lang.JavaScript, lang.TypeScript, lang.TSX: - return isJSConstantNode(kind, parentKind, parent) - case lang.Rust: - return parentKind == "source_file" && (kind == "const_item" || kind == "let_declaration") - case lang.PHP: - return parentKind == "program" && kind == "expression_statement" - case lang.Scala: - return (parentKind == "compilation_unit" || parentKind == "template_body") && kind == "val_definition" - case lang.CPP: - return parentKind == "translation_unit" && (kind == "preproc_def" || kind == "declaration") - case lang.Lua: - return parentKind == "chunk" && kind == "variable_declaration" - default: +// constantPatterns maps languages to their constant-detection patterns. +// Languages with complex logic (JS/TS) are handled separately. +var constantPatterns = map[lang.Language]constantPattern{ + lang.Go: {parentKinds: []string{"source_file"}, nodeKinds: []string{"const_declaration", "var_declaration"}}, + lang.Python: {parentKinds: []string{"module"}, nodeKinds: []string{"expression_statement"}}, + lang.Rust: {parentKinds: []string{"source_file"}, nodeKinds: []string{"const_item", "let_declaration"}}, + lang.PHP: {parentKinds: []string{"program"}, nodeKinds: []string{"expression_statement"}}, + lang.Scala: {parentKinds: []string{"compilation_unit", "template_body"}, nodeKinds: []string{"val_definition"}}, + lang.CPP: {parentKinds: []string{"translation_unit"}, nodeKinds: []string{"preproc_def", "declaration"}}, + lang.Lua: {parentKinds: []string{"chunk"}, nodeKinds: []string{"variable_declaration"}}, + lang.Erlang: {parentKinds: []string{"source_file"}, nodeKinds: []string{"pp_define", "record_decl"}}, + lang.SQL: {parentKinds: []string{"statement"}, nodeKinds: []string{"create_table", "create_view"}}, +} + +func isConstantForLanguage(kind string, parent *tree_sitter.Node, language lang.Language) bool { + // JS/TS/TSX have complex grandparent logic + if language == lang.JavaScript || language == lang.TypeScript || language == lang.TSX { + return isJSConstantNode(kind, parent.Kind(), parent) + } + + pat, ok := constantPatterns[language] + if !ok { return false } + + parentKind := parent.Kind() + parentMatch := false + for _, pk := range pat.parentKinds { + if parentKind == pk { + parentMatch = true + break + } + } + if !parentMatch { + return false + } + for _, nk := range pat.nodeKinds { + if kind == nk { + return true + } + } + return false } func isJSConstantNode(kind, parentKind string, parent *tree_sitter.Node) bool { @@ -1793,6 +2565,9 @@ var frameworkDecoratorPrefixes = []string{ "@celery.task", "@shared_task", "@task", // Signal handlers "@receiver", + // Rust Actix/Axum/Rocket route macros (#[get("/path")] → extracted as get("/path")) + "get(", "post(", "put(", "delete(", "patch(", "head(", "options(", + "route(", "connect(", "trace(", } // hasFrameworkDecorator returns true if any decorator matches a framework pattern. @@ -2143,6 +2918,15 @@ func looksLikeURL(s string) bool { // safeRowToLine converts a tree-sitter row (uint) to a 1-based line number (int). // Returns math.MaxInt if the value would overflow. +// stripBOM removes a UTF-8 BOM (0xEF 0xBB 0xBF) from the start of source. +// Common in C# and Windows-generated files; tree-sitter may choke on BOM bytes. +func stripBOM(source []byte) []byte { + if len(source) >= 3 && source[0] == 0xEF && source[1] == 0xBB && source[2] == 0xBF { + return source[3:] + } + return source +} + func safeRowToLine(row uint) int { const maxInt = int(^uint(0) >> 1) // math.MaxInt equivalent without importing math if row > uint(maxInt-1) { diff --git a/internal/pipeline/pipeline_bench_test.go b/internal/pipeline/pipeline_bench_test.go index 663d6ffee..d08a801b8 100644 --- a/internal/pipeline/pipeline_bench_test.go +++ b/internal/pipeline/pipeline_bench_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "context" "fmt" "os" "path/filepath" @@ -245,7 +246,7 @@ func BenchmarkPipelineRun(b *testing.B) { if err != nil { b.Fatal(err) } - p := New(s, repoDir) + p := New(context.Background(), s, repoDir) if err := p.Run(); err != nil { b.Fatalf("Pipeline.Run: %v", err) } @@ -264,7 +265,7 @@ func BenchmarkPipelineReindex(b *testing.B) { defer s.Close() // Initial index - p := New(s, repoDir) + p := New(context.Background(), s, repoDir) if err := p.Run(); err != nil { b.Fatalf("initial index: %v", err) } @@ -273,7 +274,7 @@ func BenchmarkPipelineReindex(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - p := New(s, repoDir) + p := New(context.Background(), s, repoDir) if err := p.Run(); err != nil { b.Fatalf("reindex %d: %v", i, err) } @@ -321,7 +322,7 @@ func (s *Struct%d) Method%d() int { if err != nil { b.Fatal(err) } - p := New(s, dir) + p := New(context.Background(), s, dir) if err := p.Run(); err != nil { b.Fatalf("Pipeline.Run: %v", err) } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index d6d9dc4ca..183e882d9 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -1,6 +1,8 @@ package pipeline import ( + "context" + "errors" "os" "path/filepath" "testing" @@ -64,7 +66,7 @@ func TestPipelineRun(t *testing.T) { } defer s.Close() - p := New(s, repoDir) + p := New(context.Background(), s, repoDir) if err := p.Run(); err != nil { t.Fatalf("Pipeline.Run: %v", err) } @@ -148,7 +150,7 @@ class DataProcessor: } defer s.Close() - p := New(s, dir) + p := New(context.Background(), s, dir) if err := p.Run(); err != nil { t.Fatalf("Pipeline.Run: %v", err) } @@ -203,7 +205,7 @@ func run() { } defer s.Close() - p := New(s, dir) + p := New(context.Background(), s, dir) if err := p.Run(); err != nil { t.Fatalf("Pipeline.Run: %v", err) } @@ -267,7 +269,7 @@ def process(): } defer s.Close() - p := New(s, dir) + p := New(context.Background(), s, dir) if err := p.Run(); err != nil { t.Fatalf("Pipeline.Run: %v", err) } @@ -333,7 +335,7 @@ def run(): } defer s.Close() - p := New(s, dir) + p := New(context.Background(), s, dir) if err := p.Run(); err != nil { t.Fatalf("Pipeline.Run: %v", err) } @@ -402,7 +404,7 @@ type ID = string } defer s.Close() - p := New(s, dir) + p := New(context.Background(), s, dir) if err := p.Run(); err != nil { t.Fatalf("Pipeline.Run: %v", err) } @@ -471,7 +473,7 @@ type ( } defer s.Close() - p := New(s, dir) + p := New(context.Background(), s, dir) if err := p.Run(); err != nil { t.Fatalf("Pipeline.Run: %v", err) } @@ -530,7 +532,7 @@ object Config { } defer s.Close() - p := New(s, dir) + p := New(context.Background(), s, dir) if err := p.Run(); err != nil { t.Fatalf("Pipeline.Run: %v", err) } @@ -574,6 +576,153 @@ object Config { } } +// TestLuaAnonymousFunctionExtraction verifies that MoonScript-style Lua code +// (local f = function(...) end) extracts Function nodes correctly. +func TestLuaAnonymousFunctionExtraction(t *testing.T) { + dir, err := os.MkdirTemp("", "cgm-lua-anon-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + writeFile(t, filepath.Join(dir, "app.lua"), `local run_before_filter +run_before_filter = function(filter, r) + return filter(r) +end + +local validate = function(data) + return data ~= nil +end + +function named_func(x) + return x + 1 +end +`) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + funcs, _ := s.FindNodesByLabel(p.ProjectName, "Function") + t.Logf("Lua functions: %d", len(funcs)) + for _, f := range funcs { + t.Logf(" %s (qn=%s)", f.Name, f.QualifiedName) + } + + // Expect: run_before_filter, validate, named_func (3 functions) + if len(funcs) < 3 { + t.Errorf("expected at least 3 functions (incl. anonymous assigned), got %d", len(funcs)) + } + + // Verify specific functions exist + for _, name := range []string{"run_before_filter", "validate", "named_func"} { + found, _ := s.FindNodesByName(p.ProjectName, name) + if len(found) == 0 { + t.Errorf("function %q not found", name) + } + } +} + +// TestCSharpModernFeatures verifies that modern C# files (file-scoped namespaces, +// primary constructors, expression-bodied members) are fully extracted. +func TestCSharpModernFeatures(t *testing.T) { + dir, err := os.MkdirTemp("", "cgm-csharp-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + writeFile(t, filepath.Join(dir, "Controller.cs"), `namespace Conduit.Features; + +public class UsersController { + public void Get() {} + public void Create(string name) {} +} +`) + + writeFile(t, filepath.Join(dir, "Model.cs"), `namespace Conduit.Models { + class User { + public string Name { get; set; } + public int GetAge() { return 0; } + } +} +`) + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + modules, _ := s.FindNodesByLabel(p.ProjectName, "Module") + t.Logf("C# modules: %d", len(modules)) + if len(modules) < 2 { + t.Errorf("expected at least 2 Module nodes for .cs files, got %d", len(modules)) + } + + classes, _ := s.FindNodesByLabel(p.ProjectName, "Class") + t.Logf("C# classes: %d", len(classes)) + if len(classes) < 2 { + t.Errorf("expected at least 2 Class nodes, got %d", len(classes)) + } + + methods, _ := s.FindNodesByLabel(p.ProjectName, "Method") + t.Logf("C# methods: %d", len(methods)) + if len(methods) < 3 { // Get, Create, GetAge + t.Errorf("expected at least 3 Method nodes, got %d", len(methods)) + } +} + +// TestBOMStripping verifies that files with UTF-8 BOM are parsed correctly. +func TestBOMStripping(t *testing.T) { + dir, err := os.MkdirTemp("", "cgm-bom-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // Write a Go file WITH UTF-8 BOM prefix + bom := []byte{0xEF, 0xBB, 0xBF} + src := []byte("package main\n\nfunc BOMFunc() {}\n") + content := make([]byte, 0, len(bom)+len(src)) + content = append(content, bom...) + content = append(content, src...) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "bom.go"), content, 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(context.Background(), s, dir) + if err := p.Run(); err != nil { + t.Fatalf("Pipeline.Run: %v", err) + } + + found, _ := s.FindNodesByName(p.ProjectName, "BOMFunc") + if len(found) == 0 { + t.Error("BOMFunc not found — BOM stripping may have failed") + } +} + // TestFunctionRegistry tests the registry in isolation. func TestFunctionRegistry(t *testing.T) { r := NewFunctionRegistry() @@ -614,3 +763,49 @@ func TestFunctionRegistry(t *testing.T) { t.Errorf("expected proj.pkg.Bar, got %s", qn) } } + +// TestPipelineRunCancellation verifies that a pre-cancelled context makes Run() return context.Canceled. +func TestPipelineRunCancellation(t *testing.T) { + repoDir, cleanup := setupTestRepo(t) + defer cleanup() + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // pre-cancel + + p := New(ctx, s, repoDir) + err = p.Run() + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +// TestProjectNameFromPath verifies unique project names from absolute paths. +func TestProjectNameFromPath(t *testing.T) { + tests := []struct{ path, want string }{ + {"/tmp/bench/erlang/lib/stdlib/src", "tmp-bench-erlang-lib-stdlib-src"}, + {"/Users/martin/projects/myapp", "Users-martin-projects-myapp"}, + {"/home/user/repo", "home-user-repo"}, + {"/single", "single"}, + } + for _, tt := range tests { + got := ProjectNameFromPath(tt.path) + if got != tt.want { + t.Errorf("ProjectNameFromPath(%q) = %q, want %q", tt.path, got, tt.want) + } + } +} + +// TestProjectNameUniqueness verifies two paths with same base name produce different project names. +func TestProjectNameUniqueness(t *testing.T) { + a := ProjectNameFromPath("/tmp/bench/zig/lib/std") + b := ProjectNameFromPath("/tmp/bench/erlang/lib/stdlib/src") + if a == b { + t.Errorf("collision: %q == %q", a, b) + } +} diff --git a/internal/pipeline/readwrite.go b/internal/pipeline/readwrite.go index 809b59009..b22259244 100644 --- a/internal/pipeline/readwrite.go +++ b/internal/pipeline/readwrite.go @@ -26,6 +26,21 @@ func extractVariables( } varTypes := toSet(spec.VariableNodeTypes) + // Build a set of QNs already claimed by Function/Method nodes to avoid + // overwriting them with Variable nodes (e.g., Lua anonymous function assignments, + // JS/TS const arrow function assignments). + var funcQNs map[string]bool + switch f.Language { + case lang.Lua, lang.JavaScript, lang.TypeScript, lang.TSX, lang.R, + lang.Haskell, lang.OCaml: + funcQNs = make(map[string]bool, len(result.Nodes)) + for _, n := range result.Nodes { + if n.Label == "Function" || n.Label == "Method" { + funcQNs[n.QualifiedName] = true + } + } + } + parser.Walk(root, func(node *tree_sitter.Node) bool { if !isModuleLevelNode(node, f.Language) || !varTypes[node.Kind()] { return true @@ -37,6 +52,9 @@ func extractVariables( continue } varQN := fqn.Compute(projectName, f.RelPath, name) + if funcQNs != nil && funcQNs[varQN] { + continue // already extracted as Function/Method (Lua anonymous function) + } startLine := safeRowToLine(node.StartPosition().Row) endLine := safeRowToLine(node.EndPosition().Row) @@ -60,7 +78,38 @@ func extractVariables( }) } -// isModuleLevelNode checks if a node is at the module/file top level. +// moduleLevelParentKinds maps languages to their simple top-level parent kinds. +// Languages with more complex logic are handled in isModuleLevelNode directly. +var moduleLevelParentKinds = map[lang.Language][]string{ + lang.Go: {"source_file"}, + lang.Rust: {"source_file"}, + lang.CPP: {"translation_unit"}, + lang.PHP: {"program"}, + lang.Kotlin: {"source_file"}, + lang.Java: {"class_body"}, + lang.CSharp: {"declaration_list"}, + lang.Scala: {"template_body"}, + lang.Ruby: {"program"}, + lang.C: {"translation_unit"}, + lang.ObjectiveC: {"translation_unit"}, + lang.Bash: {"program"}, + lang.Zig: {"source_file"}, + lang.R: {"program"}, + lang.Elixir: {"source"}, + lang.Haskell: {"declarations"}, + lang.OCaml: {"compilation_unit"}, + lang.Groovy: {"source_file"}, + lang.Dart: {"program"}, + lang.SQL: {"statement"}, + lang.Erlang: {"source_file"}, + lang.Swift: {"source_file"}, + lang.HCL: {"body"}, + lang.SCSS: {"stylesheet"}, + lang.Perl: {"source_file", "assignment_expression", "expression_statement"}, +} + +// isModuleLevelNode checks if a node is at the module/file top level +// or (for class-based languages) at the class body level. func isModuleLevelNode(node *tree_sitter.Node, language lang.Language) bool { parent := node.Parent() if parent == nil { @@ -68,44 +117,289 @@ func isModuleLevelNode(node *tree_sitter.Node, language lang.Language) bool { } parentKind := parent.Kind() + // Languages with custom grandparent logic switch language { - case lang.Go, lang.Rust: - return parentKind == "source_file" case lang.Python: - if parentKind == "module" { - return true - } - // Python wraps assignments: module → expression_statement → assignment - if parentKind == "expression_statement" { - if gp := parent.Parent(); gp != nil { - return gp.Kind() == "module" - } - } + return isModuleLevelPython(parentKind, parent) case lang.JavaScript, lang.TypeScript, lang.TSX: - if parentKind == "program" { - return true - } - // JS/TS wraps exports: program → export_statement → declaration - if parentKind == "export_statement" { - if gp := parent.Parent(); gp != nil { - return gp.Kind() == "program" + return isModuleLevelJS(parentKind, parent) + case lang.Lua: + return isModuleLevelLua(parentKind, parent) + case lang.YAML: + return isModuleLevelYAML(parentKind, parent) + } + + // Simple parent-kind lookup + if kinds, ok := moduleLevelParentKinds[language]; ok { + for _, k := range kinds { + if parentKind == k { + return true } } } return false } +func isModuleLevelPython(parentKind string, parent *tree_sitter.Node) bool { + if parentKind == "module" { + return true + } + // Python wraps assignments: module → expression_statement → assignment + if parentKind == "expression_statement" { + if gp := parent.Parent(); gp != nil { + return gp.Kind() == "module" + } + } + return false +} + +func isModuleLevelJS(parentKind string, parent *tree_sitter.Node) bool { + if parentKind == "program" { + return true + } + // JS/TS wraps exports: program → export_statement → declaration + if parentKind == "export_statement" { + if gp := parent.Parent(); gp != nil { + return gp.Kind() == "program" + } + } + return false +} + +func isModuleLevelLua(parentKind string, parent *tree_sitter.Node) bool { + if parentKind == "chunk" { + return true + } + if parentKind == "block" { + if gp := parent.Parent(); gp != nil { + return gp.Kind() == "chunk" + } + } + return false +} + +func isModuleLevelYAML(parentKind string, parent *tree_sitter.Node) bool { + if parentKind != "block_mapping" { + return false + } + gp := parent.Parent() + if gp == nil || gp.Kind() != "block_node" { + return false + } + ggp := gp.Parent() + return ggp != nil && ggp.GrammarName() == "_bgn_imp_doc" +} + +// varNameExtractor is a function type for per-language variable name extraction. +type varNameExtractor func(node *tree_sitter.Node, source []byte) []string + +// varNameExtractors maps each language to its variable name extractor. +var varNameExtractors = map[lang.Language]varNameExtractor{ + lang.Go: extractGoVarNames, + lang.Python: extractPythonVarNames, + lang.JavaScript: extractJSVarNames, + lang.TypeScript: extractJSVarNames, + lang.TSX: extractJSVarNames, + lang.Rust: extractRustVarNames, + lang.Java: extractJavaCSharpVarNames, + lang.CSharp: extractJavaCSharpVarNames, + lang.CPP: extractCPPVarNames, + lang.PHP: extractPHPVarNames, + lang.Lua: extractLuaVarNames, + lang.Scala: extractScalaVarNames, + lang.Kotlin: extractKotlinVarNames, + lang.Ruby: extractRubyVarNames, + lang.C: extractCVarNames, + lang.ObjectiveC: extractCVarNames, + lang.Bash: extractBashVarNames, + lang.Zig: extractZigVarNames, + lang.R: extractRVarNames, + lang.Elixir: extractElixirVarNames, + lang.Haskell: extractHaskellVarNames, + lang.OCaml: extractOCamlVarNames, + lang.Groovy: extractGroovyVarNames, + lang.Dart: extractDartVarNames, + lang.Perl: extractPerlVarNames, + lang.Swift: extractSwiftVarNames, + lang.SCSS: extractSCSSVarNames, + lang.HCL: extractHCLVarNames, + lang.Erlang: extractErlangVarNames, + lang.SQL: extractSQLVarNames, + lang.YAML: extractYAMLVarNames, +} + // extractVarNames extracts variable names from a declaration node. func extractVarNames(node *tree_sitter.Node, source []byte, language lang.Language) []string { - switch language { - case lang.Go: - return extractGoVarNames(node, source) - case lang.Python: - return extractPythonVarNames(node, source) - case lang.JavaScript, lang.TypeScript, lang.TSX: - return extractJSVarNames(node, source) - case lang.Rust: - return extractRustVarNames(node, source) + if fn, ok := varNameExtractors[language]; ok { + return fn(node, source) + } + return nil +} + +// extractErlangVarNames extracts names from Erlang -define() and -record() declarations. +// AST: pp_define → macro_lhs[field="lhs"] → var[field="name"] +// AST: record_decl → atom[field="name"] +func extractErlangVarNames(node *tree_sitter.Node, source []byte) []string { + switch node.Kind() { + case "pp_define": + if lhs := node.ChildByFieldName("lhs"); lhs != nil { + if nameNode := lhs.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + } + case "record_decl": + if nameNode := node.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + } + return nil +} + +// extractSQLVarNames extracts table/view names from SQL CREATE TABLE/VIEW statements. +// AST: create_table → object_reference → identifier[field="name"] +// AST: create_view → object_reference → identifier[field="name"] +func extractSQLVarNames(node *tree_sitter.Node, source []byte) []string { + // Find the first direct object_reference child (contains the table/view name) + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child != nil && child.Kind() == "object_reference" { + if nameNode := child.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + // Fallback: text of the object_reference itself + return []string{parser.NodeText(child, source)} + } + } + return nil +} + +// extractYAMLVarNames extracts top-level key names from YAML block_mapping_pair nodes. +// AST: block_mapping_pair → key field +func extractYAMLVarNames(node *tree_sitter.Node, source []byte) []string { + if key := node.ChildByFieldName("key"); key != nil { + return []string{parser.NodeText(key, source)} + } + return nil +} + +// extractRVarNames extracts names from R left_assignment / right_assignment nodes. +func extractRVarNames(node *tree_sitter.Node, source []byte) []string { + // Only extract assignments (<-, ->, =), not arithmetic/comparison operators + if op := node.ChildByFieldName("operator"); op != nil { + opText := parser.NodeText(op, source) + if opText != "<-" && opText != "<<-" && opText != "->" && opText != "->>" && opText != "=" { + return nil + } + } + if lhs := node.ChildByFieldName("lhs"); lhs != nil { + name := parser.NodeText(lhs, source) + if name != "" { + return []string{name} + } + } + return nil +} + +func extractElixirVarNames(node *tree_sitter.Node, source []byte) []string { + // Elixir: binary_operator with = — lhs is the variable name + if lhs := node.ChildByFieldName("left"); lhs != nil && lhs.Kind() == "identifier" { + return []string{parser.NodeText(lhs, source)} + } + return nil +} + +func extractHaskellVarNames(node *tree_sitter.Node, source []byte) []string { + if nameNode := node.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + return nil +} + +func extractOCamlVarNames(node *tree_sitter.Node, source []byte) []string { + // value_definition → let_binding (child) → pattern field = name + lb := findChildByKind(node, "let_binding") + if lb == nil { + lb = node // let_binding directly + } + if pat := lb.ChildByFieldName("pattern"); pat != nil { + return []string{parser.NodeText(pat, source)} + } + return nil +} + +func extractGroovyVarNames(node *tree_sitter.Node, source []byte) []string { + if nameNode := node.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + // Groovy variable_declaration: first identifier child + if id := findChildByKind(node, "identifier"); id != nil { + return []string{parser.NodeText(id, source)} + } + return nil +} + +func extractDartVarNames(node *tree_sitter.Node, source []byte) []string { + if nameNode := node.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + // Dart: initialized_identifier_list → initialized_identifier → identifier + var names []string + parser.Walk(node, func(child *tree_sitter.Node) bool { + if child.Kind() == "identifier" { + names = append(names, parser.NodeText(child, source)) + return false + } + return true + }) + return names +} + +func extractPerlVarNames(node *tree_sitter.Node, source []byte) []string { + // Perl variable_declaration: my $name = ... or my ($a, $b) = ... + var names []string + parser.Walk(node, func(child *tree_sitter.Node) bool { + if child.Kind() == "scalar" { + for i := uint(0); i < child.ChildCount(); i++ { + c := child.Child(i) + if c != nil && c.Kind() == "varname" { + names = append(names, parser.NodeText(c, source)) + } + } + return false + } + return true + }) + return names +} + +func extractSwiftVarNames(node *tree_sitter.Node, source []byte) []string { + if nameNode := node.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + // Swift: pattern → simple_identifier + if pat := findChildByKind(node, "pattern"); pat != nil { + if id := findChildByKind(pat, "simple_identifier"); id != nil { + return []string{parser.NodeText(id, source)} + } + } + return nil +} + +func extractSCSSVarNames(node *tree_sitter.Node, source []byte) []string { + // SCSS: declaration → property_name (grammar: variable) → "$name" + for i := uint(0); i < node.ChildCount(); i++ { + child := node.Child(i) + if child != nil && child.GrammarName() == "variable" { + return []string{parser.NodeText(child, source)} + } + } + return nil +} + +func extractHCLVarNames(node *tree_sitter.Node, source []byte) []string { + // HCL: attribute → identifier = expression + if id := findChildByKind(node, "identifier"); id != nil { + return []string{parser.NodeText(id, source)} } return nil } @@ -159,6 +453,182 @@ func extractRustVarNames(node *tree_sitter.Node, source []byte) []string { return nil } +// extractJavaCSharpVarNames extracts variable names from field_declaration nodes. +// AST: field_declaration → variable_declarator[declarator] → identifier[name] +func extractJavaCSharpVarNames(node *tree_sitter.Node, source []byte) []string { + var names []string + parser.Walk(node, func(child *tree_sitter.Node) bool { + if child.Kind() == "variable_declarator" { + if nameNode := child.ChildByFieldName("name"); nameNode != nil { + names = append(names, parser.NodeText(nameNode, source)) + } + return false + } + return true + }) + return names +} + +// extractCPPVarNames extracts variable names from C++ declaration nodes. +// AST: declaration → init_declarator[declarator] → identifier[declarator] +func extractCPPVarNames(node *tree_sitter.Node, source []byte) []string { + var names []string + parser.Walk(node, func(child *tree_sitter.Node) bool { + if child.Kind() == "init_declarator" { + if declNode := child.ChildByFieldName("declarator"); declNode != nil { + if declNode.Kind() == "identifier" { + names = append(names, parser.NodeText(declNode, source)) + } + } + return false + } + return true + }) + return names +} + +// extractPHPVarNames extracts variable names from PHP expression_statement assignments. +// AST: expression_statement → assignment_expression → variable_name[left] +func extractPHPVarNames(node *tree_sitter.Node, source []byte) []string { + assign := findChildByKind(node, "assignment_expression") + if assign == nil { + return nil + } + leftNode := assign.ChildByFieldName("left") + if leftNode == nil { + return nil + } + name := parser.NodeText(leftNode, source) + // Strip $ prefix from PHP variable names + name = strings.TrimPrefix(name, "$") + if name == "" { + return nil + } + return []string{name} +} + +// extractLuaVarNames extracts variable names from Lua variable_declaration. +// AST: variable_declaration → assignment_statement → variable_list → identifier +func extractLuaVarNames(node *tree_sitter.Node, source []byte) []string { + var names []string + parser.Walk(node, func(child *tree_sitter.Node) bool { + if child.Kind() == "identifier" { + // Only grab identifiers in the variable_list (left side), not values + if p := child.Parent(); p != nil && (p.Kind() == "variable_list" || p.Kind() == "variable_declaration") { + names = append(names, parser.NodeText(child, source)) + return false + } + } + return true + }) + return names +} + +// extractScalaVarNames extracts variable names from Scala val/var definitions. +// AST: val_definition → identifier[name] +func extractScalaVarNames(node *tree_sitter.Node, source []byte) []string { + if nameNode := node.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + // Fallback: find first identifier child + if nameNode := findChildByKind(node, "identifier"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + return nil +} + +// extractKotlinVarNames extracts variable names from Kotlin property_declaration. +// AST: property_declaration → variable_declaration → identifier +func extractKotlinVarNames(node *tree_sitter.Node, source []byte) []string { + if nameNode := node.ChildByFieldName("name"); nameNode != nil { + return []string{parser.NodeText(nameNode, source)} + } + // Kotlin: property_declaration → variable_declaration → identifier + for i := uint(0); i < node.NamedChildCount(); i++ { + child := node.NamedChild(i) + if child == nil { + continue + } + if child.Kind() == "variable_declaration" { + if id := findChildByKind(child, "identifier"); id != nil { + return []string{parser.NodeText(id, source)} + } + } + if child.Kind() == "simple_identifier" || child.Kind() == "identifier" { + return []string{parser.NodeText(child, source)} + } + } + return nil +} + +// extractRubyVarNames extracts variable names from Ruby assignment nodes. +// AST: assignment → constant[left="API_URL"] +func extractRubyVarNames(node *tree_sitter.Node, source []byte) []string { + leftNode := node.ChildByFieldName("left") + if leftNode == nil { + return nil + } + name := parser.NodeText(leftNode, source) + if name == "" { + return nil + } + return []string{name} +} + +// extractCVarNames extracts variable names from C declaration nodes. +// AST: declaration → init_declarator[declarator] → identifier or pointer_declarator → identifier +func extractCVarNames(node *tree_sitter.Node, source []byte) []string { + var names []string + parser.Walk(node, func(child *tree_sitter.Node) bool { + kind := child.Kind() + if kind != "init_declarator" && kind != "declaration" { + return true + } + if name := extractCDeclaratorName(child, source); name != "" { + names = append(names, name) + return false + } + return true + }) + return names +} + +// extractCDeclaratorName extracts the identifier name from a C declarator node. +func extractCDeclaratorName(node *tree_sitter.Node, source []byte) string { + declNode := node.ChildByFieldName("declarator") + if declNode == nil { + return "" + } + if declNode.Kind() == "pointer_declarator" { + if id := declNode.ChildByFieldName("declarator"); id != nil && id.Kind() == "identifier" { + return parser.NodeText(id, source) + } + } + if declNode.Kind() == "identifier" { + return parser.NodeText(declNode, source) + } + return "" +} + +// extractBashVarNames extracts variable names from Bash variable_assignment. +// AST: variable_assignment → variable_name[name] +func extractBashVarNames(node *tree_sitter.Node, source []byte) []string { + nameNode := node.ChildByFieldName("name") + if nameNode == nil { + return nil + } + return []string{parser.NodeText(nameNode, source)} +} + +// extractZigVarNames extracts variable names from Zig variable_declaration. +// AST: variable_declaration → identifier (no field name, just the first identifier child) +func extractZigVarNames(node *tree_sitter.Node, source []byte) []string { + if id := findChildByKind(node, "identifier"); id != nil { + return []string{parser.NodeText(id, source)} + } + return nil +} + // passReadsWrites creates READS and WRITES edges from functions to Variable nodes. func (p *Pipeline) passReadsWrites() { slog.Info("pass.readwrite") diff --git a/internal/pipeline/resolve.go b/internal/pipeline/resolve.go index b57d6c6ce..b8d6bcb6f 100644 --- a/internal/pipeline/resolve.go +++ b/internal/pipeline/resolve.go @@ -61,6 +61,8 @@ func resolveAssignment(node *tree_sitter.Node, source []byte, language lang.Lang return resolveCPP(node, source, symbols) case lang.Lua: return resolveLua(node, source, symbols) + case lang.CSharp: + return resolveCSharp(node, source, symbols) default: return "", "" } @@ -203,6 +205,75 @@ func resolveJava(node *tree_sitter.Node, source []byte, symbols map[string]strin return "", "" // all collected via symbols map directly } +// --- C# --- +// class_declaration → field_declaration → variable_declarator → (identifier, binary_expression|string_literal) + +func resolveCSharp(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { + if node.Kind() != "class_declaration" { + return "", "" + } + body := node.ChildByFieldName("body") + if body == nil { + return "", "" + } + // Walk field_declarations inside the class body + for i := uint(0); i < body.ChildCount(); i++ { + child := body.Child(i) + if child == nil || child.Kind() != "field_declaration" { + continue + } + resolveCSharpFieldDecl(child, source, symbols) + } + return "", "" +} + +// resolveCSharpFieldDecl resolves a single C# field_declaration into the symbols map. +func resolveCSharpFieldDecl(field *tree_sitter.Node, source []byte, symbols map[string]string) { + varDecl := findChildByKind(field, "variable_declaration") + if varDecl == nil { + return + } + decl := findChildByKind(varDecl, "variable_declarator") + if decl == nil { + return + } + nameNode := decl.ChildByFieldName("name") + if nameNode == nil { + nameNode = findChildByKind(decl, "identifier") + } + if nameNode == nil { + return + } + valueNode := findValueAfterEquals(decl) + if valueNode == nil { + return + } + name := parser.NodeText(nameNode, source) + value := resolveStringExpr(valueNode, source, symbols) + if name != "" && value != "" { + symbols[name] = value + } +} + +// findValueAfterEquals finds the first named child after a "=" token in a declarator. +func findValueAfterEquals(decl *tree_sitter.Node) *tree_sitter.Node { + foundEq := false + for ci := uint(0); ci < decl.ChildCount(); ci++ { + c := decl.Child(ci) + if c == nil { + continue + } + if c.Kind() == "=" { + foundEq = true + continue + } + if foundEq && c.IsNamed() { + return c + } + } + return nil +} + // --- PHP --- // expression_statement → assignment_expression → (variable_name, encapsed_string|binary_expression) diff --git a/internal/pipeline/resolve_test.go b/internal/pipeline/resolve_test.go index 2be9301e7..e01fa261d 100644 --- a/internal/pipeline/resolve_test.go +++ b/internal/pipeline/resolve_test.go @@ -4,47 +4,16 @@ import ( "testing" tree_sitter "github.com/tree-sitter/go-tree-sitter" - tree_sitter_go "github.com/tree-sitter/tree-sitter-go/bindings/go" - tree_sitter_java "github.com/tree-sitter/tree-sitter-java/bindings/go" - tree_sitter_javascript "github.com/tree-sitter/tree-sitter-javascript/bindings/go" - tree_sitter_php "github.com/tree-sitter/tree-sitter-php/bindings/go" - tree_sitter_python "github.com/tree-sitter/tree-sitter-python/bindings/go" - tree_sitter_rust "github.com/tree-sitter/tree-sitter-rust/bindings/go" - tree_sitter_scala "github.com/tree-sitter/tree-sitter-scala/bindings/go" - - tree_sitter_kotlin "github.com/tree-sitter-grammars/tree-sitter-kotlin/bindings/go" - tree_sitter_lua "github.com/tree-sitter-grammars/tree-sitter-lua/bindings/go" - tree_sitter_cpp "github.com/tree-sitter/tree-sitter-cpp/bindings/go" "github.com/DeusData/codebase-memory-mcp/internal/lang" + "github.com/DeusData/codebase-memory-mcp/internal/parser" ) func parseSource(t *testing.T, language lang.Language, code string) (tree *tree_sitter.Tree, source []byte) { t.Helper() - var tsLang *tree_sitter.Language - switch language { - case lang.Python: - tsLang = tree_sitter.NewLanguage(tree_sitter_python.Language()) - case lang.Go: - tsLang = tree_sitter.NewLanguage(tree_sitter_go.Language()) - case lang.JavaScript: - tsLang = tree_sitter.NewLanguage(tree_sitter_javascript.Language()) - case lang.Rust: - tsLang = tree_sitter.NewLanguage(tree_sitter_rust.Language()) - case lang.Java: - tsLang = tree_sitter.NewLanguage(tree_sitter_java.Language()) - case lang.PHP: - tsLang = tree_sitter.NewLanguage(tree_sitter_php.LanguagePHPOnly()) - case lang.Scala: - tsLang = tree_sitter.NewLanguage(tree_sitter_scala.Language()) - case lang.CPP: - tsLang = tree_sitter.NewLanguage(tree_sitter_cpp.Language()) - case lang.Lua: - tsLang = tree_sitter.NewLanguage(tree_sitter_lua.Language()) - case lang.Kotlin: - tsLang = tree_sitter.NewLanguage(tree_sitter_kotlin.Language()) - default: + tsLang, err := parser.GetLanguage(language) + if err != nil { t.Fatalf("unsupported language: %s", language) } @@ -220,6 +189,20 @@ val endpoint = base + "/orders" assertSymbol(t, symbols, "endpoint", "https://api.example.com/v1/orders") } +func TestResolveCSharpConcat(t *testing.T) { + code := `class Config { + const string BASE_URL = "https://example.com"; + const string URL = BASE_URL + "/api/orders"; +}` + tree, source := parseSource(t, lang.CSharp, code) + defer tree.Close() + + symbols := resolveModuleStrings(tree.RootNode(), source, lang.CSharp) + + assertSymbol(t, symbols, "BASE_URL", "https://example.com") + assertSymbol(t, symbols, "URL", "https://example.com/api/orders") +} + func TestResolveUnknownVariable(t *testing.T) { // When a variable can't be resolved, it should emit {} code := `URL = f"{UNKNOWN_VAR}/api/orders" diff --git a/internal/pipeline/testdetect.go b/internal/pipeline/testdetect.go index 831360935..3e1652639 100644 --- a/internal/pipeline/testdetect.go +++ b/internal/pipeline/testdetect.go @@ -7,69 +7,103 @@ import ( "github.com/DeusData/codebase-memory-mcp/internal/lang" ) +// testFilePattern defines how to detect test files for a language. +type testFilePattern struct { + // suffixes on the base filename (e.g., "_test.go") + suffixes []string + // prefixes on the base filename (e.g., "test_") + prefixes []string + // stripExtSuffixes: suffixes checked on the base name after stripping ext (e.g., ".test", ".spec") + stripExtSuffixes []string + // testDirs: directory patterns that indicate test files + testDirs []string +} + +// testFilePatterns maps languages to their test file detection patterns. +var testFilePatterns = map[lang.Language]testFilePattern{ + lang.Go: {suffixes: []string{"_test.go"}}, + lang.Python: { + prefixes: []string{"test_"}, + suffixes: []string{"_test.py"}, + testDirs: []string{"__tests__", "tests"}, + }, + lang.JavaScript: { + stripExtSuffixes: []string{".test", ".spec"}, + testDirs: []string{"__tests__"}, + }, + lang.TypeScript: { + stripExtSuffixes: []string{".test", ".spec"}, + testDirs: []string{"__tests__"}, + }, + lang.TSX: { + stripExtSuffixes: []string{".test", ".spec"}, + testDirs: []string{"__tests__"}, + }, + lang.Java: { + suffixes: []string{"Test.java", "Tests.java"}, + testDirs: []string{"src/test"}, + }, + lang.Rust: { + suffixes: []string{"_test.rs"}, + testDirs: []string{"tests"}, + }, + lang.CPP: { + stripExtSuffixes: []string{"_test"}, + testDirs: []string{"test", "tests"}, + }, + lang.PHP: { + suffixes: []string{"Test.php"}, + testDirs: []string{"tests"}, + }, + lang.Scala: { + stripExtSuffixes: []string{"Spec", "Test"}, + testDirs: []string{"src/test"}, + }, + lang.CSharp: { + stripExtSuffixes: []string{"Test", "Tests"}, + testDirs: []string{"Tests", "tests"}, + }, + lang.Kotlin: { + stripExtSuffixes: []string{"Test", "Tests", "Spec"}, + testDirs: []string{"src/test"}, + }, + lang.Lua: { + suffixes: []string{"_test.lua", "_spec.lua"}, + prefixes: []string{"test_"}, + testDirs: []string{"spec"}, + }, +} + // isTestFile returns true if the file path indicates a test file for the given language. func isTestFile(relPath string, language lang.Language) bool { - base := filepath.Base(relPath) - dir := filepath.Dir(relPath) - - switch language { - case lang.Go: - return strings.HasSuffix(base, "_test.go") - - case lang.Python: - if strings.HasPrefix(base, "test_") || strings.HasSuffix(base, "_test.py") { - return true - } - return containsTestDir(dir, "__tests__", "tests") - - case lang.JavaScript, lang.TypeScript, lang.TSX: - noExt := strings.TrimSuffix(base, filepath.Ext(base)) - // Handle double-extensions like .test.ts, .spec.tsx - if strings.HasSuffix(noExt, ".test") || strings.HasSuffix(noExt, ".spec") { - return true - } - return containsTestDir(dir, "__tests__") - - case lang.Java: - if strings.HasSuffix(base, "Test.java") || strings.HasSuffix(base, "Tests.java") { - return true - } - return containsTestDir(dir, "src/test") - - case lang.Rust: - if strings.HasSuffix(base, "_test.rs") { - return true - } - return containsTestDir(dir, "tests") - - case lang.CPP: - noExt := strings.TrimSuffix(base, filepath.Ext(base)) - if strings.HasSuffix(noExt, "_test") { - return true - } - return containsTestDir(dir, "test", "tests") - - case lang.PHP: - if strings.HasSuffix(base, "Test.php") { - return true - } - return containsTestDir(dir, "tests") - - case lang.Scala: - noExt := strings.TrimSuffix(base, filepath.Ext(base)) - if strings.HasSuffix(noExt, "Spec") || strings.HasSuffix(noExt, "Test") { - return true - } - return containsTestDir(dir, "src/test") - - case lang.CSharp: - noExt := strings.TrimSuffix(base, filepath.Ext(base)) - if strings.HasSuffix(noExt, "Test") || strings.HasSuffix(noExt, "Tests") { - return true - } - return containsTestDir(dir, "Tests", "tests") + pattern, ok := testFilePatterns[language] + if !ok { + return false } + base := filepath.Base(relPath) + + for _, s := range pattern.suffixes { + if strings.HasSuffix(base, s) { + return true + } + } + for _, p := range pattern.prefixes { + if strings.HasPrefix(base, p) { + return true + } + } + if len(pattern.stripExtSuffixes) > 0 { + noExt := strings.TrimSuffix(base, filepath.Ext(base)) + for _, s := range pattern.stripExtSuffixes { + if strings.HasSuffix(noExt, s) { + return true + } + } + } + if len(pattern.testDirs) > 0 { + return containsTestDir(filepath.Dir(relPath), pattern.testDirs...) + } return false } @@ -114,6 +148,10 @@ func isTestFunction(funcName string, language lang.Language) bool { return strings.HasPrefix(funcName, "test_") || strings.HasPrefix(funcName, "Test") + case lang.CPP: + return strings.HasPrefix(funcName, "Test") || + strings.HasPrefix(funcName, "test_") + case lang.PHP: return strings.HasPrefix(funcName, "test") || strings.HasPrefix(funcName, "Test") @@ -125,6 +163,14 @@ func isTestFunction(funcName string, language lang.Language) bool { case lang.CSharp: return strings.HasPrefix(funcName, "Test") || strings.HasSuffix(funcName, "Test") + + case lang.Kotlin: + return strings.HasPrefix(funcName, "test") || + strings.HasSuffix(funcName, "Test") + + case lang.Lua: + return strings.HasPrefix(funcName, "test_") || + strings.HasPrefix(funcName, "test") } return false diff --git a/internal/pipeline/testdetect_test.go b/internal/pipeline/testdetect_test.go new file mode 100644 index 000000000..432a85e9e --- /dev/null +++ b/internal/pipeline/testdetect_test.go @@ -0,0 +1,73 @@ +package pipeline + +import ( + "testing" + + "github.com/DeusData/codebase-memory-mcp/internal/lang" +) + +func TestIsTestFileAllLanguages(t *testing.T) { + tests := []struct { + name string + language lang.Language + testPath string // should return true + srcPath string // should return false + }{ + {"Go", lang.Go, "foo_test.go", "foo.go"}, + {"Python", lang.Python, "test_handler.py", "handler.py"}, + {"JavaScript", lang.JavaScript, "handler.test.js", "handler.js"}, + {"TypeScript", lang.TypeScript, "handler.spec.ts", "handler.ts"}, + {"TSX", lang.TSX, "Component.test.tsx", "Component.tsx"}, + {"Java", lang.Java, "OrderTest.java", "Order.java"}, + {"Rust", lang.Rust, "handler_test.rs", "handler.rs"}, + {"CPP", lang.CPP, "handler_test.cpp", "handler.cpp"}, + {"CSharp", lang.CSharp, "OrderTest.cs", "Order.cs"}, + {"PHP", lang.PHP, "OrderTest.php", "Order.php"}, + {"Scala", lang.Scala, "OrderSpec.scala", "Order.scala"}, + {"Kotlin", lang.Kotlin, "OrderTest.kt", "Order.kt"}, + {"Lua", lang.Lua, "handler_test.lua", "handler.lua"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !isTestFile(tt.testPath, tt.language) { + t.Errorf("isTestFile(%q, %s) = false, want true", tt.testPath, tt.language) + } + if isTestFile(tt.srcPath, tt.language) { + t.Errorf("isTestFile(%q, %s) = true, want false", tt.srcPath, tt.language) + } + }) + } +} + +func TestIsTestFunctionAllLanguages(t *testing.T) { + tests := []struct { + name string + language lang.Language + testFunc string // should return true + srcFunc string // should return false + }{ + {"Go", lang.Go, "TestCreate", "create"}, + {"Python", lang.Python, "test_create", "create"}, + {"JavaScript", lang.JavaScript, "test", "create"}, + {"TypeScript", lang.TypeScript, "describe", "create"}, + {"TSX", lang.TSX, "it", "create"}, + {"Java", lang.Java, "testCreate", "create"}, + {"Rust", lang.Rust, "test_create", "create"}, + {"CPP", lang.CPP, "TestCreate", "create"}, + {"CSharp", lang.CSharp, "TestCreate", "create"}, + {"PHP", lang.PHP, "testCreate", "create"}, + {"Scala", lang.Scala, "testCreate", "create"}, + {"Kotlin", lang.Kotlin, "testCreate", "create"}, + {"Lua", lang.Lua, "test_create", "create"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !isTestFunction(tt.testFunc, tt.language) { + t.Errorf("isTestFunction(%q, %s) = false, want true", tt.testFunc, tt.language) + } + if isTestFunction(tt.srcFunc, tt.language) { + t.Errorf("isTestFunction(%q, %s) = true, want false", tt.srcFunc, tt.language) + } + }) + } +} diff --git a/internal/pipeline/throws.go b/internal/pipeline/throws.go index bde0b9627..1ae637182 100644 --- a/internal/pipeline/throws.go +++ b/internal/pipeline/throws.go @@ -44,10 +44,13 @@ func (p *Pipeline) passThrows() { numWorkers = len(files) } - g := new(errgroup.Group) + g, gctx := errgroup.WithContext(p.ctx) g.SetLimit(numWorkers) for i, fe := range files { g.Go(func() error { + if gctx.Err() != nil { + return gctx.Err() + } results[i] = p.resolveFileThrows(fe.relPath, fe.cached) return nil }) @@ -70,6 +73,7 @@ func (p *Pipeline) resolveFileThrows(relPath string, cached *cachedAST) []resolv } funcTypes := toSet(spec.FunctionNodeTypes) + classTypes := toSet(spec.ClassNodeTypes) throwTypes := toSet(spec.ThrowNodeTypes) moduleQN := fqn.ModuleQN(p.ProjectName, relPath) importMap := p.importMaps[moduleQN] @@ -82,15 +86,10 @@ func (p *Pipeline) resolveFileThrows(relPath string, cached *cachedAST) []resolv return true } - nameNode := node.ChildByFieldName("name") - if nameNode == nil { - return false - } - funcName := parser.NodeText(nameNode, cached.Source) + funcQN, funcName := computeFuncQN(node, cached.Source, p.ProjectName, relPath, classTypes) if funcName == "" { return false } - funcQN := fqn.Compute(p.ProjectName, relPath, funcName) // Java: declared throws clause edges = append(edges, p.extractDeclaredThrows(node, cached.Source, spec, funcQN, moduleQN, importMap)...) @@ -104,6 +103,36 @@ func (p *Pipeline) resolveFileThrows(relPath string, cached *cachedAST) []resolv return edges } +// computeFuncQN computes the correct qualified name for a function node, +// handling both top-level functions and methods nested inside classes. +// Also handles C++ where the function name is in function_declarator. +func computeFuncQN(node *tree_sitter.Node, source []byte, projectName, relPath string, classTypes map[string]bool) (qn, name string) { + nameNode := funcNameNode(node) + if nameNode == nil { + return "", "" + } + name = parser.NodeText(nameNode, source) + if name == "" { + return "", "" + } + + // Check for enclosing class to compute correct QN + current := node.Parent() + for current != nil { + if classTypes[current.Kind()] { + classNameNode := current.ChildByFieldName("name") + if classNameNode != nil { + className := parser.NodeText(classNameNode, source) + classQN := fqn.Compute(projectName, relPath, className) + return classQN + "." + name, name + } + } + current = current.Parent() + } + + return fqn.Compute(projectName, relPath, name), name +} + // extractDeclaredThrows handles Java's "throws ExceptionA, ExceptionB" clause. func (p *Pipeline) extractDeclaredThrows(node *tree_sitter.Node, source []byte, spec *lang.LanguageSpec, funcQN, moduleQN string, importMap map[string]string) []resolvedEdge { if spec.ThrowsClauseField == "" { @@ -143,6 +172,10 @@ func (p *Pipeline) extractBodyThrows(node *tree_sitter.Node, cached *cachedAST, if bodyNode == nil { bodyNode = node.ChildByFieldName("block") } + if bodyNode == nil { + // Kotlin: function_body has no field name + bodyNode = findChildByKind(node, "function_body") + } if bodyNode == nil { return nil } @@ -189,29 +222,67 @@ func extractThrownExceptionName(throwNode *tree_sitter.Node, source []byte) stri func extractExceptionClassName(child *tree_sitter.Node, source []byte) string { switch child.Kind() { case "new_expression", "object_creation_expression": - if typeNode := child.ChildByFieldName("type"); typeNode != nil { - return cleanTypeName(parser.NodeText(typeNode, source)) - } - if child.NamedChildCount() > 0 { - first := child.NamedChild(0) - if first != nil && (first.Kind() == "identifier" || first.Kind() == "type_identifier") { - return parser.NodeText(first, source) - } - } - + return extractNewExprClassName(child, source) + case "instance_expression": + return extractInstanceExprClassName(child, source) case "call", "call_expression": - funcNode := child.ChildByFieldName("function") - if funcNode == nil { - return "" - } - name := parser.NodeText(funcNode, source) - if idx := strings.LastIndex(name, "."); idx >= 0 { - name = name[idx+1:] - } - return name - - case "identifier", "type_identifier": + return extractCallExprClassName(child, source) + case "identifier", "type_identifier", "name": return parser.NodeText(child, source) } return "" } + +// extractNewExprClassName extracts the class name from a new expression. +func extractNewExprClassName(node *tree_sitter.Node, source []byte) string { + if typeNode := node.ChildByFieldName("type"); typeNode != nil { + return cleanTypeName(parser.NodeText(typeNode, source)) + } + if node.NamedChildCount() == 0 { + return "" + } + first := node.NamedChild(0) + if first == nil { + return "" + } + switch first.Kind() { + case "identifier", "type_identifier", "name": + return parser.NodeText(first, source) + } + return "" +} + +// extractInstanceExprClassName extracts the class name from a Scala instance_expression. +func extractInstanceExprClassName(node *tree_sitter.Node, source []byte) string { + for i := uint(0); i < node.NamedChildCount(); i++ { + n := node.NamedChild(i) + if n != nil && (n.Kind() == "type_identifier" || n.Kind() == "identifier") { + return parser.NodeText(n, source) + } + } + return "" +} + +// extractCallExprClassName extracts the class name from a call/call_expression in throw context. +func extractCallExprClassName(node *tree_sitter.Node, source []byte) string { + funcNode := node.ChildByFieldName("function") + if funcNode == nil { + return extractFirstIdentifierChild(node, source) + } + name := parser.NodeText(funcNode, source) + if idx := strings.LastIndex(name, "."); idx >= 0 { + name = name[idx+1:] + } + return name +} + +// extractFirstIdentifierChild returns the text of the first identifier/simple_identifier child. +func extractFirstIdentifierChild(node *tree_sitter.Node, source []byte) string { + for i := uint(0); i < node.NamedChildCount(); i++ { + n := node.NamedChild(i) + if n != nil && (n.Kind() == "identifier" || n.Kind() == "simple_identifier") { + return parser.NodeText(n, source) + } + } + return "" +} diff --git a/internal/pipeline/usages.go b/internal/pipeline/usages.go index 49c6f9c07..95ac67b4a 100644 --- a/internal/pipeline/usages.go +++ b/internal/pipeline/usages.go @@ -42,10 +42,13 @@ func (p *Pipeline) passUsages() { numWorkers = len(files) } - g := new(errgroup.Group) + g, gctx := errgroup.WithContext(p.ctx) g.SetLimit(numWorkers) for i, fe := range files { g.Go(func() error { + if gctx.Err() != nil { + return gctx.Err() + } results[i] = p.resolveFileUsages(fe.relPath, fe.cached) return nil }) @@ -67,6 +70,9 @@ func (p *Pipeline) passUsagesForFiles(files []discover.FileInfo) { slog.Info("pass3b.usages.incremental", "files", len(files)) count := 0 for _, f := range files { + if p.ctx.Err() != nil { + return + } cached, ok := p.astCache[f.RelPath] if !ok { continue diff --git a/internal/pipeline/usages_test.go b/internal/pipeline/usages_test.go index d643418f7..460c6c0c3 100644 --- a/internal/pipeline/usages_test.go +++ b/internal/pipeline/usages_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "context" "os" "path/filepath" "testing" @@ -42,7 +43,7 @@ func Register() { } defer s.Close() - p := New(s, tmpDir) + p := New(context.Background(), s, tmpDir) if err := p.Run(); err != nil { t.Fatal(err) } @@ -106,7 +107,7 @@ func Main() { } defer s.Close() - p := New(s, tmpDir) + p := New(context.Background(), s, tmpDir) if err := p.Run(); err != nil { t.Fatal(err) } @@ -189,7 +190,7 @@ fun register() { } defer s.Close() - p := New(s, tmpDir) + p := New(context.Background(), s, tmpDir) if err := p.Run(); err != nil { t.Fatal(err) } @@ -233,7 +234,7 @@ fun main() { } defer s.Close() - p := New(s, tmpDir) + p := New(context.Background(), s, tmpDir) if err := p.Run(); err != nil { t.Fatal(err) } diff --git a/internal/selfupdate/selfupdate.go b/internal/selfupdate/selfupdate.go new file mode 100644 index 000000000..0d0acd265 --- /dev/null +++ b/internal/selfupdate/selfupdate.go @@ -0,0 +1,195 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "runtime" + "strconv" + "strings" + "time" +) + +// ReleaseURL is the GitHub API endpoint for latest release. Exported for test injection. +var ReleaseURL = "https://api.github.com/repos/DeusData/codebase-memory-mcp/releases/latest" + +// Release holds parsed GitHub release metadata. +type Release struct { + TagName string `json:"tag_name"` + Assets []Asset `json:"assets"` +} + +// Asset holds a single release artifact. +type Asset struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + Size int64 `json:"size"` +} + +// FetchLatestRelease fetches release metadata from GitHub. +func FetchLatestRelease(ctx context.Context) (*Release, error) { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", ReleaseURL, http.NoBody) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("github api status=%d", resp.StatusCode) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("read body: %w", err) + } + + var release Release + if err := json.Unmarshal(body, &release); err != nil { + return nil, fmt.Errorf("parse json: %w", err) + } + return &release, nil +} + +// LatestVersion returns the version string from the latest release (without "v" prefix). +func (r *Release) LatestVersion() string { + return strings.TrimPrefix(r.TagName, "v") +} + +// FindAsset finds a release asset by name. +func (r *Release) FindAsset(name string) *Asset { + for i := range r.Assets { + if r.Assets[i].Name == name { + return &r.Assets[i] + } + } + return nil +} + +// AssetName returns the expected release asset name for the current platform. +func AssetName() string { + if runtime.GOOS == "windows" { + return fmt.Sprintf("codebase-memory-mcp-%s-%s.zip", runtime.GOOS, runtime.GOARCH) + } + return fmt.Sprintf("codebase-memory-mcp-%s-%s.tar.gz", runtime.GOOS, runtime.GOARCH) +} + +// CompareVersions compares two semver strings (e.g. "0.2.1" vs "0.2.0"). +// Returns >0 if a > b, <0 if a < b, 0 if equal. +func CompareVersions(a, b string) int { + a = strings.TrimPrefix(a, "v") + b = strings.TrimPrefix(b, "v") + + // Strip pre-release suffixes for comparison (e.g. "0.2.1-dev" → "0.2.1") + aBase := strings.SplitN(a, "-", 2)[0] + bBase := strings.SplitN(b, "-", 2)[0] + + aParts := strings.Split(aBase, ".") + bParts := strings.Split(bBase, ".") + for i := 0; i < len(aParts) && i < len(bParts); i++ { + ai, _ := strconv.Atoi(aParts[i]) + bi, _ := strconv.Atoi(bParts[i]) + if ai != bi { + return ai - bi + } + } + if len(aParts) != len(bParts) { + return len(aParts) - len(bParts) + } + + // Same base version — non-dev beats dev (e.g. "0.2.1" > "0.2.1-dev") + aHasPre := strings.Contains(a, "-") + bHasPre := strings.Contains(b, "-") + if aHasPre && !bHasPre { + return -1 + } + if !aHasPre && bHasPre { + return 1 + } + return 0 +} + +// AllowedDownloadPrefixes controls which URL prefixes are accepted by DownloadAsset. +// Exported for test injection only. +var AllowedDownloadPrefixes = []string{ + "https://github.com/", + "https://api.github.com/", +} + +// DownloadAsset downloads a release asset and returns the body reader and content length. +// Caller must close the returned ReadCloser. +func DownloadAsset(ctx context.Context, rawURL string) (io.ReadCloser, int64, error) { + allowed := false + for _, prefix := range AllowedDownloadPrefixes { + if strings.HasPrefix(rawURL, prefix) { + allowed = true + break + } + } + if !allowed { + return nil, 0, fmt.Errorf("refusing to download from non-GitHub URL: %s", rawURL) + } + + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", rawURL, http.NoBody) + if err != nil { + return nil, 0, fmt.Errorf("create request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("download: %w", err) + } + + if resp.StatusCode != 200 { + resp.Body.Close() + return nil, 0, fmt.Errorf("download status=%d", resp.StatusCode) + } + + return resp.Body, resp.ContentLength, nil +} + +// DownloadChecksums downloads and parses the checksums.txt file from a release. +// Returns a map of filename → hex-encoded SHA-256 hash. +func DownloadChecksums(ctx context.Context, release *Release) (map[string]string, error) { + asset := release.FindAsset("checksums.txt") + if asset == nil { + return nil, fmt.Errorf("checksums.txt not found in release") + } + + body, _, err := DownloadAsset(ctx, asset.BrowserDownloadURL) + if err != nil { + return nil, fmt.Errorf("download checksums: %w", err) + } + defer body.Close() + + data, err := io.ReadAll(io.LimitReader(body, 1<<16)) + if err != nil { + return nil, fmt.Errorf("read checksums: %w", err) + } + + checksums := make(map[string]string) + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Fields(line) + if len(parts) >= 2 { + checksums[parts[1]] = parts[0] + } + } + return checksums, nil +} diff --git a/internal/selfupdate/selfupdate_test.go b/internal/selfupdate/selfupdate_test.go new file mode 100644 index 000000000..832324d91 --- /dev/null +++ b/internal/selfupdate/selfupdate_test.go @@ -0,0 +1,163 @@ +package selfupdate + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "testing" +) + +func TestCompareVersions(t *testing.T) { + tests := []struct { + a, b string + want int // >0, <0, or 0 + }{ + {"0.2.1", "0.2.0", 1}, + {"0.2.0", "0.2.0", 0}, + {"0.1.9", "0.2.0", -1}, + {"0.10.0", "0.2.0", 1}, + {"1.0.0", "0.99.99", 1}, + {"0.0.1", "0.0.2", -1}, + {"v0.2.1", "0.2.1", 0}, + {"0.2.1", "v0.2.1", 0}, + {"0.2.1-dev", "0.2.1", -1}, + {"0.2.1", "0.2.1-dev", 1}, + {"0.2.1-dev", "0.2.1-dev", 0}, + {"0.3.0", "0.2.1-dev", 1}, + {"0.2.0", "0.2.1-dev", -1}, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%s_vs_%s", tt.a, tt.b), func(t *testing.T) { + got := CompareVersions(tt.a, tt.b) + switch { + case tt.want > 0 && got <= 0: + t.Fatalf("CompareVersions(%q, %q) = %d, want > 0", tt.a, tt.b, got) + case tt.want < 0 && got >= 0: + t.Fatalf("CompareVersions(%q, %q) = %d, want < 0", tt.a, tt.b, got) + case tt.want == 0 && got != 0: + t.Fatalf("CompareVersions(%q, %q) = %d, want 0", tt.a, tt.b, got) + } + }) + } +} + +func TestAssetName(t *testing.T) { + name := AssetName() + expected := fmt.Sprintf("codebase-memory-mcp-%s-%s", runtime.GOOS, runtime.GOARCH) + if runtime.GOOS == "windows" { + expected += ".zip" + } else { + expected += ".tar.gz" + } + if name != expected { + t.Fatalf("AssetName() = %q, want %q", name, expected) + } +} + +func TestFetchLatestRelease(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{ + "tag_name": "v1.0.0", + "assets": [ + {"name": "codebase-memory-mcp-linux-amd64.tar.gz", "browser_download_url": "https://example.com/linux.tar.gz", "size": 1024}, + {"name": "checksums.txt", "browser_download_url": "https://example.com/checksums.txt", "size": 256} + ] + }`) + })) + defer ts.Close() + + old := ReleaseURL + ReleaseURL = ts.URL + defer func() { ReleaseURL = old }() + + release, err := FetchLatestRelease(context.Background()) + if err != nil { + t.Fatalf("FetchLatestRelease() error: %v", err) + } + + if release.LatestVersion() != "1.0.0" { + t.Fatalf("LatestVersion() = %q, want %q", release.LatestVersion(), "1.0.0") + } + + if len(release.Assets) != 2 { + t.Fatalf("expected 2 assets, got %d", len(release.Assets)) + } +} + +func TestFetchLatestRelease_ServerError(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer ts.Close() + + old := ReleaseURL + ReleaseURL = ts.URL + defer func() { ReleaseURL = old }() + + _, err := FetchLatestRelease(context.Background()) + if err == nil { + t.Fatal("expected error for 500 response") + } +} + +func TestRelease_FindAsset(t *testing.T) { + release := &Release{ + Assets: []Asset{ + {Name: "file-a.tar.gz"}, + {Name: "file-b.tar.gz"}, + {Name: "checksums.txt"}, + }, + } + + if a := release.FindAsset("file-b.tar.gz"); a == nil { + t.Fatal("expected to find file-b.tar.gz") + } + if a := release.FindAsset("nonexistent"); a != nil { + t.Fatal("expected nil for nonexistent asset") + } +} + +func TestDownloadChecksums(t *testing.T) { + checksumServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "abc123 file-a.tar.gz\ndef456 file-b.tar.gz\n") + })) + defer checksumServer.Close() + + // Allow local test server URLs + orig := AllowedDownloadPrefixes + AllowedDownloadPrefixes = append(AllowedDownloadPrefixes, checksumServer.URL) + t.Cleanup(func() { AllowedDownloadPrefixes = orig }) + + release := &Release{ + Assets: []Asset{ + {Name: "checksums.txt", BrowserDownloadURL: checksumServer.URL}, + }, + } + + checksums, err := DownloadChecksums(context.Background(), release) + if err != nil { + t.Fatalf("DownloadChecksums() error: %v", err) + } + + if checksums["file-a.tar.gz"] != "abc123" { + t.Fatalf("expected abc123 for file-a.tar.gz, got %q", checksums["file-a.tar.gz"]) + } + if checksums["file-b.tar.gz"] != "def456" { + t.Fatalf("expected def456 for file-b.tar.gz, got %q", checksums["file-b.tar.gz"]) + } +} + +func TestDownloadChecksums_NoChecksumsFile(t *testing.T) { + release := &Release{ + Assets: []Asset{ + {Name: "file-a.tar.gz"}, + }, + } + + _, err := DownloadChecksums(context.Background(), release) + if err == nil { + t.Fatal("expected error when checksums.txt is missing") + } +} diff --git a/internal/store/migrate.go b/internal/store/migrate.go index a950a6b3e..ae7698da8 100644 --- a/internal/store/migrate.go +++ b/internal/store/migrate.go @@ -1,6 +1,7 @@ package store import ( + "context" "database/sql" "fmt" "io" @@ -40,7 +41,8 @@ func (r *StoreRouter) migrate() error { defer legacyDB.Close() // Get list of projects - rows, err := legacyDB.Query("SELECT name FROM projects") + ctx := context.Background() + rows, err := legacyDB.QueryContext(ctx, "SELECT name FROM projects") if err != nil { return fmt.Errorf("list projects: %w", err) } @@ -94,37 +96,38 @@ func (r *StoreRouter) migrateProject(legacyDB *sql.DB, projectName, targetPath s targetStore.Close() // Use ATTACH DATABASE for efficient bulk copy - _, err = legacyDB.Exec("ATTACH DATABASE ? AS target", targetPath) + ctx := context.Background() + _, err = legacyDB.ExecContext(ctx, "ATTACH DATABASE ? AS target", targetPath) if err != nil { return fmt.Errorf("attach: %w", err) } defer func() { - _, _ = legacyDB.Exec("DETACH DATABASE target") + _, _ = legacyDB.ExecContext(ctx, "DETACH DATABASE target") }() // Copy projects - _, err = legacyDB.Exec(`INSERT OR REPLACE INTO target.projects (name, indexed_at, root_path) + _, err = legacyDB.ExecContext(ctx, `INSERT OR REPLACE INTO target.projects (name, indexed_at, root_path) SELECT name, indexed_at, root_path FROM projects WHERE name=?`, projectName) if err != nil { return fmt.Errorf("copy projects: %w", err) } // Copy nodes (preserving IDs) - _, err = legacyDB.Exec(`INSERT OR REPLACE INTO target.nodes (id, project, label, name, qualified_name, file_path, start_line, end_line, properties) + _, err = legacyDB.ExecContext(ctx, `INSERT OR REPLACE INTO target.nodes (id, project, label, name, qualified_name, file_path, start_line, end_line, properties) SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE project=?`, projectName) if err != nil { return fmt.Errorf("copy nodes: %w", err) } // Copy edges (preserving IDs) - _, err = legacyDB.Exec(`INSERT OR REPLACE INTO target.edges (id, project, source_id, target_id, type, properties) + _, err = legacyDB.ExecContext(ctx, `INSERT OR REPLACE INTO target.edges (id, project, source_id, target_id, type, properties) SELECT id, project, source_id, target_id, type, properties FROM edges WHERE project=?`, projectName) if err != nil { return fmt.Errorf("copy edges: %w", err) } // Copy file_hashes - _, err = legacyDB.Exec(`INSERT OR REPLACE INTO target.file_hashes (project, rel_path, sha256) + _, err = legacyDB.ExecContext(ctx, `INSERT OR REPLACE INTO target.file_hashes (project, rel_path, sha256) SELECT project, rel_path, sha256 FROM file_hashes WHERE project=?`, projectName) if err != nil { return fmt.Errorf("copy file_hashes: %w", err) @@ -132,15 +135,15 @@ func (r *StoreRouter) migrateProject(legacyDB *sql.DB, projectName, targetPath s // Verify row counts var srcNodes, tgtNodes int - _ = legacyDB.QueryRow("SELECT COUNT(*) FROM nodes WHERE project=?", projectName).Scan(&srcNodes) - _ = legacyDB.QueryRow("SELECT COUNT(*) FROM target.nodes WHERE project=?", projectName).Scan(&tgtNodes) + _ = legacyDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM nodes WHERE project=?", projectName).Scan(&srcNodes) + _ = legacyDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM target.nodes WHERE project=?", projectName).Scan(&tgtNodes) if srcNodes != tgtNodes { return fmt.Errorf("node count mismatch: src=%d tgt=%d", srcNodes, tgtNodes) } var srcEdges, tgtEdges int - _ = legacyDB.QueryRow("SELECT COUNT(*) FROM edges WHERE project=?", projectName).Scan(&srcEdges) - _ = legacyDB.QueryRow("SELECT COUNT(*) FROM target.edges WHERE project=?", projectName).Scan(&tgtEdges) + _ = legacyDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM edges WHERE project=?", projectName).Scan(&srcEdges) + _ = legacyDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM target.edges WHERE project=?", projectName).Scan(&tgtEdges) if srcEdges != tgtEdges { return fmt.Errorf("edge count mismatch: src=%d tgt=%d", srcEdges, tgtEdges) } diff --git a/internal/store/search.go b/internal/store/search.go index b50ab3ad1..d6f2871d0 100644 --- a/internal/store/search.go +++ b/internal/store/search.go @@ -11,7 +11,8 @@ import ( type SearchParams struct { Project string Label string - NamePattern string + NamePattern string // regex matched against short name only + QNPattern string // regex matched against qualified name only FilePattern string Relationship string Direction string // "inbound", "outbound", "any" @@ -62,15 +63,41 @@ func (s *Store) loadConnectedNames(sr *SearchResult, nodeID int64) { // Search executes a parameterized search query with pagination support. func (s *Store) Search(params *SearchParams) (*SearchOutput, error) { - // Limit=0 means use default; use a high ceiling for SQL if params.Limit <= 0 { params.Limit = 100000 } - // Build the query dynamically with parameterized values - var conditions []string - var args []any + conditions, args, nameHasLikeHints, qnHasLikeHints := buildSearchConditions(params) + where := strings.Join(conditions, " AND ") + sqlLimit := computeSQLLimit(params, nameHasLikeHints, qnHasLikeHints) + nodes, err := s.executeSearchQuery(where, args, sqlLimit) + if err != nil { + return nil, err + } + + nodes, err = applyPatternFilters(nodes, params) + if err != nil { + return nil, err + } + + allResults, err := s.buildFilteredResults(nodes, params) + if err != nil { + return nil, err + } + + sortBy := params.SortBy + if sortBy == "" { + sortBy = "relevance" + } + sortSearchResults(allResults, sortBy, params.NamePattern) + + return paginateResults(allResults, params.Offset, params.Limit), nil +} + +// buildSearchConditions builds SQL WHERE conditions and args from search params. +// Returns conditions, args, and whether LIKE hints were pushed for name/qn patterns. +func buildSearchConditions(params *SearchParams) (conditions []string, args []any, nameHasLikeHints, qnHasLikeHints bool) { conditions = append(conditions, "n.project = ?") args = append(args, params.Project) @@ -80,7 +107,6 @@ func (s *Store) Search(params *SearchParams) (*SearchOutput, error) { } if params.FilePattern != "" { - // Convert glob to SQL LIKE pattern likePattern := globToLike(params.FilePattern) conditions = append(conditions, "n.file_path LIKE ?") args = append(args, likePattern) @@ -98,43 +124,63 @@ func (s *Store) Search(params *SearchParams) (*SearchOutput, error) { // When a name pattern is provided, try to extract literal substrings for SQL LIKE // pre-filtering. This drastically reduces the rows Go needs to regex-match. // The full regex is still applied in Go for correctness. - nameHasLikeHints := false if params.NamePattern != "" { - hints := extractLikeHints(params.NamePattern) - if len(hints) > 0 { + if cond, condArgs := buildLikeHintCondition(params.NamePattern, "n.name"); cond != "" { nameHasLikeHints = true - var likeParts []string - for _, hint := range hints { - likeVal := "%" + hint + "%" - likeParts = append(likeParts, "(n.name LIKE ? OR n.qualified_name LIKE ?)") - args = append(args, likeVal, likeVal) - } - conditions = append(conditions, "("+strings.Join(likeParts, " AND ")+")") + conditions = append(conditions, cond) + args = append(args, condArgs...) } } - where := strings.Join(conditions, " AND ") + // QN pattern: same LIKE hint optimization but against qualified_name only + if params.QNPattern != "" { + if cond, condArgs := buildLikeHintCondition(params.QNPattern, "n.qualified_name"); cond != "" { + qnHasLikeHints = true + conditions = append(conditions, cond) + args = append(args, condArgs...) + } + } - // When Go-side filtering is needed (regex, degree), we must scan all - // qualifying rows since matching nodes may appear at any position. - // However, if we pushed LIKE hints into SQL, the result set is already - // narrowed and we don't need the full 200K scan for name patterns. + return conditions, args, nameHasLikeHints, qnHasLikeHints +} + +// buildLikeHintCondition extracts literal substrings from a regex pattern and returns +// a SQL condition with LIKE clauses for pre-filtering. Returns empty string if no hints. +func buildLikeHintCondition(pattern, column string) (condition string, args []any) { + hints := extractLikeHints(pattern) + if len(hints) == 0 { + return "", nil + } + likeParts := make([]string, 0, len(hints)) + for _, hint := range hints { + likeVal := "%" + hint + "%" + likeParts = append(likeParts, column+" LIKE ?") + args = append(args, likeVal) + } + return "(" + strings.Join(likeParts, " AND ") + ")", args +} + +// computeSQLLimit determines how many rows to fetch from SQL based on filtering needs. +func computeSQLLimit(params *SearchParams, nameHasLikeHints, qnHasLikeHints bool) int { hasDegreeFilter := params.MinDegree >= 0 || params.MaxDegree >= 0 - needsNameScan := params.NamePattern != "" && !nameHasLikeHints - var sqlLimit int - if needsNameScan || hasDegreeFilter { - sqlLimit = 200000 // must cover full dataset for accurate Go-side filtering - } else { - // Scan beyond offset+limit so total/has_more are accurate. - // The +1000 buffer ensures has_more is correct for result sets - // up to ~1000 beyond the requested page, at negligible cost - // (~2 extra batch degree queries ≈ 20ms worst case). - sqlLimit = params.Offset + params.Limit + 1000 - if sqlLimit > 200000 { - sqlLimit = 200000 - } - } + needsNameScan := (params.NamePattern != "" && !nameHasLikeHints) || (params.QNPattern != "" && !qnHasLikeHints) + if needsNameScan || hasDegreeFilter { + return 200000 // must cover full dataset for accurate Go-side filtering + } + // Scan beyond offset+limit so total/has_more are accurate. + // The +1000 buffer ensures has_more is correct for result sets + // up to ~1000 beyond the requested page, at negligible cost + // (~2 extra batch degree queries ≈ 20ms worst case). + sqlLimit := params.Offset + params.Limit + 1000 + if sqlLimit > 200000 { + sqlLimit = 200000 + } + return sqlLimit +} + +// executeSearchQuery runs the SQL search query and returns matching nodes. +func (s *Store) executeSearchQuery(where string, args []any, sqlLimit int) ([]*Node, error) { query := fmt.Sprintf(` SELECT n.id, n.project, n.label, n.name, n.qualified_name, n.file_path, n.start_line, n.end_line, n.properties FROM nodes n @@ -161,44 +207,42 @@ func (s *Store) Search(params *SearchParams) (*SearchOutput, error) { if err := rows.Err(); err != nil { return nil, err } + return nodes, nil +} - // Apply name pattern filter in Go (regex) +// applyPatternFilters applies name and QN regex filters in Go. +func applyPatternFilters(nodes []*Node, params *SearchParams) ([]*Node, error) { + var err error if params.NamePattern != "" { nodes, err = filterByNamePattern(nodes, params.NamePattern) if err != nil { return nil, err } } - - // Build all qualifying results (before offset/limit) for accurate total count - allResults, err := s.buildFilteredResults(nodes, params) - if err != nil { - return nil, err + if params.QNPattern != "" { + nodes, err = filterByQNPattern(nodes, params.QNPattern) + if err != nil { + return nil, err + } } + return nodes, nil +} - // Sort results before pagination - sortBy := params.SortBy - if sortBy == "" { - sortBy = "relevance" - } - sortSearchResults(allResults, sortBy, params.NamePattern) - +// paginateResults applies offset and limit to results and returns the output. +func paginateResults(allResults []*SearchResult, offset, limit int) *SearchOutput { total := len(allResults) - - // Apply offset and limit - start := params.Offset + start := offset if start > total { start = total } - end := start + params.Limit + end := start + limit if end > total { end = total } - return &SearchOutput{ Results: allResults[start:end], Total: total, - }, nil + } } // globToLike converts a glob pattern to SQL LIKE pattern. @@ -469,7 +513,7 @@ func extractLiteralQuery(pattern string) string { return p } -// filterByNamePattern filters nodes by a regex name pattern. +// filterByNamePattern filters nodes by a regex pattern against the short name only. func filterByNamePattern(nodes []*Node, pattern string) ([]*Node, error) { re, err := regexp.Compile(pattern) if err != nil { @@ -477,7 +521,22 @@ func filterByNamePattern(nodes []*Node, pattern string) ([]*Node, error) { } var filtered []*Node for _, n := range nodes { - if re.MatchString(n.Name) || re.MatchString(n.QualifiedName) { + if re.MatchString(n.Name) { + filtered = append(filtered, n) + } + } + return filtered, nil +} + +// filterByQNPattern filters nodes by a regex pattern against the qualified name. +func filterByQNPattern(nodes []*Node, pattern string) ([]*Node, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("invalid qn_pattern: %w", err) + } + var filtered []*Node + for _, n := range nodes { + if re.MatchString(n.QualifiedName) { filtered = append(filtered, n) } } diff --git a/internal/store/store.go b/internal/store/store.go index ca5135b6f..6e4e106d0 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1,6 +1,7 @@ package store import ( + "context" "database/sql" "encoding/json" "fmt" @@ -95,8 +96,9 @@ func OpenPath(dbPath string) (*Store, error) { db.SetMaxIdleConns(1) // PRAGMAs not supported in mattn DSN — set via Exec after Open. - _, _ = db.Exec("PRAGMA temp_store = MEMORY") - _, _ = db.Exec("PRAGMA mmap_size = 1073741824") // 1 GB + ctx := context.Background() + _, _ = db.ExecContext(ctx, "PRAGMA temp_store = MEMORY") + _, _ = db.ExecContext(ctx, "PRAGMA mmap_size = 1073741824") // 1 GB s := &Store{db: db, dbPath: dbPath} s.q = s.db @@ -115,7 +117,7 @@ func OpenMemory() (*Store, error) { if err != nil { return nil, fmt.Errorf("open memory db: %w", err) } - _, _ = db.Exec("PRAGMA temp_store = MEMORY") + _, _ = db.ExecContext(context.Background(), "PRAGMA temp_store = MEMORY") s := &Store{db: db, dbPath: ":memory:"} s.q = s.db if err := s.initSchema(); err != nil { @@ -129,8 +131,8 @@ func OpenMemory() (*Store, error) { // The callback receives a transaction-scoped Store — all store methods called on // txStore use the transaction. The receiver's q field is never mutated, so // concurrent read-only handlers (using s.q == s.db) are unaffected. -func (s *Store) WithTransaction(fn func(txStore *Store) error) error { - tx, err := s.db.Begin() +func (s *Store) WithTransaction(ctx context.Context, fn func(txStore *Store) error) error { + tx, err := s.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("begin tx: %w", err) } @@ -146,23 +148,23 @@ func (s *Store) WithTransaction(fn func(txStore *Store) error) error { // then runs PRAGMA optimize so the query planner has up-to-date statistics. // PRAGMA optimize (SQLite 3.46+) auto-limits sampling per index, only re-analyzing // stale stats. Cost is absorbed during indexing rather than the first read query. -func (s *Store) Checkpoint() { - _, _ = s.db.Exec("PRAGMA wal_checkpoint(TRUNCATE)") - _, _ = s.db.Exec("PRAGMA optimize") +func (s *Store) Checkpoint(ctx context.Context) { + _, _ = s.db.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)") + _, _ = s.db.ExecContext(ctx, "PRAGMA optimize") } // BeginBulkWrite switches to MEMORY journal mode for faster bulk writes. // Call EndBulkWrite when done to restore WAL mode. // MEMORY mode is rollback-safe on crash (unlike journal_mode=OFF). -func (s *Store) BeginBulkWrite() { - _, _ = s.db.Exec("PRAGMA journal_mode = MEMORY") - _, _ = s.db.Exec("PRAGMA synchronous = OFF") +func (s *Store) BeginBulkWrite(ctx context.Context) { + _, _ = s.db.ExecContext(ctx, "PRAGMA journal_mode = MEMORY") + _, _ = s.db.ExecContext(ctx, "PRAGMA synchronous = OFF") } // EndBulkWrite restores WAL journal mode and NORMAL synchronous after bulk writes. -func (s *Store) EndBulkWrite() { - _, _ = s.db.Exec("PRAGMA synchronous = NORMAL") - _, _ = s.db.Exec("PRAGMA journal_mode = WAL") +func (s *Store) EndBulkWrite(ctx context.Context) { + _, _ = s.db.ExecContext(ctx, "PRAGMA synchronous = NORMAL") + _, _ = s.db.ExecContext(ctx, "PRAGMA journal_mode = WAL") } // Close closes the database connection. @@ -181,6 +183,7 @@ func (s *Store) DBPath() string { } func (s *Store) initSchema() error { + ctx := context.Background() schema := ` CREATE TABLE IF NOT EXISTS projects ( name TEXT PRIMARY KEY, @@ -229,7 +232,7 @@ func (s *Store) initSchema() error { CREATE INDEX IF NOT EXISTS idx_edges_target_type ON edges(project, target_id, type); CREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(project, source_id, type); ` - _, err := s.db.Exec(schema) + _, err := s.db.ExecContext(ctx, schema) if err != nil { return err } @@ -238,9 +241,9 @@ func (s *Store) initSchema() error { // Generated columns require SQLite 3.31.0+ (mattn/go-sqlite3 supports this). // We check if the column already exists to make this idempotent. var colCount int - _ = s.db.QueryRow(`SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount) + _ = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount) if colCount == 0 { - _, err = s.db.Exec(`ALTER TABLE edges ADD COLUMN url_path_gen TEXT GENERATED ALWAYS AS (json_extract(properties, '$.url_path'))`) + _, err = s.db.ExecContext(ctx, `ALTER TABLE edges ADD COLUMN url_path_gen TEXT GENERATED ALWAYS AS (json_extract(properties, '$.url_path'))`) if err != nil { // If generated columns aren't supported, skip gracefully slog.Warn("schema.url_path_gen.skip", "err", err) @@ -248,7 +251,7 @@ func (s *Store) initSchema() error { } // Index on generated column (safe to CREATE IF NOT EXISTS) - _, _ = s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_edges_url_path ON edges(project, url_path_gen)`) + _, _ = s.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_edges_url_path ON edges(project, url_path_gen)`) return nil } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 62269aaac..7b5c1293a 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1,6 +1,7 @@ package store import ( + "context" "fmt" "testing" ) @@ -393,7 +394,7 @@ func TestGeneratedColumnURLPath(t *testing.T) { // Check that the generated column exists var colCount int - err = s.DB().QueryRow(`SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount) + err = s.DB().QueryRowContext(context.Background(), `SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount) if err != nil { t.Fatal(err) } @@ -475,7 +476,7 @@ func TestPragmaSettings(t *testing.T) { } for _, tt := range tests { var val string - err := s.DB().QueryRow("PRAGMA " + tt.pragma).Scan(&val) + err := s.DB().QueryRowContext(context.Background(), "PRAGMA "+tt.pragma).Scan(&val) if err != nil { t.Fatalf("PRAGMA %s: %v", tt.pragma, err) } diff --git a/internal/tools/index.go b/internal/tools/index.go index 2f58d7a04..7ccf0a9c7 100644 --- a/internal/tools/index.go +++ b/internal/tools/index.go @@ -10,7 +10,7 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" ) -func (s *Server) handleIndexRepository(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { +func (s *Server) handleIndexRepository(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { args, err := parseArgs(req) if err != nil { return errResult(err.Error()), nil @@ -30,7 +30,7 @@ func (s *Server) handleIndexRepository(_ context.Context, req *mcp.CallToolReque return errResult(fmt.Sprintf("invalid path: %v", err)), nil } - projectName := filepath.Base(absPath) + projectName := pipeline.ProjectNameFromPath(absPath) // Lock to prevent concurrent indexing with auto-sync watcher s.indexMu.Lock() @@ -43,7 +43,7 @@ func (s *Server) handleIndexRepository(_ context.Context, req *mcp.CallToolReque } // Run the indexing pipeline - p := pipeline.New(st, absPath) + p := pipeline.New(ctx, st, absPath) if err := p.Run(); err != nil { return errResult(fmt.Sprintf("indexing failed: %v", err)), nil } diff --git a/internal/tools/search.go b/internal/tools/search.go index 458d6bf1f..1b5ca50ad 100644 --- a/internal/tools/search.go +++ b/internal/tools/search.go @@ -17,6 +17,7 @@ func (s *Server) handleSearchGraph(_ context.Context, req *mcp.CallToolRequest) params := &store.SearchParams{ Label: getStringArg(args, "label"), NamePattern: getStringArg(args, "name_pattern"), + QNPattern: getStringArg(args, "qn_pattern"), FilePattern: getStringArg(args, "file_pattern"), Relationship: getStringArg(args, "relationship"), Direction: getStringArg(args, "direction"), diff --git a/internal/tools/tools.go b/internal/tools/tools.go index f3c366c46..d0c104c90 100644 --- a/internal/tools/tools.go +++ b/internal/tools/tools.go @@ -9,7 +9,6 @@ import ( "net/http" "net/url" "os" - "path/filepath" "sort" "strconv" "strings" @@ -24,7 +23,7 @@ import ( ) // Version is the current release version, referenced by MCP handshake and update checker. -const Version = "0.2.1" +const Version = "0.2.1-dev" // releaseURL is the GitHub API endpoint for latest release. Package-level var for test injection. var releaseURL = "https://api.github.com/repos/DeusData/codebase-memory-mcp/releases/latest" @@ -40,7 +39,7 @@ type Server struct { // Session-aware fields (set once via sync.Once, then immutable) sessionOnce sync.Once sessionRoot string // absolute path from client - sessionProject string // filepath.Base(sessionRoot) + sessionProject string // derived from sessionRoot via ProjectNameFromPath indexStatus atomic.Value indexStartedAt atomic.Value // time.Time — when current/last index started updateNotice atomic.Value // string — set once by checkForUpdate, cleared after first injection @@ -77,7 +76,7 @@ func (s *Server) StartWatcher(ctx context.Context) { // syncProject is called by the watcher when file changes are detected. // Uses TryLock to skip if an index operation is already in progress. -func (s *Server) syncProject(projectName, rootPath string) error { +func (s *Server) syncProject(ctx context.Context, projectName, rootPath string) error { if !s.indexMu.TryLock() { slog.Debug("watcher.skip", "path", rootPath, "reason", "index_in_progress") return nil @@ -87,7 +86,7 @@ func (s *Server) syncProject(projectName, rootPath string) error { if err != nil { return fmt.Errorf("store for %s: %w", projectName, err) } - p := pipeline.New(st, rootPath) + p := pipeline.New(ctx, st, rootPath) return p.Run() } @@ -112,7 +111,7 @@ func (s *Server) SetSessionRoot(rootPath string) { s.sessionOnce.Do(func() { s.sessionRoot = rootPath if rootPath != "" { - s.sessionProject = filepath.Base(rootPath) + s.sessionProject = pipeline.ProjectNameFromPath(rootPath) } }) } @@ -125,7 +124,7 @@ func (s *Server) onInitialized(ctx context.Context, req *mcp.InitializedRequest) s.sessionOnce.Do(func() { s.sessionRoot = s.detectSessionRoot(ctx, req.Session) if s.sessionRoot != "" { - s.sessionProject = filepath.Base(s.sessionRoot) + s.sessionProject = pipeline.ProjectNameFromPath(s.sessionRoot) s.startAutoIndex() } }) @@ -137,7 +136,7 @@ func (s *Server) onRootsChanged(ctx context.Context, req *mcp.RootsListChangedRe s.sessionOnce.Do(func() { s.sessionRoot = s.detectSessionRoot(ctx, req.Session) if s.sessionRoot != "" { - s.sessionProject = filepath.Base(s.sessionRoot) + s.sessionProject = pipeline.ProjectNameFromPath(s.sessionRoot) s.startAutoIndex() } }) @@ -210,7 +209,7 @@ func (s *Server) startAutoIndex() { slog.Warn("autoindex.store.err", "err", err) return } - p := pipeline.New(st, s.sessionRoot) + p := pipeline.New(context.Background(), st, s.sessionRoot) if err := p.Run(); err != nil { slog.Warn("autoindex.err", "err", err) return @@ -308,8 +307,8 @@ func (s *Server) checkForUpdate() { } if compareVersions(latest, Version) > 0 { notice := fmt.Sprintf( - "⚡ Update available: v%s → v%s — download at https://github.com/DeusData/codebase-memory-mcp/releases/tag/v%s", - Version, latest, latest) + "⚡ Update available: v%s → v%s — run: codebase-memory-mcp update", + Version, latest) s.updateNotice.Store(notice) slog.Info("update available", "current", Version, "latest", latest) } @@ -476,7 +475,11 @@ func (s *Server) registerSearchTools() { }, "name_pattern": { "type": "string", - "description": "Regex pattern for node name (e.g. '.*Handler', 'Send.*')" + "description": "Regex pattern matched against the short node name only (e.g. '.*Handler', 'Send.*'). Does NOT match against file paths or qualified names." + }, + "qn_pattern": { + "type": "string", + "description": "Regex pattern matched against the qualified name (includes module path, e.g. '.*tests\\.conftest\\..*', '.*services\\.order\\..*'). Use this to scope searches to specific directories or modules." }, "file_pattern": { "type": "string", diff --git a/internal/tools/update_test.go b/internal/tools/update_test.go index ec3ab0b5f..08e279abf 100644 --- a/internal/tools/update_test.go +++ b/internal/tools/update_test.go @@ -57,8 +57,8 @@ func TestCheckForUpdate_NewerAvailable(t *testing.T) { if !strings.Contains(notice, "v99.0.0") { t.Fatalf("notice should mention v99.0.0, got: %s", notice) } - if !strings.Contains(notice, "github.com/DeusData/codebase-memory-mcp/releases/tag/v99.0.0") { - t.Fatalf("notice should contain releases link, got: %s", notice) + if !strings.Contains(notice, "codebase-memory-mcp update") { + t.Fatalf("notice should contain update command, got: %s", notice) } } diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index c6dda5046..6d03f0632 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -27,13 +27,14 @@ type projectState struct { } // IndexFunc is the callback signature for triggering a re-index. -type IndexFunc func(projectName, rootPath string) error +type IndexFunc func(ctx context.Context, projectName, rootPath string) error // Watcher polls indexed projects for file changes and triggers re-indexing. type Watcher struct { router *store.StoreRouter indexFn IndexFunc projects map[string]*projectState + ctx context.Context } // New creates a Watcher. indexFn is called when file changes are detected. @@ -48,6 +49,7 @@ func New(r *store.StoreRouter, indexFn IndexFunc) *Watcher { // Run blocks until ctx is cancelled. Ticks at baseInterval, polling each // project only when its adaptive interval has elapsed. func (w *Watcher) Run(ctx context.Context) { + w.ctx = ctx ticker := time.NewTicker(baseInterval) defer ticker.Stop() @@ -133,7 +135,7 @@ func (w *Watcher) pollProject(proj *store.Project, state *projectState) { } slog.Info("watcher.changed", "project", proj.Name, "files", len(snap)) - if err := w.indexFn(proj.Name, proj.RootPath); err != nil { + if err := w.indexFn(w.ctx, proj.Name, proj.RootPath); err != nil { slog.Warn("watcher.index", "project", proj.Name, "err", err) // Keep old snapshot so we retry next cycle state.nextPoll = time.Now().Add(interval) @@ -149,7 +151,7 @@ func (w *Watcher) pollProject(proj *store.Project, state *projectState) { // captureSnapshot walks the file tree using discover.Discover and captures // mtime+size for each file. func captureSnapshot(rootPath string) (map[string]fileSnapshot, error) { - files, err := discover.Discover(rootPath, nil) + files, err := discover.Discover(context.Background(), rootPath, nil) if err != nil { return nil, err } diff --git a/internal/watcher/watcher_test.go b/internal/watcher/watcher_test.go index a74c7595b..530a99b99 100644 --- a/internal/watcher/watcher_test.go +++ b/internal/watcher/watcher_test.go @@ -179,7 +179,7 @@ func TestWatcherTriggersOnChange(t *testing.T) { defer r.CloseAll() var indexCount atomic.Int32 - indexFn := func(_, _ string) error { + indexFn := func(_ context.Context, _, _ string) error { indexCount.Add(1) return nil } @@ -226,7 +226,7 @@ func TestWatcherCancellation(t *testing.T) { } defer r.CloseAll() - w := New(r, func(_, _ string) error { return nil }) + w := New(r, func(_ context.Context, _, _ string) error { return nil }) ctx, cancel := context.WithCancel(context.Background()) @@ -251,7 +251,7 @@ func TestWatcherSkipsMissingRoot(t *testing.T) { defer r.CloseAll() var indexCount atomic.Int32 - w := New(r, func(_, _ string) error { + w := New(r, func(_ context.Context, _, _ string) error { indexCount.Add(1) return nil }) @@ -273,7 +273,7 @@ func TestWatcherNewFileTriggersIndex(t *testing.T) { defer r.CloseAll() var indexCount atomic.Int32 - w := New(r, func(_, _ string) error { + w := New(r, func(_ context.Context, _, _ string) error { indexCount.Add(1) return nil }) diff --git a/scripts/hooks/pre-commit b/scripts/hooks/pre-commit new file mode 100755 index 000000000..e2f1b58d5 --- /dev/null +++ b/scripts/hooks/pre-commit @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Pre-commit hook: mirrors the GitHub Actions lint + test jobs. +# Same commands as .github/workflows/release.yml +# +# Setup (once after clone): +# git config core.hooksPath scripts/hooks + +# --- Lint --- +if ! command -v golangci-lint &>/dev/null; then + echo "pre-commit: golangci-lint not found, skipping lint check" + echo " install: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.10" +else + echo "pre-commit: running golangci-lint..." + golangci-lint run --timeout=5m ./... +fi + +# --- Tests --- +echo "pre-commit: running tests..." +go test ./... -timeout=5m diff --git a/scripts/setup.sh b/scripts/setup.sh index d2e866781..ee27498e3 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -324,6 +324,13 @@ fi configure_claude check_path +# --- Git hooks --- +# If run from inside the repo, activate tracked hooks +if [ -d "scripts/hooks" ] && git rev-parse --git-dir &>/dev/null; then + git config core.hooksPath scripts/hooks + ok "Git hooks activated (scripts/hooks/)" +fi + echo "" ok "Done! Restart Claude Code and verify with /mcp" echo ""