From 97284a35faa994b62df4eeb962e8dbc5ae5e73ff Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 25 Feb 2026 13:43:57 +0100 Subject: [PATCH] v0.1.0: Performance, incremental indexing, async boundary detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Performance & quality: - Transaction wrapping (Querier interface, single tx per index run) - Composite indexes on edges table for faster traversals - Cypher WHERE push-down (=, CONTAINS, STARTS WITH → SQL) - golangci-lint with 25 linters, zero violations - 11 benchmarks for pipeline, search, and BFS Incremental indexing: - Content-hash based file change detection - Dependency-aware two-level invalidation (changed + dependents) - Stale file cleanup for deleted files Async boundary detection: - Project-wide env var URL scanner (Dockerfile, YAML, .tf, .sh, .env, TOML, properties files) with 3-layer secret exclusion - Env var default extraction in constant propagation (getEnv, os.environ.get, process.env with ||/?? fallback) - ASYNC_CALLS edge type for Cloud Tasks, Pub/Sub, SQS/SNS, RabbitMQ, Kafka dispatch patterns, separate from synchronous HTTP_CALLS - Tool descriptions updated with all edge types Other improvements: - Type inference for method dispatch resolution - Interface implementation detection (IMPLEMENTS edges) - Improved tool descriptions for discoverability --- .github/workflows/release.yml | 16 +- .golangci.yml | 58 +++ Makefile | 8 +- cmd/codebase-memory-mcp/main.go | 7 +- internal/cypher/ast.go | 10 +- internal/cypher/cypher_test.go | 48 +- internal/cypher/executor.go | 545 ++++++++++++-------- internal/cypher/lexer.go | 232 ++++----- internal/cypher/parser.go | 105 ++-- internal/cypher/planner.go | 91 ++-- internal/discover/discover.go | 89 ++-- internal/httplink/httplink.go | 355 +++++++++----- internal/httplink/httplink_test.go | 326 ++++++++++-- internal/lang/java.go | 4 +- internal/lang/scala.go | 4 +- internal/parser/parser.go | 25 +- internal/pipeline/envscan.go | 249 ++++++++++ internal/pipeline/envscan_test.go | 377 ++++++++++++++ internal/pipeline/implements.go | 138 +++--- internal/pipeline/imports.go | 4 +- internal/pipeline/pipeline.go | 600 ++++++++++++++++++----- internal/pipeline/pipeline_bench_test.go | 332 +++++++++++++ internal/pipeline/pipeline_test.go | 8 +- internal/pipeline/resolve.go | 124 +++-- internal/pipeline/resolve_test.go | 93 +++- internal/pipeline/resolver.go | 70 +-- internal/pipeline/typeinfer.go | 56 +++ internal/store/edges.go | 28 +- internal/store/nodes.go | 24 +- internal/store/projects.go | 34 +- internal/store/schema.go | 21 +- internal/store/search.go | 132 +++-- internal/store/search_bench_test.go | 169 +++++++ internal/store/store.go | 37 ++ internal/store/store_test.go | 52 +- internal/store/traverse.go | 83 ++-- internal/store/traverse_bench_test.go | 180 +++++++ internal/tools/code_search.go | 132 ++--- internal/tools/file.go | 125 ++--- internal/tools/projects.go | 4 +- internal/tools/schema.go | 2 +- internal/tools/search.go | 5 +- internal/tools/snippet.go | 8 +- internal/tools/tools.go | 151 +++--- internal/tools/trace.go | 6 +- 45 files changed, 3901 insertions(+), 1266 deletions(-) create mode 100644 .golangci.yml create mode 100644 internal/pipeline/envscan.go create mode 100644 internal/pipeline/envscan_test.go create mode 100644 internal/pipeline/pipeline_bench_test.go create mode 100644 internal/store/search_bench_test.go create mode 100644 internal/store/traverse_bench_test.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e06caf6b..96308dcdc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,20 @@ permissions: contents: write jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.23" + + - uses: golangci/golangci-lint-action@v6 + with: + version: latest + args: --timeout=5m + build-unix: strategy: matrix: @@ -94,7 +108,7 @@ jobs: path: "*.zip" release: - needs: [build-unix, build-windows] + needs: [lint, build-unix, build-windows] runs-on: ubuntu-latest permissions: contents: write diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..274ea3329 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,58 @@ +run: + timeout: 5m + go: "1.23" + +linters: + enable: + - errcheck + - govet + - staticcheck + - bodyclose + - nilerr + - sqlclosecheck + - rowserrcheck + - durationcheck + - makezero + - noctx + - gofmt + - goimports + - misspell + - unconvert + - unparam + - unused + - ineffassign + - gosimple + - prealloc + - cyclop + - funlen + - gocognit + - gocritic + - nestif + - gosec + +linters-settings: + cyclop: + max-complexity: 25 + funlen: + lines: 120 + statements: 80 + gocognit: + min-complexity: 30 + nestif: + min-complexity: 6 + gocritic: + enabled-tags: + - diagnostic + - style + - performance + gosec: + excludes: + - G404 + errcheck: + check-type-assertions: true + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 + exclude-dirs: + - vendor diff --git a/Makefile b/Makefile index c8a15b30a..53eba1ccb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test lint clean install +.PHONY: build test lint clean install check BINARY=codebase-memory-mcp MODULE=github.com/DeusData/codebase-memory-mcp @@ -9,8 +9,10 @@ build: test: go test ./... -v -lint: - golangci-lint run ./... +check: lint test ## Run lint + tests + +lint: ## Run golangci-lint + golangci-lint run --timeout=5m ./... clean: rm -rf bin/ diff --git a/cmd/codebase-memory-mcp/main.go b/cmd/codebase-memory-mcp/main.go index 44bef6d95..5adbe2056 100644 --- a/cmd/codebase-memory-mcp/main.go +++ b/cmd/codebase-memory-mcp/main.go @@ -22,11 +22,12 @@ func main() { if err != nil { log.Fatalf("store open err=%v", err) } - defer s.Close() srv := tools.NewServer(s) - if err := srv.MCPServer().Run(context.Background(), &mcp.StdioTransport{}); err != nil { - log.Fatalf("server err=%v", err) + runErr := srv.MCPServer().Run(context.Background(), &mcp.StdioTransport{}) + s.Close() + if runErr != nil { + log.Fatalf("server err=%v", runErr) } } diff --git a/internal/cypher/ast.go b/internal/cypher/ast.go index 54bd6a736..ad76b3bdb 100644 --- a/internal/cypher/ast.go +++ b/internal/cypher/ast.go @@ -33,11 +33,11 @@ func (*NodePattern) patternElement() {} // RelPattern matches a graph relationship with optional types, direction, and hops. type RelPattern struct { - Variable string // (optional) - Types []string // relationship types, e.g. ["CALLS", "HTTP_CALLS"] - Direction string // "outbound", "inbound", "any" - MinHops int // for variable-length, default 1 - MaxHops int // for variable-length, default 1 (0 means unbounded) + Variable string // (optional) + Types []string // relationship types, e.g. ["CALLS", "HTTP_CALLS"] + Direction string // "outbound", "inbound", "any" + MinHops int // for variable-length, default 1 + MaxHops int // for variable-length, default 1 (0 means unbounded) } func (*RelPattern) patternElement() {} diff --git a/internal/cypher/cypher_test.go b/internal/cypher/cypher_test.go index e5431079d..b5b4cca6c 100644 --- a/internal/cypher/cypher_test.go +++ b/internal/cypher/cypher_test.go @@ -73,7 +73,10 @@ func TestParseNodePattern(t *testing.T) { if len(elems) != 1 { t.Fatalf("expected 1 element, got %d", len(elems)) } - node := elems[0].(*NodePattern) + node, ok := elems[0].(*NodePattern) + if !ok { + t.Fatalf("expected *NodePattern, got %T", elems[0]) + } if node.Variable != "f" { t.Errorf("expected variable 'f', got %q", node.Variable) } @@ -94,7 +97,10 @@ func TestParseRelationship(t *testing.T) { if len(elems) != 3 { t.Fatalf("expected 3 elements (node-rel-node), got %d", len(elems)) } - rel := elems[1].(*RelPattern) + rel, ok := elems[1].(*RelPattern) + if !ok { + t.Fatalf("expected *RelPattern, got %T", elems[1]) + } if len(rel.Types) != 1 || rel.Types[0] != "CALLS" { t.Errorf("expected CALLS type, got %v", rel.Types) } @@ -111,7 +117,10 @@ func TestParseVariableLength(t *testing.T) { if err != nil { t.Fatalf("parse: %v", err) } - rel := q.Match.Pattern.Elements[1].(*RelPattern) + rel, ok := q.Match.Pattern.Elements[1].(*RelPattern) + if !ok { + t.Fatalf("expected *RelPattern, got %T", q.Match.Pattern.Elements[1]) + } if rel.MinHops != 1 { t.Errorf("expected minHops=1, got %d", rel.MinHops) } @@ -185,7 +194,10 @@ func TestParseBidirectional(t *testing.T) { if err != nil { t.Fatalf("parse: %v", err) } - rel := q.Match.Pattern.Elements[1].(*RelPattern) + rel, ok := q.Match.Pattern.Elements[1].(*RelPattern) + if !ok { + t.Fatalf("expected *RelPattern, got %T", q.Match.Pattern.Elements[1]) + } if rel.Direction != "any" { t.Errorf("expected 'any' direction, got %q", rel.Direction) } @@ -196,7 +208,10 @@ func TestParseInbound(t *testing.T) { if err != nil { t.Fatalf("parse: %v", err) } - rel := q.Match.Pattern.Elements[1].(*RelPattern) + rel, ok := q.Match.Pattern.Elements[1].(*RelPattern) + if !ok { + t.Fatalf("expected *RelPattern, got %T", q.Match.Pattern.Elements[1]) + } if rel.Direction != "inbound" { t.Errorf("expected inbound, got %q", rel.Direction) } @@ -207,7 +222,10 @@ func TestParseMultipleRelTypes(t *testing.T) { if err != nil { t.Fatalf("parse: %v", err) } - rel := q.Match.Pattern.Elements[1].(*RelPattern) + rel, ok := q.Match.Pattern.Elements[1].(*RelPattern) + if !ok { + t.Fatalf("expected *RelPattern, got %T", q.Match.Pattern.Elements[1]) + } if len(rel.Types) != 2 { t.Fatalf("expected 2 types, got %d", len(rel.Types)) } @@ -322,14 +340,22 @@ func setupTestStore(t *testing.T) *store.Store { // Edges: HandleOrder -> ValidateOrder -> SubmitOrder // HandleOrder -> LogError - s.InsertEdge(&store.Edge{Project: "test", SourceID: idA, TargetID: idB, Type: "CALLS"}) - s.InsertEdge(&store.Edge{Project: "test", SourceID: idB, TargetID: idC, Type: "CALLS"}) - s.InsertEdge(&store.Edge{Project: "test", SourceID: idA, TargetID: idE, Type: "CALLS"}) - s.InsertEdge(&store.Edge{Project: "test", SourceID: idD, TargetID: idA, Type: "DEFINES"}) + mustInsertEdge(t, s, &store.Edge{Project: "test", SourceID: idA, TargetID: idB, Type: "CALLS"}) + mustInsertEdge(t, s, &store.Edge{Project: "test", SourceID: idB, TargetID: idC, Type: "CALLS"}) + mustInsertEdge(t, s, &store.Edge{Project: "test", SourceID: idA, TargetID: idE, Type: "CALLS"}) + mustInsertEdge(t, s, &store.Edge{Project: "test", SourceID: idD, TargetID: idA, Type: "DEFINES"}) return s } +// mustInsertEdge inserts an edge and fails the test on error. +func mustInsertEdge(t *testing.T, s *store.Store, edge *store.Edge) { + t.Helper() + if _, err := s.InsertEdge(edge); err != nil { + t.Fatalf("insert edge: %v", err) + } +} + func TestExecuteSimpleMatch(t *testing.T) { s := setupTestStore(t) defer s.Close() @@ -614,7 +640,7 @@ func setupTestStoreWithHTTPCalls(t *testing.T) *store.Store { if callerNode == nil || targetNode == nil { t.Fatal("expected test nodes to exist") } - s.InsertEdge(&store.Edge{ + mustInsertEdge(t, s, &store.Edge{ Project: "test", SourceID: callerNode.ID, TargetID: targetNode.ID, diff --git a/internal/cypher/executor.go b/internal/cypher/executor.go index ec3f381fc..71d9d075b 100644 --- a/internal/cypher/executor.go +++ b/internal/cypher/executor.go @@ -1,6 +1,7 @@ package cypher import ( + "database/sql" "fmt" "regexp" "sort" @@ -84,10 +85,23 @@ func (e *Executor) executeStepsForProject(project string, steps []PlanStep) ([]b var err error switch s := step.(type) { case *ScanNodes: - bindings, err = e.execScan(project, s, bindings) + // Check if the next step is a FilterWhere that can be pushed down + var pushDown *FilterWhere + if i+1 < len(steps) { + if fw, ok := steps[i+1].(*FilterWhere); ok { + pushDown = fw + } + } + bindings, err = e.execScan(project, s, pushDown) case *ExpandRelationship: bindings, err = e.execExpand(s, bindings) case *FilterWhere: + // Skip if this was already consumed by push-down + if i > 0 { + if _, wasScan := steps[i-1].(*ScanNodes); wasScan { + continue // already handled in execScan + } + } bindings, err = e.execFilter(s, bindings) default: return nil, fmt.Errorf("unknown step type: %T", step) @@ -95,8 +109,6 @@ func (e *Executor) executeStepsForProject(project string, steps []PlanStep) ([]b if err != nil { return nil, err } - // Only cap after the last step or after expand (which can explode). - // Never cap between scan and filter — the filter needs all candidates. isLastStep := i == len(steps)-1 _, isExpand := step.(*ExpandRelationship) if isLastStep || isExpand { @@ -109,26 +121,87 @@ func (e *Executor) executeStepsForProject(project string, steps []PlanStep) ([]b return bindings, nil } -func (e *Executor) execScan(project string, s *ScanNodes, _ []binding) ([]binding, error) { - var nodes []*store.Node - var err error +// sqlPushableColumns are node properties that map directly to SQL columns. +var sqlPushableColumns = map[string]string{ + "name": "name", + "qualified_name": "qualified_name", + "label": "label", + "file_path": "file_path", +} + +func (e *Executor) execScan(project string, s *ScanNodes, pushDown *FilterWhere) ([]binding, error) { + // Build dynamic SQL query with optional push-down conditions + query := `SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE project=?` + args := []any{project} if s.Label != "" { - nodes, err = e.Store.FindNodesByLabel(project, s.Label) - } else { - nodes, err = e.Store.AllNodes(project) + query += " AND label=?" + args = append(args, s.Label) } + + // Push down WHERE conditions into SQL where possible + var unpushedConditions []Condition + if pushDown != nil { + for _, c := range pushDown.Conditions { + col, canPush := sqlPushableColumns[c.Property] + if !canPush { + unpushedConditions = append(unpushedConditions, c) + continue + } + switch c.Operator { + case "=": + query += " AND " + col + "=?" + args = append(args, c.Value) + case "CONTAINS": + query += " AND " + col + " LIKE ?" + args = append(args, "%"+c.Value+"%") + case "STARTS WITH": + query += " AND " + col + " LIKE ?" + args = append(args, c.Value+"%") + default: + // =~, numeric comparisons: can't push to SQL easily + unpushedConditions = append(unpushedConditions, c) + } + } + } + + rows, err := e.Store.DB().Query(query, args...) if err != nil { return nil, fmt.Errorf("scan nodes: %w", err) } + defer rows.Close() + + var nodes []*store.Node + for rows.Next() { + n, scanErr := scanNodeFromRows(rows) + if scanErr != nil { + return nil, scanErr + } + nodes = append(nodes, n) + } + if err := rows.Err(); err != nil { + return nil, err + } // Apply inline property filters if len(s.Props) > 0 { nodes = filterNodesByProps(nodes, s.Props) } - var bindings []binding + bindings := make([]binding, 0, len(nodes)) for _, n := range nodes { + // Apply unpushed conditions in Go + if len(unpushedConditions) > 0 { + b := newBinding() + b.nodes[s.Variable] = n + match, evalErr := evaluateConditions(b, unpushedConditions, "AND") + if evalErr != nil { + return nil, evalErr + } + if !match { + continue + } + } b := newBinding() if s.Variable != "" { b.nodes[s.Variable] = n @@ -138,6 +211,17 @@ func (e *Executor) execScan(project string, s *ScanNodes, _ []binding) ([]bindin return bindings, nil } +// scanNodeFromRows scans a single node from sql.Rows. +func scanNodeFromRows(rows *sql.Rows) (*store.Node, error) { + var n store.Node + var props string + if err := rows.Scan(&n.ID, &n.Project, &n.Label, &n.Name, &n.QualifiedName, &n.FilePath, &n.StartLine, &n.EndLine, &props); err != nil { + return nil, err + } + n.Properties = store.UnmarshalProps(props) + return &n, nil +} + func (e *Executor) execExpand(s *ExpandRelationship, bindings []binding) ([]binding, error) { if len(bindings) == 0 { return nil, nil @@ -145,7 +229,7 @@ func (e *Executor) execExpand(s *ExpandRelationship, bindings []binding) ([]bind isVariableLength := s.MinHops != 1 || s.MaxHops != 1 - var result []binding + result := make([]binding, 0, len(bindings)) for _, b := range bindings { fromNode, ok := b.nodes[s.FromVar] if !ok { @@ -180,7 +264,7 @@ func (e *Executor) expandFixedLength(b binding, fromNode *store.Node, s *ExpandR return nil, err } - var result []binding + result := make([]binding, 0, len(adjacents)) for _, adj := range adjacents { if s.ToLabel != "" && adj.Node.Label != s.ToLabel { continue @@ -221,7 +305,7 @@ func (e *Executor) expandVariableLength(b binding, fromNode *store.Node, s *Expa return nil, fmt.Errorf("bfs: %w", err) } - var result []binding + result := make([]binding, 0, len(bfsResult.Visited)) for _, nh := range bfsResult.Visited { if nh.Hop < s.MinHops { continue @@ -245,94 +329,94 @@ func (e *Executor) expandVariableLength(b binding, fromNode *store.Node, s *Expa return result, nil } +// findAdjacentNodes returns nodes connected to nodeID via the given edge types and direction. func (e *Executor) findAdjacentNodes(nodeID int64, edgeTypes []string, direction string) ([]adjacentResult, error) { - var allEdges []*store.Edge + allEdges, err := e.collectEdges(nodeID, edgeTypes, direction) + if err != nil { + return nil, err + } + return e.resolveEdgeTargets(allEdges, nodeID, direction) +} +// collectEdges gathers edges from the store based on direction and type filters. +func (e *Executor) collectEdges(nodeID int64, edgeTypes []string, direction string) ([]*store.Edge, error) { switch direction { case "outbound": - if len(edgeTypes) > 0 { - for _, et := range edgeTypes { - edges, err := e.Store.FindEdgesBySourceAndType(nodeID, et) - if err != nil { - return nil, err - } - allEdges = append(allEdges, edges...) - } - } else { - edges, err := e.Store.FindEdgesBySource(nodeID) - if err != nil { - return nil, err - } - allEdges = edges - } + return e.collectDirectionalEdges(nodeID, edgeTypes, true) case "inbound": - if len(edgeTypes) > 0 { - for _, et := range edgeTypes { - edges, err := e.Store.FindEdgesByTargetAndType(nodeID, et) - if err != nil { - return nil, err - } - allEdges = append(allEdges, edges...) + return e.collectDirectionalEdges(nodeID, edgeTypes, false) + case "any": + return e.collectBidirectionalEdges(nodeID, edgeTypes) + default: + return e.collectDirectionalEdges(nodeID, edgeTypes, true) + } +} + +// collectDirectionalEdges collects edges in one direction (outbound if isSource, inbound otherwise). +func (e *Executor) collectDirectionalEdges(nodeID int64, edgeTypes []string, isSource bool) ([]*store.Edge, error) { + if len(edgeTypes) > 0 { + var allEdges []*store.Edge + for _, et := range edgeTypes { + var edges []*store.Edge + var err error + if isSource { + edges, err = e.Store.FindEdgesBySourceAndType(nodeID, et) + } else { + edges, err = e.Store.FindEdgesByTargetAndType(nodeID, et) } - } else { - edges, err := e.Store.FindEdgesByTarget(nodeID) if err != nil { return nil, err } - allEdges = edges + allEdges = append(allEdges, edges...) } - case "any": - outEdges, err := e.Store.FindEdgesBySource(nodeID) - if err != nil { - return nil, err - } - inEdges, err := e.Store.FindEdgesByTarget(nodeID) - if err != nil { - return nil, err - } - if len(edgeTypes) > 0 { - typeSet := make(map[string]bool, len(edgeTypes)) - for _, et := range edgeTypes { - typeSet[et] = true - } - for _, edge := range outEdges { - if typeSet[edge.Type] { - allEdges = append(allEdges, edge) - } - } - for _, edge := range inEdges { - if typeSet[edge.Type] { - allEdges = append(allEdges, edge) - } - } - } else { - allEdges = append(outEdges, inEdges...) - } - default: - edges, err := e.Store.FindEdgesBySource(nodeID) - if err != nil { - return nil, err - } - allEdges = edges + return allEdges, nil } + if isSource { + return e.Store.FindEdgesBySource(nodeID) + } + return e.Store.FindEdgesByTarget(nodeID) +} - // Resolve edge targets/sources to nodes, preserving the edge - seen := make(map[int64]bool) - var results []adjacentResult - for _, edge := range allEdges { - var targetID int64 - switch direction { - case "inbound": - targetID = edge.SourceID - case "any": - if edge.SourceID == nodeID { - targetID = edge.TargetID - } else { - targetID = edge.SourceID - } - default: - targetID = edge.TargetID +// collectBidirectionalEdges collects edges in both directions, optionally filtered by type. +func (e *Executor) collectBidirectionalEdges(nodeID int64, edgeTypes []string) ([]*store.Edge, error) { + outEdges, err := e.Store.FindEdgesBySource(nodeID) + if err != nil { + return nil, err + } + inEdges, err := e.Store.FindEdgesByTarget(nodeID) + if err != nil { + return nil, err + } + if len(edgeTypes) == 0 { + result := make([]*store.Edge, 0, len(outEdges)+len(inEdges)) + result = append(result, outEdges...) + result = append(result, inEdges...) + return result, nil + } + typeSet := make(map[string]bool, len(edgeTypes)) + for _, et := range edgeTypes { + typeSet[et] = true + } + var allEdges []*store.Edge + for _, edge := range outEdges { + if typeSet[edge.Type] { + allEdges = append(allEdges, edge) } + } + for _, edge := range inEdges { + if typeSet[edge.Type] { + allEdges = append(allEdges, edge) + } + } + return allEdges, nil +} + +// resolveEdgeTargets resolves edges to adjacentResult by looking up the target/source node. +func (e *Executor) resolveEdgeTargets(allEdges []*store.Edge, nodeID int64, direction string) ([]adjacentResult, error) { + seen := make(map[int64]bool) + results := make([]adjacentResult, 0, len(allEdges)) + for _, edge := range allEdges { + targetID := edgeTargetID(edge, nodeID, direction) if seen[targetID] { continue } @@ -347,6 +431,21 @@ func (e *Executor) findAdjacentNodes(nodeID int64, edgeTypes []string, direction return results, nil } +// edgeTargetID returns the ID of the node on the "other end" of the edge relative to nodeID. +func edgeTargetID(edge *store.Edge, nodeID int64, direction string) int64 { + switch direction { + case "inbound": + return edge.SourceID + case "any": + if edge.SourceID == nodeID { + return edge.TargetID + } + return edge.SourceID + default: + return edge.TargetID + } +} + func (e *Executor) execFilter(s *FilterWhere, bindings []binding) ([]binding, error) { var result []binding for _, b := range bindings { @@ -430,10 +529,10 @@ func evaluateCondition(b binding, c Condition) (bool, error) { } } -func compareNumeric(actual any, expected string, op string) (bool, error) { +func compareNumeric(actual any, expected, op string) (bool, error) { expectedNum, err := strconv.ParseFloat(expected, 64) if err != nil { - return false, nil + return false, fmt.Errorf("invalid numeric value %q: %w", expected, err) } var actualNum float64 switch v := actual.(type) { @@ -444,9 +543,9 @@ func compareNumeric(actual any, expected string, op string) (bool, error) { case float64: actualNum = v case string: - n, err := strconv.ParseFloat(v, 64) - if err != nil { - return false, nil + n, parseErr := strconv.ParseFloat(v, 64) + if parseErr != nil { + return false, fmt.Errorf("cannot compare %q as number: %w", v, parseErr) } actualNum = n default: @@ -553,7 +652,7 @@ func (e *Executor) defaultProjection(bindings []binding) (*Result, error) { edgeVarSet[k] = true } } - var cols []string + cols := make([]string, 0, len(varSet)*3+len(edgeVarSet)) for k := range varSet { cols = append(cols, k+".name", k+".qualified_name", k+".label") } @@ -562,7 +661,7 @@ func (e *Executor) defaultProjection(bindings []binding) (*Result, error) { } sort.Strings(cols) - var rows []map[string]any + rows := make([]map[string]any, 0, len(bindings)) for _, b := range bindings { row := make(map[string]any) for varName, node := range b.nodes { @@ -584,51 +683,12 @@ func (e *Executor) defaultProjection(bindings []binding) (*Result, error) { } func (e *Executor) simpleProjection(bindings []binding, ret *ReturnClause) (*Result, error) { - var cols []string - for _, item := range ret.Items { - col := item.Variable - if item.Property != "" { - col = item.Variable + "." + item.Property - } - if item.Alias != "" { - col = item.Alias - } - cols = append(cols, col) - } + cols := buildColumnNames(ret.Items) seen := make(map[string]bool) - var rows []map[string]any + rows := make([]map[string]any, 0, len(bindings)) for _, b := range bindings { - row := make(map[string]any) - for i, item := range ret.Items { - // Try node first, then edge - if node, ok := b.nodes[item.Variable]; ok { - if item.Property == "" { - row[cols[i]] = map[string]any{ - "name": node.Name, - "qualified_name": node.QualifiedName, - "label": node.Label, - "file_path": node.FilePath, - "start_line": node.StartLine, - "end_line": node.EndLine, - } - } else { - row[cols[i]] = getNodeProperty(node, item.Property) - } - } else if edge, ok := b.edges[item.Variable]; ok { - if item.Property == "" { - row[cols[i]] = map[string]any{ - "type": edge.Type, - "source_id": edge.SourceID, - "target_id": edge.TargetID, - } - } else { - row[cols[i]] = getEdgeProperty(edge, item.Property) - } - } else { - row[cols[i]] = nil - } - } + row := buildProjectionRow(b, ret.Items, cols) // DISTINCT check if ret.Distinct { @@ -644,82 +704,20 @@ func (e *Executor) simpleProjection(bindings []binding, ret *ReturnClause) (*Res // ORDER BY if ret.OrderBy != "" { - orderCol := ret.OrderBy - // Find the matching column name - for i, item := range ret.Items { - if item.Alias == orderCol { - orderCol = cols[i] - break - } - } + orderCol := resolveOrderColumn(ret.OrderBy, ret.Items, cols) sortRows(rows, orderCol, ret.OrderDir) } // LIMIT - limit := ret.Limit - if limit <= 0 || limit > maxResultRows { - limit = maxResultRows - } - if len(rows) > limit { - rows = rows[:limit] - } + rows = applyLimit(rows, ret.Limit) return &Result{Columns: cols, Rows: rows}, nil } -func (e *Executor) aggregateResults(bindings []binding, ret *ReturnClause) (*Result, error) { - // Group by non-COUNT items - var groupItems []ReturnItem - var countItem ReturnItem - for _, item := range ret.Items { - if item.Func == "COUNT" { - countItem = item - } else { - groupItems = append(groupItems, item) - } - } - - // Build grouping key -> count - type groupEntry struct { - key string - row map[string]any - count int - } - groups := make(map[string]*groupEntry) - var order []string - - for _, b := range bindings { - row := make(map[string]any) - var keyParts []string - for _, item := range groupItems { - col := item.Variable - if item.Property != "" { - col = item.Variable + "." + item.Property - } - if item.Alias != "" { - col = item.Alias - } - var val any - if node, ok := b.nodes[item.Variable]; ok { - val = getNodeProperty(node, item.Property) - } else if edge, ok := b.edges[item.Variable]; ok { - val = getEdgeProperty(edge, item.Property) - } - row[col] = val - keyParts = append(keyParts, fmt.Sprintf("%v", val)) - } - key := strings.Join(keyParts, "\x00") - if g, ok := groups[key]; ok { - g.count++ - } else { - groups[key] = &groupEntry{key: key, row: row, count: 1} - order = append(order, key) - } - } - - // Build columns - var cols []string - for _, item := range ret.Items { +// buildColumnNames builds column names from return items. +func buildColumnNames(items []ReturnItem) []string { + cols := make([]string, 0, len(items)) + for _, item := range items { col := item.Variable if item.Property != "" { col = item.Variable + "." + item.Property @@ -729,6 +727,83 @@ func (e *Executor) aggregateResults(bindings []binding, ret *ReturnClause) (*Res } cols = append(cols, col) } + return cols +} + +// buildProjectionRow builds a single result row from a binding. +func buildProjectionRow(b binding, items []ReturnItem, cols []string) map[string]any { + row := make(map[string]any) + for i, item := range items { + row[cols[i]] = resolveItemValue(b, item) + } + return row +} + +// resolveItemValue resolves a return item value from a binding (node or edge). +func resolveItemValue(b binding, item ReturnItem) any { + if node, ok := b.nodes[item.Variable]; ok { + return resolveNodeItemValue(node, item.Property) + } + if edge, ok := b.edges[item.Variable]; ok { + return resolveEdgeItemValue(edge, item.Property) + } + return nil +} + +// resolveNodeItemValue resolves a node return item: full node map or single property. +func resolveNodeItemValue(node *store.Node, property string) any { + if property == "" { + return map[string]any{ + "name": node.Name, + "qualified_name": node.QualifiedName, + "label": node.Label, + "file_path": node.FilePath, + "start_line": node.StartLine, + "end_line": node.EndLine, + } + } + return getNodeProperty(node, property) +} + +// resolveEdgeItemValue resolves an edge return item: full edge map or single property. +func resolveEdgeItemValue(edge *store.Edge, property string) any { + if property == "" { + return map[string]any{ + "type": edge.Type, + "source_id": edge.SourceID, + "target_id": edge.TargetID, + } + } + return getEdgeProperty(edge, property) +} + +// resolveOrderColumn maps an ORDER BY field to the actual column name. +func resolveOrderColumn(orderBy string, items []ReturnItem, cols []string) string { + for i, item := range items { + if item.Alias == orderBy { + return cols[i] + } + } + return orderBy +} + +// applyLimit enforces the LIMIT clause on result rows. +func applyLimit(rows []map[string]any, limit int) []map[string]any { + if limit <= 0 || limit > maxResultRows { + limit = maxResultRows + } + if len(rows) > limit { + return rows[:limit] + } + return rows +} + +func (e *Executor) aggregateResults(bindings []binding, ret *ReturnClause) (*Result, error) { + groupItems, countItem := splitAggregateItems(ret.Items) + groups, order := buildGroups(bindings, groupItems) + + // Build columns + cols := buildColumnNames(ret.Items) // Build result rows countCol := countItem.Alias @@ -736,7 +811,7 @@ func (e *Executor) aggregateResults(bindings []binding, ret *ReturnClause) (*Res countCol = "COUNT(" + countItem.Variable + ")" } - var rows []map[string]any + rows := make([]map[string]any, 0, len(order)) for _, key := range order { g := groups[key] row := g.row @@ -750,19 +825,61 @@ func (e *Executor) aggregateResults(bindings []binding, ret *ReturnClause) (*Res } // LIMIT - limit := ret.Limit - if limit <= 0 || limit > maxResultRows { - limit = maxResultRows - } - if len(rows) > limit { - rows = rows[:limit] - } + rows = applyLimit(rows, ret.Limit) return &Result{Columns: cols, Rows: rows}, nil } +// splitAggregateItems separates return items into group-by items and the COUNT item. +func splitAggregateItems(items []ReturnItem) (groupItems []ReturnItem, countItem ReturnItem) { + for _, item := range items { + if item.Func == "COUNT" { + countItem = item + } else { + groupItems = append(groupItems, item) + } + } + return groupItems, countItem +} + +type groupEntry struct { + key string + row map[string]any + count int +} + +// buildGroups groups bindings by non-COUNT items and counts occurrences. +func buildGroups(bindings []binding, groupItems []ReturnItem) (groups map[string]*groupEntry, order []string) { + groups = make(map[string]*groupEntry) + + for _, b := range bindings { + row := make(map[string]any) + var keyParts []string + for _, item := range groupItems { + col := item.Variable + if item.Property != "" { + col = item.Variable + "." + item.Property + } + if item.Alias != "" { + col = item.Alias + } + val := resolveItemValue(b, item) + row[col] = val + keyParts = append(keyParts, fmt.Sprintf("%v", val)) + } + key := strings.Join(keyParts, "\x00") + if g, ok := groups[key]; ok { + g.count++ + } else { + groups[key] = &groupEntry{key: key, row: row, count: 1} + order = append(order, key) + } + } + return groups, order +} + // sortRows sorts rows by the given column. -func sortRows(rows []map[string]any, col string, dir string) { +func sortRows(rows []map[string]any, col, dir string) { sort.SliceStable(rows, func(i, j int) bool { a, b := rows[i][col], rows[j][col] cmp := compareValues(a, b) diff --git a/internal/cypher/lexer.go b/internal/cypher/lexer.go index 224e565ef..de171862e 100644 --- a/internal/cypher/lexer.go +++ b/internal/cypher/lexer.go @@ -30,25 +30,25 @@ const ( TokDesc // DESC // Symbols - TokLParen // ( - TokRParen // ) - TokLBracket // [ - TokRBracket // ] - TokDash // - - TokGT // > - TokLT // < - TokColon // : - TokDot // . - TokLBrace // { - TokRBrace // } - TokStar // * - TokComma // , - TokEQ // = - TokRegex // =~ - TokGTE // >= - TokLTE // <= - TokPipe // | - TokDotDot // .. + TokLParen // ( + TokRParen // ) + TokLBracket // [ + TokRBracket // ] + TokDash // - + TokGT // > + TokLT // < + TokColon // : + TokDot // . + TokLBrace // { + TokRBrace // } + TokStar // * + TokComma // , + TokEQ // = + TokRegex // =~ + TokGTE // >= + TokLTE // <= + TokPipe // | + TokDotDot // .. // Literals TokIdent // identifier @@ -60,9 +60,9 @@ const ( // Token is a single lexer token. type Token struct { - Type TokenType - Value string - Pos int // byte offset in the input + Type TokenType + Value string + Pos int // byte offset in the input } func (t Token) String() string { @@ -90,6 +90,21 @@ var keywords = map[string]TokenType{ "DESC": TokDesc, } +// singleCharTokens maps single-character symbols to their token type. +var singleCharTokens = map[byte]TokenType{ + '(': TokLParen, + ')': TokRParen, + '[': TokLBracket, + ']': TokRBracket, + '{': TokLBrace, + '}': TokRBrace, + '*': TokStar, + ',': TokComma, + '|': TokPipe, + ':': TokColon, + '-': TokDash, +} + // Lexer tokenizes a Cypher query string. type Lexer struct { input string @@ -110,106 +125,99 @@ func (l *Lexer) tokenize() error { for l.pos < len(l.input) { ch := l.input[l.pos] - // Skip whitespace - if unicode.IsSpace(rune(ch)) { - l.pos++ + if l.skipWhitespaceAndComments(ch) { continue } - // Skip line comments - if ch == '/' && l.pos+1 < len(l.input) && l.input[l.pos+1] == '/' { - for l.pos < len(l.input) && l.input[l.pos] != '\n' { - l.pos++ - } - continue + if err := l.lexNextToken(ch); err != nil { + return err } - - // Skip block comments - if ch == '/' && l.pos+1 < len(l.input) && l.input[l.pos+1] == '*' { - l.pos += 2 - for l.pos+1 < len(l.input) { - if l.input[l.pos] == '*' && l.input[l.pos+1] == '/' { - l.pos += 2 - break - } - l.pos++ - } - continue - } - - switch { - case ch == '(': - l.emit(TokLParen, "(") - case ch == ')': - l.emit(TokRParen, ")") - case ch == '[': - l.emit(TokLBracket, "[") - case ch == ']': - l.emit(TokRBracket, "]") - case ch == '{': - l.emit(TokLBrace, "{") - case ch == '}': - l.emit(TokRBrace, "}") - case ch == '*': - l.emit(TokStar, "*") - case ch == ',': - l.emit(TokComma, ",") - case ch == '|': - l.emit(TokPipe, "|") - case ch == ':': - l.emit(TokColon, ":") - case ch == '.': - if l.pos+1 < len(l.input) && l.input[l.pos+1] == '.' { - l.pos++ - l.emit(TokDotDot, "..") - } else { - l.emit(TokDot, ".") - } - case ch == '-': - l.emit(TokDash, "-") - case ch == '>': - if l.pos+1 < len(l.input) && l.input[l.pos+1] == '=' { - l.pos++ - l.emit(TokGTE, ">=") - } else { - l.emit(TokGT, ">") - } - case ch == '<': - if l.pos+1 < len(l.input) && l.input[l.pos+1] == '=' { - l.pos++ - l.emit(TokLTE, "<=") - } else { - l.emit(TokLT, "<") - } - case ch == '=': - if l.pos+1 < len(l.input) && l.input[l.pos+1] == '~' { - l.pos++ - l.emit(TokRegex, "=~") - } else { - l.emit(TokEQ, "=") - } - case ch == '"' || ch == '\'': - if err := l.lexString(ch); err != nil { - return err - } - continue // lexString advances pos itself - case isDigit(ch): - l.lexNumber() - continue // lexNumber advances pos itself - case isIdentStart(ch): - l.lexIdent() - continue // lexIdent advances pos itself - default: - return fmt.Errorf("unexpected char %q at pos %d", string(ch), l.pos) - } - - l.pos++ } l.tokens = append(l.tokens, Token{Type: TokEOF, Value: "", Pos: l.pos}) return nil } +// skipWhitespaceAndComments skips whitespace and // or /* */ comments. +// Returns true if something was skipped (caller should continue the loop). +func (l *Lexer) skipWhitespaceAndComments(ch byte) bool { + if unicode.IsSpace(rune(ch)) { + l.pos++ + return true + } + if ch == '/' && l.pos+1 < len(l.input) && l.input[l.pos+1] == '/' { + for l.pos < len(l.input) && l.input[l.pos] != '\n' { + l.pos++ + } + return true + } + if ch == '/' && l.pos+1 < len(l.input) && l.input[l.pos+1] == '*' { + l.pos += 2 + for l.pos+1 < len(l.input) { + if l.input[l.pos] == '*' && l.input[l.pos+1] == '/' { + l.pos += 2 + break + } + l.pos++ + } + return true + } + return false +} + +// lexNextToken dispatches a single token starting at l.pos. +func (l *Lexer) lexNextToken(ch byte) error { + // Single-character tokens + if tok, ok := singleCharTokens[ch]; ok { + l.emit(tok, string(ch)) + l.pos++ + return nil + } + + switch { + case ch == '.': + l.lexDot() + case ch == '>': + l.lexTwoChar('=', TokGTE, TokGT, ">") + case ch == '<': + l.lexTwoChar('=', TokLTE, TokLT, "<") + case ch == '=': + l.lexTwoChar('~', TokRegex, TokEQ, "=") + case ch == '"' || ch == '\'': + return l.lexString(ch) + case isDigit(ch): + l.lexNumber() + case isIdentStart(ch): + l.lexIdent() + default: + return fmt.Errorf("unexpected char %q at pos %d", string(ch), l.pos) + } + return nil +} + +// lexDot handles '.' and '..' tokens. +func (l *Lexer) lexDot() { + if l.pos+1 < len(l.input) && l.input[l.pos+1] == '.' { + l.emit(TokDotDot, "..") + l.pos += 2 + } else { + l.emit(TokDot, ".") + l.pos++ + } +} + +// lexTwoChar handles two-character tokens like >=, <=, =~. +// If the next char matches second, emit the compound token; otherwise emit the single token. +func (l *Lexer) lexTwoChar(second byte, compoundTok, singleTok TokenType, singleVal string) { + if l.pos+1 < len(l.input) && l.input[l.pos+1] == second { + l.emit(compoundTok, singleVal+string(second)) + l.pos += 2 + } else { + l.emit(singleTok, singleVal) + l.pos++ + } +} + func (l *Lexer) emit(typ TokenType, val string) { l.tokens = append(l.tokens, Token{Type: typ, Value: val, Pos: l.pos}) } diff --git a/internal/cypher/parser.go b/internal/cypher/parser.go index 4cd3ed79d..1d87e97e0 100644 --- a/internal/cypher/parser.go +++ b/internal/cypher/parser.go @@ -34,12 +34,12 @@ func (p *Parser) advance() Token { return t } -func (p *Parser) expect(typ TokenType) (Token, error) { +func (p *Parser) expect(typ TokenType) error { t := p.advance() if t.Type != typ { - return t, fmt.Errorf("expected token %d, got %d (%q) at pos %d", typ, t.Type, t.Value, t.Pos) + return fmt.Errorf("expected token %d, got %d (%q) at pos %d", typ, t.Type, t.Value, t.Pos) } - return t, nil + return nil } func (p *Parser) parseQuery() (*Query, error) { @@ -77,7 +77,7 @@ func (p *Parser) parseQuery() (*Query, error) { } func (p *Parser) parseMatch() (*MatchClause, error) { - if _, err := p.expect(TokMatch); err != nil { + if err := p.expect(TokMatch); err != nil { return nil, err } pat, err := p.parsePattern() @@ -132,7 +132,7 @@ func (p *Parser) parseRelAndNode() (*RelPattern, *NodePattern, error) { } // Expect dash - if _, err := p.expect(TokDash); err != nil { + if err := p.expect(TokDash); err != nil { return nil, nil, fmt.Errorf("expected '-' in relationship: %w", err) } @@ -144,7 +144,7 @@ func (p *Parser) parseRelAndNode() (*RelPattern, *NodePattern, error) { } // Expect dash - if _, err := p.expect(TokDash); err != nil { + if err := p.expect(TokDash); err != nil { return nil, nil, fmt.Errorf("expected '-' after relationship: %w", err) } @@ -201,7 +201,7 @@ func (p *Parser) parseRelBracket(rel *RelPattern) error { } // Expect ] - if _, err := p.expect(TokRBracket); err != nil { + if err := p.expect(TokRBracket); err != nil { return fmt.Errorf("expected ']' to close relationship: %w", err) } @@ -236,7 +236,8 @@ func (p *Parser) parseHopRange(rel *RelPattern) error { // *3 min=1, max=3 (shorthand) // (empty) min=1, max=0 (unbounded) - if p.peek().Type == TokNumber { + switch p.peek().Type { + case TokNumber: n, _ := strconv.Atoi(p.advance().Value) if p.peek().Type == TokDotDot { // *N..M or *N.. @@ -253,7 +254,7 @@ func (p *Parser) parseHopRange(rel *RelPattern) error { rel.MinHops = 1 rel.MaxHops = n } - } else if p.peek().Type == TokDotDot { + case TokDotDot: // *..M p.advance() // consume .. rel.MinHops = 1 @@ -263,7 +264,7 @@ func (p *Parser) parseHopRange(rel *RelPattern) error { } else { rel.MaxHops = 0 } - } else { + default: // Just * with no range: unbounded rel.MinHops = 1 rel.MaxHops = 0 @@ -273,7 +274,7 @@ func (p *Parser) parseHopRange(rel *RelPattern) error { } func (p *Parser) parseNodePattern() (*NodePattern, error) { - if _, err := p.expect(TokLParen); err != nil { + if err := p.expect(TokLParen); err != nil { return nil, fmt.Errorf("expected '(' for node pattern: %w", err) } @@ -303,7 +304,7 @@ func (p *Parser) parseNodePattern() (*NodePattern, error) { node.Props = props } - if _, err := p.expect(TokRParen); err != nil { + if err := p.expect(TokRParen); err != nil { return nil, fmt.Errorf("expected ')' to close node pattern: %w", err) } @@ -316,7 +317,7 @@ func (p *Parser) parseInlineProps() (map[string]string, error) { for p.peek().Type != TokRBrace { if len(props) > 0 { - if _, err := p.expect(TokComma); err != nil { + if err := p.expect(TokComma); err != nil { return nil, fmt.Errorf("expected ',' between properties: %w", err) } } @@ -328,7 +329,7 @@ func (p *Parser) parseInlineProps() (map[string]string, error) { } // : - if _, err := p.expect(TokColon); err != nil { + if err := p.expect(TokColon); err != nil { return nil, fmt.Errorf("expected ':' after property key: %w", err) } @@ -380,7 +381,7 @@ func (p *Parser) parseCondition() (Condition, error) { } c.Variable = varTok.Value - if _, err := p.expect(TokDot); err != nil { + if err := p.expect(TokDot); err != nil { return c, fmt.Errorf("expected '.' after variable in condition: %w", err) } @@ -469,7 +470,7 @@ func (p *Parser) parseReturn() (*ReturnClause, error) { // Optional ORDER BY if p.peek().Type == TokOrder { p.advance() // consume ORDER - if _, err := p.expect(TokBy); err != nil { + if err := p.expect(TokBy); err != nil { return nil, fmt.Errorf("expected BY after ORDER: %w", err) } orderTok := p.advance() @@ -513,35 +514,53 @@ func (p *Parser) parseReturnItem() (ReturnItem, error) { // Check for COUNT(variable) if p.peek().Type == TokCount { - p.advance() // consume COUNT - item.Func = "COUNT" - if _, err := p.expect(TokLParen); err != nil { - return item, fmt.Errorf("expected '(' after COUNT: %w", err) - } - varTok := p.advance() - if varTok.Type != TokIdent { - return item, fmt.Errorf("expected variable in COUNT(), got %q", varTok.Value) - } - item.Variable = varTok.Value - if _, err := p.expect(TokRParen); err != nil { - return item, fmt.Errorf("expected ')' after COUNT variable: %w", err) - } - } else { - // variable or variable.property - varTok := p.advance() - if varTok.Type != TokIdent { - return item, fmt.Errorf("expected variable in RETURN item, got %q at pos %d", varTok.Value, varTok.Pos) - } - item.Variable = varTok.Value + return p.parseCountItem() + } - if p.peek().Type == TokDot { - p.advance() // consume . - propTok := p.advance() - if propTok.Type != TokIdent { - return item, fmt.Errorf("expected property after '.', got %q", propTok.Value) - } - item.Property = propTok.Value + // variable or variable.property + varTok := p.advance() + if varTok.Type != TokIdent { + return item, fmt.Errorf("expected variable in RETURN item, got %q at pos %d", varTok.Value, varTok.Pos) + } + item.Variable = varTok.Value + + if p.peek().Type == TokDot { + p.advance() // consume . + propTok := p.advance() + if propTok.Type != TokIdent { + return item, fmt.Errorf("expected property after '.', got %q", propTok.Value) } + item.Property = propTok.Value + } + + // Optional AS alias + if p.peek().Type == TokAs { + p.advance() // consume AS + aliasTok := p.advance() + if aliasTok.Type != TokIdent { + return item, fmt.Errorf("expected alias after AS, got %q", aliasTok.Value) + } + item.Alias = aliasTok.Value + } + + return item, nil +} + +// parseCountItem parses a COUNT(variable) [AS alias] expression. +func (p *Parser) parseCountItem() (ReturnItem, error) { + item := ReturnItem{} + p.advance() // consume COUNT + item.Func = "COUNT" + if err := p.expect(TokLParen); err != nil { + return item, fmt.Errorf("expected '(' after COUNT: %w", err) + } + varTok := p.advance() + if varTok.Type != TokIdent { + return item, fmt.Errorf("expected variable in COUNT(), got %q", varTok.Value) + } + item.Variable = varTok.Value + if err := p.expect(TokRParen); err != nil { + return item, fmt.Errorf("expected ')' after COUNT variable: %w", err) } // Optional AS alias diff --git a/internal/cypher/planner.go b/internal/cypher/planner.go index c741a4e1b..f409abf92 100644 --- a/internal/cypher/planner.go +++ b/internal/cypher/planner.go @@ -1,5 +1,7 @@ package cypher +import "fmt" + // Plan represents an execution plan for a parsed Cypher query. type Plan struct { Steps []PlanStep @@ -22,10 +24,10 @@ func (*ScanNodes) stepType() string { return "scan" } // ExpandRelationship follows edges from bound nodes to match target nodes. type ExpandRelationship struct { - FromVar string // source variable (already bound) - ToVar string // target variable (to bind) - RelVar string // optional relationship variable (to bind edge) - ToLabel string // optional label filter on target + FromVar string // source variable (already bound) + ToVar string // target variable (to bind) + RelVar string // optional relationship variable (to bind edge) + ToLabel string // optional label filter on target ToProps map[string]string EdgeTypes []string // required edge types Direction string // "outbound", "inbound", "any" @@ -43,6 +45,24 @@ type FilterWhere struct { func (*FilterWhere) stepType() string { return "filter" } +// asNodePattern safely type-asserts a PatternElement to *NodePattern. +func asNodePattern(el PatternElement) (*NodePattern, error) { + np, ok := el.(*NodePattern) + if !ok { + return nil, fmt.Errorf("expected *NodePattern, got %T", el) + } + return np, nil +} + +// asRelPattern safely type-asserts a PatternElement to *RelPattern. +func asRelPattern(el PatternElement) (*RelPattern, error) { + rp, ok := el.(*RelPattern) + if !ok { + return nil, fmt.Errorf("expected *RelPattern, got %T", el) + } + return rp, nil +} + // BuildPlan converts a parsed Query AST into an execution Plan. func BuildPlan(q *Query) (*Plan, error) { plan := &Plan{ReturnSpec: q.Return} @@ -53,7 +73,10 @@ func BuildPlan(q *Query) (*Plan, error) { } // First element is always a node pattern - firstNode := elements[0].(*NodePattern) + firstNode, err := asNodePattern(elements[0]) + if err != nil { + return nil, fmt.Errorf("first pattern element: %w", err) + } plan.Steps = append(plan.Steps, &ScanNodes{ Variable: firstNode.Variable, Label: firstNode.Label, @@ -63,27 +86,7 @@ func BuildPlan(q *Query) (*Plan, error) { // Optimization: push WHERE conditions that reference only the first // scan variable BEFORE any expand steps. This dramatically reduces the // number of bindings that need to be expanded. - var earlyFilters []Condition - var lateFilters []Condition - - if q.Where != nil { - scanVar := firstNode.Variable - hasExpand := len(elements) > 1 - - if hasExpand && q.Where.Operator == "AND" { - // Split conditions: those referencing only scanVar go early - for _, c := range q.Where.Conditions { - if c.Variable == scanVar { - earlyFilters = append(earlyFilters, c) - } else { - lateFilters = append(lateFilters, c) - } - } - } else { - // Can't split OR conditions or when there's no expand - lateFilters = q.Where.Conditions - } - } + earlyFilters, lateFilters := splitWhereFilters(q.Where, firstNode.Variable, len(elements) > 1) // Insert early filter right after scan if len(earlyFilters) > 0 { @@ -95,11 +98,21 @@ func BuildPlan(q *Query) (*Plan, error) { // Process relationship-node pairs for i := 1; i+1 < len(elements); i += 2 { - rel := elements[i].(*RelPattern) - targetNode := elements[i+1].(*NodePattern) + rel, err := asRelPattern(elements[i]) + if err != nil { + return nil, fmt.Errorf("element[%d]: %w", i, err) + } + targetNode, err := asNodePattern(elements[i+1]) + if err != nil { + return nil, fmt.Errorf("element[%d]: %w", i+1, err) + } + fromNode, err := asNodePattern(elements[i-1]) + if err != nil { + return nil, fmt.Errorf("element[%d]: %w", i-1, err) + } plan.Steps = append(plan.Steps, &ExpandRelationship{ - FromVar: elements[i-1].(*NodePattern).Variable, + FromVar: fromNode.Variable, ToVar: targetNode.Variable, RelVar: rel.Variable, ToLabel: targetNode.Label, @@ -127,3 +140,23 @@ func BuildPlan(q *Query) (*Plan, error) { return plan, nil } + +// splitWhereFilters separates WHERE conditions into early (scan-only) and late filters. +func splitWhereFilters(where *WhereClause, scanVar string, hasExpand bool) (early, late []Condition) { + if where == nil { + return nil, nil + } + + if hasExpand && where.Operator == "AND" { + for _, c := range where.Conditions { + if c.Variable == scanVar { + early = append(early, c) + } else { + late = append(late, c) + } + } + return early, late + } + // Can't split OR conditions or when there's no expand + return nil, where.Conditions +} diff --git a/internal/discover/discover.go b/internal/discover/discover.go index fd186f5fb..2bda463da 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -43,6 +43,22 @@ type Options struct { IgnoreFile string // path to .cgrignore file (optional) } +// shouldSkipDir returns true if the directory should be skipped during discovery. +func shouldSkipDir(name, rel string, extraIgnore []string) bool { + if IGNORE_PATTERNS[name] { + return true + } + for _, pattern := range extraIgnore { + if matched, _ := filepath.Match(pattern, name); matched { + return true + } + if matched, _ := filepath.Match(pattern, rel); matched { + return true + } + } + return false +} + // Discover walks a repository and returns all source files. func Discover(repoPath string, opts *Options) ([]FileInfo, error) { repoPath, err := filepath.Abs(repoPath) @@ -63,26 +79,15 @@ func Discover(repoPath string, opts *Options) ([]FileInfo, error) { err = filepath.Walk(repoPath, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { - return nil // skip errors + return filepath.SkipDir } rel, _ := filepath.Rel(repoPath, path) if info.IsDir() { - name := info.Name() - // Check ignore patterns - if IGNORE_PATTERNS[name] { + if shouldSkipDir(info.Name(), rel, extraIgnore) { return filepath.SkipDir } - // Check .cgrignore patterns - for _, pattern := range extraIgnore { - if matched, _ := filepath.Match(pattern, name); matched { - return filepath.SkipDir - } - if matched, _ := filepath.Match(pattern, rel); matched { - return filepath.SkipDir - } - } return nil } @@ -121,35 +126,35 @@ func Discover(repoPath string, opts *Options) ([]FileInfo, error) { // ignoredJSONFiles are JSON filenames to skip (tool configs, lock files, specs). var ignoredJSONFiles = map[string]bool{ - "package.json": true, - "package-lock.json": true, - "tsconfig.json": true, - "jsconfig.json": true, - "composer.json": true, - "composer.lock": true, - "yarn.lock": true, - "openapi.json": true, - "swagger.json": true, - "jest.config.json": true, - ".eslintrc.json": true, - ".prettierrc.json": true, - ".babelrc.json": true, - "tslint.json": true, - "angular.json": true, - "firebase.json": true, - "renovate.json": true, - "lerna.json": true, - "turbo.json": true, - ".stylelintrc.json": true, - "pnpm-lock.json": true, - "deno.json": true, - "biome.json": true, - "devcontainer.json": true, - ".devcontainer.json": true, - "launch.json": true, - "settings.json": true, - "extensions.json": true, - "tasks.json": true, + "package.json": true, + "package-lock.json": true, + "tsconfig.json": true, + "jsconfig.json": true, + "composer.json": true, + "composer.lock": true, + "yarn.lock": true, + "openapi.json": true, + "swagger.json": true, + "jest.config.json": true, + ".eslintrc.json": true, + ".prettierrc.json": true, + ".babelrc.json": true, + "tslint.json": true, + "angular.json": true, + "firebase.json": true, + "renovate.json": true, + "lerna.json": true, + "turbo.json": true, + ".stylelintrc.json": true, + "pnpm-lock.json": true, + "deno.json": true, + "biome.json": true, + "devcontainer.json": true, + ".devcontainer.json": true, + "launch.json": true, + "settings.json": true, + "extensions.json": true, + "tasks.json": true, } func isIgnoredJSON(name string) bool { diff --git a/internal/httplink/httplink.go b/internal/httplink/httplink.go index 6233fb02f..aa36d08e2 100644 --- a/internal/httplink/httplink.go +++ b/internal/httplink/httplink.go @@ -15,11 +15,12 @@ import ( // RouteHandler represents a discovered HTTP route handler. type RouteHandler struct { - Path string - Method string - FunctionName string - QualifiedName string - HandlerRef string // resolved handler function reference (e.g. "h.CreateOrder") + Path string + Method string + FunctionName string + QualifiedName string + HandlerRef string // resolved handler function reference (e.g. "h.CreateOrder") + ResolvedHandlerQN string // set by createRegistrationCallEdges — actual handler QN } // HTTPCallSite represents a discovered HTTP call site. @@ -29,6 +30,7 @@ type HTTPCallSite struct { SourceName string SourceQualifiedName string SourceLabel string // "Function", "Method", or "Module" + IsAsync bool // true when source uses async dispatch keywords } // HTTPLink represents a matched HTTP call from caller to handler. @@ -37,6 +39,7 @@ type HTTPLink struct { CallerLabel string HandlerQN string URLPath string + EdgeType string // "HTTP_CALLS" or "ASYNC_CALLS" } // Linker discovers cross-service HTTP calls and creates HTTP_CALLS edges. @@ -109,12 +112,12 @@ func (l *Linker) Run() ([]HTTPLink, error) { // Resolve cross-file group prefixes before inserting Route nodes l.resolveCrossFileGroupPrefixes(routes, rootPath) - // Insert Route nodes and HANDLES edges - l.insertRouteNodes(routes) - - // Create CALLS edges for route handler references (e.g. .POST("/path", h.Handler)) + // Resolve handler references first so insertRouteNodes can use the actual handler QN l.createRegistrationCallEdges(routes) + // Insert Route nodes and HANDLES edges (uses ResolvedHandlerQN from above) + l.insertRouteNodes(routes) + callSites := l.discoverCallSites(rootPath) slog.Info("httplink.callsites", "count", len(callSites)) @@ -139,6 +142,12 @@ func (l *Linker) insertRouteNodes(routes []RouteHandler) { routeName := normalMethod + " " + rh.Path + // Use resolved handler QN if available, otherwise fall back to registering function + handlerQN := rh.QualifiedName + if rh.ResolvedHandlerQN != "" { + handlerQN = rh.ResolvedHandlerQN + } + routeID, err := l.store.UpsertNode(&store.Node{ Project: l.project, Label: "Route", @@ -147,29 +156,33 @@ func (l *Linker) insertRouteNodes(routes []RouteHandler) { Properties: map[string]any{ "method": rh.Method, "path": rh.Path, - "handler": rh.QualifiedName, + "handler": handlerQN, }, }) if err != nil || routeID == 0 { continue } - // Create HANDLES edge from handler → Route - handlerNode, _ := l.store.FindNodeByQN(l.project, rh.QualifiedName) + // Create HANDLES edge from actual handler → Route + handlerNode, _ := l.store.FindNodeByQN(l.project, handlerQN) if handlerNode != nil { - l.store.InsertEdge(&store.Edge{ + if _, edgeErr := l.store.InsertEdge(&store.Edge{ Project: l.project, SourceID: handlerNode.ID, TargetID: routeID, Type: "HANDLES", - }) + }); edgeErr != nil { + slog.Warn("httplink.handles_edge.err", "err", edgeErr) + } - // Mark handler as entry point (for Feature 4) + // Mark handler as entry point (for dead code detection) if handlerNode.Properties == nil { handlerNode.Properties = map[string]any{} } handlerNode.Properties["is_entry_point"] = true - l.store.UpsertNode(handlerNode) + if _, upsertErr := l.store.UpsertNode(handlerNode); upsertErr != nil { + slog.Warn("httplink.entry_point.err", "err", upsertErr) + } } } slog.Info("httplink.route_nodes", "count", len(routes)) @@ -255,7 +268,8 @@ func extractPythonRoutes(f *store.Node) []RouteHandler { // to the receiver of .GET/.POST calls (e.g., contracts.POST("", ...) where // contracts := rg.Group("/contracts")). func extractGoRoutes(f *store.Node, source string) []RouteHandler { - var routes []RouteHandler + // Preallocate: most functions register a few routes + routes := make([]RouteHandler, 0, 4) // Build a map of variable name → group prefix from Group() calls groupPrefixes := map[string]string{} @@ -275,23 +289,8 @@ func extractGoRoutes(f *store.Node, source string) []RouteHandler { method := strings.ToUpper(rm[1]) path := rm[2] - // Try to find the receiver variable (e.g., "contracts" in "contracts.POST") - // by looking at what's before the .METHOD( call - idx := strings.Index(line, "."+rm[1]+"(") - if idx > 0 { - prefix := strings.TrimSpace(line[:idx]) - // Take the last word (variable name) - parts := strings.Fields(prefix) - if len(parts) > 0 { - receiver := parts[len(parts)-1] - if gp, ok := groupPrefixes[receiver]; ok { - path = strings.TrimRight(gp, "/") + "/" + strings.TrimLeft(path, "/") - if path == "/" { - path = gp - } - } - } - } + // Try to resolve group prefix from the receiver variable + path = resolveGroupPrefix(line, rm[1], path, groupPrefixes) // Capture handler reference (last argument) for CALLS edge creation var handlerRef string @@ -312,10 +311,35 @@ func extractGoRoutes(f *store.Node, source string) []RouteHandler { return routes } +// resolveGroupPrefix resolves a router group prefix for a route line. +// It finds the receiver variable (e.g., "contracts" in "contracts.POST") +// and looks up its group prefix. +func resolveGroupPrefix(line, method, path string, groupPrefixes map[string]string) string { + idx := strings.Index(line, "."+method+"(") + if idx <= 0 { + return path + } + prefix := strings.TrimSpace(line[:idx]) + parts := strings.Fields(prefix) + if len(parts) == 0 { + return path + } + receiver := parts[len(parts)-1] + gp, ok := groupPrefixes[receiver] + if !ok { + return path + } + resolved := strings.TrimRight(gp, "/") + "/" + strings.TrimLeft(path, "/") + if resolved == "/" { + return gp + } + return resolved +} + // extractExpressRoutes extracts route registrations from JS/TS source (Express/Koa patterns). func extractExpressRoutes(f *store.Node, source string) []RouteHandler { - var routes []RouteHandler matches := expressRouteRe.FindAllStringSubmatch(source, -1) + routes := make([]RouteHandler, 0, len(matches)) for _, m := range matches { routes = append(routes, RouteHandler{ Path: m[2], @@ -391,8 +415,8 @@ func extractRustRoutes(f *store.Node) []RouteHandler { // extractLaravelRoutes extracts route registrations from PHP Laravel source. func extractLaravelRoutes(f *store.Node, source string) []RouteHandler { - var routes []RouteHandler matches := laravelRouteRe.FindAllStringSubmatch(source, -1) + routes := make([]RouteHandler, 0, len(matches)) for _, m := range matches { routes = append(routes, RouteHandler{ Path: m[2], @@ -532,9 +556,24 @@ var httpClientKeywords = []string{ "send_request", "http_client", } +// asyncDispatchKeywords indicate cross-service async dispatch via HTTP. +// Functions containing these keywords create ASYNC_CALLS edges instead of HTTP_CALLS. +var asyncDispatchKeywords = []string{ + // Cloud Tasks (GCP) — task body contains HTTP URL + "CreateTask", "create_task", + // Pub/Sub publish (GCP) — push subscriptions deliver via HTTP + "topic.Publish", "publisher.publish", "topic.publish", + // AWS SQS/SNS — SQS + Lambda/HTTP, SNS + HTTP subscription + "sqs.send_message", "sns.publish", + // RabbitMQ — exchange → HTTP consumer + "basic_publish", + // Kafka — consumer often fronted by HTTP + "producer.send", "producer.Send", +} + // extractFunctionCallSites extracts HTTP paths from function source code. func extractFunctionCallSites(f *store.Node, rootPath string) []HTTPCallSite { - var sites []HTTPCallSite + sites := make([]HTTPCallSite, 0, 4) if f.FilePath == "" || f.StartLine <= 0 || f.EndLine <= 0 { return sites @@ -550,8 +589,8 @@ func extractFunctionCallSites(f *store.Node, rootPath string) []HTTPCallSite { return sites } - // Require at least one HTTP client keyword to avoid false positives - // from functions that merely store URL strings in variables + // Require at least one HTTP client or async dispatch keyword to avoid + // false positives from functions that merely store URL strings in variables hasHTTPClient := false for _, kw := range httpClientKeywords { if strings.Contains(source, kw) { @@ -559,10 +598,22 @@ func extractFunctionCallSites(f *store.Node, rootPath string) []HTTPCallSite { break } } - if !hasHTTPClient { + + hasAsyncDispatch := false + for _, kw := range asyncDispatchKeywords { + if strings.Contains(source, kw) { + hasAsyncDispatch = true + break + } + } + + if !hasHTTPClient && !hasAsyncDispatch { return sites } + // Sync (HTTP client) takes precedence over async dispatch + isAsync := hasAsyncDispatch && !hasHTTPClient + method := detectHTTPMethod(source) paths := extractURLPaths(source) @@ -573,6 +624,7 @@ func extractFunctionCallSites(f *store.Node, rootPath string) []HTTPCallSite { SourceName: f.Name, SourceQualifiedName: f.QualifiedName, SourceLabel: f.Label, + IsAsync: isAsync, }) } @@ -677,51 +729,66 @@ func findJSONBounds(text string) []string { if opener == '[' { closer = ']' } - start := strings.IndexByte(text, opener) - for start >= 0 && start < len(text) { - depth := 0 - inStr := false - for i := start; i < len(text); i++ { - ch := text[i] - if inStr { - if ch == '\\' { - i++ // skip escaped char - continue - } - if ch == '"' { - inStr = false - } - continue - } - if ch == '"' { - inStr = true - } else if ch == opener { - depth++ - } else if ch == closer { - depth-- - if depth == 0 { - candidate := text[start : i+1] - if len(candidate) > 5 { // skip trivially small - results = append(results, candidate) - } - start = i + 1 - break - } - } - } - if depth != 0 { - break - } - next := strings.IndexByte(text[start:], opener) - if next < 0 { - break - } - start += next - } + results = append(results, scanJSONBlocks(text, opener, closer)...) } return results } +// scanJSONBlocks scans text for balanced JSON blocks delimited by opener/closer. +func scanJSONBlocks(text string, opener, closer byte) []string { + var results []string + start := strings.IndexByte(text, opener) + for start >= 0 && start < len(text) { + end, ok := findBalancedEnd(text, start, opener, closer) + if !ok { + break + } + candidate := text[start : end+1] + if len(candidate) > 5 { + results = append(results, candidate) + } + start = end + 1 + next := strings.IndexByte(text[start:], opener) + if next < 0 { + break + } + start += next + } + return results +} + +// findBalancedEnd finds the index of the closing bracket that balances the opener at start. +// Returns the index and true if found, or 0 and false if unbalanced. +func findBalancedEnd(text string, start int, opener, closer byte) (int, bool) { + depth := 0 + inStr := false + for i := start; i < len(text); i++ { + ch := text[i] + if inStr { + if ch == '\\' { + i++ // skip escaped char + continue + } + if ch == '"' { + inStr = false + } + continue + } + switch ch { + case '"': + inStr = true + case opener: + depth++ + case closer: + depth-- + if depth == 0 { + return i, true + } + } + } + return 0, false +} + // walkJSONForURLs recursively walks parsed JSON and extracts URL paths. func walkJSONForURLs(v any, out *[]string) { switch val := v.(type) { @@ -772,7 +839,12 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [ score = 1.0 } - // Create HTTP_CALLS edge with confidence score + // Create edge with confidence score + edgeType := "HTTP_CALLS" + if cs.IsAsync { + edgeType = "ASYNC_CALLS" + } + callerNode, _ := l.store.FindNodeByQN(l.project, cs.SourceQualifiedName) handlerNode, _ := l.store.FindNodeByQN(l.project, rh.QualifiedName) if callerNode != nil && handlerNode != nil { @@ -787,7 +859,7 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [ Project: l.project, SourceID: callerNode.ID, TargetID: handlerNode.ID, - Type: "HTTP_CALLS", + Type: edgeType, Properties: props, }) } @@ -797,6 +869,7 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [ CallerLabel: cs.SourceLabel, HandlerQN: rh.QualifiedName, URLPath: cs.Path, + EdgeType: edgeType, }) } } @@ -805,26 +878,37 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [ } // normalizePath normalizes a URL path for comparison. +// numericSegmentRe matches path segments that are pure numeric IDs. +var numericSegmentRe = regexp.MustCompile(`/\d+(/|$)`) + +// uuidSegmentRe matches path segments that are UUIDs. +var uuidSegmentRe = regexp.MustCompile(`/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(/|$)`) + func normalizePath(path string) string { path = strings.TrimRight(path, "/") path = colonParamRe.ReplaceAllString(path, "*") path = braceParamRe.ReplaceAllString(path, "*") + // Normalize UUIDs and numeric IDs to wildcards for better matching + path = uuidSegmentRe.ReplaceAllString(path, "/*$1") + path = numericSegmentRe.ReplaceAllString(path, "/*$1") return strings.ToLower(path) } // matchConfidenceThreshold is the minimum score for an HTTP_CALLS edge. -const matchConfidenceThreshold = 0.3 +const matchConfidenceThreshold = 0.45 // pathMatchScore returns a confidence score (0.0–1.0) for how well callPath // matches routePath. Returns 0 if no match. // // Multi-signal scoring (inspired by RAD/Code2DFD research): -// confidence = matchBase × (0.5 × jaccard + 0.5 × depthFactor) +// +// confidence = matchBase × (0.5 × jaccard + 0.5 × depthFactor) // // Where: -// matchBase: exact=0.95, suffix=0.75, wildcard=0.55 -// jaccard: segment Jaccard similarity (non-wildcard segments) -// depthFactor: min(matched_segments / 3.0, 1.0) — longer paths = more specific +// +// matchBase: exact=0.95, suffix=0.75, wildcard=0.55 +// jaccard: segment Jaccard similarity (non-wildcard segments) +// depthFactor: min(matched_segments / 3.0, 1.0) — longer paths = more specific func pathMatchScore(callPath, routePath string) float64 { normCall := normalizePath(callPath) normRoute := normalizePath(routePath) @@ -837,15 +921,16 @@ func pathMatchScore(callPath, routePath string) float64 { var matchBase float64 var matchedCallSegs, matchedRouteSegs []string - if normCall == normRoute { + switch { + case normCall == normRoute: matchBase = 0.95 matchedCallSegs = splitSegments(normCall) matchedRouteSegs = splitSegments(normRoute) - } else if strings.HasSuffix(normCall, normRoute) { + case strings.HasSuffix(normCall, normRoute): matchBase = 0.75 matchedCallSegs = splitSegments(normRoute) // use the route portion that matched matchedRouteSegs = splitSegments(normRoute) - } else { + default: // Segment-by-segment wildcard matching callParts := strings.Split(normCall, "/") routeParts := strings.Split(normRoute, "/") @@ -968,8 +1053,9 @@ func pathsMatch(callPath, routePath string) bool { // the nodes are in the same deployable unit. // // Example: "myapp.docker-images.cloud-runs.svcA.module.func" → dir prefix "myapp.docker-images.cloud-runs.svcA" -// "myapp.docker-images.cloud-runs.svcB.routes.handler" → dir prefix "myapp.docker-images.cloud-runs.svcB" -// → different prefix → different service → returns false +// +// "myapp.docker-images.cloud-runs.svcB.routes.handler" → dir prefix "myapp.docker-images.cloud-runs.svcB" +// → different prefix → different service → returns false func sameService(qn1, qn2 string) bool { parts1 := strings.Split(qn1, ".") parts2 := strings.Split(qn2, ".") @@ -997,58 +1083,63 @@ var crossFileGroupVarRe = regexp.MustCompile(`(\w+)\s*:?=\s*\w+\.Group\s*\(\s*"( // resolveCrossFileGroupPrefixes resolves Group() prefixes from caller functions // for routes that were registered without a group prefix within their own function. func (l *Linker) resolveCrossFileGroupPrefixes(routes []RouteHandler, rootPath string) { - // For each function that registered routes, check if callers pass a Group() result for funcQN, indices := range l.routesByFunc { funcNode, _ := l.store.FindNodeByQN(l.project, funcQN) if funcNode == nil { continue } - // Find inbound CALLS edges to this function callerEdges, _ := l.store.FindEdgesByTargetAndType(funcNode.ID, "CALLS") if len(callerEdges) == 0 { continue } - // Check each caller's source for Group() prefix passing - for _, edge := range callerEdges { - callerNode, _ := l.store.FindNodeByID(edge.SourceID) - if callerNode == nil || callerNode.FilePath == "" || callerNode.StartLine <= 0 { - continue - } + l.resolveCallerGroupPrefixes(routes, indices, callerEdges, funcNode.Name, rootPath) + } +} - callerSource := readSourceLines(rootPath, callerNode.FilePath, callerNode.StartLine, callerNode.EndLine) - if callerSource == "" { - continue - } +// resolveCallerGroupPrefixes checks each caller's source for Group() prefix passing +// and prepends the prefix to the routes at the given indices. +func (l *Linker) resolveCallerGroupPrefixes(routes []RouteHandler, indices []int, callerEdges []*store.Edge, funcName, rootPath string) { + for _, edge := range callerEdges { + callerNode, _ := l.store.FindNodeByID(edge.SourceID) + if callerNode == nil || callerNode.FilePath == "" || callerNode.StartLine <= 0 { + continue + } - // Pattern 1: direct — RegisterRoutes(router.Group("/api")) - for _, m := range crossFileGroupRe.FindAllStringSubmatch(callerSource, -1) { - calledFunc := m[1] - prefix := m[2] - if calledFunc == funcNode.Name { - l.prependPrefixToRoutes(routes, indices, prefix) - break - } - } + callerSource := readSourceLines(rootPath, callerNode.FilePath, callerNode.StartLine, callerNode.EndLine) + if callerSource == "" { + continue + } - // Pattern 2: variable-based — v1 := router.Group("/api"); RegisterRoutes(v1) - varPrefixes := map[string]string{} - for _, m := range crossFileGroupVarRe.FindAllStringSubmatch(callerSource, -1) { - varPrefixes[m[1]] = m[2] - } - if len(varPrefixes) > 0 { - // Look for funcName(varName) calls - callRe := regexp.MustCompile(regexp.QuoteMeta(funcNode.Name) + `\s*\(\s*(\w+)`) - for _, cm := range callRe.FindAllStringSubmatch(callerSource, -1) { - argName := cm[1] - if prefix, ok := varPrefixes[argName]; ok { - l.prependPrefixToRoutes(routes, indices, prefix) - break - } - } + // Pattern 1: direct — RegisterRoutes(router.Group("/api")) + for _, m := range crossFileGroupRe.FindAllStringSubmatch(callerSource, -1) { + if m[1] == funcName { + l.prependPrefixToRoutes(routes, indices, m[2]) + break } } + + // Pattern 2: variable-based — v1 := router.Group("/api"); RegisterRoutes(v1) + l.resolveVarGroupPrefix(routes, indices, callerSource, funcName) + } +} + +// resolveVarGroupPrefix resolves Group() prefixes passed via intermediate variables. +func (l *Linker) resolveVarGroupPrefix(routes []RouteHandler, indices []int, callerSource, funcName string) { + varPrefixes := map[string]string{} + for _, m := range crossFileGroupVarRe.FindAllStringSubmatch(callerSource, -1) { + varPrefixes[m[1]] = m[2] + } + if len(varPrefixes) == 0 { + return + } + callRe := regexp.MustCompile(regexp.QuoteMeta(funcName) + `\s*\(\s*(\w+)`) + for _, cm := range callRe.FindAllStringSubmatch(callerSource, -1) { + if prefix, ok := varPrefixes[cm[1]]; ok { + l.prependPrefixToRoutes(routes, indices, prefix) + break + } } } @@ -1069,7 +1160,8 @@ func (l *Linker) prependPrefixToRoutes(routes []RouteHandler, indices []int, pre // to the handler functions they reference (e.g. .POST("/path", h.CreateOrder)). func (l *Linker) createRegistrationCallEdges(routes []RouteHandler) { count := 0 - for _, rh := range routes { + for i := range routes { + rh := &routes[i] if rh.HandlerRef == "" { continue } @@ -1095,6 +1187,9 @@ func (l *Linker) createRegistrationCallEdges(routes []RouteHandler) { // Use the first match (typically unique within a project) handler := handlerNodes[0] + // Propagate resolved handler QN back to route for insertRouteNodes + rh.ResolvedHandlerQN = handler.QualifiedName + _, _ = l.store.InsertEdge(&store.Edge{ Project: l.project, SourceID: registrar.ID, diff --git a/internal/httplink/httplink_test.go b/internal/httplink/httplink_test.go index 01b173684..a93863e7c 100644 --- a/internal/httplink/httplink_test.go +++ b/internal/httplink/httplink_test.go @@ -89,19 +89,19 @@ func TestPathMatchScore(t *testing.T) { max float64 }{ // Exact matches: matchBase=0.95, confidence = 0.95 × (0.5×jaccard + 0.5×depthFactor) - {"/api/orders", "/api/orders", 0.78, 0.82}, // jaccard=1.0, depth=2/3=0.667 → 0.95×0.833 ≈ 0.79 + {"/api/orders", "/api/orders", 0.78, 0.82}, // jaccard=1.0, depth=2/3=0.667 → 0.95×0.833 ≈ 0.79 {"/integrate", "/integrate", 0.60, 0.67}, // jaccard=1.0, depth=1/3=0.333 → 0.95×0.667 ≈ 0.63 {"/api/v1/orders/items", "/api/v1/orders/items", 0.93, 0.96}, // jaccard=1.0, depth=4/3→1.0 → 0.95×1.0 = 0.95 // Suffix matches: matchBase=0.75 {"https://host/api/orders", "/api/orders", 0.60, 0.66}, // jaccard=1.0, depth=0.667 → 0.75×0.833 ≈ 0.625 - // Wildcard matches: matchBase=0.55 - {"/api/orders/123", "/api/orders/:id", 0.43, 0.48}, // jaccard({api,orders,123}∩{api,orders})=2/3=0.667, depth=1.0 → 0.55×0.833 ≈ 0.458 + // Numeric IDs normalized to wildcard → exact match with :id (also normalized to *) + {"/api/orders/123", "/api/orders/:id", 0.90, 0.96}, // both normalize to /api/orders/* → exact match // No match {"/api/users", "/api/orders", 0.0, 0.0}, - {"/", "/api/orders", 0.0, 0.0}, // empty normalized + {"/", "/api/orders", 0.0, 0.0}, // empty normalized {"", "/api/orders", 0.0, 0.0}, } for _, tt := range tests { @@ -120,14 +120,14 @@ func TestSameService(t *testing.T) { }{ // Full directory comparison: strip last 2 segments (module+name), compare rest // "a.b.c.mod.func" → dir="a.b.c", so same dir = same service - {"a.b.c.mod.Func1", "a.b.c.mod.Func2", true}, // same dir (a.b.c) - {"a.b.c.mod.Func1", "a.b.x.mod.Func2", false}, // different dir (a.b.c vs a.b.x) - {"a.b.c.d.mod.Func", "a.b.c.d.mod.Other", true}, // same deep dir (a.b.c.d) - {"a.b.c.d.mod.Func", "a.b.c.e.mod.Other", false}, // different deep dir - {"short.x", "short.y", false}, // only 2 segments → strip leaves empty → false - {"a.b", "a.b", false}, // 2 segments → not enough to determine - {"a.b.c", "a.b.c", true}, // 3 segments: dir="a", same - {"a.b.c", "x.b.c", false}, // 3 segments: dir="a" vs "x" + {"a.b.c.mod.Func1", "a.b.c.mod.Func2", true}, // same dir (a.b.c) + {"a.b.c.mod.Func1", "a.b.x.mod.Func2", false}, // different dir (a.b.c vs a.b.x) + {"a.b.c.d.mod.Func", "a.b.c.d.mod.Other", true}, // same deep dir (a.b.c.d) + {"a.b.c.d.mod.Func", "a.b.c.e.mod.Other", false}, // different deep dir + {"short.x", "short.y", false}, // only 2 segments → strip leaves empty → false + {"a.b", "a.b", false}, // 2 segments → not enough to determine + {"a.b.c", "a.b.c", true}, // 3 segments: dir="a", same + {"a.b.c", "x.b.c", false}, // 3 segments: dir="a" vs "x" // Realistic multi-service QN patterns {"myapp.docker-images.cloud-runs.order-service.main.Func", "myapp.docker-images.cloud-runs.order-service.handlers.Other", true}, {"myapp.docker-images.cloud-runs.order-service.main.Func", "myapp.docker-images.cloud-runs.notification-service.main.health_check", false}, @@ -252,7 +252,7 @@ func TestReadSourceLines(t *testing.T) { content := "line1\nline2\nline3\nline4\nline5\n" path := filepath.Join(dir, "test.go") - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatal(err) } @@ -285,7 +285,7 @@ func TestLinkerRun(t *testing.T) { } if err := os.WriteFile(filepath.Join(goDir, "client.go"), []byte(`package caller const OrderURL = "https://api.example.com/api/orders" -`), 0o644); err != nil { +`), 0o600); err != nil { t.Fatal(err) } @@ -489,7 +489,7 @@ func TestCrossFileGroupPrefix(t *testing.T) { func setup(r *gin.Engine) { RegisterRoutes(r.Group("/api")) } -`), 0o644); err != nil { +`), 0o600); err != nil { t.Fatal(err) } @@ -499,7 +499,7 @@ func RegisterRoutes(rg *gin.RouterGroup) { rg.GET("/orders", ListOrders) rg.POST("/orders", CreateOrder) } -`), 0o644); err != nil { +`), 0o600); err != nil { t.Fatal(err) } @@ -536,12 +536,14 @@ func RegisterRoutes(rg *gin.RouterGroup) { }) // Create CALLS edge: setup -> RegisterRoutes (as pipeline pass3 would) - s.InsertEdge(&store.Edge{ + if _, err := s.InsertEdge(&store.Edge{ Project: project, SourceID: setupID, TargetID: regID, Type: "CALLS", - }) + }); err != nil { + t.Fatal(err) + } linker := New(s, project) _, err = linker.Run() @@ -588,7 +590,7 @@ func setup(r *gin.Engine) { v1 := r.Group("/api") RegisterRoutes(v1) } -`), 0o644); err != nil { +`), 0o600); err != nil { t.Fatal(err) } @@ -597,7 +599,7 @@ func setup(r *gin.Engine) { func RegisterRoutes(rg *gin.RouterGroup) { rg.GET("/items", ListItems) } -`), 0o644); err != nil { +`), 0o600); err != nil { t.Fatal(err) } @@ -608,26 +610,32 @@ func RegisterRoutes(rg *gin.RouterGroup) { defer s.Close() project := "testproj" - s.UpsertProject(project, dir) + if err := s.UpsertProject(project, dir); err != nil { + t.Fatal(err) + } setupID, _ := s.UpsertNode(&store.Node{ Project: project, Label: "Function", Name: "setup", QualifiedName: "testproj.cmd.main.setup", - FilePath: "cmd/main.go", StartLine: 3, EndLine: 6, + FilePath: "cmd/main.go", StartLine: 3, EndLine: 6, }) regID, _ := s.UpsertNode(&store.Node{ Project: project, Label: "Function", Name: "RegisterRoutes", QualifiedName: "testproj.routes.routes.RegisterRoutes", - FilePath: "routes/routes.go", StartLine: 3, EndLine: 5, + FilePath: "routes/routes.go", StartLine: 3, EndLine: 5, }) - s.InsertEdge(&store.Edge{ + if _, err := s.InsertEdge(&store.Edge{ Project: project, SourceID: setupID, TargetID: regID, Type: "CALLS", - }) + }); err != nil { + t.Fatal(err) + } linker := New(s, project) - linker.Run() + if _, runErr := linker.Run(); runErr != nil { + t.Fatal(runErr) + } routeNodes, _ := s.FindNodesByLabel(project, "Route") if len(routeNodes) != 1 { @@ -657,7 +665,7 @@ func RegisterRoutes(rg *gin.RouterGroup) { rg.POST("/orders", h.CreateOrder) rg.GET("/orders/:id", h.GetOrder) } -`), 0o644); err != nil { +`), 0o600); err != nil { t.Fatal(err) } @@ -668,29 +676,39 @@ func RegisterRoutes(rg *gin.RouterGroup) { defer s.Close() project := "testproj" - s.UpsertProject(project, dir) + if err := s.UpsertProject(project, dir); err != nil { + t.Fatal(err) + } // Create the registering function - s.UpsertNode(&store.Node{ + if _, err := s.UpsertNode(&store.Node{ Project: project, Label: "Function", Name: "RegisterRoutes", QualifiedName: "testproj.routes.routes.RegisterRoutes", - FilePath: "routes/routes.go", StartLine: 3, EndLine: 6, - }) + FilePath: "routes/routes.go", StartLine: 3, EndLine: 6, + }); err != nil { + t.Fatal(err) + } // Create handler functions (as pipeline would) - s.UpsertNode(&store.Node{ + if _, err := s.UpsertNode(&store.Node{ Project: project, Label: "Method", Name: "CreateOrder", QualifiedName: "testproj.handlers.handler.CreateOrder", - FilePath: "handlers/handler.go", StartLine: 10, EndLine: 30, - }) - s.UpsertNode(&store.Node{ + FilePath: "handlers/handler.go", StartLine: 10, EndLine: 30, + }); err != nil { + t.Fatal(err) + } + if _, err := s.UpsertNode(&store.Node{ Project: project, Label: "Method", Name: "GetOrder", QualifiedName: "testproj.handlers.handler.GetOrder", - FilePath: "handlers/handler.go", StartLine: 32, EndLine: 50, - }) + FilePath: "handlers/handler.go", StartLine: 32, EndLine: 50, + }); err != nil { + t.Fatal(err) + } linker := New(s, project) - linker.Run() + if _, runErr := linker.Run(); runErr != nil { + t.Fatal(runErr) + } // Verify CALLS edges from RegisterRoutes to handlers regNode, _ := s.FindNodeByQN(project, "testproj.routes.routes.RegisterRoutes") @@ -725,6 +743,238 @@ func RegisterRoutes(rg *gin.RouterGroup) { } } +func TestAsyncDispatchKeywords(t *testing.T) { + // Test the full linker flow to verify edge types. + // Uses a Go caller with CreateTask (async) and a Python route handler. + dir, err := os.MkdirTemp("", "httplink-async-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // Write a Go file with CreateTask + URL (async dispatch) + asyncDir := filepath.Join(dir, "taskworker") + if err := os.MkdirAll(asyncDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(asyncDir, "dispatch.go"), []byte(`package taskworker + +func DispatchOrder(orderID string) { + url := "https://api.internal.com/api/orders" + client.CreateTask(ctx, url, payload) +} +`), 0o600); err != nil { + t.Fatal(err) + } + + // Write a Go file with requests.post + URL (sync HTTP call) + syncDir := filepath.Join(dir, "synccaller") + if err := os.MkdirAll(syncDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(syncDir, "caller.go"), []byte(`package synccaller + +func CallOrder() { + url := "https://api.internal.com/api/orders" + requests.post(url, data) +} +`), 0o600); err != nil { + t.Fatal(err) + } + + // Write a Go file with both sync + async keywords (sync takes precedence) + bothDir := filepath.Join(dir, "bothcaller") + if err := os.MkdirAll(bothDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bothDir, "both.go"), []byte(`package bothcaller + +func CallAndDispatch() { + url := "https://api.internal.com/api/orders" + requests.post(url, data) + client.CreateTask(ctx, url, payload) +} +`), 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + project := "testproj" + if err := s.UpsertProject(project, dir); err != nil { + t.Fatal(err) + } + + // Async caller function + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "DispatchOrder", + QualifiedName: "testproj.taskworker.dispatch.DispatchOrder", + FilePath: "taskworker/dispatch.go", StartLine: 3, EndLine: 6, + }) + + // Sync caller function + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "CallOrder", + QualifiedName: "testproj.synccaller.caller.CallOrder", + FilePath: "synccaller/caller.go", StartLine: 3, EndLine: 6, + }) + + // Both keywords caller function + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "CallAndDispatch", + QualifiedName: "testproj.bothcaller.both.CallAndDispatch", + FilePath: "bothcaller/both.go", StartLine: 3, EndLine: 7, + }) + + // Route handler (different service) + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "create_order", + QualifiedName: "testproj.handler.routes.create_order", + FilePath: "handler/routes.py", + Properties: map[string]any{ + "decorators": []any{`@app.post("/api/orders")`}, + }, + }) + + linker := New(s, project) + links, err := linker.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + + // Build a map of callerQN -> edgeType for verification + edgeTypes := map[string]string{} + for _, link := range links { + edgeTypes[link.CallerQN] = link.EdgeType + } + + // Async caller (CreateTask only) -> ASYNC_CALLS + if et, ok := edgeTypes["testproj.taskworker.dispatch.DispatchOrder"]; !ok { + t.Error("expected link from async caller DispatchOrder") + } else if et != "ASYNC_CALLS" { + t.Errorf("DispatchOrder edge type = %q, want ASYNC_CALLS", et) + } + + // Sync caller (requests.post only) -> HTTP_CALLS + if et, ok := edgeTypes["testproj.synccaller.caller.CallOrder"]; !ok { + t.Error("expected link from sync caller CallOrder") + } else if et != "HTTP_CALLS" { + t.Errorf("CallOrder edge type = %q, want HTTP_CALLS", et) + } + + // Both keywords (sync takes precedence) -> HTTP_CALLS + if et, ok := edgeTypes["testproj.bothcaller.both.CallAndDispatch"]; !ok { + t.Error("expected link from both-keyword caller CallAndDispatch") + } else if et != "HTTP_CALLS" { + t.Errorf("CallAndDispatch edge type = %q, want HTTP_CALLS", et) + } + + // Verify edges in store match + asyncNode, _ := s.FindNodeByQN(project, "testproj.taskworker.dispatch.DispatchOrder") + if asyncNode != nil { + asyncEdges, _ := s.FindEdgesBySourceAndType(asyncNode.ID, "ASYNC_CALLS") + if len(asyncEdges) != 1 { + t.Errorf("expected 1 ASYNC_CALLS edge from async caller, got %d", len(asyncEdges)) + } + httpEdges, _ := s.FindEdgesBySourceAndType(asyncNode.ID, "HTTP_CALLS") + if len(httpEdges) != 0 { + t.Errorf("expected 0 HTTP_CALLS edges from async caller, got %d", len(httpEdges)) + } + } + + syncNode, _ := s.FindNodeByQN(project, "testproj.synccaller.caller.CallOrder") + if syncNode != nil { + httpEdges, _ := s.FindEdgesBySourceAndType(syncNode.ID, "HTTP_CALLS") + if len(httpEdges) != 1 { + t.Errorf("expected 1 HTTP_CALLS edge from sync caller, got %d", len(httpEdges)) + } + asyncEdges, _ := s.FindEdgesBySourceAndType(syncNode.ID, "ASYNC_CALLS") + if len(asyncEdges) != 0 { + t.Errorf("expected 0 ASYNC_CALLS edges from sync caller, got %d", len(asyncEdges)) + } + } +} + +func TestExtractFunctionCallSitesAsync(t *testing.T) { + // Test extractFunctionCallSites directly with a temp file containing async keywords. + dir, err := os.MkdirTemp("", "httplink-extract-async-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // Write a Go file with CreateTask and a URL + if err := os.MkdirAll(filepath.Join(dir, "worker"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "worker", "task.go"), []byte(`package worker + +func EnqueueJob(ctx context.Context) { + url := "https://backend.internal.com/api/process" + client.CreateTask(ctx, &taskspb.CreateTaskRequest{ + HttpRequest: &taskspb.HttpRequest{Url: url}, + }) +} +`), 0o600); err != nil { + t.Fatal(err) + } + + node := &store.Node{ + Project: "testproj", Label: "Function", Name: "EnqueueJob", + QualifiedName: "testproj.worker.task.EnqueueJob", + FilePath: "worker/task.go", StartLine: 3, EndLine: 7, + } + + sites := extractFunctionCallSites(node, dir) + if len(sites) == 0 { + t.Fatal("expected at least 1 call site, got 0") + } + + foundAsync := false + for _, s := range sites { + if s.IsAsync { + foundAsync = true + if s.Path != "/api/process" { + t.Errorf("async site path = %q, want /api/process", s.Path) + } + } + } + if !foundAsync { + t.Error("expected at least one call site with IsAsync=true") + } + + // Also test that a function with only sync keywords gets IsAsync=false + if err := os.WriteFile(filepath.Join(dir, "worker", "sync.go"), []byte(`package worker + +func SyncCall(ctx context.Context) { + url := "https://backend.internal.com/api/process" + requests.post(url, data) +} +`), 0o600); err != nil { + t.Fatal(err) + } + + syncNode := &store.Node{ + Project: "testproj", Label: "Function", Name: "SyncCall", + QualifiedName: "testproj.worker.sync.SyncCall", + FilePath: "worker/sync.go", StartLine: 3, EndLine: 6, + } + + syncSites := extractFunctionCallSites(syncNode, dir) + if len(syncSites) == 0 { + t.Fatal("expected at least 1 sync call site, got 0") + } + for _, s := range syncSites { + if s.IsAsync { + t.Errorf("sync call site should have IsAsync=false, got true (path=%s)", s.Path) + } + } +} + func TestLinkerSkipsSameService(t *testing.T) { s, err := store.OpenMemory() if err != nil { diff --git a/internal/lang/java.go b/internal/lang/java.go index 107c154af..992013dd5 100644 --- a/internal/lang/java.go +++ b/internal/lang/java.go @@ -2,8 +2,8 @@ package lang func init() { Register(&LanguageSpec{ - Language: Java, - FileExtensions: []string{".java"}, + Language: Java, + FileExtensions: []string{".java"}, FunctionNodeTypes: []string{"method_declaration", "constructor_declaration"}, ClassNodeTypes: []string{ "class_declaration", diff --git a/internal/lang/scala.go b/internal/lang/scala.go index 12e40f0c0..e59a33fec 100644 --- a/internal/lang/scala.go +++ b/internal/lang/scala.go @@ -2,8 +2,8 @@ package lang func init() { Register(&LanguageSpec{ - Language: Scala, - FileExtensions: []string{".scala", ".sc"}, + Language: Scala, + FileExtensions: []string{".scala", ".sc"}, FunctionNodeTypes: []string{"function_definition", "function_declaration"}, ClassNodeTypes: []string{ "class_definition", diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 07f124c00..e8a2529e2 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -3,15 +3,14 @@ package parser import ( "fmt" "sync" - "unsafe" tree_sitter "github.com/tree-sitter/go-tree-sitter" + tree_sitter_lua "github.com/tree-sitter-grammars/tree-sitter-lua/bindings/go" tree_sitter_cpp "github.com/tree-sitter/tree-sitter-cpp/bindings/go" tree_sitter_go "github.com/tree-sitter/tree-sitter-go/bindings/go" tree_sitter_java "github.com/tree-sitter/tree-sitter-java/bindings/go" tree_sitter_javascript "github.com/tree-sitter/tree-sitter-javascript/bindings/go" - tree_sitter_lua "github.com/tree-sitter-grammars/tree-sitter-lua/bindings/go" tree_sitter_php "github.com/tree-sitter/tree-sitter-php/bindings/go" tree_sitter_python "github.com/tree-sitter/tree-sitter-python/bindings/go" tree_sitter_rust "github.com/tree-sitter/tree-sitter-rust/bindings/go" @@ -29,18 +28,18 @@ var ( func initLanguages() { languagesOnce.Do(func() { languages = map[lang.Language]*tree_sitter.Language{ - lang.Python: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_python.Language())), - lang.JavaScript: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_javascript.Language())), - lang.TypeScript: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_typescript.LanguageTypescript())), - lang.TSX: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_typescript.LanguageTSX())), - lang.Go: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_go.Language())), - lang.Rust: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_rust.Language())), - lang.Java: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_java.Language())), - lang.CPP: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_cpp.Language())), + lang.Python: tree_sitter.NewLanguage(tree_sitter_python.Language()), + lang.JavaScript: tree_sitter.NewLanguage(tree_sitter_javascript.Language()), + lang.TypeScript: tree_sitter.NewLanguage(tree_sitter_typescript.LanguageTypescript()), + lang.TSX: tree_sitter.NewLanguage(tree_sitter_typescript.LanguageTSX()), + lang.Go: tree_sitter.NewLanguage(tree_sitter_go.Language()), + lang.Rust: tree_sitter.NewLanguage(tree_sitter_rust.Language()), + lang.Java: tree_sitter.NewLanguage(tree_sitter_java.Language()), + lang.CPP: tree_sitter.NewLanguage(tree_sitter_cpp.Language()), // C# skipped: upstream module path mismatch (tree-sitter-c-sharp vs tree-sitter-c_sharp) - lang.PHP: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_php.LanguagePHPOnly())), - lang.Lua: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_lua.Language())), - lang.Scala: tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_scala.Language())), + lang.PHP: tree_sitter.NewLanguage(tree_sitter_php.LanguagePHPOnly()), + lang.Lua: tree_sitter.NewLanguage(tree_sitter_lua.Language()), + lang.Scala: tree_sitter.NewLanguage(tree_sitter_scala.Language()), } }) } diff --git a/internal/pipeline/envscan.go b/internal/pipeline/envscan.go new file mode 100644 index 000000000..d04a529ac --- /dev/null +++ b/internal/pipeline/envscan.go @@ -0,0 +1,249 @@ +package pipeline + +import ( + "bufio" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/DeusData/codebase-memory-mcp/internal/discover" +) + +// EnvBinding represents an extracted environment variable with a URL value. +type EnvBinding struct { + Key string + Value string + FilePath string // relative path where found +} + +// ScanProjectEnvURLs walks the project root, scanning all non-ignored files for +// env var assignments where the value looks like a URL. +func ScanProjectEnvURLs(rootPath string) []EnvBinding { + var bindings []EnvBinding + + _ = filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil // skip errors + } + + name := info.Name() + + // Skip ignored directories (reuse discover package patterns) + if info.IsDir() { + if discover.IGNORE_PATTERNS[name] { + return filepath.SkipDir + } + return nil + } + + // Skip secret files + if isSecretFile(name) { + return nil + } + + // Skip ignored suffixes + for suffix := range discover.IGNORE_SUFFIXES { + if strings.HasSuffix(name, suffix) { + return nil + } + } + + // Skip binary/large files + if info.Size() > 1<<20 { // 1MB + return nil + } + + relPath, _ := filepath.Rel(rootPath, path) + if relPath == "" { + return nil + } + + fileBindings := scanFileForEnvURLs(path, relPath) + bindings = append(bindings, fileBindings...) + + return nil + }) + + return bindings +} + +func scanFileForEnvURLs(absPath, relPath string) []EnvBinding { + name := filepath.Base(absPath) + ext := filepath.Ext(name) + lowerName := strings.ToLower(name) + + // Determine which patterns to use based on file type + var patterns []*regexp.Regexp + + switch { + case isDockerfile(lowerName): + patterns = dockerfilePatterns + case ext == ".yaml" || ext == ".yml": + patterns = yamlPatterns + case ext == ".tf" || ext == ".hcl": + patterns = terraformPatterns + case ext == ".sh" || ext == ".bash" || ext == ".zsh": + patterns = shellPatterns + case isEnvFile(lowerName): + patterns = envFilePatterns + case ext == ".toml": + patterns = tomlPatterns + case ext == ".properties" || ext == ".cfg" || ext == ".ini": + patterns = propertiesPatterns + default: + // Not a config file type we scan + return nil + } + + return scanWithPatterns(absPath, relPath, patterns) +} + +// Pattern definitions (compiled once as package-level vars). +var ( + // Dockerfile: ENV KEY=VALUE or ENV KEY VALUE, ARG KEY=VALUE + dockerfilePatterns = []*regexp.Regexp{ + regexp.MustCompile(`^(?:ENV|ARG)\s+(\w+)[= ](.*)`), + } + + // YAML: key: "https://..." or --set-env-vars KEY=VALUE + yamlPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(\w+):\s*["']?(https?://[^\s"']+)`), + regexp.MustCompile(`--set-env-vars\s+(\w+)=(\S+)`), + } + + // Terraform/HCL: default = "https://..." or value = "https://..." + terraformPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?:default|value)\s*=\s*"(https?://[^"]+)"`), + } + + // Shell: export KEY="https://..." or KEY="https://..." + shellPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?:export\s+)?(\w+)=["']?(https?://[^\s"']+)`), + } + + // .env files: KEY=https://... + envFilePatterns = []*regexp.Regexp{ + regexp.MustCompile(`^(\w+)=(https?://[^\s]+)`), + } + + // TOML: key = "https://..." + tomlPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(\w+)\s*=\s*"(https?://[^"]+)"`), + } + + // Properties/cfg/ini: key=https://... + propertiesPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(\w+)\s*=\s*(https?://\S+)`), + } +) + +func isDockerfile(name string) bool { + lower := strings.ToLower(name) + if lower == "dockerfile" { + return true + } + if strings.HasPrefix(lower, "dockerfile.") || strings.HasSuffix(lower, ".dockerfile") { + return true + } + return false +} + +func isEnvFile(name string) bool { + lower := strings.ToLower(name) + if lower == ".env" || strings.HasPrefix(lower, ".env.") || strings.HasSuffix(lower, ".env") { + return true + } + return false +} + +// Secret exclusion patterns. +var secretKeyPattern = regexp.MustCompile( + `(?i)(secret|password|passwd|token|api_key|apikey|private_key|` + + `credential|auth_token|access_key|client_secret|signing_key|` + + `encryption_key|ssh_key|deploy_key|service_account|bearer|jwt_secret)`) + +var secretValuePattern = regexp.MustCompile( + `(?i)(-----BEGIN|AKIA[0-9A-Z]{16}|sk-[a-zA-Z0-9]{20,}|` + + `ghp_[a-zA-Z0-9]{36}|glpat-[a-zA-Z0-9\-]{20,}|xox[bps]-[a-zA-Z0-9\-]+)`) + +func isSecretFile(name string) bool { + lower := strings.ToLower(name) + secretFilePatterns := []string{ + "service_account", "credentials", "key.json", "key.pem", + "id_rsa", "id_ed25519", ".pem", ".key", + } + for _, p := range secretFilePatterns { + if strings.Contains(lower, p) { + return true + } + } + return false +} + +func isSecretBinding(key, value string) bool { + if secretKeyPattern.MatchString(key) { + return true + } + if secretValuePattern.MatchString(value) { + return true + } + return false +} + +func scanWithPatterns(absPath, relPath string, patterns []*regexp.Regexp) []EnvBinding { + f, err := os.Open(absPath) + if err != nil { + return nil + } + defer f.Close() + + var bindings []EnvBinding + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + // Skip comments + if strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") { + continue + } + + for _, pat := range patterns { + matches := pat.FindStringSubmatch(line) + if matches == nil { + continue + } + + var key, value string + switch len(matches) { + case 2: + // Pattern with only value capture (terraform: default = "URL") + key = "_tf_default" + value = matches[1] + case 3: + key = matches[1] + value = strings.Trim(matches[2], `"'`) + default: + continue + } + + // Validate: must be a URL + if !strings.HasPrefix(value, "http://") && !strings.HasPrefix(value, "https://") { + continue + } + + // Secret exclusion + if isSecretBinding(key, value) { + continue + } + + bindings = append(bindings, EnvBinding{ + Key: key, + Value: value, + FilePath: relPath, + }) + } + } + + return bindings +} diff --git a/internal/pipeline/envscan_test.go b/internal/pipeline/envscan_test.go new file mode 100644 index 000000000..e860f713b --- /dev/null +++ b/internal/pipeline/envscan_test.go @@ -0,0 +1,377 @@ +package pipeline + +import ( + "os" + "path/filepath" + "testing" +) + +func createTempFile(t *testing.T, dir, name, content string) { + t.Helper() + fullPath := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(fullPath, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestScanDockerfileEnvURLs(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "Dockerfile", `FROM python:3.9-slim +ENV ORDER_URL=https://api.example.com/api/orders +ENV DB_HOST=localhost +ARG WEBHOOK_URL=https://hooks.example.com/webhook +`) + + bindings := ScanProjectEnvURLs(dir) + + found := map[string]string{} + for _, b := range bindings { + found[b.Key] = b.Value + } + + if v, ok := found["ORDER_URL"]; !ok || v != "https://api.example.com/api/orders" { + t.Errorf("expected ORDER_URL=https://api.example.com/api/orders, got %q", v) + } + if v, ok := found["WEBHOOK_URL"]; !ok || v != "https://hooks.example.com/webhook" { + t.Errorf("expected WEBHOOK_URL=https://hooks.example.com/webhook, got %q", v) + } + if _, ok := found["DB_HOST"]; ok { + t.Error("DB_HOST=localhost should not be extracted (not a URL)") + } +} + +func TestScanYAMLEnvURLs(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "config.yaml", ` +service: + service_url: "https://api.internal.com/api/process" + timeout: 30 + callback_url: "https://hooks.internal.com/callback" +`) + + bindings := ScanProjectEnvURLs(dir) + + found := map[string]string{} + for _, b := range bindings { + found[b.Key] = b.Value + } + + if v, ok := found["service_url"]; !ok || v != "https://api.internal.com/api/process" { + t.Errorf("expected service_url, got %q (ok=%v)", v, ok) + } + if v, ok := found["callback_url"]; !ok || v != "https://hooks.internal.com/callback" { + t.Errorf("expected callback_url, got %q (ok=%v)", v, ok) + } +} + +func TestScanShellEnvURLs(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "setup.sh", `#!/bin/bash +export DB_URL="https://db.example.com/api/sync" +APP_NAME="my-service" +CALLBACK_URL=https://hooks.example.com/notify +`) + + bindings := ScanProjectEnvURLs(dir) + + found := map[string]string{} + for _, b := range bindings { + found[b.Key] = b.Value + } + + if v, ok := found["DB_URL"]; !ok || v != "https://db.example.com/api/sync" { + t.Errorf("expected DB_URL=https://db.example.com/api/sync, got %q", v) + } + if v, ok := found["CALLBACK_URL"]; !ok || v != "https://hooks.example.com/notify" { + t.Errorf("expected CALLBACK_URL=https://hooks.example.com/notify, got %q", v) + } + if _, ok := found["APP_NAME"]; ok { + t.Error("APP_NAME=my-service should not be extracted (not a URL)") + } +} + +func TestScanEnvFileURLs(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, ".env", ` +API_URL=https://api.example.com/v1 +DEBUG=true +SERVICE_URL=https://service.example.com/api +`) + + bindings := ScanProjectEnvURLs(dir) + + found := map[string]string{} + for _, b := range bindings { + found[b.Key] = b.Value + } + + if v, ok := found["API_URL"]; !ok || v != "https://api.example.com/v1" { + t.Errorf("expected API_URL=https://api.example.com/v1, got %q", v) + } + if v, ok := found["SERVICE_URL"]; !ok || v != "https://service.example.com/api" { + t.Errorf("expected SERVICE_URL=https://service.example.com/api, got %q", v) + } + if _, ok := found["DEBUG"]; ok { + t.Error("DEBUG=true should not be extracted (not a URL)") + } +} + +func TestScanTerraformURLs(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "variables.tf", ` +variable "webhook_url" { + description = "Webhook endpoint" + default = "https://api.example.com/webhook" +} + +variable "region" { + default = "us-east-1" +} + +resource "aws_lambda_function" "handler" { + environment { + variables = { + API_ENDPOINT = "https://api.example.com/v2" + } + } +} +`) + + bindings := ScanProjectEnvURLs(dir) + + if len(bindings) == 0 { + t.Fatal("expected at least 1 binding from terraform file") + } + + foundURL := false + for _, b := range bindings { + if b.Value == "https://api.example.com/webhook" { + foundURL = true + } + } + if !foundURL { + t.Error("expected to find https://api.example.com/webhook in terraform bindings") + } +} + +func TestScanTomlURLs(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "config.toml", ` +[service] +base_url = "https://api.example.com" +name = "my-service" +callback_url = "https://hooks.example.com/notify" +`) + + bindings := ScanProjectEnvURLs(dir) + + found := map[string]string{} + for _, b := range bindings { + found[b.Key] = b.Value + } + + if v, ok := found["base_url"]; !ok || v != "https://api.example.com" { + t.Errorf("expected base_url=https://api.example.com, got %q", v) + } + if v, ok := found["callback_url"]; !ok || v != "https://hooks.example.com/notify" { + t.Errorf("expected callback_url=https://hooks.example.com/notify, got %q", v) + } + if _, ok := found["name"]; ok { + t.Error("name=my-service should not be extracted (not a URL)") + } +} + +func TestScanPropertiesURLs(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "app.properties", ` +api.url=https://api.example.com/health +app.name=myapp +service.endpoint=https://service.example.com/api +`) + + bindings := ScanProjectEnvURLs(dir) + + found := map[string]string{} + for _, b := range bindings { + // Properties patterns use dot-separated keys; the regex captures the last key segment + found[b.Key] = b.Value + } + + foundHealthURL := false + foundServiceURL := false + for _, b := range bindings { + if b.Value == "https://api.example.com/health" { + foundHealthURL = true + } + if b.Value == "https://service.example.com/api" { + foundServiceURL = true + } + } + + if !foundHealthURL { + t.Error("expected to find https://api.example.com/health") + } + if !foundServiceURL { + t.Error("expected to find https://service.example.com/api") + } +} + +func TestSecretKeyExclusion(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "Dockerfile", `FROM node:18 +ENV SECRET_TOKEN=https://api.example.com/api +ENV API_KEY=https://api.example.com/v1 +ENV PASSWORD=https://auth.example.com/login +ENV NORMAL_URL=https://api.example.com/orders +`) + + bindings := ScanProjectEnvURLs(dir) + + for _, b := range bindings { + if b.Key == "SECRET_TOKEN" || b.Key == "API_KEY" || b.Key == "PASSWORD" { + t.Errorf("secret key %q should have been excluded", b.Key) + } + } + + found := false + for _, b := range bindings { + if b.Key == "NORMAL_URL" { + found = true + } + } + if !found { + t.Error("NORMAL_URL should not be excluded") + } +} + +func TestSecretValueExclusion(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "deploy.sh", `#!/bin/bash +export GH_URL="https://ghp_abcdefghijklmnopqrstuvwxyz1234567890@github.com/repo" +export NORMAL_ENDPOINT="https://api.example.com/orders" +`) + + bindings := ScanProjectEnvURLs(dir) + + for _, b := range bindings { + if b.Key == "GH_URL" { + t.Error("value containing ghp_ token should have been excluded") + } + } + + found := false + for _, b := range bindings { + if b.Key == "NORMAL_ENDPOINT" { + found = true + } + } + if !found { + t.Error("NORMAL_ENDPOINT should not be excluded") + } +} + +func TestSecretFileExclusion(t *testing.T) { + dir := t.TempDir() + + // Create a file that matches secret file patterns + createTempFile(t, dir, "credentials.sh", `#!/bin/bash +export API_URL="https://api.example.com/v1" +`) + + // Also create a normal file for comparison + createTempFile(t, dir, "setup.sh", `#!/bin/bash +export API_URL="https://api.example.com/v1" +`) + + bindings := ScanProjectEnvURLs(dir) + + for _, b := range bindings { + if b.FilePath == "credentials.sh" { + t.Error("files matching secret patterns (credentials.sh) should be skipped") + } + } + + found := false + for _, b := range bindings { + if b.FilePath == "setup.sh" { + found = true + } + } + if !found { + t.Error("setup.sh should be scanned normally") + } +} + +func TestSkipsIgnoredDirs(t *testing.T) { + dir := t.TempDir() + + // File inside .git directory should be skipped + createTempFile(t, dir, ".git/config.sh", `#!/bin/bash +export API_URL="https://api.example.com/v1" +`) + + // File inside node_modules should be skipped + createTempFile(t, dir, "node_modules/pkg/config.sh", `#!/bin/bash +export API_URL="https://api.example.com/v1" +`) + + // File at root level should be scanned + createTempFile(t, dir, "deploy.sh", `#!/bin/bash +export API_URL="https://api.example.com/v1" +`) + + bindings := ScanProjectEnvURLs(dir) + + for _, b := range bindings { + if filepath.Dir(b.FilePath) == ".git" || filepath.Dir(b.FilePath) == "node_modules/pkg" { + t.Errorf("file in ignored directory should be skipped: %s", b.FilePath) + } + } + + found := false + for _, b := range bindings { + if b.FilePath == "deploy.sh" { + found = true + } + } + if !found { + t.Error("deploy.sh at root level should be scanned") + } +} + +func TestNonURLValuesSkipped(t *testing.T) { + dir := t.TempDir() + + createTempFile(t, dir, "Dockerfile", `FROM python:3.9 +ENV APP_NAME=my-service +ENV PORT=8080 +ENV DEBUG=true +ENV LOG_LEVEL=info +`) + + createTempFile(t, dir, "config.sh", `#!/bin/bash +export REGION="us-east-1" +export COUNT=42 +`) + + bindings := ScanProjectEnvURLs(dir) + + if len(bindings) != 0 { + t.Errorf("expected 0 bindings for non-URL values, got %d", len(bindings)) + for _, b := range bindings { + t.Logf(" unexpected: %s=%s (%s)", b.Key, b.Value, b.FilePath) + } + } +} diff --git a/internal/pipeline/implements.go b/internal/pipeline/implements.go index 609afc95e..6d88766c3 100644 --- a/internal/pipeline/implements.go +++ b/internal/pipeline/implements.go @@ -7,32 +7,43 @@ import ( "github.com/DeusData/codebase-memory-mcp/internal/store" ) +// ifaceInfo holds an interface node and its required method names. +type ifaceInfo struct { + node *store.Node + methods []string +} + // passImplements detects Go interface satisfaction and creates IMPLEMENTS edges. // A struct implements an interface if it has methods matching all interface methods. -func (p *Pipeline) passImplements() error { +func (p *Pipeline) passImplements() { slog.Info("pass5.implements") - interfaces, err := p.Store.FindNodesByLabel(p.ProjectName, "Interface") - if err != nil || len(interfaces) == 0 { + ifaces := p.collectGoInterfaces() + if len(ifaces) == 0 { + return + } + + structMethods, structQNPrefix := p.collectStructMethods() + linkCount := p.matchImplements(ifaces, structMethods, structQNPrefix) + + slog.Info("pass5.implements.done", "links", linkCount) +} + +// collectGoInterfaces returns Go interfaces with their method names. +func (p *Pipeline) collectGoInterfaces() []ifaceInfo { + interfaces, findErr := p.Store.FindNodesByLabel(p.ProjectName, "Interface") + if findErr != nil || len(interfaces) == 0 { return nil } - // Build interface -> method names map - type ifaceInfo struct { - node *store.Node - methods []string - } var ifaces []ifaceInfo - for _, iface := range interfaces { - // Only process Go interfaces (check file extension) if !strings.HasSuffix(iface.FilePath, ".go") { continue } - // Find methods defined by this interface via DEFINES_METHOD edges - edges, err := p.Store.FindEdgesBySourceAndType(iface.ID, "DEFINES_METHOD") - if err != nil || len(edges) == 0 { + edges, edgeErr := p.Store.FindEdgesBySourceAndType(iface.ID, "DEFINES_METHOD") + if edgeErr != nil || len(edges) == 0 { continue } @@ -48,25 +59,21 @@ func (p *Pipeline) passImplements() error { ifaces = append(ifaces, ifaceInfo{node: iface, methods: methodNames}) } } + return ifaces +} - if len(ifaces) == 0 { - return nil +// collectStructMethods builds maps of receiver type -> method names and QN prefixes +// from Go methods with receiver properties. +func (p *Pipeline) collectStructMethods() (structMethods map[string]map[string]bool, structQNPrefix map[string]string) { + methodNodes, findErr := p.Store.FindNodesByLabel(p.ProjectName, "Method") + if findErr != nil { + return nil, nil } - // Build struct -> method names map from Go receiver methods. - // Go methods are stored as "Method" nodes with a "receiver" property - // containing the receiver type like "(h *Handlers)". - methods, err := p.Store.FindNodesByLabel(p.ProjectName, "Method") - if err != nil { - return nil - } + structMethods = make(map[string]map[string]bool) + structQNPrefix = make(map[string]string) - // receiverType -> set of method names - structMethods := make(map[string]map[string]bool) - // receiverType -> one sample method's QN prefix (to find the struct node) - structQNPrefix := make(map[string]string) - - for _, m := range methods { + for _, m := range methodNodes { if !strings.HasSuffix(m.FilePath, ".go") { continue } @@ -89,55 +96,60 @@ func (p *Pipeline) passImplements() error { } structMethods[typeName][m.Name] = true - // Store QN prefix for finding the struct. The method QN is like - // "project.path.module.MethodName" — we want the module QN prefix. if _, exists := structQNPrefix[typeName]; !exists { - // Take everything before the last dot (the method name) if idx := strings.LastIndex(m.QualifiedName, "."); idx > 0 { structQNPrefix[typeName] = m.QualifiedName[:idx] } } } + return +} - // Check each struct against each interface +// matchImplements checks each struct against each interface and creates IMPLEMENTS edges. +func (p *Pipeline) matchImplements( + ifaces []ifaceInfo, + structMethods map[string]map[string]bool, + structQNPrefix map[string]string, +) int { linkCount := 0 for _, iface := range ifaces { for typeName, methodSet := range structMethods { - if satisfies(iface.methods, methodSet) { - // Find or create the struct node. In Go, structs are stored - // as "Class" nodes (since tree-sitter classifies type_declaration - // as a class-like node). - structQN := structQNPrefix[typeName] + "." + typeName - structNode, _ := p.Store.FindNodeByQN(p.ProjectName, structQN) - - // If we can't find a class node by that QN, try to find it - // by searching Class nodes with the matching name - if structNode == nil { - classes, _ := p.Store.FindNodesByLabel(p.ProjectName, "Class") - for _, c := range classes { - if c.Name == typeName && strings.HasSuffix(c.FilePath, ".go") { - structNode = c - break - } - } - } - - if structNode == nil { - continue - } - - _, _ = p.Store.InsertEdge(&store.Edge{ - Project: p.ProjectName, - SourceID: structNode.ID, - TargetID: iface.node.ID, - Type: "IMPLEMENTS", - }) - linkCount++ + if !satisfies(iface.methods, methodSet) { + continue } + + structNode := p.findStructNode(typeName, structQNPrefix) + if structNode == nil { + continue + } + + _, _ = p.Store.InsertEdge(&store.Edge{ + Project: p.ProjectName, + SourceID: structNode.ID, + TargetID: iface.node.ID, + Type: "IMPLEMENTS", + }) + linkCount++ + } + } + return linkCount +} + +// findStructNode looks up the struct/class node for a given receiver type name. +func (p *Pipeline) findStructNode(typeName string, structQNPrefix map[string]string) *store.Node { + if prefix, ok := structQNPrefix[typeName]; ok { + structQN := prefix + "." + typeName + if n, _ := p.Store.FindNodeByQN(p.ProjectName, structQN); n != nil { + return n } } - slog.Info("pass5.implements.done", "links", linkCount) + classes, _ := p.Store.FindNodesByLabel(p.ProjectName, "Class") + for _, c := range classes { + if c.Name == typeName && strings.HasSuffix(c.FilePath, ".go") { + return c + } + } return nil } diff --git a/internal/pipeline/imports.go b/internal/pipeline/imports.go index a777fa61a..7f2d290a1 100644 --- a/internal/pipeline/imports.go +++ b/internal/pipeline/imports.go @@ -21,7 +21,7 @@ func parseImports( ) map[string]string { switch language { case lang.Go: - return parseGoImports(root, source, projectName, relPath) + return parseGoImports(root, source, projectName) case lang.Python: return parsePythonImports(root, source, projectName, relPath) default: @@ -42,7 +42,7 @@ func parseImports( func parseGoImports( root *tree_sitter.Node, source []byte, - projectName, relPath string, + projectName string, ) map[string]string { imports := make(map[string]string) diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 0e575c491..929c57602 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -54,66 +54,22 @@ func New(s *store.Store, repoPath string) *Pipeline { } } -// Run executes the full 3-pass pipeline. +// Run executes the full 3-pass pipeline within a single transaction. +// If file hashes from a previous run exist, only changed files are re-processed. func (p *Pipeline) Run() error { slog.Info("pipeline.start", "project", p.ProjectName, "path", p.RepoPath) - // Create/update project record - if err := p.Store.UpsertProject(p.ProjectName, p.RepoPath); err != nil { - return fmt.Errorf("upsert project: %w", err) - } - - // Discover source files + // Discover source files (filesystem, no DB — runs outside transaction) files, err := discover.Discover(p.RepoPath, nil) if err != nil { return fmt.Errorf("discover: %w", err) } slog.Info("pipeline.discovered", "files", len(files)) - // Pass 1: Structure (directories, packages, folders) - if err := p.passStructure(files); err != nil { - return fmt.Errorf("pass1 structure: %w", err) - } - - // Pass 2: Definitions (functions, classes, modules) - if err := p.passDefinitions(files); err != nil { - return fmt.Errorf("pass2 definitions: %w", err) - } - - // Build function registry from all definition nodes - p.buildRegistry() - - // Store IMPORTS edges from the import maps built during pass 2 - p.passImports() - - // Pass 3: Calls (resolve call targets, create CALLS edges) - if err := p.passCalls(); err != nil { - return fmt.Errorf("pass3 calls: %w", err) - } - - // Clean up AST cache - for _, cached := range p.astCache { - cached.Tree.Close() - } - p.astCache = nil - - // Pass 4: HTTP linking (cross-service HTTP calls) - if err := p.passHTTPLinks(); err != nil { - slog.Warn("pass4.httplink.err", "err", err) - // Don't fail the whole pipeline for HTTP linking - } - - // Pass 5: Interface implementation detection (Go) - if err := p.passImplements(); err != nil { - slog.Warn("pass5.implements.err", "err", err) - } - - // Update file hashes - for _, f := range files { - hash, hashErr := fileHash(f.Path) - if hashErr == nil { - _ = p.Store.UpsertFileHash(p.ProjectName, f.RelPath, hash) - } + if err := p.Store.WithTransaction(func() error { + return p.runPasses(files) + }); err != nil { + return err } nc, _ := p.Store.CountNodes(p.ProjectName) @@ -122,6 +78,259 @@ func (p *Pipeline) Run() error { return nil } +// runPasses executes all indexing passes (called within a transaction). +func (p *Pipeline) runPasses(files []discover.FileInfo) error { + if err := p.Store.UpsertProject(p.ProjectName, p.RepoPath); err != nil { + return fmt.Errorf("upsert project: %w", err) + } + + // Classify files as changed/unchanged using stored hashes + changed, unchanged := p.classifyFiles(files) + + // If all files are changed (first index or no hashes), do full pass + isFullIndex := len(unchanged) == 0 + if isFullIndex { + return p.runFullPasses(files) + } + + slog.Info("incremental.classify", "changed", len(changed), "unchanged", len(unchanged), "total", len(files)) + return p.runIncrementalPasses(files, changed, unchanged) +} + +// runFullPasses runs the complete pipeline (no incremental optimization). +func (p *Pipeline) runFullPasses(files []discover.FileInfo) error { + if err := p.passStructure(files); err != nil { + return fmt.Errorf("pass1 structure: %w", err) + } + p.passDefinitions(files) + p.buildRegistry() + p.passImports() + p.passCalls() + p.cleanupASTCache() + if err := p.passHTTPLinks(); err != nil { + slog.Warn("pass4.httplink.err", "err", err) + } + p.passImplements() + p.updateFileHashes(files) + return nil +} + +// runIncrementalPasses re-indexes only changed files + their dependents. +func (p *Pipeline) runIncrementalPasses( + allFiles []discover.FileInfo, + changed, unchanged []discover.FileInfo, +) error { + // Pass 1: Structure always runs on all files (fast, idempotent upserts) + if err := p.passStructure(allFiles); err != nil { + return fmt.Errorf("pass1 structure: %w", err) + } + + // Remove stale nodes/edges for deleted files + p.removeDeletedFiles(allFiles) + + // Delete nodes for changed files (will be re-created in pass 2) + for _, f := range changed { + _ = p.Store.DeleteNodesByFile(p.ProjectName, f.RelPath) + } + + // Pass 2: Parse changed files only + p.passDefinitions(changed) + + // Build full registry: includes nodes from unchanged files (already in DB) + // plus newly parsed nodes from changed files + p.buildRegistry() + + // Re-build import maps for changed files (already done in passDefinitions) + // Also load import maps for unchanged files from their AST (not cached) + // For correctness, we need the full import map, but unchanged files don't + // have ASTs cached. Rebuild imports only for changed files is sufficient + // since unchanged file import edges still exist in DB. + p.passImports() + + // Determine which files need call re-resolution: + // changed files + files that import any changed module + dependents := p.findDependentFiles(changed, unchanged) + filesToResolve := mergeFiles(changed, dependents) + slog.Info("incremental.resolve", "changed", len(changed), "dependents", len(dependents)) + + // Delete CALLS edges for files being re-resolved + for _, f := range filesToResolve { + _ = p.Store.DeleteEdgesBySourceFile(p.ProjectName, f.RelPath, "CALLS") + } + + // Pass 3: Re-resolve calls for changed + dependent files + p.passCallsForFiles(filesToResolve) + + p.cleanupASTCache() + + // HTTP linking and implements always run fully (they clean up first) + if err := p.passHTTPLinks(); err != nil { + slog.Warn("pass4.httplink.err", "err", err) + } + p.passImplements() + + p.updateFileHashes(allFiles) + return nil +} + +// classifyFiles splits files into changed and unchanged based on stored hashes. +func (p *Pipeline) classifyFiles(files []discover.FileInfo) (changed, unchanged []discover.FileInfo) { + storedHashes, err := p.Store.GetFileHashes(p.ProjectName) + if err != nil || len(storedHashes) == 0 { + return files, nil // no hashes → full index + } + + for _, f := range files { + currentHash, hashErr := fileHash(f.Path) + if hashErr != nil { + changed = append(changed, f) + continue + } + if stored, ok := storedHashes[f.RelPath]; ok && stored == currentHash { + unchanged = append(unchanged, f) + } else { + changed = append(changed, f) + } + } + return changed, unchanged +} + +// findDependentFiles finds unchanged files that import any changed file's module. +func (p *Pipeline) findDependentFiles(changed, unchanged []discover.FileInfo) []discover.FileInfo { + // Build set of module QNs for changed files + changedModules := make(map[string]bool, len(changed)) + for _, f := range changed { + mqn := fqn.ModuleQN(p.ProjectName, f.RelPath) + changedModules[mqn] = true + // Also add folder QN (for Go package-level imports) + dir := filepath.Dir(f.RelPath) + if dir != "." { + changedModules[fqn.FolderQN(p.ProjectName, dir)] = true + } + } + + var dependents []discover.FileInfo + for _, f := range unchanged { + mqn := fqn.ModuleQN(p.ProjectName, f.RelPath) + importMap := p.importMaps[mqn] + // If no cached import map, check the store for IMPORTS edges + if len(importMap) == 0 { + importMap = p.loadImportMapFromDB(mqn) + } + for _, targetQN := range importMap { + if changedModules[targetQN] { + dependents = append(dependents, f) + break + } + } + } + return dependents +} + +// loadImportMapFromDB reconstructs an import map from stored IMPORTS edges. +func (p *Pipeline) loadImportMapFromDB(moduleQN string) map[string]string { + moduleNode, err := p.Store.FindNodeByQN(p.ProjectName, moduleQN) + if err != nil || moduleNode == nil { + return nil + } + edges, err := p.Store.FindEdgesBySourceAndType(moduleNode.ID, "IMPORTS") + if err != nil { + return nil + } + result := make(map[string]string, len(edges)) + for _, e := range edges { + target, tErr := p.Store.FindNodeByID(e.TargetID) + if tErr != nil || target == nil { + continue + } + alias := "" + if a, ok := e.Properties["alias"].(string); ok { + alias = a + } + if alias != "" { + result[alias] = target.QualifiedName + } + } + return result +} + +// passCallsForFiles resolves calls only for the specified files. +func (p *Pipeline) passCallsForFiles(files []discover.FileInfo) { + slog.Info("pass3.calls.incremental", "files", len(files)) + for _, f := range files { + cached, ok := p.astCache[f.RelPath] + if !ok { + // File not in AST cache — need to parse it + source, err := os.ReadFile(f.Path) + if err != nil { + continue + } + tree, err := parser.Parse(f.Language, source) + if err != nil { + continue + } + cached = &cachedAST{Tree: tree, Source: source, Language: f.Language} + p.astCache[f.RelPath] = cached + } + spec := lang.ForLanguage(cached.Language) + if spec == nil { + continue + } + p.processFileCalls(f.RelPath, cached, spec) + } +} + +// removeDeletedFiles removes nodes/edges for files that no longer exist on disk. +func (p *Pipeline) removeDeletedFiles(currentFiles []discover.FileInfo) { + currentSet := make(map[string]bool, len(currentFiles)) + for _, f := range currentFiles { + currentSet[f.RelPath] = true + } + indexed, err := p.Store.ListFilesForProject(p.ProjectName) + if err != nil { + return + } + for _, filePath := range indexed { + if !currentSet[filePath] { + _ = p.Store.DeleteNodesByFile(p.ProjectName, filePath) + _ = p.Store.DeleteFileHash(p.ProjectName, filePath) + slog.Info("incremental.removed", "file", filePath) + } + } +} + +func (p *Pipeline) cleanupASTCache() { + for _, cached := range p.astCache { + cached.Tree.Close() + } + p.astCache = nil +} + +func (p *Pipeline) updateFileHashes(files []discover.FileInfo) { + for _, f := range files { + hash, hashErr := fileHash(f.Path) + if hashErr == nil { + _ = p.Store.UpsertFileHash(p.ProjectName, f.RelPath, hash) + } + } +} + +// mergeFiles returns the union of two file slices (deduped by RelPath). +func mergeFiles(a, b []discover.FileInfo) []discover.FileInfo { + seen := make(map[string]bool, len(a)) + result := make([]discover.FileInfo, 0, len(a)+len(b)) + for _, f := range a { + seen[f.RelPath] = true + result = append(result, f) + } + for _, f := range b { + if !seen[f.RelPath] { + result = append(result, f) + } + } + return result +} + // passStructure creates Project, Folder, Package, File nodes and containment edges. func (p *Pipeline) passStructure(files []discover.FileInfo) error { slog.Info("pass1.structure") @@ -188,7 +397,7 @@ func (p *Pipeline) passStructure(files []discover.FileInfo) error { return nil } -func (p *Pipeline) ensureDirHierarchy(relDir string, created map[string]bool, pkgIndicators map[string]bool) { +func (p *Pipeline) ensureDirHierarchy(relDir string, created, pkgIndicators map[string]bool) { if relDir == "." || relDir == "" || created[relDir] { return } @@ -257,7 +466,7 @@ func (p *Pipeline) dirQN(relDir string) string { } // passDefinitions parses each file and extracts function/class/method/module nodes. -func (p *Pipeline) passDefinitions(files []discover.FileInfo) error { +func (p *Pipeline) passDefinitions(files []discover.FileInfo) { slog.Info("pass2.definitions") for _, f := range files { @@ -272,7 +481,6 @@ func (p *Pipeline) passDefinitions(files []discover.FileInfo) error { // Continue with other files } } - return nil } func (p *Pipeline) processFileDefinitions(f discover.FileInfo) error { @@ -324,13 +532,13 @@ func (p *Pipeline) processFileDefinitions(f discover.FileInfo) error { // Functions / methods if funcTypes[kind] { - p.extractFunction(node, source, f, moduleQN, moduleID, spec) + p.extractFunction(node, source, f, moduleID) return false // don't recurse into function body } // Classes / types / interfaces if classTypes[kind] { - p.extractClass(node, source, f, moduleQN, moduleID, spec) + p.extractClass(node, source, f, moduleID, spec) return false // don't recurse; extractClass handles methods internally } @@ -388,7 +596,7 @@ func (p *Pipeline) processFileDefinitions(f discover.FileInfo) error { func (p *Pipeline) extractFunction( node *tree_sitter.Node, source []byte, f discover.FileInfo, - moduleQN string, moduleID int64, spec *lang.LanguageSpec, + moduleID int64, ) { nameNode := node.ChildByFieldName("name") if nameNode == nil { @@ -446,8 +654,8 @@ func (p *Pipeline) extractFunction( props["is_entry_point"] = true } - startLine := int(node.StartPosition().Row) + 1 - endLine := int(node.EndPosition().Row) + 1 + startLine := safeRowToLine(node.StartPosition().Row) + endLine := safeRowToLine(node.EndPosition().Row) funcID, err := p.Store.UpsertNode(&store.Node{ Project: p.ProjectName, @@ -478,7 +686,7 @@ func (p *Pipeline) extractFunction( func (p *Pipeline) extractClass( node *tree_sitter.Node, source []byte, f discover.FileInfo, - moduleQN string, moduleID int64, spec *lang.LanguageSpec, + moduleID int64, spec *lang.LanguageSpec, ) { nameNode := node.ChildByFieldName("name") if nameNode == nil { @@ -506,8 +714,8 @@ func (p *Pipeline) extractClass( } } - startLine := int(node.StartPosition().Row) + 1 - endLine := int(node.EndPosition().Row) + 1 + startLine := safeRowToLine(node.StartPosition().Row) + endLine := safeRowToLine(node.EndPosition().Row) classID, err := p.Store.UpsertNode(&store.Node{ Project: p.ProjectName, @@ -532,61 +740,70 @@ func (p *Pipeline) extractClass( }) // Extract methods inside the class + p.extractClassMethods(node, source, f, classQN, classID, spec) +} + +// extractClassMethods walks a class AST node and creates Method nodes for each method found. +func (p *Pipeline) extractClassMethods( + classNode *tree_sitter.Node, source []byte, f discover.FileInfo, + classQN string, classID int64, spec *lang.LanguageSpec, +) { funcTypes := toSet(spec.FunctionNodeTypes) - parser.Walk(node, func(child *tree_sitter.Node) bool { - if child.Id() == node.Id() { - return true // skip the class node itself + parser.Walk(classNode, func(child *tree_sitter.Node) bool { + if child.Id() == classNode.Id() { + return true + } + if !funcTypes[child.Kind()] { + return true } - if funcTypes[child.Kind()] { - mn := child.ChildByFieldName("name") - if mn == nil { - return false - } - methodName := parser.NodeText(mn, source) - if methodName == "" { - return false - } - methodQN := classQN + "." + methodName - props := map[string]any{} - - paramsNode := child.ChildByFieldName("parameters") - if paramsNode != nil { - props["signature"] = parser.NodeText(paramsNode, source) - } - for _, field := range []string{"result", "return_type", "type"} { - rtNode := child.ChildByFieldName(field) - if rtNode != nil { - props["return_type"] = parser.NodeText(rtNode, source) - break - } - } - props["is_exported"] = isExported(methodName, f.Language) - - mStartLine := int(child.StartPosition().Row) + 1 - mEndLine := int(child.EndPosition().Row) + 1 - - methodID, upsertErr := p.Store.UpsertNode(&store.Node{ - Project: p.ProjectName, - Label: "Method", - Name: methodName, - QualifiedName: methodQN, - FilePath: f.RelPath, - StartLine: mStartLine, - EndLine: mEndLine, - Properties: props, - }) - if upsertErr == nil { - _, _ = p.Store.InsertEdge(&store.Edge{ - Project: p.ProjectName, - SourceID: classID, - TargetID: methodID, - Type: "DEFINES_METHOD", - }) - } + mn := child.ChildByFieldName("name") + if mn == nil { return false } - return true + methodName := parser.NodeText(mn, source) + if methodName == "" { + return false + } + + methodQN := classQN + "." + methodName + props := map[string]any{} + + paramsNode := child.ChildByFieldName("parameters") + if paramsNode != nil { + props["signature"] = parser.NodeText(paramsNode, source) + } + for _, field := range []string{"result", "return_type", "type"} { + rtNode := child.ChildByFieldName(field) + if rtNode != nil { + props["return_type"] = parser.NodeText(rtNode, source) + break + } + } + props["is_exported"] = isExported(methodName, f.Language) + + mStartLine := safeRowToLine(child.StartPosition().Row) + mEndLine := safeRowToLine(child.EndPosition().Row) + + methodID, upsertErr := p.Store.UpsertNode(&store.Node{ + Project: p.ProjectName, + Label: "Method", + Name: methodName, + QualifiedName: methodQN, + FilePath: f.RelPath, + StartLine: mStartLine, + EndLine: mEndLine, + Properties: props, + }) + if upsertErr == nil { + _, _ = p.Store.InsertEdge(&store.Edge{ + Project: p.ProjectName, + SourceID: classID, + TargetID: methodID, + Type: "DEFINES_METHOD", + }) + } + return false }) } @@ -607,7 +824,7 @@ func (p *Pipeline) buildRegistry() { } // passCalls resolves call targets and creates CALLS edges. -func (p *Pipeline) passCalls() error { +func (p *Pipeline) passCalls() { slog.Info("pass3.calls") for relPath, cached := range p.astCache { @@ -617,11 +834,11 @@ func (p *Pipeline) passCalls() error { } p.processFileCalls(relPath, cached, spec) } - return nil } func (p *Pipeline) processFileCalls(relPath string, cached *cachedAST, spec *lang.LanguageSpec) { callTypes := toSet(spec.CallNodeTypes) + funcTypes := toSet(spec.FunctionNodeTypes) moduleQN := fqn.ModuleQN(p.ProjectName, relPath) root := cached.Tree.RootNode() @@ -649,28 +866,65 @@ func (p *Pipeline) processFileCalls(relPath string, cached *cachedAST, spec *lan callerQN = moduleQN } + // Build a local type map that includes receiver/self scope + localTypeMap := typeMap + + // Python self.method() resolution + if cached.Language == lang.Python && strings.HasPrefix(calleeName, "self.") { + classQN := findEnclosingClassQN(node, cached.Source, p.ProjectName, relPath) + if classQN != "" { + methodName := calleeName[5:] + candidate := classQN + "." + methodName + if n, _ := p.Store.FindNodeByQN(p.ProjectName, candidate); n != nil { + p.createCallEdge(callerQN, candidate) + return false + } + } + } + + // Go receiver scoping — add receiver variable to local type map + if cached.Language == lang.Go { + enclosing := findEnclosingFuncNode(node, funcTypes) + if enclosing != nil { + varName, typeName := parseGoReceiverType(enclosing, cached.Source) + if varName != "" && typeName != "" { + classQN := resolveAsClass(typeName, p.registry, moduleQN, importMap) + if classQN != "" { + localTypeMap = make(TypeMap, len(typeMap)+1) + for k, v := range typeMap { + localTypeMap[k] = v + } + localTypeMap[varName] = classQN + } + } + } + } + // Try type-based resolution for method calls (obj.method) - targetQN := p.resolveCallWithTypes(calleeName, moduleQN, importMap, typeMap) + targetQN := p.resolveCallWithTypes(calleeName, moduleQN, importMap, localTypeMap) if targetQN == "" { return false } - // Create CALLS edge - callerNode, _ := p.Store.FindNodeByQN(p.ProjectName, callerQN) - targetNode, _ := p.Store.FindNodeByQN(p.ProjectName, targetQN) - if callerNode != nil && targetNode != nil { - _, _ = p.Store.InsertEdge(&store.Edge{ - Project: p.ProjectName, - SourceID: callerNode.ID, - TargetID: targetNode.ID, - Type: "CALLS", - }) - } - + p.createCallEdge(callerQN, targetQN) return false }) } +// createCallEdge creates a CALLS edge between caller and target by QN. +func (p *Pipeline) createCallEdge(callerQN, targetQN string) { + callerNode, _ := p.Store.FindNodeByQN(p.ProjectName, callerQN) + targetNode, _ := p.Store.FindNodeByQN(p.ProjectName, targetQN) + if callerNode != nil && targetNode != nil { + _, _ = p.Store.InsertEdge(&store.Edge{ + Project: p.ProjectName, + SourceID: callerNode.ID, + TargetID: targetNode.ID, + Type: "CALLS", + }) + } +} + // resolveCallWithTypes resolves a callee name using the registry, import maps, // and type inference for method dispatch. func (p *Pipeline) resolveCallWithTypes( @@ -904,10 +1158,17 @@ func (p *Pipeline) passImports() { // passHTTPLinks runs the HTTP linker to discover cross-service HTTP calls. func (p *Pipeline) passHTTPLinks() error { - // Clean up stale Route nodes and HTTP_CALLS/HANDLES edges before re-running + // Clean up stale Route nodes and HTTP_CALLS/HANDLES/ASYNC_CALLS edges before re-running _ = p.Store.DeleteNodesByLabel(p.ProjectName, "Route") _ = p.Store.DeleteEdgesByType(p.ProjectName, "HTTP_CALLS") _ = p.Store.DeleteEdgesByType(p.ProjectName, "HANDLES") + _ = p.Store.DeleteEdgesByType(p.ProjectName, "ASYNC_CALLS") + + // Scan config files for env var URLs and create synthetic Module nodes + envBindings := ScanProjectEnvURLs(p.RepoPath) + if len(envBindings) > 0 { + p.injectEnvBindings(envBindings) + } linker := httplink.New(p.Store, p.ProjectName) links, err := linker.Run() @@ -918,6 +1179,72 @@ func (p *Pipeline) passHTTPLinks() error { return nil } +// injectEnvBindings creates or updates Module nodes for config files that contain +// environment variable URL bindings. These synthetic constants feed into the +// HTTP linker's call site discovery. +func (p *Pipeline) injectEnvBindings(bindings []EnvBinding) { + // Group by file path + byFile := make(map[string][]EnvBinding) + for _, b := range bindings { + byFile[b.FilePath] = append(byFile[b.FilePath], b) + } + + count := 0 + for filePath, fileBindings := range byFile { + moduleQN := fqn.ModuleQN(p.ProjectName, filePath) + + // Build constants list from bindings + var constants []string + for _, b := range fileBindings { + constants = append(constants, b.Key+" = "+b.Value) + } + + // Cap at 50 constants per config file + if len(constants) > 50 { + constants = constants[:50] + } + + // Merge with existing constants if module already exists + existing, _ := p.Store.FindNodeByQN(p.ProjectName, moduleQN) + if existing != nil { + if existConsts, ok := existing.Properties["constants"].([]any); ok { + seen := make(map[string]bool, len(existConsts)) + for _, c := range existConsts { + if s, ok := c.(string); ok { + seen[s] = true + } + } + for _, c := range constants { + if !seen[c] { + existConsts = append(existConsts, c) + } + } + if existing.Properties == nil { + existing.Properties = map[string]any{} + } + existing.Properties["constants"] = existConsts + _, _ = p.Store.UpsertNode(existing) + count += len(fileBindings) + continue + } + } + + _, _ = p.Store.UpsertNode(&store.Node{ + Project: p.ProjectName, + Label: "Module", + Name: filepath.Base(filePath), + QualifiedName: moduleQN, + FilePath: filePath, + Properties: map[string]any{"constants": constants}, + }) + count += len(fileBindings) + } + + if count > 0 { + slog.Info("envscan.injected", "bindings", count, "files", len(byFile)) + } +} + // jsonURLKeyPattern matches JSON keys that likely contain URL/endpoint values. var jsonURLKeyPattern = regexp.MustCompile(`(?i)(url|endpoint|base_url|host|api_url|service_url|target_url|callback_url|webhook|href|uri|address|server|origin|proxy|redirect|forward|destination)`) @@ -931,7 +1258,7 @@ func (p *Pipeline) processJSONFile(f discover.FileInfo) error { var parsed any if err := json.Unmarshal(data, &parsed); err != nil { - return nil // skip malformed JSON silently + return fmt.Errorf("json parse: %w", err) } var constants []string @@ -999,14 +1326,21 @@ func looksLikeURL(s string) bool { if strings.HasPrefix(s, "/") && strings.Count(s, "/") >= 2 { // Skip version-like paths: /1.0.0, /v2, /en seg := strings.TrimPrefix(s, "/") - if len(seg) <= 3 { - return false - } - return true + return len(seg) > 3 } return false } +// safeRowToLine converts a tree-sitter row (uint) to a 1-based line number (int). +// Returns math.MaxInt if the value would overflow. +func safeRowToLine(row uint) int { + const maxInt = int(^uint(0) >> 1) // math.MaxInt equivalent without importing math + if row > uint(maxInt-1) { + return maxInt + } + return int(row) + 1 +} + func fileHash(path string) (string, error) { f, err := os.Open(path) if err != nil { diff --git a/internal/pipeline/pipeline_bench_test.go b/internal/pipeline/pipeline_bench_test.go new file mode 100644 index 000000000..663d6ffee --- /dev/null +++ b/internal/pipeline/pipeline_bench_test.go @@ -0,0 +1,332 @@ +package pipeline + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +// setupBenchRepo creates a temp directory with 10 Go files across 2 packages. +// Package "model" defines structs and methods; package "handler" calls into model. +func setupBenchRepo(b *testing.B) (dir string, cleanup func()) { + b.Helper() + dir, err := os.MkdirTemp("", "cgm-bench-*") + if err != nil { + b.Fatal(err) + } + cleanup = func() { os.RemoveAll(dir) } + + writeBenchModelFiles(b, dir) + writeBenchHandlerFiles(b, dir) + + return dir, cleanup +} + +func writeBenchModelFiles(b *testing.B, dir string) { + b.Helper() + + writeBenchFile(b, filepath.Join(dir, "model", "order.go"), `package model + +type Order struct { + ID string + Customer string + Total float64 + Items []Item +} + +func NewOrder(id, customer string) *Order { + return &Order{ID: id, Customer: customer} +} + +func (o *Order) AddItem(item Item) { + o.Items = append(o.Items, item) + o.Total += item.Price +} + +func (o *Order) Validate() error { + if o.ID == "" { + return nil + } + return nil +} +`) + + writeBenchFile(b, filepath.Join(dir, "model", "item.go"), `package model + +type Item struct { + SKU string + Name string + Price float64 + Qty int +} + +func NewItem(sku, name string, price float64, qty int) Item { + return Item{SKU: sku, Name: name, Price: price, Qty: qty} +} + +func (i Item) Subtotal() float64 { + return i.Price * float64(i.Qty) +} +`) + + writeBenchFile(b, filepath.Join(dir, "model", "customer.go"), `package model + +type Customer struct { + ID string + Name string + Email string +} + +func NewCustomer(id, name, email string) *Customer { + return &Customer{ID: id, Name: name, Email: email} +} + +func (c *Customer) DisplayName() string { + if c.Name != "" { + return c.Name + } + return c.Email +} +`) + + writeBenchFile(b, filepath.Join(dir, "model", "inventory.go"), `package model + +type Inventory struct { + Stock map[string]int +} + +func NewInventory() *Inventory { + return &Inventory{Stock: make(map[string]int)} +} + +func (inv *Inventory) Reserve(sku string, qty int) bool { + if inv.Stock[sku] >= qty { + inv.Stock[sku] -= qty + return true + } + return false +} + +func (inv *Inventory) Restock(sku string, qty int) { + inv.Stock[sku] += qty +} +`) + + writeBenchFile(b, filepath.Join(dir, "model", "pricing.go"), `package model + +const ( + TaxRate = 0.19 + DiscountCap = 0.25 +) + +func ApplyTax(amount float64) float64 { + return amount * (1 + TaxRate) +} + +func ApplyDiscount(amount, discount float64) float64 { + if discount > DiscountCap { + discount = DiscountCap + } + return amount * (1 - discount) +} + +func FinalPrice(amount, discount float64) float64 { + return ApplyTax(ApplyDiscount(amount, discount)) +} +`) +} + +func writeBenchHandlerFiles(b *testing.B, dir string) { + b.Helper() + + writeBenchFile(b, filepath.Join(dir, "handler", "order_handler.go"), `package handler + +import "example.com/bench/model" + +func CreateOrder(customerID string, items []model.Item) (*model.Order, error) { + c := model.NewCustomer(customerID, "", "") + order := model.NewOrder("ord-1", c.DisplayName()) + for _, item := range items { + order.AddItem(item) + } + if err := order.Validate(); err != nil { + return nil, err + } + return order, nil +} + +func GetOrderTotal(order *model.Order) float64 { + return model.ApplyTax(order.Total) +} +`) + + writeBenchFile(b, filepath.Join(dir, "handler", "inventory_handler.go"), `package handler + +import "example.com/bench/model" + +func CheckAndReserve(inv *model.Inventory, sku string, qty int) bool { + return inv.Reserve(sku, qty) +} + +func RestockItem(inv *model.Inventory, sku string, qty int) { + inv.Restock(sku, qty) +} +`) + + writeBenchFile(b, filepath.Join(dir, "handler", "pricing_handler.go"), `package handler + +import "example.com/bench/model" + +func CalculatePrice(amount, discount float64) float64 { + return model.FinalPrice(amount, discount) +} + +func QuotePrice(items []model.Item, discount float64) float64 { + var total float64 + for _, item := range items { + total += item.Subtotal() + } + return model.ApplyDiscount(total, discount) +} +`) + + writeBenchFile(b, filepath.Join(dir, "handler", "customer_handler.go"), `package handler + +import "example.com/bench/model" + +func RegisterCustomer(id, name, email string) *model.Customer { + return model.NewCustomer(id, name, email) +} + +func CustomerLabel(c *model.Customer) string { + return c.DisplayName() +} +`) + + writeBenchFile(b, filepath.Join(dir, "handler", "admin_handler.go"), `package handler + +import "example.com/bench/model" + +func AdminRestock(inv *model.Inventory, sku string, qty int) { + inv.Restock(sku, qty) +} + +func AdminCreateOrder(custName string) *model.Order { + order := model.NewOrder("admin-1", custName) + item := model.NewItem("ADMIN-SKU", "Admin Item", 99.99, 1) + order.AddItem(item) + return order +} +`) +} + +func writeBenchFile(b *testing.B, path, content string) { + b.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + b.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + b.Fatal(err) + } +} + +func BenchmarkPipelineRun(b *testing.B) { + repoDir, cleanup := setupBenchRepo(b) + defer cleanup() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + s, err := store.OpenMemory() + if err != nil { + b.Fatal(err) + } + p := New(s, repoDir) + if err := p.Run(); err != nil { + b.Fatalf("Pipeline.Run: %v", err) + } + s.Close() + } +} + +func BenchmarkPipelineReindex(b *testing.B) { + repoDir, cleanup := setupBenchRepo(b) + defer cleanup() + + s, err := store.OpenMemory() + if err != nil { + b.Fatal(err) + } + defer s.Close() + + // Initial index + p := New(s, repoDir) + if err := p.Run(); err != nil { + b.Fatalf("initial index: %v", err) + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + p := New(s, repoDir) + if err := p.Run(); err != nil { + b.Fatalf("reindex %d: %v", i, err) + } + } +} + +func BenchmarkPipelineRunScaled(b *testing.B) { + for _, fileCount := range []int{5, 20, 50} { + b.Run(fmt.Sprintf("files=%d", fileCount), func(b *testing.B) { + dir, err := os.MkdirTemp("", "cgm-bench-scale-*") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(dir) + + for i := 0; i < fileCount; i++ { + pkg := fmt.Sprintf("pkg%d", i%5) + name := fmt.Sprintf("file%d.go", i) + content := fmt.Sprintf(`package %s + +func Func%d() int { + return %d +} + +func Caller%d() int { + return Func%d() +} + +type Struct%d struct { + Field int +} + +func (s *Struct%d) Method%d() int { + return s.Field + Func%d() +} +`, pkg, i, i, i, i, i, i, i, i) + writeBenchFile(b, filepath.Join(dir, pkg, name), content) + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + s, err := store.OpenMemory() + if err != nil { + b.Fatal(err) + } + p := New(s, dir) + if err := p.Run(); err != nil { + b.Fatalf("Pipeline.Run: %v", err) + } + s.Close() + } + }) + } +} diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 1a18de408..55788c0bb 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -8,13 +8,13 @@ import ( "github.com/DeusData/codebase-memory-mcp/internal/store" ) -func setupTestRepo(t *testing.T) (string, func()) { +func setupTestRepo(t *testing.T) (dir string, cleanup func()) { t.Helper() dir, err := os.MkdirTemp("", "cgm-test-*") if err != nil { t.Fatal(err) } - cleanup := func() { os.RemoveAll(dir) } + cleanup = func() { os.RemoveAll(dir) } // Create a simple Go project writeFile(t, filepath.Join(dir, "main.go"), `package main @@ -44,12 +44,12 @@ func SubmitOrder(order interface{}) error { return dir, cleanup } -func writeFile(t *testing.T, path string, content string) { +func writeFile(t *testing.T, path, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatal(err) } } diff --git a/internal/pipeline/resolve.go b/internal/pipeline/resolve.go index 1d45c88c8..c35408768 100644 --- a/internal/pipeline/resolve.go +++ b/internal/pipeline/resolve.go @@ -39,7 +39,7 @@ func resolveModuleStrings(root *tree_sitter.Node, source []byte, language lang.L // resolveAssignment tries to extract a (name, resolved_value) pair from // a top-level AST node. Returns ("","") if the node isn't a string assignment. -func resolveAssignment(node *tree_sitter.Node, source []byte, language lang.Language, symbols map[string]string) (string, string) { +func resolveAssignment(node *tree_sitter.Node, source []byte, language lang.Language, symbols map[string]string) (name, value string) { switch language { case lang.Python: return resolvePython(node, source, symbols) @@ -67,7 +67,7 @@ func resolveAssignment(node *tree_sitter.Node, source []byte, language lang.Lang // --- Python --- // expression_statement → assignment → (identifier, string|binary_operator) -func resolvePython(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolvePython(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { if node.Kind() != "expression_statement" { return "", "" } @@ -80,8 +80,8 @@ func resolvePython(node *tree_sitter.Node, source []byte, symbols map[string]str if nameNode == nil || valueNode == nil || nameNode.Kind() != "identifier" { return "", "" } - name := parser.NodeText(nameNode, source) - value := resolveStringExpr(valueNode, source, symbols) + name = parser.NodeText(nameNode, source) + value = resolveStringExpr(valueNode, source, symbols) return name, value } @@ -89,7 +89,7 @@ func resolvePython(node *tree_sitter.Node, source []byte, symbols map[string]str // const_declaration → const_spec → (identifier, expression_list) // var_declaration → var_spec → (identifier, expression_list) -func resolveGo(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolveGo(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { var spec *tree_sitter.Node switch node.Kind() { case "const_declaration": @@ -114,15 +114,15 @@ func resolveGo(node *tree_sitter.Node, source []byte, symbols map[string]string) return "", "" } } - name := parser.NodeText(nameNode, source) - value := resolveStringExpr(valueNode, source, symbols) + name = parser.NodeText(nameNode, source) + value = resolveStringExpr(valueNode, source, symbols) return name, value } // --- JavaScript / TypeScript / TSX --- // lexical_declaration → variable_declarator → (identifier, value) -func resolveJS(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolveJS(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { if node.Kind() != "lexical_declaration" { return "", "" } @@ -135,8 +135,8 @@ func resolveJS(node *tree_sitter.Node, source []byte, symbols map[string]string) if nameNode == nil || valueNode == nil { return "", "" } - name := parser.NodeText(nameNode, source) - value := resolveStringExpr(valueNode, source, symbols) + name = parser.NodeText(nameNode, source) + value = resolveStringExpr(valueNode, source, symbols) return name, value } @@ -144,7 +144,7 @@ func resolveJS(node *tree_sitter.Node, source []byte, symbols map[string]string) // const_item → (identifier, string_literal|binary_expression|macro_invocation) // let_declaration → (identifier, value) -func resolveRust(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolveRust(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { switch node.Kind() { case "const_item", "let_declaration": // both use field "name" for identifier and "value" for the expression @@ -160,8 +160,8 @@ func resolveRust(node *tree_sitter.Node, source []byte, symbols map[string]strin if nameNode == nil || valueNode == nil { return "", "" } - name := parser.NodeText(nameNode, source) - value := resolveStringExpr(valueNode, source, symbols) + name = parser.NodeText(nameNode, source) + value = resolveStringExpr(valueNode, source, symbols) return name, value } @@ -169,7 +169,7 @@ func resolveRust(node *tree_sitter.Node, source []byte, symbols map[string]strin // class_declaration → class_body → field_declaration → variable_declarator // We need to look inside class bodies for static final fields. -func resolveJava(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolveJava(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { if node.Kind() != "class_declaration" { return "", "" } @@ -192,8 +192,8 @@ func resolveJava(node *tree_sitter.Node, source []byte, symbols map[string]strin if nameNode == nil || valueNode == nil { continue } - name := parser.NodeText(nameNode, source) - value := resolveStringExpr(valueNode, source, symbols) + name = parser.NodeText(nameNode, source) + value = resolveStringExpr(valueNode, source, symbols) if name != "" && value != "" { symbols[name] = value } @@ -204,7 +204,7 @@ func resolveJava(node *tree_sitter.Node, source []byte, symbols map[string]strin // --- PHP --- // expression_statement → assignment_expression → (variable_name, encapsed_string|binary_expression) -func resolvePHP(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolvePHP(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { if node.Kind() != "expression_statement" { return "", "" } @@ -218,11 +218,11 @@ func resolvePHP(node *tree_sitter.Node, source []byte, symbols map[string]string return "", "" } // PHP variable names include $, extract just the name part - name := extractPHPVarName(nameNode, source) + name = extractPHPVarName(nameNode, source) if name == "" { return "", "" } - value := resolveStringExpr(valueNode, source, symbols) + value = resolveStringExpr(valueNode, source, symbols) return name, value } @@ -243,7 +243,7 @@ func extractPHPVarName(node *tree_sitter.Node, source []byte) string { // --- Scala --- // val_definition → (identifier, string|interpolated_string_expression|infix_expression) -func resolveScala(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolveScala(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { if node.Kind() != "val_definition" { return "", "" } @@ -252,8 +252,8 @@ func resolveScala(node *tree_sitter.Node, source []byte, symbols map[string]stri if nameNode == nil || valueNode == nil { return "", "" } - name := parser.NodeText(nameNode, source) - value := resolveStringExpr(valueNode, source, symbols) + name = parser.NodeText(nameNode, source) + value = resolveStringExpr(valueNode, source, symbols) return name, value } @@ -261,7 +261,7 @@ func resolveScala(node *tree_sitter.Node, source []byte, symbols map[string]stri // preproc_def → identifier + preproc_arg (for #define) // declaration → init_declarator → identifier + value (for const std::string x = "...") -func resolveCPP(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolveCPP(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { switch node.Kind() { case "preproc_def": // #define NAME "value" @@ -270,7 +270,7 @@ func resolveCPP(node *tree_sitter.Node, source []byte, symbols map[string]string if nameNode == nil || valueNode == nil { return "", "" } - name := parser.NodeText(nameNode, source) + name = parser.NodeText(nameNode, source) // preproc_arg contains the raw text — try to extract string content argText := strings.TrimSpace(parser.NodeText(valueNode, source)) if len(argText) >= 2 && argText[0] == '"' && argText[len(argText)-1] == '"' { @@ -294,8 +294,8 @@ func resolveCPP(node *tree_sitter.Node, source []byte, symbols map[string]string if nameNode == nil || valueNode == nil { return "", "" } - name := parser.NodeText(nameNode, source) - value := resolveStringExpr(valueNode, source, symbols) + name = parser.NodeText(nameNode, source) + value = resolveStringExpr(valueNode, source, symbols) return name, value } return "", "" @@ -305,7 +305,7 @@ func resolveCPP(node *tree_sitter.Node, source []byte, symbols map[string]string // variable_declaration → assignment_statement → variable_list + expression_list // Local: local x = "value" → variable_declaration containing assignment_statement -func resolveLua(node *tree_sitter.Node, source []byte, symbols map[string]string) (string, string) { +func resolveLua(node *tree_sitter.Node, source []byte, symbols map[string]string) (name, value string) { if node.Kind() != "variable_declaration" { return "", "" } @@ -326,8 +326,8 @@ func resolveLua(node *tree_sitter.Node, source []byte, symbols map[string]string return "", "" } - name := parser.NodeText(nameNode, source) - value := resolveStringExpr(valueNode, source, symbols) + name = parser.NodeText(nameNode, source) + value = resolveStringExpr(valueNode, source, symbols) return name, value } @@ -394,8 +394,8 @@ func resolveStringExpr(node *tree_sitter.Node, source []byte, symbols map[string return resolveBinaryConcat(node, source, symbols) } - // Go fmt.Sprintf / Java String.format / Lua string.format / Rust format! macro - if kind == "call_expression" || kind == "function_call" { + // Go fmt.Sprintf / Java String.format / Lua string.format / Python calls / Rust format! macro + if kind == "call_expression" || kind == "function_call" || kind == "call" { return resolveCallExpr(node, source, symbols) } if kind == "macro_invocation" { @@ -511,21 +511,34 @@ func resolveScalaInterpolated(node *tree_sitter.Node, source []byte, symbols map } // resolveBinaryConcat resolves string concatenation: left + right or left . right (PHP). +// Also handles || and ?? (nullish coalescing / logical OR default patterns). func resolveBinaryConcat(node *tree_sitter.Node, source []byte, symbols map[string]string) string { opNode := node.ChildByFieldName("operator") if opNode == nil { return "" } op := parser.NodeText(opNode, source) - if op != "+" && op != "." && op != ".." { + + switch op { + case "+", ".", "..": + left := resolveStringExpr(node.ChildByFieldName("left"), source, symbols) + right := resolveStringExpr(node.ChildByFieldName("right"), source, symbols) + if left == "" && right == "" { + return "" + } + return left + right + case "||", "??": + // For || and ??, return the right operand (default value) when left can't be resolved. + // This handles: process.env.KEY || "https://fallback" + // envVar ?? "https://default" + left := resolveStringExpr(node.ChildByFieldName("left"), source, symbols) + if left != "" { + return left + } + return resolveStringExpr(node.ChildByFieldName("right"), source, symbols) + default: return "" } - left := resolveStringExpr(node.ChildByFieldName("left"), source, symbols) - right := resolveStringExpr(node.ChildByFieldName("right"), source, symbols) - if left == "" && right == "" { - return "" - } - return left + right } // resolveCallExpr resolves format-style calls: Go fmt.Sprintf, Lua string.format, Java String.format. @@ -540,13 +553,6 @@ func resolveCallExpr(node *tree_sitter.Node, source []byte, symbols map[string]s } funcName := parser.NodeText(funcNode, source) - switch funcName { - case "fmt.Sprintf", "String.format", "string.format": - // continue — supported format functions - default: - return "" - } - // Try "arguments" field (Go, Java, Lua) args := node.ChildByFieldName("arguments") if args == nil { @@ -570,6 +576,15 @@ func resolveCallExpr(node *tree_sitter.Node, source []byte, symbols map[string]s return "" } + switch funcName { + case "fmt.Sprintf", "String.format", "string.format": + // continue — supported format functions + default: + // For unknown functions with URL-like string arguments, + // extract the URL as a fallback. Handles getEnv(), os.environ.get(), etc. + return extractURLArgFallback(argNodes, source, symbols) + } + // First arg is the format string fmtStr := extractStringContent(argNodes[0], source) if fmtStr == "" { @@ -596,6 +611,19 @@ func resolveCallExpr(node *tree_sitter.Node, source []byte, symbols map[string]s return b.String() } +// extractURLArgFallback scans function arguments for URL-like string literals. +// Used for unknown function calls like getEnv("KEY", "https://..."), +// os.environ.get("KEY", "https://..."), etc. +func extractURLArgFallback(argNodes []*tree_sitter.Node, source []byte, symbols map[string]string) string { + for _, arg := range argNodes { + val := resolveStringExpr(arg, source, symbols) + if val != "" && looksLikeURL(val) { + return val + } + } + return "" +} + // resolveRustFormatMacro resolves Rust format!("...", args) macros. func resolveRustFormatMacro(node *tree_sitter.Node, source []byte, symbols map[string]string) string { macroName := node.ChildByFieldName("macro") @@ -671,9 +699,9 @@ func extractStringContent(node *tree_sitter.Node, source []byte) string { } // Look for content children (language-specific names for inner text) contentKinds := map[string]bool{ - "string_content": true, // Python, Rust, PHP, Scala - "string_fragment": true, // JS/TS, Java - "interpreted_string_literal_content": true, // Go + "string_content": true, // Python, Rust, PHP, Scala + "string_fragment": true, // JS/TS, Java + "interpreted_string_literal_content": true, // Go } for i := uint(0); i < node.ChildCount(); i++ { child := node.Child(i) diff --git a/internal/pipeline/resolve_test.go b/internal/pipeline/resolve_test.go index 0a3989766..8da674dd9 100644 --- a/internal/pipeline/resolve_test.go +++ b/internal/pipeline/resolve_test.go @@ -2,7 +2,6 @@ package pipeline import ( "testing" - "unsafe" tree_sitter "github.com/tree-sitter/go-tree-sitter" tree_sitter_go "github.com/tree-sitter/tree-sitter-go/bindings/go" @@ -13,35 +12,35 @@ import ( tree_sitter_rust "github.com/tree-sitter/tree-sitter-rust/bindings/go" tree_sitter_scala "github.com/tree-sitter/tree-sitter-scala/bindings/go" - tree_sitter_cpp "github.com/tree-sitter/tree-sitter-cpp/bindings/go" tree_sitter_lua "github.com/tree-sitter-grammars/tree-sitter-lua/bindings/go" + tree_sitter_cpp "github.com/tree-sitter/tree-sitter-cpp/bindings/go" "github.com/DeusData/codebase-memory-mcp/internal/lang" ) -func parseSource(t *testing.T, language lang.Language, code string) (*tree_sitter.Tree, []byte) { +func parseSource(t *testing.T, language lang.Language, code string) (tree *tree_sitter.Tree, source []byte) { t.Helper() var tsLang *tree_sitter.Language switch language { case lang.Python: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_python.Language())) + tsLang = tree_sitter.NewLanguage(tree_sitter_python.Language()) case lang.Go: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_go.Language())) + tsLang = tree_sitter.NewLanguage(tree_sitter_go.Language()) case lang.JavaScript: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_javascript.Language())) + tsLang = tree_sitter.NewLanguage(tree_sitter_javascript.Language()) case lang.Rust: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_rust.Language())) + tsLang = tree_sitter.NewLanguage(tree_sitter_rust.Language()) case lang.Java: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_java.Language())) + tsLang = tree_sitter.NewLanguage(tree_sitter_java.Language()) case lang.PHP: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_php.LanguagePHPOnly())) + tsLang = tree_sitter.NewLanguage(tree_sitter_php.LanguagePHPOnly()) case lang.Scala: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_scala.Language())) + tsLang = tree_sitter.NewLanguage(tree_sitter_scala.Language()) case lang.CPP: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_cpp.Language())) + tsLang = tree_sitter.NewLanguage(tree_sitter_cpp.Language()) case lang.Lua: - tsLang = tree_sitter.NewLanguage(unsafe.Pointer(tree_sitter_lua.Language())) + tsLang = tree_sitter.NewLanguage(tree_sitter_lua.Language()) default: t.Fatalf("unsupported language: %s", language) } @@ -51,8 +50,8 @@ func parseSource(t *testing.T, language lang.Language, code string) (*tree_sitte if err := p.SetLanguage(tsLang); err != nil { t.Fatal(err) } - source := []byte(code) - tree := p.Parse(source, nil) + source = []byte(code) + tree = p.Parse(source, nil) return tree, source } @@ -139,10 +138,9 @@ let concat = String::from(BASE_URL) + "/api/orders"; assertSymbol(t, symbols, "BASE_URL", "https://example.com") assertSymbol(t, symbols, "url", "https://example.com/api/orders") - // concat: String::from(BASE_URL) is a call_expression with "String::from", not "fmt.Sprintf" - // The resolver sees binary_expression: call_expression + string_literal - // call_expression resolves to "" (not fmt.Sprintf), so concat = "" + "/api/orders" - assertSymbol(t, symbols, "concat", "/api/orders") + // concat: String::from(BASE_URL) — extractURLArgFallback resolves BASE_URL from symbols, + // so the full binary_expression resolves to "https://example.com" + "/api/orders" + assertSymbol(t, symbols, "concat", "https://example.com/api/orders") } func TestResolveJavaConcat(t *testing.T) { @@ -253,6 +251,65 @@ local formatted = string.format("%s/api/items", base_url) assertSymbol(t, symbols, "formatted", "https://example.com/api/items") } +func TestResolveEnvVarDefault(t *testing.T) { + // Python: os.environ.get("KEY", "https://example.com/api/orders") + code := `ORDER_URL = os.environ.get("ORDER_SERVICE_URL", "https://example.com/api/orders") +` + tree, source := parseSource(t, lang.Python, code) + defer tree.Close() + + symbols := resolveModuleStrings(tree.RootNode(), source, lang.Python) + assertSymbol(t, symbols, "ORDER_URL", "https://example.com/api/orders") +} + +func TestResolveGoGetenvDefault(t *testing.T) { + // Go: getEnv("KEY", "https://example.com/api/orders") — custom wrapper + code := `package main +var orderURL = getEnv("ORDER_URL", "https://example.com/api/orders") +` + tree, source := parseSource(t, lang.Go, code) + defer tree.Close() + + symbols := resolveModuleStrings(tree.RootNode(), source, lang.Go) + assertSymbol(t, symbols, "orderURL", "https://example.com/api/orders") +} + +func TestResolveJSNullishCoalescing(t *testing.T) { + // JS: process.env.KEY ?? "https://fallback" + code := `const apiUrl = process.env.API_URL ?? "https://example.com/api"; +` + tree, source := parseSource(t, lang.JavaScript, code) + defer tree.Close() + + symbols := resolveModuleStrings(tree.RootNode(), source, lang.JavaScript) + assertSymbol(t, symbols, "apiUrl", "https://example.com/api") +} + +func TestResolveJSLogicalOrDefault(t *testing.T) { + // JS: process.env.KEY || "https://fallback" + code := `const apiUrl = process.env.API_URL || "https://example.com/api"; +` + tree, source := parseSource(t, lang.JavaScript, code) + defer tree.Close() + + symbols := resolveModuleStrings(tree.RootNode(), source, lang.JavaScript) + assertSymbol(t, symbols, "apiUrl", "https://example.com/api") +} + +func TestResolveUnknownCallNoURL(t *testing.T) { + // Unknown function with non-URL arguments should resolve to empty + code := `package main +var x = someFunc("key", "value") +` + tree, source := parseSource(t, lang.Go, code) + defer tree.Close() + + symbols := resolveModuleStrings(tree.RootNode(), source, lang.Go) + if _, ok := symbols["x"]; ok { + t.Error("x should not be in symbols (no URL-like args)") + } +} + func assertSymbol(t *testing.T, symbols map[string]string, name, want string) { t.Helper() got, ok := symbols[name] diff --git a/internal/pipeline/resolver.go b/internal/pipeline/resolver.go index 6e2ff9d69..4de9a188e 100644 --- a/internal/pipeline/resolver.go +++ b/internal/pipeline/resolver.go @@ -59,46 +59,62 @@ func (r *FunctionRegistry) Resolve(calleeName, moduleQN string, importMap map[st suffix = parts[1] } - // Strategy 1: Import map lookup - if importMap != nil { - if resolved, ok := importMap[prefix]; ok { - var candidate string - if suffix != "" { - // Qualified call: pkg.Func -> resolved + "." + Func - candidate = resolved + "." + suffix - } else { - // Direct import: from X import func -> resolved is the full QN - candidate = resolved - } - if _, exists := r.exact[candidate]; exists { - return candidate - } - // If the resolved path is a module, try appending the calleeName - if suffix != "" { - // Also try looking up just the suffix under the resolved module - for qn := range r.exact { - if strings.HasPrefix(qn, resolved+".") && strings.HasSuffix(qn, "."+suffix) { - return qn - } - } + if result := r.resolveViaImportMap(prefix, suffix, importMap); result != "" { + return result + } + + if result := r.resolveViaSameModule(calleeName, suffix, moduleQN); result != "" { + return result + } + + return r.resolveViaNameLookup(calleeName, suffix, moduleQN) +} + +// resolveViaImportMap tries to resolve a callee using the import map (Strategy 1). +func (r *FunctionRegistry) resolveViaImportMap(prefix, suffix string, importMap map[string]string) string { + if importMap == nil { + return "" + } + resolved, ok := importMap[prefix] + if !ok { + return "" + } + var candidate string + if suffix != "" { + candidate = resolved + "." + suffix + } else { + candidate = resolved + } + if _, exists := r.exact[candidate]; exists { + return candidate + } + if suffix != "" { + for qn := range r.exact { + if strings.HasPrefix(qn, resolved+".") && strings.HasSuffix(qn, "."+suffix) { + return qn } } } + return "" +} - // Strategy 2: Same-module match +// resolveViaSameModule tries to resolve a callee within the same module (Strategy 2). +func (r *FunctionRegistry) resolveViaSameModule(calleeName, suffix, moduleQN string) string { sameModule := moduleQN + "." + calleeName if _, exists := r.exact[sameModule]; exists { return sameModule } - // For qualified calls in the same module, try the full calleeName if suffix != "" { sameModuleQualified := moduleQN + "." + suffix if _, exists := r.exact[sameModuleQualified]; exists { return sameModuleQualified } } + return "" +} - // Strategy 3: Project-wide single match by simple name +// resolveViaNameLookup tries project-wide name lookup and suffix matching (Strategies 3+4). +func (r *FunctionRegistry) resolveViaNameLookup(calleeName, suffix, moduleQN string) string { lookupName := calleeName if suffix != "" { lookupName = suffix @@ -109,12 +125,11 @@ func (r *FunctionRegistry) Resolve(calleeName, moduleQN string, importMap map[st return candidates[0] } - // Strategy 4: Suffix match with import distance scoring if suffix != "" { var matches []string for _, qn := range candidates { if strings.HasSuffix(qn, "."+calleeName) { - return qn // exact suffix match + return qn } if strings.HasSuffix(qn, "."+suffix) { matches = append(matches, qn) @@ -128,7 +143,6 @@ func (r *FunctionRegistry) Resolve(calleeName, moduleQN string, importMap map[st } } - // For non-qualified calls with multiple candidates, use import distance if len(candidates) > 1 { return bestByImportDistance(candidates, moduleQN) } diff --git a/internal/pipeline/typeinfer.go b/internal/pipeline/typeinfer.go index 5a9e156ac..5014d5d1c 100644 --- a/internal/pipeline/typeinfer.go +++ b/internal/pipeline/typeinfer.go @@ -5,6 +5,7 @@ import ( tree_sitter "github.com/tree-sitter/go-tree-sitter" + "github.com/DeusData/codebase-memory-mcp/internal/fqn" "github.com/DeusData/codebase-memory-mcp/internal/lang" "github.com/DeusData/codebase-memory-mcp/internal/parser" ) @@ -267,3 +268,58 @@ func extractCompositeLiteralType(node *tree_sitter.Node, source []byte) string { typeName = strings.TrimPrefix(typeName, "*") return typeName } + +// findEnclosingClassQN walks up the AST from a call node to find the enclosing +// class_definition (Python) and returns the class's qualified name. +// Returns "" if the call is not inside a class. +func findEnclosingClassQN(node *tree_sitter.Node, source []byte, project, relPath string) string { + current := node.Parent() + for current != nil { + if current.Kind() == "class_definition" { + nameNode := current.ChildByFieldName("name") + if nameNode != nil { + className := parser.NodeText(nameNode, source) + return fqn.Compute(project, relPath, className) + } + } + current = current.Parent() + } + return "" +} + +// parseGoReceiverType extracts the receiver type name from a Go method's +// function_declaration node. Returns the variable name and type name. +// E.g., "(s *Server)" -> ("s", "Server") +func parseGoReceiverType(funcNode *tree_sitter.Node, source []byte) (varName, typeName string) { + recvNode := funcNode.ChildByFieldName("receiver") + if recvNode == nil { + return "", "" + } + recvText := parser.NodeText(recvNode, source) + // Strip parens: "(s *Server)" -> "s *Server" + recvText = strings.TrimPrefix(recvText, "(") + recvText = strings.TrimSuffix(recvText, ")") + recvText = strings.TrimSpace(recvText) + + parts := strings.Fields(recvText) + if len(parts) < 2 { + return "", "" + } + varName = parts[0] + typeName = parts[1] + typeName = strings.TrimPrefix(typeName, "*") + return varName, typeName +} + +// findEnclosingFuncNode walks up the AST to find the nearest function_declaration +// or method_declaration ancestor. +func findEnclosingFuncNode(node *tree_sitter.Node, funcTypes map[string]bool) *tree_sitter.Node { + current := node.Parent() + for current != nil { + if funcTypes[current.Kind()] { + return current + } + current = current.Parent() + } + return nil +} diff --git a/internal/store/edges.go b/internal/store/edges.go index d6852860c..9cc144f3e 100644 --- a/internal/store/edges.go +++ b/internal/store/edges.go @@ -7,7 +7,7 @@ import ( // InsertEdge inserts an edge (dedup by source_id, target_id, type). func (s *Store) InsertEdge(e *Edge) (int64, error) { - res, err := s.db.Exec(` + res, err := s.q.Exec(` INSERT INTO edges (project, source_id, target_id, type, properties) VALUES (?, ?, ?, ?, ?) ON CONFLICT(source_id, target_id, type) DO UPDATE SET properties=excluded.properties`, @@ -20,7 +20,7 @@ func (s *Store) InsertEdge(e *Edge) (int64, error) { // FindEdgesBySource finds all edges from a given source node. func (s *Store) FindEdgesBySource(sourceID int64) ([]*Edge, error) { - rows, err := s.db.Query(`SELECT id, project, source_id, target_id, type, properties + rows, err := s.q.Query(`SELECT id, project, source_id, target_id, type, properties FROM edges WHERE source_id=?`, sourceID) if err != nil { return nil, fmt.Errorf("find edges by source: %w", err) @@ -31,7 +31,7 @@ func (s *Store) FindEdgesBySource(sourceID int64) ([]*Edge, error) { // FindEdgesByTarget finds all edges to a given target node. func (s *Store) FindEdgesByTarget(targetID int64) ([]*Edge, error) { - rows, err := s.db.Query(`SELECT id, project, source_id, target_id, type, properties + rows, err := s.q.Query(`SELECT id, project, source_id, target_id, type, properties FROM edges WHERE target_id=?`, targetID) if err != nil { return nil, fmt.Errorf("find edges by target: %w", err) @@ -42,7 +42,7 @@ func (s *Store) FindEdgesByTarget(targetID int64) ([]*Edge, error) { // FindEdgesBySourceAndType finds edges from a source with a specific type. func (s *Store) FindEdgesBySourceAndType(sourceID int64, edgeType string) ([]*Edge, error) { - rows, err := s.db.Query(`SELECT id, project, source_id, target_id, type, properties + rows, err := s.q.Query(`SELECT id, project, source_id, target_id, type, properties FROM edges WHERE source_id=? AND type=?`, sourceID, edgeType) if err != nil { return nil, fmt.Errorf("find edges by source+type: %w", err) @@ -53,7 +53,7 @@ func (s *Store) FindEdgesBySourceAndType(sourceID int64, edgeType string) ([]*Ed // FindEdgesByTargetAndType finds edges to a target with a specific type. func (s *Store) FindEdgesByTargetAndType(targetID int64, edgeType string) ([]*Edge, error) { - rows, err := s.db.Query(`SELECT id, project, source_id, target_id, type, properties + rows, err := s.q.Query(`SELECT id, project, source_id, target_id, type, properties FROM edges WHERE target_id=? AND type=?`, targetID, edgeType) if err != nil { return nil, fmt.Errorf("find edges by target+type: %w", err) @@ -65,19 +65,31 @@ func (s *Store) FindEdgesByTargetAndType(targetID int64, edgeType string) ([]*Ed // CountEdges returns the number of edges in a project. func (s *Store) CountEdges(project string) (int, error) { var count int - err := s.db.QueryRow("SELECT COUNT(*) FROM edges WHERE project=?", project).Scan(&count) + err := s.q.QueryRow("SELECT COUNT(*) FROM edges WHERE project=?", project).Scan(&count) return count, err } // DeleteEdgesByProject deletes all edges for a project. func (s *Store) DeleteEdgesByProject(project string) error { - _, err := s.db.Exec("DELETE FROM edges WHERE project=?", project) + _, err := s.q.Exec("DELETE FROM edges WHERE project=?", project) return err } // DeleteEdgesByType deletes all edges of a given type for a project. func (s *Store) DeleteEdgesByType(project, edgeType string) error { - _, err := s.db.Exec("DELETE FROM edges WHERE project=? AND type=?", project, edgeType) + _, err := s.q.Exec("DELETE FROM edges WHERE project=? AND type=?", project, edgeType) + return err +} + +// DeleteEdgesBySourceFile deletes edges of a given type where the source node +// belongs to a specific file. Used for incremental re-indexing of CALLS edges. +func (s *Store) DeleteEdgesBySourceFile(project, filePath, edgeType string) error { + _, err := s.q.Exec(` + DELETE FROM edges WHERE id IN ( + SELECT e.id FROM edges e + JOIN nodes n ON e.source_id = n.id + WHERE e.project=? AND n.file_path=? AND e.type=? + )`, project, filePath, edgeType) return err } diff --git a/internal/store/nodes.go b/internal/store/nodes.go index ed3bb83c3..11d948c56 100644 --- a/internal/store/nodes.go +++ b/internal/store/nodes.go @@ -7,7 +7,7 @@ import ( // UpsertNode inserts or replaces a node (dedup by qualified_name). func (s *Store) UpsertNode(n *Node) (int64, error) { - res, err := s.db.Exec(` + res, err := s.q.Exec(` INSERT INTO nodes (project, label, name, qualified_name, file_path, start_line, end_line, properties) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(project, qualified_name) DO UPDATE SET @@ -23,7 +23,7 @@ func (s *Store) UpsertNode(n *Node) (int64, error) { } // On conflict, LastInsertId may return 0; query the actual id if id == 0 { - err = s.db.QueryRow("SELECT id FROM nodes WHERE project=? AND qualified_name=?", n.Project, n.QualifiedName).Scan(&id) + err = s.q.QueryRow("SELECT id FROM nodes WHERE project=? AND qualified_name=?", n.Project, n.QualifiedName).Scan(&id) if err != nil { return 0, fmt.Errorf("get node id: %w", err) } @@ -33,21 +33,21 @@ func (s *Store) UpsertNode(n *Node) (int64, error) { // FindNodeByID finds a node by its primary key ID. func (s *Store) FindNodeByID(id int64) (*Node, error) { - row := s.db.QueryRow(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties + row := s.q.QueryRow(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE id=?`, id) return scanNode(row) } // FindNodeByQN finds a node by project and qualified name. func (s *Store) FindNodeByQN(project, qualifiedName string) (*Node, error) { - row := s.db.QueryRow(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties + row := s.q.QueryRow(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE project=? AND qualified_name=?`, project, qualifiedName) return scanNode(row) } // FindNodesByName finds nodes by project and name. func (s *Store) FindNodesByName(project, name string) ([]*Node, error) { - rows, err := s.db.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties + rows, err := s.q.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE project=? AND name=?`, project, name) if err != nil { return nil, fmt.Errorf("find by name: %w", err) @@ -58,7 +58,7 @@ func (s *Store) FindNodesByName(project, name string) ([]*Node, error) { // FindNodesByLabel finds all nodes with a given label in a project. func (s *Store) FindNodesByLabel(project, label string) ([]*Node, error) { - rows, err := s.db.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties + rows, err := s.q.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE project=? AND label=?`, project, label) if err != nil { return nil, fmt.Errorf("find by label: %w", err) @@ -69,7 +69,7 @@ func (s *Store) FindNodesByLabel(project, label string) ([]*Node, error) { // FindNodesByFile finds all nodes in a given file. func (s *Store) FindNodesByFile(project, filePath string) ([]*Node, error) { - rows, err := s.db.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties + rows, err := s.q.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE project=? AND file_path=?`, project, filePath) if err != nil { return nil, fmt.Errorf("find by file: %w", err) @@ -81,31 +81,31 @@ func (s *Store) FindNodesByFile(project, filePath string) ([]*Node, error) { // CountNodes returns the number of nodes in a project. func (s *Store) CountNodes(project string) (int, error) { var count int - err := s.db.QueryRow("SELECT COUNT(*) FROM nodes WHERE project=?", project).Scan(&count) + err := s.q.QueryRow("SELECT COUNT(*) FROM nodes WHERE project=?", project).Scan(&count) return count, err } // DeleteNodesByProject deletes all nodes for a project. func (s *Store) DeleteNodesByProject(project string) error { - _, err := s.db.Exec("DELETE FROM nodes WHERE project=?", project) + _, err := s.q.Exec("DELETE FROM nodes WHERE project=?", project) return err } // DeleteNodesByFile deletes all nodes for a specific file in a project. func (s *Store) DeleteNodesByFile(project, filePath string) error { - _, err := s.db.Exec("DELETE FROM nodes WHERE project=? AND file_path=?", project, filePath) + _, err := s.q.Exec("DELETE FROM nodes WHERE project=? AND file_path=?", project, filePath) return err } // DeleteNodesByLabel deletes all nodes with a given label in a project. func (s *Store) DeleteNodesByLabel(project, label string) error { - _, err := s.db.Exec("DELETE FROM nodes WHERE project=? AND label=?", project, label) + _, err := s.q.Exec("DELETE FROM nodes WHERE project=? AND label=?", project, label) return err } // AllNodes returns all nodes for a project. func (s *Store) AllNodes(project string) ([]*Node, error) { - rows, err := s.db.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties + rows, err := s.q.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE project=?`, project) if err != nil { return nil, err diff --git a/internal/store/projects.go b/internal/store/projects.go index a3cfcfe95..2394aaaa5 100644 --- a/internal/store/projects.go +++ b/internal/store/projects.go @@ -11,7 +11,7 @@ type Project struct { // UpsertProject creates or updates a project record. func (s *Store) UpsertProject(name, rootPath string) error { - _, err := s.db.Exec(` + _, err := s.q.Exec(` INSERT INTO projects (name, indexed_at, root_path) VALUES (?, ?, ?) ON CONFLICT(name) DO UPDATE SET indexed_at=excluded.indexed_at, root_path=excluded.root_path`, name, Now(), rootPath) @@ -21,7 +21,7 @@ func (s *Store) UpsertProject(name, rootPath string) error { // GetProject returns a project by name. func (s *Store) GetProject(name string) (*Project, error) { var p Project - err := s.db.QueryRow("SELECT name, indexed_at, root_path FROM projects WHERE name=?", name). + err := s.q.QueryRow("SELECT name, indexed_at, root_path FROM projects WHERE name=?", name). Scan(&p.Name, &p.IndexedAt, &p.RootPath) if err != nil { return nil, err @@ -31,7 +31,7 @@ func (s *Store) GetProject(name string) (*Project, error) { // ListProjects returns all indexed projects. func (s *Store) ListProjects() ([]*Project, error) { - rows, err := s.db.Query("SELECT name, indexed_at, root_path FROM projects ORDER BY name") + rows, err := s.q.Query("SELECT name, indexed_at, root_path FROM projects ORDER BY name") if err != nil { return nil, err } @@ -49,7 +49,7 @@ func (s *Store) ListProjects() ([]*Project, error) { // DeleteProject deletes a project and all associated data (CASCADE). func (s *Store) DeleteProject(name string) error { - _, err := s.db.Exec("DELETE FROM projects WHERE name=?", name) + _, err := s.q.Exec("DELETE FROM projects WHERE name=?", name) return err } @@ -62,7 +62,7 @@ type FileHash struct { // UpsertFileHash stores a file's content hash. func (s *Store) UpsertFileHash(project, relPath, sha256 string) error { - _, err := s.db.Exec(` + _, err := s.q.Exec(` INSERT INTO file_hashes (project, rel_path, sha256) VALUES (?, ?, ?) ON CONFLICT(project, rel_path) DO UPDATE SET sha256=excluded.sha256`, project, relPath, sha256) @@ -71,7 +71,7 @@ func (s *Store) UpsertFileHash(project, relPath, sha256 string) error { // GetFileHashes returns all file hashes for a project. func (s *Store) GetFileHashes(project string) (map[string]string, error) { - rows, err := s.db.Query("SELECT rel_path, sha256 FROM file_hashes WHERE project=?", project) + rows, err := s.q.Query("SELECT rel_path, sha256 FROM file_hashes WHERE project=?", project) if err != nil { return nil, fmt.Errorf("get file hashes: %w", err) } @@ -87,14 +87,32 @@ func (s *Store) GetFileHashes(project string) (map[string]string, error) { return result, rows.Err() } +// ListFilesForProject returns all distinct file paths indexed for a project. +func (s *Store) ListFilesForProject(project string) ([]string, error) { + rows, err := s.q.Query("SELECT DISTINCT file_path FROM nodes WHERE project=? AND file_path != ''", project) + if err != nil { + return nil, fmt.Errorf("list files: %w", err) + } + defer rows.Close() + var result []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + result = append(result, path) + } + return result, rows.Err() +} + // DeleteFileHash deletes a single file hash entry. func (s *Store) DeleteFileHash(project, relPath string) error { - _, err := s.db.Exec("DELETE FROM file_hashes WHERE project=? AND rel_path=?", project, relPath) + _, err := s.q.Exec("DELETE FROM file_hashes WHERE project=? AND rel_path=?", project, relPath) return err } // DeleteFileHashes deletes all file hashes for a project. func (s *Store) DeleteFileHashes(project string) error { - _, err := s.db.Exec("DELETE FROM file_hashes WHERE project=?", project) + _, err := s.q.Exec("DELETE FROM file_hashes WHERE project=?", project) return err } diff --git a/internal/store/schema.go b/internal/store/schema.go index 4d8a648f6..6785fc667 100644 --- a/internal/store/schema.go +++ b/internal/store/schema.go @@ -29,7 +29,7 @@ func (s *Store) GetSchema(project string) (*SchemaInfo, error) { info := &SchemaInfo{} // Node label counts - rows, err := s.db.Query("SELECT label, COUNT(*) as cnt FROM nodes WHERE project=? GROUP BY label ORDER BY cnt DESC", project) + rows, err := s.q.Query("SELECT label, COUNT(*) as cnt FROM nodes WHERE project=? GROUP BY label ORDER BY cnt DESC", project) if err != nil { return nil, fmt.Errorf("schema labels: %w", err) } @@ -46,7 +46,7 @@ func (s *Store) GetSchema(project string) (*SchemaInfo, error) { } // Edge type counts - rows2, err := s.db.Query("SELECT type, COUNT(*) as cnt FROM edges WHERE project=? GROUP BY type ORDER BY cnt DESC", project) + rows2, err := s.q.Query("SELECT type, COUNT(*) as cnt FROM edges WHERE project=? GROUP BY type ORDER BY cnt DESC", project) if err != nil { return nil, fmt.Errorf("schema edge types: %w", err) } @@ -63,7 +63,7 @@ func (s *Store) GetSchema(project string) (*SchemaInfo, error) { } // Relationship patterns: (src_label)-[type]->(tgt_label) with counts - rows3, err := s.db.Query(` + rows3, err := s.q.Query(` SELECT sn.label, e.type, tn.label, COUNT(*) as cnt FROM edges e JOIN nodes sn ON e.source_id = sn.id @@ -89,7 +89,7 @@ func (s *Store) GetSchema(project string) (*SchemaInfo, error) { } // Sample function names - rows4, err := s.db.Query("SELECT name FROM nodes WHERE project=? AND label='Function' ORDER BY name LIMIT 30", project) + rows4, err := s.q.Query("SELECT name FROM nodes WHERE project=? AND label='Function' ORDER BY name LIMIT 30", project) if err != nil { return nil, fmt.Errorf("schema sample funcs: %w", err) } @@ -101,9 +101,12 @@ func (s *Store) GetSchema(project string) (*SchemaInfo, error) { } info.SampleFunctionNames = append(info.SampleFunctionNames, name) } + if err := rows4.Err(); err != nil { + return nil, err + } // Sample class names - rows5, err := s.db.Query("SELECT name FROM nodes WHERE project=? AND label='Class' ORDER BY name LIMIT 20", project) + rows5, err := s.q.Query("SELECT name FROM nodes WHERE project=? AND label='Class' ORDER BY name LIMIT 20", project) if err != nil { return nil, fmt.Errorf("schema sample classes: %w", err) } @@ -115,9 +118,12 @@ func (s *Store) GetSchema(project string) (*SchemaInfo, error) { } info.SampleClassNames = append(info.SampleClassNames, name) } + if err := rows5.Err(); err != nil { + return nil, err + } // Sample qualified names - rows6, err := s.db.Query("SELECT qualified_name FROM nodes WHERE project=? LIMIT 5", project) + rows6, err := s.q.Query("SELECT qualified_name FROM nodes WHERE project=? LIMIT 5", project) if err != nil { return nil, fmt.Errorf("schema sample qns: %w", err) } @@ -129,6 +135,9 @@ func (s *Store) GetSchema(project string) (*SchemaInfo, error) { } info.SampleQualifiedNames = append(info.SampleQualifiedNames, qn) } + if err := rows6.Err(); err != nil { + return nil, err + } return info, nil } diff --git a/internal/store/search.go b/internal/store/search.go index 0eb252acf..3748573c7 100644 --- a/internal/store/search.go +++ b/internal/store/search.go @@ -19,6 +19,7 @@ type SearchParams struct { Limit int Offset int ExcludeEntryPoints bool // when true, exclude nodes with is_entry_point=true + IncludeConnected bool // when true, load connected node names (expensive, off by default) } // SearchResult is a node with edge degree info. @@ -35,8 +36,49 @@ type SearchOutput struct { Total int } +// countDegrees populates in/out degree counts on a SearchResult. +func (s *Store) countDegrees(sr *SearchResult, nodeID int64, relationship string) error { + if relationship != "" { + if err := s.q.QueryRow("SELECT COUNT(*) FROM edges WHERE target_id=? AND type=?", nodeID, relationship).Scan(&sr.InDegree); err != nil { + return fmt.Errorf("count in-degree: %w", err) + } + if err := s.q.QueryRow("SELECT COUNT(*) FROM edges WHERE source_id=? AND type=?", nodeID, relationship).Scan(&sr.OutDegree); err != nil { + return fmt.Errorf("count out-degree: %w", err) + } + } else { + if err := s.q.QueryRow("SELECT COUNT(*) FROM edges WHERE target_id=?", nodeID).Scan(&sr.InDegree); err != nil { + return fmt.Errorf("count in-degree: %w", err) + } + if err := s.q.QueryRow("SELECT COUNT(*) FROM edges WHERE source_id=?", nodeID).Scan(&sr.OutDegree); err != nil { + return fmt.Errorf("count out-degree: %w", err) + } + } + return nil +} + +// loadConnectedNames fetches up to 10 connected node names for display. +func (s *Store) loadConnectedNames(sr *SearchResult, nodeID int64) { + connRows, connErr := s.q.Query(` + SELECT DISTINCT n2.name FROM edges e + JOIN nodes n2 ON (e.target_id = n2.id OR e.source_id = n2.id) + WHERE (e.source_id = ? OR e.target_id = ?) AND n2.id != ? + LIMIT 10`, nodeID, nodeID, nodeID) + if connErr != nil { + return + } + defer connRows.Close() + for connRows.Next() { + var name string + if err := connRows.Scan(&name); err != nil { + break + } + sr.ConnectedNames = append(sr.ConnectedNames, name) + } + _ = connRows.Err() +} + // Search executes a parameterized search query with pagination support. -func (s *Store) Search(params SearchParams) (*SearchOutput, error) { +func (s *Store) Search(params *SearchParams) (*SearchOutput, error) { // Limit=0 means use default; use a high ceiling for SQL if params.Limit <= 0 { params.Limit = 100000 @@ -84,7 +126,7 @@ func (s *Store) Search(params SearchParams) (*SearchOutput, error) { LIMIT ?`, where) args = append(args, sqlLimit) - rows, err := s.db.Query(query, args...) + rows, err := s.q.Query(query, args...) if err != nil { return nil, fmt.Errorf("search: %w", err) } @@ -113,55 +155,9 @@ func (s *Store) Search(params SearchParams) (*SearchOutput, error) { } // Build all qualifying results (before offset/limit) for accurate total count - var allResults []*SearchResult - for _, n := range nodes { - sr := &SearchResult{Node: n} - - // Count degrees - if params.Relationship != "" { - var inCount, outCount int - s.db.QueryRow("SELECT COUNT(*) FROM edges WHERE target_id=? AND type=?", n.ID, params.Relationship).Scan(&inCount) - s.db.QueryRow("SELECT COUNT(*) FROM edges WHERE source_id=? AND type=?", n.ID, params.Relationship).Scan(&outCount) - sr.InDegree = inCount - sr.OutDegree = outCount - } else { - s.db.QueryRow("SELECT COUNT(*) FROM edges WHERE target_id=?", n.ID).Scan(&sr.InDegree) - s.db.QueryRow("SELECT COUNT(*) FROM edges WHERE source_id=?", n.ID).Scan(&sr.OutDegree) - } - - // Apply degree filters (-1 means "not set") - degree := sr.InDegree - if params.Direction == "outbound" { - degree = sr.OutDegree - } - if params.MinDegree >= 0 && degree < params.MinDegree { - continue - } - if params.MaxDegree >= 0 && degree > params.MaxDegree { - continue - } - - // Exclude entry points from dead code results - if params.ExcludeEntryPoints && isEntryPoint(n) { - continue - } - - // Get connected node names (limit to 10 for display) - connRows, connErr := s.db.Query(` - SELECT DISTINCT n2.name FROM edges e - JOIN nodes n2 ON (e.target_id = n2.id OR e.source_id = n2.id) - WHERE (e.source_id = ? OR e.target_id = ?) AND n2.id != ? - LIMIT 10`, n.ID, n.ID, n.ID) - if connErr == nil { - for connRows.Next() { - var name string - connRows.Scan(&name) - sr.ConnectedNames = append(sr.ConnectedNames, name) - } - connRows.Close() - } - - allResults = append(allResults, sr) + allResults, err := s.buildFilteredResults(nodes, params) + if err != nil { + return nil, err } total := len(allResults) @@ -204,6 +200,40 @@ func isEntryPoint(n *Node) bool { return ok && b } +// buildFilteredResults applies degree, direction, and entry-point filters to nodes, +// counts degrees, and loads connected names for each qualifying result. +func (s *Store) buildFilteredResults(nodes []*Node, params *SearchParams) ([]*SearchResult, error) { + results := make([]*SearchResult, 0, len(nodes)) + for _, n := range nodes { + sr := &SearchResult{Node: n} + + if err := s.countDegrees(sr, n.ID, params.Relationship); err != nil { + return nil, err + } + + degree := sr.InDegree + if params.Direction == "outbound" { + degree = sr.OutDegree + } + if params.MinDegree >= 0 && degree < params.MinDegree { + continue + } + if params.MaxDegree >= 0 && degree > params.MaxDegree { + continue + } + + if params.ExcludeEntryPoints && isEntryPoint(n) { + continue + } + + if params.IncludeConnected { + s.loadConnectedNames(sr, n.ID) + } + results = append(results, sr) + } + return results, nil +} + // filterByNamePattern filters nodes by a regex name pattern. func filterByNamePattern(nodes []*Node, pattern string) ([]*Node, error) { re, err := regexp.Compile(pattern) diff --git a/internal/store/search_bench_test.go b/internal/store/search_bench_test.go new file mode 100644 index 000000000..87456a0e0 --- /dev/null +++ b/internal/store/search_bench_test.go @@ -0,0 +1,169 @@ +package store + +import ( + "fmt" + "testing" +) + +// populateSearchBench creates a project with ~500 nodes and ~1000 edges. +func populateSearchBench(b *testing.B) *Store { + b.Helper() + s, err := OpenMemory() + if err != nil { + b.Fatal(err) + } + if err := s.UpsertProject("bench", "/tmp/bench"); err != nil { + b.Fatal(err) + } + + labels := []string{"Function", "Method", "Class", "Module"} + nodeIDs := make([]int64, 0, 500) + + for i := 0; i < 500; i++ { + label := labels[i%len(labels)] + id, err := s.UpsertNode(&Node{ + Project: "bench", + Label: label, + Name: fmt.Sprintf("Node%d", i), + QualifiedName: fmt.Sprintf("bench.pkg%d.Node%d", i/50, i), + FilePath: fmt.Sprintf("pkg%d/file%d.go", i/50, i%50), + StartLine: i*10 + 1, + EndLine: i*10 + 8, + Properties: map[string]any{"index": i}, + }) + if err != nil { + b.Fatal(err) + } + nodeIDs = append(nodeIDs, id) + } + + // Create ~1000 edges: each node calls 2 others, modules define functions + for i := 0; i < 500; i++ { + target1 := (i + 1) % 500 + target2 := (i + 7) % 500 + edgeType := "CALLS" + if i%4 == 3 { + edgeType = "DEFINES" + } + if _, err := s.InsertEdge(&Edge{ + Project: "bench", + SourceID: nodeIDs[i], + TargetID: nodeIDs[target1], + Type: edgeType, + }); err != nil { + b.Fatal(err) + } + if _, err := s.InsertEdge(&Edge{ + Project: "bench", + SourceID: nodeIDs[i], + TargetID: nodeIDs[target2], + Type: "CALLS", + }); err != nil { + b.Fatal(err) + } + } + + return s +} + +func BenchmarkSearch100Results(b *testing.B) { + s := populateSearchBench(b) + defer s.Close() + + params := SearchParams{ + Project: "bench", + Label: "Function", + MinDegree: -1, + MaxDegree: -1, + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + out, err := s.Search(¶ms) + if err != nil { + b.Fatal(err) + } + if len(out.Results) == 0 { + b.Fatal("expected results") + } + } +} + +func BenchmarkSearchWithDegreeFilter(b *testing.B) { + s := populateSearchBench(b) + defer s.Close() + + params := SearchParams{ + Project: "bench", + Relationship: "CALLS", + Direction: "outbound", + MinDegree: 1, + MaxDegree: -1, + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + out, err := s.Search(¶ms) + if err != nil { + b.Fatal(err) + } + if len(out.Results) == 0 { + b.Fatal("expected results") + } + } +} + +func BenchmarkSearchNamePattern(b *testing.B) { + s := populateSearchBench(b) + defer s.Close() + + params := SearchParams{ + Project: "bench", + NamePattern: ".*Node1[0-9]{2}.*", + MinDegree: -1, + MaxDegree: -1, + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + out, err := s.Search(¶ms) + if err != nil { + b.Fatal(err) + } + if len(out.Results) == 0 { + b.Fatal("expected results") + } + } +} + +func BenchmarkSearchPagination(b *testing.B) { + s := populateSearchBench(b) + defer s.Close() + + params := SearchParams{ + Project: "bench", + Limit: 20, + Offset: 100, + MinDegree: -1, + MaxDegree: -1, + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + out, err := s.Search(¶ms) + if err != nil { + b.Fatal(err) + } + if out.Total == 0 { + b.Fatal("expected results") + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 3c3b2f03c..6b53825ea 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -11,9 +11,17 @@ import ( _ "modernc.org/sqlite" ) +// Querier abstracts *sql.DB and *sql.Tx so store methods work in both contexts. +type Querier interface { + Exec(query string, args ...any) (sql.Result, error) + Query(query string, args ...any) (*sql.Rows, error) + QueryRow(query string, args ...any) *sql.Row +} + // Store wraps a SQLite connection for graph storage. type Store struct { db *sql.DB + q Querier // active querier: db or tx dbPath string } @@ -70,6 +78,7 @@ func OpenPath(dbPath string) (*Store, error) { return nil, fmt.Errorf("open db: %w", err) } s := &Store{db: db, dbPath: dbPath} + s.q = s.db if err := s.initSchema(); err != nil { db.Close() return nil, fmt.Errorf("init schema: %w", err) @@ -84,6 +93,7 @@ func OpenMemory() (*Store, error) { return nil, fmt.Errorf("open memory db: %w", err) } s := &Store{db: db, dbPath: ":memory:"} + s.q = s.db if err := s.initSchema(); err != nil { db.Close() return nil, fmt.Errorf("init schema: %w", err) @@ -91,6 +101,25 @@ func OpenMemory() (*Store, error) { return s, nil } +// WithTransaction executes fn within a single SQLite transaction. +// All store methods called inside fn use the transaction automatically. +func (s *Store) WithTransaction(fn func() error) error { + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + s.q = tx + defer func() { + s.q = s.db + }() + + if err := fn(); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() +} + // Close closes the database connection. func (s *Store) Close() error { return s.db.Close() @@ -146,6 +175,9 @@ func (s *Store) initSchema() error { CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source_id, type); CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_id, type); CREATE INDEX IF NOT EXISTS idx_edges_type ON edges(project, type); + + CREATE INDEX IF NOT EXISTS idx_edges_target_type ON edges(project, target_id, type); + CREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(project, source_id, type); ` _, err := s.db.Exec(schema) return err @@ -163,6 +195,11 @@ func marshalProps(props map[string]any) string { return string(b) } +// UnmarshalProps deserializes JSON properties. Exported for use by cypher executor. +func UnmarshalProps(data string) map[string]any { + return unmarshalProps(data) +} + // unmarshalProps deserializes JSON properties. func unmarshalProps(data string) map[string]any { if data == "" { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 327f0ac1e..f5bf5b3ba 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -7,7 +7,7 @@ func TestOpenMemory(t *testing.T) { if err != nil { t.Fatalf("OpenMemory: %v", err) } - defer s.Close() + s.Close() } func TestNodeCRUD(t *testing.T) { @@ -90,8 +90,12 @@ func TestNodeDedup(t *testing.T) { n1 := &Node{Project: "test", Label: "Function", Name: "Foo", QualifiedName: "test.main.Foo"} n2 := &Node{Project: "test", Label: "Function", Name: "Foo", QualifiedName: "test.main.Foo", Properties: map[string]any{"updated": true}} - s.UpsertNode(n1) - s.UpsertNode(n2) + if _, err := s.UpsertNode(n1); err != nil { + t.Fatalf("UpsertNode n1: %v", err) + } + if _, err := s.UpsertNode(n2); err != nil { + t.Fatalf("UpsertNode n2: %v", err) + } count, _ := s.CountNodes("test") if count != 1 { @@ -153,10 +157,14 @@ func TestCascadeDelete(t *testing.T) { defer s.Close() // Create project with nodes and edges - s.UpsertProject("test", "/tmp/test") + if err := s.UpsertProject("test", "/tmp/test"); err != nil { + t.Fatalf("UpsertProject: %v", err) + } id1, _ := s.UpsertNode(&Node{Project: "test", Label: "Function", Name: "A", QualifiedName: "test.A"}) id2, _ := s.UpsertNode(&Node{Project: "test", Label: "Function", Name: "B", QualifiedName: "test.B"}) - s.InsertEdge(&Edge{Project: "test", SourceID: id1, TargetID: id2, Type: "CALLS"}) + if _, err := s.InsertEdge(&Edge{Project: "test", SourceID: id1, TargetID: id2, Type: "CALLS"}); err != nil { + t.Fatalf("InsertEdge: %v", err) + } // Delete project — should cascade if err := s.DeleteProject("test"); err != nil { @@ -214,7 +222,9 @@ func TestFileHashes(t *testing.T) { } defer s.Close() - s.UpsertProject("test", "/tmp/test") + if err := s.UpsertProject("test", "/tmp/test"); err != nil { + t.Fatalf("UpsertProject: %v", err) + } // Upsert if err := s.UpsertFileHash("test", "main.go", "abc123"); err != nil { @@ -231,7 +241,9 @@ func TestFileHashes(t *testing.T) { } // Update - s.UpsertFileHash("test", "main.go", "def456") + if err := s.UpsertFileHash("test", "main.go", "def456"); err != nil { + t.Fatalf("UpsertFileHash update: %v", err) + } hashes, _ = s.GetFileHashes("test") if hashes["main.go"] != "def456" { t.Errorf("expected def456, got %s", hashes["main.go"]) @@ -245,13 +257,21 @@ func TestSearch(t *testing.T) { } defer s.Close() - s.UpsertProject("test", "/tmp/test") - s.UpsertNode(&Node{Project: "test", Label: "Function", Name: "SubmitOrder", QualifiedName: "test.main.SubmitOrder", FilePath: "main.go"}) - s.UpsertNode(&Node{Project: "test", Label: "Function", Name: "ProcessOrder", QualifiedName: "test.service.ProcessOrder", FilePath: "service.go"}) - s.UpsertNode(&Node{Project: "test", Label: "Class", Name: "OrderService", QualifiedName: "test.service.OrderService", FilePath: "service.go"}) + if err := s.UpsertProject("test", "/tmp/test"); err != nil { + t.Fatalf("UpsertProject: %v", err) + } + if _, err := s.UpsertNode(&Node{Project: "test", Label: "Function", Name: "SubmitOrder", QualifiedName: "test.main.SubmitOrder", FilePath: "main.go"}); err != nil { + t.Fatalf("UpsertNode SubmitOrder: %v", err) + } + if _, err := s.UpsertNode(&Node{Project: "test", Label: "Function", Name: "ProcessOrder", QualifiedName: "test.service.ProcessOrder", FilePath: "service.go"}); err != nil { + t.Fatalf("UpsertNode ProcessOrder: %v", err) + } + if _, err := s.UpsertNode(&Node{Project: "test", Label: "Class", Name: "OrderService", QualifiedName: "test.service.OrderService", FilePath: "service.go"}); err != nil { + t.Fatalf("UpsertNode OrderService: %v", err) + } // Search by label - output, err := s.Search(SearchParams{Project: "test", Label: "Function"}) + output, err := s.Search(&SearchParams{Project: "test", Label: "Function"}) if err != nil { t.Fatalf("Search: %v", err) } @@ -263,7 +283,7 @@ func TestSearch(t *testing.T) { } // Search by name pattern - output, err = s.Search(SearchParams{Project: "test", NamePattern: ".*Submit.*"}) + output, err = s.Search(&SearchParams{Project: "test", NamePattern: ".*Submit.*"}) if err != nil { t.Fatalf("Search: %v", err) } @@ -272,7 +292,7 @@ func TestSearch(t *testing.T) { } // Search by file pattern - output, err = s.Search(SearchParams{Project: "test", FilePattern: "service*"}) + output, err = s.Search(&SearchParams{Project: "test", FilePattern: "service*"}) if err != nil { t.Fatalf("Search: %v", err) } @@ -281,7 +301,7 @@ func TestSearch(t *testing.T) { } // Search with offset/limit pagination - output, err = s.Search(SearchParams{Project: "test", Limit: 1}) + output, err = s.Search(&SearchParams{Project: "test", Limit: 1}) if err != nil { t.Fatalf("Search: %v", err) } @@ -292,7 +312,7 @@ func TestSearch(t *testing.T) { t.Errorf("expected total=3, got %d", output.Total) } - output, err = s.Search(SearchParams{Project: "test", Limit: 1, Offset: 1}) + output, err = s.Search(&SearchParams{Project: "test", Limit: 1, Offset: 1}) if err != nil { t.Fatalf("Search: %v", err) } diff --git a/internal/store/traverse.go b/internal/store/traverse.go index 10943602a..9d99da63f 100644 --- a/internal/store/traverse.go +++ b/internal/store/traverse.go @@ -20,6 +20,30 @@ type EdgeInfo struct { Type string } +type bfsQueue struct { + nodeID int64 + hop int +} + +// fetchEdgesForNode retrieves edges from a node in the given direction and edge types. +func (s *Store) fetchEdgesForNode(nodeID int64, direction string, edgeTypes []string) ([]*Edge, error) { + var edges []*Edge + for _, et := range edgeTypes { + var found []*Edge + var err error + if direction == "outbound" { + found, err = s.FindEdgesBySourceAndType(nodeID, et) + } else { + found, err = s.FindEdgesByTargetAndType(nodeID, et) + } + if err != nil { + return nil, err + } + edges = append(edges, found...) + } + return edges, nil +} + // BFS performs breadth-first traversal following edges of given types. // direction: "outbound" follows source->target, "inbound" follows target->source. // maxDepth caps the BFS depth, maxResults caps total visited nodes. @@ -32,14 +56,17 @@ func (s *Store) BFS(startNodeID int64, direction string, edgeTypes []string, max } result := &TraverseResult{} - visited := make(map[int64]int) // nodeID -> hop + visited := make(map[int64]int) // nodeID -> hop + nodeCache := make(map[int64]*Node) // nodeID -> resolved node visited[startNodeID] = 0 - type queueItem struct { - nodeID int64 - hop int + // Cache the start node for edge name resolution + startNode, err := s.FindNodeByID(startNodeID) + if err == nil && startNode != nil { + nodeCache[startNodeID] = startNode } - queue := []queueItem{{startNodeID, 0}} + + queue := []bfsQueue{{startNodeID, 0}} for len(queue) > 0 && len(result.Visited) < maxResults { item := queue[0] @@ -49,20 +76,9 @@ func (s *Store) BFS(startNodeID int64, direction string, edgeTypes []string, max continue } - // Get edges from this node - var edges []*Edge - for _, et := range edgeTypes { - var found []*Edge - var err error - if direction == "outbound" { - found, err = s.FindEdgesBySourceAndType(item.nodeID, et) - } else { - found, err = s.FindEdgesByTargetAndType(item.nodeID, et) - } - if err != nil { - return nil, err - } - edges = append(edges, found...) + edges, err := s.fetchEdgesForNode(item.nodeID, direction, edgeTypes) + if err != nil { + return nil, err } for _, e := range edges { @@ -76,28 +92,39 @@ func (s *Store) BFS(startNodeID int64, direction string, edgeTypes []string, max if _, seen := visited[nextID]; !seen { visited[nextID] = item.hop + 1 - row := s.db.QueryRow(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties - FROM nodes WHERE id=?`, nextID) - nextNode, err := scanNode(row) - if err != nil || nextNode == nil { + nextNode, lookupErr := s.FindNodeByID(nextID) + if lookupErr != nil || nextNode == nil { continue } + nodeCache[nextID] = nextNode result.Visited = append(result.Visited, &NodeHop{Node: nextNode, Hop: item.hop + 1}) - queue = append(queue, queueItem{nextID, item.hop + 1}) + queue = append(queue, bfsQueue{nextID, item.hop + 1}) if len(result.Visited) >= maxResults { break } } - // Record edge info - var fromName, toName string - s.db.QueryRow("SELECT name FROM nodes WHERE id=?", e.SourceID).Scan(&fromName) - s.db.QueryRow("SELECT name FROM nodes WHERE id=?", e.TargetID).Scan(&toName) + // Record edge info using cached nodes (avoid extra queries) + fromName := resolveNodeName(nodeCache, s, e.SourceID) + toName := resolveNodeName(nodeCache, s, e.TargetID) result.Edges = append(result.Edges, EdgeInfo{FromName: fromName, ToName: toName, Type: e.Type}) } } return result, nil } + +// resolveNodeName returns the name for a node ID, using the cache first. +func resolveNodeName(cache map[int64]*Node, s *Store, id int64) string { + if n, ok := cache[id]; ok { + return n.Name + } + n, err := s.FindNodeByID(id) + if err != nil || n == nil { + return "" + } + cache[id] = n + return n.Name +} diff --git a/internal/store/traverse_bench_test.go b/internal/store/traverse_bench_test.go new file mode 100644 index 000000000..9da3a5982 --- /dev/null +++ b/internal/store/traverse_bench_test.go @@ -0,0 +1,180 @@ +package store + +import ( + "fmt" + "testing" +) + +// populateTraverseBench creates a graph with controlled fan-out for BFS benchmarks. +// Root node calls nodes 1-fanOut, each of which calls fanOut more nodes (depth 2). +func populateTraverseBench(b *testing.B, fanOut int) (s *Store, rootID int64) { + b.Helper() + var err error + s, err = OpenMemory() + if err != nil { + b.Fatal(err) + } + if err := s.UpsertProject("bench", "/tmp/bench"); err != nil { + b.Fatal(err) + } + + nextID := 0 + makeNode := func(name string) int64 { + id, nodeErr := s.UpsertNode(&Node{ + Project: "bench", + Label: "Function", + Name: name, + QualifiedName: fmt.Sprintf("bench.pkg.%s", name), + FilePath: "pkg/funcs.go", + StartLine: nextID*5 + 1, + EndLine: nextID*5 + 4, + }) + if nodeErr != nil { + b.Fatal(nodeErr) + } + nextID++ + return id + } + + rootID = makeNode("Root") + + // Depth 1: root -> children + depth1IDs := make([]int64, fanOut) + for i := 0; i < fanOut; i++ { + depth1IDs[i] = makeNode(fmt.Sprintf("Child%d", i)) + if _, err := s.InsertEdge(&Edge{ + Project: "bench", + SourceID: rootID, + TargetID: depth1IDs[i], + Type: "CALLS", + }); err != nil { + b.Fatal(err) + } + } + + // Depth 2: each depth-1 node -> fanOut leaf nodes + for i, parentID := range depth1IDs { + for j := 0; j < fanOut; j++ { + leafID := makeNode(fmt.Sprintf("Leaf%d_%d", i, j)) + if _, err := s.InsertEdge(&Edge{ + Project: "bench", + SourceID: parentID, + TargetID: leafID, + Type: "CALLS", + }); err != nil { + b.Fatal(err) + } + } + } + + return s, rootID +} + +func BenchmarkBFS50Edges(b *testing.B) { + // fanOut=7 gives 1 + 7 + 49 = 57 nodes and 56 edges (close to 50) + s, rootID := populateTraverseBench(b, 7) + defer s.Close() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + result, err := s.BFS(rootID, "outbound", []string{"CALLS"}, 3, 200) + if err != nil { + b.Fatal(err) + } + if len(result.Visited) == 0 { + b.Fatal("expected visited nodes") + } + } +} + +func BenchmarkBFS200Edges(b *testing.B) { + // fanOut=14 gives 1 + 14 + 196 = 211 nodes, 210 edges + s, rootID := populateTraverseBench(b, 14) + defer s.Close() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + result, err := s.BFS(rootID, "outbound", []string{"CALLS"}, 3, 300) + if err != nil { + b.Fatal(err) + } + if len(result.Visited) == 0 { + b.Fatal("expected visited nodes") + } + } +} + +func BenchmarkBFSInbound(b *testing.B) { + s, err := OpenMemory() + if err != nil { + b.Fatal(err) + } + defer s.Close() + + if err := s.UpsertProject("bench", "/tmp/bench"); err != nil { + b.Fatal(err) + } + + // Create a "popular" function called by 50 callers + targetID, _ := s.UpsertNode(&Node{ + Project: "bench", + Label: "Function", + Name: "PopularFunc", + QualifiedName: "bench.pkg.PopularFunc", + FilePath: "pkg/popular.go", + }) + + for i := 0; i < 50; i++ { + callerID, _ := s.UpsertNode(&Node{ + Project: "bench", + Label: "Function", + Name: fmt.Sprintf("Caller%d", i), + QualifiedName: fmt.Sprintf("bench.callers.Caller%d", i), + FilePath: fmt.Sprintf("callers/caller%d.go", i), + }) + if _, err := s.InsertEdge(&Edge{ + Project: "bench", + SourceID: callerID, + TargetID: targetID, + Type: "CALLS", + }); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + result, err := s.BFS(targetID, "inbound", []string{"CALLS"}, 2, 200) + if err != nil { + b.Fatal(err) + } + if len(result.Visited) == 0 { + b.Fatal("expected visited nodes") + } + } +} + +func BenchmarkBFSDepthScaled(b *testing.B) { + for _, depth := range []int{1, 2, 3, 5} { + b.Run(fmt.Sprintf("depth=%d", depth), func(b *testing.B) { + s, rootID := populateTraverseBench(b, 5) + defer s.Close() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := s.BFS(rootID, "outbound", []string{"CALLS"}, depth, 500) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/internal/tools/code_search.go b/internal/tools/code_search.go index bf1bcbc45..1553ab773 100644 --- a/internal/tools/code_search.go +++ b/internal/tools/code_search.go @@ -18,44 +18,99 @@ type codeMatch struct { Content string `json:"content"` } -func (s *Server) handleSearchCode(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { +// searchCodeParams holds parsed parameters for a code search request. +type searchCodeParams struct { + pattern string + fileGlob string + maxResults int + offset int + isRegex bool + re *regexp.Regexp +} + +// parseSearchCodeParams extracts and validates search_code parameters from the request. +func parseSearchCodeParams(req *mcp.CallToolRequest) (*searchCodeParams, *mcp.CallToolResult) { args, err := parseArgs(req) if err != nil { - return errResult(err.Error()), nil + return nil, errResult(err.Error()) } - pattern := getStringArg(args, "pattern") - if pattern == "" { - return errResult("pattern is required"), nil + p := &searchCodeParams{ + pattern: getStringArg(args, "pattern"), + fileGlob: getStringArg(args, "file_pattern"), + maxResults: getIntArg(args, "max_results", 10), + offset: getIntArg(args, "offset", 0), + isRegex: getBoolArg(args, "regex"), } - fileGlob := getStringArg(args, "file_pattern") - maxResults := getIntArg(args, "max_results", 100) - offset := getIntArg(args, "offset", 0) + if p.pattern == "" { + return nil, errResult("pattern is required") + } - isRegex := false - if v, ok := args["regex"]; ok { - if b, ok := v.(bool); ok { - isRegex = b + if p.isRegex { + p.re, err = regexp.Compile(p.pattern) + if err != nil { + return nil, errResult(fmt.Sprintf("invalid regex: %v", err)) } } + return p, nil +} + +func (s *Server) handleSearchCode(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + params, errRes := parseSearchCodeParams(req) + if errRes != nil { + return errRes, nil + } + // Resolve project root root, err := s.resolveProjectRoot() if err != nil { return errResult(fmt.Sprintf("resolve root: %v", err)), nil } - // Compile regex or prepare literal search - var re *regexp.Regexp - if isRegex { - re, err = regexp.Compile(pattern) - if err != nil { - return errResult(fmt.Sprintf("invalid regex: %v", err)), nil + filePaths := s.collectSearchFilePaths(params.fileGlob) + + // Collect all matches up to offset+maxResults for accurate total count + fetchLimit := params.offset + params.maxResults + var allMatches []codeMatch + for _, relPath := range filePaths { + if len(allMatches) >= fetchLimit { + break } + + absPath := filepath.Join(root, relPath) + fileMatches := searchFile(absPath, relPath, params.pattern, params.re, params.isRegex, fetchLimit-len(allMatches)) + allMatches = append(allMatches, fileMatches...) } - // Get indexed file paths from the store + total := len(allMatches) + hasMore := total >= fetchLimit + + // Apply offset and limit + start := params.offset + if start > total { + start = total + } + end := start + params.maxResults + if end > total { + end = total + } + pageMatches := allMatches[start:end] + + return jsonResult(map[string]any{ + "pattern": params.pattern, + "total": total, + "limit": params.maxResults, + "offset": params.offset, + "has_more": hasMore, + "matches": pageMatches, + "files_count": len(filePaths), + }), nil +} + +// collectSearchFilePaths gathers indexed file paths, optionally filtered by a glob pattern. +func (s *Server) collectSearchFilePaths(fileGlob string) []string { projects, _ := s.store.ListProjects() var filePaths []string for _, p := range projects { @@ -66,7 +121,6 @@ func (s *Server) handleSearchCode(_ context.Context, req *mcp.CallToolRequest) ( } if fileGlob != "" { matched, _ := filepath.Match(fileGlob, filepath.Base(f.FilePath)) - // Also try against the full relative path using double-star simulation if !matched { matched = globMatch(fileGlob, f.FilePath) } @@ -77,43 +131,7 @@ func (s *Server) handleSearchCode(_ context.Context, req *mcp.CallToolRequest) ( filePaths = append(filePaths, f.FilePath) } } - - // Collect all matches up to offset+maxResults for accurate total count - fetchLimit := offset + maxResults - var allMatches []codeMatch - for _, relPath := range filePaths { - if len(allMatches) >= fetchLimit { - break - } - - absPath := filepath.Join(root, relPath) - fileMatches := searchFile(absPath, relPath, pattern, re, isRegex, fetchLimit-len(allMatches)) - allMatches = append(allMatches, fileMatches...) - } - - total := len(allMatches) - hasMore := total >= fetchLimit - - // Apply offset and limit - start := offset - if start > total { - start = total - } - end := start + maxResults - if end > total { - end = total - } - pageMatches := allMatches[start:end] - - return jsonResult(map[string]any{ - "pattern": pattern, - "total": total, - "limit": maxResults, - "offset": offset, - "has_more": hasMore, - "matches": pageMatches, - "files_count": len(filePaths), - }), nil + return filePaths } func searchFile(absPath, relPath, pattern string, re *regexp.Regexp, isRegex bool, limit int) []codeMatch { diff --git a/internal/tools/file.go b/internal/tools/file.go index 3e5939945..49eaebf1b 100644 --- a/internal/tools/file.go +++ b/internal/tools/file.go @@ -36,8 +36,8 @@ func (s *Server) handleReadFile(_ context.Context, req *mcp.CallToolRequest) (*m } // Check file exists and is not a directory - info, err := os.Stat(absPath) - if err != nil { + info, statErr := os.Stat(absPath) + if statErr != nil { return errResult(fmt.Sprintf("file not found: %s", absPath)), nil } if info.IsDir() { @@ -113,70 +113,17 @@ func (s *Server) handleListDirectory(_ context.Context, req *mcp.CallToolRequest } } - info, err := os.Stat(absPath) - if err != nil { + info, _ := os.Stat(absPath) + if info == nil { return errResult(fmt.Sprintf("path not found: %s", absPath)), nil } if !info.IsDir() { return errResult("path is a file, use read_file instead"), nil } - type entry struct { - Name string `json:"name"` - Path string `json:"path"` - IsDir bool `json:"is_dir"` - Size int64 `json:"size,omitempty"` - } - - var entries []entry - - if globPattern != "" { - // Glob matching within the directory - matches, globErr := filepath.Glob(filepath.Join(absPath, globPattern)) - if globErr != nil { - return errResult(fmt.Sprintf("glob: %v", globErr)), nil - } - for _, m := range matches { - fi, statErr := os.Stat(m) - if statErr != nil { - continue - } - relPath, _ := filepath.Rel(absPath, m) - e := entry{ - Name: relPath, - Path: m, - IsDir: fi.IsDir(), - } - if !fi.IsDir() { - e.Size = fi.Size() - } - entries = append(entries, e) - } - } else { - // List immediate children - dirEntries, readErr := os.ReadDir(absPath) - if readErr != nil { - return errResult(fmt.Sprintf("read dir: %v", readErr)), nil - } - for _, de := range dirEntries { - // Skip hidden files/dirs - if strings.HasPrefix(de.Name(), ".") { - continue - } - fi, statErr := de.Info() - if statErr != nil { - continue - } - e := entry{ - Name: de.Name(), - Path: filepath.Join(absPath, de.Name()), - IsDir: de.IsDir(), - } - if !de.IsDir() { - e.Size = fi.Size() - } - entries = append(entries, e) - } + entries, listErr := listEntries(absPath, globPattern) + if listErr != nil { + return errResult(listErr.Error()), nil } return jsonResult(map[string]any{ @@ -186,6 +133,64 @@ func (s *Server) handleListDirectory(_ context.Context, req *mcp.CallToolRequest }), nil } +type dirEntry struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size,omitempty"` +} + +func listEntries(absPath, globPattern string) ([]dirEntry, error) { + if globPattern != "" { + return listGlobEntries(absPath, globPattern) + } + return listDirChildren(absPath) +} + +func listGlobEntries(absPath, globPattern string) ([]dirEntry, error) { + matches, globErr := filepath.Glob(filepath.Join(absPath, globPattern)) + if globErr != nil { + return nil, fmt.Errorf("glob: %w", globErr) + } + entries := make([]dirEntry, 0, len(matches)) + for _, m := range matches { + fi, _ := os.Stat(m) + if fi == nil { + continue + } + relPath, _ := filepath.Rel(absPath, m) + e := dirEntry{Name: relPath, Path: m, IsDir: fi.IsDir()} + if !fi.IsDir() { + e.Size = fi.Size() + } + entries = append(entries, e) + } + return entries, nil +} + +func listDirChildren(absPath string) ([]dirEntry, error) { + dirEntries, readErr := os.ReadDir(absPath) + if readErr != nil { + return nil, fmt.Errorf("read dir: %w", readErr) + } + entries := make([]dirEntry, 0, len(dirEntries)) + for _, de := range dirEntries { + if strings.HasPrefix(de.Name(), ".") { + continue + } + fi, _ := de.Info() + if fi == nil { + continue + } + e := dirEntry{Name: de.Name(), Path: filepath.Join(absPath, de.Name()), IsDir: de.IsDir()} + if !de.IsDir() { + e.Size = fi.Size() + } + entries = append(entries, e) + } + return entries, nil +} + // resolveProjectRoot finds the first indexed project root path. func (s *Server) resolveProjectRoot() (string, error) { projects, err := s.store.ListProjects() diff --git a/internal/tools/projects.go b/internal/tools/projects.go index 92332a3b1..a2bfd25cb 100644 --- a/internal/tools/projects.go +++ b/internal/tools/projects.go @@ -49,8 +49,8 @@ func (s *Server) handleDeleteProject(_ context.Context, req *mcp.CallToolRequest } // Verify project exists - proj, err := s.store.GetProject(name) - if err != nil || proj == nil { + proj, _ := s.store.GetProject(name) + if proj == nil { return errResult(fmt.Sprintf("project not found: %s", name)), nil } diff --git a/internal/tools/schema.go b/internal/tools/schema.go index ae8fb5cad..8e780ac8a 100644 --- a/internal/tools/schema.go +++ b/internal/tools/schema.go @@ -26,7 +26,7 @@ func (s *Server) handleGetGraphSchema(_ context.Context, _ *mcp.CallToolRequest) Schema *store.SchemaInfo `json:"schema"` } - var schemas []projectSchema + schemas := make([]projectSchema, 0, len(projects)) for _, p := range projects { schema, schemaErr := s.store.GetSchema(p.Name) if schemaErr != nil { diff --git a/internal/tools/search.go b/internal/tools/search.go index e5986aabd..9bf45de42 100644 --- a/internal/tools/search.go +++ b/internal/tools/search.go @@ -14,7 +14,7 @@ func (s *Server) handleSearchGraph(_ context.Context, req *mcp.CallToolRequest) return errResult(err.Error()), nil } - params := store.SearchParams{ + params := &store.SearchParams{ Label: getStringArg(args, "label"), NamePattern: getStringArg(args, "name_pattern"), FilePattern: getStringArg(args, "file_pattern"), @@ -22,9 +22,10 @@ func (s *Server) handleSearchGraph(_ context.Context, req *mcp.CallToolRequest) Direction: getStringArg(args, "direction"), MinDegree: getIntArg(args, "min_degree", -1), MaxDegree: getIntArg(args, "max_degree", -1), - Limit: getIntArg(args, "limit", 100), + Limit: getIntArg(args, "limit", 10), Offset: getIntArg(args, "offset", 0), ExcludeEntryPoints: getBoolArg(args, "exclude_entry_points"), + IncludeConnected: getBoolArg(args, "include_connected"), } projectFilter := getStringArg(args, "project") diff --git a/internal/tools/snippet.go b/internal/tools/snippet.go index 322355025..57aeecea0 100644 --- a/internal/tools/snippet.go +++ b/internal/tools/snippet.go @@ -23,8 +23,8 @@ func (s *Server) handleGetCodeSnippet(_ context.Context, req *mcp.CallToolReques } // Find the node across all projects - node, project, err := s.findNodeByQNAcrossProjects(qn) - if err != nil { + node, project, _ := s.findNodeByQNAcrossProjects(qn) + if node == nil { return errResult(fmt.Sprintf("node not found: %s", qn)), nil } @@ -37,8 +37,8 @@ func (s *Server) handleGetCodeSnippet(_ context.Context, req *mcp.CallToolReques } // Resolve file path against the project's root path - proj, projErr := s.store.GetProject(project) - if projErr != nil { + proj, _ := s.store.GetProject(project) + if proj == nil { return errResult(fmt.Sprintf("project not found: %s", project)), nil } diff --git a/internal/tools/tools.go b/internal/tools/tools.go index 85ebcf5cc..4d561c616 100644 --- a/internal/tools/tools.go +++ b/internal/tools/tools.go @@ -36,7 +36,20 @@ func (s *Server) MCPServer() *mcp.Server { } func (s *Server) registerTools() { - // 1. index_repository + s.registerGraphTools() + s.registerFileTools() + s.registerProjectTools() +} + +// registerGraphTools registers tools for graph querying, searching, and tracing. +func (s *Server) registerGraphTools() { + s.registerIndexAndTraceTool() + s.registerSchemaAndSnippetTools() + s.registerSearchTools() + s.registerQueryTool() +} + +func (s *Server) registerIndexAndTraceTool() { s.mcp.AddTool(&mcp.Tool{ Name: "index_repository", Description: "Index a repository into the code graph. Parses source files, extracts functions/classes/modules, resolves call relationships, and stores the graph for querying. Supports incremental reindex via content hashing.", @@ -51,10 +64,9 @@ func (s *Server) registerTools() { }`), }, s.handleIndexRepository) - // 2. trace_call_path s.mcp.AddTool(&mcp.Tool{ Name: "trace_call_path", - Description: "Trace call paths from/to a function using BFS traversal. Returns the root function with signature and module constants, hop-by-hop callees/callers, and call edges with type (CALLS or HTTP_CALLS). Use for understanding call chains and data flow.", + Description: "Trace call paths from/to a function. Requires EXACT function name — if unsure, call search_graph(name_pattern=...) first to discover the correct name. Returns hop-by-hop callees/callers with edge types (CALLS, HTTP_CALLS, ASYNC_CALLS). Use depth=1 first, increase only if needed. IMPORTANT: If the function is not found, use search_graph with a name_pattern regex to find similar names before retrying.", InputSchema: json.RawMessage(`{ "type": "object", "properties": { @@ -75,15 +87,15 @@ func (s *Server) registerTools() { "required": ["function_name"] }`), }, s.handleTraceCallPath) +} - // 3. get_graph_schema +func (s *Server) registerSchemaAndSnippetTools() { s.mcp.AddTool(&mcp.Tool{ Name: "get_graph_schema", Description: "Return the schema of the indexed code graph: node label counts, edge type counts, relationship patterns (e.g. Function-CALLS->Function), and sample function/class names. Use to understand what's in the graph before querying.", InputSchema: json.RawMessage(`{"type": "object"}`), }, s.handleGetGraphSchema) - // 4. get_code_snippet s.mcp.AddTool(&mcp.Tool{ Name: "get_code_snippet", Description: "Retrieve source code for a function/class by qualified name. Reads directly from disk using the stored file path and line range. Returns the source code with line numbers.", @@ -98,11 +110,12 @@ func (s *Server) registerTools() { "required": ["qualified_name"] }`), }, s.handleGetCodeSnippet) +} - // 5. search_graph +func (s *Server) registerSearchTools() { s.mcp.AddTool(&mcp.Tool{ Name: "search_graph", - Description: "Search the code graph with structured filters. Replaces raw Cypher queries with safe, parameterized search. Supports filtering by project, node label, name pattern (regex), file pattern (glob), relationship type, direction, and degree (fan-in/fan-out). Returns matching nodes with properties and edge counts.", + Description: "Search the code graph. Returns 10 results per page (use offset to paginate, has_more indicates more pages). For dead code: use relationship='CALLS', direction='inbound', max_degree=0, exclude_entry_points=true. For fan-out: use relationship='CALLS', direction='outbound', min_degree=N. Route nodes: properties.handler contains the actual handler function name. Prefer this over query_graph for counting — no row cap. Relationship types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE, IMPLEMENTS.", InputSchema: json.RawMessage(`{ "type": "object", "properties": { @@ -124,7 +137,7 @@ func (s *Server) registerTools() { }, "relationship": { "type": "string", - "description": "Filter by relationship type: CALLS, HTTP_CALLS, IMPORTS, DEFINES, etc." + "description": "Filter by relationship type: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE, IMPLEMENTS" }, "direction": { "type": "string", @@ -145,39 +158,71 @@ func (s *Server) registerTools() { }, "limit": { "type": "integer", - "description": "Max results to return per page (default: 100)." + "description": "Max results per page (default: 10). Use small limits and paginate with offset — response includes has_more flag." }, "offset": { "type": "integer", - "description": "Number of results to skip for pagination (default: 0). Use with limit to page through results." + "description": "Skip N results for pagination (default: 0). Check has_more in response to know if more pages exist." + }, + "include_connected": { + "type": "boolean", + "description": "Include connected node names in results (default: false). Expensive — only enable when you need to see neighbor names." } } }`), }, s.handleSearchGraph) - // 6. list_projects s.mcp.AddTool(&mcp.Tool{ - Name: "list_projects", - Description: "List all indexed projects with their indexed_at timestamp, root path, and node/edge counts.", - InputSchema: json.RawMessage(`{"type": "object"}`), - }, s.handleListProjects) - - // 7. delete_project - s.mcp.AddTool(&mcp.Tool{ - Name: "delete_project", - Description: "Delete an indexed project and all its graph data (nodes, edges, file hashes). This action is irreversible.", + Name: "search_code", + Description: "Search for text in source code files (like grep, scoped to indexed project). Returns 10 matches per page — use offset to paginate, has_more indicates more pages. Use for string literals, error messages, TODO comments.", InputSchema: json.RawMessage(`{ "type": "object", "properties": { - "project_name": { + "pattern": { "type": "string", - "description": "Name of the project to delete" + "description": "Text to search for (literal string, or regex if regex=true)" + }, + "file_pattern": { + "type": "string", + "description": "Glob pattern to filter files (e.g. '*.go', '*.py')" + }, + "regex": { + "type": "boolean", + "description": "Treat pattern as a regular expression (default: false)" + }, + "max_results": { + "type": "integer", + "description": "Max matches per page (default: 10). Response includes has_more flag for pagination." + }, + "offset": { + "type": "integer", + "description": "Skip N matches for pagination (default: 0). Check has_more in response." } }, - "required": ["project_name"] + "required": ["pattern"] }`), - }, s.handleDeleteProject) + }, s.handleSearchCode) +} +func (s *Server) registerQueryTool() { + s.mcp.AddTool(&mcp.Tool{ + Name: "query_graph", + Description: "Execute a Cypher-like graph query. WARNING: 200-row cap applies BEFORE aggregation — COUNT queries on large codebases silently undercount. For fan-out/fan-in counting, use search_graph with min_degree/max_degree instead. Best for: relationship patterns, filtered joins, path queries. Always use LIMIT. Edge types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, IMPLEMENTS.", + InputSchema: json.RawMessage(`{ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Cypher query, e.g. MATCH (f:Function)-[:CALLS]->(g:Function) WHERE f.name = 'main' RETURN g.name, g.qualified_name LIMIT 20" + } + }, + "required": ["query"] + }`), + }, s.handleQueryGraph) +} + +// registerFileTools registers tools for file and directory operations. +func (s *Server) registerFileTools() { // 8. read_file s.mcp.AddTool(&mcp.Tool{ Name: "read_file", @@ -220,54 +265,32 @@ func (s *Server) registerTools() { } }`), }, s.handleListDirectory) +} - // 10. search_code +// registerProjectTools registers tools for project management. +func (s *Server) registerProjectTools() { + // 6. list_projects s.mcp.AddTool(&mcp.Tool{ - Name: "search_code", - Description: "Search for text patterns within source code files. Like grep/ripgrep but scoped to indexed project files. Returns matching lines with file paths and line numbers. Use for finding string literals, error messages, TODO comments, or any text within function bodies.", + Name: "list_projects", + Description: "List all indexed projects with their indexed_at timestamp, root path, and node/edge counts.", + InputSchema: json.RawMessage(`{"type": "object"}`), + }, s.handleListProjects) + + // 7. delete_project + s.mcp.AddTool(&mcp.Tool{ + Name: "delete_project", + Description: "Delete an indexed project and all its graph data (nodes, edges, file hashes). This action is irreversible.", InputSchema: json.RawMessage(`{ "type": "object", "properties": { - "pattern": { + "project_name": { "type": "string", - "description": "Text to search for (literal string, or regex if regex=true)" - }, - "file_pattern": { - "type": "string", - "description": "Glob pattern to filter files (e.g. '*.go', '*.py')" - }, - "regex": { - "type": "boolean", - "description": "Treat pattern as a regular expression (default: false)" - }, - "max_results": { - "type": "integer", - "description": "Maximum number of matches to return per page (default: 100)." - }, - "offset": { - "type": "integer", - "description": "Number of matches to skip for pagination (default: 0). Use with max_results to page through results." + "description": "Name of the project to delete" } }, - "required": ["pattern"] + "required": ["project_name"] }`), - }, s.handleSearchCode) - - // 11. query_graph - s.mcp.AddTool(&mcp.Tool{ - Name: "query_graph", - Description: "Execute a Cypher-like graph query. Supports MATCH patterns with node labels, relationship types, variable-length paths, WHERE filters (=, =~, CONTAINS, STARTS WITH, >, <), and RETURN with COUNT/ORDER BY/LIMIT/DISTINCT. Read-only (no CREATE/DELETE).", - InputSchema: json.RawMessage(`{ - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Cypher query, e.g. MATCH (f:Function)-[:CALLS]->(g:Function) WHERE f.name = 'main' RETURN g.name, g.qualified_name LIMIT 20" - } - }, - "required": ["query"] - }`), - }, s.handleQueryGraph) + }, s.handleDeleteProject) } // jsonResult marshals data to JSON and returns as tool result. @@ -295,7 +318,7 @@ func errResult(msg string) *mcp.CallToolResult { // parseArgs unmarshals the raw JSON arguments into a map. func parseArgs(req *mcp.CallToolRequest) (map[string]any, error) { - if req.Params.Arguments == nil || len(req.Params.Arguments) == 0 { + if len(req.Params.Arguments) == 0 { return map[string]any{}, nil } var m map[string]any diff --git a/internal/tools/trace.go b/internal/tools/trace.go index 5d2bc76e5..c2684b3d1 100644 --- a/internal/tools/trace.go +++ b/internal/tools/trace.go @@ -33,12 +33,12 @@ func (s *Server) handleTraceCallPath(_ context.Context, req *mcp.CallToolRequest } // Find the function node across all projects - rootNode, project, err := s.findNodeAcrossProjects(funcName) - if err != nil { + rootNode, project, _ := s.findNodeAcrossProjects(funcName) + if rootNode == nil { return errResult(fmt.Sprintf("function not found: %s", funcName)), nil } - edgeTypes := []string{"CALLS", "HTTP_CALLS"} + edgeTypes := []string{"CALLS", "HTTP_CALLS", "ASYNC_CALLS"} // Build root info root := buildNodeInfo(rootNode)