diff --git a/internal/httplink/httplink_test.go b/internal/httplink/httplink_test.go index a93863e7c..7ac3d55d0 100644 --- a/internal/httplink/httplink_test.go +++ b/internal/httplink/httplink_test.go @@ -744,59 +744,34 @@ func RegisterRoutes(rg *gin.RouterGroup) { } func TestAsyncDispatchKeywords(t *testing.T) { - // Test the full linker flow to verify edge types. - // Uses a Go caller with CreateTask (async) and a Python route handler. dir, err := os.MkdirTemp("", "httplink-async-*") if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) - // Write a Go file with CreateTask + URL (async dispatch) - asyncDir := filepath.Join(dir, "taskworker") - if err := os.MkdirAll(asyncDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(asyncDir, "dispatch.go"), []byte(`package taskworker + writeTestFile(t, dir, "taskworker", "dispatch.go", `package taskworker func DispatchOrder(orderID string) { url := "https://api.internal.com/api/orders" client.CreateTask(ctx, url, payload) } -`), 0o600); err != nil { - t.Fatal(err) - } - - // Write a Go file with requests.post + URL (sync HTTP call) - syncDir := filepath.Join(dir, "synccaller") - if err := os.MkdirAll(syncDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(syncDir, "caller.go"), []byte(`package synccaller +`) + writeTestFile(t, dir, "synccaller", "caller.go", `package synccaller func CallOrder() { url := "https://api.internal.com/api/orders" requests.post(url, data) } -`), 0o600); err != nil { - t.Fatal(err) - } - - // Write a Go file with both sync + async keywords (sync takes precedence) - bothDir := filepath.Join(dir, "bothcaller") - if err := os.MkdirAll(bothDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(bothDir, "both.go"), []byte(`package bothcaller +`) + writeTestFile(t, dir, "bothcaller", "both.go", `package bothcaller func CallAndDispatch() { url := "https://api.internal.com/api/orders" requests.post(url, data) client.CreateTask(ctx, url, payload) } -`), 0o600); err != nil { - t.Fatal(err) - } +`) s, err := store.OpenMemory() if err != nil { @@ -809,35 +784,15 @@ func CallAndDispatch() { t.Fatal(err) } - // Async caller function - _, _ = s.UpsertNode(&store.Node{ - Project: project, Label: "Function", Name: "DispatchOrder", - QualifiedName: "testproj.taskworker.dispatch.DispatchOrder", - FilePath: "taskworker/dispatch.go", StartLine: 3, EndLine: 6, - }) + createTestNode(t, s, project, "DispatchOrder", "testproj.taskworker.dispatch.DispatchOrder", "taskworker/dispatch.go", 3, 6) + createTestNode(t, s, project, "CallOrder", "testproj.synccaller.caller.CallOrder", "synccaller/caller.go", 3, 6) + createTestNode(t, s, project, "CallAndDispatch", "testproj.bothcaller.both.CallAndDispatch", "bothcaller/both.go", 3, 7) - // Sync caller function - _, _ = s.UpsertNode(&store.Node{ - Project: project, Label: "Function", Name: "CallOrder", - QualifiedName: "testproj.synccaller.caller.CallOrder", - FilePath: "synccaller/caller.go", StartLine: 3, EndLine: 6, - }) - - // Both keywords caller function - _, _ = s.UpsertNode(&store.Node{ - Project: project, Label: "Function", Name: "CallAndDispatch", - QualifiedName: "testproj.bothcaller.both.CallAndDispatch", - FilePath: "bothcaller/both.go", StartLine: 3, EndLine: 7, - }) - - // Route handler (different service) _, _ = s.UpsertNode(&store.Node{ Project: project, Label: "Function", Name: "create_order", QualifiedName: "testproj.handler.routes.create_order", FilePath: "handler/routes.py", - Properties: map[string]any{ - "decorators": []any{`@app.post("/api/orders")`}, - }, + Properties: map[string]any{"decorators": []any{`@app.post("/api/orders")`}}, }) linker := New(s, project) @@ -846,56 +801,68 @@ func CallAndDispatch() { t.Fatalf("Run: %v", err) } - // Build a map of callerQN -> edgeType for verification edgeTypes := map[string]string{} for _, link := range links { edgeTypes[link.CallerQN] = link.EdgeType } - // Async caller (CreateTask only) -> ASYNC_CALLS - if et, ok := edgeTypes["testproj.taskworker.dispatch.DispatchOrder"]; !ok { - t.Error("expected link from async caller DispatchOrder") - } else if et != "ASYNC_CALLS" { - t.Errorf("DispatchOrder edge type = %q, want ASYNC_CALLS", et) - } + assertEdgeType(t, edgeTypes, "testproj.taskworker.dispatch.DispatchOrder", "ASYNC_CALLS") + assertEdgeType(t, edgeTypes, "testproj.synccaller.caller.CallOrder", "HTTP_CALLS") + assertEdgeType(t, edgeTypes, "testproj.bothcaller.both.CallAndDispatch", "HTTP_CALLS") - // Sync caller (requests.post only) -> HTTP_CALLS - if et, ok := edgeTypes["testproj.synccaller.caller.CallOrder"]; !ok { - t.Error("expected link from sync caller CallOrder") - } else if et != "HTTP_CALLS" { - t.Errorf("CallOrder edge type = %q, want HTTP_CALLS", et) - } + assertStoredEdgeCounts(t, s, project, "testproj.taskworker.dispatch.DispatchOrder", 1, 0) + assertStoredEdgeCounts(t, s, project, "testproj.synccaller.caller.CallOrder", 0, 1) +} - // Both keywords (sync takes precedence) -> HTTP_CALLS - if et, ok := edgeTypes["testproj.bothcaller.both.CallAndDispatch"]; !ok { - t.Error("expected link from both-keyword caller CallAndDispatch") - } else if et != "HTTP_CALLS" { - t.Errorf("CallAndDispatch edge type = %q, want HTTP_CALLS", et) +func writeTestFile(t *testing.T, dir, subdir, filename, content string) { + t.Helper() + d := filepath.Join(dir, subdir) + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) } - - // Verify edges in store match - asyncNode, _ := s.FindNodeByQN(project, "testproj.taskworker.dispatch.DispatchOrder") - if asyncNode != nil { - asyncEdges, _ := s.FindEdgesBySourceAndType(asyncNode.ID, "ASYNC_CALLS") - if len(asyncEdges) != 1 { - t.Errorf("expected 1 ASYNC_CALLS edge from async caller, got %d", len(asyncEdges)) - } - httpEdges, _ := s.FindEdgesBySourceAndType(asyncNode.ID, "HTTP_CALLS") - if len(httpEdges) != 0 { - t.Errorf("expected 0 HTTP_CALLS edges from async caller, got %d", len(httpEdges)) - } + if err := os.WriteFile(filepath.Join(d, filename), []byte(content), 0o600); err != nil { + t.Fatal(err) } +} - syncNode, _ := s.FindNodeByQN(project, "testproj.synccaller.caller.CallOrder") - if syncNode != nil { - httpEdges, _ := s.FindEdgesBySourceAndType(syncNode.ID, "HTTP_CALLS") - if len(httpEdges) != 1 { - t.Errorf("expected 1 HTTP_CALLS edge from sync caller, got %d", len(httpEdges)) - } - asyncEdges, _ := s.FindEdgesBySourceAndType(syncNode.ID, "ASYNC_CALLS") - if len(asyncEdges) != 0 { - t.Errorf("expected 0 ASYNC_CALLS edges from sync caller, got %d", len(asyncEdges)) - } +func createTestNode(t *testing.T, s *store.Store, project, name, qn, filePath string, startLine, endLine int) { + t.Helper() + _, err := s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: name, + QualifiedName: qn, FilePath: filePath, + StartLine: startLine, EndLine: endLine, + }) + if err != nil { + t.Fatal(err) + } +} + +func assertEdgeType(t *testing.T, edgeTypes map[string]string, qn, wantType string) { + t.Helper() + et, ok := edgeTypes[qn] + if !ok { + t.Errorf("expected link from %s", qn) + return + } + if et != wantType { + t.Errorf("%s edge type = %q, want %q", qn, et, wantType) + } +} + +func assertStoredEdgeCounts(t *testing.T, s *store.Store, project, qn string, wantAsync, wantHTTP int) { + t.Helper() + node, _ := s.FindNodeByQN(project, qn) + if node == nil { + t.Errorf("node not found: %s", qn) + return + } + asyncEdges, _ := s.FindEdgesBySourceAndType(node.ID, "ASYNC_CALLS") + if len(asyncEdges) != wantAsync { + t.Errorf("%s: ASYNC_CALLS edges = %d, want %d", qn, len(asyncEdges), wantAsync) + } + httpEdges, _ := s.FindEdgesBySourceAndType(node.ID, "HTTP_CALLS") + if len(httpEdges) != wantHTTP { + t.Errorf("%s: HTTP_CALLS edges = %d, want %d", qn, len(httpEdges), wantHTTP) } } diff --git a/internal/pipeline/envscan.go b/internal/pipeline/envscan.go index d04a529ac..6230adac8 100644 --- a/internal/pipeline/envscan.go +++ b/internal/pipeline/envscan.go @@ -24,7 +24,7 @@ func ScanProjectEnvURLs(rootPath string) []EnvBinding { _ = filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { if err != nil { - return nil // skip errors + return err } name := info.Name() @@ -124,7 +124,7 @@ var ( // .env files: KEY=https://... envFilePatterns = []*regexp.Regexp{ - regexp.MustCompile(`^(\w+)=(https?://[^\s]+)`), + regexp.MustCompile(`^(\w+)=(https?://\S+)`), } // TOML: key = "https://..." diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 929c57602..45cbd8275 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -838,7 +838,6 @@ func (p *Pipeline) passCalls() { func (p *Pipeline) processFileCalls(relPath string, cached *cachedAST, spec *lang.LanguageSpec) { callTypes := toSet(spec.CallNodeTypes) - funcTypes := toSet(spec.FunctionNodeTypes) moduleQN := fqn.ModuleQN(p.ProjectName, relPath) root := cached.Tree.RootNode() @@ -860,47 +859,19 @@ func (p *Pipeline) processFileCalls(relPath string, cached *cachedAST, spec *lan return false } - // Find the enclosing function callerQN := findEnclosingFunction(node, cached.Source, p.ProjectName, relPath, spec) if callerQN == "" { callerQN = moduleQN } - // Build a local type map that includes receiver/self scope - localTypeMap := typeMap - // Python self.method() resolution - if cached.Language == lang.Python && strings.HasPrefix(calleeName, "self.") { - classQN := findEnclosingClassQN(node, cached.Source, p.ProjectName, relPath) - if classQN != "" { - methodName := calleeName[5:] - candidate := classQN + "." + methodName - if n, _ := p.Store.FindNodeByQN(p.ProjectName, candidate); n != nil { - p.createCallEdge(callerQN, candidate) - return false - } - } + if p.resolvePythonSelfCall(node, cached, calleeName, callerQN, relPath) { + return false } - // Go receiver scoping — add receiver variable to local type map - if cached.Language == lang.Go { - enclosing := findEnclosingFuncNode(node, funcTypes) - if enclosing != nil { - varName, typeName := parseGoReceiverType(enclosing, cached.Source) - if varName != "" && typeName != "" { - classQN := resolveAsClass(typeName, p.registry, moduleQN, importMap) - if classQN != "" { - localTypeMap = make(TypeMap, len(typeMap)+1) - for k, v := range typeMap { - localTypeMap[k] = v - } - localTypeMap[varName] = classQN - } - } - } - } + // Go receiver scoping — extend type map with receiver variable + localTypeMap := p.extendTypeMapWithReceiver(node, cached, typeMap, spec, moduleQN, importMap) - // Try type-based resolution for method calls (obj.method) targetQN := p.resolveCallWithTypes(calleeName, moduleQN, importMap, localTypeMap) if targetQN == "" { return false @@ -911,6 +882,54 @@ func (p *Pipeline) processFileCalls(relPath string, cached *cachedAST, spec *lan }) } +// resolvePythonSelfCall handles Python self.method() resolution via enclosing class scope. +// Returns true if the call was resolved. +func (p *Pipeline) resolvePythonSelfCall(node *tree_sitter.Node, cached *cachedAST, calleeName, callerQN, relPath string) bool { + if cached.Language != lang.Python || !strings.HasPrefix(calleeName, "self.") { + return false + } + classQN := findEnclosingClassQN(node, cached.Source, p.ProjectName, relPath) + if classQN == "" { + return false + } + candidate := classQN + "." + calleeName[5:] + if n, _ := p.Store.FindNodeByQN(p.ProjectName, candidate); n != nil { + p.createCallEdge(callerQN, candidate) + return true + } + return false +} + +// extendTypeMapWithReceiver augments the type map with the Go receiver variable +// from the enclosing method declaration, if applicable. +func (p *Pipeline) extendTypeMapWithReceiver( + node *tree_sitter.Node, cached *cachedAST, typeMap TypeMap, + spec *lang.LanguageSpec, moduleQN string, importMap map[string]string, +) TypeMap { + if cached.Language != lang.Go { + return typeMap + } + funcTypes := toSet(spec.FunctionNodeTypes) + enclosing := findEnclosingFuncNode(node, funcTypes) + if enclosing == nil { + return typeMap + } + varName, typeName := parseGoReceiverType(enclosing, cached.Source) + if varName == "" || typeName == "" { + return typeMap + } + classQN := resolveAsClass(typeName, p.registry, moduleQN, importMap) + if classQN == "" { + return typeMap + } + localTypeMap := make(TypeMap, len(typeMap)+1) + for k, v := range typeMap { + localTypeMap[k] = v + } + localTypeMap[varName] = classQN + return localTypeMap +} + // createCallEdge creates a CALLS edge between caller and target by QN. func (p *Pipeline) createCallEdge(callerQN, targetQN string) { callerNode, _ := p.Store.FindNodeByQN(p.ProjectName, callerQN) @@ -1183,7 +1202,6 @@ func (p *Pipeline) passHTTPLinks() error { // environment variable URL bindings. These synthetic constants feed into the // HTTP linker's call site discovery. func (p *Pipeline) injectEnvBindings(bindings []EnvBinding) { - // Group by file path byFile := make(map[string][]EnvBinding) for _, b := range bindings { byFile[b.FilePath] = append(byFile[b.FilePath], b) @@ -1192,41 +1210,11 @@ func (p *Pipeline) injectEnvBindings(bindings []EnvBinding) { count := 0 for filePath, fileBindings := range byFile { moduleQN := fqn.ModuleQN(p.ProjectName, filePath) + constants := buildConstantsList(fileBindings) - // Build constants list from bindings - var constants []string - for _, b := range fileBindings { - constants = append(constants, b.Key+" = "+b.Value) - } - - // Cap at 50 constants per config file - if len(constants) > 50 { - constants = constants[:50] - } - - // Merge with existing constants if module already exists - existing, _ := p.Store.FindNodeByQN(p.ProjectName, moduleQN) - if existing != nil { - if existConsts, ok := existing.Properties["constants"].([]any); ok { - seen := make(map[string]bool, len(existConsts)) - for _, c := range existConsts { - if s, ok := c.(string); ok { - seen[s] = true - } - } - for _, c := range constants { - if !seen[c] { - existConsts = append(existConsts, c) - } - } - if existing.Properties == nil { - existing.Properties = map[string]any{} - } - existing.Properties["constants"] = existConsts - _, _ = p.Store.UpsertNode(existing) - count += len(fileBindings) - continue - } + if p.mergeWithExistingModule(moduleQN, constants) { + count += len(fileBindings) + continue } _, _ = p.Store.UpsertNode(&store.Node{ @@ -1245,6 +1233,48 @@ func (p *Pipeline) injectEnvBindings(bindings []EnvBinding) { } } +// buildConstantsList converts env bindings to "KEY = VALUE" constant strings, capped at 50. +func buildConstantsList(bindings []EnvBinding) []string { + constants := make([]string, 0, len(bindings)) + for _, b := range bindings { + constants = append(constants, b.Key+" = "+b.Value) + } + if len(constants) > 50 { + constants = constants[:50] + } + return constants +} + +// mergeWithExistingModule merges new constants into an existing Module node's constant list. +// Returns true if the module existed and was updated. +func (p *Pipeline) mergeWithExistingModule(moduleQN string, constants []string) bool { + existing, _ := p.Store.FindNodeByQN(p.ProjectName, moduleQN) + if existing == nil { + return false + } + existConsts, ok := existing.Properties["constants"].([]any) + if !ok { + return false + } + seen := make(map[string]bool, len(existConsts)) + for _, c := range existConsts { + if s, ok := c.(string); ok { + seen[s] = true + } + } + for _, c := range constants { + if !seen[c] { + existConsts = append(existConsts, c) + } + } + if existing.Properties == nil { + existing.Properties = map[string]any{} + } + existing.Properties["constants"] = existConsts + _, _ = p.Store.UpsertNode(existing) + return true +} + // jsonURLKeyPattern matches JSON keys that likely contain URL/endpoint values. var jsonURLKeyPattern = regexp.MustCompile(`(?i)(url|endpoint|base_url|host|api_url|service_url|target_url|callback_url|webhook|href|uri|address|server|origin|proxy|redirect|forward|destination)`)