mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 08:13:37 +08:00
v0.1.5: Performance optimizations, mattn driver, batch passStructure, CTE BFS, SQL COUNT, fix delete_project file_hashes
This commit is contained in:
Martin Vogel
@@ -3,6 +3,7 @@ module github.com/DeusData/codebase-memory-mcp
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/modelcontextprotocol/go-sdk v1.3.1
|
||||
github.com/tree-sitter-grammars/tree-sitter-lua v0.4.1
|
||||
github.com/tree-sitter/go-tree-sitter v0.25.0
|
||||
@@ -18,25 +19,16 @@ require (
|
||||
github.com/zeebo/xxh3 v1.1.0
|
||||
golang.org/x/sync v0.17.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.46.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/jsonschema-go v0.4.2 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-pointer v0.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/segmentio/asm v1.1.3 // indirect
|
||||
github.com/segmentio/encoding v0.5.3 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/sys v0.37.0 // indirect
|
||||
modernc.org/libc v1.67.6 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
golang.org/x/tools v0.38.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,33 +1,21 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8=
|
||||
github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0=
|
||||
github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc=
|
||||
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
|
||||
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/modelcontextprotocol/go-sdk v1.3.1 h1:TfqtNKOIWN4Z1oqmPAiWDC2Jq7K9OdJaooe0teoXASI=
|
||||
github.com/modelcontextprotocol/go-sdk v1.3.1/go.mod h1:DgVX498dMD8UJlseK1S5i1T4tFz2fkBk4xogC3D15nw=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
|
||||
github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
|
||||
github.com/segmentio/encoding v0.5.3 h1:OjMgICtcSFuNvQCdwqMCv9Tg7lEOXGwm1J5RPQccx6w=
|
||||
@@ -72,15 +60,10 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
|
||||
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
||||
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
|
||||
@@ -89,31 +72,3 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
|
||||
modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
|
||||
modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc=
|
||||
modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM=
|
||||
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
|
||||
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE=
|
||||
modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI=
|
||||
modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
|
||||
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU=
|
||||
modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
|
||||
+403
-152
@@ -15,7 +15,8 @@ const maxResultRows = 200
|
||||
|
||||
// Executor runs Cypher execution plans against a store.
|
||||
type Executor struct {
|
||||
Store *store.Store
|
||||
Store *store.Store
|
||||
regexCache map[string]*regexp.Regexp
|
||||
}
|
||||
|
||||
// Result holds the tabular output of a query.
|
||||
@@ -37,12 +38,6 @@ func newBinding() binding {
|
||||
}
|
||||
}
|
||||
|
||||
// adjacentResult pairs a matched node with the edge that reached it.
|
||||
type adjacentResult struct {
|
||||
Node *store.Node
|
||||
Edge *store.Edge
|
||||
}
|
||||
|
||||
// Execute parses, plans, and executes a Cypher query across all projects.
|
||||
func (e *Executor) Execute(query string) (*Result, error) {
|
||||
q, err := Parse(query)
|
||||
@@ -57,6 +52,11 @@ func (e *Executor) Execute(query string) (*Result, error) {
|
||||
}
|
||||
|
||||
func (e *Executor) executePlan(plan *Plan) (*Result, error) {
|
||||
// Fast path: push COUNT aggregation to SQL when pattern is fusible.
|
||||
if result, ok := e.tryAggregateSQL(plan); ok {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
projects, err := e.Store.ListProjects()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list projects: %w", err)
|
||||
@@ -78,6 +78,247 @@ func (e *Executor) executePlan(plan *Plan) (*Result, error) {
|
||||
return e.projectResults(allBindings, plan.ReturnSpec)
|
||||
}
|
||||
|
||||
// tryAggregateSQL attempts to push a COUNT aggregation query entirely to SQL.
|
||||
// Returns (result, true) if the pattern is fusible, (nil, false) otherwise.
|
||||
// Fusible pattern: ScanNodes + ExpandRelationship (fixed-length 1 hop) with
|
||||
// RETURN containing COUNT and group-by items that map to SQL columns.
|
||||
func (e *Executor) tryAggregateSQL(plan *Plan) (*Result, bool) {
|
||||
if plan.ReturnSpec == nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Must have COUNT in return
|
||||
var countItem *ReturnItem
|
||||
var groupItems []ReturnItem
|
||||
for i := range plan.ReturnSpec.Items {
|
||||
item := &plan.ReturnSpec.Items[i]
|
||||
if item.Func == "COUNT" {
|
||||
countItem = item
|
||||
} else {
|
||||
groupItems = append(groupItems, *item)
|
||||
}
|
||||
}
|
||||
if countItem == nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Identify the fusible step pattern: optional FilterWhere between Scan and Expand
|
||||
steps := plan.Steps
|
||||
var scan *ScanNodes
|
||||
var expand *ExpandRelationship
|
||||
var filter *FilterWhere
|
||||
|
||||
switch len(steps) {
|
||||
case 2:
|
||||
s, ok1 := steps[0].(*ScanNodes)
|
||||
ex, ok2 := steps[1].(*ExpandRelationship)
|
||||
if !ok1 || !ok2 {
|
||||
return nil, false
|
||||
}
|
||||
scan, expand = s, ex
|
||||
case 3:
|
||||
s, ok1 := steps[0].(*ScanNodes)
|
||||
f, ok2 := steps[1].(*FilterWhere)
|
||||
ex, ok3 := steps[2].(*ExpandRelationship)
|
||||
if !ok1 || !ok3 {
|
||||
return nil, false
|
||||
}
|
||||
scan, expand = s, ex
|
||||
if ok2 {
|
||||
filter = f
|
||||
}
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Must be fixed-length (1 hop)
|
||||
if expand.MinHops != 1 || expand.MaxHops != 1 {
|
||||
return nil, false
|
||||
}
|
||||
if len(expand.EdgeTypes) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
if len(scan.Props) > 0 || len(expand.ToProps) > 0 {
|
||||
return nil, false // inline property filters not supported in SQL push-down
|
||||
}
|
||||
|
||||
// All group-by items must reference SQL-pushable columns
|
||||
for _, gi := range groupItems {
|
||||
if gi.Property == "" {
|
||||
return nil, false
|
||||
}
|
||||
if _, ok := sqlPushableColumns[gi.Property]; !ok {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// Filter conditions must all be pushable
|
||||
if filter != nil {
|
||||
for _, c := range filter.Conditions {
|
||||
if _, ok := sqlPushableColumns[c.Property]; !ok {
|
||||
return nil, false
|
||||
}
|
||||
switch c.Operator {
|
||||
case "=", "CONTAINS", "STARTS WITH":
|
||||
// OK
|
||||
default:
|
||||
return nil, false // regex, numeric not pushable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build the SQL query
|
||||
projects, err := e.Store.ListProjects()
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
dir := expand.Direction
|
||||
if dir == "" {
|
||||
dir = "outbound"
|
||||
}
|
||||
var srcCol, tgtCol string
|
||||
if dir == "outbound" {
|
||||
srcCol, tgtCol = "source_id", "target_id"
|
||||
} else {
|
||||
srcCol, tgtCol = "target_id", "source_id"
|
||||
}
|
||||
|
||||
// Map variable names to table aliases
|
||||
srcAlias := "src" // scan variable
|
||||
tgtAlias := "tgt" // expand target variable
|
||||
|
||||
// Build SELECT columns for group-by items
|
||||
selectParts := make([]string, 0, len(groupItems)+1)
|
||||
groupByParts := make([]string, 0, len(groupItems))
|
||||
for _, gi := range groupItems {
|
||||
col := sqlPushableColumns[gi.Property]
|
||||
var alias string
|
||||
if gi.Variable == scan.Variable {
|
||||
alias = srcAlias
|
||||
} else {
|
||||
alias = tgtAlias
|
||||
}
|
||||
selectParts = append(selectParts, alias+"."+col)
|
||||
groupByParts = append(groupByParts, alias+"."+col)
|
||||
}
|
||||
selectParts = append(selectParts, "COUNT(*) as cnt")
|
||||
|
||||
// Accumulate results across all projects
|
||||
cols := buildColumnNames(plan.ReturnSpec.Items)
|
||||
countCol := cols[len(cols)-1] // COUNT column is last
|
||||
|
||||
allRows := make([]map[string]any, 0)
|
||||
|
||||
for _, proj := range projects {
|
||||
var sb strings.Builder
|
||||
args := make([]any, 0, 10)
|
||||
|
||||
sb.WriteString("SELECT ")
|
||||
sb.WriteString(strings.Join(selectParts, ", "))
|
||||
sb.WriteString(" FROM edges e")
|
||||
sb.WriteString(fmt.Sprintf(" JOIN nodes %s ON %s.id = e.%s", srcAlias, srcAlias, srcCol))
|
||||
sb.WriteString(fmt.Sprintf(" JOIN nodes %s ON %s.id = e.%s", tgtAlias, tgtAlias, tgtCol))
|
||||
sb.WriteString(" WHERE e.project = ?")
|
||||
args = append(args, proj.Name)
|
||||
|
||||
// Edge type filter
|
||||
typePlaceholders := make([]string, len(expand.EdgeTypes))
|
||||
for i, et := range expand.EdgeTypes {
|
||||
typePlaceholders[i] = "?"
|
||||
args = append(args, et)
|
||||
}
|
||||
sb.WriteString(" AND e.type IN (" + strings.Join(typePlaceholders, ",") + ")")
|
||||
|
||||
// Label filters
|
||||
if scan.Label != "" {
|
||||
sb.WriteString(fmt.Sprintf(" AND %s.label = ?", srcAlias))
|
||||
args = append(args, scan.Label)
|
||||
}
|
||||
if expand.ToLabel != "" {
|
||||
sb.WriteString(fmt.Sprintf(" AND %s.label = ?", tgtAlias))
|
||||
args = append(args, expand.ToLabel)
|
||||
}
|
||||
|
||||
// Push-down WHERE conditions
|
||||
if filter != nil {
|
||||
for _, c := range filter.Conditions {
|
||||
col := sqlPushableColumns[c.Property]
|
||||
var alias string
|
||||
if c.Variable == scan.Variable {
|
||||
alias = srcAlias
|
||||
} else {
|
||||
alias = tgtAlias
|
||||
}
|
||||
switch c.Operator {
|
||||
case "=":
|
||||
sb.WriteString(fmt.Sprintf(" AND %s.%s = ?", alias, col))
|
||||
args = append(args, c.Value)
|
||||
case "CONTAINS":
|
||||
sb.WriteString(fmt.Sprintf(" AND %s.%s LIKE ?", alias, col))
|
||||
args = append(args, "%"+c.Value+"%")
|
||||
case "STARTS WITH":
|
||||
sb.WriteString(fmt.Sprintf(" AND %s.%s LIKE ?", alias, col))
|
||||
args = append(args, c.Value+"%")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sb.WriteString(" GROUP BY " + strings.Join(groupByParts, ", "))
|
||||
|
||||
rows, qErr := e.Store.DB().Query(sb.String(), args...)
|
||||
if qErr != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
// Scan group-by values + count
|
||||
vals := make([]any, len(groupItems)+1)
|
||||
ptrs := make([]any, len(groupItems)+1)
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if sErr := rows.Scan(ptrs...); sErr != nil {
|
||||
rows.Close()
|
||||
continue
|
||||
}
|
||||
row := make(map[string]any)
|
||||
for i, gi := range groupItems {
|
||||
col := gi.Variable + "." + gi.Property
|
||||
if gi.Alias != "" {
|
||||
col = gi.Alias
|
||||
}
|
||||
// Normalize []byte to string (mattn returns []byte for TEXT cols)
|
||||
if b, ok := vals[i].([]byte); ok {
|
||||
row[col] = string(b)
|
||||
} else {
|
||||
row[col] = vals[i]
|
||||
}
|
||||
}
|
||||
// Normalize COUNT to int for consistency with Go-side aggregation
|
||||
switch v := vals[len(vals)-1].(type) {
|
||||
case int64:
|
||||
row[countCol] = int(v)
|
||||
default:
|
||||
row[countCol] = v
|
||||
}
|
||||
allRows = append(allRows, row)
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
|
||||
// ORDER BY
|
||||
if plan.ReturnSpec.OrderBy != "" {
|
||||
orderCol := resolveOrderColumn(plan.ReturnSpec.OrderBy, plan.ReturnSpec.Items, cols)
|
||||
sortRows(allRows, orderCol, plan.ReturnSpec.OrderDir)
|
||||
}
|
||||
|
||||
// LIMIT
|
||||
allRows = applyLimit(allRows, plan.ReturnSpec.Limit)
|
||||
|
||||
return &Result{Columns: cols, Rows: allRows}, true
|
||||
}
|
||||
|
||||
//nolint:gocognit // step dispatch with fusion/push-down is inherently branchy
|
||||
func (e *Executor) executeStepsForProject(project string, steps []PlanStep) ([]binding, error) {
|
||||
var bindings []binding
|
||||
@@ -335,7 +576,7 @@ func (e *Executor) execScan(project string, s *ScanNodes, pushDown *FilterWhere)
|
||||
if len(unpushedConditions) > 0 {
|
||||
b := newBinding()
|
||||
b.nodes[s.Variable] = n
|
||||
match, evalErr := evaluateConditions(b, unpushedConditions, "AND")
|
||||
match, evalErr := e.evaluateConditions(b, unpushedConditions, "AND")
|
||||
if evalErr != nil {
|
||||
return nil, evalErr
|
||||
}
|
||||
@@ -370,25 +611,145 @@ func (e *Executor) execExpand(s *ExpandRelationship, bindings []binding) ([]bind
|
||||
|
||||
isVariableLength := s.MinHops != 1 || s.MaxHops != 1
|
||||
|
||||
if isVariableLength {
|
||||
// Variable-length: use BFS per binding (already uses recursive CTE)
|
||||
result := make([]binding, 0, len(bindings))
|
||||
for _, b := range bindings {
|
||||
fromNode, ok := b.nodes[s.FromVar]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
expanded, err := e.expandVariableLength(b, fromNode, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, expanded...)
|
||||
if len(result) > maxResultRows*2 {
|
||||
result = result[:maxResultRows*2]
|
||||
break
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Fixed-length (1 hop): batch all source IDs, 2 queries total
|
||||
return e.expandFixedLengthBatch(s, bindings)
|
||||
}
|
||||
|
||||
// expandFixedLengthBatch performs fixed-length (1 hop) expansion for all bindings
|
||||
// in just 2 SQL queries: one for edges, one for target nodes.
|
||||
func (e *Executor) expandFixedLengthBatch(s *ExpandRelationship, bindings []binding) ([]binding, error) {
|
||||
// Collect all unique source node IDs
|
||||
sourceIDs := make([]int64, 0, len(bindings))
|
||||
idSeen := make(map[int64]bool, len(bindings))
|
||||
for _, b := range bindings {
|
||||
fromNode, ok := b.nodes[s.FromVar]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !idSeen[fromNode.ID] {
|
||||
idSeen[fromNode.ID] = true
|
||||
sourceIDs = append(sourceIDs, fromNode.ID)
|
||||
}
|
||||
}
|
||||
|
||||
if len(sourceIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
direction := s.Direction
|
||||
if direction == "" {
|
||||
direction = "outbound"
|
||||
}
|
||||
|
||||
// Batch 1: fetch all edges for all source IDs
|
||||
var edgesByNode map[int64][]*store.Edge
|
||||
var err error
|
||||
switch direction {
|
||||
case "outbound":
|
||||
edgesByNode, err = e.Store.FindEdgesBySourceIDs(sourceIDs, s.EdgeTypes)
|
||||
case "inbound":
|
||||
edgesByNode, err = e.Store.FindEdgesByTargetIDs(sourceIDs, s.EdgeTypes)
|
||||
case "any":
|
||||
outEdges, outErr := e.Store.FindEdgesBySourceIDs(sourceIDs, s.EdgeTypes)
|
||||
if outErr != nil {
|
||||
return nil, outErr
|
||||
}
|
||||
inEdges, inErr := e.Store.FindEdgesByTargetIDs(sourceIDs, s.EdgeTypes)
|
||||
if inErr != nil {
|
||||
return nil, inErr
|
||||
}
|
||||
edgesByNode = outEdges
|
||||
for id, edges := range inEdges {
|
||||
edgesByNode[id] = append(edgesByNode[id], edges...)
|
||||
}
|
||||
default:
|
||||
edgesByNode, err = e.Store.FindEdgesBySourceIDs(sourceIDs, s.EdgeTypes)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Collect all target node IDs
|
||||
targetIDSet := make(map[int64]bool)
|
||||
for _, edges := range edgesByNode {
|
||||
for _, edge := range edges {
|
||||
tid := edgeTargetID(edge, 0, direction)
|
||||
if direction == "any" {
|
||||
// For "any", we need to check per-source
|
||||
targetIDSet[edge.SourceID] = true
|
||||
targetIDSet[edge.TargetID] = true
|
||||
} else {
|
||||
targetIDSet[tid] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
targetIDs := make([]int64, 0, len(targetIDSet))
|
||||
for id := range targetIDSet {
|
||||
targetIDs = append(targetIDs, id)
|
||||
}
|
||||
|
||||
// Batch 2: fetch all target nodes
|
||||
nodeMap, err := e.Store.FindNodesByIDs(targetIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Build results per binding
|
||||
result := make([]binding, 0, len(bindings))
|
||||
for _, b := range bindings {
|
||||
fromNode, ok := b.nodes[s.FromVar]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
edges := edgesByNode[fromNode.ID]
|
||||
seen := make(map[int64]bool)
|
||||
|
||||
if isVariableLength {
|
||||
expanded, err := e.expandVariableLength(b, fromNode, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
for _, edge := range edges {
|
||||
targetID := edgeTargetID(edge, fromNode.ID, direction)
|
||||
if seen[targetID] {
|
||||
continue
|
||||
}
|
||||
result = append(result, expanded...)
|
||||
} else {
|
||||
expanded, err := e.expandFixedLength(b, fromNode, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
seen[targetID] = true
|
||||
|
||||
node, exists := nodeMap[targetID]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
result = append(result, expanded...)
|
||||
if s.ToLabel != "" && node.Label != s.ToLabel {
|
||||
continue
|
||||
}
|
||||
if len(s.ToProps) > 0 && !nodeMatchesProps(node, s.ToProps) {
|
||||
continue
|
||||
}
|
||||
newB := copyBinding(b)
|
||||
if s.ToVar != "" {
|
||||
newB.nodes[s.ToVar] = node
|
||||
}
|
||||
if s.RelVar != "" {
|
||||
newB.edges[s.RelVar] = edge
|
||||
}
|
||||
result = append(result, newB)
|
||||
}
|
||||
|
||||
if len(result) > maxResultRows*2 {
|
||||
@@ -399,32 +760,6 @@ func (e *Executor) execExpand(s *ExpandRelationship, bindings []binding) ([]bind
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (e *Executor) expandFixedLength(b binding, fromNode *store.Node, s *ExpandRelationship) ([]binding, error) {
|
||||
adjacents, err := e.findAdjacentNodes(fromNode.ID, s.EdgeTypes, s.Direction)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make([]binding, 0, len(adjacents))
|
||||
for _, adj := range adjacents {
|
||||
if s.ToLabel != "" && adj.Node.Label != s.ToLabel {
|
||||
continue
|
||||
}
|
||||
if len(s.ToProps) > 0 && !nodeMatchesProps(adj.Node, s.ToProps) {
|
||||
continue
|
||||
}
|
||||
newB := copyBinding(b)
|
||||
if s.ToVar != "" {
|
||||
newB.nodes[s.ToVar] = adj.Node
|
||||
}
|
||||
if s.RelVar != "" && adj.Edge != nil {
|
||||
newB.edges[s.RelVar] = adj.Edge
|
||||
}
|
||||
result = append(result, newB)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (e *Executor) expandVariableLength(b binding, fromNode *store.Node, s *ExpandRelationship) ([]binding, error) {
|
||||
maxDepth := s.MaxHops
|
||||
if maxDepth == 0 {
|
||||
@@ -470,107 +805,6 @@ func (e *Executor) expandVariableLength(b binding, fromNode *store.Node, s *Expa
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// findAdjacentNodes returns nodes connected to nodeID via the given edge types and direction.
|
||||
func (e *Executor) findAdjacentNodes(nodeID int64, edgeTypes []string, direction string) ([]adjacentResult, error) {
|
||||
allEdges, err := e.collectEdges(nodeID, edgeTypes, direction)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.resolveEdgeTargets(allEdges, nodeID, direction)
|
||||
}
|
||||
|
||||
// collectEdges gathers edges from the store based on direction and type filters.
|
||||
func (e *Executor) collectEdges(nodeID int64, edgeTypes []string, direction string) ([]*store.Edge, error) {
|
||||
switch direction {
|
||||
case "outbound":
|
||||
return e.collectDirectionalEdges(nodeID, edgeTypes, true)
|
||||
case "inbound":
|
||||
return e.collectDirectionalEdges(nodeID, edgeTypes, false)
|
||||
case "any":
|
||||
return e.collectBidirectionalEdges(nodeID, edgeTypes)
|
||||
default:
|
||||
return e.collectDirectionalEdges(nodeID, edgeTypes, true)
|
||||
}
|
||||
}
|
||||
|
||||
// collectDirectionalEdges collects edges in one direction (outbound if isSource, inbound otherwise).
|
||||
func (e *Executor) collectDirectionalEdges(nodeID int64, edgeTypes []string, isSource bool) ([]*store.Edge, error) {
|
||||
if len(edgeTypes) > 0 {
|
||||
var allEdges []*store.Edge
|
||||
for _, et := range edgeTypes {
|
||||
var edges []*store.Edge
|
||||
var err error
|
||||
if isSource {
|
||||
edges, err = e.Store.FindEdgesBySourceAndType(nodeID, et)
|
||||
} else {
|
||||
edges, err = e.Store.FindEdgesByTargetAndType(nodeID, et)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allEdges = append(allEdges, edges...)
|
||||
}
|
||||
return allEdges, nil
|
||||
}
|
||||
if isSource {
|
||||
return e.Store.FindEdgesBySource(nodeID)
|
||||
}
|
||||
return e.Store.FindEdgesByTarget(nodeID)
|
||||
}
|
||||
|
||||
// collectBidirectionalEdges collects edges in both directions, optionally filtered by type.
|
||||
func (e *Executor) collectBidirectionalEdges(nodeID int64, edgeTypes []string) ([]*store.Edge, error) {
|
||||
outEdges, err := e.Store.FindEdgesBySource(nodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inEdges, err := e.Store.FindEdgesByTarget(nodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(edgeTypes) == 0 {
|
||||
result := make([]*store.Edge, 0, len(outEdges)+len(inEdges))
|
||||
result = append(result, outEdges...)
|
||||
result = append(result, inEdges...)
|
||||
return result, nil
|
||||
}
|
||||
typeSet := make(map[string]bool, len(edgeTypes))
|
||||
for _, et := range edgeTypes {
|
||||
typeSet[et] = true
|
||||
}
|
||||
var allEdges []*store.Edge
|
||||
for _, edge := range outEdges {
|
||||
if typeSet[edge.Type] {
|
||||
allEdges = append(allEdges, edge)
|
||||
}
|
||||
}
|
||||
for _, edge := range inEdges {
|
||||
if typeSet[edge.Type] {
|
||||
allEdges = append(allEdges, edge)
|
||||
}
|
||||
}
|
||||
return allEdges, nil
|
||||
}
|
||||
|
||||
// resolveEdgeTargets resolves edges to adjacentResult by looking up the target/source node.
|
||||
func (e *Executor) resolveEdgeTargets(allEdges []*store.Edge, nodeID int64, direction string) ([]adjacentResult, error) {
|
||||
seen := make(map[int64]bool)
|
||||
results := make([]adjacentResult, 0, len(allEdges))
|
||||
for _, edge := range allEdges {
|
||||
targetID := edgeTargetID(edge, nodeID, direction)
|
||||
if seen[targetID] {
|
||||
continue
|
||||
}
|
||||
seen[targetID] = true
|
||||
|
||||
node, err := e.Store.FindNodeByID(targetID)
|
||||
if err != nil || node == nil {
|
||||
continue
|
||||
}
|
||||
results = append(results, adjacentResult{Node: node, Edge: edge})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// edgeTargetID returns the ID of the node on the "other end" of the edge relative to nodeID.
|
||||
func edgeTargetID(edge *store.Edge, nodeID int64, direction string) int64 {
|
||||
@@ -590,7 +824,7 @@ func edgeTargetID(edge *store.Edge, nodeID int64, direction string) int64 {
|
||||
func (e *Executor) execFilter(s *FilterWhere, bindings []binding) ([]binding, error) {
|
||||
var result []binding
|
||||
for _, b := range bindings {
|
||||
match, err := evaluateConditions(b, s.Conditions, s.Operator)
|
||||
match, err := e.evaluateConditions(b, s.Conditions, s.Operator)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -601,10 +835,10 @@ func (e *Executor) execFilter(s *FilterWhere, bindings []binding) ([]binding, er
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func evaluateConditions(b binding, conditions []Condition, op string) (bool, error) {
|
||||
func (e *Executor) evaluateConditions(b binding, conditions []Condition, op string) (bool, error) {
|
||||
if op == "OR" {
|
||||
for _, c := range conditions {
|
||||
ok, err := evaluateCondition(b, c)
|
||||
ok, err := e.evaluateCondition(b, c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -616,7 +850,7 @@ func evaluateConditions(b binding, conditions []Condition, op string) (bool, err
|
||||
}
|
||||
// AND (default)
|
||||
for _, c := range conditions {
|
||||
ok, err := evaluateCondition(b, c)
|
||||
ok, err := e.evaluateCondition(b, c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -627,7 +861,24 @@ func evaluateConditions(b binding, conditions []Condition, op string) (bool, err
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func evaluateCondition(b binding, c Condition) (bool, error) {
|
||||
// compiledRegex returns a compiled regex for the pattern, using a cache
|
||||
// to avoid recompiling the same pattern on every binding.
|
||||
func (e *Executor) compiledRegex(pattern string) (*regexp.Regexp, error) {
|
||||
if e.regexCache == nil {
|
||||
e.regexCache = make(map[string]*regexp.Regexp)
|
||||
}
|
||||
if re, ok := e.regexCache[pattern]; ok {
|
||||
return re, nil
|
||||
}
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.regexCache[pattern] = re
|
||||
return re, nil
|
||||
}
|
||||
|
||||
func (e *Executor) evaluateCondition(b binding, c Condition) (bool, error) {
|
||||
// Try node first, then edge
|
||||
var actual any
|
||||
if node, ok := b.nodes[c.Variable]; ok {
|
||||
@@ -646,11 +897,11 @@ func evaluateCondition(b binding, c Condition) (bool, error) {
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
matched, err := regexp.MatchString(c.Value, s)
|
||||
re, err := e.compiledRegex(c.Value)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("regex %q: %w", c.Value, err)
|
||||
}
|
||||
return matched, nil
|
||||
return re.MatchString(s), nil
|
||||
case "CONTAINS":
|
||||
s, ok := actual.(string)
|
||||
if !ok {
|
||||
|
||||
@@ -17,6 +17,7 @@ func init() {
|
||||
"union_specifier",
|
||||
"enum_specifier",
|
||||
},
|
||||
FieldNodeTypes: []string{"field_declaration"},
|
||||
ModuleNodeTypes: []string{
|
||||
"translation_unit",
|
||||
"namespace_definition",
|
||||
|
||||
@@ -6,6 +6,7 @@ func init() {
|
||||
FileExtensions: []string{".go"},
|
||||
FunctionNodeTypes: []string{"function_declaration", "method_declaration"},
|
||||
ClassNodeTypes: []string{"type_spec", "type_alias"},
|
||||
FieldNodeTypes: []string{"field_declaration"},
|
||||
ModuleNodeTypes: []string{"source_file"},
|
||||
CallNodeTypes: []string{"call_expression"},
|
||||
ImportNodeTypes: []string{"import_declaration"},
|
||||
|
||||
@@ -12,6 +12,7 @@ func init() {
|
||||
"annotation_type_declaration",
|
||||
"record_declaration",
|
||||
},
|
||||
FieldNodeTypes: []string{"field_declaration"},
|
||||
ModuleNodeTypes: []string{"program"},
|
||||
CallNodeTypes: []string{"method_invocation"},
|
||||
ImportNodeTypes: []string{"import_declaration"},
|
||||
|
||||
@@ -30,6 +30,7 @@ type LanguageSpec struct {
|
||||
FileExtensions []string
|
||||
FunctionNodeTypes []string
|
||||
ClassNodeTypes []string
|
||||
FieldNodeTypes []string // tree-sitter node kinds for struct/class fields
|
||||
ModuleNodeTypes []string
|
||||
CallNodeTypes []string
|
||||
ImportNodeTypes []string
|
||||
|
||||
@@ -17,6 +17,7 @@ func init() {
|
||||
"impl_item",
|
||||
"type_item",
|
||||
},
|
||||
FieldNodeTypes: []string{"field_declaration"},
|
||||
ModuleNodeTypes: []string{"source_file", "mod_item"},
|
||||
CallNodeTypes: []string{"call_expression", "macro_invocation"},
|
||||
ImportNodeTypes: []string{"use_declaration", "extern_crate_declaration"},
|
||||
|
||||
+344
-95
@@ -11,6 +11,7 @@ import (
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
tree_sitter "github.com/tree-sitter/go-tree-sitter"
|
||||
"github.com/zeebo/xxh3"
|
||||
@@ -68,6 +69,9 @@ func (p *Pipeline) Run() error {
|
||||
}
|
||||
slog.Info("pipeline.discovered", "files", len(files))
|
||||
|
||||
// Use MEMORY journal mode during fresh indexing for faster bulk writes.
|
||||
p.Store.BeginBulkWrite()
|
||||
|
||||
wroteData := false
|
||||
if err := p.Store.WithTransaction(func(txStore *store.Store) error {
|
||||
origStore := p.Store
|
||||
@@ -77,9 +81,12 @@ func (p *Pipeline) Run() error {
|
||||
wroteData, passErr = p.runPasses(files)
|
||||
return passErr
|
||||
}); err != nil {
|
||||
p.Store.EndBulkWrite()
|
||||
return err
|
||||
}
|
||||
|
||||
p.Store.EndBulkWrite()
|
||||
|
||||
// Only checkpoint + optimize when actual data was written.
|
||||
// No-op incremental reindexes skip this to avoid ANALYZE overhead.
|
||||
if wroteData {
|
||||
@@ -121,21 +128,52 @@ func (p *Pipeline) runPasses(files []discover.FileInfo) (bool, error) {
|
||||
|
||||
// runFullPasses runs the complete pipeline (no incremental optimization).
|
||||
func (p *Pipeline) runFullPasses(files []discover.FileInfo) error {
|
||||
t := time.Now()
|
||||
if err := p.passStructure(files); err != nil {
|
||||
return fmt.Errorf("pass1 structure: %w", err)
|
||||
}
|
||||
slog.Info("pass.timing", "pass", "structure", "elapsed", time.Since(t))
|
||||
|
||||
t = time.Now()
|
||||
p.passDefinitions(files)
|
||||
slog.Info("pass.timing", "pass", "definitions", "elapsed", time.Since(t))
|
||||
|
||||
t = time.Now()
|
||||
p.buildRegistry()
|
||||
slog.Info("pass.timing", "pass", "registry", "elapsed", time.Since(t))
|
||||
|
||||
t = time.Now()
|
||||
p.passImports()
|
||||
slog.Info("pass.timing", "pass", "imports", "elapsed", time.Since(t))
|
||||
|
||||
t = time.Now()
|
||||
p.passCalls()
|
||||
slog.Info("pass.timing", "pass", "calls", "elapsed", time.Since(t))
|
||||
|
||||
t = time.Now()
|
||||
p.passUsages()
|
||||
slog.Info("pass.timing", "pass", "usages", "elapsed", time.Since(t))
|
||||
|
||||
p.cleanupASTCache()
|
||||
|
||||
t = time.Now()
|
||||
if err := p.passHTTPLinks(); err != nil {
|
||||
slog.Warn("pass4.httplink.err", "err", err)
|
||||
}
|
||||
slog.Info("pass.timing", "pass", "httplinks", "elapsed", time.Since(t))
|
||||
|
||||
t = time.Now()
|
||||
p.passImplements()
|
||||
slog.Info("pass.timing", "pass", "implements", "elapsed", time.Since(t))
|
||||
|
||||
t = time.Now()
|
||||
p.passGitHistory()
|
||||
slog.Info("pass.timing", "pass", "githistory", "elapsed", time.Since(t))
|
||||
|
||||
t = time.Now()
|
||||
p.updateFileHashes(files)
|
||||
slog.Info("pass.timing", "pass", "filehashes", "elapsed", time.Since(t))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -410,25 +448,12 @@ func mergeFiles(a, b []discover.FileInfo) []discover.FileInfo {
|
||||
}
|
||||
|
||||
// passStructure creates Project, Folder, Package, File nodes and containment edges.
|
||||
// Collects all nodes/edges in memory first, then batch-writes to DB.
|
||||
func (p *Pipeline) passStructure(files []discover.FileInfo) error {
|
||||
slog.Info("pass1.structure")
|
||||
|
||||
// Create project node
|
||||
_, err := p.Store.UpsertNode(&store.Node{
|
||||
Project: p.ProjectName,
|
||||
Label: "Project",
|
||||
Name: p.ProjectName,
|
||||
QualifiedName: p.ProjectName,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Track which directories we've created
|
||||
createdDirs := map[string]bool{}
|
||||
|
||||
// Collect all package indicators
|
||||
packageIndicators := map[string]bool{}
|
||||
packageIndicators := make(map[string]bool)
|
||||
for _, l := range lang.AllLanguages() {
|
||||
spec := lang.ForLanguage(l)
|
||||
if spec != nil {
|
||||
@@ -438,104 +463,127 @@ func (p *Pipeline) passStructure(files []discover.FileInfo) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Phase A: Collect all unique directories from file paths
|
||||
dirSet := make(map[string]bool)
|
||||
for _, f := range files {
|
||||
dir := filepath.Dir(f.RelPath)
|
||||
for dir != "." && dir != "" && !dirSet[dir] {
|
||||
dirSet[dir] = true
|
||||
dir = filepath.Dir(dir)
|
||||
}
|
||||
}
|
||||
|
||||
// Create directory hierarchy
|
||||
p.ensureDirHierarchy(dir, createdDirs, packageIndicators)
|
||||
// Phase B: Check package indicators via os.Stat (batched, not interleaved with SQL)
|
||||
dirIsPackage := make(map[string]bool, len(dirSet))
|
||||
for dir := range dirSet {
|
||||
absDir := filepath.Join(p.RepoPath, dir)
|
||||
for indicator := range packageIndicators {
|
||||
if _, err := os.Stat(filepath.Join(absDir, indicator)); err == nil {
|
||||
dirIsPackage[dir] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create File node
|
||||
fileQN := fqn.Compute(p.ProjectName, f.RelPath, "")
|
||||
fileID, fileErr := p.Store.UpsertNode(&store.Node{
|
||||
// Phase C: Build all nodes and pending edges in memory
|
||||
var nodes []*store.Node
|
||||
var edges []pendingEdge
|
||||
|
||||
// Project node
|
||||
projectQN := p.ProjectName
|
||||
nodes = append(nodes, &store.Node{
|
||||
Project: p.ProjectName,
|
||||
Label: "Project",
|
||||
Name: p.ProjectName,
|
||||
QualifiedName: projectQN,
|
||||
})
|
||||
|
||||
// Directory/Package nodes + containment edges
|
||||
for dir := range dirSet {
|
||||
dirName := filepath.Base(dir)
|
||||
label := "Folder"
|
||||
if dirIsPackage[dir] {
|
||||
label = "Package"
|
||||
}
|
||||
qn := fqn.FolderQN(p.ProjectName, dir)
|
||||
nodes = append(nodes, &store.Node{
|
||||
Project: p.ProjectName,
|
||||
Label: label,
|
||||
Name: dirName,
|
||||
QualifiedName: qn,
|
||||
FilePath: dir,
|
||||
})
|
||||
|
||||
// Containment edge from parent
|
||||
parent := filepath.Dir(dir)
|
||||
var parentQN string
|
||||
if parent == "." || parent == "" {
|
||||
parentQN = projectQN
|
||||
} else {
|
||||
parentQN = fqn.FolderQN(p.ProjectName, parent)
|
||||
}
|
||||
|
||||
edgeType := "CONTAINS_FOLDER"
|
||||
if dirIsPackage[dir] {
|
||||
edgeType = "CONTAINS_PACKAGE"
|
||||
}
|
||||
edges = append(edges, pendingEdge{
|
||||
SourceQN: parentQN,
|
||||
TargetQN: qn,
|
||||
Type: edgeType,
|
||||
})
|
||||
}
|
||||
|
||||
// File nodes + containment edges
|
||||
for _, f := range files {
|
||||
fileQN := fqn.Compute(p.ProjectName, f.RelPath, "") + ".__file__"
|
||||
nodes = append(nodes, &store.Node{
|
||||
Project: p.ProjectName,
|
||||
Label: "File",
|
||||
Name: filepath.Base(f.RelPath),
|
||||
QualifiedName: fileQN + ".__file__",
|
||||
QualifiedName: fileQN,
|
||||
FilePath: f.RelPath,
|
||||
Properties: map[string]any{"extension": filepath.Ext(f.RelPath)},
|
||||
})
|
||||
if fileErr != nil {
|
||||
slog.Warn("pass1.file.err", "path", f.RelPath, "err", fileErr)
|
||||
continue
|
||||
}
|
||||
|
||||
// Create CONTAINS_FILE edge from parent dir
|
||||
dir := filepath.Dir(f.RelPath)
|
||||
parentQN := p.dirQN(dir)
|
||||
parentNode, _ := p.Store.FindNodeByQN(p.ProjectName, parentQN)
|
||||
if parentNode != nil {
|
||||
_, _ = p.Store.InsertEdge(&store.Edge{
|
||||
Project: p.ProjectName,
|
||||
SourceID: parentNode.ID,
|
||||
TargetID: fileID,
|
||||
Type: "CONTAINS_FILE",
|
||||
edges = append(edges, pendingEdge{
|
||||
SourceQN: parentQN,
|
||||
TargetQN: fileQN,
|
||||
Type: "CONTAINS_FILE",
|
||||
})
|
||||
}
|
||||
|
||||
// Phase D: Batch write all nodes
|
||||
idMap, err := p.Store.UpsertNodeBatch(nodes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pass1 batch upsert: %w", err)
|
||||
}
|
||||
|
||||
// Resolve pending edges to real edges using ID map
|
||||
realEdges := make([]*store.Edge, 0, len(edges))
|
||||
for _, pe := range edges {
|
||||
srcID, srcOK := idMap[pe.SourceQN]
|
||||
tgtID, tgtOK := idMap[pe.TargetQN]
|
||||
if srcOK && tgtOK {
|
||||
realEdges = append(realEdges, &store.Edge{
|
||||
Project: p.ProjectName,
|
||||
SourceID: srcID,
|
||||
TargetID: tgtID,
|
||||
Type: pe.Type,
|
||||
Properties: pe.Properties,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err := p.Store.InsertEdgeBatch(realEdges); err != nil {
|
||||
return fmt.Errorf("pass1 batch edges: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Pipeline) ensureDirHierarchy(relDir string, created, pkgIndicators map[string]bool) {
|
||||
if relDir == "." || relDir == "" || created[relDir] {
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure parent first
|
||||
parent := filepath.Dir(relDir)
|
||||
if parent != "." && parent != relDir {
|
||||
p.ensureDirHierarchy(parent, created, pkgIndicators)
|
||||
}
|
||||
|
||||
created[relDir] = true
|
||||
|
||||
// Check if this is a package
|
||||
absDir := filepath.Join(p.RepoPath, relDir)
|
||||
isPackage := false
|
||||
for indicator := range pkgIndicators {
|
||||
if _, err := os.Stat(filepath.Join(absDir, indicator)); err == nil {
|
||||
isPackage = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
dirName := filepath.Base(relDir)
|
||||
label := "Folder"
|
||||
if isPackage {
|
||||
label = "Package"
|
||||
}
|
||||
qn := fqn.FolderQN(p.ProjectName, relDir)
|
||||
|
||||
nodeID, _ := p.Store.UpsertNode(&store.Node{
|
||||
Project: p.ProjectName,
|
||||
Label: label,
|
||||
Name: dirName,
|
||||
QualifiedName: qn,
|
||||
FilePath: relDir,
|
||||
})
|
||||
|
||||
// Create containment edge from parent
|
||||
var parentQN string
|
||||
if parent == "." || parent == "" {
|
||||
parentQN = p.ProjectName
|
||||
} else {
|
||||
parentQN = fqn.FolderQN(p.ProjectName, parent)
|
||||
}
|
||||
|
||||
parentNode, _ := p.Store.FindNodeByQN(p.ProjectName, parentQN)
|
||||
if parentNode != nil && nodeID > 0 {
|
||||
edgeType := "CONTAINS_FOLDER"
|
||||
if isPackage {
|
||||
edgeType = "CONTAINS_PACKAGE"
|
||||
}
|
||||
_, _ = p.Store.InsertEdge(&store.Edge{
|
||||
Project: p.ProjectName,
|
||||
SourceID: parentNode.ID,
|
||||
TargetID: nodeID,
|
||||
Type: edgeType,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pipeline) dirQN(relDir string) string {
|
||||
if relDir == "." || relDir == "" {
|
||||
return p.ProjectName
|
||||
@@ -698,6 +746,10 @@ func parseFileAST(projectName string, f discover.FileInfo) *parseResult {
|
||||
|
||||
var constants []string
|
||||
|
||||
// C/C++ macro tracking: extract macro definitions
|
||||
isCPP := f.Language == lang.CPP
|
||||
macroNames := make(map[string]bool) // track macro names for call site resolution
|
||||
|
||||
parser.Walk(root, func(node *tree_sitter.Node) bool {
|
||||
kind := node.Kind()
|
||||
|
||||
@@ -711,6 +763,12 @@ func parseFileAST(projectName string, f discover.FileInfo) *parseResult {
|
||||
return false
|
||||
}
|
||||
|
||||
// Macro definitions (C/C++ only)
|
||||
if isCPP && (kind == "preproc_def" || kind == "preproc_function_def") {
|
||||
extractMacroDef(node, source, f, projectName, moduleQN, macroNames, result)
|
||||
return false
|
||||
}
|
||||
|
||||
if isConstantNode(node, f.Language) {
|
||||
c := extractConstant(node, source)
|
||||
if c != "" && len(c) > 1 {
|
||||
@@ -721,6 +779,18 @@ func parseFileAST(projectName string, f discover.FileInfo) *parseResult {
|
||||
return true
|
||||
})
|
||||
|
||||
// Store macro names in result for later call resolution (pass 3)
|
||||
if len(macroNames) > 0 {
|
||||
if moduleNode.Properties == nil {
|
||||
moduleNode.Properties = make(map[string]any)
|
||||
}
|
||||
macroList := make([]string, 0, len(macroNames))
|
||||
for name := range macroNames {
|
||||
macroList = append(macroList, name)
|
||||
}
|
||||
moduleNode.Properties["macros"] = macroList
|
||||
}
|
||||
|
||||
// Resolve interpolated/concatenated strings
|
||||
resolved := resolveModuleStrings(root, source, f.Language)
|
||||
|
||||
@@ -879,6 +949,9 @@ func extractClassDef(
|
||||
|
||||
// Extract methods inside the class
|
||||
extractClassMethodDefs(node, source, f, projectName, classQN, spec, result)
|
||||
|
||||
// Extract fields inside the class/struct
|
||||
extractClassFieldDefs(node, source, f, projectName, classQN, spec, result)
|
||||
}
|
||||
|
||||
// extractClassMethodDefs walks a class AST node and extracts Method nodes (no DB).
|
||||
@@ -942,10 +1015,186 @@ func extractClassMethodDefs(
|
||||
})
|
||||
}
|
||||
|
||||
// extractClassFieldDefs walks a class/struct AST node and extracts Field nodes (no DB).
|
||||
func extractClassFieldDefs(
|
||||
classNode *tree_sitter.Node, source []byte, f discover.FileInfo,
|
||||
projectName, classQN string, spec *lang.LanguageSpec, result *parseResult,
|
||||
) {
|
||||
if len(spec.FieldNodeTypes) == 0 {
|
||||
return
|
||||
}
|
||||
fieldTypes := toSet(spec.FieldNodeTypes)
|
||||
funcTypes := toSet(spec.FunctionNodeTypes)
|
||||
|
||||
parser.Walk(classNode, func(child *tree_sitter.Node) bool {
|
||||
if child.Id() == classNode.Id() {
|
||||
return true
|
||||
}
|
||||
// Skip nested class/method definitions — they have their own extraction
|
||||
if funcTypes[child.Kind()] {
|
||||
return false
|
||||
}
|
||||
if !fieldTypes[child.Kind()] {
|
||||
return true
|
||||
}
|
||||
|
||||
fieldName := extractFieldName(child, source, f.Language)
|
||||
if fieldName == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
fieldQN := classQN + "." + fieldName
|
||||
props := map[string]any{}
|
||||
|
||||
// Extract type annotation if present
|
||||
fieldType := extractFieldType(child, source, f.Language)
|
||||
if fieldType != "" {
|
||||
props["type"] = fieldType
|
||||
}
|
||||
|
||||
startLine := safeRowToLine(child.StartPosition().Row)
|
||||
endLine := safeRowToLine(child.EndPosition().Row)
|
||||
|
||||
result.Nodes = append(result.Nodes, &store.Node{
|
||||
Project: projectName,
|
||||
Label: "Field",
|
||||
Name: fieldName,
|
||||
QualifiedName: fieldQN,
|
||||
FilePath: f.RelPath,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Properties: props,
|
||||
})
|
||||
result.PendingEdges = append(result.PendingEdges, pendingEdge{
|
||||
SourceQN: classQN,
|
||||
TargetQN: fieldQN,
|
||||
Type: "DEFINES_FIELD",
|
||||
})
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
// extractFieldName extracts the name from a field declaration node.
|
||||
func extractFieldName(node *tree_sitter.Node, source []byte, l lang.Language) string {
|
||||
// Go: field_declaration has named children, first identifier is the name
|
||||
// C++/Java: field_declaration has a "declarator" field
|
||||
// Rust: field_declaration has a "name" field
|
||||
|
||||
// Try "name" field first (Rust, some others)
|
||||
if nameNode := node.ChildByFieldName("name"); nameNode != nil {
|
||||
return parser.NodeText(nameNode, source)
|
||||
}
|
||||
|
||||
// Try "declarator" field (C++, Java)
|
||||
if declNode := node.ChildByFieldName("declarator"); declNode != nil {
|
||||
// The declarator might be a pointer_declarator, array_declarator, etc.
|
||||
// Walk to find the identifier
|
||||
name := extractIdentifierFromDeclarator(declNode, source)
|
||||
if name != "" {
|
||||
return name
|
||||
}
|
||||
}
|
||||
|
||||
// Go struct fields: first child that is an identifier (field_identifier)
|
||||
if l == lang.Go {
|
||||
for i := uint(0); i < node.ChildCount(); i++ {
|
||||
child := node.Child(i)
|
||||
if child != nil && (child.Kind() == "field_identifier" || child.Kind() == "identifier") {
|
||||
return parser.NodeText(child, source)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractIdentifierFromDeclarator walks a declarator subtree to find the identifier name.
|
||||
func extractIdentifierFromDeclarator(node *tree_sitter.Node, source []byte) string {
|
||||
if node == nil {
|
||||
return ""
|
||||
}
|
||||
switch node.Kind() {
|
||||
case "identifier", "field_identifier":
|
||||
return parser.NodeText(node, source)
|
||||
case "pointer_declarator", "reference_declarator", "array_declarator":
|
||||
if declNode := node.ChildByFieldName("declarator"); declNode != nil {
|
||||
return extractIdentifierFromDeclarator(declNode, source)
|
||||
}
|
||||
// Fall through to child walk
|
||||
}
|
||||
// Walk children
|
||||
for i := uint(0); i < node.ChildCount(); i++ {
|
||||
child := node.Child(i)
|
||||
if child != nil && (child.Kind() == "identifier" || child.Kind() == "field_identifier") {
|
||||
return parser.NodeText(child, source)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractFieldType extracts the type annotation from a field declaration.
|
||||
func extractFieldType(node *tree_sitter.Node, source []byte, l lang.Language) string {
|
||||
// Try "type" field (Go, Rust, Java)
|
||||
if typeNode := node.ChildByFieldName("type"); typeNode != nil {
|
||||
return parser.NodeText(typeNode, source)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractMacroDef extracts a Macro node from a C/C++ preprocessor definition.
|
||||
func extractMacroDef(
|
||||
node *tree_sitter.Node, source []byte, f discover.FileInfo,
|
||||
projectName, moduleQN string, macroNames map[string]bool, result *parseResult,
|
||||
) {
|
||||
nameNode := node.ChildByFieldName("name")
|
||||
if nameNode == nil {
|
||||
return
|
||||
}
|
||||
name := parser.NodeText(nameNode, source)
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
|
||||
macroNames[name] = true
|
||||
|
||||
isFunctionLike := node.Kind() == "preproc_function_def"
|
||||
macroQN := moduleQN + "::macro::" + name
|
||||
|
||||
props := map[string]any{
|
||||
"is_function_like": isFunctionLike,
|
||||
}
|
||||
|
||||
if isFunctionLike {
|
||||
if paramsNode := node.ChildByFieldName("parameters"); paramsNode != nil {
|
||||
props["parameter_count"] = paramsNode.ChildCount()
|
||||
}
|
||||
}
|
||||
|
||||
startLine := safeRowToLine(node.StartPosition().Row)
|
||||
endLine := safeRowToLine(node.EndPosition().Row)
|
||||
|
||||
result.Nodes = append(result.Nodes, &store.Node{
|
||||
Project: projectName,
|
||||
Label: "Macro",
|
||||
Name: name,
|
||||
QualifiedName: macroQN,
|
||||
FilePath: f.RelPath,
|
||||
StartLine: startLine,
|
||||
EndLine: endLine,
|
||||
Properties: props,
|
||||
})
|
||||
|
||||
result.PendingEdges = append(result.PendingEdges, pendingEdge{
|
||||
SourceQN: moduleQN,
|
||||
TargetQN: macroQN,
|
||||
Type: "DEFINES",
|
||||
})
|
||||
}
|
||||
|
||||
// buildRegistry populates the FunctionRegistry from all Function, Method,
|
||||
// and Class nodes in the store.
|
||||
func (p *Pipeline) buildRegistry() {
|
||||
labels := []string{"Function", "Method", "Class", "Type", "Interface", "Enum"}
|
||||
labels := []string{"Function", "Method", "Class", "Type", "Interface", "Enum", "Macro"}
|
||||
for _, label := range labels {
|
||||
nodes, err := p.Store.FindNodesByLabel(p.ProjectName, label)
|
||||
if err != nil {
|
||||
|
||||
+109
-2
@@ -120,8 +120,9 @@ func (s *Store) FindEdgesByURLPath(project, pathSubstring string) ([]*Edge, erro
|
||||
return scanEdges(rows)
|
||||
}
|
||||
|
||||
// edgesBatchSize is the max rows per batch INSERT for edges (5 cols × 150 = 750 vars < 999).
|
||||
const edgesBatchSize = 150
|
||||
// Formula-derived batch size: SQLite has a 999 bind variable limit.
|
||||
const numEdgeCols = 5
|
||||
const edgesBatchSize = 999 / numEdgeCols // = 199
|
||||
|
||||
// InsertEdgeBatch inserts multiple edges in batched multi-row INSERTs.
|
||||
func (s *Store) InsertEdgeBatch(edges []*Edge) error {
|
||||
@@ -161,6 +162,112 @@ func (s *Store) insertEdgeChunk(batch []*Edge) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindEdgesBySourceIDs returns all edges where source_id is in the given set,
|
||||
// optionally filtered by edge types. Groups results by source_id for efficient lookup.
|
||||
func (s *Store) FindEdgesBySourceIDs(sourceIDs []int64, edgeTypes []string) (map[int64][]*Edge, error) {
|
||||
if len(sourceIDs) == 0 {
|
||||
return map[int64][]*Edge{}, nil
|
||||
}
|
||||
|
||||
result := make(map[int64][]*Edge, len(sourceIDs))
|
||||
const batchSize = 500 // leave room for type args
|
||||
|
||||
for i := 0; i < len(sourceIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(sourceIDs) {
|
||||
end = len(sourceIDs)
|
||||
}
|
||||
chunk := sourceIDs[i:end]
|
||||
|
||||
placeholders := make([]string, len(chunk))
|
||||
args := make([]any, 0, len(chunk)+len(edgeTypes))
|
||||
for j, id := range chunk {
|
||||
placeholders[j] = "?"
|
||||
args = append(args, id)
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT id, project, source_id, target_id, type, properties FROM edges WHERE source_id IN (%s)",
|
||||
strings.Join(placeholders, ","))
|
||||
|
||||
if len(edgeTypes) > 0 {
|
||||
typePH := make([]string, len(edgeTypes))
|
||||
for j, et := range edgeTypes {
|
||||
typePH[j] = "?"
|
||||
args = append(args, et)
|
||||
}
|
||||
query += " AND type IN (" + strings.Join(typePH, ",") + ")"
|
||||
}
|
||||
|
||||
rows, err := s.q.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find edges by source ids: %w", err)
|
||||
}
|
||||
edges, err := scanEdges(rows)
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range edges {
|
||||
result[e.SourceID] = append(result[e.SourceID], e)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// FindEdgesByTargetIDs returns all edges where target_id is in the given set,
|
||||
// optionally filtered by edge types. Groups results by target_id.
|
||||
func (s *Store) FindEdgesByTargetIDs(targetIDs []int64, edgeTypes []string) (map[int64][]*Edge, error) {
|
||||
if len(targetIDs) == 0 {
|
||||
return map[int64][]*Edge{}, nil
|
||||
}
|
||||
|
||||
result := make(map[int64][]*Edge, len(targetIDs))
|
||||
const batchSize = 500
|
||||
|
||||
for i := 0; i < len(targetIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(targetIDs) {
|
||||
end = len(targetIDs)
|
||||
}
|
||||
chunk := targetIDs[i:end]
|
||||
|
||||
placeholders := make([]string, len(chunk))
|
||||
args := make([]any, 0, len(chunk)+len(edgeTypes))
|
||||
for j, id := range chunk {
|
||||
placeholders[j] = "?"
|
||||
args = append(args, id)
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT id, project, source_id, target_id, type, properties FROM edges WHERE target_id IN (%s)",
|
||||
strings.Join(placeholders, ","))
|
||||
|
||||
if len(edgeTypes) > 0 {
|
||||
typePH := make([]string, len(edgeTypes))
|
||||
for j, et := range edgeTypes {
|
||||
typePH[j] = "?"
|
||||
args = append(args, et)
|
||||
}
|
||||
query += " AND type IN (" + strings.Join(typePH, ",") + ")"
|
||||
}
|
||||
|
||||
rows, err := s.q.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find edges by target ids: %w", err)
|
||||
}
|
||||
edges, err := scanEdges(rows)
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range edges {
|
||||
result[e.TargetID] = append(result[e.TargetID], e)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanEdges(rows *sql.Rows) ([]*Edge, error) {
|
||||
var result []*Edge
|
||||
for rows.Next() {
|
||||
|
||||
+45
-2
@@ -104,6 +104,48 @@ func (s *Store) DeleteNodesByLabel(project, label string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// FindNodesByIDs returns a map of nodeID → *Node for the given IDs.
|
||||
func (s *Store) FindNodesByIDs(ids []int64) (map[int64]*Node, error) {
|
||||
if len(ids) == 0 {
|
||||
return map[int64]*Node{}, nil
|
||||
}
|
||||
result := make(map[int64]*Node, len(ids))
|
||||
const batchSize = 998 // leave room under 999 limit
|
||||
|
||||
for i := 0; i < len(ids); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(ids) {
|
||||
end = len(ids)
|
||||
}
|
||||
chunk := ids[i:end]
|
||||
|
||||
placeholders := make([]string, len(chunk))
|
||||
args := make([]any, len(chunk))
|
||||
for j, id := range chunk {
|
||||
placeholders[j] = "?"
|
||||
args[j] = id
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties FROM nodes WHERE id IN (%s)",
|
||||
strings.Join(placeholders, ","))
|
||||
|
||||
rows, err := s.q.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find nodes by ids: %w", err)
|
||||
}
|
||||
nodes, err := scanNodes(rows)
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, n := range nodes {
|
||||
result[n.ID] = n
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// AllNodes returns all nodes for a project.
|
||||
func (s *Store) AllNodes(project string) ([]*Node, error) {
|
||||
rows, err := s.q.Query(`SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, properties
|
||||
@@ -147,8 +189,9 @@ func scanNodes(rows *sql.Rows) ([]*Node, error) {
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
// nodesBatchSize is the max rows per batch INSERT for nodes (8 cols × 100 = 800 vars < 999).
|
||||
const nodesBatchSize = 100
|
||||
// Formula-derived batch size: SQLite has a 999 bind variable limit.
|
||||
const numNodeCols = 8
|
||||
const nodesBatchSize = 999 / numNodeCols // = 124
|
||||
|
||||
// UpsertNodeBatch inserts or updates multiple nodes in batched multi-row INSERTs.
|
||||
// Returns a map of qualifiedName → ID for all upserted nodes.
|
||||
|
||||
+36
-17
@@ -9,7 +9,7 @@ import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// Querier abstracts *sql.DB and *sql.Tx so store methods work in both contexts.
|
||||
@@ -74,20 +74,24 @@ func Open(project string) (*Store, error) {
|
||||
|
||||
// OpenPath opens a SQLite database at the given path.
|
||||
func OpenPath(dbPath string) (*Store, error) {
|
||||
dsn := dbPath + "?_pragma=journal_mode(WAL)" +
|
||||
"&_pragma=busy_timeout(5000)" +
|
||||
"&_pragma=foreign_keys(ON)" +
|
||||
"&_pragma=synchronous(NORMAL)" +
|
||||
"&_pragma=cache_size(-8192)" +
|
||||
"&_pragma=temp_store(MEMORY)" +
|
||||
"&_pragma=mmap_size(268435456)" // 256 MB mmap for fast reads
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
dsn := dbPath + "?_journal_mode=WAL" +
|
||||
"&_busy_timeout=5000" +
|
||||
"&_foreign_keys=1" +
|
||||
"&_synchronous=NORMAL" +
|
||||
"&_cache_size=-65536" + // 64 MB
|
||||
"&_txlock=immediate"
|
||||
db, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open db: %w", err)
|
||||
}
|
||||
// Single connection: SQLite is single-writer, pool adds lock contention.
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
|
||||
// PRAGMAs not supported in mattn DSN — set via Exec after Open.
|
||||
_, _ = db.Exec("PRAGMA temp_store = MEMORY")
|
||||
_, _ = db.Exec("PRAGMA mmap_size = 1073741824") // 1 GB
|
||||
|
||||
s := &Store{db: db, dbPath: dbPath}
|
||||
s.q = s.db
|
||||
if err := s.initSchema(); err != nil {
|
||||
@@ -99,13 +103,13 @@ func OpenPath(dbPath string) (*Store, error) {
|
||||
|
||||
// OpenMemory opens an in-memory SQLite database (for testing).
|
||||
func OpenMemory() (*Store, error) {
|
||||
dsn := ":memory:?_pragma=foreign_keys(ON)" +
|
||||
"&_pragma=synchronous(OFF)" +
|
||||
"&_pragma=temp_store(MEMORY)"
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
dsn := ":memory:?_foreign_keys=1" +
|
||||
"&_synchronous=OFF"
|
||||
db, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open memory db: %w", err)
|
||||
}
|
||||
_, _ = db.Exec("PRAGMA temp_store = MEMORY")
|
||||
s := &Store{db: db, dbPath: ":memory:"}
|
||||
s.q = s.db
|
||||
if err := s.initSchema(); err != nil {
|
||||
@@ -133,11 +137,26 @@ func (s *Store) WithTransaction(fn func(txStore *Store) error) error {
|
||||
}
|
||||
|
||||
// Checkpoint forces a WAL checkpoint, moving pages from WAL to the main DB,
|
||||
// then runs ANALYZE so the query planner has up-to-date statistics.
|
||||
// Cost is absorbed during indexing rather than charged to the first read query.
|
||||
// then runs PRAGMA optimize so the query planner has up-to-date statistics.
|
||||
// PRAGMA optimize (SQLite 3.46+) auto-limits sampling per index, only re-analyzing
|
||||
// stale stats. Cost is absorbed during indexing rather than the first read query.
|
||||
func (s *Store) Checkpoint() {
|
||||
_, _ = s.db.Exec("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
_, _ = s.db.Exec("ANALYZE")
|
||||
_, _ = s.db.Exec("PRAGMA optimize")
|
||||
}
|
||||
|
||||
// BeginBulkWrite switches to MEMORY journal mode for faster bulk writes.
|
||||
// Call EndBulkWrite when done to restore WAL mode.
|
||||
// MEMORY mode is rollback-safe on crash (unlike journal_mode=OFF).
|
||||
func (s *Store) BeginBulkWrite() {
|
||||
_, _ = s.db.Exec("PRAGMA journal_mode = MEMORY")
|
||||
_, _ = s.db.Exec("PRAGMA synchronous = OFF")
|
||||
}
|
||||
|
||||
// EndBulkWrite restores WAL journal mode and NORMAL synchronous after bulk writes.
|
||||
func (s *Store) EndBulkWrite() {
|
||||
_, _ = s.db.Exec("PRAGMA synchronous = NORMAL")
|
||||
_, _ = s.db.Exec("PRAGMA journal_mode = WAL")
|
||||
}
|
||||
|
||||
// Close closes the database connection.
|
||||
@@ -210,7 +229,7 @@ func (s *Store) initSchema() error {
|
||||
}
|
||||
|
||||
// Migration: add url_path generated column to edges table.
|
||||
// Generated columns require SQLite 3.31.0+ (modernc.org/sqlite supports this).
|
||||
// Generated columns require SQLite 3.31.0+ (mattn/go-sqlite3 supports this).
|
||||
// We check if the column already exists to make this idempotent.
|
||||
var colCount int
|
||||
_ = s.db.QueryRow(`SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount)
|
||||
|
||||
@@ -777,6 +777,18 @@ func TestBatchCountDegrees(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchSizeSafety(t *testing.T) {
|
||||
// Verify formula-derived batch sizes stay under SQLite's 999 bind variable limit.
|
||||
if numNodeCols*nodesBatchSize >= 999 {
|
||||
t.Errorf("node batch exceeds limit: %d cols × %d rows = %d (max 998)",
|
||||
numNodeCols, nodesBatchSize, numNodeCols*nodesBatchSize)
|
||||
}
|
||||
if numEdgeCols*edgesBatchSize >= 999 {
|
||||
t.Errorf("edge batch exceeds limit: %d cols × %d rows = %d (max 998)",
|
||||
numEdgeCols, edgesBatchSize, numEdgeCols*edgesBatchSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchExcludeLabels(t *testing.T) {
|
||||
s, err := OpenMemory()
|
||||
if err != nil {
|
||||
|
||||
+107
-87
@@ -1,5 +1,10 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TraverseResult holds BFS traversal results.
|
||||
type TraverseResult struct {
|
||||
Root *Node
|
||||
@@ -20,31 +25,9 @@ type EdgeInfo struct {
|
||||
Type string
|
||||
}
|
||||
|
||||
type bfsQueue struct {
|
||||
nodeID int64
|
||||
hop int
|
||||
}
|
||||
|
||||
// fetchEdgesForNode retrieves edges from a node in the given direction and edge types.
|
||||
func (s *Store) fetchEdgesForNode(nodeID int64, direction string, edgeTypes []string) ([]*Edge, error) {
|
||||
var edges []*Edge
|
||||
for _, et := range edgeTypes {
|
||||
var found []*Edge
|
||||
var err error
|
||||
if direction == "outbound" {
|
||||
found, err = s.FindEdgesBySourceAndType(nodeID, et)
|
||||
} else {
|
||||
found, err = s.FindEdgesByTargetAndType(nodeID, et)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
edges = append(edges, found...)
|
||||
}
|
||||
return edges, nil
|
||||
}
|
||||
|
||||
// BFS performs breadth-first traversal following edges of given types.
|
||||
// BFS performs breadth-first traversal following edges of given types using a
|
||||
// recursive CTE, replacing the previous per-node Go-side loop with a single
|
||||
// SQL round-trip.
|
||||
// direction: "outbound" follows source->target, "inbound" follows target->source.
|
||||
// maxDepth caps the BFS depth, maxResults caps total visited nodes.
|
||||
func (s *Store) BFS(startNodeID int64, direction string, edgeTypes []string, maxDepth, maxResults int) (*TraverseResult, error) {
|
||||
@@ -54,77 +37,114 @@ func (s *Store) BFS(startNodeID int64, direction string, edgeTypes []string, max
|
||||
if maxResults <= 0 {
|
||||
maxResults = 200
|
||||
}
|
||||
if len(edgeTypes) == 0 {
|
||||
edgeTypes = []string{"CALLS"}
|
||||
}
|
||||
|
||||
// Build type filter placeholders
|
||||
typePlaceholders := make([]string, len(edgeTypes))
|
||||
typeArgs := make([]any, len(edgeTypes))
|
||||
for i, et := range edgeTypes {
|
||||
typePlaceholders[i] = "?"
|
||||
typeArgs[i] = et
|
||||
}
|
||||
typeFilter := strings.Join(typePlaceholders, ",")
|
||||
|
||||
// Determine join columns based on direction
|
||||
var joinCol, nextCol string
|
||||
if direction == "inbound" {
|
||||
joinCol, nextCol = "target_id", "source_id"
|
||||
} else {
|
||||
joinCol, nextCol = "source_id", "target_id"
|
||||
}
|
||||
|
||||
// Recursive CTE: traverse edges up to maxDepth hops, collect node IDs + edges
|
||||
query := fmt.Sprintf(`
|
||||
WITH RECURSIVE bfs(node_id, hop) AS (
|
||||
SELECT ?, 0
|
||||
UNION ALL
|
||||
SELECT e.%s, b.hop + 1
|
||||
FROM bfs b
|
||||
JOIN edges e ON e.%s = b.node_id AND e.type IN (%s)
|
||||
WHERE b.hop < ?
|
||||
)
|
||||
SELECT DISTINCT n.id, n.project, n.label, n.name, n.qualified_name,
|
||||
n.file_path, n.start_line, n.end_line, n.properties, bfs.hop
|
||||
FROM bfs
|
||||
JOIN nodes n ON n.id = bfs.node_id
|
||||
WHERE bfs.hop > 0
|
||||
ORDER BY bfs.hop, n.name
|
||||
LIMIT ?`,
|
||||
nextCol, joinCol, typeFilter)
|
||||
|
||||
// Build args: startNodeID, ...typeArgs, maxDepth, maxResults
|
||||
args := make([]any, 0, 3+len(typeArgs))
|
||||
args = append(args, startNodeID)
|
||||
args = append(args, typeArgs...)
|
||||
args = append(args, maxDepth, maxResults)
|
||||
|
||||
rows, err := s.q.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bfs cte: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := &TraverseResult{}
|
||||
visited := make(map[int64]int) // nodeID -> hop
|
||||
nodeCache := make(map[int64]*Node) // nodeID -> resolved node
|
||||
visited[startNodeID] = 0
|
||||
|
||||
// Cache the start node for edge name resolution
|
||||
startNode, err := s.FindNodeByID(startNodeID)
|
||||
if err == nil && startNode != nil {
|
||||
nodeCache[startNodeID] = startNode
|
||||
}
|
||||
|
||||
queue := []bfsQueue{{startNodeID, 0}}
|
||||
|
||||
for len(queue) > 0 && len(result.Visited) < maxResults {
|
||||
item := queue[0]
|
||||
queue = queue[1:]
|
||||
|
||||
if item.hop >= maxDepth {
|
||||
continue
|
||||
}
|
||||
|
||||
edges, err := s.fetchEdgesForNode(item.nodeID, direction, edgeTypes)
|
||||
if err != nil {
|
||||
for rows.Next() {
|
||||
var n Node
|
||||
var props string
|
||||
var hop int
|
||||
if err := rows.Scan(&n.ID, &n.Project, &n.Label, &n.Name, &n.QualifiedName,
|
||||
&n.FilePath, &n.StartLine, &n.EndLine, &props, &hop); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n.Properties = unmarshalProps(props)
|
||||
result.Visited = append(result.Visited, &NodeHop{Node: &n, Hop: hop})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, e := range edges {
|
||||
var nextID int64
|
||||
if direction == "outbound" {
|
||||
nextID = e.TargetID
|
||||
} else {
|
||||
nextID = e.SourceID
|
||||
}
|
||||
// Collect edge info with a second CTE query that returns actual edges
|
||||
edgeQuery := fmt.Sprintf(`
|
||||
WITH RECURSIVE bfs(node_id, hop) AS (
|
||||
SELECT ?, 0
|
||||
UNION ALL
|
||||
SELECT e.%s, b.hop + 1
|
||||
FROM bfs b
|
||||
JOIN edges e ON e.%s = b.node_id AND e.type IN (%s)
|
||||
WHERE b.hop < ?
|
||||
)
|
||||
SELECT DISTINCT src.name, tgt.name, e.type
|
||||
FROM bfs b
|
||||
JOIN edges e ON e.%s = b.node_id AND e.type IN (%s)
|
||||
JOIN nodes src ON src.id = e.source_id
|
||||
JOIN nodes tgt ON tgt.id = e.target_id
|
||||
WHERE b.hop < ?`,
|
||||
nextCol, joinCol, typeFilter,
|
||||
joinCol, typeFilter)
|
||||
|
||||
if _, seen := visited[nextID]; !seen {
|
||||
visited[nextID] = item.hop + 1
|
||||
// Build edge args: startNodeID, ...typeArgs, maxDepth, ...typeArgs, maxDepth
|
||||
edgeArgs := make([]any, 0, 4+2*len(typeArgs))
|
||||
edgeArgs = append(edgeArgs, startNodeID)
|
||||
edgeArgs = append(edgeArgs, typeArgs...)
|
||||
edgeArgs = append(edgeArgs, maxDepth)
|
||||
edgeArgs = append(edgeArgs, typeArgs...)
|
||||
edgeArgs = append(edgeArgs, maxDepth)
|
||||
|
||||
nextNode, lookupErr := s.FindNodeByID(nextID)
|
||||
if lookupErr != nil || nextNode == nil {
|
||||
continue
|
||||
}
|
||||
nodeCache[nextID] = nextNode
|
||||
edgeRows, err := s.q.Query(edgeQuery, edgeArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bfs edges: %w", err)
|
||||
}
|
||||
defer edgeRows.Close()
|
||||
|
||||
result.Visited = append(result.Visited, &NodeHop{Node: nextNode, Hop: item.hop + 1})
|
||||
queue = append(queue, bfsQueue{nextID, item.hop + 1})
|
||||
|
||||
if len(result.Visited) >= maxResults {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Record edge info using cached nodes (avoid extra queries)
|
||||
fromName := resolveNodeName(nodeCache, s, e.SourceID)
|
||||
toName := resolveNodeName(nodeCache, s, e.TargetID)
|
||||
result.Edges = append(result.Edges, EdgeInfo{FromName: fromName, ToName: toName, Type: e.Type})
|
||||
for edgeRows.Next() {
|
||||
var ei EdgeInfo
|
||||
if err := edgeRows.Scan(&ei.FromName, &ei.ToName, &ei.Type); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Edges = append(result.Edges, ei)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// resolveNodeName returns the name for a node ID, using the cache first.
|
||||
func resolveNodeName(cache map[int64]*Node, s *Store, id int64) string {
|
||||
if n, ok := cache[id]; ok {
|
||||
return n.Name
|
||||
}
|
||||
n, err := s.FindNodeByID(id)
|
||||
if err != nil || n == nil {
|
||||
return ""
|
||||
}
|
||||
cache[id] = n
|
||||
return n.Name
|
||||
return result, edgeRows.Err()
|
||||
}
|
||||
|
||||
@@ -58,6 +58,11 @@ func (s *Server) handleDeleteProject(_ context.Context, req *mcp.CallToolRequest
|
||||
return errResult(fmt.Sprintf("delete failed: %v", err)), nil
|
||||
}
|
||||
|
||||
// file_hashes table is not cascaded from projects — delete separately
|
||||
if err := s.store.DeleteFileHashes(name); err != nil {
|
||||
return errResult(fmt.Sprintf("delete file hashes failed: %v", err)), nil
|
||||
}
|
||||
|
||||
return jsonResult(map[string]any{
|
||||
"deleted": name,
|
||||
"status": "ok",
|
||||
|
||||
Reference in New Issue
Block a user