diff --git a/CHANGELOG.md b/CHANGELOG.md index 95b72160f..997e2deed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph status` now shows how many of a language's files yielded symbols when that isn't all of them, and warns when none did, so a parser that isn't working no longer looks healthy, and `codegraph status --strict` exits non-zero on that or on an index that's missing, unfinished or out of date, so packaging and CI can gate on it (#1658). Thanks @ferrine. - `codegraph serve --mcp --no-telemetry` turns telemetry off for one MCP server entry, so a shared MCP config can opt out without an environment variable; a background daemon that's already running for the project keeps its own setting until it exits. Thanks @danusha2345. (#1908) - `codegraph implementations ` lists the types that implement or extend a trait, interface or class, with each one's file and line, how it's tied to the type, and whether it's test code, a Rust mock in a `#[cfg(test)]` module included; `--depth` follows subclasses of subclasses. Thanks @danusha2345. (#2833) +- Set `CODEGRAPH_SYNTH_MIN_INTERVAL_MS` to limit how often live syncs rebuild the callback and event links across the whole project, which on a large TypeScript or Vue repo otherwise happens on almost every edit; a rebuild that's put off runs when the interval is up even if no edit follows, and until then those links can be that far behind your code. A full `codegraph index` is never held. Thanks @danusha2345. (#2911) + - New `codegraph cycles` command lists the groups of files that import each other in a circle, as text or `--json`, and exits 1 when it finds any, so it works as a pre-commit or CI check (#888). Thanks @jiangyq9 and @goutham80808. diff --git a/README.md b/README.md index b2eb521d7..0c10d51cb 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ CodeGraph's parsing engine is a **native Rust kernel**: 20 languages — TypeScr When your agent (Claude Code, Cursor, Codex, opencode) launches `codegraph serve --mcp`, three layers keep the index in step with your code — and make sure the agent never gets a silent wrong answer in the brief window between an edit and the next sync: -1. **File watcher with debounced auto-sync.** A native FSEvents / inotify / ReadDirectoryChangesW watcher captures every source-file create / modify / delete and triggers a re-index after a debounce window (default `2000ms`, tunable via `CODEGRAPH_WATCH_DEBOUNCE_MS`, clamped to `[100ms, 60s]`). Bursts of edits collapse into a single sync. +1. **File watcher with debounced auto-sync.** A native FSEvents / inotify / ReadDirectoryChangesW watcher captures every source-file create / modify / delete and triggers a re-index after a debounce window (default `2000ms`, tunable via `CODEGRAPH_WATCH_DEBOUNCE_MS`, clamped to `[100ms, 60s]`). Bursts of edits collapse into a single sync. On a large TypeScript or Vue repo nearly every edit also asks for a repo-wide rebuild of the synthesized callback and event edges; `CODEGRAPH_SYNTH_MIN_INTERVAL_MS` (1s to 24h) bounds how often that runs, and a rebuild a sync puts off runs when the interval is up. Until then those edges can be that far behind the code, and tool output doesn't say so. 2. **Per-file staleness banner.** During the brief debounce window, MCP tool responses that would reference a still-pending file prepend a `⚠️` banner naming it and telling the agent to `Read` it directly. Pending files NOT referenced by the response surface as a small footer instead. Either way, the agent gets an explicit signal — validated with Claude Code, where the agent literally says "Reading the file directly for the live content" before opening it. diff --git a/__tests__/sync-synthesis-interval.test.ts b/__tests__/sync-synthesis-interval.test.ts new file mode 100644 index 000000000..5f57bd924 --- /dev/null +++ b/__tests__/sync-synthesis-interval.test.ts @@ -0,0 +1,226 @@ +/** + * CODEGRAPH_SYNTH_MIN_INTERVAL_MS bounds how often a long-lived instance's + * syncs rebuild the synthesized edges (#2911). The synthesis passes read the + * whole graph, and nearly every TypeScript or Vue file is one of their inputs, + * so the daemon's watcher paid for a repo-wide pass on almost every edit. With + * the variable set, a sync within the interval of the last synthesis puts the + * rebuild off: `synthesis_pending` stays set, and a timer runs it when the + * interval is up even if no edit follows. + */ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph, parseSynthMinIntervalEnv } from '../src'; +import { createDatabase } from '../src/db/sqlite-adapter'; + +// A sync on a saturated machine can take seconds, and it has to fall inside this. +const HOLD_MS = 10_000; +const SHORTEST_MS = 1000; +const ENV = 'CODEGRAPH_SYNTH_MIN_INTERVAL_MS'; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** The parts of an instance these tests reach past the public API. */ +interface Internals { + resolver: { refreshSynthesis(...args: unknown[]): Promise }; + db: { getWalAutocheckpoint(): number }; + deferredSynthesisTimer: NodeJS.Timeout | null; + runDeferredSynthesis(): Promise; +} +const internals = (graph: CodeGraph) => graph as unknown as Internals; + +function write(dir: string, rel: string, content: string): void { + const full = path.join(dir, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); +} + +/** A bus, a handler registered on it, and a service that emits: one event-emitter edge. */ +function fixture(dir: string): void { + write(dir, 'package.json', JSON.stringify({ name: 'app' })); + write(dir, 'src/bus.js', "const { EventEmitter } = require('events');\nconst bus = new EventEmitter();\nmodule.exports = { bus };\n"); + write(dir, 'src/handlers.js', "const { bus } = require('./bus');\nfunction onSaved(doc) { return doc; }\nbus.on('saved', onSaved);\nmodule.exports = { onSaved };\n"); + write(dir, 'src/service.js', "const { bus } = require('./bus');\nfunction save(doc) { bus.emit('saved', doc); return doc; }\nmodule.exports = { save };\n"); +} + +/** A second event on the bus: the edit that makes a sync rebuild the synthesized edges. */ +function addRemovedEvent(dir: string): void { + write(dir, 'src/handlers.js', "const { bus } = require('./bus');\nfunction onSaved(doc) { return doc; }\nfunction onRemoved(doc) { return doc; }\nbus.on('saved', onSaved);\nbus.on('removed', onRemoved);\nmodule.exports = { onSaved, onRemoved };\n"); + write(dir, 'src/service.js', "const { bus } = require('./bus');\nfunction save(doc) { bus.emit('saved', doc); return doc; }\nfunction remove(doc) { bus.emit('removed', doc); return doc; }\nmodule.exports = { save, remove };\n"); +} + +/** `synthesis_pending` as stored, read through a connection of its own. */ +function pending(dir: string): string | null { + const { db } = createDatabase(path.join(dir, '.codegraph', 'codegraph.db'), { readOnly: true }); + try { + const row = db.prepare("SELECT value FROM project_metadata WHERE key = 'synthesis_pending'").get() as { value: string } | undefined; + return row?.value ?? null; + } finally { + db.close(); + } +} + +/** Every synthesized edge between functions, as `caller->callee`. */ +function synthesized(cg: CodeGraph): string[] { + const out: string[] = []; + for (const n of cg.getNodesByKind('function')) { + for (const e of cg.getOutgoingEdges(n.id)) { + if (e.metadata?.synthesizedBy) out.push(`${n.name}->${cg.getNode(e.target)!.name}`); + } + } + return out.sort(); +} + +async function waitUntil(check: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (!check() && Date.now() < deadline) await sleep(50); +} + +describe('CODEGRAPH_SYNTH_MIN_INTERVAL_MS holds a sync\'s synthesis (#2911)', () => { + let dir = ''; + let cg: CodeGraph | null = null; + + afterEach(async () => { + delete process.env[ENV]; + vi.restoreAllMocks(); + // A put-off synthesis clears the flag and then runs its maintenance off-thread: + // close only when it has finished, or the database file is still busy on Windows. + const open = cg; + if (open) await waitUntil(() => !open.isIndexing(), 20_000); + cg?.close(); + cg = null; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + dir = ''; + }); + + async function indexed(): Promise { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-synth-interval-')); + fixture(dir); + cg = await CodeGraph.init(dir, { index: true }); + expect(synthesized(cg)).toEqual(['save->onSaved']); + expect(pending(dir)).not.toBe('1'); + return cg; + } + + it('a sync within the interval keeps the synthesis pending, and the timer runs it when the interval is up', async () => { + const graph = await indexed(); + // The index just synthesized: this sync falls inside the interval. + process.env[ENV] = String(HOLD_MS); + addRemovedEvent(dir); + const result = await graph.sync(); + expect(result.filesModified).toBe(2); + expect(pending(dir)).toBe('1'); + expect(synthesized(graph)).not.toContain('remove->onRemoved'); + + await waitUntil(() => pending(dir) === '0', HOLD_MS + 20_000); + expect(pending(dir)).toBe('0'); + expect(synthesized(graph)).toContain('remove->onRemoved'); + }, 60_000); + + it('a sync after the interval synthesizes at once', async () => { + const graph = await indexed(); + process.env[ENV] = String(SHORTEST_MS); + await sleep(SHORTEST_MS + 100); + addRemovedEvent(dir); + await graph.sync(); + expect(pending(dir)).toBe('0'); + expect(synthesized(graph)).toContain('remove->onRemoved'); + }, 60_000); + + it('without the variable a sync synthesizes as before', async () => { + const graph = await indexed(); + addRemovedEvent(dir); + await graph.sync(); + expect(pending(dir)).toBe('0'); + expect(synthesized(graph)).toContain('remove->onRemoved'); + }, 60_000); + + it('close() cancels the timer; the pending flag stays for the next sync', async () => { + const graph = await indexed(); + process.env[ENV] = String(HOLD_MS); + addRemovedEvent(dir); + await graph.sync(); + expect(pending(dir)).toBe('1'); + expect(internals(graph).deferredSynthesisTimer).not.toBeNull(); + graph.close(); + cg = null; + expect(internals(graph).deferredSynthesisTimer).toBeNull(); + expect(pending(dir)).toBe('1'); + + // The next instance's first sync runs it: it has no synthesis of its own to measure from. + const again = await CodeGraph.open(dir); + cg = again; + await again.sync(); + expect(pending(dir)).toBe('0'); + expect(synthesized(again)).toContain('remove->onRemoved'); + }, 60_000); + + // The run comes from a timer, so a rejection reaches nothing but the process's + // fatal handler: a `database is locked` here used to end the daemon. + it('a put-off synthesis that fails stays pending and tries again, without rejecting', async () => { + const graph = await indexed(); + process.env[ENV] = String(HOLD_MS); + addRemovedEvent(dir); + await graph.sync(); + const inner = internals(graph); + clearTimeout(inner.deferredSynthesisTimer!); + inner.deferredSynthesisTimer = null; + + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(inner.resolver, 'refreshSynthesis').mockRejectedValueOnce(new Error('database is locked')); + await expect(inner.runDeferredSynthesis()).resolves.toBeUndefined(); + expect(pending(dir)).toBe('1'); + expect(inner.deferredSynthesisTimer).not.toBeNull(); + + await inner.runDeferredSynthesis(); + expect(pending(dir)).toBe('0'); + expect(synthesized(graph)).toContain('remove->onRemoved'); + }, 60_000); + + it('a put-off synthesis that fails after close() arms nothing', async () => { + const graph = await indexed(); + process.env[ENV] = String(HOLD_MS); + addRemovedEvent(dir); + await graph.sync(); + graph.close(); + cg = null; + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await expect(internals(graph).runDeferredSynthesis()).resolves.toBeUndefined(); + expect(internals(graph).deferredSynthesisTimer).toBeNull(); + }, 60_000); + + it('a put-off synthesis defers WAL checkpoints as a sync does', async () => { + const graph = await indexed(); + process.env[ENV] = String(HOLD_MS); + addRemovedEvent(dir); + await graph.sync(); + const inner = internals(graph); + const before = inner.db.getWalAutocheckpoint(); + const real = inner.resolver.refreshSynthesis.bind(inner.resolver); + let during: number | undefined; + let backpressure: unknown; + vi.spyOn(inner.resolver, 'refreshSynthesis').mockImplementation(async (...args: unknown[]) => { + during = inner.db.getWalAutocheckpoint(); + backpressure = args[2]; + return real(...args); + }); + + await inner.runDeferredSynthesis(); + expect(pending(dir)).toBe('0'); + expect(during).toBe(0); + expect(typeof backpressure).toBe('function'); + expect(inner.db.getWalAutocheckpoint()).toBe(before); + }, 60_000); + + it('parseSynthMinIntervalEnv reads whole milliseconds from one second to a day', () => { + expect(parseSynthMinIntervalEnv(undefined)).toBeUndefined(); + expect(parseSynthMinIntervalEnv('')).toBeUndefined(); + expect(parseSynthMinIntervalEnv('abc')).toBeUndefined(); + expect(parseSynthMinIntervalEnv('500')).toBeUndefined(); + expect(parseSynthMinIntervalEnv('1500.5')).toBeUndefined(); + expect(parseSynthMinIntervalEnv('86400001')).toBeUndefined(); + expect(parseSynthMinIntervalEnv('1000')).toBe(1000); + expect(parseSynthMinIntervalEnv('300000')).toBe(300_000); + }); +}); diff --git a/src/index.ts b/src/index.ts index 8cff94d95..76a5b3c51 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,6 +31,7 @@ import { } from './types'; import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db'; import { WalCheckpointValve, resolveWalValveMb } from './db/wal-valve'; +import { logWarn } from './errors'; import { QueryBuilder, type DeprioritizedPathMatcher } from './db/queries'; import { importPathKeys, moduleReferenceKeys } from './db/reference-tail'; import { @@ -185,6 +186,18 @@ function parseNameList(value: string | null): string[] | null { } } +/** + * CODEGRAPH_SYNTH_MIN_INTERVAL_MS: the least time between two repo-wide + * synthesis passes a long-lived instance's syncs run (#2911). Whole + * milliseconds, 1000 to 24 hours; unset, empty or out of range means no hold. + */ +export function parseSynthMinIntervalEnv(raw: string | undefined): number | undefined { + if (!raw || !raw.trim()) return undefined; + const n = Number(raw); + if (!Number.isInteger(n) || n < 1000 || n > 86_400_000) return undefined; + return n; +} + export class CodeGraph { private db: DatabaseConnection; private queries: QueryBuilder; @@ -207,6 +220,12 @@ export class CodeGraph { // File watcher for auto-sync on file changes private watcher: FileWatcher | null = null; + // When this instance last rebuilt the synthesized edges, and the timer that + // runs a synthesis a sync put off under CODEGRAPH_SYNTH_MIN_INTERVAL_MS + // (#2911). Per instance: the daemon's watcher is what syncs on every edit. + private lastSynthesisAt = 0; + private deferredSynthesisTimer: NodeJS.Timeout | null = null; + private constructor( db: DatabaseConnection, queries: QueryBuilder, @@ -586,6 +605,10 @@ export class CodeGraph { */ close(): void { this.unwatch(); + if (this.deferredSynthesisTimer) { + clearTimeout(this.deferredSynthesisTimer); + this.deferredSynthesisTimer = null; + } // Release file lock if held this.fileLock.release(); this.db.close(); @@ -788,6 +811,7 @@ export class CodeGraph { walValve ? () => walValve!.backpressure() : undefined ); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] resolution: ${Date.now() - tResolve}ms`); + this.lastSynthesisAt = Date.now(); // Second pass: chained calls whose method lives on a supertype the // receiver conforms to (protocol-extension / inherited / default- @@ -1729,16 +1753,32 @@ export class CodeGraph { await this.resolver.resolveDeferredThisMemberRefs(); } + // The synthesis passes read the whole graph, whatever this sync + // changed, and nearly every TypeScript or Vue file is one of their + // inputs, so on a large repo almost every edit paid for a whole-repo + // synthesis (#2911). With CODEGRAPH_SYNTH_MIN_INTERVAL_MS set, a + // synthesis within that long of the last one is put off, not + // dropped: `synthesis_pending` stays set, so a later sync runs it, + // and a timer runs it when the interval is up even if no edit + // follows. A full index is never held. + let synthesized = false; if (refreshSynthesis) { - await this.resolver.refreshSynthesis(this.db.getPath(), (done, total) => { - options.onProgress?.({ phase: 'linking', current: done, total }); - }, backpressure); - this.queries.setMetadata('synthesis_pending', '0'); + const hold = this.synthesisHoldMs(); + if (hold > 0) { + this.armDeferredSynthesis(hold); + } else { + await this.resolver.refreshSynthesis(this.db.getPath(), (done, total) => { + options.onProgress?.({ phase: 'linking', current: done, total }); + }, backpressure); + this.queries.setMetadata('synthesis_pending', '0'); + this.lastSynthesisAt = Date.now(); + synthesized = true; + } } // Refresh planner stats + checkpoint the WAL after bulk writes. // Off-thread — see indexAll's call site. - if (filesChanged || result.filesRemoved > 0 || orphanCount > 0 || refreshSynthesis) { + if (filesChanged || result.filesRemoved > 0 || orphanCount > 0 || synthesized) { await this.db.runMaintenance(); } @@ -1791,6 +1831,103 @@ export class CodeGraph { return this.indexMutex.isLocked(); } + /** + * How much longer a sync must wait before it may rebuild the synthesized + * edges: the rest of CODEGRAPH_SYNTH_MIN_INTERVAL_MS since this instance last + * did (#2911). Zero when the variable is unset or invalid, or the interval + * has passed. + */ + private synthesisHoldMs(): number { + const interval = parseSynthMinIntervalEnv(process.env.CODEGRAPH_SYNTH_MIN_INTERVAL_MS); + if (!interval || this.lastSynthesisAt === 0) return 0; + return Math.max(0, this.lastSynthesisAt + interval - Date.now()); + } + + /** Run the synthesis a sync put off once the interval is up, unless a sync runs it first. */ + private armDeferredSynthesis(delayMs: number): void { + if (this.deferredSynthesisTimer) return; + this.deferredSynthesisTimer = setTimeout(() => { + this.deferredSynthesisTimer = null; + void this.runDeferredSynthesis(); + }, delayMs); + this.deferredSynthesisTimer.unref?.(); + } + + /** + * The put-off synthesis, under the locks a sync holds. Nothing to do when a + * sync ran it meanwhile. When the file lock is taken by another process, + * try again after the interval; `synthesis_pending` stays set throughout. + * + * It runs from a timer, so nothing above it catches a failure, and an + * unhandled rejection ends the process (`installFatalHandlers`): that would + * be the daemon every session shares, over a `database is locked`. A failure + * here is logged and tried again after the interval, as lock contention is. + */ + private async runDeferredSynthesis(): Promise { + const retryLater = (): void => { + if (!this.db.isOpen()) return; + this.armDeferredSynthesis(parseSynthMinIntervalEnv(process.env.CODEGRAPH_SYNTH_MIN_INTERVAL_MS) ?? 1000); + }; + try { + await this.indexMutex.withLock(async () => { + if (this.queries.getMetadata('synthesis_pending') !== '1') return; + try { + this.fileLock.acquire(); + } catch { + retryLater(); + return; + } + try { + // A rebuild may have replaced the file meanwhile, as sync checks. + this.reopenReplacedDatabase(); + if (this.queries.getMetadata('synthesis_pending') !== '1') return; + await this.synthesizeWithWalDeferred(); + } finally { + this.fileLock.release(); + } + }); + } catch (err) { + logWarn(`Put-off synthesis failed, trying again after the interval: ${err instanceof Error ? err.message : String(err)}`); + retryLater(); + } + } + + /** + * The rebuild, with the WAL handling a sync gives it (#1231, #1248): a whole + * graph's edges rewritten must not be checkpointed inline on this thread by + * the next commit on this connection, so the valve bounds the WAL off-thread + * and runMaintenance folds it up. + */ + private async synthesizeWithWalDeferred(): Promise { + const deferWal = process.env.CODEGRAPH_NO_WAL_DEFER !== '1' && this.db.getJournalMode() === 'wal'; + let walValve: WalCheckpointValve | null = null; + let priorAutocheckpoint = 1000; + if (deferWal) { + priorAutocheckpoint = this.db.getWalAutocheckpoint(); + this.db.setWalAutocheckpoint(0); + walValve = new WalCheckpointValve( + this.db, + resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()) + ); + walValve.start(); + } + try { + await this.resolver.refreshSynthesis( + this.db.getPath(), + undefined, + walValve ? () => walValve!.backpressure() : undefined + ); + this.queries.setMetadata('synthesis_pending', '0'); + this.lastSynthesisAt = Date.now(); + await this.db.runMaintenance(); + } finally { + if (walValve) { walValve.stop(); await walValve.drain(); } + if (deferWal) { + try { this.db.setWalAutocheckpoint(priorAutocheckpoint); } catch { /* connection may be closing */ } + } + } + } + // =========================================================================== // File Watching // ===========================================================================