From 7889da6697e843bdff9fda737d62dc3d15c9ba40 Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Fri, 30 Oct 2026 18:53:03 +0000 Subject: [PATCH] feat(status): show files parsed beside files found, and add --strict (#3096) A language's file count says what the files are, not whether anything came of them, so a language with nothing in the graph could read as healthy. `codegraph status` and `codegraph_status` now show how many of a language's files yielded symbols when that isn't all of them, and warn about a language where none did. `status --json` gains filesByLanguage, parsedFilesByLanguage and unparsedLanguages. `status --strict` exits 1 unless the index is finished, up to date and every language that should yield symbols did. The count stays out of getStats(), which every explore runs. A plain EXISTS probe per file would have doubled getStats() on a 14,569-file index, from about 40 ms to about 80 ms. The query status runs instead takes 4.3 ms there, because the stored node count settles nearly every row, and only status runs it. The gate covers more than a zero count because a grammar that can't load no longer leaves one: since PR 2335 nothing is stored for such a file, the language drops out of the rows, and the only trace is pending files. A gate checking zeroes alone would pass that. Text changes: a language row gains " (N with symbols)" only when the two numbers differ, and a warning line appears for a language with no symbols. Every other line, and every existing JSON field, is unchanged. Across 115 real indexes, 9 repos have a code language with no symbols, all of them stubs in languages with at most 3 files, such as ripgrep's .nvim.lua. A file-count threshold would have hidden a real failure in a small smoke-test project, so the rule stays plain and the warning names the files instead. Reimplements PR 1658 by @ferrine. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + README.md | 2 +- __tests__/status-language-coverage.test.ts | 354 +++++++++++++++++++++ site/src/content/docs/reference/cli.md | 16 +- src/bin/codegraph.ts | 58 +++- src/db/queries.ts | 24 ++ src/extraction/grammars.ts | 9 + src/index.ts | 18 +- src/mcp/tools.ts | 15 +- src/types.ts | 15 + 10 files changed, 499 insertions(+), 13 deletions(-) create mode 100644 __tests__/status-language-coverage.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index fcbd764b8..53d2dfaaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph files --format package` and `codegraph_files` with `format: "package"` now list a project's files by the package that owns them, the nearest `package.json` or `composer.json` above each file that names one, so a monorepo splits into its workspace members with no setup and files nothing claims get a section of their own (#1871). Thanks @LuisAlbertoMK. - CodeGraph's MCP server now gives agents and editors its display name and icon, so one that shows them, like the Codex app, lists CodeGraph by name and logo instead of a plain `codegraph`. Thanks @fedosov. +- `codegraph status` now shows how many of a language's files yielded symbols when that isn't all of them, and warns when none did, so a parser that isn't working no longer looks healthy, and `codegraph status --strict` exits non-zero on that or on an index that's missing, unfinished or out of date, so packaging and CI can gate on it (#1658). Thanks @ferrine. ### Fixes diff --git a/README.md b/README.md index 3f5a5a598..54ce5ba02 100644 --- a/README.md +++ b/README.md @@ -533,7 +533,7 @@ codegraph init [path] # Initialize a project + build its graph (one codegraph uninit [path] # Remove CodeGraph from a project (--force to skip prompt) codegraph index [path] # Full index (--force to re-index, --quiet for less output) codegraph sync [path] # Incremental update -codegraph status [path] # Show statistics +codegraph status [path] # Show statistics (--json, --strict) codegraph unlock [path] # Remove a stale lock file that's blocking indexing codegraph query # Search symbols (--kind, --limit, --json) codegraph explore # Relevant symbols' source + call paths in one shot (same output as the codegraph_explore MCP tool) diff --git a/__tests__/status-language-coverage.test.ts b/__tests__/status-language-coverage.test.ts new file mode 100644 index 000000000..a63c38b35 --- /dev/null +++ b/__tests__/status-language-coverage.test.ts @@ -0,0 +1,354 @@ +/** + * `status`: files found against files that yielded symbols, and `--strict`. + * + * A language's file count says what the files are, not whether anything came of + * them, so a language with nothing in the graph could read as healthy. These pin + * the count, what is left out of it on purpose, the text lines scripts may read, + * and every way the gate fails: the failure that started this no longer shows up + * as a zero, because a grammar that can't load stores no row at all (#2335). + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { MAX_SOURCE_FILE_SIZE_BYTES } from '../src/file-limits'; +import { ToolHandler } from '../src/mcp/tools'; +import { endFreshnessMeasurements } from '../src/mcp/index-freshness'; + +const BIN = path.resolve(__dirname, '../dist/bin/codegraph.js'); + +// These index a project and spawn the real CLI, up to several times a test. +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const HAS_SYMBOL = 'export function alpha(): number { return 1; }\n'; +const NO_SYMBOL = '// nothing here but a comment\n'; + +describe('status language coverage', () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-language-coverage-')); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function write(files: Record): void { + for (const [file, content] of Object.entries(files)) { + const full = path.join(dir, file); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); + } + } + + async function indexed(files: Record): Promise { + write(files); + const cg = await CodeGraph.init(dir); + await cg.indexAll(); + return cg; + } + + async function indexedAndClosed(files: Record): Promise { + (await indexed(files)).close(); + } + + function status(...args: string[]): { code: number | null; stdout: string; stderr: string } { + const run = spawnSync(process.execPath, [BIN, 'status', dir, ...args], { + encoding: 'utf-8', + env: { ...process.env, CODEGRAPH_NO_DAEMON: '1', NO_COLOR: '1' }, + }); + return { code: run.status, stdout: run.stdout, stderr: run.stderr }; + } + + function statusJson(...args: string[]): { code: number | null; json: Record } { + const { code, stdout } = status('--json', ...args); + return { code, json: JSON.parse(stdout) }; + } + + /** The whole row a language prints under `Files by Language:`. */ + function languageRow(stdout: string, language: string): string | undefined { + return stdout.split(/\r?\n/).find((line) => line.startsWith(` ${language} `)); + } + + describe('the count', () => { + it('counts a file with a symbol, and not one with only its own file node', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL }); + + expect(cg.getStats().filesByLanguage.typescript).toBe(2); + expect(cg.getLanguageCoverage()).toEqual({ parsedFilesByLanguage: { typescript: 1 }, unparsedLanguages: [] }); + cg.close(); + }); + + it('names a language none of whose files yielded a symbol', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'stub.lua': '-- nothing here\n' }); + + expect(cg.getLanguageCoverage()).toEqual({ + parsedFilesByLanguage: { lua: 0, typescript: 1 }, + unparsedLanguages: ['lua'], + }); + cg.close(); + }); + + it('leaves out the languages tracked at file level, which yield nothing by design', async () => { + const cg = await indexed({ + 'a.ts': HAS_SYMBOL, + 'config.yaml': 'key: value\n', + 'application.properties': 'key=value\n', + 'pom.xml': '\n', + }); + + expect(cg.getStats().filesByLanguage).toMatchObject({ yaml: 1, properties: 1, xml: 1 }); + expect(cg.getLanguageCoverage()).toEqual({ parsedFilesByLanguage: { typescript: 1 }, unparsedLanguages: [] }); + cg.close(); + }); + + it('asks the nodes when the stored count cannot say', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL }); + // A file whose one symbol is its only node reads like a file with none. + const db = (cg as unknown as { db: { getDb(): { prepare(sql: string): { run(): unknown } } } }).db.getDb(); + db.prepare('UPDATE files SET node_count = 1').run(); + + expect(cg.getLanguageCoverage().parsedFilesByLanguage.typescript).toBe(1); + cg.close(); + }); + + it('counts a file over the size limit as found, not parsed', async () => { + const cg = await indexed({ + 'a.ts': HAS_SYMBOL, + 'huge.ts': `// ${'x'.repeat(MAX_SOURCE_FILE_SIZE_BYTES)}\n${HAS_SYMBOL}`, + }); + + expect(cg.getStats().filesByLanguage.typescript).toBe(2); + expect(cg.getLanguageCoverage().parsedFilesByLanguage.typescript).toBe(1); + cg.close(); + }); + + it('costs an explore nothing: only status runs it', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL }); + const queries = (cg as unknown as { queries: { getParsedFileCounts(): unknown } }).queries; + let calls = 0; + const original = queries.getParsedFileCounts.bind(queries); + queries.getParsedFileCounts = () => { + calls++; + return original(); + }; + const handler = new ToolHandler(cg); + + const explored = await handler.execute('codegraph_explore', { query: 'alpha' }); + expect(explored.isError).toBeFalsy(); + cg.getStats(); + expect(calls).toBe(0); + + await handler.execute('codegraph_status', {}); + expect(calls).toBe(1); + await endFreshnessMeasurements(); + cg.close(); + }); + }); + + describe('status text', () => { + it('leaves a language row as it was when every file yielded symbols', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'b.ts': HAS_SYMBOL }); + + expect(languageRow(status().stdout, 'typescript')).toBe(' typescript 2'); + }); + + it('adds the count to a row when the two differ', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL }); + + expect(languageRow(status().stdout, 'typescript')).toBe(' typescript 2 (1 with symbols)'); + }); + + it('leaves a file-level language row as it was', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'config.yaml': 'key: value\n' }); + + expect(languageRow(status().stdout, 'yaml')).toBe(' yaml 1'); + }); + + it('says which language yielded nothing and names its files, without failing', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, '.nvim.lua': '-- settings\n' }); + + const { code, stdout } = status(); + + expect(code).toBe(0); + expect(languageRow(stdout, 'lua')).toBe(' lua 1 (0 with symbols)'); + expect(stdout).toMatch(/No symbols came from any lua file .* \.nvim\.lua/); + expect(stdout).toContain('Index is up to date'); + }); + + it('says nothing of the kind about a healthy index', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL }); + + expect(status().stdout).not.toContain('No symbols came from'); + }); + }); + + describe('status --json', () => { + it('adds the coverage fields and keeps every field it had', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL, 'c.yaml': 'key: value\n' }); + + const { json } = statusJson(); + + expect(json).toMatchObject({ + filesByLanguage: { typescript: 2, yaml: 1 }, + parsedFilesByLanguage: { typescript: 1 }, + unparsedLanguages: [], + }); + expect(json.parsedFilesByLanguage).not.toHaveProperty('yaml'); + expect(Object.keys(json)).toEqual( + expect.arrayContaining([ + 'initialized', 'version', 'projectPath', 'indexPath', 'lastIndexed', 'fileCount', 'nodeCount', 'edgeCount', + 'dbSizeBytes', 'walSizeBytes', 'backend', 'journalMode', 'nodesByKind', 'languages', 'pendingChanges', + 'worktreeMismatch', 'rebuildInProgress', 'index', + ]), + ); + expect(json.languages).toEqual(expect.arrayContaining(['typescript', 'yaml'])); + }); + + it('keeps the uninitialized answer as it was', () => { + const { code, json } = statusJson(); + + expect(code).toBe(0); + expect(Object.keys(json).sort()).toEqual(['indexPath', 'initialized', 'lastIndexed', 'projectPath', 'version']); + }); + }); + + describe('status --strict', () => { + const healthy = { 'a.ts': HAS_SYMBOL, 'config.yaml': 'key: value\n', 'pom.xml': '\n' }; + + function setIndexState(state: string): void { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(path.join(dir, '.codegraph', 'codegraph.db')); + db.prepare( + "INSERT INTO project_metadata (key, value, updated_at) VALUES ('index_state', ?, 0) " + + 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', + ).run(state); + db.close(); + } + + it('passes a finished, current index whose languages all yielded symbols', async () => { + await indexedAndClosed(healthy); + + expect(status('--strict').code).toBe(0); + expect(statusJson('--strict').code).toBe(0); + }); + + it('prints the same text as a plain status', async () => { + await indexedAndClosed(healthy); + + expect(status('--strict').stdout).toBe(status().stdout); + }); + + it('fails a language that yielded nothing, in either output mode', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'stub.lua': '-- nothing here\n' }); + + expect(status('--strict').code).toBe(1); + const { code, json } = statusJson('--strict'); + expect(code).toBe(1); + expect(json.unparsedLanguages).toEqual(['lua']); + }); + + it('does not fail a language for one file that yielded nothing', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL }); + + expect(status('--strict').code).toBe(0); + }); + + it('fails with no index to read, in either output mode', () => { + expect(status('--strict').code).toBe(1); + expect(statusJson('--strict').code).toBe(1); + }); + + it.each(['indexing', 'partial', 'failed'])('fails an index left %s', async (state) => { + await indexedAndClosed(healthy); + setIndexState(state); + + expect(status().code).toBe(0); + expect(status('--strict').code).toBe(1); + }); + + it('fails while a rebuild runs: its numbers are a snapshot', async () => { + await indexedAndClosed(healthy); + const lock = JSON.stringify({ pid: process.pid, mode: 'rebuild', startedAt: Date.now(), ready: false }); + fs.writeFileSync(path.join(dir, '.codegraph', 'rebuild.pid'), lock); + fs.writeFileSync(path.join(dir, '.codegraph', 'writer.pid'), lock); + + expect(status().code).toBe(0); + expect(status('--strict').code).toBe(1); + expect(statusJson('--strict').code).toBe(1); + }); + + it('fails files the index does not have, which is all a grammar that cannot load leaves', async () => { + await indexedAndClosed(healthy); + write({ 'b.ts': HAS_SYMBOL }); + + expect(status().code).toBe(0); + expect(status('--strict').code).toBe(1); + }); + + it('fails a file that lost its symbols', async () => { + const cg = await indexed(healthy); + const db = (cg as unknown as { db: { getDb(): { prepare(sql: string): { run(...args: unknown[]): unknown } } } }).db.getDb(); + db.prepare('DELETE FROM nodes WHERE file_path = ?').run('a.ts'); + db.prepare('UPDATE files SET node_count = 0 WHERE path = ?').run('a.ts'); + cg.close(); + + expect(status().code).toBe(0); + expect(status('--strict').code).toBe(1); + }); + + it('does not fail one file that cannot be parsed', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'bad.ts': '@@@ ((( }}} class 1 = ;\n' }); + + const { code, stdout } = status('--strict'); + + expect(stdout).toContain('could not be parsed'); + expect(code).toBe(0); + }); + }); + + describe('codegraph_status', () => { + afterEach(async () => { + await endFreshnessMeasurements(); + }); + + async function statusText(cg: CodeGraph): Promise { + const result = await new ToolHandler(cg).execute('codegraph_status', {}); + expect(result.isError).toBeFalsy(); + return result.content[0]!.text as string; + } + + it('leaves a language row as it was when every file yielded symbols', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'config.yaml': 'key: value\n' }); + + const text = await statusText(cg); + + expect(text).toContain('- typescript: 1\n'); + expect(text).toMatch(/- yaml: 1(\n|$)/); + expect(text).not.toContain('No symbols from'); + cg.close(); + }); + + it('shows the count when the two differ', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL }); + + expect(await statusText(cg)).toContain('- typescript: 2 (1 with symbols)'); + cg.close(); + }); + + it('tells an agent which languages it cannot search, as a normal answer', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'stub.lua': '-- nothing here\n' }); + + const text = await statusText(cg); + + expect(text).toContain('- lua: 1 (0 with symbols)'); + expect(text).toContain('**No symbols from:** ⚠ lua'); + cg.close(); + }); + }); +}); diff --git a/site/src/content/docs/reference/cli.md b/site/src/content/docs/reference/cli.md index 26f4d52aa..5b335da82 100644 --- a/site/src/content/docs/reference/cli.md +++ b/site/src/content/docs/reference/cli.md @@ -11,7 +11,7 @@ codegraph init [path] # Initialize a project + build its graph (one codegraph uninit [path] # Remove CodeGraph from a project (--force to skip prompt) codegraph index [path] # Full re-index from scratch (--force, --quiet, --verbose) codegraph sync [path] # Incremental update (--quiet) -codegraph status [path] # Show statistics (--json) +codegraph status [path] # Show statistics (--json, --strict) codegraph ui [path] # Open the browser viewer (not released yet — see below) codegraph unlock [path] # Remove a stale lock file that's blocking indexing codegraph query # Search symbols (--kind, --limit, --json) @@ -35,6 +35,20 @@ The MCP server (`codegraph serve --mcp`) is launched automatically by your agent `codegraph init` creates the local `.codegraph/` directory **and** builds the full graph in one step. (The old `-i`/`--index` flag is now a no-op, accepted only so existing scripts don't break.) After that the file watcher keeps the graph current automatically — `index` (a full rebuild from scratch) and `sync` (an incremental update) are only needed when the watcher is disabled or you're scripting against the index outside an agent session. +## status + +`codegraph status` shows what is in the index, including how many files each language has. When only some of a language's files yielded symbols, its row says how many did: + +```text +Files by Language: + typescript 1,204 + python 500 (420 with symbols) +``` + +Empty files and stubs explain most gaps. A language where no file yielded a symbol gets a warning naming its files, because that is also what a parser that isn't working looks like. Formats tracked at file level, such as YAML, never show a count. + +`codegraph status --strict` exits with code 1 unless the index is finished, up to date, and every language that should yield symbols did, so packaging and CI can gate on it. A dirty working tree counts as out of date, and one file that can't be parsed doesn't fail it. With `--json` the same counts are in `filesByLanguage`, `parsedFilesByLanguage` and `unparsedLanguages`. + ## Query commands `query`, `callers`, `callees`, and `impact` all accept `--json` for machine-readable output. diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index d9a91aa8a..f8d5fd01e 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -75,7 +75,7 @@ import { lookupSymbolNodes, describeSymbolNode, groupDefinitions } from '../grap import { callSiteLines, formatCallSiteLines } from '../graph/call-sites'; import { goPackageReaderFor } from '../graph/go-package'; import { groupFilesByPackage, packageGroupTitle, packageListingTitle } from '../graph/owning-package'; -import type { Node, Edge } from '../types'; +import type { Node, Edge, Language } from '../types'; import { isTestPath } from '../search/query-utils'; // Decided once, before `--color`/`--no-color` are stripped from argv below @@ -1092,6 +1092,11 @@ program } }); +/** Up to three of `paths`, and how many more there are. */ +function samplePaths(paths: string[]): string { + return paths.slice(0, 3).join(', ') + (paths.length > 3 ? ` (+${formatNumber(paths.length - 3)} more)` : ''); +} + /** * The `status` warnings for indexed files whose symbols are missing although * their content is current (#2335, #2336): one line per group, saying what is @@ -1099,21 +1104,19 @@ program */ function describeFilesMissingSymbols(health: { needsReindex: string[]; parseErrors: string[] }): string[] { const dash = getGlyphs().dash; - const sample = (paths: string[]): string => - paths.slice(0, 3).join(', ') + (paths.length > 3 ? ` (+${formatNumber(paths.length - 3)} more)` : ''); const lines: string[] = []; const reindex = health.needsReindex.length; if (reindex > 0) { lines.push( `${formatNumber(reindex)} ${reindex === 1 ? 'file is missing its' : 'files are missing their'} symbols ${dash} ` + - `run "codegraph sync" to re-index ${reindex === 1 ? 'it' : 'them'}: ${sample(health.needsReindex)}` + `run "codegraph sync" to re-index ${reindex === 1 ? 'it' : 'them'}: ${samplePaths(health.needsReindex)}` ); } const unparsed = health.parseErrors.length; if (unparsed > 0) { lines.push( `${formatNumber(unparsed)} ${unparsed === 1 ? 'file could not be parsed, so its symbols are' : 'files could not be parsed, so their symbols are'} ` + - `missing: ${sample(health.parseErrors)} ${dash} "codegraph files --json" shows the errors` + `missing: ${samplePaths(health.parseErrors)} ${dash} "codegraph files --json" shows the errors` ); } return lines; @@ -1126,7 +1129,8 @@ program .command('status [path]') .description('Show index status and statistics') .option('-j, --json', 'Output as JSON') - .action(async (pathArg: string | undefined, options: { json?: boolean }) => { + .option('--strict', 'Exit 1 unless the index is finished, up to date, and every language that should yield symbols did') + .action(async (pathArg: string | undefined, options: { json?: boolean; strict?: boolean }) => { const projectPath = resolveProjectPath(pathArg); // The directory the user actually ran from, before walking up to the index // root. Used to detect when the resolved index lives in a different git @@ -1136,6 +1140,8 @@ program try { if (!isInitialized(projectPath)) { + // A gate with no index to read has proved nothing. + if (options.strict) process.exitCode = 1; if (options.json) { console.log(JSON.stringify({ initialized: false, @@ -1183,6 +1189,24 @@ program // Files whose content is current but whose symbols are missing — the // content-hash check behind "up to date" cannot see them (#2336). const health = cg.getIndexHealth(); + // Files found against files that yielded symbols: a language can be + // listed with its full count and have nothing in the graph. + const coverage = cg.getLanguageCoverage(); + + // Reached by every output mode below. A grammar that can't load leaves no + // row and no zero to count, only files the index doesn't have yet, so + // "up to date" is part of the gate as much as the language check is. + // Parse errors are not: one bad fixture shouldn't fail a whole project. + const totalChanges = changes.added.length + changes.modified.length + changes.removed.length; + if (options.strict && ( + rebuildInProgress || + indexState === 'indexing' || indexState === 'partial' || indexState === 'failed' || + totalChanges > 0 || + health.needsReindex.length > 0 || + coverage.unparsedLanguages.length > 0 + )) { + process.exitCode = 1; + } // JSON output mode if (options.json) { @@ -1202,6 +1226,11 @@ program journalMode, nodesByKind: stats.nodesByKind, languages: Object.entries(stats.filesByLanguage).filter(([, count]) => count > 0).map(([lang]) => lang), + filesByLanguage: stats.filesByLanguage, + // Files that yielded a symbol, for the languages expected to. + parsedFilesByLanguage: coverage.parsedFilesByLanguage, + // Of those, the languages with files and no symbols at all. + unparsedLanguages: coverage.unparsedLanguages, pendingChanges: { added: changes.added.length, modified: changes.modified.length, @@ -1313,7 +1342,9 @@ program .filter(([, count]) => count > 0) .sort((a, b) => b[1] - a[1]); for (const [lang, count] of filesByLang) { - console.log(` ${lang.padEnd(15)} ${formatNumber(count)}`); + const parsed = coverage.parsedFilesByLanguage[lang as Language]; + const withSymbols = parsed === undefined || parsed === count ? '' : ` (${formatNumber(parsed)} with symbols)`; + console.log(` ${lang.padEnd(15)} ${formatNumber(count)}${chalk.dim(withSymbols)}`); } console.log(); @@ -1324,8 +1355,19 @@ program return; } + // Worded for both causes: a lone config stub with nothing to extract + // (a `.nvim.lua`, a comment-only `site.js`) reads exactly like a parser + // that isn't working, and the file names are what tell them apart. + if (coverage.unparsedLanguages.length > 0) { + const files = cg.getFiles(); + for (const language of coverage.unparsedLanguages) { + const paths = files.filter((file) => file.language === language).map((file) => file.path); + warn(`No symbols came from any ${language} file ${getGlyphs().dash} fine if they hold no code, otherwise its parser isn't working: ${samplePaths(paths)}`); + } + console.log(); + } + // Pending changes - const totalChanges = changes.added.length + changes.modified.length + changes.removed.length; if (totalChanges > 0) { console.log(chalk.bold('Pending Changes:')); if (changes.added.length > 0) { diff --git a/src/db/queries.ts b/src/db/queries.ts index 7c794bc32..2e242d7ca 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -5593,6 +5593,30 @@ export class QueryBuilder { .get() as { nodes: number; edges: number }; } + /** + * For each language, how many files it has and how many yielded a symbol: a + * node other than the file's own. More than one stored node settles a row, so + * `nodes` is only probed for the few it can't, such as a file whose one symbol + * came from a framework pass. Probing every row costs as much as the whole of + * getStats() on a 14,000-file index; this costs a tenth of that. Not part of + * getStats() either way, because every explore runs that. + */ + getParsedFileCounts(): Array<{ language: Language; files: number; parsed: number }> { + return this.db + .prepare(` + SELECT f.language AS language, COUNT(*) AS files, + SUM(CASE + WHEN f.node_count > 1 THEN 1 + WHEN EXISTS (SELECT 1 FROM nodes n WHERE n.file_path = f.path AND n.kind <> 'file') THEN 1 + ELSE 0 + END) AS parsed + FROM files f + GROUP BY f.language + ORDER BY f.language + `) + .all() as Array<{ language: Language; files: number; parsed: number }>; + } + /** * Get graph statistics */ diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 66c69687b..44a18994d 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -837,6 +837,15 @@ export function isFileLevelOnlyLanguage(language: Language): boolean { return language === 'yaml' || language === 'twig' || language === 'properties'; } +/** + * Whether files of this language are expected to yield symbols, so that a + * language with none is news. Not so for the file-level languages, nor for + * `xml`, where only MyBatis mappers yield anything. + */ +export function yieldsSymbols(language: Language): boolean { + return !isFileLevelOnlyLanguage(language) && language !== 'xml' && language !== 'unknown'; +} + /** * Get all supported languages (those with grammar definitions). */ diff --git a/src/index.ts b/src/index.ts index c78458c93..777f5efd9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,6 +26,7 @@ import { FindRelevantContextOptions, UnresolvedReference, IndexHealth, + LanguageCoverage, Language, } from './types'; import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db'; @@ -46,7 +47,7 @@ import { extractFromSource, initGrammars, } from './extraction'; -import { detectLanguage, hasGrammarLoadFailure, isFileLevelOnlyLanguage } from './extraction/grammars'; +import { detectLanguage, hasGrammarLoadFailure, isFileLevelOnlyLanguage, yieldsSymbols } from './extraction/grammars'; import { ReferenceResolver, changedRoutes, @@ -2770,6 +2771,21 @@ export class CodeGraph { return { needsReindex, parseErrors }; } + /** + * Files found against files that yielded symbols, by language. Reported by + * `status`, whose `--strict` fails on a language that yielded none. + */ + getLanguageCoverage(): LanguageCoverage { + const parsedFilesByLanguage: LanguageCoverage['parsedFilesByLanguage'] = {}; + const unparsedLanguages: Language[] = []; + for (const { language, parsed } of this.queries.getParsedFileCounts()) { + if (!yieldsSymbols(language)) continue; + parsedFilesByLanguage[language] = parsed; + if (parsed === 0) unparsedLanguages.push(language); + } + return { parsedFilesByLanguage, unparsedLanguages }; + } + // =========================================================================== // Graph Query Methods // =========================================================================== diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 3be1f39b8..080e4f34a 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -35,7 +35,7 @@ import { type WorktreeIndexMismatch, } from '../sync/worktree'; import type { PendingFile } from '../sync'; -import type { Node, Edge, SearchResult, Subgraph, NodeKind } from '../types'; +import type { Node, Edge, SearchResult, Subgraph, NodeKind, Language } from '../types'; import { isTestFile, normalizeNameToken, splitAtCjkSwitches, isQueryNameToken } from '../search/query-utils'; import { groupDefinitions, isQualifiedSymbol, lastQualifierPart, matchesSymbol, qualifiedSymbolMatches } from '../graph/symbol-lookup'; import { goPackageReaderFor } from '../graph/go-package'; @@ -8548,12 +8548,23 @@ export class ToolHandler { } } + // Beside the file count, how many yielded symbols: it is what tells a + // language that parsed from one that is only listed. + const coverage = cg.getLanguageCoverage(); lines.push('', '**Languages:**'); for (const [lang, count] of Object.entries(stats.filesByLanguage)) { if ((count as number) > 0) { - lines.push(`- ${lang}: ${count}`); + const parsed = coverage.parsedFilesByLanguage[lang as Language]; + lines.push(parsed === undefined || parsed === count ? `- ${lang}: ${count}` : `- ${lang}: ${count} (${parsed} with symbols)`); } } + if (coverage.unparsedLanguages.length > 0) { + lines.push( + '', + `**No symbols from:** ⚠ ${coverage.unparsedLanguages.join(', ')} ` + + `— those files are indexed, but nothing in them can be searched or traced; Read them directly.` + ); + } // Whole-index degradation (#876): when live watching has permanently // stopped, getPendingFiles() is empty (so no "Pending sync" section below) diff --git a/src/types.ts b/src/types.ts index 9c4c4461f..05313675f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -272,6 +272,21 @@ export interface FileRecord { generated?: boolean; } +/** + * Files found against files that yielded symbols, by language, for `status`. + * `GraphStats.filesByLanguage` counts files by what they are, so on its own it + * reads the same for a language that parsed and one that yielded nothing. + */ +export interface LanguageCoverage { + /** + * Files that yielded a symbol, for the languages expected to. A language + * tracked at file level (yaml, most xml) has no entry: nothing is missing. + */ + parsedFilesByLanguage: Partial>; + /** Languages expected to yield symbols that have files, none of which did. */ + unparsedLanguages: Language[]; +} + /** * Indexed files whose symbols are missing even though their content hash is * current — what a hash comparison alone calls "up to date" (#2336). Paths are