diff --git a/CHANGELOG.md b/CHANGELOG.md index fcbd764b8..53d2dfaaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph files --format package` and `codegraph_files` with `format: "package"` now list a project's files by the package that owns them, the nearest `package.json` or `composer.json` above each file that names one, so a monorepo splits into its workspace members with no setup and files nothing claims get a section of their own (#1871). Thanks @LuisAlbertoMK. - CodeGraph's MCP server now gives agents and editors its display name and icon, so one that shows them, like the Codex app, lists CodeGraph by name and logo instead of a plain `codegraph`. Thanks @fedosov. +- `codegraph status` now shows how many of a language's files yielded symbols when that isn't all of them, and warns when none did, so a parser that isn't working no longer looks healthy, and `codegraph status --strict` exits non-zero on that or on an index that's missing, unfinished or out of date, so packaging and CI can gate on it (#1658). Thanks @ferrine. ### Fixes diff --git a/README.md b/README.md index 3f5a5a598..54ce5ba02 100644 --- a/README.md +++ b/README.md @@ -533,7 +533,7 @@ codegraph init [path] # Initialize a project + build its graph (one codegraph uninit [path] # Remove CodeGraph from a project (--force to skip prompt) codegraph index [path] # Full index (--force to re-index, --quiet for less output) codegraph sync [path] # Incremental update -codegraph status [path] # Show statistics +codegraph status [path] # Show statistics (--json, --strict) codegraph unlock [path] # Remove a stale lock file that's blocking indexing codegraph query # Search symbols (--kind, --limit, --json) codegraph explore # Relevant symbols' source + call paths in one shot (same output as the codegraph_explore MCP tool) diff --git a/__tests__/status-language-coverage.test.ts b/__tests__/status-language-coverage.test.ts new file mode 100644 index 000000000..a63c38b35 --- /dev/null +++ b/__tests__/status-language-coverage.test.ts @@ -0,0 +1,354 @@ +/** + * `status`: files found against files that yielded symbols, and `--strict`. + * + * A language's file count says what the files are, not whether anything came of + * them, so a language with nothing in the graph could read as healthy. These pin + * the count, what is left out of it on purpose, the text lines scripts may read, + * and every way the gate fails: the failure that started this no longer shows up + * as a zero, because a grammar that can't load stores no row at all (#2335). + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { MAX_SOURCE_FILE_SIZE_BYTES } from '../src/file-limits'; +import { ToolHandler } from '../src/mcp/tools'; +import { endFreshnessMeasurements } from '../src/mcp/index-freshness'; + +const BIN = path.resolve(__dirname, '../dist/bin/codegraph.js'); + +// These index a project and spawn the real CLI, up to several times a test. +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const HAS_SYMBOL = 'export function alpha(): number { return 1; }\n'; +const NO_SYMBOL = '// nothing here but a comment\n'; + +describe('status language coverage', () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-language-coverage-')); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function write(files: Record): void { + for (const [file, content] of Object.entries(files)) { + const full = path.join(dir, file); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); + } + } + + async function indexed(files: Record): Promise { + write(files); + const cg = await CodeGraph.init(dir); + await cg.indexAll(); + return cg; + } + + async function indexedAndClosed(files: Record): Promise { + (await indexed(files)).close(); + } + + function status(...args: string[]): { code: number | null; stdout: string; stderr: string } { + const run = spawnSync(process.execPath, [BIN, 'status', dir, ...args], { + encoding: 'utf-8', + env: { ...process.env, CODEGRAPH_NO_DAEMON: '1', NO_COLOR: '1' }, + }); + return { code: run.status, stdout: run.stdout, stderr: run.stderr }; + } + + function statusJson(...args: string[]): { code: number | null; json: Record } { + const { code, stdout } = status('--json', ...args); + return { code, json: JSON.parse(stdout) }; + } + + /** The whole row a language prints under `Files by Language:`. */ + function languageRow(stdout: string, language: string): string | undefined { + return stdout.split(/\r?\n/).find((line) => line.startsWith(` ${language} `)); + } + + describe('the count', () => { + it('counts a file with a symbol, and not one with only its own file node', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL }); + + expect(cg.getStats().filesByLanguage.typescript).toBe(2); + expect(cg.getLanguageCoverage()).toEqual({ parsedFilesByLanguage: { typescript: 1 }, unparsedLanguages: [] }); + cg.close(); + }); + + it('names a language none of whose files yielded a symbol', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'stub.lua': '-- nothing here\n' }); + + expect(cg.getLanguageCoverage()).toEqual({ + parsedFilesByLanguage: { lua: 0, typescript: 1 }, + unparsedLanguages: ['lua'], + }); + cg.close(); + }); + + it('leaves out the languages tracked at file level, which yield nothing by design', async () => { + const cg = await indexed({ + 'a.ts': HAS_SYMBOL, + 'config.yaml': 'key: value\n', + 'application.properties': 'key=value\n', + 'pom.xml': '\n', + }); + + expect(cg.getStats().filesByLanguage).toMatchObject({ yaml: 1, properties: 1, xml: 1 }); + expect(cg.getLanguageCoverage()).toEqual({ parsedFilesByLanguage: { typescript: 1 }, unparsedLanguages: [] }); + cg.close(); + }); + + it('asks the nodes when the stored count cannot say', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL }); + // A file whose one symbol is its only node reads like a file with none. + const db = (cg as unknown as { db: { getDb(): { prepare(sql: string): { run(): unknown } } } }).db.getDb(); + db.prepare('UPDATE files SET node_count = 1').run(); + + expect(cg.getLanguageCoverage().parsedFilesByLanguage.typescript).toBe(1); + cg.close(); + }); + + it('counts a file over the size limit as found, not parsed', async () => { + const cg = await indexed({ + 'a.ts': HAS_SYMBOL, + 'huge.ts': `// ${'x'.repeat(MAX_SOURCE_FILE_SIZE_BYTES)}\n${HAS_SYMBOL}`, + }); + + expect(cg.getStats().filesByLanguage.typescript).toBe(2); + expect(cg.getLanguageCoverage().parsedFilesByLanguage.typescript).toBe(1); + cg.close(); + }); + + it('costs an explore nothing: only status runs it', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL }); + const queries = (cg as unknown as { queries: { getParsedFileCounts(): unknown } }).queries; + let calls = 0; + const original = queries.getParsedFileCounts.bind(queries); + queries.getParsedFileCounts = () => { + calls++; + return original(); + }; + const handler = new ToolHandler(cg); + + const explored = await handler.execute('codegraph_explore', { query: 'alpha' }); + expect(explored.isError).toBeFalsy(); + cg.getStats(); + expect(calls).toBe(0); + + await handler.execute('codegraph_status', {}); + expect(calls).toBe(1); + await endFreshnessMeasurements(); + cg.close(); + }); + }); + + describe('status text', () => { + it('leaves a language row as it was when every file yielded symbols', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'b.ts': HAS_SYMBOL }); + + expect(languageRow(status().stdout, 'typescript')).toBe(' typescript 2'); + }); + + it('adds the count to a row when the two differ', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL }); + + expect(languageRow(status().stdout, 'typescript')).toBe(' typescript 2 (1 with symbols)'); + }); + + it('leaves a file-level language row as it was', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'config.yaml': 'key: value\n' }); + + expect(languageRow(status().stdout, 'yaml')).toBe(' yaml 1'); + }); + + it('says which language yielded nothing and names its files, without failing', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, '.nvim.lua': '-- settings\n' }); + + const { code, stdout } = status(); + + expect(code).toBe(0); + expect(languageRow(stdout, 'lua')).toBe(' lua 1 (0 with symbols)'); + expect(stdout).toMatch(/No symbols came from any lua file .* \.nvim\.lua/); + expect(stdout).toContain('Index is up to date'); + }); + + it('says nothing of the kind about a healthy index', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL }); + + expect(status().stdout).not.toContain('No symbols came from'); + }); + }); + + describe('status --json', () => { + it('adds the coverage fields and keeps every field it had', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL, 'c.yaml': 'key: value\n' }); + + const { json } = statusJson(); + + expect(json).toMatchObject({ + filesByLanguage: { typescript: 2, yaml: 1 }, + parsedFilesByLanguage: { typescript: 1 }, + unparsedLanguages: [], + }); + expect(json.parsedFilesByLanguage).not.toHaveProperty('yaml'); + expect(Object.keys(json)).toEqual( + expect.arrayContaining([ + 'initialized', 'version', 'projectPath', 'indexPath', 'lastIndexed', 'fileCount', 'nodeCount', 'edgeCount', + 'dbSizeBytes', 'walSizeBytes', 'backend', 'journalMode', 'nodesByKind', 'languages', 'pendingChanges', + 'worktreeMismatch', 'rebuildInProgress', 'index', + ]), + ); + expect(json.languages).toEqual(expect.arrayContaining(['typescript', 'yaml'])); + }); + + it('keeps the uninitialized answer as it was', () => { + const { code, json } = statusJson(); + + expect(code).toBe(0); + expect(Object.keys(json).sort()).toEqual(['indexPath', 'initialized', 'lastIndexed', 'projectPath', 'version']); + }); + }); + + describe('status --strict', () => { + const healthy = { 'a.ts': HAS_SYMBOL, 'config.yaml': 'key: value\n', 'pom.xml': '\n' }; + + function setIndexState(state: string): void { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(path.join(dir, '.codegraph', 'codegraph.db')); + db.prepare( + "INSERT INTO project_metadata (key, value, updated_at) VALUES ('index_state', ?, 0) " + + 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', + ).run(state); + db.close(); + } + + it('passes a finished, current index whose languages all yielded symbols', async () => { + await indexedAndClosed(healthy); + + expect(status('--strict').code).toBe(0); + expect(statusJson('--strict').code).toBe(0); + }); + + it('prints the same text as a plain status', async () => { + await indexedAndClosed(healthy); + + expect(status('--strict').stdout).toBe(status().stdout); + }); + + it('fails a language that yielded nothing, in either output mode', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'stub.lua': '-- nothing here\n' }); + + expect(status('--strict').code).toBe(1); + const { code, json } = statusJson('--strict'); + expect(code).toBe(1); + expect(json.unparsedLanguages).toEqual(['lua']); + }); + + it('does not fail a language for one file that yielded nothing', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL }); + + expect(status('--strict').code).toBe(0); + }); + + it('fails with no index to read, in either output mode', () => { + expect(status('--strict').code).toBe(1); + expect(statusJson('--strict').code).toBe(1); + }); + + it.each(['indexing', 'partial', 'failed'])('fails an index left %s', async (state) => { + await indexedAndClosed(healthy); + setIndexState(state); + + expect(status().code).toBe(0); + expect(status('--strict').code).toBe(1); + }); + + it('fails while a rebuild runs: its numbers are a snapshot', async () => { + await indexedAndClosed(healthy); + const lock = JSON.stringify({ pid: process.pid, mode: 'rebuild', startedAt: Date.now(), ready: false }); + fs.writeFileSync(path.join(dir, '.codegraph', 'rebuild.pid'), lock); + fs.writeFileSync(path.join(dir, '.codegraph', 'writer.pid'), lock); + + expect(status().code).toBe(0); + expect(status('--strict').code).toBe(1); + expect(statusJson('--strict').code).toBe(1); + }); + + it('fails files the index does not have, which is all a grammar that cannot load leaves', async () => { + await indexedAndClosed(healthy); + write({ 'b.ts': HAS_SYMBOL }); + + expect(status().code).toBe(0); + expect(status('--strict').code).toBe(1); + }); + + it('fails a file that lost its symbols', async () => { + const cg = await indexed(healthy); + const db = (cg as unknown as { db: { getDb(): { prepare(sql: string): { run(...args: unknown[]): unknown } } } }).db.getDb(); + db.prepare('DELETE FROM nodes WHERE file_path = ?').run('a.ts'); + db.prepare('UPDATE files SET node_count = 0 WHERE path = ?').run('a.ts'); + cg.close(); + + expect(status().code).toBe(0); + expect(status('--strict').code).toBe(1); + }); + + it('does not fail one file that cannot be parsed', async () => { + await indexedAndClosed({ 'a.ts': HAS_SYMBOL, 'bad.ts': '@@@ ((( }}} class 1 = ;\n' }); + + const { code, stdout } = status('--strict'); + + expect(stdout).toContain('could not be parsed'); + expect(code).toBe(0); + }); + }); + + describe('codegraph_status', () => { + afterEach(async () => { + await endFreshnessMeasurements(); + }); + + async function statusText(cg: CodeGraph): Promise { + const result = await new ToolHandler(cg).execute('codegraph_status', {}); + expect(result.isError).toBeFalsy(); + return result.content[0]!.text as string; + } + + it('leaves a language row as it was when every file yielded symbols', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'config.yaml': 'key: value\n' }); + + const text = await statusText(cg); + + expect(text).toContain('- typescript: 1\n'); + expect(text).toMatch(/- yaml: 1(\n|$)/); + expect(text).not.toContain('No symbols from'); + cg.close(); + }); + + it('shows the count when the two differ', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'b.ts': NO_SYMBOL }); + + expect(await statusText(cg)).toContain('- typescript: 2 (1 with symbols)'); + cg.close(); + }); + + it('tells an agent which languages it cannot search, as a normal answer', async () => { + const cg = await indexed({ 'a.ts': HAS_SYMBOL, 'stub.lua': '-- nothing here\n' }); + + const text = await statusText(cg); + + expect(text).toContain('- lua: 1 (0 with symbols)'); + expect(text).toContain('**No symbols from:** ⚠ lua'); + cg.close(); + }); + }); +}); diff --git a/site/src/content/docs/reference/cli.md b/site/src/content/docs/reference/cli.md index 26f4d52aa..5b335da82 100644 --- a/site/src/content/docs/reference/cli.md +++ b/site/src/content/docs/reference/cli.md @@ -11,7 +11,7 @@ codegraph init [path] # Initialize a project + build its graph (one codegraph uninit [path] # Remove CodeGraph from a project (--force to skip prompt) codegraph index [path] # Full re-index from scratch (--force, --quiet, --verbose) codegraph sync [path] # Incremental update (--quiet) -codegraph status [path] # Show statistics (--json) +codegraph status [path] # Show statistics (--json, --strict) codegraph ui [path] # Open the browser viewer (not released yet — see below) codegraph unlock [path] # Remove a stale lock file that's blocking indexing codegraph query # Search symbols (--kind, --limit, --json) @@ -35,6 +35,20 @@ The MCP server (`codegraph serve --mcp`) is launched automatically by your agent `codegraph init` creates the local `.codegraph/` directory **and** builds the full graph in one step. (The old `-i`/`--index` flag is now a no-op, accepted only so existing scripts don't break.) After that the file watcher keeps the graph current automatically — `index` (a full rebuild from scratch) and `sync` (an incremental update) are only needed when the watcher is disabled or you're scripting against the index outside an agent session. +## status + +`codegraph status` shows what is in the index, including how many files each language has. When only some of a language's files yielded symbols, its row says how many did: + +```text +Files by Language: + typescript 1,204 + python 500 (420 with symbols) +``` + +Empty files and stubs explain most gaps. A language where no file yielded a symbol gets a warning naming its files, because that is also what a parser that isn't working looks like. Formats tracked at file level, such as YAML, never show a count. + +`codegraph status --strict` exits with code 1 unless the index is finished, up to date, and every language that should yield symbols did, so packaging and CI can gate on it. A dirty working tree counts as out of date, and one file that can't be parsed doesn't fail it. With `--json` the same counts are in `filesByLanguage`, `parsedFilesByLanguage` and `unparsedLanguages`. + ## Query commands `query`, `callers`, `callees`, and `impact` all accept `--json` for machine-readable output. diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts index d9a91aa8a..f8d5fd01e 100644 --- a/src/bin/codegraph.ts +++ b/src/bin/codegraph.ts @@ -75,7 +75,7 @@ import { lookupSymbolNodes, describeSymbolNode, groupDefinitions } from '../grap import { callSiteLines, formatCallSiteLines } from '../graph/call-sites'; import { goPackageReaderFor } from '../graph/go-package'; import { groupFilesByPackage, packageGroupTitle, packageListingTitle } from '../graph/owning-package'; -import type { Node, Edge } from '../types'; +import type { Node, Edge, Language } from '../types'; import { isTestPath } from '../search/query-utils'; // Decided once, before `--color`/`--no-color` are stripped from argv below @@ -1092,6 +1092,11 @@ program } }); +/** Up to three of `paths`, and how many more there are. */ +function samplePaths(paths: string[]): string { + return paths.slice(0, 3).join(', ') + (paths.length > 3 ? ` (+${formatNumber(paths.length - 3)} more)` : ''); +} + /** * The `status` warnings for indexed files whose symbols are missing although * their content is current (#2335, #2336): one line per group, saying what is @@ -1099,21 +1104,19 @@ program */ function describeFilesMissingSymbols(health: { needsReindex: string[]; parseErrors: string[] }): string[] { const dash = getGlyphs().dash; - const sample = (paths: string[]): string => - paths.slice(0, 3).join(', ') + (paths.length > 3 ? ` (+${formatNumber(paths.length - 3)} more)` : ''); const lines: string[] = []; const reindex = health.needsReindex.length; if (reindex > 0) { lines.push( `${formatNumber(reindex)} ${reindex === 1 ? 'file is missing its' : 'files are missing their'} symbols ${dash} ` + - `run "codegraph sync" to re-index ${reindex === 1 ? 'it' : 'them'}: ${sample(health.needsReindex)}` + `run "codegraph sync" to re-index ${reindex === 1 ? 'it' : 'them'}: ${samplePaths(health.needsReindex)}` ); } const unparsed = health.parseErrors.length; if (unparsed > 0) { lines.push( `${formatNumber(unparsed)} ${unparsed === 1 ? 'file could not be parsed, so its symbols are' : 'files could not be parsed, so their symbols are'} ` + - `missing: ${sample(health.parseErrors)} ${dash} "codegraph files --json" shows the errors` + `missing: ${samplePaths(health.parseErrors)} ${dash} "codegraph files --json" shows the errors` ); } return lines; @@ -1126,7 +1129,8 @@ program .command('status [path]') .description('Show index status and statistics') .option('-j, --json', 'Output as JSON') - .action(async (pathArg: string | undefined, options: { json?: boolean }) => { + .option('--strict', 'Exit 1 unless the index is finished, up to date, and every language that should yield symbols did') + .action(async (pathArg: string | undefined, options: { json?: boolean; strict?: boolean }) => { const projectPath = resolveProjectPath(pathArg); // The directory the user actually ran from, before walking up to the index // root. Used to detect when the resolved index lives in a different git @@ -1136,6 +1140,8 @@ program try { if (!isInitialized(projectPath)) { + // A gate with no index to read has proved nothing. + if (options.strict) process.exitCode = 1; if (options.json) { console.log(JSON.stringify({ initialized: false, @@ -1183,6 +1189,24 @@ program // Files whose content is current but whose symbols are missing — the // content-hash check behind "up to date" cannot see them (#2336). const health = cg.getIndexHealth(); + // Files found against files that yielded symbols: a language can be + // listed with its full count and have nothing in the graph. + const coverage = cg.getLanguageCoverage(); + + // Reached by every output mode below. A grammar that can't load leaves no + // row and no zero to count, only files the index doesn't have yet, so + // "up to date" is part of the gate as much as the language check is. + // Parse errors are not: one bad fixture shouldn't fail a whole project. + const totalChanges = changes.added.length + changes.modified.length + changes.removed.length; + if (options.strict && ( + rebuildInProgress || + indexState === 'indexing' || indexState === 'partial' || indexState === 'failed' || + totalChanges > 0 || + health.needsReindex.length > 0 || + coverage.unparsedLanguages.length > 0 + )) { + process.exitCode = 1; + } // JSON output mode if (options.json) { @@ -1202,6 +1226,11 @@ program journalMode, nodesByKind: stats.nodesByKind, languages: Object.entries(stats.filesByLanguage).filter(([, count]) => count > 0).map(([lang]) => lang), + filesByLanguage: stats.filesByLanguage, + // Files that yielded a symbol, for the languages expected to. + parsedFilesByLanguage: coverage.parsedFilesByLanguage, + // Of those, the languages with files and no symbols at all. + unparsedLanguages: coverage.unparsedLanguages, pendingChanges: { added: changes.added.length, modified: changes.modified.length, @@ -1313,7 +1342,9 @@ program .filter(([, count]) => count > 0) .sort((a, b) => b[1] - a[1]); for (const [lang, count] of filesByLang) { - console.log(` ${lang.padEnd(15)} ${formatNumber(count)}`); + const parsed = coverage.parsedFilesByLanguage[lang as Language]; + const withSymbols = parsed === undefined || parsed === count ? '' : ` (${formatNumber(parsed)} with symbols)`; + console.log(` ${lang.padEnd(15)} ${formatNumber(count)}${chalk.dim(withSymbols)}`); } console.log(); @@ -1324,8 +1355,19 @@ program return; } + // Worded for both causes: a lone config stub with nothing to extract + // (a `.nvim.lua`, a comment-only `site.js`) reads exactly like a parser + // that isn't working, and the file names are what tell them apart. + if (coverage.unparsedLanguages.length > 0) { + const files = cg.getFiles(); + for (const language of coverage.unparsedLanguages) { + const paths = files.filter((file) => file.language === language).map((file) => file.path); + warn(`No symbols came from any ${language} file ${getGlyphs().dash} fine if they hold no code, otherwise its parser isn't working: ${samplePaths(paths)}`); + } + console.log(); + } + // Pending changes - const totalChanges = changes.added.length + changes.modified.length + changes.removed.length; if (totalChanges > 0) { console.log(chalk.bold('Pending Changes:')); if (changes.added.length > 0) { diff --git a/src/db/queries.ts b/src/db/queries.ts index 7c794bc32..2e242d7ca 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -5593,6 +5593,30 @@ export class QueryBuilder { .get() as { nodes: number; edges: number }; } + /** + * For each language, how many files it has and how many yielded a symbol: a + * node other than the file's own. More than one stored node settles a row, so + * `nodes` is only probed for the few it can't, such as a file whose one symbol + * came from a framework pass. Probing every row costs as much as the whole of + * getStats() on a 14,000-file index; this costs a tenth of that. Not part of + * getStats() either way, because every explore runs that. + */ + getParsedFileCounts(): Array<{ language: Language; files: number; parsed: number }> { + return this.db + .prepare(` + SELECT f.language AS language, COUNT(*) AS files, + SUM(CASE + WHEN f.node_count > 1 THEN 1 + WHEN EXISTS (SELECT 1 FROM nodes n WHERE n.file_path = f.path AND n.kind <> 'file') THEN 1 + ELSE 0 + END) AS parsed + FROM files f + GROUP BY f.language + ORDER BY f.language + `) + .all() as Array<{ language: Language; files: number; parsed: number }>; + } + /** * Get graph statistics */ diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 66c69687b..44a18994d 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -837,6 +837,15 @@ export function isFileLevelOnlyLanguage(language: Language): boolean { return language === 'yaml' || language === 'twig' || language === 'properties'; } +/** + * Whether files of this language are expected to yield symbols, so that a + * language with none is news. Not so for the file-level languages, nor for + * `xml`, where only MyBatis mappers yield anything. + */ +export function yieldsSymbols(language: Language): boolean { + return !isFileLevelOnlyLanguage(language) && language !== 'xml' && language !== 'unknown'; +} + /** * Get all supported languages (those with grammar definitions). */ diff --git a/src/index.ts b/src/index.ts index c78458c93..777f5efd9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,6 +26,7 @@ import { FindRelevantContextOptions, UnresolvedReference, IndexHealth, + LanguageCoverage, Language, } from './types'; import { DatabaseConnection, getDatabasePath, removeDatabaseFiles } from './db'; @@ -46,7 +47,7 @@ import { extractFromSource, initGrammars, } from './extraction'; -import { detectLanguage, hasGrammarLoadFailure, isFileLevelOnlyLanguage } from './extraction/grammars'; +import { detectLanguage, hasGrammarLoadFailure, isFileLevelOnlyLanguage, yieldsSymbols } from './extraction/grammars'; import { ReferenceResolver, changedRoutes, @@ -2770,6 +2771,21 @@ export class CodeGraph { return { needsReindex, parseErrors }; } + /** + * Files found against files that yielded symbols, by language. Reported by + * `status`, whose `--strict` fails on a language that yielded none. + */ + getLanguageCoverage(): LanguageCoverage { + const parsedFilesByLanguage: LanguageCoverage['parsedFilesByLanguage'] = {}; + const unparsedLanguages: Language[] = []; + for (const { language, parsed } of this.queries.getParsedFileCounts()) { + if (!yieldsSymbols(language)) continue; + parsedFilesByLanguage[language] = parsed; + if (parsed === 0) unparsedLanguages.push(language); + } + return { parsedFilesByLanguage, unparsedLanguages }; + } + // =========================================================================== // Graph Query Methods // =========================================================================== diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 3be1f39b8..080e4f34a 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -35,7 +35,7 @@ import { type WorktreeIndexMismatch, } from '../sync/worktree'; import type { PendingFile } from '../sync'; -import type { Node, Edge, SearchResult, Subgraph, NodeKind } from '../types'; +import type { Node, Edge, SearchResult, Subgraph, NodeKind, Language } from '../types'; import { isTestFile, normalizeNameToken, splitAtCjkSwitches, isQueryNameToken } from '../search/query-utils'; import { groupDefinitions, isQualifiedSymbol, lastQualifierPart, matchesSymbol, qualifiedSymbolMatches } from '../graph/symbol-lookup'; import { goPackageReaderFor } from '../graph/go-package'; @@ -8548,12 +8548,23 @@ export class ToolHandler { } } + // Beside the file count, how many yielded symbols: it is what tells a + // language that parsed from one that is only listed. + const coverage = cg.getLanguageCoverage(); lines.push('', '**Languages:**'); for (const [lang, count] of Object.entries(stats.filesByLanguage)) { if ((count as number) > 0) { - lines.push(`- ${lang}: ${count}`); + const parsed = coverage.parsedFilesByLanguage[lang as Language]; + lines.push(parsed === undefined || parsed === count ? `- ${lang}: ${count}` : `- ${lang}: ${count} (${parsed} with symbols)`); } } + if (coverage.unparsedLanguages.length > 0) { + lines.push( + '', + `**No symbols from:** ⚠ ${coverage.unparsedLanguages.join(', ')} ` + + `— those files are indexed, but nothing in them can be searched or traced; Read them directly.` + ); + } // Whole-index degradation (#876): when live watching has permanently // stopped, getPendingFiles() is empty (so no "Pending sync" section below) diff --git a/src/types.ts b/src/types.ts index 9c4c4461f..05313675f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -272,6 +272,21 @@ export interface FileRecord { generated?: boolean; } +/** + * Files found against files that yielded symbols, by language, for `status`. + * `GraphStats.filesByLanguage` counts files by what they are, so on its own it + * reads the same for a language that parsed and one that yielded nothing. + */ +export interface LanguageCoverage { + /** + * Files that yielded a symbol, for the languages expected to. A language + * tracked at file level (yaml, most xml) has no entry: nothing is missing. + */ + parsedFilesByLanguage: Partial>; + /** Languages expected to yield symbols that have files, none of which did. */ + unparsedLanguages: Language[]; +} + /** * Indexed files whose symbols are missing even though their content hash is * current — what a hash comparison alone calls "up to date" (#2336). Paths are