From 2d1b64413381342707fa2e69043a13f1f63f97dd Mon Sep 17 00:00:00 2001 From: Colby Mchenry Date: Wed, 30 Sep 2026 11:55:43 +0000 Subject: [PATCH] fix(extraction): Flow-typed JavaScript is read as TSX (#2161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tree-sitter-javascript can't parse Flow annotations; `render(): React.Node` ended a class early (segmented-control's component had no methods). A `.js` / `.jsx` whose leading comments carry `@flow` (not `@noflow`) now detects as `tsx`, and the TS extractor's offset-preserving preParse blanks Flow-only syntax for those files: exact object `{| |}`, a maybe type's `?`, an inexact object's bare `...`, `import typeof`, `opaque type`. preParse is hoisted before the kernel, so both paths parse the same bytes; kernel-parity now routes .js/.jsx through detectLanguage too (RN core Libraries: 0 diffs over 641 files). Grammar preload adds tsx whenever JavaScript is present. Parse errors over 400 RN-core Flow files: 10,626 (JS grammar) → 391 (TSX). A/B edges: RN core Libraries 29,649 → 38,194 (+12,639 / −4,094, the removals mostly `root.render(…)` → the one DrawerLayoutAndroid::render the old parse left, and edges re-keyed onto recovered methods); segmented-control +35/−5; express and react-native-netinfo byte-identical. Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + __tests__/flow-js.test.ts | 82 ++++++++++++++++++++++++++ scripts/kernel-parity.mjs | 7 ++- src/extraction/grammars.ts | 12 ++++ src/extraction/index.ts | 2 + src/extraction/languages/typescript.ts | 20 +++++++ 6 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 __tests__/flow-js.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ec0b964b..740e4b56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- Flow-typed JavaScript files (those with `@flow` in their header comment) are now parsed like TypeScript, so type annotations like `render(): React.Node` no longer cut classes short. This mostly affects React Native code: on React Native's own libraries the graph gained over 8,000 links, and component classes keep their methods. Files without the pragma are unchanged. Re-index after upgrading. - In Scala, a call reached by its name alone now stays in reach. A name that the enclosing function binds as a parameter or local (`f(true)` for `f: A => B`) is that local. A call later in a chain, like `fa.iterator.map(…)`, reaches only a member of what its receiver is named after. A plain call reaches only the enclosing class and its supertypes (an anonymous subclass's base included), a companion, or an imported object. On cats, parameter calls like `f(a)` used to land on a case class's field `f` over 400 times, and chained `.map(…)` on one ops class's `map`. - A link made by a framework or bridge resolver now records which one made it (`metadata.framework`, such as `swift-objc-bridge` or `react-native-bridge`), as the README describes. Before, only synthesized event and view channels were named. - ASP.NET apps built with FastEndpoints now have their routes: each endpoint class's `Get(…)` / `Post(…)` in `Configure()` becomes a route linked to its own `HandleAsync` / `ExecuteAsync`, including paths kept in a request class's `Route` constant. ardalis/CleanArchitecture went from no routes to 24. Minimal API routes written without a leading slash (`app.MapGet("api/todos", …)`) are now named `/api/todos`. diff --git a/__tests__/flow-js.test.ts b/__tests__/flow-js.test.ts new file mode 100644 index 00000000..1c77e147 --- /dev/null +++ b/__tests__/flow-js.test.ts @@ -0,0 +1,82 @@ +/** + * A Flow-typed `.js` file (`// @flow` in its leading comments) is read with the + * TSX grammar: the JavaScript grammar can't parse its annotations, and a + * `render(): React.Node` cut the class around it short — segmented-control's + * component came out with no methods. Flow's own syntax (`{| |}`, `?T`, an + * inexact object's `...`, `import typeof`, `opaque type`) is blanked first. + * A `.js` without the pragma is untouched. + */ +import { describe, it, expect, afterAll, beforeAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { detectLanguage } from '../src/extraction/grammars'; +import { blankFlowSyntax } from '../src/extraction/languages/typescript'; + +let root = ''; +let cg: CodeGraph; + +const FLOW = `/** + * @flow strict-local + * @format + */ +import typeof * as React from 'react'; +import * as React from 'react'; + +type Props = $ReadOnly<{| + values: ?Array, + onChange?: ?(event: {nativeEvent: {value: string}, ...}) => mixed, +|}>; + +opaque type Token = string; + +class SegmentedControl extends React.Component { + render(): React.Node { + return this._renderItems(); + } + + _renderItems(): React.Node { + return null; + } +} + +module.exports = SegmentedControl; +`; + +beforeAll(async () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-flow-')); + fs.mkdirSync(path.join(root, 'js'), { recursive: true }); + fs.writeFileSync(path.join(root, 'js/SegmentedControl.js'), FLOW); + fs.writeFileSync(path.join(root, 'js/plain.js'), `function plain() { return 1; }\nmodule.exports = plain;\n`); + cg = await CodeGraph.init(root, { index: true }); +}); + +afterAll(() => { + cg?.close(); + if (root) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe('Flow-typed JavaScript', () => { + it('is detected by its pragma, and only then', () => { + expect(detectLanguage('js/SegmentedControl.js', FLOW)).toBe('tsx'); + expect(detectLanguage('js/plain.js', 'function plain() {}')).toBe('javascript'); + expect(detectLanguage('js/x.js', '// @noflow\nconst a = 1;')).toBe('javascript'); + }); + + it('keeps its class and methods', () => { + const nodes = cg.getNodesInFile('js/SegmentedControl.js'); + const methods = nodes.filter((n) => n.kind === 'method').map((n) => n.name).sort(); + expect(methods).toEqual(['_renderItems', 'render']); + const render = nodes.find((n) => n.name === 'render')!; + const calls = cg.getOutgoingEdges(render.id).filter((e) => e.kind === 'calls').map((e) => cg.getNode(e.target)!.name); + expect(calls).toContain('_renderItems'); + }); + + it('blanks Flow-only syntax to spaces, keeping every offset', () => { + const out = blankFlowSyntax(FLOW, 'js/SegmentedControl.js'); + expect(out.length).toBe(FLOW.length); + expect(out).not.toMatch(/\{\||\|\}|import\s+typeof|opaque\s+type|:\s*\?Array/); + expect(blankFlowSyntax('const a = {| b |};', 'plain.js')).toBe('const a = {| b |};'); + }); +}); diff --git a/scripts/kernel-parity.mjs b/scripts/kernel-parity.mjs index db7cfa04..d02ab6d6 100644 --- a/scripts/kernel-parity.mjs +++ b/scripts/kernel-parity.mjs @@ -88,9 +88,12 @@ function collect(p, out) { const lang = EXTS.get(path.extname(p).toLowerCase()); // 'detect' (.h) resolves per file in the run loop; under --lang it rides // along whenever either C-family language is requested. + // A Flow-typed `.js` routes as tsx (detectLanguage reads its pragma), so + // it rides along when tsx is requested and is re-detected per file below. const passes = !langFilter || - (lang === 'detect' ? langFilter.has('c') || langFilter.has('cpp') : langFilter.has(lang)); + (lang === 'detect' ? langFilter.has('c') || langFilter.has('cpp') : langFilter.has(lang)) || + ((lang === 'javascript' || lang === 'jsx') && langFilter.has('tsx')); if (passes) out.push({ file: p, lang }); } } @@ -196,7 +199,7 @@ for (const { file, lang: extLang } of files) { const source = fs.readFileSync(file, 'utf8'); const rel = path.relative(ROOT, file); // `.h` resolves C vs C++ by content — the same call the indexer makes. - const lang = extLang === 'detect' ? detectLanguage(rel, source) : extLang; + const lang = extLang === 'detect' || extLang === 'javascript' || extLang === 'jsx' ? detectLanguage(rel, source) : extLang; if (!KERNEL_LANGS.has(lang)) continue; if (langFilter && !langFilter.has(lang)) continue; processed++; diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 84596d81..f0702a91 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -548,6 +548,11 @@ export function detectLanguage(filePath: string, source?: string, overrides?: Re if (isErlangAppFile(filePath)) return 'erlang'; const lang = (overrides && overrides[ext]) || EXTENSION_MAP[ext] || 'unknown'; + // A Flow-typed `.js` (`// @flow` in its leading comments) parses as TSX: + // the JavaScript grammar can't read its annotations — `render(): React.Node` + // cut a class short — and TypeScript's syntax covers most of Flow's. + if ((lang === 'javascript' || lang === 'jsx') && source && hasFlowPragma(source)) return 'tsx'; + // .h files could be C, C++, or Objective-C — check source content if (lang === 'c' && ext === '.h' && source) { if (looksLikeCpp(source)) return 'cpp'; @@ -557,6 +562,13 @@ export function detectLanguage(filePath: string, source?: string, overrides?: Re return lang; } +/** Whether a JavaScript file's leading comments carry Flow's `@flow` pragma (and not `@noflow`). */ +export function hasFlowPragma(source: string): boolean { + const head = source.slice(0, 4096).replace(/^#![^\n]*\n/, ''); + const lead = /^(?:\s*(?:\/\/[^\n]*|\/\*[\s\S]*?\*\/))*/.exec(head)?.[0] ?? ''; + return /@flow\b/.test(lead) && !/@noflow\b/.test(lead); +} + /** * A class/struct BASE CLAUSE — `struct Derived : Base {`, `class Foo final : * public Bar, private Baz {`, `struct D : ns::B {` — which is never valid diff --git a/src/extraction/index.ts b/src/extraction/index.ts index bb5b9c0f..e71ffabe 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -801,6 +801,8 @@ export function preloadLanguagesForFiles( overrides?: Record ): Language[] { const languages = [...new Set(files.map((f) => detectLanguage(f, undefined, overrides)))]; + // A Flow-typed `.js` is read with the TSX grammar (see detectLanguage). + if ((languages.includes('javascript') || languages.includes('jsx')) && !languages.includes('tsx')) languages.push('tsx'); if (languages.includes('c')) { for (const ambiguous of ['cpp', 'objc'] as const) { if (!languages.includes(ambiguous)) languages.push(ambiguous); diff --git a/src/extraction/languages/typescript.ts b/src/extraction/languages/typescript.ts index c59b4ca7..a56e0205 100644 --- a/src/extraction/languages/typescript.ts +++ b/src/extraction/languages/typescript.ts @@ -1,5 +1,6 @@ import { getNodeText, getChildByField } from '../tree-sitter-helpers'; import type { LanguageExtractor } from '../tree-sitter-types'; +import { hasFlowPragma } from '../grammars'; import type { Node as SyntaxNode } from 'web-tree-sitter'; /** @@ -38,7 +39,26 @@ export function classifyTsClassMember(node: SyntaxNode): 'method' | 'property' { return 'property'; } +/** + * Flow's own syntax, blanked to spaces (offsets survive) so a Flow file reads + * as TSX: exact object types `{| a: T |}`, a maybe type's `?` (`x: ?string`, + * `Array`), an inexact object's `...`, `import typeof`, `opaque type`. Only files with the `@flow` + * pragma — the ones `detectLanguage` sends here from `.js`. + */ +export function blankFlowSyntax(source: string, filePath?: string): string { + if (!filePath || !/\.(?:jsx?|mjs|cjs)$/.test(filePath) || !hasFlowPragma(source)) return source; + return source + .replace(/\{\|/g, '{ ') + .replace(/\|\}/g, ' }') + .replace(/([:<,]\s*)\?(?=[\w$({[])/g, '$1 ') + .replace(/\bimport\s+typeof\b/g, (m) => 'import' + ' '.repeat(m.length - 'import'.length)) + .replace(/\bopaque(?=\s+type\b)/g, ' ') + // An inexact object type's bare `...` (`{a: T, ...}`) — never a value spread. + .replace(/\.\.\.(?=\s*[},])/g, ' '); +} + export const typescriptExtractor: LanguageExtractor = { + preParse: blankFlowSyntax, functionTypes: ['function_declaration', 'generator_function_declaration', 'arrow_function', 'function_expression', 'generator_function'], classTypes: ['class_declaration', 'abstract_class_declaration'], // `method_signature` is the interface/type-literal form of a method; without it