Files
e2e/scripts/benchmark-coverage.ts
Dery FerdikaandDonny/강동윤 6d1ce27d47 test(benchmark): check every scenario has its test (#1791)
A coverage check in the pnpm check chain and its own step in spec.yml's check job: every web slug keeps its tests/<slug>.e2e.ts twin, and every mobile scenario its tests-agent entry outside the documented Control Inventory exclusion. Both catalogs parse in either quote style; a catalog that parses to no scenarios fails the check, a `slug:` or `name:` property the parser cannot read (a template literal, a bare identifier) fails it too, and a commented-out `name:` no longer counts as an entry, so a format change or a disabled entry cannot pass the check by hiding a scenario. The web agentic side stays unchecked on purpose: Refs #1777 leaves that part to the issue. file-round-trip gains the twin test the check was missing, opening the page in beforeEach, asserting the input resets after a rejected upload, and covering the voucher that then verifies.

Co-authored-by: Donny/강동윤 <kdy.1997.dev@gmail.com>
2026-11-20 01:35:11 +00:00

116 lines
4.8 KiB
TypeScript

/**
* The pairing rules the benchmark READMEs promise, as functions a test can
* call: every web scenario has a deterministic twin, and every mobile scenario
* has an agentic entry. The runner (`check-benchmark-coverage.ts`) reads the
* catalogs and suites and reports the gaps; this module holds the parsing and
* the comparison so the rules are covered beside the other check helpers.
*/
/** The scenario slugs a web catalog source declares, in file order: every quoted `slug:` property, in either quote style. */
export function webSlugs(source: string): string[] {
return [...source.matchAll(/\bslug:\s*(['"])(.*?)\1/g)].map((match) => match[2]!);
}
/** The scenario names a mobile catalog source declares, in file order: every quoted `name:` property, whatever else the entry orders around it. */
export function mobileNames(source: string): string[] {
return [...source.matchAll(/\bname:\s*(['"])(.*?)\1/g)].map((match) => match[2]!);
}
/**
* The `slug:` properties a web catalog declares that `webSlugs` cannot read,
* as problems, so a reformatted entry fails the check instead of quietly
* hiding a scenario.
*/
export function unreadableWebSlugs(source: string): string[] {
return unreadable(source, 'slug', 'web');
}
/** The `name:` properties a mobile catalog declares that `mobileNames` cannot read, as problems, like `unreadableWebSlugs`. */
export function unreadableMobileNames(source: string): string[] {
return unreadable(source, 'name', 'mobile');
}
/** Every `key:` property in `source` whose value is neither a quoted string nor a type declaration's `key: <type>;` field. */
function unreadable(source: string, key: 'slug' | 'name', side: 'web' | 'mobile'): string[] {
return [...source.matchAll(new RegExp(`\\b${key}\\s*:([^\\n]*)`, 'g'))]
.map((match) => (match[1] ?? '').trim())
.filter((value) => !/^(['"])[^\n]*?\1/.test(value) && !/^[A-Za-z_][\w.$<>[\]|]*\s*;$/.test(value))
.map((value) => `${side}: the catalog declares a ${key} this check cannot read: ${bounded(value)}; write it as a quoted ${key}, such as ${key}: "name"`);
}
/** `value` up to the entry's separator, short enough for one line of output. */
function bounded(value: string): string {
const [first = ''] = value.split(/[,}]/);
const trimmed = first.trim();
return trimmed.length > 40 ? `${trimmed.slice(0, 40)}...` : trimmed;
}
/** One web pairing rule: the catalog's slugs and the deterministic suite's files. */
export interface WebCoverage {
readonly slugs: readonly string[];
readonly testFiles: ReadonlySet<string>;
}
/**
* The web benchmark's gaps: a catalog slug with no `tests/<slug>.e2e.ts`.
* The agentic suite is intentionally partial (no web agentic check), only
* the deterministic twin the README promises every scenario.
*/
export function webProblems(coverage: WebCoverage): string[] {
const problems: string[] = [];
for (const slug of coverage.slugs) {
if (!coverage.testFiles.has(`${slug}.e2e.ts`)) {
problems.push(`web: ${slug} has no tests/${slug}.e2e.ts; the README promises every scenario a deterministic twin`);
}
}
return problems;
}
/** One mobile pairing rule: the catalog's names, the agentic suite's source, and the names allowed to have no agentic entry. */
export interface MobileCoverage {
readonly names: readonly string[];
readonly scenariosSource: string;
readonly withoutAgent: readonly string[];
}
/**
* The mobile benchmark's gaps: a catalog name with no entry in
* `tests-agent/scenarios.e2e.ts`, outside the names allowed to have none.
*/
export function mobileProblems(coverage: MobileCoverage): string[] {
const excluded = new Set(coverage.withoutAgent);
return coverage.names
.filter((name) => !excluded.has(name) && !declaresName(coverage.scenariosSource, name))
.map((name) => `mobile: ${name} has no entry in tests-agent/scenarios.e2e.ts; the README promises one agent.act per scenario`);
}
/**
* Whether `source` declares a `name` property for `name`, in either quote
* style and not commented out, so formatting or a commented-out entry never
* hides one. Comment lines are dropped first, which is enough for a catalog
* that keeps one property per line.
*/
function declaresName(source: string, name: string): boolean {
return mobileNames(uncommented(source)).includes(name);
}
/** `source` without its `//` line comments and its block comments, which stand in as empty lines. */
function uncommented(source: string): string {
let inBlock = false;
return source
.split('\n')
.map((line) => {
const trimmed = line.trimStart();
if (inBlock) {
inBlock = !trimmed.includes('*/');
return '';
}
if (trimmed.startsWith('/*')) {
inBlock = !trimmed.includes('*/');
return '';
}
return trimmed.startsWith('//') ? '' : line;
})
.join('\n');
}