mirror of
https://github.com/tester-army/e2e.git
synced 2026-11-22 07:55:36 +08:00
A coverage check in the pnpm check chain and its own step in spec.yml's check job: every web slug keeps its tests/<slug>.e2e.ts twin, and every mobile scenario its tests-agent entry outside the documented Control Inventory exclusion. Both catalogs parse in either quote style; a catalog that parses to no scenarios fails the check, a `slug:` or `name:` property the parser cannot read (a template literal, a bare identifier) fails it too, and a commented-out `name:` no longer counts as an entry, so a format change or a disabled entry cannot pass the check by hiding a scenario. The web agentic side stays unchecked on purpose: Refs #1777 leaves that part to the issue. file-round-trip gains the twin test the check was missing, opening the page in beforeEach, asserting the input resets after a rejected upload, and covering the voucher that then verifies. Co-authored-by: Donny/강동윤 <kdy.1997.dev@gmail.com>
57 lines
2.6 KiB
TypeScript
57 lines
2.6 KiB
TypeScript
/**
|
|
* Fails when a benchmark scenario ships without its test.
|
|
*
|
|
* `apps/web-benchmark/README.md` promises every scenario a deterministic twin
|
|
* (`tests/<slug>.e2e.ts`), and `apps/mobile-benchmark/README.md` promises one
|
|
* `agent.act` per scenario. The web side checks the deterministic twin only,
|
|
* since the web agentic suite is intentionally partial; the mobile side
|
|
* checks the agentic list with one documented exception, the Control
|
|
* Inventory the agentic suite deliberately leaves out (a new exception needs
|
|
* a written reason). A catalog whose parse finds no scenarios fails too, so a
|
|
* format change cannot pass the check by checking nothing.
|
|
*
|
|
* Usage: `node scripts/check-benchmark-coverage.ts`, part of `pnpm check`.
|
|
* Exits 1 listing every gap.
|
|
*/
|
|
|
|
import { readdirSync, readFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { mobileNames, mobileProblems, unreadableMobileNames, unreadableWebSlugs, webProblems, webSlugs } from './benchmark-coverage.ts';
|
|
|
|
const ROOT = fileURLToPath(new URL('..', import.meta.url));
|
|
|
|
/** Mobile names the agentic suite deliberately leaves out, from `apps/mobile-benchmark/README.md`. */
|
|
const MOBILE_WITHOUT_AGENT = ['Control Inventory'];
|
|
|
|
const webDir = join(ROOT, 'apps', 'web-benchmark');
|
|
const mobileDir = join(ROOT, 'apps', 'mobile-benchmark');
|
|
|
|
const webSource = readFileSync(join(webDir, 'app', 'src', 'examples.ts'), 'utf8');
|
|
const testFiles = new Set(readdirSync(join(webDir, 'tests')).filter((file) => file.endsWith('.e2e.ts')));
|
|
|
|
const mobileSource = readFileSync(join(mobileDir, 'src', 'examples.ts'), 'utf8');
|
|
const scenariosSource = readFileSync(join(mobileDir, 'tests-agent', 'scenarios.e2e.ts'), 'utf8');
|
|
|
|
const slugs = webSlugs(webSource);
|
|
const names = mobileNames(mobileSource);
|
|
|
|
if (slugs.length === 0 || names.length === 0) {
|
|
const empty = [slugs.length === 0 ? 'web' : undefined, names.length === 0 ? 'mobile' : undefined].filter(Boolean).join(' and ');
|
|
process.stderr.write(`benchmark coverage: the ${empty} catalog parses to no scenarios; nothing was checked\n`);
|
|
process.exitCode = 1;
|
|
} else {
|
|
const problems = [
|
|
...unreadableWebSlugs(webSource),
|
|
...unreadableMobileNames(mobileSource),
|
|
...webProblems({ slugs, testFiles }),
|
|
...mobileProblems({ names, scenariosSource, withoutAgent: MOBILE_WITHOUT_AGENT }),
|
|
];
|
|
if (problems.length > 0) {
|
|
process.stderr.write(`${problems.join('\n')}\n`);
|
|
process.exitCode = 1;
|
|
} else {
|
|
process.stdout.write(`benchmark coverage: ${slugs.length} web scenarios and ${names.length} mobile scenarios have their tests\n`);
|
|
}
|
|
}
|