OpenMAIC/scripts/assert-pg-contract-suites.mjs

import { createRequire } from 'node:module';
import { readFileSync, writeFileSync } from 'node:fs';

/**
 * Assert that @openmaic/storage's PostgreSQL contract suites really exercised a
 * real PostgreSQL, from evidence that cannot be produced by editing the tests.
 *
 * WHY THIS EXISTS AT ALL. The suites already refuse to skip when
 * STORAGE_PG_CONTRACT_REQUIRED=1, but that refusal is a `throw` inside the test
 * modules, so it cannot fire if vitest stops collecting them. Collection is
 * decided by `packages/@openmaic/storage/vitest.config.ts`, and both that file
 * and the whole `test/` directory are on the ignore list of publishable inputs
 * in `check-package-version-bumps.mjs`. The entire surface those suites live on
 * can therefore be rewritten with no version bump and no release gate noticing.
 *
 * WHY THE VITEST RESULTS ARE NOT ENOUGH ON THEIR OWN. `assertionResults` are
 * test CASES, not `expect()` calls, so a file named `pg-document-store.pg.test.ts`
 * containing nothing but `test('x', () => {})` satisfies every check in phase 1
 * below. And because `test/setup.ts` is already wired as `setupFiles`, a single
 * `vi.mock('pg', ...)` there makes both suites collect, run and pass green
 * against an in-memory fake. Both edits are one line, in `test/`, and need no
 * version bump. Phase 1 therefore proves only that two files with those names
 * ran and reported passing cases — nothing whatsoever about a database.
 *
 * WHAT PHASE 2 ADDS. It connects to the contract database from OUTSIDE the
 * vitest process and asks PostgreSQL itself what happened: the five tables
 * these two backends own must exist, and each must have gained inserts DURING
 * the run. Nothing inside `test/` can forge that, because producing it requires
 * actually writing to the database this script independently connects to.
 *
 * Insert counters rather than surviving rows: the suites clean up after
 * themselves, so counting rows would prove nothing, while `n_tup_ins` survives
 * the cleanup. Counted as a delta against a baseline captured before the run
 * rather than as an absolute, so a non-ephemeral database cannot satisfy the
 * check forever on the strength of some earlier run. Both current workflows use
 * a fresh per-job service container, but this check should not depend on that
 * staying true.
 *
 * ── THREAT MODEL, STATED HONESTLY ────────────────────────────────────────────
 *
 * What this proves: during this run, rows were inserted into those five tables
 * in a real PostgreSQL, and two files with the contract suites' names ran and
 * passed.
 *
 * What it does NOT prove: that the built `PgDocumentStore` and `PgRuntimeStore`
 * were the code that inserted them. The whole `test/` directory is on the
 * publishable-input ignore list, so test code can create the schema and insert
 * directly, and this audit would read the same either way. Closing that needs a
 * harness living outside the ignored `test/` surface — separate work, not
 * attempted here.
 *
 * That limit is acceptable because of who each threat is. This guard exists to
 * catch ACCIDENTAL silencing: a vitest `include`/`exclude` change, a missing
 * environment variable, a renamed suite file, a dropped workflow step. Those
 * are the ways this coverage actually disappears, and they are all caught.
 * It is not a defence against someone deliberately faking coverage from inside
 * `test/` — and it does not need to be, because that person can merge changes
 * to the production sources just as easily.
 */

const REQUIRED_SUITES = [
  'packages/@openmaic/storage/test/pg-document-store.pg.test.ts',
  'packages/@openmaic/storage/test/pg-runtime-store.pg.test.ts',
];

/**
 * The tables created by `DOCUMENT_PG_SCHEMA` and `RUNTIME_PG_SCHEMA`. Kept
 * explicit rather than parsed out of those sources: this list is the
 * independent statement of what the contract must have touched, and deriving it
 * from the code under test would let that code narrow its own audit.
 */
const REQUIRED_TABLES = [
  'document_stages',
  'document_scenes',
  'document_outlines',
  'runtime_sessions',
  'runtime_records',
];

const usage = [
  'Usage:',
  '  assert-pg-contract-suites.mjs --capture-baseline <file>',
  '      Record the current insert counters. Run BEFORE the vitest invocation.',
  '  assert-pg-contract-suites.mjs <vitest-json-results> --baseline <file>',
  '      Audit the run against that baseline. Run AFTER the vitest invocation.',
].join('\n');

// Parsed by consuming each flag with its value, so the remaining positionals
// are exactly the positionals. Scanning for "the first argument that does not
// start with --" would silently accept a flag's value as the results path.
const argv = process.argv.slice(2);
let capturingBaseline = false;
let baselinePath;
const positionals = [];
for (let i = 0; i < argv.length; i += 1) {
  const arg = argv[i];
  if (arg === '--baseline' || arg === '--capture-baseline') {
    capturingBaseline ||= arg === '--capture-baseline';
    i += 1;
    baselinePath = argv[i];
    if (baselinePath === undefined) {
      console.error(`${arg} needs a file path.\n\n${usage}`);
      process.exit(2);
    }
    continue;
  }
  if (arg.startsWith('--')) {
    console.error(`Unknown option ${arg}.\n\n${usage}`);
    process.exit(2);
  }
  positionals.push(arg);
}

const expectedPositionals = capturingBaseline ? 0 : 1;
if (!baselinePath || positionals.length !== expectedPositionals) {
  console.error(usage);
  process.exit(2);
}
const resultsPath = positionals[0];

const contractUrl = process.env.PG_CONTRACT_URL;
if (!contractUrl) {
  console.error(
    'PG_CONTRACT_URL is unset, so there is no database to audit and this check cannot ' +
      'establish that the PostgreSQL contract ran. Invoke it in the same job, with the ' +
      'same PG_CONTRACT_URL, as the vitest run it is auditing.',
  );
  process.exit(2);
}

// The database side -----------------------------------------------------------

// `pg` is a devDependency of @openmaic/storage, not of the repository root, so
// resolve it from the package that owns it rather than assuming hoisting.
const requireFromStorage = createRequire(
  new URL('../packages/@openmaic/storage/package.json', import.meta.url),
);
const { Client } = requireFromStorage('pg');

async function collectInsertCounts(client) {
  const { rows } = await client.query(
    `SELECT t.relname AS table_name,
            to_regclass('public.' || t.relname) IS NOT NULL AS present,
            COALESCE(s.n_tup_ins, 0)::bigint AS inserts
       FROM unnest($1::text[]) AS t(relname)
       LEFT JOIN pg_stat_user_tables s
              ON s.schemaname = 'public' AND s.relname = t.relname`,
    [REQUIRED_TABLES],
  );
  return Object.fromEntries(
    rows.map((row) => [row.table_name, { present: row.present, inserts: Number(row.inserts) }]),
  );
}

/**
 * Read the counters, optionally waiting for them to move past `baseline`.
 *
 * Backends flush statistics at transaction end and on exit, so by the time
 * vitest has returned they are normally already visible. Re-read a few times
 * anyway rather than racing a slow flush, discarding the per-session snapshot
 * each round because a backend caches it for the whole transaction. A flush
 * that never arrives fails the check rather than passing it.
 */
async function readCounters({ waitFor } = {}) {
  const client = new Client({ connectionString: contractUrl });
  try {
    await client.connect();
    let counts = await collectInsertCounts(client);
    if (!waitFor) return counts;
    for (let attempt = 1; attempt < 5; attempt += 1) {
      const unproven = REQUIRED_TABLES.filter(
        (table) => !(counts[table]?.inserts > (waitFor[table]?.inserts ?? 0)),
      );
      if (unproven.length === 0) break;
      await new Promise((resolve) => setTimeout(resolve, 500));
      await client.query('SELECT pg_stat_clear_snapshot()');
      counts = await collectInsertCounts(client);
    }
    return counts;
  } catch (error) {
    console.error(
      `Cannot reach the contract database at PG_CONTRACT_URL: ${error.message}. ` +
        'Without it there is no evidence the suites touched a real PostgreSQL.',
    );
    process.exit(2);
  } finally {
    await client.end().catch(() => {});
  }
}

if (capturingBaseline) {
  const baseline = await readCounters();
  writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`);
  const summary = REQUIRED_TABLES.map((table) => `${table}=${baseline[table]?.inserts ?? 0}`);
  console.log(`Captured contract-database baseline to ${baselinePath}: ${summary.join(' ')}.`);
  process.exit(0);
}

// Phase 1 ---------------------------------------------------------------------
// The two suite files were collected and reported passing cases.

let results;
try {
  results = JSON.parse(readFileSync(resultsPath, 'utf8'));
} catch (error) {
  console.error(`Cannot read the vitest results at ${resultsPath}: ${error.message}`);
  process.exit(2);
}

const files = Array.isArray(results.testResults) ? results.testResults : undefined;
if (!files) {
  console.error(
    `${resultsPath} has no testResults array, so it cannot show which suites ran. ` +
      'Was the run invoked with the json reporter?',
  );
  process.exit(2);
}

const failures = [];
for (const suite of REQUIRED_SUITES) {
  // The reporter records absolute paths; match on the repository-relative tail.
  const entry = files.find((file) => typeof file.name === 'string' && file.name.endsWith(suite));
  if (!entry) {
    failures.push(
      `${suite} did not run, so the PostgreSQL contract is unverified. ` +
        "Check vitest's `include`/`exclude` in packages/@openmaic/storage/vitest.config.ts.",
    );
    continue;
  }
  const cases = Array.isArray(entry.assertionResults) ? entry.assertionResults : [];
  const passed = cases.filter((testCase) => testCase.status === 'passed').length;
  const pending = cases.filter((testCase) => testCase.status !== 'passed');
  if (entry.status !== 'passed') {
    failures.push(`${suite} reported status "${entry.status}".`);
    continue;
  }
  if (passed === 0) {
    failures.push(`${suite} ran but reported no passing test cases, so it asserted nothing.`);
    continue;
  }
  if (pending.length > 0) {
    failures.push(
      `${suite} left ${pending.length} test case(s) not passing ` +
        `(${[...new Set(pending.map((testCase) => testCase.status))].join(', ')}).`,
    );
    continue;
  }
  console.log(`${suite}: ${passed} test cases ran and passed.`);
}

if (failures.length > 0) {
  console.error(
    [
      'The PostgreSQL contract suites did not run as required:',
      ...failures.map((failure) => `- ${failure}`),
    ].join('\n'),
  );
  process.exit(1);
}

// Phase 2 ---------------------------------------------------------------------
// PostgreSQL's own account of what changed during the run.

let baseline;
try {
  baseline = JSON.parse(readFileSync(baselinePath, 'utf8'));
} catch (error) {
  console.error(
    `Cannot read the pre-run baseline at ${baselinePath}: ${error.message}. Capture it with ` +
      '`--capture-baseline` before the vitest step; without it, counters left by an earlier ' +
      'run against a non-ephemeral database would satisfy this check forever.',
  );
  process.exit(2);
}

const counts = await readCounters({ waitFor: baseline });

const databaseFailures = [];
for (const table of REQUIRED_TABLES) {
  const observed = counts[table];
  const before = baseline[table]?.inserts ?? 0;
  if (!observed?.present) {
    databaseFailures.push(
      `${table} does not exist in the contract database, so the suites never created it ` +
        'and did not run against this PostgreSQL.',
    );
    continue;
  }
  const gained = observed.inserts - before;
  if (!(gained > 0)) {
    databaseFailures.push(
      `${table} exists but gained no inserts during this run (before ${before}, after ` +
        `${observed.inserts}), so the suites did not write to this database. Check for a ` +
        'mocked driver in packages/@openmaic/storage/test/setup.ts, or a stubbed-out suite body.',
    );
    continue;
  }
  console.log(`${table}: ${gained} inserts during this run (${before} -> ${observed.inserts}).`);
}

if (databaseFailures.length > 0) {
  console.error(
    [
      'The contract database shows no evidence that these suites wrote to it during this run:',
      ...databaseFailures.map((failure) => `- ${failure}`),
    ].join('\n'),
  );
  process.exit(1);
}

console.log(
  `Verified: both contract suite files ran and passed, and all ${REQUIRED_TABLES.length} tables ` +
    'the two PostgreSQL backends own gained inserts in a real database during this run. ' +
    'This does not attribute those inserts to the built PgDocumentStore / PgRuntimeStore ' +
    'specifically — see the threat model at the top of this script.',
);