scriptc/tests/harness/ci-test-plan.test.ts

import { expect, test } from "vitest";
import { ciTestPlan, corpusFiles, effectFiles, fileExclusions } from "../../scripts/ci-test-plan.mjs";
import { runCiTestPlan } from "../../scripts/ci-test.mjs";

test.for([1, 3, 5, 10])("CI retains every suite with %i shards without mixing partition axes", (count) => {
  for (const flavor of ["plain", "san"]) {
    const plans = Array.from({ length: count }, (_, index) => ciTestPlan({ shard: `${index + 1}/${count}`, flavor }));
    const sideFiles = plans.flatMap((plan) => plan.side.filter((task) => task.args.length === 2).map((task) => task.args[1]));
    for (const file of fileExclusions.filter((file) => !corpusFiles.includes(file) && !effectFiles.includes(file) && !file.includes("native-driver") && !file.includes("coverage") && !file.includes("native-toolchain"))) {
      expect(sideFiles.filter((selected) => selected === file), file).toHaveLength(1);
    }
    for (const plan of plans) {
      expect(plan.corpus.args).toEqual(["test", ...corpusFiles]);
      expect(plan.side[0]!.args).toContain(`--shard=${plan.corpus.env.SCRIPTC_TEST_SHARD}`);
      for (const file of fileExclusions) expect(plan.side[0]!.args).toContain(`--exclude=${file}`);
      const cache = plan.side.find((task) => task.name === "native-cache correctness");
      expect(Boolean(cache)).toBe(flavor === "plain");
      if (cache) {
        expect(cache.env.SCRIPTC_CACHE_TEST_SHARD).toBe(plan.corpus.env.SCRIPTC_TEST_SHARD);
        expect(cache.env.SCRIPTC_TEST_STABLE_TOOLCHAIN).toBe("0");
      }
      expect(plan.side.some((task) => task.name === "coverage sweep")).toBe(flavor === "plain");
    }
  }
});

test.for([1, 2, 4, 8])("CI overlaps reserved lanes within a budget of %i workers", async (workers) => {
  const plan = ciTestPlan({ shard: "1/10", flavor: "plain", workers });
  const events: string[] = [];
  let active = 0;
  let peak = 0;
  let finishCorpus!: () => void;
  const corpusWait = new Promise<void>((resolve) => { finishCorpus = resolve; });
  await runCiTestPlan(plan, async (task) => {
    const cost = Number(task.env.SCRIPTC_TEST_WORKERS);
    active += cost;
    peak = Math.max(peak, active);
    events.push(`start:${task.name}`);
    if (task === plan.corpus && workers > 1) await corpusWait;
    else await Promise.resolve();
    events.push(`end:${task.name}`);
    active -= cost;
    if (task === plan.side.at(-1)) finishCorpus();
  });
  expect(peak).toBeLessThanOrEqual(workers);
  expect(events.filter((event) => event.startsWith("start:")).sort()).toEqual([plan.corpus, ...plan.side].map((task) => `start:${task.name}`).sort());
  for (let index = 1; index < plan.side.length; index++) {
    expect(events.indexOf(`start:${plan.side[index]!.name}`)).toBeGreaterThan(events.indexOf(`end:${plan.side[index - 1]!.name}`));
  }
  if (workers === 1) expect(events.indexOf(`start:${plan.side[0]!.name}`)).toBeGreaterThan(events.indexOf(`end:${plan.corpus.name}`));
  else expect(events.indexOf(`start:${plan.side[0]!.name}`)).toBeLessThan(events.indexOf(`end:${plan.corpus.name}`));
});

test("CI reports failures after running every scheduled suite", async () => {
  const plan = ciTestPlan({ shard: "1/10", flavor: "plain" });
  const seen: string[] = [];
  let failure: AggregateError | undefined;
  try {
    await runCiTestPlan(plan, async (task) => {
      seen.push(task.name);
      if (task === plan.corpus || task === plan.side[0]) throw new Error(task.name);
    });
  } catch (error) { failure = error as AggregateError; }
  expect(failure).toBeInstanceOf(AggregateError);
  expect(failure!.errors).toHaveLength(2);
  expect(seen.sort()).toEqual([plan.corpus, ...plan.side].map((task) => task.name).sort());
});

test("CI rejects malformed scheduling inputs", () => {
  for (const shard of [undefined, "", "0/10", "11/10", "2/x"]) expect(() => ciTestPlan({ shard, flavor: "plain" })).toThrow();
  for (const workers of [0, -1, 1.5, NaN, Infinity]) expect(() => ciTestPlan({ shard: "1/10", flavor: "plain", workers })).toThrow();
  expect(() => ciTestPlan({ shard: "1/10", flavor: "unknown" })).toThrow();
});