diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index 055500fdb..3158de997 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -478,10 +478,13 @@ export function compareCounts(baseline, candidate) { /** One program's verdict from its build results and artifact comparisons. * `allowed` marks programs that match the change's pattern signature. */ -export function programVerdict({ baseline, candidate, artifacts, allowed }) { +/** `required` programs must build with both checkouts: a failure both share + * means the gate compared nothing for them. Other programs may fail the same + * way on both sides, as a corpus program the compiler refuses does. */ +export function programVerdict({ baseline, candidate, artifacts, allowed, required = false }) { if (!baseline.ok || !candidate.ok) { if (!baseline.ok && !candidate.ok && baseline.error === candidate.error) - return { verdict: "both-fail", fails: false }; + return { verdict: "both-fail", fails: required }; return { verdict: "build-error", fails: true }; } const differing = Object.entries(artifacts).filter(([, r]) => !r.identical); diff --git a/scripts/zero-cost-gate.mjs b/scripts/zero-cost-gate.mjs index 1575fc5ee..99b1bc468 100644 --- a/scripts/zero-cost-gate.mjs +++ b/scripts/zero-cost-gate.mjs @@ -448,7 +448,15 @@ await pool( allow !== null && programInputs(program.entry).some((f) => allow.test(readFileSync(f, "utf8"))); const artifacts = baseline.ok && candidate.ok ? compareBuilds(baseline, candidate) : {}; - const { verdict, fails } = programVerdict({ baseline, candidate, artifacts, allowed }); + // The benchmark workloads and the self-hosted compiler always build. + const required = !program.id.startsWith("corpus/"); + const { verdict, fails } = programVerdict({ + baseline, + candidate, + artifacts, + allowed, + required, + }); const entry = { id: program.id, verdict, @@ -536,9 +544,19 @@ function printProgram(entry) { } else if (entry.cachegrind?.skipped) notes.push(`cachegrind skipped: ${entry.cachegrind.skipped}`); for (const failure of entry.dynamicTier?.failures ?? []) notes.push(`SYMBOLS: ${failure}`); - if (entry.baselineError) notes.push(`baseline: ${entry.baselineError.split("\n")[0]}`); - if (entry.candidateError) notes.push(`candidate: ${entry.candidateError.split("\n")[0]}`); console.log(`${label} ${entry.id}${notes.length ? ` ${notes.join("; ")}` : ""}`); + // A build error's first line is often a warning; print all of it. + const errors = + entry.baselineError !== undefined && entry.baselineError === entry.candidateError + ? [["baseline and candidate", entry.baselineError]] + : [ + ["baseline", entry.baselineError], + ["candidate", entry.candidateError], + ].filter(([, error]) => error !== undefined); + for (const [side, error] of errors) { + console.log(` ${side}:`); + for (const line of error.split("\n")) console.log(` ${line}`); + } } /* Compile time: paired --emit=llvm builds, the baseline and the candidate @@ -613,6 +631,10 @@ if (timingRuns > 0) { const counts = {}; for (const p of report.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1; +if (!report.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict))) { + failed = true; + report.failures.push("no program built with both checkouts, so nothing was compared"); +} report.counts = counts; report.verdict = failed ? "fail" : "pass"; report.elapsedMs = Date.now() - started; diff --git a/tests/harness/zero-cost-gate.test.ts b/tests/harness/zero-cost-gate.test.ts index 3da134010..565f8ba71 100644 --- a/tests/harness/zero-cost-gate.test.ts +++ b/tests/harness/zero-cost-gate.test.ts @@ -179,11 +179,15 @@ test("the zero-cost gate fails differing programs unless they match the signatur allowed = false, baseline: object = ok, candidate: object = ok, - ) => programVerdict({ baseline, candidate, artifacts, allowed }); + required = false, + ) => programVerdict({ baseline, candidate, artifacts, allowed, required }); expect(verdict(same)).toEqual({ verdict: "identical", fails: false }); expect(verdict(differs)).toEqual({ verdict: "differs", fails: true }); expect(verdict(differs, true)).toEqual({ verdict: "allowed", fails: false }); const failed = { ok: false, error: "SC1090" }; expect(verdict({}, false, failed, failed)).toEqual({ verdict: "both-fail", fails: false }); + // A benchmark or the self-hosted compiler that neither checkout builds + // was never compared. + expect(verdict({}, false, failed, failed, true)).toEqual({ verdict: "both-fail", fails: true }); expect(verdict({}, true, ok, failed)).toEqual({ verdict: "build-error", fails: true }); });