Fail the zero-cost gate when it compares nothing (#1637)

scripts/zero-cost-compare.mjs

@@ -478,10 +478,13 @@ export function compareCounts(baseline, candidate) {
 
 /** One program's verdict from its build results and artifact comparisons.
  * `allowed` marks programs that match the change's pattern signature. */
-export function programVerdict({ baseline, candidate, artifacts, allowed }) {
+/** `required` programs must build with both checkouts: a failure both share
+ * means the gate compared nothing for them. Other programs may fail the same
+ * way on both sides, as a corpus program the compiler refuses does. */
+export function programVerdict({ baseline, candidate, artifacts, allowed, required = false }) {
   if (!baseline.ok || !candidate.ok) {
     if (!baseline.ok && !candidate.ok && baseline.error === candidate.error)
-      return { verdict: "both-fail", fails: false };
+      return { verdict: "both-fail", fails: required };
     return { verdict: "build-error", fails: true };
   }
   const differing = Object.entries(artifacts).filter(([, r]) => !r.identical);

scripts/zero-cost-gate.mjs

@@ -448,7 +448,15 @@ await pool(
         allow !== null &&
         programInputs(program.entry).some((f) => allow.test(readFileSync(f, "utf8")));
       const artifacts = baseline.ok && candidate.ok ? compareBuilds(baseline, candidate) : {};
-      const { verdict, fails } = programVerdict({ baseline, candidate, artifacts, allowed });
+      // The benchmark workloads and the self-hosted compiler always build.
+      const required = !program.id.startsWith("corpus/");
+      const { verdict, fails } = programVerdict({
+        baseline,
+        candidate,
+        artifacts,
+        allowed,
+        required,
+      });
       const entry = {
         id: program.id,
         verdict,
@@ -536,9 +544,19 @@ function printProgram(entry) {
   } else if (entry.cachegrind?.skipped)
     notes.push(`cachegrind skipped: ${entry.cachegrind.skipped}`);
   for (const failure of entry.dynamicTier?.failures ?? []) notes.push(`SYMBOLS: ${failure}`);
-  if (entry.baselineError) notes.push(`baseline: ${entry.baselineError.split("\n")[0]}`);
-  if (entry.candidateError) notes.push(`candidate: ${entry.candidateError.split("\n")[0]}`);
   console.log(`${label}  ${entry.id}${notes.length ? `  ${notes.join("; ")}` : ""}`);
+  // A build error's first line is often a warning; print all of it.
+  const errors =
+    entry.baselineError !== undefined && entry.baselineError === entry.candidateError
+      ? [["baseline and candidate", entry.baselineError]]
+      : [
+          ["baseline", entry.baselineError],
+          ["candidate", entry.candidateError],
+        ].filter(([, error]) => error !== undefined);
+  for (const [side, error] of errors) {
+    console.log(`           ${side}:`);
+    for (const line of error.split("\n")) console.log(`             ${line}`);
+  }
 }
 
 /* Compile time: paired --emit=llvm builds, the baseline and the candidate
@@ -613,6 +631,10 @@ if (timingRuns > 0) {
 
 const counts = {};
 for (const p of report.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1;
+if (!report.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict))) {
+  failed = true;
+  report.failures.push("no program built with both checkouts, so nothing was compared");
+}
 report.counts = counts;
 report.verdict = failed ? "fail" : "pass";
 report.elapsedMs = Date.now() - started;

tests/harness/zero-cost-gate.test.ts

@@ -179,11 +179,15 @@ test("the zero-cost gate fails differing programs unless they match the signatur
     allowed = false,
     baseline: object = ok,
     candidate: object = ok,
-  ) => programVerdict({ baseline, candidate, artifacts, allowed });
+    required = false,
+  ) => programVerdict({ baseline, candidate, artifacts, allowed, required });
   expect(verdict(same)).toEqual({ verdict: "identical", fails: false });
   expect(verdict(differs)).toEqual({ verdict: "differs", fails: true });
   expect(verdict(differs, true)).toEqual({ verdict: "allowed", fails: false });
   const failed = { ok: false, error: "SC1090" };
   expect(verdict({}, false, failed, failed)).toEqual({ verdict: "both-fail", fails: false });
+  // A benchmark or the self-hosted compiler that neither checkout builds
+  // was never compared.
+  expect(verdict({}, false, failed, failed, true)).toEqual({ verdict: "both-fail", fails: true });
   expect(verdict({}, true, ok, failed)).toEqual({ verdict: "build-error", fails: true });
 });