fix(ci): accept the testing-sha staleness fallback in chain evidence (#8056)

The 09-22 nightly chain failed all 12 lanes in seconds at the 'Bind
functional candidate' step:

  ValueError: private script pin differs from chain

resolve_functional_candidate.py's >24h staleness fallback (added by
#8026, made functional by #8041's token fix) legitimately sets
manifest.testing_sha to auto-testing main HEAD, but current_chain()
still required it to equal .config/functional-script-revision.txt -
a check written for the pre-fallback world where the two could never
diverge. Once the fallback finally fired, prepare produced testing_sha
21edcf4 while the pin file still holds 27e9584 and every lane aborted
before checking out the test scripts.

Drop the pin-file comparison and keep what the lane actually needs to
guarantee: testing_sha is a valid commit sha (current_chain), the lane
checked out exactly that sha (record: private_head == testing_sha, kept
as-is), and the health checker validates the same format instead of
re-reading the pin file. Tests updated: a fallback testing_sha that
differs from the pin is accepted; a non-sha testing_sha is rejected.

Verified: python3 -m unittest test_functional_chain
test_functional_chain_health -> 39 tests OK.

scripts/functional_chain_evidence.py

@@ -28,7 +28,12 @@ def current_chain():
     require(sha(chain["workflow_sha"]) and chain["workflow_sha"] == os.environ["GITHUB_SHA"], "chain workflow source mismatch")
     head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
     require(head == chain["workflow_sha"], "lane checkout differs from chain workflow source")
-    require(chain["testing_sha"] == (ROOT / ".config/functional-script-revision.txt").read_text().strip() and sha(chain["testing_sha"]), "private script pin differs from chain")
+    # testing_sha is either the committed pin or auto-testing main HEAD via
+    # resolve_functional_candidate.py's >24h staleness fallback, so pin
+    # equality is no longer an invariant (the 09-21 chain died on exactly
+    # that check once the fallback finally fired). Lanes check out exactly
+    # this sha, which is what the format check guards.
+    require(sha(chain["testing_sha"]), "private script revision is not a valid commit sha")
     candidate = chain["candidate"]
     require(isinstance(candidate, dict) and set(candidate) == {"manifest", "artifact_id", "artifact_digest", "workflow_sha", "workflow_ref", "build_started_at"}, "invalid candidate envelope")
     manifest = candidate["manifest"]

scripts/functional_chain_health.py

@@ -32,8 +32,9 @@ def validate_summary(summary, run):
     candidate = chain["candidate"]
     manifest = candidate["manifest"]
     require(resolve(manifest["build_run_id"], manifest["build_run_attempt"]) == candidate, "producer candidate identity changed")
-    config = api(f"repos/{REPOSITORY}/contents/.config/functional-script-revision.txt?ref={run['head_sha']}")
-    require(base64.b64decode(config["content"]).decode().strip() == chain["testing_sha"], "private pin differs from workflow source")
+    # testing_sha may legitimately be auto-testing main HEAD via the prepare
+    # step's >24h staleness fallback (checked for sha format above), so pin
+    # equality is not an invariant; drop the pin-file comparison.
     completed = timestamp(summary["completed_at"])
     require(timestamp(run["run_started_at"]) <= completed <= datetime.now(timezone.utc) + timedelta(minutes=5), "invalid completion timestamp")
     source_ref = manifest.get("source_ref", candidate["workflow_ref"])

scripts/test_functional_chain.py

@@ -226,6 +226,20 @@ class EnvelopeTests(unittest.TestCase):
                 evidence.current_chain()
         self.assertFalse((self.root / "env").exists())
 
+    def test_testing_sha_fallback_is_accepted_while_garbage_is_rejected(self):
+        # prepare's >24h staleness fallback legitimately sets testing_sha to
+        # auto-testing main HEAD, which differs from the committed pin; only
+        # the sha format is an invariant now.
+        for testing_sha, ok in (("d" * 40, True), ("1" * 40, True), ("xyz", False), ("", False)):
+            chain = dict(self.chain, testing_sha=testing_sha)
+            env = dict(self.env, CHAIN_MANIFEST=json.dumps(chain))
+            if ok:
+                with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40):
+                    evidence.consume(evidence.current_chain())
+            else:
+                with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40), self.assertRaises(ValueError):
+                    evidence.current_chain()
+
     def test_report_or_swallowed_test_failure_cannot_produce_valid_evidence(self):
         report = self.root / "cases.md"
         report.write_text("| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n")

scripts/test_functional_chain_health.py

@@ -39,8 +39,14 @@ class HealthTests(unittest.TestCase):
     def test_substituted_producer_pin_attempt_or_empty_suite_fails(self):
         with self.assertRaises(ValueError):
             self.validate(candidate={**self.candidate, "workflow_sha": "e" * 40})
+        # The prepare step's >24h staleness fallback legitimately sets
+        # testing_sha to auto-testing main HEAD, so the pin FILE is no longer
+        # consulted at all; the sha FORMAT of the chain's testing_sha is the
+        # remaining invariant.
+        wrong = copy.deepcopy(self.summary)
+        wrong["chain"]["testing_sha"] = "short"
         with self.assertRaises(ValueError):
-            self.validate(config={"content": base64.b64encode(b"wrong pin").decode()})
+            self.validate(wrong)
         wrong = copy.deepcopy(self.summary)
         wrong["chain"]["attempt"] = 1
         with self.assertRaises(ValueError):