Merge pull request #1350 from DeusData/fix/skip-tests-skips-smoke

fix(release): skip_tests silently skipped smoke and soak too

.github/workflows/release.yml

@@ -80,7 +80,15 @@ jobs:
       attest: true
 
   # ── 4. Smoke test every binary ──────────────────────────────────
+  # !cancelled() && !failure(): GitHub propagates "skipped" TRANSITIVELY down the
+  # needs graph, so with skip_tests=true a skipped `test` skipped smoke and soak
+  # too — even though `build` overrode the same condition and succeeded. Nothing
+  # failed and nothing said so; the pipeline simply carried on toward publishing
+  # artifacts that had never been smoke-tested or soaked. Every job downstream of
+  # an optional phase needs this override, or the phase being optional silently
+  # makes the phases after it optional as well.
   smoke:
+    if: ${{ !cancelled() && !failure() }}
     needs: [build]
     uses: ./.github/workflows/_smoke.yml
     with:
@@ -88,7 +96,7 @@ jobs:
 
   # ── 5. Soak tests ──────────────────────────────────────────────
   soak:
-    if: ${{ inputs.soak_level != 'none' }}
+    if: ${{ !cancelled() && !failure() && inputs.soak_level != 'none' }}
     needs: [build]
     uses: ./.github/workflows/_soak.yml
     with:
@@ -97,9 +105,17 @@ jobs:
       version: ${{ inputs.version }}
 
   # ── 6. Create DRAFT release ────────────────────────────────────
+  # Requires smoke to have actually SUCCEEDED, not merely "not failed". The bare
+  # !cancelled() && !failure() form is fail-OPEN: a skipped smoke is neither
+  # cancelled nor failed, so a draft was one condition away from being cut from
+  # binaries nobody had run. soak may legitimately be 'skipped' (soak_level=none),
+  # so that one is enumerated explicitly rather than lumped in.
   release-draft:
     needs: [smoke, soak]
-    if: ${{ !cancelled() && !failure() }}
+    if: >-
+      ${{ !cancelled() && !failure()
+      && needs.smoke.result == 'success'
+      && (needs.soak.result == 'success' || needs.soak.result == 'skipped') }}
     runs-on: ubuntu-latest
     permissions:
       contents: write

tests/test_release_gate_chain_contract.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+# Contract: making a release phase optional must not silently make the phases
+# AFTER it optional too, and a draft must require smoke to have actually run.
+#
+# This exists because it nearly shipped untested binaries. `skip_tests: true`
+# skipped `test`, and GitHub propagates "skipped" TRANSITIVELY down the needs
+# graph: `build` overrode the condition and ran, but `smoke` and `soak` had no
+# override and were skipped. Nothing failed. Nothing said so. `release-draft`
+# then ran anyway, because `!cancelled() && !failure()` is fail-OPEN — a skipped
+# job is neither cancelled nor failed — so the pipeline was one gate away from
+# publishing artifacts nobody had smoke-tested or soaked.
+#
+# Two properties are pinned:
+#   1. every job downstream of an optional phase carries the
+#      `!cancelled() && !failure()` override, so it runs when an ancestor was
+#      deliberately skipped;
+#   2. release-draft requires `needs.smoke.result == 'success'` explicitly, so a
+#      skipped smoke BLOCKS the draft instead of sailing past it.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+WF="$ROOT/.github/workflows/release.yml"
+[ -f "$WF" ] || { echo "FAIL: $WF not found" >&2; exit 2; }
+
+python3 - "$WF" <<'PY'
+import pathlib
+import re
+import sys
+
+text = pathlib.Path(sys.argv[1]).read_text()
+
+# Slice the file into top-level job blocks: two-space indented "name:".
+blocks, current, name = {}, [], None
+for line in text.splitlines():
+    m = re.match(r"^  ([A-Za-z0-9_-]+):\s*$", line)
+    if m:
+        if name:
+            blocks[name] = "\n".join(current)
+        name, current = m.group(1), []
+        continue
+    if name is not None:
+        current.append(line)
+if name:
+    blocks[name] = "\n".join(current)
+
+failures = []
+
+def cond(job):
+    """The job's `if:` expression, block scalars folded onto one line."""
+    body = blocks.get(job, "")
+    m = re.search(r"^    if:\s*(>-|>|\|-|\|)?\s*(.*?)(?=^    [a-z_-]+:|\Z)",
+                  body, re.S | re.M)
+    if not m:
+        return ""
+    return " ".join(m.group(2).split())
+
+# 1. Downstream-of-optional jobs must tolerate a deliberately skipped ancestor.
+#    `test` is the optional phase (if: !inputs.skip_tests); everything after it
+#    in the chain has to survive that.
+TOLERATE = ["build", "smoke", "soak", "release-draft"]
+for job in TOLERATE:
+    if job not in blocks:
+        failures.append(f"{job}: job missing from release.yml — update this contract")
+        continue
+    c = cond(job)
+    if "!cancelled()" not in c or "!failure()" not in c:
+        failures.append(
+            f"{job}: `if:` lacks `!cancelled() && !failure()` (got: {c or '<none>'}).\n"
+            f"      With skip_tests=true a skipped ancestor SKIPS this job silently.")
+
+# 2. The draft must require smoke to have genuinely succeeded.
+draft = cond("release-draft")
+if "needs.smoke.result == 'success'" not in draft:
+    failures.append(
+        "release-draft: `if:` must require needs.smoke.result == 'success'.\n"
+        "      `!cancelled() && !failure()` alone is fail-open: a SKIPPED smoke\n"
+        "      passes it, and the draft gets cut from unsmoked binaries.")
+
+# soak is allowed to be skipped (soak_level=none) but must be enumerated, not
+# ignored — otherwise 'none' and 'silently never ran' are indistinguishable.
+if "needs.soak.result" not in draft:
+    failures.append(
+        "release-draft: `if:` must mention needs.soak.result explicitly so a\n"
+        "      legitimately skipped soak (soak_level=none) is distinguishable\n"
+        "      from a soak that never ran.")
+
+if failures:
+    for f in failures:
+        print("FAIL: " + f, file=sys.stderr)
+    print(f"release gate-chain contract FAILED with {len(failures)} violation(s)",
+          file=sys.stderr)
+    sys.exit(1)
+print("PASS: optional phases cannot silently disable the phases after them")
+PY