From 4e9f36bc7d60e0bbead1114e68c729bbf09685e0 Mon Sep 17 00:00:00 2001 From: Jiajun Li Date: Sun, 16 Aug 2026 22:36:29 -0700 Subject: [PATCH] refactor(ci): require domain labels for GPU tests (#2548) --- .github/workflows/pr-test.yml | 5 ++ docs/ci/00-stage.md | 2 +- docs/ci/01-label.md | 9 +- docs/ci/03-metric-history-gate.md | 2 +- docs/ci/contributor-guide.md | 6 +- docs/developer/contributor-guide.md | 4 +- tests/ci/ci_policy.py | 5 +- tests/ci/ci_register.py | 31 ++++--- tests/ci/run_suite.py | 19 +++-- tests/ci/test/test_ci_register.py | 47 ++++------- tests/ci/test/test_gate_integration.py | 4 +- tests/ci/test/test_history_gate.py | 6 +- tests/ci/test/test_run_suite.py | 82 ++++++++++--------- tests/ci/test/test_stage_selection.py | 30 ++++--- tests/fast-gpu/test_det_process_group.py | 4 +- tests/fast-gpu/test_mxfp8_quantizer.py | 2 +- tests/fast-gpu/test_nvfp4_quantizer.py | 2 +- tests/fast-gpu/test_quantizer_ci.py | 2 +- .../fast-gpu/test_run_megatron_worker_main.py | 4 +- tests/fast-gpu/test_semaphore.py | 2 +- 20 files changed, 142 insertions(+), 126 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 7f4c7bcd87..3423500a34 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -210,6 +210,7 @@ jobs: python tests/ci/run_suite.py --hw cuda --suite stage-b-2-gpu-h200 --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} + ${{ github.event_name == 'workflow_dispatch' && '--match-all-labels' || '' }} secrets: inherit stage-c-8-gpu-h100: @@ -236,6 +237,7 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} + ${{ github.event_name == 'workflow_dispatch' && '--match-all-labels' || '' }} secrets: inherit stage-c-8-gpu-h200: @@ -262,6 +264,7 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} + ${{ github.event_name == 'workflow_dispatch' && '--match-all-labels' || '' }} secrets: inherit stage-c-4-gpu-h200: @@ -288,6 +291,7 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 3 --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} + ${{ github.event_name == 'workflow_dispatch' && '--match-all-labels' || '' }} secrets: inherit stage-c-2-gpu-h200: @@ -314,4 +318,5 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} + ${{ github.event_name == 'workflow_dispatch' && '--match-all-labels' || '' }} secrets: inherit diff --git a/docs/ci/00-stage.md b/docs/ci/00-stage.md index 0dfc6fc29f..5c79bf2b46 100644 --- a/docs/ci/00-stage.md +++ b/docs/ci/00-stage.md @@ -50,7 +50,7 @@ Distinct from image selection, the **`run-ci-image` label** selects the test sco - Each Miles PR workflow passes trigger facts and, for PRs, the diff to `tests/ci/ci_policy.py`, which publishes the resolved policy and `skipped_stages` for `run_suite.py` and GPU job gates. - A PR `nightly` label maps to nightly cadence. - A scheduled run maps its exact UTC `github.event.schedule` cron: `0 15 * * 0-5` maps to nightly and `0 15 * * 6` maps to weekly; an unknown cron fails. -- A manual dispatch keeps regular cadence and has no PR labels. `pr-test.yml` therefore runs the ordinary always-on selection, while the dedicated ROCm dispatch adds `--match-all-labels` to preserve its full regular MI350 run. +- A manual dispatch keeps regular cadence and has no PR labels. Both GPU workflows add `--match-all-labels` so an explicit manual operation runs the full regular GPU suites; CPU selection remains unchanged. - A reusable `workflow_call` supplies an explicit cadence override because the called workflow inherits the caller's event name. Policy resolution deliberately runs the caller commit's `ci_policy.py`; only suite jobs check out the requested release ref. A **nightly** policy selects every enabled tag except `long` and `ft-long`, admits both regular and `nightly=True` registrations, and disables fast-fail. **Weekly** and **release** select every enabled tag, admit both registration types, and disable fast-fail; release differs by never writing the rolling performance baseline. Regular cadence admits only regular registrations. All four cadences use the same stage inventory. diff --git a/docs/ci/01-label.md b/docs/ci/01-label.md index dec5e73817..6bde85a808 100644 --- a/docs/ci/01-label.md +++ b/docs/ci/01-label.md @@ -20,13 +20,15 @@ A test declares its labels: `register_cuda_ci(..., labels=["megatron"])`. The PR | Test declares | Runs when | |---|---| -| `labels=[]` (or omitted) | every run whose cadence admits the test (always-on within that cadence) | +| CPU `labels=[]` (or omitted) | every run whose cadence admits the test (always-on within that cadence) | | `labels=["megatron"]` | PR has `run-ci-megatron` | | `labels=["sglang"]` | PR has `run-ci-sglang` | | `labels=["fsdp", "lora"]` | PR has `run-ci-fsdp` or `run-ci-lora` | PR labels without the `run-ci-` prefix are ignored. +CUDA and ROCm registrations must declare at least one domain label. GPU runners are scarce and expensive, so an always-on GPU test would defeat the purpose of selecting only the GPU coverage a PR requests. + ### The canonical label list Domain labels live in `tests/ci/labels.py` (`KNOWN_LABELS`); a `labels=[...]` value outside it is a hard error. Current set: `megatron`, `model-scripts`, `sglang`, `fsdp`, `short`, `long`, `ckpt`, `lora`, `precision`, `ft-short`, `ft-long`, `weight-update`, `replay`, `qwen35`, `mooncake`. @@ -55,7 +57,7 @@ Release and weekly have the same selection and fast-fail policy. Release is sepa Rows are in precedence order: when scope signals overlap, the higher row wins (`run-ci-all` > weekly/release full scope > nightly > `run-ci-image`, the branch order of `resolve_policy`). `run-ci-all` widens only the domain scope; regular cadence still does not admit `nightly=True` registrations. -The generic triggers carry no policy. All scheduled runs use UTC: nightly is identified by the exact cron `0 15 * * 0-5`, and weekly by `0 15 * * 6`. Saturday weekly replaces that day's nightly rather than starting alongside it. A manual dispatch uses regular cadence and no PR labels, so it receives only the ordinary always-on scope; its operation inputs do not imply all, nightly, weekly, or release. A called workflow instead supplies its cadence explicitly, which is how `release-branch-cut.yml` selects release. +The generic triggers carry no policy. All scheduled runs use UTC: nightly is identified by the exact cron `0 15 * * 0-5`, and weekly by `0 15 * * 6`. Saturday weekly replaces that day's nightly rather than starting alongside it. A manual dispatch uses regular cadence and no PR labels; the GPU workflow commands explicitly add `--match-all-labels` so that operation runs the full regular GPU suites without changing cadence. A called workflow instead supplies its cadence explicitly, which is how `release-branch-cut.yml` selects release. A subtraction is not a per-test veto — it only stops that label from granting inclusion. A test carrying a subtracted label still runs when another of its labels is in the set, so a test that must stay outside the standard nightly scope must carry only labels that nightly subtracts. @@ -63,9 +65,10 @@ A domain label explicitly requested on the PR wins over a scope subtraction: `ru ## Registration and scan scope -Labels are optional; registration is not. The runner scans `tests/fast`, `tests/fast-gpu`, `tests/e2e`, `tests/ci` recursively for `test_*.py`. Every file must resolve to a registration or collection fails: +Labels are optional for CPU registrations and required for GPU registrations; registration itself is not optional. The runner scans `tests/fast`, `tests/fast-gpu`, `tests/e2e`, `tests/ci` recursively for `test_*.py`. Every file must resolve to a registration or collection fails: - A file outside `tests/fast/` with no `register_*_ci()` call → `No CI registry found`. +- A CUDA or ROCm registration with missing, `None`, or empty `labels` → `labels ... must contain at least one domain label`. - A `labels=[...]` value not in `KNOWN_LABELS` → `unknown labels [...]`. ## `tests/fast/` auto-registers as CPU diff --git a/docs/ci/03-metric-history-gate.md b/docs/ci/03-metric-history-gate.md index 0a0a0c37a2..ae1671df4a 100644 --- a/docs/ci/03-metric-history-gate.md +++ b/docs/ci/03-metric-history-gate.md @@ -39,7 +39,7 @@ A declaration sits at top level of the test file, next to its CI registration from tests.ci.ci_register import register_cuda_ci from tests.ci.metric_history import register_ci_gate -register_cuda_ci(est_time=300, suite="stage-c-8-gpu-h100") +register_cuda_ci(est_time=300, suite="stage-c-8-gpu-h100", labels=["megatron"]) register_ci_gate( metric_key="train/ppo_kl", # must be a captured key (whitelist) diff --git a/docs/ci/contributor-guide.md b/docs/ci/contributor-guide.md index 5afbe3fbc5..6a6b21a31b 100644 --- a/docs/ci/contributor-guide.md +++ b/docs/ci/contributor-guide.md @@ -18,7 +18,7 @@ from tests.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=600, # rough seconds the test takes; used to balance + time-out suite="stage-c-4-gpu-h200", # which hardware bucket runs it (table below) - labels=["megatron"], # see "Will it run on my PR?"; use [] for always-on + labels=["megatron"], # required; see "Will it run on my PR?" ) ``` @@ -51,9 +51,7 @@ If your file does **not** show up, check, in order: ### Will it run on my PR? -`labels` gates *which PRs* trigger your test within its eligible cadence: - -- `labels=[]` (or omitted) → **always-on** within the eligible cadence; with the default `nightly=False`, this includes every PR. +`labels` gates *which PRs* trigger your test within its eligible cadence. GPU registrations require at least one domain label: - `labels=["megatron"]` → runs only when the PR carries the GitHub label **`run-ci-megatron`** (the `run-ci-` prefix is added on the PR side). This keeps the heavy GPU matrix off unrelated PRs. Cadence is independent of labels: `nightly=True` excludes a registration from regular cadence, while nightly, weekly, and release runs include both ordinary and `nightly=True` registrations. diff --git a/docs/developer/contributor-guide.md b/docs/developer/contributor-guide.md index 4b7df9335c..cbc61f0d70 100644 --- a/docs/developer/contributor-guide.md +++ b/docs/developer/contributor-guide.md @@ -158,11 +158,11 @@ from tests.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=600, # rough seconds; balances shards and sets the per-file timeout suite="stage-c-4-gpu-h200", # the hardware bucket that runs it - labels=["megatron"], # [] or omitted means always-on + labels=["megatron"], # required for CUDA and ROCm tests ) ``` -`register_cpu_ci`, `register_cuda_ci` and `register_rocm_ci` share that signature, plus `nightly=True` (nightly, weekly, and release cadence only) and `disabled=""` (reported as skipped rather than deleted). The calls are parsed from the AST, so they must be top-level, literal, and unaliased. +`register_cpu_ci` allows empty labels for always-on CPU coverage; `register_cuda_ci` and `register_rocm_ci` require a non-empty domain-label list. All three also accept `nightly=True` (nightly, weekly, and release cadence only) and `disabled=""` (reported as skipped rather than deleted). The calls are parsed from the AST, so they must be top-level, literal, and unaliased. The runner scans `tests/fast`, `tests/fast-gpu`, `tests/e2e` and `tests/ci` for `test_*.py`, and a file outside `tests/fast/` with no registration fails collection with diff --git a/tests/ci/ci_policy.py b/tests/ci/ci_policy.py index 0e724e998f..a09b3cc230 100644 --- a/tests/ci/ci_policy.py +++ b/tests/ci/ci_policy.py @@ -79,8 +79,9 @@ def resolve_policy(cadence: str, raw_labels: set[str]) -> RunPolicy: The workflow adapter resolves trigger-specific facts into a cadence and raw labels; this function never infers policy from a GitHub event name. A - test runs iff it is cadence-eligible and declares no labels (always-run) - or any of its labels is in the effective include set. + test runs iff it is cadence-eligible and declares no labels (the CPU + always-on case) or any of its labels is in the effective include set. GPU + registrations are validated separately to require a non-empty label set. Broad scopes are large include sets: diff --git a/tests/ci/ci_register.py b/tests/ci/ci_register.py index b522289c2c..b8891e9892 100644 --- a/tests/ci/ci_register.py +++ b/tests/ci/ci_register.py @@ -60,12 +60,12 @@ def register_cpu_ci( ): """Marker for CPU CI registration (parsed via AST; runtime no-op). - `labels=None` and `labels=[]` are equivalent: the test is always-on within - every cadence that admits it. A non-empty `labels` list gates the test on - the resolved domain scope; a PR can include `` with `run-ci-`, while - broad scopes include many domain labels at once. `nightly=True` adds a - cadence gate: regular runs exclude the test, while nightly and weekly runs - include it alongside regular registrations. + `labels=None` and `labels=[]` are equivalent for CPU tests: the test is + always-on within every cadence that admits it. A non-empty `labels` list + gates the test on the resolved domain scope; a PR can include `` with + `run-ci-`, while broad scopes include many domain labels at once. + `nightly=True` adds a cadence gate: regular runs exclude the test, while + nightly and weekly runs include it alongside regular registrations. """ return None @@ -74,13 +74,14 @@ def register_cuda_ci( est_time: float, suite: str, *, - labels: list[str] | None = None, + labels: list[str], nightly: bool = False, disabled: str | None = None, ): """Marker for CUDA CI registration (parsed via AST; runtime no-op). - See `register_cpu_ci` for label semantics. + `labels` must contain at least one domain label so GPU tests run only when + an explicit or broad scope selects them. """ return None @@ -89,13 +90,14 @@ def register_rocm_ci( est_time: float, suite: str, *, - labels: list[str] | None = None, + labels: list[str], nightly: bool = False, disabled: str | None = None, ): """Marker for ROCm CI registration (parsed via AST; runtime no-op). - See `register_cpu_ci` for label semantics. + `labels` must contain at least one domain label so GPU tests run only when + an explicit or broad scope selects them. """ return None @@ -194,11 +196,14 @@ class RegistryVisitor(ast.NodeVisitor): if not isinstance(parsed["suite"], str): raise ValueError(f"{self.filename}: suite must be a string in {func_name}()") - # `labels` is optional. Missing / None / [] all mean "always-on within - # the eligible cadence"; only a non-empty list adds a domain gate. + # CPU labels remain optional; GPU registrations require an explicit + # non-empty domain so they cannot consume runners on every PR. labels = parsed.get("labels", []) if not isinstance(labels, list): raise ValueError(f"{self.filename}: labels must be a list or None in {func_name}()") + backend = _REGISTER_BACKEND_MAP[func_name] + if backend != HWBackend.CPU and not labels: + raise ValueError(f"{self.filename}: labels in {func_name}() must contain at least one domain label") nightly = parsed.get("nightly", False) if not isinstance(nightly, bool): @@ -219,7 +224,7 @@ class RegistryVisitor(ast.NodeVisitor): ) return CIRegistry( - backend=_REGISTER_BACKEND_MAP[func_name], + backend=backend, filename=self.filename, est_time=float(parsed["est_time"]), suite=parsed["suite"], diff --git a/tests/ci/run_suite.py b/tests/ci/run_suite.py index f5473a6fd4..a984a7b677 100644 --- a/tests/ci/run_suite.py +++ b/tests/ci/run_suite.py @@ -68,13 +68,14 @@ def filter_tests( """Filter registered tests down to the set that should run. The base predicate (hw / suite / cadence eligibility / disabled) is applied first. - Label selection then keeps a test iff it declares no labels (always-run) - or any of its labels is in `labels` -- the effective include set from - `resolve_policy` (the requested domain labels for a plain PR, near-total - registry sets for broad scopes). There is no separate exclusion pass: a - label a scope subtracted simply grants no inclusion, so a test whose - only labels were subtracted drops out (including from the skip report), - while a test that also carries an included label still runs. + Label selection then keeps a test iff it declares no labels (the CPU + always-on case) or any of its labels is in `labels` -- the effective + include set from `resolve_policy` (the requested domain labels for a plain + PR, near-total registry sets for broad scopes). GPU registrations require + at least one label. There is no separate exclusion pass: a label a scope + subtracted simply grants no inclusion, so a test whose only labels were + subtracted drops out (including from the skip report), while a test that + also carries an included label still runs. """ valid_suites = CI_SUITES.get(hw, []) if suite not in valid_suites: @@ -343,8 +344,8 @@ def main(): "Raw PR-side labels (e.g. `run-ci-megatron run-ci-fsdp`). The " "`run-ci-` prefix is stripped on the Python side; the resulting " "domain-label set is intersected with each test's `labels` to " - "decide what runs. An empty list keeps only registrations with " - "no domain labels." + "decide what runs. An empty list keeps only CPU registrations " + "with no domain labels; it selects no GPU tests." ), ) parser.add_argument( diff --git a/tests/ci/test/test_ci_register.py b/tests/ci/test/test_ci_register.py index 7168df9053..394be79179 100644 --- a/tests/ci/test/test_ci_register.py +++ b/tests/ci/test/test_ci_register.py @@ -84,30 +84,6 @@ class TestRegisterPositive: assert r.suite == "stage-a-cpu" assert r.labels == [] - def test_labels_none_is_always_run(self, tmp_path): - # Explicit `labels=None` is equivalent to omitting / `labels=[]`. - path = _make_fixture( - """ - from tests.ci.ci_register import register_cuda_ci - register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=None) - """, - tmp_path, - ) - r = ut_parse_one_file(path)[0] - assert r.labels == [] - - def test_labels_empty_list_is_always_run(self, tmp_path): - # `labels=[]` is also legal and means always-run; no never-run rule. - path = _make_fixture( - """ - from tests.ci.ci_register import register_cuda_ci - register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=[]) - """, - tmp_path, - ) - r = ut_parse_one_file(path)[0] - assert r.labels == [] - def test_cuda_multiple_labels(self, tmp_path): path = _make_fixture( """ @@ -158,6 +134,19 @@ class TestRegisterPositive: class TestRegisterNegative: + @pytest.mark.parametrize("func_name", ["register_cuda_ci", "register_rocm_ci"]) + @pytest.mark.parametrize("labels_arg", ["", ", labels=None", ", labels=[]"]) + def test_gpu_labels_must_be_non_empty(self, tmp_path, func_name, labels_arg): + path = _make_fixture( + f""" + from tests.ci.ci_register import {func_name} + {func_name}(est_time=60, suite="stage-b-2-gpu-h200"{labels_arg}) + """, + tmp_path, + ) + with pytest.raises(ValueError, match=r"labels.*must contain at least one domain label"): + ut_parse_one_file(path) + def test_unknown_label_rejected(self, tmp_path): path = _make_fixture( """ @@ -279,7 +268,7 @@ class TestSelectionHelpers: assert _extract_list_constant(node) == [] def test_extract_list_constant_none_is_empty(self): - # Treat literal `None` as equivalent to `[]` (always-run intent). + # Literal `None` remains the CPU always-run spelling. node = ast.parse("None", mode="eval").body assert _extract_list_constant(node) == [] @@ -356,7 +345,7 @@ class TestFileTextMentionsRegister: def test_file_with_cuda_call_matches(self, tmp_path): p = tmp_path / "f.py" p.write_text( - "from tests.ci.ci_register import register_cuda_ci\nregister_cuda_ci(est_time=60, suite='stage-b-2-gpu-h200', labels=[])\n" + "from tests.ci.ci_register import register_cuda_ci\nregister_cuda_ci(est_time=60, suite='stage-b-2-gpu-h200', labels=['precision'])\n" ) assert _file_text_mentions_register(str(p)) @@ -479,7 +468,7 @@ class TestCollectTestsFastGpuStrict: "test_gpu_thing.py", """ from tests.ci.ci_register import register_cuda_ci - register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=[]) + register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=["precision"]) """, ) registries = collect_tests([path], sanity_check=True) @@ -523,7 +512,7 @@ class TestCollectTestsCudaBanInFast: "test_misplaced.py", """ from tests.ci.ci_register import register_cuda_ci - register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=[]) + register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=["precision"]) """, ) with pytest.raises(ValueError, match=r"register_cuda_ci is forbidden in tests/fast/"): @@ -536,7 +525,7 @@ class TestCollectTestsCudaBanInFast: "test_ok.py", """ from tests.ci.ci_register import register_cuda_ci - register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=[]) + register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=["precision"]) """, ) registries = collect_tests([path], sanity_check=True) diff --git a/tests/ci/test/test_gate_integration.py b/tests/ci/test/test_gate_integration.py index c3c3cc6e7a..1629620fd6 100644 --- a/tests/ci/test/test_gate_integration.py +++ b/tests/ci/test/test_gate_integration.py @@ -67,7 +67,9 @@ def _write_test_file(tmp_path: Path, gate_lines: str, *, name: str = "test_e2e_g body = ( "from tests.ci.ci_register import register_cuda_ci\n" "from tests.ci.metric_history import register_ci_gate\n" - 'register_cuda_ci(est_time=600, suite="stage-c-8-gpu-h100")\n' + textwrap.dedent(gate_lines).strip() + "\n" + 'register_cuda_ci(est_time=600, suite="stage-c-8-gpu-h100", labels=["megatron"])\n' + + textwrap.dedent(gate_lines).strip() + + "\n" ) p = tmp_path / name p.write_text(body) diff --git a/tests/ci/test/test_history_gate.py b/tests/ci/test/test_history_gate.py index 59b19a2af6..89eedcf2bd 100644 --- a/tests/ci/test/test_history_gate.py +++ b/tests/ci/test/test_history_gate.py @@ -51,7 +51,9 @@ def _write_test_file(tmp_path: Path, gate_lines: str, *, name: str = "test_e2e_f body = ( "from tests.ci.ci_register import register_cuda_ci\n" "from tests.ci.metric_history import register_ci_gate\n" - 'register_cuda_ci(est_time=600, suite="stage-c-8-gpu-h100")\n' + textwrap.dedent(gate_lines).strip() + "\n" + 'register_cuda_ci(est_time=600, suite="stage-c-8-gpu-h100", labels=["megatron"])\n' + + textwrap.dedent(gate_lines).strip() + + "\n" ) p = tmp_path / name p.write_text(body) @@ -580,7 +582,7 @@ def test_no_gate_specs_is_vacuously_trusted(tmp_path, store): body = textwrap.dedent( """ from tests.ci.ci_register import register_cuda_ci - register_cuda_ci(est_time=600, suite="stage-c-8-gpu-h100") + register_cuda_ci(est_time=600, suite="stage-c-8-gpu-h100", labels=["megatron"]) """ ).lstrip("\n") p = tmp_path / "test_nogate.py" diff --git a/tests/ci/test/test_run_suite.py b/tests/ci/test/test_run_suite.py index 3788b6763e..53861c1881 100644 --- a/tests/ci/test/test_run_suite.py +++ b/tests/ci/test/test_run_suite.py @@ -6,10 +6,9 @@ These cover the Python-side policy and label pipeline: workflow-only labels, warning on other non-prefixed inputs. * `resolve_policy`: explicit cadence + raw labels -> selection and fast-fail. * The PR workflow seams: one adapter resolves trigger facts and every CUDA/ROCm stage consumes its outputs. -* `filter_tests`: include-set selection with the "empty labels means always - run" semantic; a scope subtraction is not a per-test veto. -* `CI_SUITES`: locked to the new taxonomy including the - always-run GPU bucket `stage-b-2-gpu-h200`. +* `filter_tests`: include-set selection, including CPU always-on coverage and + GPU domain labels; a scope subtraction is not a per-test veto. +* `CI_SUITES`: locked to the current hardware taxonomy. We build `CIRegistry` instances directly via a small factory rather than parsing fixture files -- the AST-side validation lives in @@ -55,14 +54,17 @@ def _make( ) -> CIRegistry: """Minimal `CIRegistry` factory for filter tests. - `labels=None` and `labels=[]` are equivalent (always-run semantics). + CUDA fixtures default to the `megatron` domain; CPU fixtures default to + the always-on empty label set. """ + if labels is None: + labels = [] if backend == HWBackend.CPU else ["megatron"] return CIRegistry( backend=backend, filename=filename, est_time=est_time, suite=suite, - labels=list(labels) if labels is not None else [], + labels=list(labels), nightly=nightly, disabled=disabled, implicit=False, @@ -383,11 +385,14 @@ class TestWorkflowScopeSeam: ) assert expected in block - def test_dispatch_has_no_implicit_scope(self): + def test_dispatch_has_no_scope_input_but_runs_all_cuda_domains(self): workflow = self._workflow() dispatch_inputs = workflow.split("workflow_dispatch:", 1)[1].split("permissions:", 1)[0] assert "ci_cadence" not in dispatch_inputs assert "ci_scope" not in dispatch_inputs + manual_scope = "${{ github.event_name == 'workflow_dispatch' && '--match-all-labels' || '' }}" + cuda_stages = workflow.split(" stage-b-2-gpu-h200:", 1)[1] + assert cuda_stages.count(manual_scope) == 5 def test_gpu_gates_consume_shared_bypass_output(self): workflow = self._workflow() @@ -688,15 +693,15 @@ def cuda_h100_tests(): """A representative `stage-c-8-gpu-h100` registry used across scenarios. Composition: - * 2 always-run tests (`labels=[]`) + * 2 precision tests * 1 megatron-only test * 1 fsdp-only test * 1 megatron+sglang test (multi-label, exercises OR semantics) * 1 disabled megatron test (must always be classified as skipped) """ return [ - _make("tests/e2e/fast1.py", labels=[]), - _make("tests/e2e/fast2.py", labels=[]), + _make("tests/e2e/precision1.py", labels=["precision"]), + _make("tests/e2e/precision2.py", labels=["precision"]), _make("tests/e2e/megatron/m1.py", labels=["megatron"]), _make("tests/e2e/fsdp/f1.py", labels=["fsdp"]), _make("tests/e2e/megatron/m_or_s.py", labels=["megatron", "sglang"]), @@ -709,20 +714,20 @@ def _names(tests: list[CIRegistry]) -> set[str]: class TestFilterTestsLabels: - def test_case1_no_labels_keeps_only_always_run(self, cuda_h100_tests): - # Empty --labels (after stripping) -> tests with empty `labels` - # survive (always run); labelled tests are filtered out. + def test_case1_no_labels_selects_no_gpu_tests(self, cuda_h100_tests): + # Every GPU registration has a domain, so an empty include set selects + # no GPU tests. enabled, skipped = filter_tests( cuda_h100_tests, HWBackend.CUDA, "stage-c-8-gpu-h100", labels=set(), ) - assert _names(enabled) == {"tests/e2e/fast1.py", "tests/e2e/fast2.py"} + assert enabled == [] assert skipped == [] def test_case2_single_domain_label(self, cuda_h100_tests): - # `run-ci-megatron` -> always-run + megatron-labelled tests. + # `run-ci-megatron` selects only megatron-labelled tests. enabled, skipped = filter_tests( cuda_h100_tests, HWBackend.CUDA, @@ -730,8 +735,6 @@ class TestFilterTestsLabels: labels={"megatron"}, ) assert _names(enabled) == { - "tests/e2e/fast1.py", - "tests/e2e/fast2.py", "tests/e2e/megatron/m1.py", "tests/e2e/megatron/m_or_s.py", } @@ -740,7 +743,7 @@ class TestFilterTestsLabels: assert _names(skipped) == {"tests/e2e/megatron/disabled.py"} def test_case3_multiple_domain_labels_or_semantics(self, cuda_h100_tests): - # {megatron, fsdp} -> union (OR) of matches plus always-run tests. + # {megatron, fsdp} -> union (OR) of domain matches. enabled, _ = filter_tests( cuda_h100_tests, HWBackend.CUDA, @@ -748,8 +751,6 @@ class TestFilterTestsLabels: labels={"megatron", "fsdp"}, ) assert _names(enabled) == { - "tests/e2e/fast1.py", - "tests/e2e/fast2.py", "tests/e2e/megatron/m1.py", "tests/e2e/fsdp/f1.py", "tests/e2e/megatron/m_or_s.py", @@ -765,8 +766,8 @@ class TestFilterTestsLabels: labels=_ALL, ) assert _names(enabled) == { - "tests/e2e/fast1.py", - "tests/e2e/fast2.py", + "tests/e2e/precision1.py", + "tests/e2e/precision2.py", "tests/e2e/megatron/m1.py", "tests/e2e/fsdp/f1.py", "tests/e2e/megatron/m_or_s.py", @@ -774,16 +775,15 @@ class TestFilterTestsLabels: assert _names(skipped) == {"tests/e2e/megatron/disabled.py"} def test_case5_unknown_pr_side_label_is_silent_noop(self, cuda_h100_tests): - # Unknown PR-side label (e.g. `run-ci-foo`) -- after stripping, - # `foo` simply produces an empty intersection. No error; only - # always-run tests survive. + # Unknown PR-side label (e.g. `run-ci-foo`) produces an empty + # intersection and selects no GPU tests. enabled, _ = filter_tests( cuda_h100_tests, HWBackend.CUDA, "stage-c-8-gpu-h100", labels={"foo"}, ) - assert _names(enabled) == {"tests/e2e/fast1.py", "tests/e2e/fast2.py"} + assert enabled == [] # --- filter_tests: broad CI scopes as include sets --------------------------- @@ -792,7 +792,7 @@ class TestFilterTestsLabels: @pytest.fixture def broad_scope_tests(): return [ - _make("tests/e2e/always.py", labels=[]), + _make("tests/e2e/precision.py", labels=["precision"]), _make("tests/e2e/megatron.py", labels=["megatron"]), _make("tests/e2e/long.py", labels=["long"]), _make("tests/e2e/ft/short.py", labels=["ft-short"]), @@ -809,7 +809,7 @@ class TestFilterTestsBroadScopes: labels=set(resolve_policy(REGULAR_CADENCE, {"run-ci-image"}).include_labels), ) assert _names(enabled) == { - "tests/e2e/always.py", + "tests/e2e/precision.py", "tests/e2e/megatron.py", } @@ -822,14 +822,14 @@ class TestFilterTestsBroadScopes: labels=set(resolve_policy(NIGHTLY_CADENCE, set()).include_labels), ) assert _names(enabled) == { - "tests/e2e/always.py", + "tests/e2e/precision.py", "tests/e2e/megatron.py", "tests/e2e/ft/short.py", } def test_subtracted_only_test_drops_out_entirely(self): tests = [ - _make("tests/e2e/always.py", labels=[]), + _make("tests/e2e/precision.py", labels=["precision"]), _make("tests/e2e/ft/soak.py", labels=["ft-long"]), _make("tests/e2e/ft/soak_disabled.py", labels=["ft-long"], disabled="flaky"), ] @@ -842,7 +842,7 @@ class TestFilterTestsBroadScopes: ) # A test whose only labels were subtracted is out of scope entirely, # including from the skip report. - assert _names(enabled) == {"tests/e2e/always.py"} + assert _names(enabled) == {"tests/e2e/precision.py"} assert skipped == [] def test_all_scope_includes_every_label(self, broad_scope_tests): @@ -863,8 +863,8 @@ class TestFilterTestsBaseDimensions: # A test registered to stage-c-4-gpu-h200 must not surface in # stage-c-8-gpu-h100, even with the full include set. tests = [ - _make("tests/e2e/h100/t.py", suite="stage-c-8-gpu-h100", labels=[]), - _make("tests/e2e/h200/t.py", suite="stage-c-4-gpu-h200", labels=[]), + _make("tests/e2e/h100/t.py", suite="stage-c-8-gpu-h100", labels=["precision"]), + _make("tests/e2e/h200/t.py", suite="stage-c-4-gpu-h200", labels=["precision"]), ] enabled, _ = filter_tests( tests, @@ -875,10 +875,15 @@ class TestFilterTestsBaseDimensions: assert _names(enabled) == {"tests/e2e/h100/t.py"} def test_cross_backend_isolation(self): - # CPU suite must not pull in CUDA-registered always-run tests. + # CPU suite must not pull in CUDA registrations. tests = [ _make("tests/fast/t.py", backend=HWBackend.CPU, suite="stage-a-cpu", labels=[]), - _make("tests/e2e/h100/t.py", backend=HWBackend.CUDA, suite="stage-c-8-gpu-h100", labels=[]), + _make( + "tests/e2e/h100/t.py", + backend=HWBackend.CUDA, + suite="stage-c-8-gpu-h100", + labels=["precision"], + ), ] enabled, _ = filter_tests( tests, @@ -969,15 +974,14 @@ class TestFilterTestsBaseDimensions: assert skipped == [] def test_stage_b_2_gpu_h200_is_addressable(self): - # The always-run GPU bucket must be a first-class suite that - # filter_tests can route to without a "unknown suite" warning fail. + # The fast GPU bucket remains a first-class suite. tests = [ - _make("tests/fast/q.py", suite="stage-b-2-gpu-h200", labels=[]), + _make("tests/fast/q.py", suite="stage-b-2-gpu-h200", labels=["precision"]), ] enabled, _ = filter_tests( tests, HWBackend.CUDA, "stage-b-2-gpu-h200", - labels=set(), + labels={"precision"}, ) assert _names(enabled) == {"tests/fast/q.py"} diff --git a/tests/ci/test/test_stage_selection.py b/tests/ci/test/test_stage_selection.py index 29c8ec7549..ad82c00e8a 100644 --- a/tests/ci/test/test_stage_selection.py +++ b/tests/ci/test/test_stage_selection.py @@ -27,7 +27,7 @@ def _registration( filename=filename, est_time=1, suite=suite, - labels=labels or [], + labels=["precision"] if labels is None else labels, disabled=disabled, ) @@ -40,7 +40,7 @@ def _select( changed_files: tuple[ChangedFile, ...] | None, registrations: list[CIRegistry], *, - raw_labels: tuple[str, ...] = (), + raw_labels: tuple[str, ...] = ("run-ci-precision",), event_name: str = "pull_request", ) -> tuple[str, ...]: return select_skipped_gpu_stages( @@ -73,7 +73,7 @@ def test_read_changed_files_fails_open_on_unusable_input(tmp_path, payload): def test_docs_tooling_and_cpu_only_tests_skip_every_gpu_stage(): cpu_test = "tests/fast/doc/test_sync_example_docs.py" registrations = [ - _registration("tests/fast-gpu/test_always.py", "stage-b-2-gpu-h200"), + _registration("tests/fast-gpu/test_precision.py", "stage-b-2-gpu-h200"), CIRegistry(HWBackend.CPU, cpu_test, 1, "stage-a-cpu"), ] changed_files = ( @@ -84,25 +84,29 @@ def test_docs_tooling_and_cpu_only_tests_skip_every_gpu_stage(): ChangedFile("M", (cpu_test,)), ) - assert set(_select(changed_files, registrations)) == PR_GPU_STAGES + assert set(_select(changed_files, registrations, raw_labels=())) == PR_GPU_STAGES -def test_changed_multi_backend_test_runs_only_its_local_stages(): +def test_changed_multi_backend_test_and_explicit_domain_run_their_stages(): shared_test = "tests/e2e/test_shared.py" registrations = [ - _registration("tests/fast-gpu/test_always.py", "stage-b-2-gpu-h200"), + _registration("tests/fast-gpu/test_precision.py", "stage-b-2-gpu-h200"), _registration(shared_test, "stage-c-8-gpu-h100"), _registration(shared_test, "stage-c-4-gpu-mi350"), ] skipped = set(_select((ChangedFile("M", (shared_test,)),), registrations)) - assert skipped == PR_GPU_STAGES - {"stage-c-8-gpu-h100", "stage-c-4-gpu-mi350"} + assert skipped == PR_GPU_STAGES - { + "stage-b-2-gpu-h200", + "stage-c-8-gpu-h100", + "stage-c-4-gpu-mi350", + } def test_domain_label_adds_only_stages_with_matching_tests(): registrations = [ - _registration("tests/fast-gpu/test_always.py", "stage-b-2-gpu-h200"), + _registration("tests/fast-gpu/test_precision.py", "stage-b-2-gpu-h200"), _registration("tests/e2e/test_megatron.py", "stage-c-8-gpu-h100", labels=["megatron"]), ] changed_files = (ChangedFile("M", ("docs/index.md",)),) @@ -114,7 +118,7 @@ def test_domain_label_adds_only_stages_with_matching_tests(): def test_broad_scope_adds_every_runnable_stage(): registrations = [ - _registration("tests/fast-gpu/test_always.py", "stage-b-2-gpu-h200"), + _registration("tests/fast-gpu/test_precision.py", "stage-b-2-gpu-h200"), _registration("tests/e2e/test_megatron.py", "stage-c-8-gpu-h100", labels=["megatron"]), ] changed_files = (ChangedFile("M", ("docs/index.md",)),) @@ -125,7 +129,7 @@ def test_broad_scope_adds_every_runnable_stage(): def test_bypass_fastfail_does_not_make_a_docs_change_affect_gpu_stages(): - registrations = [_registration("tests/fast-gpu/test_always.py", "stage-b-2-gpu-h200")] + registrations = [_registration("tests/fast-gpu/test_precision.py", "stage-b-2-gpu-h200")] changed_files = (ChangedFile("M", ("docs/index.md",)),) assert set(_select(changed_files, registrations, raw_labels=("bypass-fastfail",))) == PR_GPU_STAGES @@ -158,11 +162,13 @@ def test_non_pr_or_missing_diff_never_prunes(event_name, changed_files): def test_changed_labeled_test_without_its_label_keeps_current_selection_semantics(): changed_test = "tests/e2e/test_megatron.py" registrations = [ - _registration("tests/fast-gpu/test_always.py", "stage-b-2-gpu-h200"), + _registration("tests/fast-gpu/test_precision.py", "stage-b-2-gpu-h200"), _registration(changed_test, "stage-c-8-gpu-h100", labels=["megatron"]), ] - assert set(_select((ChangedFile("M", (changed_test,)),), registrations)) == PR_GPU_STAGES + assert set(_select((ChangedFile("M", (changed_test,)),), registrations)) == ( + PR_GPU_STAGES - {"stage-b-2-gpu-h200"} + ) def test_cli_publishes_all_gpu_stages_for_docs_diff(tmp_path): diff --git a/tests/fast-gpu/test_det_process_group.py b/tests/fast-gpu/test_det_process_group.py index 30d0a5a144..d140e3d182 100644 --- a/tests/fast-gpu/test_det_process_group.py +++ b/tests/fast-gpu/test_det_process_group.py @@ -1,7 +1,7 @@ from tests.ci.ci_register import register_cuda_ci, register_rocm_ci -register_cuda_ci(est_time=240, suite="stage-c-4-gpu-h200", labels=[]) -register_rocm_ci(est_time=40, suite="nightly-stage-c-4-gpu-mi350", labels=[]) +register_cuda_ci(est_time=240, suite="stage-c-4-gpu-h200", labels=["precision"]) +register_rocm_ci(est_time=40, suite="nightly-stage-c-4-gpu-mi350", labels=["precision"]) import os import socket diff --git a/tests/fast-gpu/test_mxfp8_quantizer.py b/tests/fast-gpu/test_mxfp8_quantizer.py index 73976dc6b8..0550792b58 100644 --- a/tests/fast-gpu/test_mxfp8_quantizer.py +++ b/tests/fast-gpu/test_mxfp8_quantizer.py @@ -3,7 +3,7 @@ from tests.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=60, suite="stage-b-2-gpu-h200", - labels=[], + labels=["precision"], disabled="FIXME: re-enable after the MXFP8 H200 reference path is settled.", ) diff --git a/tests/fast-gpu/test_nvfp4_quantizer.py b/tests/fast-gpu/test_nvfp4_quantizer.py index b69158f9d5..3e393e1b87 100644 --- a/tests/fast-gpu/test_nvfp4_quantizer.py +++ b/tests/fast-gpu/test_nvfp4_quantizer.py @@ -3,7 +3,7 @@ from tests.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=60, suite="stage-b-2-gpu-h200", - labels=[], + labels=["precision"], disabled="Requires Blackwell/B200 CI runner for NVFP4.", ) diff --git a/tests/fast-gpu/test_quantizer_ci.py b/tests/fast-gpu/test_quantizer_ci.py index e83ce5bf0e..89ee117614 100644 --- a/tests/fast-gpu/test_quantizer_ci.py +++ b/tests/fast-gpu/test_quantizer_ci.py @@ -9,7 +9,7 @@ from tests.ci.ci_register import register_cuda_ci # The quantizer hardcodes `device="cuda"` throughout; this test drives it with # real CUDA tensors to exercise the ignore-rule name-matching path. Fast enough # for the GPU fast suite; only needs 1 GPU. -register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=[]) +register_cuda_ci(est_time=60, suite="stage-b-2-gpu-h200", labels=["precision"]) import pytest diff --git a/tests/fast-gpu/test_run_megatron_worker_main.py b/tests/fast-gpu/test_run_megatron_worker_main.py index daf55a5035..feb75d866d 100644 --- a/tests/fast-gpu/test_run_megatron_worker_main.py +++ b/tests/fast-gpu/test_run_megatron_worker_main.py @@ -13,12 +13,12 @@ from tests.ci.ci_register import register_cuda_ci, register_rocm_ci register_cuda_ci( est_time=30, suite="stage-b-2-gpu-h200", - labels=[], + labels=["megatron"], ) register_rocm_ci( est_time=20, suite="nightly-stage-c-2-gpu-mi350", - labels=[], + labels=["megatron"], ) import argparse diff --git a/tests/fast-gpu/test_semaphore.py b/tests/fast-gpu/test_semaphore.py index 9678257f74..ca7fda458a 100644 --- a/tests/fast-gpu/test_semaphore.py +++ b/tests/fast-gpu/test_semaphore.py @@ -8,7 +8,7 @@ from tests.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=60, suite="stage-b-2-gpu-h200", - labels=[], + labels=["sglang"], disabled="FIXME: re-enable after shared HTTP client concurrency is reset between cases.", )