mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-31 07:44:57 +08:00
build(cargo): keep conformance lockfile aligned with the root
`cargo --locked` only checks a lockfile against its own manifest. The conformance suites build openshell-sdk from a separate workspace with its own Cargo.lock, so the two lockfiles could resolve shared crates to different versions without any check failing. That is how aws-lc-sys 0.46 was picked up, which does not link against the AWS-LC Nix provides. - Add check_cargo_lockfile_alignment.py: aligned workspaces must lock every registry crate they share with the root at a version the root lockfile has. `--fix` re-locks a workspace from the root lockfile, which keeps coupled crate families such as futures-* consistent. - Enroll tests/suites/conformance and re-lock it from the root lockfile. The other workspaces that path-depend on root crates also drift and can be enrolled once aligned. - Run the check in rust:lockfiles:check and the Cargo lockfiles CI job, and its tests in `mise run test`. - Build test archives and run the test runner with --locked so a stale lockfile fails instead of being re-resolved. Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
@@ -239,6 +239,10 @@ jobs:
|
||||
shell: nix develop -c bash -euo pipefail {0}
|
||||
run: tasks/scripts/check-cargo-lockfiles.sh
|
||||
|
||||
- name: Verify Cargo lockfile alignment
|
||||
shell: nix develop -c bash -euo pipefail {0}
|
||||
run: uv run tasks/scripts/check_cargo_lockfile_alignment.py
|
||||
|
||||
rust-lint:
|
||||
name: Rust lint (${{ matrix.system }})
|
||||
needs: pr_metadata
|
||||
|
||||
+11
@@ -48,6 +48,17 @@ mise run test:rust # cargo test --workspace
|
||||
|
||||
Rust validation checks tracked Cargo lockfiles; run `mise run rust:lockfiles:check` to check them directly. If one is stale, refresh it with Cargo using its adjacent manifest, review the diff, and commit the update.
|
||||
|
||||
Workspaces that build root crates through path dependencies keep their own
|
||||
lockfiles, which `--locked` does not compare with the root's. The check also
|
||||
requires the workspaces listed in
|
||||
`tasks/scripts/check_cargo_lockfile_alignment.py` to lock shared crates at the
|
||||
root's versions. If it fails, re-lock them from the root lockfile and commit the
|
||||
result:
|
||||
|
||||
```shell
|
||||
uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix
|
||||
```
|
||||
|
||||
### PostgreSQL-backed tests
|
||||
|
||||
Tests that need a real PostgreSQL server, such as advisory-lock concurrency
|
||||
|
||||
+5
-2
@@ -18,8 +18,11 @@ run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts
|
||||
hide = true
|
||||
|
||||
["rust:lockfiles:check"]
|
||||
description = "Verify all tracked Cargo lockfiles are current"
|
||||
run = "tasks/scripts/check-cargo-lockfiles.sh"
|
||||
description = "Verify all tracked Cargo lockfiles are current and aligned with the root"
|
||||
run = [
|
||||
"tasks/scripts/check-cargo-lockfiles.sh",
|
||||
"uv run tasks/scripts/check_cargo_lockfile_alignment.py",
|
||||
]
|
||||
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/check-cargo-lockfiles.ps1"
|
||||
hide = true
|
||||
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# ///
|
||||
|
||||
"""Check that aligned Cargo workspaces lock shared crates like the root does.
|
||||
|
||||
Some workspaces outside the root build root crates through path
|
||||
dependencies, for example the conformance suites build `openshell-sdk`. Each
|
||||
workspace has its own Cargo.lock, and `cargo --locked` only checks a lockfile
|
||||
against its own manifest, so nothing stops the two lockfiles from resolving
|
||||
the same crate to different versions. The root's versions are the ones CI and
|
||||
the Nix toolchain are built for; a newer `aws-lc-sys`, for example, does not
|
||||
link against the AWS-LC that Nix provides.
|
||||
|
||||
This check fails when an aligned workspace locks a registry crate that the
|
||||
root lockfile also contains at a version the root lockfile does not have.
|
||||
With `--fix`, it re-locks each aligned workspace starting from the root
|
||||
lockfile.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
# Workspaces whose shared crates must match the root lockfile. Add a
|
||||
# workspace here once its lockfile has been aligned with `--fix`.
|
||||
ALIGNED_WORKSPACES = ("tests/suites/conformance",)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mismatch:
|
||||
workspace: str
|
||||
name: str
|
||||
version: str
|
||||
root_versions: tuple[str, ...]
|
||||
|
||||
def target(self) -> str | None:
|
||||
"""The root version semver-compatible with the locked one, if any."""
|
||||
key = compat_key(self.version)
|
||||
candidates = [v for v in self.root_versions if compat_key(v) == key]
|
||||
return max(candidates, key=version_tuple) if candidates else None
|
||||
|
||||
|
||||
def version_tuple(version: str) -> tuple[int, ...]:
|
||||
core = re.split(r"[-+]", version, maxsplit=1)[0]
|
||||
return tuple(int(part) for part in core.split("."))
|
||||
|
||||
|
||||
def compat_key(version: str) -> tuple[int, ...]:
|
||||
"""Cargo's caret-compatibility class: the leftmost non-zero component."""
|
||||
major, minor, patch = (*version_tuple(version), 0, 0, 0)[:3]
|
||||
if major:
|
||||
return (major,)
|
||||
if minor:
|
||||
return (0, minor)
|
||||
return (0, 0, patch)
|
||||
|
||||
|
||||
def registry_versions(lockfile: Path) -> dict[str, set[str]]:
|
||||
"""Versions of registry (non-path) packages in a lockfile, by name."""
|
||||
data = tomllib.loads(lockfile.read_text())
|
||||
versions: dict[str, set[str]] = {}
|
||||
for package in data.get("package", []):
|
||||
if "source" in package:
|
||||
versions.setdefault(package["name"], set()).add(package["version"])
|
||||
return versions
|
||||
|
||||
|
||||
def mismatches(root: Path, workspace: str) -> list[Mismatch]:
|
||||
root_versions = registry_versions(root / "Cargo.lock")
|
||||
found = []
|
||||
for name, versions in sorted(
|
||||
registry_versions(root / workspace / "Cargo.lock").items()
|
||||
):
|
||||
shared = root_versions.get(name)
|
||||
if shared is None:
|
||||
continue
|
||||
for version in sorted(versions - shared, key=version_tuple):
|
||||
found.append(
|
||||
Mismatch(
|
||||
workspace, name, version, tuple(sorted(shared, key=version_tuple))
|
||||
)
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
def fix(root: Path, workspace: str) -> list[Mismatch]:
|
||||
"""Re-lock the workspace starting from the root lockfile.
|
||||
|
||||
Coupled crates such as the `futures-*` family pin each other, so they
|
||||
cannot be moved one at a time. Seeding the workspace lockfile with the
|
||||
root's lets Cargo keep every root version that still fits, add only the
|
||||
crates the root does not have, and drop the rest. Returns what is still
|
||||
misaligned afterwards.
|
||||
"""
|
||||
shutil.copyfile(root / "Cargo.lock", root / workspace / "Cargo.lock")
|
||||
subprocess.run(
|
||||
[
|
||||
"cargo",
|
||||
"metadata",
|
||||
"--format-version",
|
||||
"1",
|
||||
"--manifest-path",
|
||||
str(root / workspace / "Cargo.toml"),
|
||||
],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
return mismatches(root, workspace)
|
||||
|
||||
|
||||
def report(problems: list[Mismatch]) -> None:
|
||||
for m in problems:
|
||||
target = m.target()
|
||||
hint = (
|
||||
f"; root has {target}"
|
||||
if target
|
||||
else f"; root has {', '.join(m.root_versions)}"
|
||||
)
|
||||
print(
|
||||
f"error: {m.workspace}: {m.name} {m.version} is not in the root Cargo.lock{hint}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument(
|
||||
"--fix", action="store_true", help="update aligned lockfiles to root versions"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
root = Path(
|
||||
subprocess.run(
|
||||
["git", "rev-parse", "--show-toplevel"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
)
|
||||
problems: list[Mismatch] = []
|
||||
for workspace in ALIGNED_WORKSPACES:
|
||||
print(f"Checking {workspace}/Cargo.lock against Cargo.lock")
|
||||
problems += fix(root, workspace) if args.fix else mismatches(root, workspace)
|
||||
|
||||
if problems:
|
||||
report(problems)
|
||||
if not args.fix:
|
||||
print(
|
||||
"Run `uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix` to align them.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""Tests for the Cargo lockfile alignment check."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SPEC = importlib.util.spec_from_file_location(
|
||||
"check_cargo_lockfile_alignment",
|
||||
Path(__file__).with_name("check_cargo_lockfile_alignment.py"),
|
||||
)
|
||||
assert _SPEC and _SPEC.loader
|
||||
alignment = importlib.util.module_from_spec(_SPEC)
|
||||
sys.modules[_SPEC.name] = alignment
|
||||
_SPEC.loader.exec_module(alignment)
|
||||
|
||||
REGISTRY = 'source = "registry+https://github.com/rust-lang/crates.io-index"'
|
||||
|
||||
|
||||
def lockfile(path: Path, packages: list[tuple[str, str, bool]]) -> None:
|
||||
"""Write a lockfile of (name, version, from_registry) packages."""
|
||||
entries = []
|
||||
for name, version, from_registry in packages:
|
||||
entry = f'[[package]]\nname = "{name}"\nversion = "{version}"\n'
|
||||
if from_registry:
|
||||
entry += REGISTRY + "\n"
|
||||
entries.append(entry)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("version = 4\n\n" + "\n".join(entries))
|
||||
|
||||
|
||||
def test_compat_key_follows_cargo_caret_rules() -> None:
|
||||
assert alignment.compat_key("1.52.1") == (1,)
|
||||
assert alignment.compat_key("0.44.0") == (0, 44)
|
||||
assert alignment.compat_key("0.0.3") == (0, 0, 3)
|
||||
assert alignment.compat_key("1.0.4+wasi-0.2.12") == (1,)
|
||||
|
||||
|
||||
def test_reports_shared_registry_crates_at_other_versions(tmp_path: Path) -> None:
|
||||
lockfile(
|
||||
tmp_path / "Cargo.lock",
|
||||
[
|
||||
("tokio", "1.52.1", True),
|
||||
("rand", "0.8.6", True),
|
||||
("rand", "0.9.4", True),
|
||||
("openshell-sdk", "0.0.0", False),
|
||||
],
|
||||
)
|
||||
lockfile(
|
||||
tmp_path / "ws" / "Cargo.lock",
|
||||
[
|
||||
("tokio", "1.53.1", True),
|
||||
("rand", "0.9.5", True),
|
||||
("rand", "0.8.6", True),
|
||||
("only-here", "2.0.0", True),
|
||||
("openshell-sdk", "0.0.0", False),
|
||||
],
|
||||
)
|
||||
|
||||
found = alignment.mismatches(tmp_path, "ws")
|
||||
|
||||
assert [(m.name, m.version, m.target()) for m in found] == [
|
||||
("rand", "0.9.5", "0.9.4"),
|
||||
("tokio", "1.53.1", "1.52.1"),
|
||||
]
|
||||
|
||||
|
||||
def test_no_target_without_a_compatible_root_version(tmp_path: Path) -> None:
|
||||
lockfile(tmp_path / "Cargo.lock", [("syn", "1.0.109", True)])
|
||||
lockfile(tmp_path / "ws" / "Cargo.lock", [("syn", "2.0.119", True)])
|
||||
|
||||
(found,) = alignment.mismatches(tmp_path, "ws")
|
||||
|
||||
assert found.target() is None
|
||||
|
||||
|
||||
def test_aligned_lockfiles_pass(tmp_path: Path) -> None:
|
||||
lockfile(tmp_path / "Cargo.lock", [("tokio", "1.52.1", True)])
|
||||
lockfile(tmp_path / "ws" / "Cargo.lock", [("tokio", "1.52.1", True)])
|
||||
|
||||
assert alignment.mismatches(tmp_path, "ws") == []
|
||||
@@ -23,6 +23,7 @@ depends = [
|
||||
"test:docs-website",
|
||||
"test:docs-nav",
|
||||
"test:maintainer-approval",
|
||||
"test:cargo-lockfile-alignment",
|
||||
]
|
||||
|
||||
["test:docs-website"]
|
||||
@@ -31,6 +32,10 @@ description = "Test the docs-website sync script"
|
||||
# dependencies and the script's runtime dependency, which live outside the project env.
|
||||
run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/sync_docs_website_test.py"
|
||||
|
||||
["test:cargo-lockfile-alignment"]
|
||||
description = "Test the Cargo lockfile alignment check"
|
||||
run = "uv run --no-project --with pytest pytest tasks/scripts/check_cargo_lockfile_alignment_test.py"
|
||||
|
||||
["test:docs-nav"]
|
||||
description = "Test the docs navigation check"
|
||||
run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/check_docs_nav_test.py"
|
||||
|
||||
@@ -58,6 +58,7 @@ let
|
||||
mkdir -p "$(dirname "$output")"
|
||||
cd "$root"
|
||||
cargo nextest archive \
|
||||
--locked \
|
||||
--manifest-path "$manifest_path" \
|
||||
--target ${target} \
|
||||
-p ${package} \
|
||||
|
||||
+1
-1
@@ -259,7 +259,7 @@ let
|
||||
text = ''
|
||||
root=$(git rev-parse --show-toplevel)
|
||||
export ANSIBLE_CONFIG="$root/tests/ansible/ansible.cfg"
|
||||
exec cargo run --quiet --manifest-path "$root/tests/runner/Cargo.toml" --bin openshell-test -- "$@"
|
||||
exec cargo run --quiet --locked --manifest-path "$root/tests/runner/Cargo.toml" --bin openshell-test -- "$@"
|
||||
'';
|
||||
};
|
||||
in
|
||||
|
||||
@@ -60,7 +60,7 @@ pub fn build(
|
||||
|
||||
let mut command = Command::new("cargo");
|
||||
command
|
||||
.args(["nextest", "archive", "--manifest-path"])
|
||||
.args(["nextest", "archive", "--locked", "--manifest-path"])
|
||||
.arg(&manifest)
|
||||
.args(["--target", target, "--archive-file"])
|
||||
.arg(&archive);
|
||||
|
||||
Generated
+457
-343
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user