build(cargo): keep conformance lockfile aligned with the root

`cargo --locked` only checks a lockfile against its own manifest. The
conformance suites build openshell-sdk from a separate workspace with its
own Cargo.lock, so the two lockfiles could resolve shared crates to
different versions without any check failing. That is how aws-lc-sys 0.46
was picked up, which does not link against the AWS-LC Nix provides.

- Add check_cargo_lockfile_alignment.py: aligned workspaces must lock every
  registry crate they share with the root at a version the root lockfile
  has. `--fix` re-locks a workspace from the root lockfile, which keeps
  coupled crate families such as futures-* consistent.
- Enroll tests/suites/conformance and re-lock it from the root lockfile.
  The other workspaces that path-depend on root crates also drift and can
  be enrolled once aligned.
- Run the check in rust:lockfiles:check and the Cargo lockfiles CI job,
  and its tests in `mise run test`.
- Build test archives and run the test runner with --locked so a stale
  lockfile fails instead of being re-resolved.

Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar
2026-10-15 23:44:58 +02:00
parent 898ca460ed
commit a37ebe9174
10 changed files with 739 additions and 347 deletions
+4
View File
@@ -239,6 +239,10 @@ jobs:
shell: nix develop -c bash -euo pipefail {0} shell: nix develop -c bash -euo pipefail {0}
run: tasks/scripts/check-cargo-lockfiles.sh run: tasks/scripts/check-cargo-lockfiles.sh
- name: Verify Cargo lockfile alignment
shell: nix develop -c bash -euo pipefail {0}
run: uv run tasks/scripts/check_cargo_lockfile_alignment.py
rust-lint: rust-lint:
name: Rust lint (${{ matrix.system }}) name: Rust lint (${{ matrix.system }})
needs: pr_metadata needs: pr_metadata
+11
View File
@@ -48,6 +48,17 @@ mise run test:rust # cargo test --workspace
Rust validation checks tracked Cargo lockfiles; run `mise run rust:lockfiles:check` to check them directly. If one is stale, refresh it with Cargo using its adjacent manifest, review the diff, and commit the update. Rust validation checks tracked Cargo lockfiles; run `mise run rust:lockfiles:check` to check them directly. If one is stale, refresh it with Cargo using its adjacent manifest, review the diff, and commit the update.
Workspaces that build root crates through path dependencies keep their own
lockfiles, which `--locked` does not compare with the root's. The check also
requires the workspaces listed in
`tasks/scripts/check_cargo_lockfile_alignment.py` to lock shared crates at the
root's versions. If it fails, re-lock them from the root lockfile and commit the
result:
```shell
uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix
```
### PostgreSQL-backed tests ### PostgreSQL-backed tests
Tests that need a real PostgreSQL server, such as advisory-lock concurrency Tests that need a real PostgreSQL server, such as advisory-lock concurrency
+5 -2
View File
@@ -18,8 +18,11 @@ run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts
hide = true hide = true
["rust:lockfiles:check"] ["rust:lockfiles:check"]
description = "Verify all tracked Cargo lockfiles are current" description = "Verify all tracked Cargo lockfiles are current and aligned with the root"
run = "tasks/scripts/check-cargo-lockfiles.sh" run = [
"tasks/scripts/check-cargo-lockfiles.sh",
"uv run tasks/scripts/check_cargo_lockfile_alignment.py",
]
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/check-cargo-lockfiles.ps1" run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/check-cargo-lockfiles.ps1"
hide = true hide = true
@@ -0,0 +1,169 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# /// script
# requires-python = ">=3.11"
# ///
"""Check that aligned Cargo workspaces lock shared crates like the root does.
Some workspaces outside the root build root crates through path
dependencies, for example the conformance suites build `openshell-sdk`. Each
workspace has its own Cargo.lock, and `cargo --locked` only checks a lockfile
against its own manifest, so nothing stops the two lockfiles from resolving
the same crate to different versions. The root's versions are the ones CI and
the Nix toolchain are built for; a newer `aws-lc-sys`, for example, does not
link against the AWS-LC that Nix provides.
This check fails when an aligned workspace locks a registry crate that the
root lockfile also contains at a version the root lockfile does not have.
With `--fix`, it re-locks each aligned workspace starting from the root
lockfile.
"""
from __future__ import annotations
import argparse
import re
import shutil
import subprocess
import sys
import tomllib
from dataclasses import dataclass
from pathlib import Path
# Workspaces whose shared crates must match the root lockfile. Add a
# workspace here once its lockfile has been aligned with `--fix`.
ALIGNED_WORKSPACES = ("tests/suites/conformance",)
@dataclass(frozen=True)
class Mismatch:
workspace: str
name: str
version: str
root_versions: tuple[str, ...]
def target(self) -> str | None:
"""The root version semver-compatible with the locked one, if any."""
key = compat_key(self.version)
candidates = [v for v in self.root_versions if compat_key(v) == key]
return max(candidates, key=version_tuple) if candidates else None
def version_tuple(version: str) -> tuple[int, ...]:
core = re.split(r"[-+]", version, maxsplit=1)[0]
return tuple(int(part) for part in core.split("."))
def compat_key(version: str) -> tuple[int, ...]:
"""Cargo's caret-compatibility class: the leftmost non-zero component."""
major, minor, patch = (*version_tuple(version), 0, 0, 0)[:3]
if major:
return (major,)
if minor:
return (0, minor)
return (0, 0, patch)
def registry_versions(lockfile: Path) -> dict[str, set[str]]:
"""Versions of registry (non-path) packages in a lockfile, by name."""
data = tomllib.loads(lockfile.read_text())
versions: dict[str, set[str]] = {}
for package in data.get("package", []):
if "source" in package:
versions.setdefault(package["name"], set()).add(package["version"])
return versions
def mismatches(root: Path, workspace: str) -> list[Mismatch]:
root_versions = registry_versions(root / "Cargo.lock")
found = []
for name, versions in sorted(
registry_versions(root / workspace / "Cargo.lock").items()
):
shared = root_versions.get(name)
if shared is None:
continue
for version in sorted(versions - shared, key=version_tuple):
found.append(
Mismatch(
workspace, name, version, tuple(sorted(shared, key=version_tuple))
)
)
return found
def fix(root: Path, workspace: str) -> list[Mismatch]:
"""Re-lock the workspace starting from the root lockfile.
Coupled crates such as the `futures-*` family pin each other, so they
cannot be moved one at a time. Seeding the workspace lockfile with the
root's lets Cargo keep every root version that still fits, add only the
crates the root does not have, and drop the rest. Returns what is still
misaligned afterwards.
"""
shutil.copyfile(root / "Cargo.lock", root / workspace / "Cargo.lock")
subprocess.run(
[
"cargo",
"metadata",
"--format-version",
"1",
"--manifest-path",
str(root / workspace / "Cargo.toml"),
],
check=True,
stdout=subprocess.DEVNULL,
)
return mismatches(root, workspace)
def report(problems: list[Mismatch]) -> None:
for m in problems:
target = m.target()
hint = (
f"; root has {target}"
if target
else f"; root has {', '.join(m.root_versions)}"
)
print(
f"error: {m.workspace}: {m.name} {m.version} is not in the root Cargo.lock{hint}",
file=sys.stderr,
)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument(
"--fix", action="store_true", help="update aligned lockfiles to root versions"
)
args = parser.parse_args(argv)
root = Path(
subprocess.run(
["git", "rev-parse", "--show-toplevel"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
)
problems: list[Mismatch] = []
for workspace in ALIGNED_WORKSPACES:
print(f"Checking {workspace}/Cargo.lock against Cargo.lock")
problems += fix(root, workspace) if args.fix else mismatches(root, workspace)
if problems:
report(problems)
if not args.fix:
print(
"Run `uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix` to align them.",
file=sys.stderr,
)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,85 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Tests for the Cargo lockfile alignment check."""
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
_SPEC = importlib.util.spec_from_file_location(
"check_cargo_lockfile_alignment",
Path(__file__).with_name("check_cargo_lockfile_alignment.py"),
)
assert _SPEC and _SPEC.loader
alignment = importlib.util.module_from_spec(_SPEC)
sys.modules[_SPEC.name] = alignment
_SPEC.loader.exec_module(alignment)
REGISTRY = 'source = "registry+https://github.com/rust-lang/crates.io-index"'
def lockfile(path: Path, packages: list[tuple[str, str, bool]]) -> None:
"""Write a lockfile of (name, version, from_registry) packages."""
entries = []
for name, version, from_registry in packages:
entry = f'[[package]]\nname = "{name}"\nversion = "{version}"\n'
if from_registry:
entry += REGISTRY + "\n"
entries.append(entry)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("version = 4\n\n" + "\n".join(entries))
def test_compat_key_follows_cargo_caret_rules() -> None:
assert alignment.compat_key("1.52.1") == (1,)
assert alignment.compat_key("0.44.0") == (0, 44)
assert alignment.compat_key("0.0.3") == (0, 0, 3)
assert alignment.compat_key("1.0.4+wasi-0.2.12") == (1,)
def test_reports_shared_registry_crates_at_other_versions(tmp_path: Path) -> None:
lockfile(
tmp_path / "Cargo.lock",
[
("tokio", "1.52.1", True),
("rand", "0.8.6", True),
("rand", "0.9.4", True),
("openshell-sdk", "0.0.0", False),
],
)
lockfile(
tmp_path / "ws" / "Cargo.lock",
[
("tokio", "1.53.1", True),
("rand", "0.9.5", True),
("rand", "0.8.6", True),
("only-here", "2.0.0", True),
("openshell-sdk", "0.0.0", False),
],
)
found = alignment.mismatches(tmp_path, "ws")
assert [(m.name, m.version, m.target()) for m in found] == [
("rand", "0.9.5", "0.9.4"),
("tokio", "1.53.1", "1.52.1"),
]
def test_no_target_without_a_compatible_root_version(tmp_path: Path) -> None:
lockfile(tmp_path / "Cargo.lock", [("syn", "1.0.109", True)])
lockfile(tmp_path / "ws" / "Cargo.lock", [("syn", "2.0.119", True)])
(found,) = alignment.mismatches(tmp_path, "ws")
assert found.target() is None
def test_aligned_lockfiles_pass(tmp_path: Path) -> None:
lockfile(tmp_path / "Cargo.lock", [("tokio", "1.52.1", True)])
lockfile(tmp_path / "ws" / "Cargo.lock", [("tokio", "1.52.1", True)])
assert alignment.mismatches(tmp_path, "ws") == []
+5
View File
@@ -23,6 +23,7 @@ depends = [
"test:docs-website", "test:docs-website",
"test:docs-nav", "test:docs-nav",
"test:maintainer-approval", "test:maintainer-approval",
"test:cargo-lockfile-alignment",
] ]
["test:docs-website"] ["test:docs-website"]
@@ -31,6 +32,10 @@ description = "Test the docs-website sync script"
# dependencies and the script's runtime dependency, which live outside the project env. # dependencies and the script's runtime dependency, which live outside the project env.
run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/sync_docs_website_test.py" run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/sync_docs_website_test.py"
["test:cargo-lockfile-alignment"]
description = "Test the Cargo lockfile alignment check"
run = "uv run --no-project --with pytest pytest tasks/scripts/check_cargo_lockfile_alignment_test.py"
["test:docs-nav"] ["test:docs-nav"]
description = "Test the docs navigation check" description = "Test the docs navigation check"
run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/check_docs_nav_test.py" run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/check_docs_nav_test.py"
+1
View File
@@ -58,6 +58,7 @@ let
mkdir -p "$(dirname "$output")" mkdir -p "$(dirname "$output")"
cd "$root" cd "$root"
cargo nextest archive \ cargo nextest archive \
--locked \
--manifest-path "$manifest_path" \ --manifest-path "$manifest_path" \
--target ${target} \ --target ${target} \
-p ${package} \ -p ${package} \
+1 -1
View File
@@ -259,7 +259,7 @@ let
text = '' text = ''
root=$(git rev-parse --show-toplevel) root=$(git rev-parse --show-toplevel)
export ANSIBLE_CONFIG="$root/tests/ansible/ansible.cfg" export ANSIBLE_CONFIG="$root/tests/ansible/ansible.cfg"
exec cargo run --quiet --manifest-path "$root/tests/runner/Cargo.toml" --bin openshell-test -- "$@" exec cargo run --quiet --locked --manifest-path "$root/tests/runner/Cargo.toml" --bin openshell-test -- "$@"
''; '';
}; };
in in
+1 -1
View File
@@ -60,7 +60,7 @@ pub fn build(
let mut command = Command::new("cargo"); let mut command = Command::new("cargo");
command command
.args(["nextest", "archive", "--manifest-path"]) .args(["nextest", "archive", "--locked", "--manifest-path"])
.arg(&manifest) .arg(&manifest)
.args(["--target", target, "--archive-file"]) .args(["--target", target, "--archive-file"])
.arg(&archive); .arg(&archive);
+457 -343
View File
File diff suppressed because it is too large Load Diff