mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-11-02 08:04:37 +08:00
test(runner): run test plans in tmachine environments
Add the openshell-test runner, which runs a test plan end to end: nix run .#test-runner -- test --plan tests/plans/cpu-tests.yaml - Plans (tests/plans/*.yaml) select families, suites, and environments. Unknown families, suites, environments, and fields fail before anything is built or booted. - The catalog is read from source: each tests/*.rs file of a family workspace containing #[testkit::test] functions is a suite. Parsing moves into testkit-spec, shared by the macro and the runner, so they cannot disagree about a test's ID, description, or requirements. - Jobs are grouped by environment so each one is provisioned once. Tests whose requirements the environment lacks are excluded by exact name and reported as unsupported, as are environments the host cannot boot. - Each family is cross-built into a nextest archive bundle for the guest, checked against the catalog, run in a tmachine session against the target formed from the installer's gateway section and the environment's capabilities, and its JUnit report, log, and (on failure) gateway diagnostics are copied to artifacts/test-results/<plan>/<env>/<family>/. - --dry-run prints resolved jobs and filtersets; --env and --suite narrow a plan; --keep leaves each VM up after its tests; `catalog` prints every suite and test as JSON. - The test-runner Nix package provides the cross toolchains, QEMU, Ansible, firmware, and tmachine configuration. Document test plans and conformance suites in TESTING.md. Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar
+58
@@ -387,6 +387,64 @@ Secrets and Vault storage backends one at a time:
|
||||
mise run e2e:kubernetes:credential-drivers
|
||||
```
|
||||
|
||||
### Test plans (`tests/plans/`)
|
||||
|
||||
A test plan selects suites and the environments to run them in. The runner
|
||||
(`tests/runner`) builds each family's suites for the guest, boots each
|
||||
environment in a tmachine VM, runs the suites inside it, and copies the
|
||||
results back:
|
||||
|
||||
```shell
|
||||
nix run .#build-artifacts
|
||||
nix run .#test-runner -- test --plan tests/plans/cpu-tests.yaml
|
||||
```
|
||||
|
||||
```yaml
|
||||
matrix:
|
||||
- type: conformance # family: a workspace under tests/suites/
|
||||
suites: [sandbox-lifecycle] # omit to run every suite in the family
|
||||
environments: [tmachine.ubuntu2404-arm64.docker-rootful.deb]
|
||||
```
|
||||
|
||||
Add `--dry-run` to print the resolved jobs and nextest filtersets without
|
||||
building or booting anything, `--env` or `--suite` to narrow a plan, and
|
||||
`--keep` to leave each VM running after its tests. Results, including JUnit
|
||||
reports and gateway logs for failures, go to
|
||||
`artifacts/test-results/<plan>/<environment>/<family>/`.
|
||||
|
||||
Environments are defined in `tests/environments/*.yaml`. tmachine boots guests
|
||||
of the host's architecture only, so the runner reports environments for other
|
||||
architectures as unsupported instead of failing.
|
||||
|
||||
Conformance suites (`tests/suites/conformance/`) are black-box tests that drive
|
||||
the gateway through the Rust SDK. Each suite is one test binary of
|
||||
`#[testkit::test]` functions:
|
||||
|
||||
```rust
|
||||
/// A long-running sandbox becomes ready, runs a command with exec, and is gone
|
||||
/// from get and list after delete.
|
||||
#[testkit::test]
|
||||
async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result {
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
The function name is the test ID, and the doc comment is its required
|
||||
description. Declare capabilities a test needs with
|
||||
`#[testkit::test(requires = [service_exposure])]`; the runner excludes the test
|
||||
in environments without them and reports it as unsupported. Tests read the
|
||||
system under test from the target file named by `OPENSHELL_TEST_TARGET`, so you
|
||||
can also run a suite directly against a gateway you already have:
|
||||
|
||||
```shell
|
||||
OPENSHELL_TEST_TARGET=target.yaml cargo nextest run \
|
||||
--manifest-path tests/suites/conformance/Cargo.toml \
|
||||
-p openshell-conformance-sandbox-lifecycle
|
||||
```
|
||||
|
||||
`cargo run --manifest-path tests/runner/Cargo.toml --bin openshell-test -- catalog`
|
||||
prints every suite and test with its description and requirements.
|
||||
|
||||
### Kubernetes E2E (`e2e/rust/e2e-kubernetes.sh`)
|
||||
|
||||
Kubernetes e2e tests deploy an OpenShell gateway into a real Kubernetes cluster
|
||||
|
||||
@@ -127,7 +127,7 @@
|
||||
firmwarePkgs = testGuestPkgs;
|
||||
};
|
||||
testMachines = import ./tests/config.nix {
|
||||
inherit pkgs toolchains;
|
||||
inherit pkgs toolchains rustToolchain;
|
||||
qemuPkgs = tmachineRuntimePkgs;
|
||||
firmwarePkgs = tmachineRuntimePkgs;
|
||||
};
|
||||
@@ -199,6 +199,7 @@
|
||||
tmachine = testMachines.package;
|
||||
tmachine-config = testMachines.config;
|
||||
tmachine-unwrapped = testMachines.unwrapped;
|
||||
test-runner = testMachines.testRunner;
|
||||
};
|
||||
|
||||
devShells = {
|
||||
|
||||
+31
-1
@@ -4,6 +4,7 @@
|
||||
{
|
||||
pkgs,
|
||||
toolchains,
|
||||
rustToolchain,
|
||||
qemuPkgs ? pkgs,
|
||||
firmwarePkgs ? pkgs,
|
||||
}:
|
||||
@@ -232,9 +233,38 @@ let
|
||||
exec ${tmachine}/bin/tmachine --config ${config} "$@"
|
||||
'';
|
||||
};
|
||||
# Runs test plans with tests/runner. It cross-builds guest test bundles and
|
||||
# drives tmachine sessions, so it needs the toolchains and the VM tools.
|
||||
testRunner = pkgs.writeShellApplication {
|
||||
name = "test-runner";
|
||||
runtimeInputs = [
|
||||
qemu
|
||||
pkgs.ansible
|
||||
pkgs.cargo-nextest
|
||||
pkgs.coreutils
|
||||
pkgs.git
|
||||
pkgs.gnutar
|
||||
pkgs.sshpass
|
||||
rustToolchain
|
||||
];
|
||||
runtimeEnv =
|
||||
pkgs.lib.foldl' (env: toolchain: env // toolchain.env) { } (builtins.attrValues toolchains)
|
||||
// {
|
||||
TMACHINE_CONFIG = "${config}";
|
||||
}
|
||||
// pkgs.lib.optionalAttrs isAarch64 {
|
||||
TMACHINE_FIRMWARE_CODE = firmwarePkgs.OVMF.firmware;
|
||||
TMACHINE_FIRMWARE_VARS = firmwarePkgs.OVMF.variables;
|
||||
};
|
||||
text = ''
|
||||
root=$(git rev-parse --show-toplevel)
|
||||
export ANSIBLE_CONFIG="$root/tests/ansible/ansible.cfg"
|
||||
exec cargo run --quiet --manifest-path "$root/tests/runner/Cargo.toml" --bin openshell-test -- "$@"
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
package = runner;
|
||||
unwrapped = tmachine;
|
||||
inherit config;
|
||||
inherit config testRunner;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
matrix:
|
||||
- type: conformance
|
||||
suites: [sandbox-lifecycle]
|
||||
environments:
|
||||
- tmachine.ubuntu2404-amd64.docker-rootful.deb
|
||||
- tmachine.ubuntu2404-arm64.docker-rootful.deb
|
||||
Generated
+669
-1
@@ -2,24 +2,304 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "annotate-snippets"
|
||||
version = "0.12.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"memchr",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "arraydeque"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||
|
||||
[[package]]
|
||||
name = "blake3"
|
||||
version = "1.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"constant_time_eq",
|
||||
"cpufeatures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
|
||||
|
||||
[[package]]
|
||||
name = "core_detect"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "directories"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d"
|
||||
dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"core_detect",
|
||||
"multiversion_no_op",
|
||||
"rustversion",
|
||||
"scopeguard",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs_io"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "granit-parser"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e20f99e46474f56bd905c56e817ebddcf377a611f94c53ac4649e4d3fa3c0cd0"
|
||||
dependencies = [
|
||||
"arraydeque",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.2"
|
||||
@@ -32,18 +312,68 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.190"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multiversion_no_op"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
|
||||
|
||||
[[package]]
|
||||
name = "noyalib"
|
||||
version = "0.0.53"
|
||||
@@ -59,15 +389,63 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "openshell-test-runner"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"noyalib",
|
||||
"openshell-testkit-spec",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tmachine",
|
||||
"tokio",
|
||||
"toml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openshell-testkit-spec"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "option-ext"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
@@ -86,12 +464,53 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0"
|
||||
dependencies = [
|
||||
"libredox",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
@@ -102,6 +521,20 @@ dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde-saphyr"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8050abb251097357e24aff63ba2c52a6309ecb7d23a5474023df960a02694d8"
|
||||
dependencies = [
|
||||
"annotate-snippets",
|
||||
"encoding_rs_io",
|
||||
"granit-parser",
|
||||
"num-traits",
|
||||
"serde_core",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
@@ -119,15 +552,86 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.154"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
|
||||
dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
@@ -139,8 +643,172 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tmachine"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
"clap",
|
||||
"directories",
|
||||
"serde",
|
||||
"serde-saphyr",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.53.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
"mio",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
"tokio-macros",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.9.12+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"serde_core",
|
||||
"serde_spanned",
|
||||
"toml_datetime",
|
||||
"toml_parser",
|
||||
"toml_writer",
|
||||
"winnow 0.7.15",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "0.7.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_parser"
|
||||
version = "1.1.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c"
|
||||
dependencies = [
|
||||
"winnow 1.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_writer"
|
||||
version = "1.1.3+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
@@ -11,9 +11,20 @@ license = "Apache-2.0"
|
||||
repository = "https://github.com/NVIDIA/OpenShell"
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-test"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
openshell-testkit-spec = { path = "../suites/conformance/testkit-spec" }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] }
|
||||
tempfile = "3"
|
||||
tmachine = { path = "../tmachine" }
|
||||
tokio = { version = "1", features = ["macros", "rt"] }
|
||||
toml = "0.9"
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Building a family's test bundle for the guest.
|
||||
//!
|
||||
//! A bundle is a tar holding a nextest archive built for the guest target,
|
||||
//! the family workspace's `Cargo.toml` files (for `--workspace-remap`), and a
|
||||
//! `nextest` config with the runner's `JUnit` profile. The guest needs only the
|
||||
//! `cargo-nextest` binary to run it.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use serde::Deserialize;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use crate::catalog::Family;
|
||||
use crate::environments::Arch;
|
||||
|
||||
/// The nextest profile the runner uses in the guest.
|
||||
pub const PROFILE: &str = "openshell";
|
||||
|
||||
const NEXTEST_CONFIG: &str = r#"# Written by the OpenShell test runner.
|
||||
[profile.openshell]
|
||||
fail-fast = false
|
||||
|
||||
[profile.openshell.junit]
|
||||
path = "junit.xml"
|
||||
"#;
|
||||
|
||||
pub struct Bundle {
|
||||
pub path: PathBuf,
|
||||
_dir: TempDir,
|
||||
}
|
||||
|
||||
/// The Rust target for guests of `arch`. Static musl binaries don't depend
|
||||
/// on the guest's libc.
|
||||
pub const fn guest_target(arch: Arch) -> &'static str {
|
||||
match arch {
|
||||
Arch::Amd64 => "x86_64-unknown-linux-musl",
|
||||
Arch::Arm64 => "aarch64-unknown-linux-musl",
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the bundle for `packages`, and check that the archive contains a
|
||||
/// binary for every suite in `binaries`.
|
||||
pub fn build(
|
||||
family: &Family,
|
||||
packages: &BTreeSet<String>,
|
||||
binaries: &BTreeSet<String>,
|
||||
target: &str,
|
||||
) -> Result<Bundle> {
|
||||
let dir = tempfile::tempdir().context("create bundle directory")?;
|
||||
let stage = dir.path().join("stage");
|
||||
std::fs::create_dir(&stage)?;
|
||||
let manifest = family.workspace.join("Cargo.toml");
|
||||
let archive = stage.join("tests.tar.zst");
|
||||
|
||||
let mut command = Command::new("cargo");
|
||||
command
|
||||
.args(["nextest", "archive", "--manifest-path"])
|
||||
.arg(&manifest)
|
||||
.args(["--target", target, "--archive-file"])
|
||||
.arg(&archive);
|
||||
for package in packages {
|
||||
command.args(["-p", package]);
|
||||
}
|
||||
run(&mut command).with_context(|| format!("build {} test archive", family.name))?;
|
||||
|
||||
let archived = archived_binaries(&manifest, &archive)?;
|
||||
let missing: Vec<_> = binaries.difference(&archived).collect();
|
||||
ensure!(
|
||||
missing.is_empty(),
|
||||
"the {} catalog lists suites the archive has no test binary for: {missing:?}; archived: {archived:?}",
|
||||
family.name
|
||||
);
|
||||
|
||||
copy_manifests(&family.workspace, &stage)?;
|
||||
std::fs::write(stage.join("nextest.toml"), NEXTEST_CONFIG)?;
|
||||
|
||||
let path = dir.path().join(format!("{}-tests.tar", family.name));
|
||||
run(Command::new("tar")
|
||||
.arg("-C")
|
||||
.arg(&stage)
|
||||
.arg("-cf")
|
||||
.arg(&path)
|
||||
.arg("."))
|
||||
.context("pack test bundle")?;
|
||||
Ok(Bundle { path, _dir: dir })
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ListOutput {
|
||||
#[serde(rename = "rust-binaries")]
|
||||
rust_binaries: std::collections::BTreeMap<String, ListSuite>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ListSuite {
|
||||
#[serde(rename = "binary-name")]
|
||||
binary_name: String,
|
||||
kind: String,
|
||||
}
|
||||
|
||||
/// Names of the test binaries in an archive, read without executing them.
|
||||
fn archived_binaries(manifest: &Path, archive: &Path) -> Result<BTreeSet<String>> {
|
||||
let output = Command::new("cargo")
|
||||
.args(["nextest", "list", "--manifest-path"])
|
||||
.arg(manifest)
|
||||
.arg("--archive-file")
|
||||
.arg(archive)
|
||||
.args(["--list-type", "binaries-only", "--message-format", "json"])
|
||||
.output()
|
||||
.context("run cargo nextest list")?;
|
||||
ensure!(
|
||||
output.status.success(),
|
||||
"cargo nextest list exited with {}: {}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let list: ListOutput =
|
||||
serde_json::from_slice(&output.stdout).context("parse cargo nextest list output")?;
|
||||
Ok(list
|
||||
.rust_binaries
|
||||
.into_values()
|
||||
.filter(|suite| suite.kind == "test")
|
||||
.map(|suite| suite.binary_name)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Copy every `Cargo.toml` under `workspace` into `stage`, keeping relative
|
||||
/// paths and skipping build output.
|
||||
fn copy_manifests(workspace: &Path, stage: &Path) -> Result<()> {
|
||||
fn walk(root: &Path, dir: &Path, stage: &Path) -> Result<()> {
|
||||
for entry in std::fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? {
|
||||
let path = entry?.path();
|
||||
let name = path.file_name().unwrap_or_default();
|
||||
if path.is_dir() {
|
||||
if name != "target" && name != ".git" {
|
||||
walk(root, &path, stage)?;
|
||||
}
|
||||
} else if name == "Cargo.toml" {
|
||||
let dest = stage.join(path.strip_prefix(root)?);
|
||||
std::fs::create_dir_all(dest.parent().unwrap_or(stage))?;
|
||||
std::fs::copy(&path, &dest).with_context(|| format!("copy {}", path.display()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
walk(workspace, workspace, stage)
|
||||
}
|
||||
|
||||
fn run(command: &mut Command) -> Result<()> {
|
||||
let status = command
|
||||
.status()
|
||||
.with_context(|| format!("run {command:?}"))?;
|
||||
if !status.success() {
|
||||
bail!("{command:?} exited with {status}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! The test catalog: every family, suite, and test, read from source.
|
||||
//!
|
||||
//! A family is a Cargo workspace under `tests/suites/`. A suite is one test
|
||||
//! binary (`<crate>/tests/<suite_name>.rs`) containing `#[testkit::test]`
|
||||
//! functions; the suite name is the binary name with `_` replaced by `-`.
|
||||
//! Tests are parsed with `testkit-spec`, the same parser the macro uses, so
|
||||
//! the runner never executes test binaries on the host to learn about them.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Families the runner knows how to run, and their workspaces relative to
|
||||
/// the repository root.
|
||||
const FAMILIES: &[(&str, &str)] = &[("conformance", "tests/suites/conformance")];
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Family {
|
||||
pub name: String,
|
||||
#[serde(skip)]
|
||||
pub workspace: PathBuf,
|
||||
pub suites: BTreeMap<String, Suite>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct Suite {
|
||||
pub suite: String,
|
||||
pub binary: String,
|
||||
pub package: String,
|
||||
pub tests: Vec<Test>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct Test {
|
||||
pub id: String,
|
||||
pub description: String,
|
||||
pub requires: Vec<String>,
|
||||
}
|
||||
|
||||
/// Every known family, keyed by name.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Catalog {
|
||||
pub families: BTreeMap<String, Family>,
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
pub fn load(root: &Path) -> Result<Self> {
|
||||
let mut families = BTreeMap::new();
|
||||
for (name, workspace) in FAMILIES {
|
||||
let family = Family::load(name, &root.join(workspace))
|
||||
.with_context(|| format!("load {name} test catalog"))?;
|
||||
families.insert((*name).to_string(), family);
|
||||
}
|
||||
Ok(Self { families })
|
||||
}
|
||||
|
||||
pub fn family(&self, name: &str) -> Result<&Family> {
|
||||
self.families.get(name).with_context(|| {
|
||||
format!(
|
||||
"unknown test family {name:?}; known families: {}",
|
||||
self.families.keys().cloned().collect::<Vec<_>>().join(", ")
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct WorkspaceManifest {
|
||||
workspace: WorkspaceSection,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct WorkspaceSection {
|
||||
members: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PackageManifest {
|
||||
package: PackageSection,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PackageSection {
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl Family {
|
||||
fn load(name: &str, workspace: &Path) -> Result<Self> {
|
||||
let manifest: WorkspaceManifest = read_toml(&workspace.join("Cargo.toml"))?;
|
||||
let mut suites = BTreeMap::new();
|
||||
for member in manifest.workspace.members {
|
||||
let member_dir = workspace.join(&member);
|
||||
for suite in member_suites(&member_dir)? {
|
||||
if let Some(existing) = suites.insert(suite.suite.clone(), suite) {
|
||||
bail!(
|
||||
"suite {:?} is defined more than once (again in {member})",
|
||||
existing.suite
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
name: name.to_string(),
|
||||
workspace: workspace.to_path_buf(),
|
||||
suites,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn suite(&self, name: &str) -> Result<&Suite> {
|
||||
self.suites.get(name).with_context(|| {
|
||||
format!(
|
||||
"unknown {} suite {name:?}; known suites: {}",
|
||||
self.name,
|
||||
self.suites.keys().cloned().collect::<Vec<_>>().join(", ")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Packages containing the given suites.
|
||||
pub fn packages<'a>(&self, suites: impl IntoIterator<Item = &'a str>) -> BTreeSet<String> {
|
||||
suites
|
||||
.into_iter()
|
||||
.filter_map(|suite| self.suites.get(suite))
|
||||
.map(|suite| suite.package.clone())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Suites in one workspace member: its `tests/*.rs` files that contain
|
||||
/// `#[testkit::test]` functions.
|
||||
fn member_suites(member_dir: &Path) -> Result<Vec<Suite>> {
|
||||
let tests_dir = member_dir.join("tests");
|
||||
if !tests_dir.is_dir() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let package: PackageManifest = read_toml(&member_dir.join("Cargo.toml"))?;
|
||||
|
||||
let mut files: Vec<_> = std::fs::read_dir(&tests_dir)
|
||||
.with_context(|| format!("read {}", tests_dir.display()))?
|
||||
.map(|entry| entry.map(|entry| entry.path()))
|
||||
.collect::<std::io::Result<_>>()?;
|
||||
files.retain(|path| path.extension().is_some_and(|ext| ext == "rs"));
|
||||
files.sort();
|
||||
|
||||
let mut suites = Vec::new();
|
||||
for file in files {
|
||||
let source =
|
||||
std::fs::read_to_string(&file).with_context(|| format!("read {}", file.display()))?;
|
||||
let specs = testkit_spec::scan(&source)
|
||||
.map_err(|error| anyhow::anyhow!("{}: {error}", file.display()))?;
|
||||
if specs.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let binary = file
|
||||
.file_stem()
|
||||
.and_then(|stem| stem.to_str())
|
||||
.with_context(|| format!("test file name {} is not UTF-8", file.display()))?
|
||||
.to_string();
|
||||
suites.push(Suite {
|
||||
suite: binary.replace('_', "-"),
|
||||
binary,
|
||||
package: package.package.name.clone(),
|
||||
tests: specs
|
||||
.into_iter()
|
||||
.map(|spec| Test {
|
||||
id: spec.id.to_string(),
|
||||
description: spec.description,
|
||||
requires: spec.requires.iter().map(ToString::to_string).collect(),
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
Ok(suites)
|
||||
}
|
||||
|
||||
fn read_toml<T: serde::de::DeserializeOwned>(path: &Path) -> Result<T> {
|
||||
let text = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
|
||||
toml::from_str(&text).with_context(|| format!("parse {}", path.display()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn repo_root() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repository_catalog_has_sandbox_lifecycle() {
|
||||
let catalog = Catalog::load(&repo_root()).unwrap();
|
||||
let family = catalog.family("conformance").unwrap();
|
||||
let suite = family.suite("sandbox-lifecycle").unwrap();
|
||||
assert_eq!(suite.binary, "sandbox_lifecycle");
|
||||
assert_eq!(suite.package, "openshell-conformance-sandbox-lifecycle");
|
||||
assert!(
|
||||
suite
|
||||
.tests
|
||||
.iter()
|
||||
.any(|test| test.id == "long_running_exec_delete" && !test.description.is_empty())
|
||||
);
|
||||
// CLI-driven tests in the same workspace are not testkit suites.
|
||||
assert!(family.suites.keys().all(|name| name != "file-transfer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_names_list_known_ones() {
|
||||
let catalog = Catalog::load(&repo_root()).unwrap();
|
||||
let err = catalog.family("features").unwrap_err();
|
||||
assert!(err.to_string().contains("conformance"), "{err}");
|
||||
let err = catalog
|
||||
.family("conformance")
|
||||
.unwrap()
|
||||
.suite("sandbox-lifecyle")
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("sandbox-lifecycle"), "{err}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Running a job: provision the environment, run each family's bundle in it,
|
||||
//! and bring the results back.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
use std::process::Output;
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use tmachine::Session;
|
||||
use tmachine::config::Config;
|
||||
|
||||
use crate::bundle::{Bundle, PROFILE};
|
||||
use crate::jobs::{FamilyJob, Job};
|
||||
use crate::report::{Outcome, Status, junit_counts};
|
||||
|
||||
/// Where the installer describes the gateway in the guest.
|
||||
const GATEWAY_SECTION: &str = "/etc/openshell-test/gateway.yaml";
|
||||
const GUEST_ROOT: &str = "/var/tmp/openshell-tests";
|
||||
|
||||
pub struct Options<'a> {
|
||||
pub tmachine: &'a Config,
|
||||
pub bundles: &'a BTreeMap<String, Bundle>,
|
||||
pub results: &'a Path,
|
||||
pub keep: bool,
|
||||
}
|
||||
|
||||
/// Run one job. Every family gets an outcome, even when provisioning fails.
|
||||
pub async fn run(job: &Job, options: &Options<'_>) -> Vec<Outcome> {
|
||||
let env_id = &job.environment.id;
|
||||
let outcome = |family: &FamilyJob, status| Outcome {
|
||||
environment: env_id.clone(),
|
||||
family: family.family.clone(),
|
||||
status,
|
||||
unsupported_tests: family
|
||||
.suites
|
||||
.iter()
|
||||
.map(|suite| suite.unsupported.len())
|
||||
.sum(),
|
||||
};
|
||||
|
||||
if let Some(reason) = &job.unsupported {
|
||||
return job
|
||||
.families
|
||||
.iter()
|
||||
.map(|family| outcome(family, Status::Unsupported(reason.clone())))
|
||||
.collect();
|
||||
}
|
||||
if !job.families.iter().any(FamilyJob::has_tests) {
|
||||
return job
|
||||
.families
|
||||
.iter()
|
||||
.map(|family| outcome(family, Status::NothingToRun))
|
||||
.collect();
|
||||
}
|
||||
|
||||
eprintln!("==> {env_id}: provisioning");
|
||||
let session = match provision(job, options.tmachine).await {
|
||||
Ok(session) => session,
|
||||
Err(error) => {
|
||||
let message = format!("provision environment: {error:#}");
|
||||
eprintln!("==> {env_id}: {message}");
|
||||
return job
|
||||
.families
|
||||
.iter()
|
||||
.map(|family| outcome(family, Status::Error(message.clone())))
|
||||
.collect();
|
||||
}
|
||||
};
|
||||
|
||||
let mut outcomes = Vec::new();
|
||||
for family in &job.families {
|
||||
if !family.has_tests() {
|
||||
outcomes.push(outcome(family, Status::NothingToRun));
|
||||
continue;
|
||||
}
|
||||
let dir = options.results.join(env_id).join(&family.family);
|
||||
eprintln!(
|
||||
"==> {env_id}: running {} ({})",
|
||||
family.family,
|
||||
family.filterset()
|
||||
);
|
||||
let status = match run_family(&session, job, family, options, &dir).await {
|
||||
Ok(status) => status,
|
||||
Err(error) => Status::Error(format!("{error:#}")),
|
||||
};
|
||||
if status.is_failure()
|
||||
&& let Err(error) = session.collect_diagnostics(&dir.join("diagnostics")).await
|
||||
{
|
||||
eprintln!("==> {env_id}: collecting diagnostics failed: {error:#}");
|
||||
}
|
||||
eprintln!("==> {env_id}: {} {status:?}", family.family);
|
||||
outcomes.push(outcome(family, status));
|
||||
}
|
||||
|
||||
if options.keep {
|
||||
eprintln!(
|
||||
"==> {env_id}: keeping the VM. Connect with `ssh -p 2222 tmachine@127.0.0.1` (password `tmachine`). Press Enter to tear it down."
|
||||
);
|
||||
let mut line = String::new();
|
||||
let _ = std::io::stdin().read_line(&mut line);
|
||||
}
|
||||
eprintln!("==> {env_id}: tearing down");
|
||||
session.down().await;
|
||||
outcomes
|
||||
}
|
||||
|
||||
async fn provision(job: &Job, config: &Config) -> Result<Session> {
|
||||
let params = job
|
||||
.environment
|
||||
.tmachine
|
||||
.as_ref()
|
||||
.context("environment has no tmachine section")?;
|
||||
let environment = config
|
||||
.environments
|
||||
.iter()
|
||||
.find(|candidate| candidate.name == params.environment)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"tmachine environment {:?} is not defined",
|
||||
params.environment
|
||||
)
|
||||
})?;
|
||||
let machine = config
|
||||
.machines
|
||||
.iter()
|
||||
.find(|candidate| candidate.name == environment.machine)
|
||||
.with_context(|| format!("tmachine machine {:?} is not defined", environment.machine))?;
|
||||
let installer = config
|
||||
.installers
|
||||
.iter()
|
||||
.find(|candidate| candidate.name == params.installer)
|
||||
.with_context(|| format!("tmachine installer {:?} is not defined", params.installer))?;
|
||||
Session::up(machine, environment, installer).await
|
||||
}
|
||||
|
||||
async fn run_family(
|
||||
session: &Session,
|
||||
job: &Job,
|
||||
family: &FamilyJob,
|
||||
options: &Options<'_>,
|
||||
dir: &Path,
|
||||
) -> Result<Status> {
|
||||
if dir.exists() {
|
||||
std::fs::remove_dir_all(dir).with_context(|| format!("clear {}", dir.display()))?;
|
||||
}
|
||||
std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
|
||||
let bundle = options
|
||||
.bundles
|
||||
.get(&family.family)
|
||||
.with_context(|| format!("no bundle was built for {}", family.family))?;
|
||||
|
||||
// The installer knows how to reach the gateway; the environment
|
||||
// definition knows what it provides. Together they are the target.
|
||||
let gateway = checked(
|
||||
"read the installer's gateway description",
|
||||
session.exec(&["cat", GATEWAY_SECTION]).await?,
|
||||
)?;
|
||||
let capabilities = serde_yml::to_string(&BTreeMap::from([(
|
||||
"capabilities",
|
||||
&job.environment.capabilities,
|
||||
)]))?;
|
||||
let target = dir.join("target.yaml");
|
||||
std::fs::write(&target, format!("{gateway}{capabilities}"))?;
|
||||
|
||||
let guest = format!("{GUEST_ROOT}/{}", family.family);
|
||||
checked(
|
||||
"prepare guest directory",
|
||||
session
|
||||
.exec(&[
|
||||
"sh",
|
||||
"-c",
|
||||
&format!("rm -rf {guest} && mkdir -p {guest}/extract"),
|
||||
])
|
||||
.await?,
|
||||
)?;
|
||||
session
|
||||
.copy_in(&bundle.path, &format!("{guest}/bundle.tar"))
|
||||
.await?;
|
||||
session
|
||||
.copy_in(&target, &format!("{guest}/target.yaml"))
|
||||
.await?;
|
||||
checked(
|
||||
"unpack bundle",
|
||||
session
|
||||
.exec(&["tar", "-C", &guest, "-xf", &format!("{guest}/bundle.tar")])
|
||||
.await?,
|
||||
)?;
|
||||
|
||||
let filterset = family.filterset();
|
||||
let output = session
|
||||
.exec(&[
|
||||
"env",
|
||||
&format!("OPENSHELL_TEST_TARGET={guest}/target.yaml"),
|
||||
"cargo-nextest",
|
||||
"nextest",
|
||||
"run",
|
||||
"--archive-file",
|
||||
&format!("{guest}/tests.tar.zst"),
|
||||
"--workspace-remap",
|
||||
&guest,
|
||||
"--extract-to",
|
||||
&format!("{guest}/extract"),
|
||||
"--extract-overwrite",
|
||||
"--config-file",
|
||||
&format!("{guest}/nextest.toml"),
|
||||
"--profile",
|
||||
PROFILE,
|
||||
"--no-capture",
|
||||
"-E",
|
||||
&filterset,
|
||||
])
|
||||
.await?;
|
||||
let mut log = output.stdout.clone();
|
||||
log.extend_from_slice(&output.stderr);
|
||||
std::fs::write(dir.join("nextest.log"), &log)?;
|
||||
|
||||
let junit = dir.join("junit.xml");
|
||||
let found = checked(
|
||||
"find JUnit report",
|
||||
session
|
||||
.exec(&["find", &guest, "-name", "junit.xml"])
|
||||
.await?,
|
||||
)?;
|
||||
let counts = match found.lines().next() {
|
||||
Some(path) => {
|
||||
session.copy_out(path, &junit).await?;
|
||||
junit_counts(&std::fs::read_to_string(&junit)?)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
Ok(match (output.status.success(), counts) {
|
||||
(true, Some((tests, 0))) => Status::Passed { tests },
|
||||
(true, counts) => bail!(
|
||||
"nextest succeeded but the JUnit report is missing or reports failures ({counts:?}); see {}",
|
||||
dir.join("nextest.log").display()
|
||||
),
|
||||
(false, Some((tests, failures))) if failures > 0 => Status::Failed { tests, failures },
|
||||
(false, _) => Status::Error(format!(
|
||||
"nextest exited with {}; see {}",
|
||||
output.status,
|
||||
dir.join("nextest.log").display()
|
||||
)),
|
||||
})
|
||||
}
|
||||
|
||||
fn checked(step: &str, output: Output) -> Result<String> {
|
||||
ensure!(
|
||||
output.status.success(),
|
||||
"{step}: exited with {}: {}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
);
|
||||
String::from_utf8(output.stdout).with_context(|| format!("{step}: output is not UTF-8"))
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Expanding a plan into jobs.
|
||||
//!
|
||||
//! Jobs are grouped by environment so each environment is provisioned once
|
||||
//! and runs every family and suite the plan selects for it. Tests whose
|
||||
//! requirements the environment lacks are excluded by exact name and
|
||||
//! reported as unsupported; libtest has no runtime skip, so this is the only
|
||||
//! place they can be skipped honestly.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
|
||||
use crate::catalog::{Catalog, Suite};
|
||||
use crate::environments::{Arch, Catalog as Environments, Environment};
|
||||
use crate::plan::Plan;
|
||||
|
||||
/// Everything the plan runs in one environment.
|
||||
#[derive(Debug)]
|
||||
pub struct Job {
|
||||
pub environment: Environment,
|
||||
/// Why the environment can't be provisioned on this host, if it can't.
|
||||
pub unsupported: Option<String>,
|
||||
pub families: Vec<FamilyJob>,
|
||||
}
|
||||
|
||||
/// The suites of one family selected for an environment.
|
||||
#[derive(Debug)]
|
||||
pub struct FamilyJob {
|
||||
pub family: String,
|
||||
pub suites: Vec<SuiteJob>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SuiteJob {
|
||||
pub suite: Suite,
|
||||
/// Test IDs to run.
|
||||
pub run: Vec<String>,
|
||||
/// Tests excluded because the environment lacks a capability they need.
|
||||
pub unsupported: Vec<UnsupportedTest>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct UnsupportedTest {
|
||||
pub id: String,
|
||||
pub missing: Vec<String>,
|
||||
}
|
||||
|
||||
/// Filters applied on top of the plan, from the command line.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Narrowing {
|
||||
pub environments: Vec<String>,
|
||||
pub suites: Vec<String>,
|
||||
}
|
||||
|
||||
/// Validate the plan against the catalogs and expand it into jobs.
|
||||
pub fn expand(
|
||||
plan: &Plan,
|
||||
catalog: &Catalog,
|
||||
environments: &Environments,
|
||||
host: Option<Arch>,
|
||||
narrowing: &Narrowing,
|
||||
) -> Result<Vec<Job>> {
|
||||
// environment ID -> family -> suite names, in plan order where it matters.
|
||||
let mut selected: BTreeMap<String, BTreeMap<String, Vec<String>>> = BTreeMap::new();
|
||||
|
||||
for entry in &plan.matrix {
|
||||
let family = catalog.family(&entry.family)?;
|
||||
let suites: Vec<String> = match &entry.suites {
|
||||
Some(suites) => {
|
||||
for suite in suites {
|
||||
family.suite(suite)?;
|
||||
}
|
||||
suites.clone()
|
||||
}
|
||||
None => family.suites.keys().cloned().collect(),
|
||||
};
|
||||
for id in &entry.environments {
|
||||
if environments.get(id).is_none() {
|
||||
bail!(
|
||||
"unknown environment {id:?}; known environments: {}",
|
||||
environments
|
||||
.iter()
|
||||
.map(|env| env.id.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
);
|
||||
}
|
||||
let family_suites = selected
|
||||
.entry(id.clone())
|
||||
.or_default()
|
||||
.entry(entry.family.clone())
|
||||
.or_default();
|
||||
for suite in &suites {
|
||||
if !family_suites.contains(suite) {
|
||||
family_suites.push(suite.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for id in &narrowing.environments {
|
||||
if !selected.contains_key(id) {
|
||||
bail!("--env {id:?} is not in the plan");
|
||||
}
|
||||
}
|
||||
for suite in &narrowing.suites {
|
||||
let in_plan = selected
|
||||
.values()
|
||||
.flat_map(BTreeMap::values)
|
||||
.any(|suites| suites.contains(suite));
|
||||
if !in_plan {
|
||||
bail!("--suite {suite:?} is not in the plan");
|
||||
}
|
||||
}
|
||||
|
||||
let mut jobs = Vec::new();
|
||||
for (id, families) in selected {
|
||||
if !narrowing.environments.is_empty() && !narrowing.environments.contains(&id) {
|
||||
continue;
|
||||
}
|
||||
let environment = environments.get(&id).expect("validated above").clone();
|
||||
let mut family_jobs = Vec::new();
|
||||
for (family_name, suites) in families {
|
||||
let family = catalog.family(&family_name)?;
|
||||
let suites: Vec<SuiteJob> = suites
|
||||
.iter()
|
||||
.filter(|suite| narrowing.suites.is_empty() || narrowing.suites.contains(suite))
|
||||
.map(|suite| suite_job(family.suite(suite).expect("validated above"), &environment))
|
||||
.collect();
|
||||
if !suites.is_empty() {
|
||||
family_jobs.push(FamilyJob {
|
||||
family: family_name,
|
||||
suites,
|
||||
});
|
||||
}
|
||||
}
|
||||
if family_jobs.is_empty() {
|
||||
continue;
|
||||
}
|
||||
jobs.push(Job {
|
||||
unsupported: environment.unsupported_reason(host),
|
||||
environment,
|
||||
families: family_jobs,
|
||||
});
|
||||
}
|
||||
Ok(jobs)
|
||||
}
|
||||
|
||||
fn suite_job(suite: &Suite, environment: &Environment) -> SuiteJob {
|
||||
let mut run = Vec::new();
|
||||
let mut unsupported = Vec::new();
|
||||
for test in &suite.tests {
|
||||
let missing: Vec<String> = test
|
||||
.requires
|
||||
.iter()
|
||||
.filter(|capability| !provides(environment, capability))
|
||||
.cloned()
|
||||
.collect();
|
||||
if missing.is_empty() {
|
||||
run.push(test.id.clone());
|
||||
} else {
|
||||
unsupported.push(UnsupportedTest {
|
||||
id: test.id.clone(),
|
||||
missing,
|
||||
});
|
||||
}
|
||||
}
|
||||
SuiteJob {
|
||||
suite: suite.clone(),
|
||||
run,
|
||||
unsupported,
|
||||
}
|
||||
}
|
||||
|
||||
/// Requirements name boolean capabilities.
|
||||
fn provides(environment: &Environment, capability: &str) -> bool {
|
||||
environment.capabilities.get(capability) == Some(&serde_yml::Value::Bool(true))
|
||||
}
|
||||
|
||||
impl FamilyJob {
|
||||
/// Whether any test will run.
|
||||
pub fn has_tests(&self) -> bool {
|
||||
self.suites.iter().any(|suite| !suite.run.is_empty())
|
||||
}
|
||||
|
||||
/// The nextest filterset selecting exactly the tests to run.
|
||||
pub fn filterset(&self) -> String {
|
||||
self.suites
|
||||
.iter()
|
||||
.filter(|suite| !suite.run.is_empty())
|
||||
.map(|suite| {
|
||||
let binary = format!("binary(={})", suite.suite.binary);
|
||||
if suite.unsupported.is_empty() {
|
||||
binary
|
||||
} else {
|
||||
let tests = suite
|
||||
.run
|
||||
.iter()
|
||||
.map(|id| format!("test(={id})"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" | ");
|
||||
format!("({binary} & ({tests}))")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" | ")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::PathBuf;
|
||||
|
||||
use super::*;
|
||||
use crate::catalog::{Family, Test};
|
||||
|
||||
fn catalog() -> Catalog {
|
||||
let suite = |name: &str, tests: &[(&str, &[&str])]| Suite {
|
||||
suite: name.to_string(),
|
||||
binary: name.replace('-', "_"),
|
||||
package: format!("pkg-{name}"),
|
||||
tests: tests
|
||||
.iter()
|
||||
.map(|(id, requires)| Test {
|
||||
id: (*id).to_string(),
|
||||
description: "d".to_string(),
|
||||
requires: requires.iter().map(ToString::to_string).collect(),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let family = Family {
|
||||
name: "conformance".to_string(),
|
||||
workspace: PathBuf::new(),
|
||||
suites: [
|
||||
suite(
|
||||
"sandbox-lifecycle",
|
||||
&[("long_running", &[]), ("expose", &["service_exposure"])],
|
||||
),
|
||||
suite("providers", &[("create_provider", &[])]),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|suite| (suite.suite.clone(), suite))
|
||||
.collect(),
|
||||
};
|
||||
Catalog {
|
||||
families: [("conformance".to_string(), family)].into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn environments() -> Environments {
|
||||
use std::fmt::Write as _;
|
||||
|
||||
let mut yaml = String::from("environments:\n");
|
||||
for arch in ["amd64", "arm64"] {
|
||||
let _ = write!(
|
||||
yaml,
|
||||
" - id: tmachine.ubuntu2404-{arch}.docker-rootful.deb\n backend: tmachine\n machine: {{ os: ubuntu, version: \"24.04\", arch: {arch} }}\n runtime: docker-rootful\n installer: deb\n tmachine: {{ environment: ubuntu-docker-rootful, installer: deb }}\n capabilities: {{ compute_driver: docker }}\n"
|
||||
);
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("docker.yaml"), yaml).unwrap();
|
||||
Environments::load_dir(dir.path()).unwrap()
|
||||
}
|
||||
|
||||
const AMD64: &str = "tmachine.ubuntu2404-amd64.docker-rootful.deb";
|
||||
const ARM64: &str = "tmachine.ubuntu2404-arm64.docker-rootful.deb";
|
||||
|
||||
fn expand_plan(yaml: &str, narrowing: &Narrowing) -> Result<Vec<Job>> {
|
||||
expand(
|
||||
&Plan::parse(yaml).unwrap(),
|
||||
&catalog(),
|
||||
&environments(),
|
||||
Some(Arch::Amd64),
|
||||
narrowing,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excludes_tests_with_unmet_requirements() {
|
||||
let jobs = expand_plan(
|
||||
&format!("matrix:\n - type: conformance\n suites: [sandbox-lifecycle]\n environments: [{AMD64}]\n"),
|
||||
&Narrowing::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.len(), 1);
|
||||
let suite = &jobs[0].families[0].suites[0];
|
||||
assert_eq!(suite.run, ["long_running"]);
|
||||
assert_eq!(suite.unsupported[0].id, "expose");
|
||||
assert_eq!(suite.unsupported[0].missing, ["service_exposure"]);
|
||||
assert_eq!(
|
||||
jobs[0].families[0].filterset(),
|
||||
"(binary(=sandbox_lifecycle) & (test(=long_running)))"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn omitted_suites_select_the_whole_family() {
|
||||
let jobs = expand_plan(
|
||||
&format!("matrix:\n - type: conformance\n environments: [{AMD64}]\n"),
|
||||
&Narrowing::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let suites: Vec<_> = jobs[0].families[0]
|
||||
.suites
|
||||
.iter()
|
||||
.map(|suite| suite.suite.suite.as_str())
|
||||
.collect();
|
||||
assert_eq!(suites, ["providers", "sandbox-lifecycle"]);
|
||||
assert_eq!(
|
||||
jobs[0].families[0].filterset(),
|
||||
"binary(=providers) | (binary(=sandbox_lifecycle) & (test(=long_running)))"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn groups_entries_by_environment() {
|
||||
let jobs = expand_plan(
|
||||
&format!(
|
||||
"matrix:\n - type: conformance\n suites: [providers]\n environments: [{AMD64}, {ARM64}]\n - type: conformance\n suites: [providers, sandbox-lifecycle]\n environments: [{AMD64}]\n"
|
||||
),
|
||||
&Narrowing::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.len(), 2);
|
||||
let amd64 = jobs.iter().find(|job| job.environment.id == AMD64).unwrap();
|
||||
assert_eq!(amd64.families.len(), 1);
|
||||
assert_eq!(amd64.families[0].suites.len(), 2);
|
||||
assert!(amd64.unsupported.is_none());
|
||||
let arm64 = jobs.iter().find(|job| job.environment.id == ARM64).unwrap();
|
||||
assert!(arm64.unsupported.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn narrows_to_environment_and_suite() {
|
||||
let narrowing = Narrowing {
|
||||
environments: vec![AMD64.to_string()],
|
||||
suites: vec!["providers".to_string()],
|
||||
};
|
||||
let jobs = expand_plan(
|
||||
&format!("matrix:\n - type: conformance\n environments: [{AMD64}, {ARM64}]\n"),
|
||||
&narrowing,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(jobs.len(), 1);
|
||||
assert_eq!(jobs[0].families[0].suites.len(), 1);
|
||||
assert_eq!(jobs[0].families[0].filterset(), "binary(=providers)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_names() {
|
||||
let unknown_env = expand_plan(
|
||||
"matrix:\n - type: conformance\n environments: [nope]\n",
|
||||
&Narrowing::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
unknown_env.to_string().contains("unknown environment"),
|
||||
"{unknown_env}"
|
||||
);
|
||||
|
||||
let unknown_suite = expand_plan(
|
||||
&format!("matrix:\n - type: conformance\n suites: [sandbox-lifecyle]\n environments: [{AMD64}]\n"),
|
||||
&Narrowing::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
unknown_suite
|
||||
.to_string()
|
||||
.contains("unknown conformance suite"),
|
||||
"{unknown_suite}"
|
||||
);
|
||||
|
||||
let unknown_family = expand_plan(
|
||||
&format!("matrix:\n - type: features\n environments: [{AMD64}]\n"),
|
||||
&Narrowing::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
unknown_family.to_string().contains("unknown test family"),
|
||||
"{unknown_family}"
|
||||
);
|
||||
|
||||
let narrowing = Narrowing {
|
||||
environments: vec![ARM64.to_string()],
|
||||
suites: vec![],
|
||||
};
|
||||
let not_in_plan = expand_plan(
|
||||
&format!("matrix:\n - type: conformance\n environments: [{AMD64}]\n"),
|
||||
&narrowing,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
not_in_plan.to_string().contains("not in the plan"),
|
||||
"{not_in_plan}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -5,4 +5,10 @@
|
||||
//! environment through its backend, runs the selected suites inside it, and
|
||||
//! reports the results.
|
||||
|
||||
pub mod bundle;
|
||||
pub mod catalog;
|
||||
pub mod environments;
|
||||
pub mod execute;
|
||||
pub mod jobs;
|
||||
pub mod plan;
|
||||
pub mod report;
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! `openshell-test`: run `OpenShell` test plans.
|
||||
//!
|
||||
//! Run it through `nix run .#test-runner -- <command>`, which provides the
|
||||
//! guest cross-compilation toolchain, QEMU, Ansible, and the tmachine
|
||||
//! configuration.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::{Parser, Subcommand};
|
||||
use openshell_test_runner::bundle::{self, Bundle};
|
||||
use openshell_test_runner::catalog::Catalog;
|
||||
use openshell_test_runner::environments::{Arch, Catalog as Environments};
|
||||
use openshell_test_runner::execute::{self, Options};
|
||||
use openshell_test_runner::jobs::{self, Job, Narrowing};
|
||||
use openshell_test_runner::plan::Plan;
|
||||
use openshell_test_runner::report;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "openshell-test", about = "Run OpenShell test plans")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Command {
|
||||
/// Run a test plan.
|
||||
Test {
|
||||
/// The plan to run, e.g. tests/plans/cpu-tests.yaml.
|
||||
#[arg(long)]
|
||||
plan: PathBuf,
|
||||
/// Print the resolved jobs without building or provisioning anything.
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
/// Keep each environment running after its tests until Enter is pressed.
|
||||
#[arg(long)]
|
||||
keep: bool,
|
||||
/// Only run these environments from the plan.
|
||||
#[arg(long = "env", value_name = "ID")]
|
||||
environments: Vec<String>,
|
||||
/// Only run these suites from the plan.
|
||||
#[arg(long = "suite", value_name = "NAME")]
|
||||
suites: Vec<String>,
|
||||
/// Results directory [default: artifacts/test-results/<plan name>].
|
||||
#[arg(long)]
|
||||
results: Option<PathBuf>,
|
||||
},
|
||||
/// Print the test catalog as JSON.
|
||||
Catalog,
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> ExitCode {
|
||||
match run(Cli::parse()).await {
|
||||
Ok(code) => code,
|
||||
Err(error) => {
|
||||
eprintln!("error: {error:#}");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(cli: Cli) -> Result<ExitCode> {
|
||||
let root = repo_root()?;
|
||||
let catalog = Catalog::load(&root)?;
|
||||
|
||||
match cli.command {
|
||||
Command::Catalog => {
|
||||
println!("{}", serde_json::to_string_pretty(&catalog)?);
|
||||
Ok(ExitCode::SUCCESS)
|
||||
}
|
||||
Command::Test {
|
||||
plan,
|
||||
dry_run,
|
||||
keep,
|
||||
environments,
|
||||
suites,
|
||||
results,
|
||||
} => {
|
||||
let plan_path = plan;
|
||||
let plan = Plan::load(&plan_path)?;
|
||||
let environments_catalog = Environments::load_dir(&root.join("tests/environments"))?;
|
||||
let host = Arch::host();
|
||||
let narrowing = Narrowing {
|
||||
environments,
|
||||
suites,
|
||||
};
|
||||
let jobs = jobs::expand(&plan, &catalog, &environments_catalog, host, &narrowing)?;
|
||||
print_jobs(&jobs);
|
||||
if dry_run {
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
|
||||
let results = match results {
|
||||
Some(results) => results,
|
||||
None => root.join("artifacts/test-results").join(
|
||||
plan_path
|
||||
.file_stem()
|
||||
.context("plan path has no file name")?,
|
||||
),
|
||||
};
|
||||
std::fs::create_dir_all(&results)
|
||||
.with_context(|| format!("create {}", results.display()))?;
|
||||
std::fs::write(
|
||||
results.join("suites.json"),
|
||||
serde_json::to_string_pretty(&catalog)?,
|
||||
)?;
|
||||
|
||||
let bundles = build_bundles(&jobs, &catalog, host)?;
|
||||
let tmachine = load_tmachine(&root)?;
|
||||
let options = Options {
|
||||
tmachine: &tmachine,
|
||||
bundles: &bundles,
|
||||
results: &results,
|
||||
keep,
|
||||
};
|
||||
let mut outcomes = Vec::new();
|
||||
for job in &jobs {
|
||||
outcomes.extend(execute::run(job, &options).await);
|
||||
}
|
||||
|
||||
eprintln!("{}", report::summary(&outcomes));
|
||||
eprintln!("Results: {}", results.display());
|
||||
Ok(
|
||||
if outcomes.iter().any(|outcome| outcome.status.is_failure()) {
|
||||
ExitCode::FAILURE
|
||||
} else {
|
||||
ExitCode::SUCCESS
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn print_jobs(jobs: &[Job]) {
|
||||
for job in jobs {
|
||||
eprintln!("{}", job.environment.id);
|
||||
if let Some(reason) = &job.unsupported {
|
||||
eprintln!(" unsupported on this host: {reason}");
|
||||
}
|
||||
for family in &job.families {
|
||||
eprintln!(" {}: -E '{}'", family.family, family.filterset());
|
||||
for suite in &family.suites {
|
||||
eprintln!(" {}: {}", suite.suite.suite, suite.run.join(", "));
|
||||
for test in &suite.unsupported {
|
||||
eprintln!(
|
||||
" {}: {} unsupported (needs {})",
|
||||
suite.suite.suite,
|
||||
test.id,
|
||||
test.missing.join(", ")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build one bundle per family, covering every suite that will run.
|
||||
fn build_bundles(
|
||||
jobs: &[Job],
|
||||
catalog: &Catalog,
|
||||
host: Option<Arch>,
|
||||
) -> Result<BTreeMap<String, Bundle>> {
|
||||
let mut suites: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
|
||||
for job in jobs.iter().filter(|job| job.unsupported.is_none()) {
|
||||
for family in &job.families {
|
||||
for suite in family.suites.iter().filter(|suite| !suite.run.is_empty()) {
|
||||
suites
|
||||
.entry(family.family.as_str())
|
||||
.or_default()
|
||||
.insert(suite.suite.suite.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut bundles = BTreeMap::new();
|
||||
if suites.is_empty() {
|
||||
return Ok(bundles);
|
||||
}
|
||||
let target = bundle::guest_target(host.context("the host architecture is not supported")?);
|
||||
for (family_name, suite_names) in suites {
|
||||
let family = catalog.family(family_name)?;
|
||||
let binaries = suite_names
|
||||
.iter()
|
||||
.map(|name| family.suite(name).map(|suite| suite.binary.clone()))
|
||||
.collect::<Result<_>>()?;
|
||||
eprintln!("==> building {family_name} bundle for {target}");
|
||||
let bundle = bundle::build(
|
||||
family,
|
||||
&family.packages(suite_names.iter().copied()),
|
||||
&binaries,
|
||||
target,
|
||||
)?;
|
||||
bundles.insert(family_name.to_string(), bundle);
|
||||
}
|
||||
Ok(bundles)
|
||||
}
|
||||
|
||||
/// Load the tmachine configuration and prepare the process to drive it.
|
||||
///
|
||||
/// tmachine configs name playbooks and inputs relative to `tests/`, and its
|
||||
/// Ansible runs read `ANSIBLE_CONFIG`.
|
||||
fn load_tmachine(root: &Path) -> Result<tmachine::config::Config> {
|
||||
let path = std::env::var_os("TMACHINE_CONFIG").context(
|
||||
"TMACHINE_CONFIG is not set; run through `nix run .#test-runner`, which provides it",
|
||||
)?;
|
||||
let config = tmachine::config::Config::load(Path::new(&path))?;
|
||||
let tests = root.join("tests");
|
||||
std::env::set_current_dir(&tests).with_context(|| format!("enter {}", tests.display()))?;
|
||||
if std::env::var_os("ANSIBLE_CONFIG").is_none() {
|
||||
// SAFETY: single-threaded at this point; no other thread reads the
|
||||
// environment concurrently.
|
||||
unsafe { std::env::set_var("ANSIBLE_CONFIG", tests.join("ansible/ansible.cfg")) };
|
||||
}
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn repo_root() -> Result<PathBuf> {
|
||||
let output = std::process::Command::new("git")
|
||||
.args(["rev-parse", "--show-toplevel"])
|
||||
.output()
|
||||
.context("run git rev-parse")?;
|
||||
anyhow::ensure!(
|
||||
output.status.success(),
|
||||
"not inside the OpenShell repository"
|
||||
);
|
||||
Ok(PathBuf::from(String::from_utf8(output.stdout)?.trim()))
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Test plans: `tests/plans/*.yaml`.
|
||||
//!
|
||||
//! ```yaml
|
||||
//! matrix:
|
||||
//! - type: conformance
|
||||
//! suites: [sandbox-lifecycle] # omit for every suite in the family
|
||||
//! environments: [tmachine.ubuntu2404-amd64.docker-rootful.deb]
|
||||
//! ```
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Plan {
|
||||
pub matrix: Vec<Entry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Entry {
|
||||
/// The test family, e.g. `conformance`.
|
||||
#[serde(rename = "type")]
|
||||
pub family: String,
|
||||
/// Suites to run. `None` selects every suite in the family.
|
||||
#[serde(default)]
|
||||
pub suites: Option<Vec<String>>,
|
||||
pub environments: Vec<String>,
|
||||
}
|
||||
|
||||
impl Plan {
|
||||
pub fn load(path: &Path) -> Result<Self> {
|
||||
let yaml = std::fs::read_to_string(path)
|
||||
.with_context(|| format!("read plan {}", path.display()))?;
|
||||
Self::parse(&yaml).with_context(|| format!("load plan {}", path.display()))
|
||||
}
|
||||
|
||||
pub fn parse(yaml: &str) -> Result<Self> {
|
||||
let plan: Self = serde_yml::from_str(yaml)?;
|
||||
ensure!(!plan.matrix.is_empty(), "plan has an empty matrix");
|
||||
for (index, entry) in plan.matrix.iter().enumerate() {
|
||||
ensure!(
|
||||
!entry.environments.is_empty(),
|
||||
"matrix entry {index} ({}) names no environments",
|
||||
entry.family
|
||||
);
|
||||
ensure!(
|
||||
entry
|
||||
.suites
|
||||
.as_ref()
|
||||
.is_none_or(|suites| !suites.is_empty()),
|
||||
"matrix entry {index} ({}) has an empty suite list; omit `suites` to select every suite",
|
||||
entry.family
|
||||
);
|
||||
}
|
||||
Ok(plan)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_plan() {
|
||||
let plan = Plan::parse(
|
||||
"matrix:\n - type: conformance\n suites: [sandbox-lifecycle]\n environments: [e1, e2]\n - type: conformance\n environments: [e3]\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(plan.matrix.len(), 2);
|
||||
assert_eq!(plan.matrix[0].family, "conformance");
|
||||
assert_eq!(
|
||||
plan.matrix[0].suites.as_deref(),
|
||||
Some(&["sandbox-lifecycle".to_string()][..])
|
||||
);
|
||||
assert_eq!(plan.matrix[1].suites, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_fields() {
|
||||
let err =
|
||||
Plan::parse("matrix:\n - type: conformance\n suite: [x]\n environments: [e]\n")
|
||||
.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("suite"), "{err:#}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_entry_without_environments() {
|
||||
let err =
|
||||
Plan::parse("matrix:\n - type: conformance\n environments: []\n").unwrap_err();
|
||||
assert!(err.to_string().contains("names no environments"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_suite_list() {
|
||||
let err =
|
||||
Plan::parse("matrix:\n - type: conformance\n suites: []\n environments: [e]\n")
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("empty suite list"), "{err}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Job outcomes and the run summary.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
|
||||
/// The outcome of one family in one environment.
|
||||
#[derive(Debug)]
|
||||
pub struct Outcome {
|
||||
pub environment: String,
|
||||
pub family: String,
|
||||
pub status: Status,
|
||||
/// Tests excluded because the environment lacks a capability.
|
||||
pub unsupported_tests: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Status {
|
||||
Passed {
|
||||
tests: u32,
|
||||
},
|
||||
Failed {
|
||||
tests: u32,
|
||||
failures: u32,
|
||||
},
|
||||
/// The environment can't be provisioned on this host.
|
||||
Unsupported(String),
|
||||
/// Every selected test needs a capability the environment lacks.
|
||||
NothingToRun,
|
||||
/// The runner could not complete the job.
|
||||
Error(String),
|
||||
}
|
||||
|
||||
impl Status {
|
||||
pub const fn is_failure(&self) -> bool {
|
||||
matches!(self, Self::Failed { .. } | Self::Error(_))
|
||||
}
|
||||
}
|
||||
|
||||
/// Test and failure counts from a `JUnit` report's root element.
|
||||
pub fn junit_counts(xml: &str) -> Option<(u32, u32)> {
|
||||
let start = xml.find("<testsuites")?;
|
||||
let end = start + xml[start..].find('>')?;
|
||||
let root = &xml[start..end];
|
||||
let attr = |name: &str| -> Option<u32> {
|
||||
let key = format!(" {name}=\"");
|
||||
let value_start = root.find(&key)? + key.len();
|
||||
let value_end = value_start + root[value_start..].find('"')?;
|
||||
root[value_start..value_end].parse().ok()
|
||||
};
|
||||
let tests = attr("tests")?;
|
||||
let failures = attr("failures").unwrap_or(0) + attr("errors").unwrap_or(0);
|
||||
Some((tests, failures))
|
||||
}
|
||||
|
||||
/// A human-readable summary of every outcome.
|
||||
pub fn summary(outcomes: &[Outcome]) -> String {
|
||||
let mut out = String::from("\nSummary\n");
|
||||
for outcome in outcomes {
|
||||
let status = match &outcome.status {
|
||||
Status::Passed { tests } => format!("PASS {tests} passed"),
|
||||
Status::Failed { tests, failures } => {
|
||||
format!("FAIL {failures} of {tests} failed")
|
||||
}
|
||||
Status::Unsupported(reason) => format!("UNSUPPORTED {reason}"),
|
||||
Status::NothingToRun => {
|
||||
"UNSUPPORTED every selected test needs a missing capability".to_string()
|
||||
}
|
||||
Status::Error(error) => format!("ERROR {error}"),
|
||||
};
|
||||
let unsupported = if outcome.unsupported_tests > 0 {
|
||||
format!(" ({} tests unsupported)", outcome.unsupported_tests)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" {} / {}: {status}{unsupported}",
|
||||
outcome.environment, outcome.family
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reads_junit_counts() {
|
||||
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuites name="nextest-run" tests="3" failures="1" errors="1" uuid="x" timestamp="t" time="1.0">
|
||||
<testsuite name="a" tests="3"></testsuite></testsuites>"#;
|
||||
assert_eq!(junit_counts(xml), Some((3, 2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_root_has_no_counts() {
|
||||
assert_eq!(junit_counts("<testsuite tests=\"1\"/>"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarizes_outcomes() {
|
||||
let text = summary(&[
|
||||
Outcome {
|
||||
environment: "env-a".to_string(),
|
||||
family: "conformance".to_string(),
|
||||
status: Status::Passed { tests: 2 },
|
||||
unsupported_tests: 1,
|
||||
},
|
||||
Outcome {
|
||||
environment: "env-b".to_string(),
|
||||
family: "conformance".to_string(),
|
||||
status: Status::Unsupported("arch".to_string()),
|
||||
unsupported_tests: 0,
|
||||
},
|
||||
]);
|
||||
assert!(
|
||||
text.contains("env-a / conformance: PASS 2 passed (1 tests unsupported)"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("env-b / conformance: UNSUPPORTED arch"),
|
||||
"{text}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Generated
+9
@@ -1247,11 +1247,20 @@ dependencies = [
|
||||
name = "openshell-testkit-macros"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"openshell-testkit-spec",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openshell-testkit-spec"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.2.1"
|
||||
|
||||
@@ -3,4 +3,4 @@
|
||||
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = ["cli", "sandbox-lifecycle", "testkit", "testkit-macros"]
|
||||
members = ["cli", "sandbox-lifecycle", "testkit", "testkit-macros", "testkit-spec"]
|
||||
|
||||
@@ -16,6 +16,7 @@ name = "testkit_macros"
|
||||
proc-macro = true
|
||||
|
||||
[dependencies]
|
||||
openshell-testkit-spec = { path = "../testkit-spec" }
|
||||
proc-macro2 = "1"
|
||||
quote = "1"
|
||||
syn = { version = "2", features = ["full"] }
|
||||
|
||||
@@ -16,14 +16,13 @@
|
||||
//!
|
||||
//! The attribute expands to a `#[tokio::test]` wrapper with the same name, so
|
||||
//! the nextest test name is the test ID. The original function is nested in
|
||||
//! the wrapper and passed to `testkit::run`.
|
||||
//! the wrapper and passed to `testkit::run`. Parsing lives in `testkit-spec`,
|
||||
//! which the test runner's catalog also uses.
|
||||
|
||||
use proc_macro::TokenStream;
|
||||
use proc_macro2::{Span, TokenStream as TokenStream2};
|
||||
use proc_macro2::TokenStream as TokenStream2;
|
||||
use quote::{format_ident, quote};
|
||||
use syn::parse::{Parse, ParseStream};
|
||||
use syn::punctuated::Punctuated;
|
||||
use syn::{Expr, ExprLit, Ident, ItemFn, Lit, LitStr, Meta, Token, bracketed};
|
||||
use syn::{Ident, ItemFn, LitStr};
|
||||
|
||||
#[proc_macro_attribute]
|
||||
pub fn test(attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||
@@ -32,62 +31,20 @@ pub fn test(attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Arguments accepted by `#[testkit::test(...)]`.
|
||||
#[derive(Default)]
|
||||
struct Args {
|
||||
requires: Vec<Ident>,
|
||||
}
|
||||
|
||||
impl Parse for Args {
|
||||
fn parse(input: ParseStream<'_>) -> syn::Result<Self> {
|
||||
let mut args = Self::default();
|
||||
while !input.is_empty() {
|
||||
let key: Ident = input.parse()?;
|
||||
if key != "requires" {
|
||||
return Err(syn::Error::new(
|
||||
key.span(),
|
||||
format!("unknown argument `{key}`; expected `requires = [...]`"),
|
||||
));
|
||||
}
|
||||
input.parse::<Token![=]>()?;
|
||||
let content;
|
||||
bracketed!(content in input);
|
||||
args.requires
|
||||
.extend(Punctuated::<Ident, Token![,]>::parse_terminated(&content)?);
|
||||
if !input.is_empty() {
|
||||
input.parse::<Token![,]>()?;
|
||||
}
|
||||
}
|
||||
Ok(args)
|
||||
}
|
||||
}
|
||||
|
||||
fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result<TokenStream2> {
|
||||
let args: Args = syn::parse2(attr)?;
|
||||
let mut body: ItemFn = syn::parse2(item)?;
|
||||
let spec = testkit_spec::parse(attr, &body)?;
|
||||
|
||||
if body.sig.asyncness.is_none() {
|
||||
return Err(syn::Error::new_spanned(
|
||||
body.sig.fn_token,
|
||||
"conformance tests must be `async fn`",
|
||||
));
|
||||
}
|
||||
if body.sig.inputs.len() != 1 {
|
||||
return Err(syn::Error::new_spanned(
|
||||
&body.sig.inputs,
|
||||
"conformance tests take exactly one argument: `cx: &mut testkit::Context`",
|
||||
));
|
||||
}
|
||||
|
||||
let id = body.sig.ident.clone();
|
||||
let description = description(&body)?;
|
||||
let requires = args.requires.iter().map(|capability| {
|
||||
let id = spec.id;
|
||||
let id_str = LitStr::new(&id.to_string(), id.span());
|
||||
let description = LitStr::new(&spec.description, id.span());
|
||||
let requires = spec.requires.iter().map(|capability| {
|
||||
let variant = Ident::new(&to_camel_case(&capability.to_string()), capability.span());
|
||||
quote!(::testkit::Capability::#variant)
|
||||
});
|
||||
|
||||
let vis = body.vis.clone();
|
||||
let wrapper_attrs: Vec<_> = body
|
||||
let doc_attrs: Vec<_> = body
|
||||
.attrs
|
||||
.iter()
|
||||
.filter(|attr| attr.path().is_ident("doc"))
|
||||
@@ -96,10 +53,9 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result<TokenStream2> {
|
||||
body.attrs.retain(|attr| !attr.path().is_ident("doc"));
|
||||
body.vis = syn::Visibility::Inherited;
|
||||
body.sig.ident = format_ident!("body");
|
||||
let id_str = LitStr::new(&id.to_string(), id.span());
|
||||
|
||||
Ok(quote! {
|
||||
#(#wrapper_attrs)*
|
||||
#(#doc_attrs)*
|
||||
#[::testkit::__private::tokio::test(crate = "::testkit::__private::tokio")]
|
||||
#vis async fn #id() {
|
||||
#body
|
||||
@@ -116,33 +72,6 @@ fn expand(attr: TokenStream2, item: TokenStream2) -> syn::Result<TokenStream2> {
|
||||
})
|
||||
}
|
||||
|
||||
/// The test description: the doc comment, joined into one line.
|
||||
fn description(item: &ItemFn) -> syn::Result<LitStr> {
|
||||
let lines: Vec<String> = item
|
||||
.attrs
|
||||
.iter()
|
||||
.filter_map(|attr| match &attr.meta {
|
||||
Meta::NameValue(meta) if meta.path.is_ident("doc") => match &meta.value {
|
||||
Expr::Lit(ExprLit {
|
||||
lit: Lit::Str(line),
|
||||
..
|
||||
}) => Some(line.value().trim().to_string()),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.filter(|line| !line.is_empty())
|
||||
.collect();
|
||||
|
||||
if lines.is_empty() {
|
||||
return Err(syn::Error::new(
|
||||
item.sig.ident.span(),
|
||||
"conformance tests need a doc comment describing the behavior they verify",
|
||||
));
|
||||
}
|
||||
Ok(LitStr::new(&lines.join(" "), Span::call_site()))
|
||||
}
|
||||
|
||||
fn to_camel_case(snake: &str) -> String {
|
||||
snake
|
||||
.split('_')
|
||||
@@ -176,16 +105,6 @@ mod tests {
|
||||
assert!(out.contains("description : \"Does a thing.\""), "{out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn joins_multi_line_doc_comments() {
|
||||
let out = expand_str(
|
||||
"",
|
||||
"/// First line\n/// second line.\nasync fn t(cx: &mut Context) -> testkit::Result { Ok(()) }",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(out.contains("\"First line second line.\""), "{out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_requirements_to_capabilities() {
|
||||
let out = expand_str(
|
||||
@@ -202,7 +121,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_missing_doc_comment() {
|
||||
fn reports_spec_errors() {
|
||||
let err = expand_str(
|
||||
"",
|
||||
"async fn t(cx: &mut Context) -> testkit::Result { Ok(()) }",
|
||||
@@ -210,31 +129,4 @@ mod tests {
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("doc comment"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_sync_fn() {
|
||||
let err = expand_str(
|
||||
"",
|
||||
"/// Doc.\nfn t(cx: &mut Context) -> testkit::Result { Ok(()) }",
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("async fn"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_wrong_arity() {
|
||||
let err =
|
||||
expand_str("", "/// Doc.\nasync fn t() -> testkit::Result { Ok(()) }").unwrap_err();
|
||||
assert!(err.to_string().contains("exactly one argument"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_argument() {
|
||||
let err = expand_str(
|
||||
"timeout = [x]",
|
||||
"/// Doc.\nasync fn t(cx: &mut Context) -> testkit::Result { Ok(()) }",
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("unknown argument"), "{err}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
[package]
|
||||
name = "openshell-testkit-spec"
|
||||
description = "Parses #[testkit::test] functions for the macro and the test catalog"
|
||||
version = "0.0.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.94"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/NVIDIA/OpenShell"
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
name = "testkit_spec"
|
||||
|
||||
[dependencies]
|
||||
proc-macro2 = "1"
|
||||
syn = { version = "2", features = ["full"] }
|
||||
@@ -0,0 +1,226 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! The single definition of what a `#[testkit::test]` function declares.
|
||||
//!
|
||||
//! The `#[testkit::test]` macro and the test runner's catalog both parse
|
||||
//! tests through this crate, so they cannot disagree about a test's ID,
|
||||
//! description, or requirements.
|
||||
|
||||
use proc_macro2::TokenStream;
|
||||
use syn::parse::{Parse, ParseStream};
|
||||
use syn::punctuated::Punctuated;
|
||||
use syn::{Attribute, Expr, ExprLit, Ident, Item, ItemFn, Lit, Meta, Token, bracketed};
|
||||
|
||||
/// What a test declares.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct TestSpec {
|
||||
/// The function name, which is the test ID.
|
||||
pub id: Ident,
|
||||
/// The doc comment, joined into one line.
|
||||
pub description: String,
|
||||
/// Capability names, as written in `requires = [...]`.
|
||||
pub requires: Vec<Ident>,
|
||||
}
|
||||
|
||||
/// Arguments accepted by `#[testkit::test(...)]`.
|
||||
#[derive(Default)]
|
||||
struct Args {
|
||||
requires: Vec<Ident>,
|
||||
}
|
||||
|
||||
impl Parse for Args {
|
||||
fn parse(input: ParseStream<'_>) -> syn::Result<Self> {
|
||||
let mut args = Self::default();
|
||||
while !input.is_empty() {
|
||||
let key: Ident = input.parse()?;
|
||||
if key != "requires" {
|
||||
return Err(syn::Error::new(
|
||||
key.span(),
|
||||
format!("unknown argument `{key}`; expected `requires = [...]`"),
|
||||
));
|
||||
}
|
||||
input.parse::<Token![=]>()?;
|
||||
let content;
|
||||
bracketed!(content in input);
|
||||
args.requires
|
||||
.extend(Punctuated::<Ident, Token![,]>::parse_terminated(&content)?);
|
||||
if !input.is_empty() {
|
||||
input.parse::<Token![,]>()?;
|
||||
}
|
||||
}
|
||||
Ok(args)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a test from the attribute's arguments and the function it marks.
|
||||
pub fn parse(args: TokenStream, item: &ItemFn) -> syn::Result<TestSpec> {
|
||||
let args: Args = syn::parse2(args)?;
|
||||
|
||||
if item.sig.asyncness.is_none() {
|
||||
return Err(syn::Error::new_spanned(
|
||||
item.sig.fn_token,
|
||||
"conformance tests must be `async fn`",
|
||||
));
|
||||
}
|
||||
if item.sig.inputs.len() != 1 {
|
||||
return Err(syn::Error::new_spanned(
|
||||
&item.sig.inputs,
|
||||
"conformance tests take exactly one argument: `cx: &mut testkit::Context`",
|
||||
));
|
||||
}
|
||||
|
||||
let description = doc_comment(&item.attrs);
|
||||
if description.is_empty() {
|
||||
return Err(syn::Error::new(
|
||||
item.sig.ident.span(),
|
||||
"conformance tests need a doc comment describing the behavior they verify",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(TestSpec {
|
||||
id: item.sig.ident.clone(),
|
||||
description,
|
||||
requires: args.requires,
|
||||
})
|
||||
}
|
||||
|
||||
/// Find every `#[testkit::test]` function in a source file.
|
||||
pub fn scan(source: &str) -> syn::Result<Vec<TestSpec>> {
|
||||
let file = syn::parse_file(source)?;
|
||||
let mut tests = Vec::new();
|
||||
for item in &file.items {
|
||||
let Item::Fn(item) = item else {
|
||||
continue;
|
||||
};
|
||||
let Some(attr) = item.attrs.iter().find(|attr| is_testkit_test(attr)) else {
|
||||
continue;
|
||||
};
|
||||
let args = match &attr.meta {
|
||||
Meta::Path(_) => TokenStream::new(),
|
||||
Meta::List(list) => list.tokens.clone(),
|
||||
Meta::NameValue(meta) => {
|
||||
return Err(syn::Error::new_spanned(
|
||||
meta,
|
||||
"expected `#[testkit::test]` or `#[testkit::test(...)]`",
|
||||
));
|
||||
}
|
||||
};
|
||||
tests.push(parse(args, item)?);
|
||||
}
|
||||
Ok(tests)
|
||||
}
|
||||
|
||||
fn is_testkit_test(attr: &Attribute) -> bool {
|
||||
let segments: Vec<_> = attr.path().segments.iter().collect();
|
||||
segments.len() == 2 && segments[0].ident == "testkit" && segments[1].ident == "test"
|
||||
}
|
||||
|
||||
/// The doc comment, joined into one line. Empty if there is none.
|
||||
pub fn doc_comment(attrs: &[Attribute]) -> String {
|
||||
attrs
|
||||
.iter()
|
||||
.filter_map(|attr| match &attr.meta {
|
||||
Meta::NameValue(meta) if meta.path.is_ident("doc") => match &meta.value {
|
||||
Expr::Lit(ExprLit {
|
||||
lit: Lit::Str(line),
|
||||
..
|
||||
}) => Some(line.value().trim().to_string()),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
})
|
||||
.filter(|line| !line.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse_str(args: &str, item: &str) -> syn::Result<TestSpec> {
|
||||
parse(args.parse().unwrap(), &syn::parse_str(item).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_id_description_and_requirements() {
|
||||
let spec = parse_str(
|
||||
"requires = [service_exposure, gpu]",
|
||||
"/// First line\n/// second line.\nasync fn t(cx: &mut Context) -> testkit::Result { Ok(()) }",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(spec.id, "t");
|
||||
assert_eq!(spec.description, "First line second line.");
|
||||
assert_eq!(spec.requires, ["service_exposure", "gpu"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_missing_doc_comment() {
|
||||
let err = parse_str(
|
||||
"",
|
||||
"async fn t(cx: &mut Context) -> testkit::Result { Ok(()) }",
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("doc comment"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_sync_fn() {
|
||||
let err = parse_str(
|
||||
"",
|
||||
"/// Doc.\nfn t(cx: &mut Context) -> testkit::Result { Ok(()) }",
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("async fn"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_wrong_arity() {
|
||||
let err =
|
||||
parse_str("", "/// Doc.\nasync fn t() -> testkit::Result { Ok(()) }").unwrap_err();
|
||||
assert!(err.to_string().contains("exactly one argument"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_argument() {
|
||||
let err = parse_str(
|
||||
"timeout = [x]",
|
||||
"/// Doc.\nasync fn t(cx: &mut Context) -> testkit::Result { Ok(()) }",
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("unknown argument"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scans_only_testkit_tests() {
|
||||
let tests = scan(
|
||||
"
|
||||
use testkit::Context;
|
||||
|
||||
/// Plain.
|
||||
#[testkit::test]
|
||||
async fn plain(cx: &mut Context) -> testkit::Result { Ok(()) }
|
||||
|
||||
/// Needs exposure.
|
||||
#[testkit::test(requires = [service_exposure])]
|
||||
async fn exposed(cx: &mut Context) -> testkit::Result { Ok(()) }
|
||||
|
||||
#[test]
|
||||
fn not_a_conformance_test() {}
|
||||
|
||||
async fn helper(cx: &mut Context) {}
|
||||
",
|
||||
)
|
||||
.unwrap();
|
||||
let ids: Vec<_> = tests.iter().map(|test| test.id.to_string()).collect();
|
||||
assert_eq!(ids, ["plain", "exposed"]);
|
||||
assert_eq!(tests[1].requires, ["service_exposure"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_reports_invalid_tests() {
|
||||
let err = scan("#[testkit::test]\nasync fn t(cx: &mut Context) {}").unwrap_err();
|
||||
assert!(err.to_string().contains("doc comment"), "{err}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user