diff --git a/tests/ansible/roles/openshell_client/defaults/main.yaml b/tests/ansible/roles/openshell_client/defaults/main.yaml new file mode 100644 index 000000000..8e9095494 --- /dev/null +++ b/tests/ansible/roles/openshell_client/defaults/main.yaml @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +openshell_client_config_dir: >- + /home/{{ ansible_user }}/.config/openshell/gateways/{{ openshell_client_gateway_name | default('tmachine') }} diff --git a/tests/ansible/roles/openshell_client/tasks/main.yaml b/tests/ansible/roles/openshell_client/tasks/main.yaml index 6586f23ac..66ce6464d 100644 --- a/tests/ansible/roles/openshell_client/tasks/main.yaml +++ b/tests/ansible/roles/openshell_client/tasks/main.yaml @@ -12,3 +12,40 @@ - --local - --name - "{{ openshell_client_gateway_name | default('tmachine') }}" + +# Describe the registered gateway for conformance tests. The test runner +# combines this with the environment's capabilities into the target file +# that tests read through OPENSHELL_TEST_TARGET. +- name: Read registered gateway metadata + ansible.builtin.slurp: + src: "{{ openshell_client_config_dir }}/metadata.json" + register: openshell_client_metadata + +- name: Check for registered mTLS client credentials + ansible.builtin.stat: + path: "{{ openshell_client_config_dir }}/mtls/tls.crt" + register: openshell_client_mtls + +- name: Create test target directory + become: true + ansible.builtin.file: + path: /etc/openshell-test + state: directory + mode: "0755" + +- name: Write gateway section of the test target + become: true + ansible.builtin.copy: + dest: /etc/openshell-test/gateway.yaml + mode: "0644" + content: | + # Written by the openshell_client role. + {{ {'gateway': openshell_client_target_gateway} | to_nice_yaml(indent=2) }} + vars: + openshell_client_target_tls: + ca: "{{ openshell_client_config_dir }}/mtls/ca.crt" + client_cert: "{{ openshell_client_config_dir }}/mtls/tls.crt" + client_key: "{{ openshell_client_config_dir }}/mtls/tls.key" + openshell_client_target_gateway: >- + {{ {'endpoint': (openshell_client_metadata.content | b64decode | from_json).gateway_endpoint} + | combine({'tls': openshell_client_target_tls} if openshell_client_mtls.stat.exists else {}) }} diff --git a/tests/environments/docker.yaml b/tests/environments/docker.yaml new file mode 100644 index 000000000..d847b84fa --- /dev/null +++ b/tests/environments/docker.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Docker-backed environments. The ID must equal +# .-... +environments: + - id: tmachine.ubuntu2404-amd64.docker-rootful.deb + backend: tmachine + machine: { os: ubuntu, version: "24.04", arch: amd64 } + runtime: docker-rootful + installer: deb + tmachine: { environment: ubuntu-docker-rootful, installer: deb } + capabilities: { compute_driver: docker, rootless: false, gpu: false } + - id: tmachine.ubuntu2404-arm64.docker-rootful.deb + backend: tmachine + machine: { os: ubuntu, version: "24.04", arch: arm64 } + runtime: docker-rootful + installer: deb + tmachine: { environment: ubuntu-docker-rootful, installer: deb } + capabilities: { compute_driver: docker, rootless: false, gpu: false } diff --git a/tests/runner/Cargo.lock b/tests/runner/Cargo.lock new file mode 100644 index 000000000..41626bb91 --- /dev/null +++ b/tests/runner/Cargo.lock @@ -0,0 +1,146 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", + "serde", + "serde_core", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "noyalib" +version = "0.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962838abbd8dd2cd3844d2208dc502458bba6985f8bf1dc21761b30260fcbd7f" +dependencies = [ + "hashbrown", + "indexmap", + "libm", + "memchr", + "rustc-hash", + "serde_core", + "smallvec", +] + +[[package]] +name = "openshell-test-runner" +version = "0.0.0" +dependencies = [ + "anyhow", + "noyalib", + "serde", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" diff --git a/tests/runner/Cargo.toml b/tests/runner/Cargo.toml new file mode 100644 index 000000000..226071039 --- /dev/null +++ b/tests/runner/Cargo.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-test-runner" +description = "Runs OpenShell test plans against provisioned environments" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +anyhow = "1" +serde = { version = "1", features = ["derive"] } +serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } + +[workspace] diff --git a/tests/runner/src/environments.rs b/tests/runner/src/environments.rs new file mode 100644 index 000000000..026381e7a --- /dev/null +++ b/tests/runner/src/environments.rs @@ -0,0 +1,288 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Environment definitions: `tests/environments/*.yaml`. +//! +//! Each entry describes one environment a plan can name: the machine, the +//! container runtime, the installer, how the backend provisions it, and the +//! capabilities it gives tests. Capabilities pass through to the target file +//! unchanged; testkit validates their names. + +use std::collections::BTreeMap; +use std::fmt; +use std::path::Path; + +use anyhow::{Context, Result, bail, ensure}; +use serde::Deserialize; + +/// All environments, keyed by ID. +#[derive(Debug, Default)] +pub struct Catalog { + environments: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct File { + environments: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Environment { + pub id: String, + pub backend: Backend, + pub machine: Machine, + pub runtime: String, + pub installer: String, + /// Backend parameters when `backend` is `tmachine`. + #[serde(default)] + pub tmachine: Option, + #[serde(default)] + pub capabilities: BTreeMap, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Backend { + Tmachine, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Machine { + pub os: String, + pub version: String, + pub arch: Arch, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Arch { + Amd64, + Arm64, +} + +/// The tmachine environment and installer that provision this environment. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Tmachine { + pub environment: String, + pub installer: String, +} + +impl fmt::Display for Backend { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Tmachine => "tmachine", + }) + } +} + +impl fmt::Display for Arch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Amd64 => "amd64", + Self::Arm64 => "arm64", + }) + } +} + +impl Arch { + /// The architecture of the machine running this process. + pub const fn host() -> Option { + if cfg!(target_arch = "x86_64") { + Some(Self::Amd64) + } else if cfg!(target_arch = "aarch64") { + Some(Self::Arm64) + } else { + None + } + } +} + +impl Environment { + /// The ID implied by the environment's fields. + pub fn expected_id(&self) -> String { + format!( + "{}.{}{}-{}.{}.{}", + self.backend, + self.machine.os, + self.machine.version.replace('.', ""), + self.machine.arch, + self.runtime, + self.installer + ) + } + + fn validate(&self) -> Result<()> { + let expected = self.expected_id(); + ensure!( + self.id == expected, + "environment {:?} does not match its fields; expected ID {expected:?}", + self.id + ); + match self.backend { + Backend::Tmachine => ensure!( + self.tmachine.is_some(), + "environment {:?} uses the tmachine backend but has no `tmachine` section", + self.id + ), + } + Ok(()) + } + + /// Why the backend can't provision this environment on this host, if it + /// can't. tmachine boots guests of the host's architecture only. + pub fn unsupported_reason(&self, host: Option) -> Option { + match self.backend { + Backend::Tmachine if host != Some(self.machine.arch) => Some(format!( + "tmachine boots {} guests only on {} hosts", + self.machine.arch, self.machine.arch + )), + Backend::Tmachine => None, + } + } +} + +impl Catalog { + /// Load every `*.yaml` file in `dir`. + pub fn load_dir(dir: &Path) -> Result { + let mut paths: Vec<_> = std::fs::read_dir(dir) + .with_context(|| format!("read environments directory {}", dir.display()))? + .map(|entry| entry.map(|entry| entry.path())) + .collect::>() + .with_context(|| format!("read environments directory {}", dir.display()))?; + paths.retain(|path| path.extension().is_some_and(|ext| ext == "yaml")); + paths.sort(); + + let mut catalog = Self::default(); + for path in paths { + let yaml = std::fs::read_to_string(&path) + .with_context(|| format!("read {}", path.display()))?; + catalog + .add_yaml(&yaml) + .with_context(|| format!("load environments from {}", path.display()))?; + } + Ok(catalog) + } + + fn add_yaml(&mut self, yaml: &str) -> Result<()> { + let file: File = serde_yml::from_str(yaml)?; + for environment in file.environments { + environment.validate()?; + if self.environments.contains_key(&environment.id) { + bail!("environment {:?} is defined more than once", environment.id); + } + self.environments + .insert(environment.id.clone(), environment); + } + Ok(()) + } + + pub fn get(&self, id: &str) -> Option<&Environment> { + self.environments.get(id) + } + + pub fn iter(&self) -> impl Iterator { + self.environments.values() + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + + const UBUNTU_DOCKER: &str = " +environments: + - id: tmachine.ubuntu2404-amd64.docker-rootful.deb + backend: tmachine + machine: { os: ubuntu, version: \"24.04\", arch: amd64 } + runtime: docker-rootful + installer: deb + tmachine: { environment: ubuntu-docker-rootful, installer: deb } + capabilities: { compute_driver: docker, gpu: false } +"; + + fn catalog(yaml: &str) -> Result { + let mut catalog = Catalog::default(); + catalog.add_yaml(yaml)?; + Ok(catalog) + } + + #[test] + fn loads_environment() { + let catalog = catalog(UBUNTU_DOCKER).unwrap(); + let env = catalog + .get("tmachine.ubuntu2404-amd64.docker-rootful.deb") + .unwrap(); + assert_eq!(env.machine.arch, Arch::Amd64); + assert_eq!( + env.tmachine, + Some(Tmachine { + environment: "ubuntu-docker-rootful".to_string(), + installer: "deb".to_string(), + }) + ); + assert_eq!( + env.capabilities.get("compute_driver"), + Some(&serde_yml::Value::from("docker")) + ); + } + + #[test] + fn rejects_id_that_does_not_match_fields() { + let yaml = UBUNTU_DOCKER.replace("docker-rootful.deb", "docker-rootless.deb"); + let err = catalog(&yaml).unwrap_err(); + assert!( + err.to_string().contains("does not match its fields"), + "{err}" + ); + } + + #[test] + fn rejects_tmachine_environment_without_tmachine_section() { + let yaml = UBUNTU_DOCKER.replace( + " tmachine: { environment: ubuntu-docker-rootful, installer: deb }\n", + "", + ); + let err = catalog(&yaml).unwrap_err(); + assert!(err.to_string().contains("no `tmachine` section"), "{err}"); + } + + #[test] + fn rejects_unknown_fields() { + let yaml = UBUNTU_DOCKER.replace("runtime:", "runtme:"); + assert!(catalog(&yaml).is_err()); + } + + #[test] + fn rejects_duplicate_ids() { + let mut catalog = catalog(UBUNTU_DOCKER).unwrap(); + let err = catalog.add_yaml(UBUNTU_DOCKER).unwrap_err(); + assert!(err.to_string().contains("more than once"), "{err}"); + } + + #[test] + fn reports_arch_mismatch_as_unsupported() { + let catalog = catalog(UBUNTU_DOCKER).unwrap(); + let env = catalog + .get("tmachine.ubuntu2404-amd64.docker-rootful.deb") + .unwrap(); + assert_eq!(env.unsupported_reason(Some(Arch::Amd64)), None); + assert!(env.unsupported_reason(Some(Arch::Arm64)).is_some()); + } + + #[test] + fn repository_environments_are_valid() { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../environments"); + let catalog = Catalog::load_dir(&dir).unwrap(); + assert!( + catalog + .get("tmachine.ubuntu2404-amd64.docker-rootful.deb") + .is_some() + ); + } +} diff --git a/tests/runner/src/lib.rs b/tests/runner/src/lib.rs new file mode 100644 index 000000000..db3c9f103 --- /dev/null +++ b/tests/runner/src/lib.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runs `OpenShell` test plans: expands a plan into jobs, provisions each +//! environment through its backend, runs the selected suites inside it, and +//! reports the results. + +pub mod environments; diff --git a/tests/tmachine/Cargo.toml b/tests/tmachine/Cargo.toml index 46358ea4e..785ee5ea9 100644 --- a/tests/tmachine/Cargo.toml +++ b/tests/tmachine/Cargo.toml @@ -18,6 +18,6 @@ directories = { version = "6.0.0", default-features = false } serde = { version = "1.0.229", default-features = false, features = ["std", "derive"] } serde-saphyr = { version = "1.2.0", default-features = false, features = ["deserialize"] } tempfile = { version = "3.27.0", default-features = false } -tokio = { version = "1.53.1", default-features = false, features = ["macros", "rt", "process", "net", "io-util"] } +tokio = { version = "1.53.1", default-features = false, features = ["macros", "rt", "process", "net", "io-util", "time"] } [workspace] diff --git a/tests/tmachine/src/lib.rs b/tests/tmachine/src/lib.rs new file mode 100644 index 000000000..ff1db593b --- /dev/null +++ b/tests/tmachine/src/lib.rs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Isolated QEMU test machines provisioned with Ansible. +//! +//! The `tmachine` binary drives one-shot `setup`, `install`, and `test` +//! commands. Callers that need to act on a running machine use [`Session`]. + +pub mod ansible; +pub mod config; +mod qemu; + +pub use qemu::{Session, install, setup, test}; diff --git a/tests/tmachine/src/main.rs b/tests/tmachine/src/main.rs index 3c0d1a4ee..69f9f006c 100644 --- a/tests/tmachine/src/main.rs +++ b/tests/tmachine/src/main.rs @@ -5,11 +5,8 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; -use config::{Config, Environment, Installer, Machine}; - -mod ansible; -mod config; -mod qemu; +use tmachine::config::{Config, Environment, Installer, Machine}; +use tmachine::{install, setup, test}; #[derive(Parser)] #[command(name = "tmachine")] @@ -45,7 +42,7 @@ async fn main() -> Result<()> { match cli.command { Command::Setup { environment } => { let (machine, environment) = find_environment(&config, &environment)?; - qemu::setup(&machine, &environment).await?; + setup(&machine, &environment).await?; } Command::Install { environment, @@ -53,7 +50,7 @@ async fn main() -> Result<()> { } => { let (machine, environment) = find_environment(&config, &environment)?; let installer = find_installer(&config, &installer)?; - qemu::install(&machine, &environment, &installer).await?; + install(&machine, &environment, &installer).await?; } Command::Test { environment, @@ -67,7 +64,7 @@ async fn main() -> Result<()> { .iter() .find(|candidate| candidate.name == testsuite) .with_context(|| format!("testsuite {testsuite:?} is not defined"))?; - qemu::test(&machine, &environment, &installer, testsuite).await?; + test(&machine, &environment, &installer, testsuite).await?; } } diff --git a/tests/tmachine/src/qemu/mod.rs b/tests/tmachine/src/qemu/mod.rs index c13ebbf56..5be890226 100644 --- a/tests/tmachine/src/qemu/mod.rs +++ b/tests/tmachine/src/qemu/mod.rs @@ -5,10 +5,12 @@ mod ansible_hash; mod img; mod install; mod layer; +mod session; mod setup; mod test; mod vm; pub use install::install; +pub use session::Session; pub use setup::setup; pub use test::test; diff --git a/tests/tmachine/src/qemu/session.rs b/tests/tmachine/src/qemu/session.rs new file mode 100644 index 000000000..25d4b74a1 --- /dev/null +++ b/tests/tmachine/src/qemu/session.rs @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A running machine that callers act on over SSH. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::{ExitStatus, Output, Stdio}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail, ensure}; +use tempfile::{TempDir, tempdir}; +use tokio::process::Command; + +use crate::config::{Environment, Installer, Machine}; + +use super::img::QemuImage; +use super::install::install; +use super::layer::run_playbooks; +use super::vm::QemuVm; + +// Must match the QEMU port forward in `vm.rs` and `ansible/inventory.ini`. +const SSH_HOST: &str = "127.0.0.1"; +const SSH_PORT: &str = "2222"; +const SSH_USER: &str = "tmachine"; +const SSH_PASSWORD: &str = "tmachine"; +// Password authentication only: keys from the caller's agent or ~/.ssh would +// otherwise use up the guest's authentication attempts first. +const SSH_OPTIONS: [&str; 14] = [ + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "LogLevel=ERROR", + "-o", + "ConnectTimeout=5", + "-o", + "PreferredAuthentications=password", + "-o", + "PubkeyAuthentication=no", + "-o", + "IdentityAgent=none", +]; +const SSH_READY_TIMEOUT: Duration = Duration::from_secs(300); +const SSH_READY_INTERVAL: Duration = Duration::from_secs(2); + +/// A machine booted from a throwaway overlay of the installed disk. +/// +/// The VM stops when the session is dropped, so a caller that fails or +/// panics cannot leak it. Call [`Session::down`] for a clean shutdown. +pub struct Session { + // Declared before `_dir` so the VM stops before its disk is removed. + vm: Option, + _image: QemuImage, + _dir: TempDir, +} + +impl Session { + /// Set up and install the environment (reusing cached disks), boot a VM + /// from a throwaway overlay, and wait until it accepts SSH. + pub async fn up( + machine: &Machine, + environment: &Environment, + installer: &Installer, + ) -> Result { + let install_disk = install(machine, environment, installer).await?; + let dir = tempdir().context("create session directory")?; + let image = QemuImage::create(&install_disk, dir.path().join("session.qcow2")).await; + let vm = QemuVm::start(&image).await; + let mut session = Self { + vm: Some(vm), + _image: image, + _dir: dir, + }; + session.wait_for_ssh().await?; + Ok(session) + } + + async fn wait_for_ssh(&mut self) -> Result<()> { + let started = Instant::now(); + loop { + if let Some(status) = self.vm.as_mut().and_then(QemuVm::try_wait) { + bail!( + "QEMU exited with {status} before SSH was ready; is another tmachine VM using port {SSH_PORT}?" + ); + } + match self.exec(&["true"]).await { + Ok(output) if output.status.success() => return Ok(()), + Ok(_) | Err(_) if started.elapsed() < SSH_READY_TIMEOUT => { + tokio::time::sleep(SSH_READY_INTERVAL).await; + } + Ok(output) => bail!( + "SSH was not ready after {SSH_READY_TIMEOUT:?}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + Err(error) => { + return Err( + error.context(format!("SSH was not ready after {SSH_READY_TIMEOUT:?}")) + ); + } + } + } + } + + /// Run Ansible playbooks against the machine. + pub async fn run_playbooks( + &self, + playbooks: &[PathBuf], + inputs: &BTreeMap, + ) -> Result<()> { + run_playbooks(playbooks, inputs).await + } + + /// Run a command in the machine as the `tmachine` user and capture its + /// output. A non-zero exit is returned in the output, not as an error. + pub async fn exec(&self, argv: &[&str]) -> Result { + ensure!(!argv.is_empty(), "exec needs a command"); + let remote = argv + .iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" "); + ssh_command("ssh", &["-p", SSH_PORT]) + .arg(format!("{SSH_USER}@{SSH_HOST}")) + .arg("--") + .arg(remote) + .stdin(Stdio::null()) + .output() + .await + .with_context(|| format!("run {argv:?} in tmachine VM")) + } + + /// Copy a local file or directory into the machine. + pub async fn copy_in(&self, src: &Path, dst: &str) -> Result<()> { + let remote = format!("{SSH_USER}@{SSH_HOST}:{dst}"); + scp(src.as_os_str().to_string_lossy().as_ref(), &remote) + .await + .with_context(|| format!("copy {} to {dst} in tmachine VM", src.display())) + } + + /// Copy a file or directory out of the machine. + pub async fn copy_out(&self, src: &str, dst: &Path) -> Result<()> { + let remote = format!("{SSH_USER}@{SSH_HOST}:{src}"); + scp(&remote, dst.as_os_str().to_string_lossy().as_ref()) + .await + .with_context(|| format!("copy {src} from tmachine VM to {}", dst.display())) + } + + /// Write the OpenShell gateway's journal to `dir/gateway.log`. + pub async fn collect_diagnostics(&self, dir: &Path) -> Result<()> { + std::fs::create_dir_all(dir) + .with_context(|| format!("create diagnostics directory {}", dir.display()))?; + let output = self + .exec(&[ + "sudo", + "-n", + "journalctl", + "--no-pager", + "--lines", + "2000", + "_SYSTEMD_UNIT=openshell-gateway.service", + "+", + "_SYSTEMD_USER_UNIT=openshell-gateway.service", + ]) + .await?; + let log = dir.join("gateway.log"); + let mut contents = output.stdout; + contents.extend_from_slice(&output.stderr); + std::fs::write(&log, contents).with_context(|| format!("write {}", log.display())) + } + + /// Open an interactive SSH shell in the machine. + pub async fn shell(&self) -> Result { + ssh_command("ssh", &["-tt", "-p", SSH_PORT]) + .arg(format!("{SSH_USER}@{SSH_HOST}")) + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .await + .context("open SSH shell in tmachine VM") + } + + /// Power the machine off and wait for QEMU to exit. + pub async fn down(mut self) { + if let Some(vm) = self.vm.take() { + vm.shutdown().await; + vm.wait().await; + } + } +} + +fn ssh_command(program: &str, args: &[&str]) -> Command { + let mut command = Command::new("sshpass"); + command + .env("SSHPASS", SSH_PASSWORD) + .arg("-e") + .arg(program) + .args(args) + .args(SSH_OPTIONS); + command +} + +async fn scp(src: &str, dst: &str) -> Result<()> { + let output = ssh_command("scp", &["-r", "-P", SSH_PORT]) + .arg(src) + .arg(dst) + .stdin(Stdio::null()) + .output() + .await + .context("run scp")?; + ensure!( + output.status.success(), + "scp exited with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + Ok(()) +} + +/// Quote an argument for the remote POSIX shell that SSH runs commands in. +fn shell_quote(arg: &str) -> String { + let safe = !arg.is_empty() + && arg + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_./=:@,+%".contains(&b)); + if safe { + arg.to_string() + } else { + format!("'{}'", arg.replace('\'', r"'\''")) + } +} + +#[cfg(test)] +mod tests { + use super::shell_quote; + + #[test] + fn leaves_plain_arguments_unquoted() { + assert_eq!(shell_quote("cargo-nextest"), "cargo-nextest"); + assert_eq!(shell_quote("/var/lib/x.tar.zst"), "/var/lib/x.tar.zst"); + assert_eq!(shell_quote("KEY=value"), "KEY=value"); + } + + #[test] + fn quotes_shell_metacharacters() { + assert_eq!( + shell_quote("binary(a) | binary(b)"), + "'binary(a) | binary(b)'" + ); + assert_eq!(shell_quote("$HOME"), "'$HOME'"); + assert_eq!(shell_quote(""), "''"); + } + + #[test] + fn escapes_single_quotes() { + assert_eq!(shell_quote("it's"), r"'it'\''s'"); + } +} diff --git a/tests/tmachine/src/qemu/test.rs b/tests/tmachine/src/qemu/test.rs index ac8a8bdfc..cd4cb95d4 100644 --- a/tests/tmachine/src/qemu/test.rs +++ b/tests/tmachine/src/qemu/test.rs @@ -1,18 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::process::Stdio; - -use anyhow::{Context, Result}; -use tempfile::tempdir; -use tokio::process::Command; +use anyhow::Result; use crate::config::{Environment, Installer, Machine, Testsuite}; -use super::img::QemuImage; -use super::install::install; -use super::layer::run_playbooks; -use super::vm::QemuVm; +use super::session::Session; pub async fn test( machine: &Machine, @@ -20,46 +13,20 @@ pub async fn test( installer: &Installer, testsuite: &Testsuite, ) -> Result<()> { - let install_disk = install(machine, environment, installer).await?; - let test_dir = tempdir().unwrap(); - let test_disk = test_dir.path().join("test.qcow2"); - let image = QemuImage::create(&install_disk, test_disk).await; - let vm = QemuVm::start(&image).await; - - run_playbooks(&testsuite.playbooks, &testsuite.inputs).await?; + let session = Session::up(machine, environment, installer).await?; + session + .run_playbooks(&testsuite.playbooks, &testsuite.inputs) + .await?; if testsuite.interactive { println!("Opening an SSH shell in the tmachine VM."); - let ssh_status = Command::new("sshpass") - .env("SSHPASS", "tmachine") - .args([ - "-e", - "ssh", - "-tt", - "-p", - "2222", - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - "LogLevel=ERROR", - "tmachine@127.0.0.1", - ]) - .stdin(Stdio::inherit()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .status() - .await - .context("open SSH shell in tmachine VM")?; - - vm.shutdown().await; - vm.wait().await; + let ssh_status = session.shell().await; + session.down().await; + let ssh_status = ssh_status?; anyhow::ensure!(ssh_status.success(), "SSH shell exited with {ssh_status}"); return Ok(()); } - vm.shutdown().await; - vm.wait().await; + session.down().await; Ok(()) } diff --git a/tests/tmachine/src/qemu/vm.rs b/tests/tmachine/src/qemu/vm.rs index 1e035632e..c65ba7d7b 100644 --- a/tests/tmachine/src/qemu/vm.rs +++ b/tests/tmachine/src/qemu/vm.rs @@ -38,6 +38,11 @@ impl QemuVm { Self { child, runtime_dir } } + /// The exit status, if QEMU has already exited. + pub(super) fn try_wait(&mut self) -> Option { + self.child.try_wait().ok().flatten() + } + pub(super) async fn wait(mut self) { let status = self.child.wait().await.unwrap(); assert!(status.success()); @@ -117,8 +122,10 @@ fn command(image: &QemuImage, qmp_socket: &Path, firmware_vars: Option<&Path>) - .arg(format!("unix:{},server=on,wait=off", qmp_socket.display())); if let Some(firmware_vars) = firmware_vars { - let firmware_code = option_env!("TMACHINE_FIRMWARE_CODE") - .expect("TMACHINE_FIRMWARE_CODE must be set for ARM builds"); + let firmware_code = firmware_path( + "TMACHINE_FIRMWARE_CODE", + option_env!("TMACHINE_FIRMWARE_CODE"), + ); command .arg("-drive") .arg(format!( @@ -154,11 +161,24 @@ fn command(image: &QemuImage, qmp_socket: &Path, firmware_vars: Option<&Path>) - command } +/// A firmware path from the runtime environment, falling back to the value +/// compiled in. The fallback lets library users built outside Nix (such as +/// the test runner) supply firmware at runtime. +#[cfg(target_arch = "aarch64")] +fn firmware_path(name: &str, compiled: Option<&'static str>) -> String { + std::env::var(name) + .ok() + .or_else(|| compiled.map(str::to_string)) + .unwrap_or_else(|| panic!("{name} must be set for ARM builds")) +} + fn prepare_firmware(runtime_dir: &Path) -> Option { #[cfg(target_arch = "aarch64")] { - let template = option_env!("TMACHINE_FIRMWARE_VARS") - .expect("TMACHINE_FIRMWARE_VARS must be set for ARM builds"); + let template = firmware_path( + "TMACHINE_FIRMWARE_VARS", + option_env!("TMACHINE_FIRMWARE_VARS"), + ); let firmware_vars = runtime_dir.join("firmware-vars.fd"); fs::copy(template, &firmware_vars).unwrap(); fs::set_permissions(&firmware_vars, fs::Permissions::from_mode(0o600)).unwrap();