mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-02 07:34:45 +08:00
test(conformance): cover sandbox lifecycle in archives (#3375)
* test(conformance): add sandbox lifecycle coverage Signed-off-by: Evan Lezar <elezar@nvidia.com> * test(tmachine): rename smoke suite to conformance Signed-off-by: Evan Lezar <elezar@nvidia.com> * test(tmachine): configure client during scenario install Signed-off-by: Evan Lezar <elezar@nvidia.com> --------- Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar
@@ -23,7 +23,7 @@ use tokio::time::sleep;
|
||||
|
||||
use self::executor::{CliExecutionError, CliExecutor, ProcessCli};
|
||||
|
||||
pub use scenarios::SMOKE_SCENARIO;
|
||||
pub use scenarios::{SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO};
|
||||
|
||||
/// An installed conformance scenario.
|
||||
#[derive(Debug)]
|
||||
@@ -41,7 +41,7 @@ impl Scenario {
|
||||
}
|
||||
}
|
||||
|
||||
const SCENARIOS: &[Scenario] = &[SMOKE_SCENARIO];
|
||||
const SCENARIOS: &[Scenario] = &[SMOKE_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO];
|
||||
|
||||
/// Returns every scenario compiled into this distribution.
|
||||
pub fn scenarios() -> &'static [Scenario] {
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
//! Registered, portable conformance scenarios.
|
||||
|
||||
mod sandbox_lifecycle;
|
||||
mod smoke;
|
||||
|
||||
pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO;
|
||||
pub use smoke::SMOKE_SCENARIO;
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Portable sandbox lifecycle conformance scenarios.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{OpenShellRunner, Poll, Scenario, ScenarioFuture};
|
||||
|
||||
const CREATE_TIMEOUT: Duration = Duration::from_mins(10);
|
||||
const COMMAND_TIMEOUT: Duration = Duration::from_mins(2);
|
||||
const TRANSITION_TIMEOUT: Duration = Duration::from_mins(4);
|
||||
const TRANSITION_INTERVAL: Duration = Duration::from_secs(2);
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SandboxState {
|
||||
name: String,
|
||||
phase: String,
|
||||
}
|
||||
|
||||
/// Certify sandbox stop, start, and deletion lifecycle behavior.
|
||||
pub const SANDBOX_LIFECYCLE_SCENARIO: Scenario = Scenario {
|
||||
name: "sandbox-lifecycle",
|
||||
description: "Verify sandbox stop, start, and deletion lifecycle behavior.",
|
||||
run: run_sandbox_lifecycle,
|
||||
};
|
||||
|
||||
fn run_sandbox_lifecycle(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
|
||||
Box::pin(async move {
|
||||
stop_start_preserves_workspace(runner).await?;
|
||||
stopped_can_be_deleted(runner).await
|
||||
})
|
||||
}
|
||||
|
||||
async fn stop_start_preserves_workspace(runner: &mut OpenShellRunner) -> Result<(), String> {
|
||||
let sandbox_name = format!("ct-{}-ss", runner.id());
|
||||
let sentinel = format!("openshell-stop-start-{}", runner.id());
|
||||
let sentinel_path = "/sandbox/.openshell-stop-start-sentinel";
|
||||
let run_count_path = "/sandbox/.openshell-main-run-count";
|
||||
let main = format!(
|
||||
"count=0; test ! -f '{run_count_path}' || count=$(cat '{run_count_path}'); \
|
||||
count=$((count + 1)); printf '%s\\n' \"$count\" > '{run_count_path}'; \
|
||||
exec sleep infinity"
|
||||
);
|
||||
|
||||
create_running_sandbox(runner, &sandbox_name, &main, "stop-start").await?;
|
||||
exec_expect_exact(
|
||||
runner,
|
||||
&sandbox_name,
|
||||
"write-sentinel",
|
||||
&[
|
||||
"sh",
|
||||
"-lc",
|
||||
&format!("printf '%s\\n' '{sentinel}' > '{sentinel_path}' && sync"),
|
||||
],
|
||||
"",
|
||||
)
|
||||
.await?;
|
||||
|
||||
run_lifecycle_command(runner, "stop", &sandbox_name, "stop-start/stop").await?;
|
||||
wait_for_phase(runner, &sandbox_name, "Stopped", "stop-start/stopped").await?;
|
||||
|
||||
let stopped_exec = runner
|
||||
.step("stop-start/exec-while-stopped")
|
||||
.description(format!(
|
||||
"sandbox '{sandbox_name}' rejects exec while stopped"
|
||||
))
|
||||
.with_timeout(COMMAND_TIMEOUT)
|
||||
.run(&[
|
||||
"sandbox",
|
||||
"exec",
|
||||
"--name",
|
||||
&sandbox_name,
|
||||
"--no-tty",
|
||||
"--",
|
||||
"cat",
|
||||
sentinel_path,
|
||||
])
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if stopped_exec.success() {
|
||||
return Err(
|
||||
stopped_exec.failure_diagnostic("sandbox exec fails while the sandbox is stopped")
|
||||
);
|
||||
}
|
||||
|
||||
run_lifecycle_command(runner, "start", &sandbox_name, "stop-start/start").await?;
|
||||
wait_for_phase(runner, &sandbox_name, "Ready", "stop-start/restarted").await?;
|
||||
|
||||
exec_expect_exact(
|
||||
runner,
|
||||
&sandbox_name,
|
||||
"read-sentinel",
|
||||
&["cat", sentinel_path],
|
||||
&format!("{sentinel}\n"),
|
||||
)
|
||||
.await?;
|
||||
exec_expect_exact(
|
||||
runner,
|
||||
&sandbox_name,
|
||||
"read-main-run-count",
|
||||
&["cat", run_count_path],
|
||||
"2\n",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn stopped_can_be_deleted(runner: &mut OpenShellRunner) -> Result<(), String> {
|
||||
let sandbox_name = format!("ct-{}-sd", runner.id());
|
||||
create_running_sandbox(
|
||||
runner,
|
||||
&sandbox_name,
|
||||
"exec sleep infinity",
|
||||
"stopped-delete",
|
||||
)
|
||||
.await?;
|
||||
|
||||
run_lifecycle_command(runner, "stop", &sandbox_name, "stopped-delete/stop").await?;
|
||||
wait_for_phase(runner, &sandbox_name, "Stopped", "stopped-delete/stopped").await?;
|
||||
run_lifecycle_command(runner, "delete", &sandbox_name, "stopped-delete/delete").await?;
|
||||
wait_for_absence(runner, &sandbox_name, "stopped-delete/deleted").await?;
|
||||
runner.forget_sandbox(&sandbox_name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_running_sandbox(
|
||||
runner: &mut OpenShellRunner,
|
||||
sandbox_name: &str,
|
||||
main: &str,
|
||||
step: &str,
|
||||
) -> Result<(), String> {
|
||||
runner.track_sandbox(sandbox_name);
|
||||
let create = runner
|
||||
.step(format!("{step}/create"))
|
||||
.description(format!("sandbox '{sandbox_name}' is created"))
|
||||
.with_timeout(CREATE_TIMEOUT)
|
||||
.run(&[
|
||||
"sandbox",
|
||||
"create",
|
||||
"--name",
|
||||
sandbox_name,
|
||||
"--from",
|
||||
"base",
|
||||
"--detach",
|
||||
"--no-tty",
|
||||
"--",
|
||||
"sh",
|
||||
"-lc",
|
||||
main,
|
||||
])
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
create.require_success()?;
|
||||
wait_for_phase(runner, sandbox_name, "Ready", &format!("{step}/ready")).await
|
||||
}
|
||||
|
||||
async fn run_lifecycle_command(
|
||||
runner: &OpenShellRunner,
|
||||
operation: &str,
|
||||
sandbox_name: &str,
|
||||
step: &str,
|
||||
) -> Result<(), String> {
|
||||
let result = runner
|
||||
.step(step)
|
||||
.description(format!("sandbox '{sandbox_name}' {operation} succeeds"))
|
||||
.with_timeout(COMMAND_TIMEOUT)
|
||||
.run(&["sandbox", operation, sandbox_name])
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
result.require_success()
|
||||
}
|
||||
|
||||
async fn exec_expect_exact(
|
||||
runner: &OpenShellRunner,
|
||||
sandbox_name: &str,
|
||||
step: &str,
|
||||
command: &[&str],
|
||||
expected_stdout: &str,
|
||||
) -> Result<(), String> {
|
||||
let mut args = vec!["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"];
|
||||
args.extend_from_slice(command);
|
||||
let result = runner
|
||||
.step(format!("stop-start/{step}"))
|
||||
.description(format!("sandbox '{sandbox_name}' exec {step} succeeds"))
|
||||
.with_timeout(COMMAND_TIMEOUT)
|
||||
.run(&args)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
result.require_success()?;
|
||||
if result.stdout() == expected_stdout {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(result.failure_diagnostic(&format!("stdout is exactly {expected_stdout:?}")))
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_phase(
|
||||
runner: &mut OpenShellRunner,
|
||||
sandbox_name: &str,
|
||||
expected_phase: &str,
|
||||
step: &str,
|
||||
) -> Result<(), String> {
|
||||
let sandbox_name = sandbox_name.to_string();
|
||||
let expected_phase = expected_phase.to_string();
|
||||
let step = step.to_string();
|
||||
let poll_step = step.clone();
|
||||
runner
|
||||
.poll_until(
|
||||
&poll_step,
|
||||
TRANSITION_TIMEOUT,
|
||||
TRANSITION_INTERVAL,
|
||||
async move |runner| {
|
||||
let result = runner
|
||||
.step(format!("{step}/get"))
|
||||
.description(format!(
|
||||
"sandbox '{sandbox_name}' reaches phase {expected_phase}"
|
||||
))
|
||||
.with_timeout(COMMAND_TIMEOUT)
|
||||
.run(&["sandbox", "get", &sandbox_name, "--output", "json"])
|
||||
.await;
|
||||
match result {
|
||||
Ok(result) if !result.success() => {
|
||||
Poll::Pending(result.failure_diagnostic(&format!(
|
||||
"sandbox '{sandbox_name}' can be retrieved"
|
||||
)))
|
||||
}
|
||||
Ok(result) => match result.json::<SandboxState>() {
|
||||
Ok(state) if state.name != sandbox_name => Poll::Failed(format!(
|
||||
"sandbox get returned {:?}; expected '{sandbox_name}'",
|
||||
state.name
|
||||
)),
|
||||
Ok(state) if state.phase == expected_phase => Poll::Ready(()),
|
||||
Ok(state) => Poll::Pending(format!(
|
||||
"sandbox '{sandbox_name}' phase is {:?}; expected {expected_phase:?}",
|
||||
state.phase
|
||||
)),
|
||||
Err(error) => Poll::Failed(error.to_string()),
|
||||
},
|
||||
Err(error) => Poll::Pending(error.to_string()),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
async fn wait_for_absence(
|
||||
runner: &mut OpenShellRunner,
|
||||
sandbox_name: &str,
|
||||
step: &str,
|
||||
) -> Result<(), String> {
|
||||
let sandbox_name = sandbox_name.to_string();
|
||||
let step = step.to_string();
|
||||
let poll_step = step.clone();
|
||||
runner
|
||||
.poll_until(
|
||||
&poll_step,
|
||||
TRANSITION_TIMEOUT,
|
||||
TRANSITION_INTERVAL,
|
||||
async move |runner| {
|
||||
let result = runner
|
||||
.step(format!("{step}/get"))
|
||||
.description(format!("sandbox '{sandbox_name}' is no longer retrievable"))
|
||||
.with_timeout(COMMAND_TIMEOUT)
|
||||
.run(&["sandbox", "get", &sandbox_name, "--output", "json"])
|
||||
.await;
|
||||
match result {
|
||||
Ok(result) if !result.success() => Poll::Ready(()),
|
||||
Ok(_) => {
|
||||
Poll::Pending(format!("sandbox '{sandbox_name}' is still retrievable"))
|
||||
}
|
||||
Err(error) => Poll::Pending(error.to_string()),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
---
|
||||
- name: Run OpenShell smoke tests
|
||||
- name: Run OpenShell conformance tests
|
||||
hosts: all
|
||||
gather_facts: false
|
||||
tasks:
|
||||
@@ -18,7 +18,7 @@
|
||||
group: tmachine
|
||||
mode: "0700"
|
||||
|
||||
- name: Install OpenShell conformance test bundle
|
||||
- name: Extract OpenShell conformance test bundle
|
||||
become: true
|
||||
ansible.builtin.unarchive:
|
||||
src: "{{ openshell_conformance_test_bundle }}"
|
||||
@@ -26,24 +26,20 @@
|
||||
owner: tmachine
|
||||
group: tmachine
|
||||
|
||||
- name: Wait for OpenShell gateway
|
||||
ansible.builtin.wait_for:
|
||||
host: 127.0.0.1
|
||||
port: 17670
|
||||
timeout: 60
|
||||
# Report a malformed outer bundle directly instead of failing later with
|
||||
# an opaque cargo-nextest missing-archive error.
|
||||
- name: Check OpenShell conformance nextest archive
|
||||
ansible.builtin.stat:
|
||||
path: /var/lib/openshell-conformance/tests/tests.tar.zst
|
||||
register: conformance_archive
|
||||
|
||||
- name: Register OpenShell gateway
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- /usr/local/bin/openshell
|
||||
- gateway
|
||||
- add
|
||||
- http://127.0.0.1:17670
|
||||
- --local
|
||||
- --name
|
||||
- tmachine
|
||||
- name: Require OpenShell conformance nextest archive
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- conformance_archive.stat.isreg | default(false)
|
||||
fail_msg: OpenShell conformance test bundle did not contain tests.tar.zst
|
||||
|
||||
- name: Run OpenShell smoke conformance archive
|
||||
- name: Run OpenShell conformance archive
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- cargo-nextest
|
||||
@@ -60,13 +56,12 @@
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Show OpenShell smoke conformance diagnostics
|
||||
# Preserve both streams for failures without adding passing-test output to
|
||||
# every tmachine run.
|
||||
- name: Show OpenShell conformance diagnostics
|
||||
ansible.builtin.debug:
|
||||
var: conformance_result.stderr_lines
|
||||
|
||||
- name: Show OpenShell smoke conformance result
|
||||
ansible.builtin.debug:
|
||||
var: conformance_result.stdout_lines
|
||||
var: conformance_result
|
||||
when: conformance_result.rc != 0
|
||||
|
||||
- name: Read OpenShell gateway logs
|
||||
become: true
|
||||
@@ -88,8 +83,8 @@
|
||||
var: openshell_gateway_logs.stdout_lines
|
||||
when: conformance_result.rc != 0
|
||||
|
||||
- name: Require OpenShell smoke conformance success
|
||||
- name: Require OpenShell conformance success
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- conformance_result.rc == 0
|
||||
fail_msg: OpenShell smoke conformance test failed
|
||||
fail_msg: OpenShell conformance test failed
|
||||
@@ -7,3 +7,4 @@
|
||||
gather_facts: false
|
||||
roles:
|
||||
- openshell_gateway
|
||||
- openshell_client
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
---
|
||||
- name: Register OpenShell gateway for tmachine test client
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- /usr/local/bin/openshell
|
||||
- gateway
|
||||
- add
|
||||
- http://127.0.0.1:17670
|
||||
- --local
|
||||
- --name
|
||||
- tmachine
|
||||
+2
-2
@@ -108,8 +108,8 @@ let
|
||||
|
||||
testsuites = [
|
||||
{
|
||||
name = "smoke";
|
||||
playbooks = [ "ansible/playbooks/smoke.yaml" ];
|
||||
name = "conformance";
|
||||
playbooks = [ "ansible/playbooks/conformance/cli.yaml" ];
|
||||
inputs = {
|
||||
openshell_conformance_test_bundle = "../artifacts/test-archives/${muslTarget}/openshell-conformance-tests.tar";
|
||||
};
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Driver-agnostic sandbox lifecycle conformance tests.
|
||||
|
||||
use openshell_conformance::{OpenShellRunner, SANDBOX_LIFECYCLE_SCENARIO};
|
||||
|
||||
/// Exercise stop, start, and stopped-deletion behavior through the candidate CLI.
|
||||
#[tokio::test]
|
||||
async fn sandbox_lifecycle() {
|
||||
let mut runner = OpenShellRunner::from_env(SANDBOX_LIFECYCLE_SCENARIO.name)
|
||||
.expect("candidate openshell CLI is available");
|
||||
|
||||
let result = async {
|
||||
runner.check_gateway_status().await?;
|
||||
SANDBOX_LIFECYCLE_SCENARIO.run(&mut runner).await
|
||||
}
|
||||
.await;
|
||||
if let Err(error) = runner.finish(result).await {
|
||||
panic!("sandbox lifecycle conformance scenario failed:\n{error}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user