// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 //! Standalone runner for `OpenShell` CLI conformance scenarios. use std::collections::BTreeSet; use std::path::PathBuf; use std::process::ExitCode; use clap::{Parser, Subcommand, ValueEnum}; use openshell_conformance::{OpenShellRunner, Scenario, scenario, scenarios}; use serde::Serialize; #[derive(Debug, Parser)] #[command( name = "openshell-conformance", about = "Run OpenShell CLI conformance scenarios", version )] struct Cli { #[command(subcommand)] command: Command, } #[derive(Debug, Subcommand)] enum Command { /// List registered scenarios. List { #[arg(long, value_enum, default_value_t = OutputFormat::Text)] output: OutputFormat, }, /// Run all registered scenarios, or named scenarios. Run { /// Leaf scenario names or family prefixes. Omit to run every leaf. scenarios: Vec, /// Explicit path to the `OpenShell` CLI. Defaults to `openshell` on PATH. #[arg(long)] openshell_bin: Option, #[arg(long, value_enum, default_value_t = OutputFormat::Text)] output: OutputFormat, }, } #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] enum OutputFormat { Text, Json, } #[derive(Serialize)] struct ScenarioDescription<'a> { name: &'a str, description: &'a str, } #[derive(Serialize)] struct ScenarioResult<'a> { name: &'a str, passed: bool, diagnostic: Option, } #[derive(Serialize)] struct RunReport<'a> { scenarios: Vec>, passed: bool, } #[tokio::main] async fn main() -> ExitCode { match execute(Cli::parse()).await { Ok(()) => ExitCode::SUCCESS, Err(error) => { eprintln!("openshell-conformance: {error}"); ExitCode::FAILURE } } } async fn execute(cli: Cli) -> Result<(), String> { match cli.command { Command::List { output } => list(output), Command::Run { scenarios: requested, openshell_bin, output, } => run(&requested, openshell_bin, output).await, } } fn list(output: OutputFormat) -> Result<(), String> { match output { OutputFormat::Text => { for candidate in scenarios() { println!("{:<24} {}", candidate.name, candidate.description); } } OutputFormat::Json => { let result = scenarios() .iter() .map(|candidate| ScenarioDescription { name: candidate.name, description: candidate.description, }) .collect::>(); println!( "{}", serde_json::to_string_pretty(&result).map_err(|error| error.to_string())? ); } } Ok(()) } async fn run( requested: &[String], binary: Option, output: OutputFormat, ) -> Result<(), String> { let selected = select_scenarios(requested)?; let mut results = Vec::with_capacity(selected.len()); for candidate in selected { results.push(run_scenario(candidate, binary.as_ref()).await); } render_results(results, output) } async fn run_scenario( candidate: &'static Scenario, binary: Option<&PathBuf>, ) -> ScenarioResult<'static> { let runner = binary.map_or_else( || OpenShellRunner::new(candidate.name), |path| OpenShellRunner::with_binary(path.clone(), candidate.name), ); let mut runner = match runner { Ok(runner) => runner, Err(error) => { return ScenarioResult { name: candidate.name, passed: false, diagnostic: Some(error.to_string()), }; } }; eprintln!("CLI conformance run ID: {}", runner.id()); let scenario_result = match runner.check_gateway_status().await { Ok(()) => candidate.run(&mut runner).await, Err(error) => Err(error), }; let outcome = runner.finish(scenario_result).await; ScenarioResult { name: candidate.name, passed: outcome.is_ok(), diagnostic: outcome.err(), } } fn render_results( results: Vec>, output: OutputFormat, ) -> Result<(), String> { let passed = results.iter().all(|result| result.passed); match output { OutputFormat::Text => { for result in &results { if result.passed { println!("PASS {}", result.name); } else { println!( "FAIL {}\n{}", result.name, result.diagnostic.as_deref().unwrap_or("unknown failure") ); } } } OutputFormat::Json => println!( "{}", serde_json::to_string_pretty(&RunReport { scenarios: results, passed }) .map_err(|error| error.to_string())? ), } if passed { Ok(()) } else { Err("one or more scenarios failed".to_string()) } } fn select_scenarios(requested: &[String]) -> Result, String> { if requested.is_empty() { return Ok(scenarios().iter().collect()); } let mut selected = Vec::new(); let mut selected_names = BTreeSet::new(); for selector in requested { if let Some(candidate) = scenario(selector) { if selected_names.insert(candidate.name) { selected.push(candidate); } continue; } let prefix = format!("{selector}/"); let mut matched = false; for candidate in scenarios() .iter() .filter(|candidate| candidate.name.starts_with(&prefix)) { matched = true; if selected_names.insert(candidate.name) { selected.push(candidate); } } if !matched { return Err(format!( "unknown scenario or group '{selector}'; run `openshell-conformance list`" )); } } Ok(selected) } #[cfg(test)] mod tests { use clap::Parser; use super::*; #[test] fn selects_all_scenarios_by_default() { let selected = select_scenarios(&[]).expect("select all"); assert_eq!(selected.len(), 10); assert_eq!(selected.len(), scenarios().len()); } #[test] fn selects_named_scenario() { let selected = select_scenarios(&["smoke/exec".to_string()]).expect("select smoke exec"); assert_eq!(selected.len(), 1); assert_eq!(selected[0].name, "smoke/exec"); } #[test] fn expands_group_to_all_leaf_scenarios() { let selected = select_scenarios(&["smoke".to_string()]).expect("select smoke group"); let names = selected .iter() .map(|candidate| candidate.name) .collect::>(); assert_eq!(names, ["smoke/control-plane", "smoke/exec"]); } #[test] fn expands_lifecycle_group_to_independent_capabilities() { let selected = select_scenarios(&["sandbox-lifecycle".to_string()]).expect("select lifecycle group"); let names = selected .iter() .map(|candidate| candidate.name) .collect::>(); assert_eq!( names, [ "sandbox-lifecycle/control-plane", "sandbox-lifecycle/restart-persistence", ] ); } #[test] fn overlapping_selectors_do_not_run_a_leaf_twice() { let selected = select_scenarios(&[ "smoke".to_string(), "smoke/exec".to_string(), "policy-advisor".to_string(), ]) .expect("select overlapping scenarios"); let names = selected .iter() .map(|candidate| candidate.name) .collect::>(); assert_eq!( names, [ "smoke/control-plane", "smoke/exec", "policy-advisor/mechanistic-proposal", "policy-advisor/new-hostname-proposal", "policy-advisor/sandbox-local" ] ); } #[test] fn unknown_scenario_has_actionable_diagnostic() { let error = select_scenarios(&["missing".to_string()]).expect_err("unknown scenario"); assert!(error.contains("unknown scenario or group 'missing'")); assert!(error.contains("openshell-conformance list")); } #[test] fn parses_binary_override_and_json_output() { let cli = Cli::try_parse_from([ "openshell-conformance", "run", "smoke/exec", "--openshell-bin", "/opt/openshell", "--output", "json", ]) .expect("parse CLI"); let Command::Run { openshell_bin, output, .. } = cli.command else { panic!("expected run") }; assert_eq!(openshell_bin, Some(PathBuf::from("/opt/openshell"))); assert_eq!(output, OutputFormat::Json); } }