OpenShell/crates/openshell-conformance-cli/src/main.rs

// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Standalone runner for `OpenShell` CLI conformance scenarios.

use std::collections::BTreeSet;
use std::path::PathBuf;
use std::process::ExitCode;

use clap::{Parser, Subcommand, ValueEnum};
use openshell_conformance::{OpenShellRunner, Scenario, scenario, scenarios};
use serde::Serialize;

#[derive(Debug, Parser)]
#[command(
    name = "openshell-conformance",
    about = "Run OpenShell CLI conformance scenarios",
    version
)]
struct Cli {
    #[command(subcommand)]
    command: Command,
}

#[derive(Debug, Subcommand)]
enum Command {
    /// List registered scenarios.
    List {
        #[arg(long, value_enum, default_value_t = OutputFormat::Text)]
        output: OutputFormat,
    },
    /// Run all registered scenarios, or named scenarios.
    Run {
        /// Leaf scenario names or family prefixes. Omit to run every leaf.
        scenarios: Vec<String>,
        /// Explicit path to the `OpenShell` CLI. Defaults to `openshell` on PATH.
        #[arg(long)]
        openshell_bin: Option<PathBuf>,
        #[arg(long, value_enum, default_value_t = OutputFormat::Text)]
        output: OutputFormat,
    },
}

#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
enum OutputFormat {
    Text,
    Json,
}

#[derive(Serialize)]
struct ScenarioDescription<'a> {
    name: &'a str,
    description: &'a str,
}

#[derive(Serialize)]
struct ScenarioResult<'a> {
    name: &'a str,
    passed: bool,
    diagnostic: Option<String>,
}

#[derive(Serialize)]
struct RunReport<'a> {
    scenarios: Vec<ScenarioResult<'a>>,
    passed: bool,
}

#[tokio::main]
async fn main() -> ExitCode {
    match execute(Cli::parse()).await {
        Ok(()) => ExitCode::SUCCESS,
        Err(error) => {
            eprintln!("openshell-conformance: {error}");
            ExitCode::FAILURE
        }
    }
}

async fn execute(cli: Cli) -> Result<(), String> {
    match cli.command {
        Command::List { output } => list(output),
        Command::Run {
            scenarios: requested,
            openshell_bin,
            output,
        } => run(&requested, openshell_bin, output).await,
    }
}

fn list(output: OutputFormat) -> Result<(), String> {
    match output {
        OutputFormat::Text => {
            for candidate in scenarios() {
                println!("{:<24} {}", candidate.name, candidate.description);
            }
        }
        OutputFormat::Json => {
            let result = scenarios()
                .iter()
                .map(|candidate| ScenarioDescription {
                    name: candidate.name,
                    description: candidate.description,
                })
                .collect::<Vec<_>>();
            println!(
                "{}",
                serde_json::to_string_pretty(&result).map_err(|error| error.to_string())?
            );
        }
    }
    Ok(())
}

async fn run(
    requested: &[String],
    binary: Option<PathBuf>,
    output: OutputFormat,
) -> Result<(), String> {
    let selected = select_scenarios(requested)?;
    let mut results = Vec::with_capacity(selected.len());
    for candidate in selected {
        results.push(run_scenario(candidate, binary.as_ref()).await);
    }

    render_results(results, output)
}

async fn run_scenario(
    candidate: &'static Scenario,
    binary: Option<&PathBuf>,
) -> ScenarioResult<'static> {
    let runner = binary.map_or_else(
        || OpenShellRunner::new(candidate.name),
        |path| OpenShellRunner::with_binary(path.clone(), candidate.name),
    );
    let mut runner = match runner {
        Ok(runner) => runner,
        Err(error) => {
            return ScenarioResult {
                name: candidate.name,
                passed: false,
                diagnostic: Some(error.to_string()),
            };
        }
    };
    eprintln!("CLI conformance run ID: {}", runner.id());
    let scenario_result = match runner.check_gateway_status().await {
        Ok(()) => candidate.run(&mut runner).await,
        Err(error) => Err(error),
    };
    let outcome = runner.finish(scenario_result).await;
    ScenarioResult {
        name: candidate.name,
        passed: outcome.is_ok(),
        diagnostic: outcome.err(),
    }
}

fn render_results(
    results: Vec<ScenarioResult<'static>>,
    output: OutputFormat,
) -> Result<(), String> {
    let passed = results.iter().all(|result| result.passed);
    match output {
        OutputFormat::Text => {
            for result in &results {
                if result.passed {
                    println!("PASS {}", result.name);
                } else {
                    println!(
                        "FAIL {}\n{}",
                        result.name,
                        result.diagnostic.as_deref().unwrap_or("unknown failure")
                    );
                }
            }
        }
        OutputFormat::Json => println!(
            "{}",
            serde_json::to_string_pretty(&RunReport {
                scenarios: results,
                passed
            })
            .map_err(|error| error.to_string())?
        ),
    }
    if passed {
        Ok(())
    } else {
        Err("one or more scenarios failed".to_string())
    }
}

fn select_scenarios(requested: &[String]) -> Result<Vec<&'static Scenario>, String> {
    if requested.is_empty() {
        return Ok(scenarios().iter().collect());
    }

    let mut selected = Vec::new();
    let mut selected_names = BTreeSet::new();
    for selector in requested {
        if let Some(candidate) = scenario(selector) {
            if selected_names.insert(candidate.name) {
                selected.push(candidate);
            }
            continue;
        }

        let prefix = format!("{selector}/");
        let mut matched = false;
        for candidate in scenarios()
            .iter()
            .filter(|candidate| candidate.name.starts_with(&prefix))
        {
            matched = true;
            if selected_names.insert(candidate.name) {
                selected.push(candidate);
            }
        }
        if !matched {
            return Err(format!(
                "unknown scenario or group '{selector}'; run `openshell-conformance list`"
            ));
        }
    }
    Ok(selected)
}

#[cfg(test)]
mod tests {
    use clap::Parser;

    use super::*;

    #[test]
    fn selects_all_scenarios_by_default() {
        let selected = select_scenarios(&[]).expect("select all");
        assert_eq!(selected.len(), 10);
        assert_eq!(selected.len(), scenarios().len());
    }

    #[test]
    fn selects_named_scenario() {
        let selected = select_scenarios(&["smoke/exec".to_string()]).expect("select smoke exec");
        assert_eq!(selected.len(), 1);
        assert_eq!(selected[0].name, "smoke/exec");
    }

    #[test]
    fn expands_group_to_all_leaf_scenarios() {
        let selected = select_scenarios(&["smoke".to_string()]).expect("select smoke group");
        let names = selected
            .iter()
            .map(|candidate| candidate.name)
            .collect::<Vec<_>>();
        assert_eq!(names, ["smoke/control-plane", "smoke/exec"]);
    }

    #[test]
    fn expands_lifecycle_group_to_independent_capabilities() {
        let selected =
            select_scenarios(&["sandbox-lifecycle".to_string()]).expect("select lifecycle group");
        let names = selected
            .iter()
            .map(|candidate| candidate.name)
            .collect::<Vec<_>>();
        assert_eq!(
            names,
            [
                "sandbox-lifecycle/control-plane",
                "sandbox-lifecycle/restart-persistence",
            ]
        );
    }

    #[test]
    fn overlapping_selectors_do_not_run_a_leaf_twice() {
        let selected = select_scenarios(&[
            "smoke".to_string(),
            "smoke/exec".to_string(),
            "policy-advisor".to_string(),
        ])
        .expect("select overlapping scenarios");
        let names = selected
            .iter()
            .map(|candidate| candidate.name)
            .collect::<Vec<_>>();
        assert_eq!(
            names,
            [
                "smoke/control-plane",
                "smoke/exec",
                "policy-advisor/mechanistic-proposal",
                "policy-advisor/new-hostname-proposal",
                "policy-advisor/sandbox-local"
            ]
        );
    }

    #[test]
    fn unknown_scenario_has_actionable_diagnostic() {
        let error = select_scenarios(&["missing".to_string()]).expect_err("unknown scenario");
        assert!(error.contains("unknown scenario or group 'missing'"));
        assert!(error.contains("openshell-conformance list"));
    }

    #[test]
    fn parses_binary_override_and_json_output() {
        let cli = Cli::try_parse_from([
            "openshell-conformance",
            "run",
            "smoke/exec",
            "--openshell-bin",
            "/opt/openshell",
            "--output",
            "json",
        ])
        .expect("parse CLI");
        let Command::Run {
            openshell_bin,
            output,
            ..
        } = cli.command
        else {
            panic!("expected run")
        };
        assert_eq!(openshell_bin, Some(PathBuf::from("/opt/openshell")));
        assert_eq!(output, OutputFormat::Json);
    }
}