mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-30 07:33:59 +08:00
test(conformance): run driver suites with cargo (#3866)
* test(conformance): run driver suites with cargo Signed-off-by: Evan Lezar <elezar@nvidia.com> * fix(e2e): resolve conformance CLI path before cargo tests Signed-off-by: Evan Lezar <elezar@nvidia.com> * refactor(conformance): remove unused CLI scenario APIs Signed-off-by: Evan Lezar <elezar@nvidia.com> --------- Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar
@@ -1,12 +1,6 @@
|
||||
name: Setup E2E CLI
|
||||
description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests
|
||||
|
||||
inputs:
|
||||
conformance-artifact-prefix:
|
||||
description: Optional conformance CLI artifact name prefix; linux-<arch> is appended automatically
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
@@ -23,27 +17,3 @@ runs:
|
||||
chmod +x "$cli"
|
||||
"$cli" --version
|
||||
echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Download prebuilt conformance CLI
|
||||
if: inputs.conformance-artifact-prefix != ''
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: ${{ format('{0}-{1}', inputs.conformance-artifact-prefix, runner.arch == 'X64' && 'x86_64-unknown-linux-musl' || 'aarch64-unknown-linux-musl') }}
|
||||
path: .e2e/prebuilt-conformance
|
||||
|
||||
- name: Configure prebuilt conformance CLI
|
||||
if: inputs.conformance-artifact-prefix != ''
|
||||
shell: bash
|
||||
run: | # zizmor: ignore[github-env] validated filename under the trusted workspace path
|
||||
set -euo pipefail
|
||||
conformance="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance"
|
||||
if [[ ! -f "$conformance" ]]; then
|
||||
echo "downloaded artifact is missing $conformance" >&2
|
||||
exit 1
|
||||
fi
|
||||
# TODO: Remove this temporary mode diagnostic after CI confirms artifact permission handling.
|
||||
echo "conformance binary mode before chmod: $(ls -l "$conformance")"
|
||||
chmod +x "$conformance"
|
||||
echo "conformance binary mode after chmod: $(ls -l "$conformance")"
|
||||
"$conformance" list --output json >/dev/null
|
||||
echo "OPENSHELL_CONFORMANCE_BIN=$conformance" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -310,7 +310,6 @@ jobs:
|
||||
with:
|
||||
image-tag: ${{ github.sha }}
|
||||
runner: linux-arm64-cpu8
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
podman-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-images]
|
||||
@@ -322,7 +321,6 @@ jobs:
|
||||
uses: ./.github/workflows/e2e-podman-test.yml
|
||||
with:
|
||||
image-tag: ${{ github.sha }}
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
suite-matrix: >-
|
||||
[{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}]
|
||||
|
||||
@@ -334,8 +332,6 @@ jobs:
|
||||
contents: read
|
||||
packages: read
|
||||
uses: ./.github/workflows/e2e-vm-test.yml
|
||||
with:
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
docker-external-driver-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images]
|
||||
@@ -350,7 +346,6 @@ jobs:
|
||||
runner: linux-arm64-cpu8
|
||||
gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu
|
||||
external-driver-binary: openshell-driver-docker
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
suite-matrix: >-
|
||||
[{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}]
|
||||
|
||||
@@ -366,7 +361,6 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
|
||||
external-driver-binary: openshell-driver-podman
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
suite-matrix: >-
|
||||
[{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}]
|
||||
|
||||
@@ -382,7 +376,6 @@ jobs:
|
||||
gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
|
||||
suite-name: external-driver
|
||||
e2e-task: e2e:vm:external-driver
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
gpu-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-images]
|
||||
@@ -394,7 +387,6 @@ jobs:
|
||||
uses: ./.github/workflows/e2e-gpu-test.yaml
|
||||
with:
|
||||
image-tag: ${{ github.sha }}
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-images]
|
||||
@@ -416,7 +408,6 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }})
|
||||
agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-workspace-managed-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-images]
|
||||
@@ -430,7 +421,6 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes E2E (workspace managed mode)
|
||||
e2e-task: e2e:kubernetes:workspace-managed
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-external-driver-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images]
|
||||
@@ -447,7 +437,6 @@ jobs:
|
||||
gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
|
||||
external-driver-binary: openshell-driver-kubernetes
|
||||
cluster-images: sandbox supervisor
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-workspace-operator-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-images]
|
||||
@@ -461,7 +450,6 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes E2E (workspace operator mode)
|
||||
e2e-task: e2e:kubernetes:workspace-operator
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-ha-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-images]
|
||||
@@ -479,7 +467,6 @@ jobs:
|
||||
test-name: kubernetes_ha_rebalancing
|
||||
kubernetes-features: e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha
|
||||
use-envoy-gateway: true
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-credential-drivers-e2e:
|
||||
needs: [pr_metadata, build-binaries, build-images]
|
||||
@@ -493,7 +480,6 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes Credential Drivers E2E
|
||||
e2e-task: e2e:kubernetes:credential-drivers
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
core-e2e-result:
|
||||
name: Core E2E result
|
||||
|
||||
@@ -62,22 +62,6 @@ jobs:
|
||||
supervisor-image-tag: ${{ inputs.supervisor-image-tag }}
|
||||
extra-cargo-flags: ${{ env.EXTRA_CARGO_FLAGS }}
|
||||
|
||||
conformance:
|
||||
name: openshell-conformance (${{ matrix.triple }})
|
||||
env:
|
||||
PACKAGE: openshell-conformance-cli
|
||||
BINARY: openshell-conformance
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- triple: x86_64-unknown-linux-musl
|
||||
runner: linux-amd64-cpu8
|
||||
- triple: aarch64-unknown-linux-musl
|
||||
runner: linux-arm64-cpu8
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 60
|
||||
steps: *build_steps
|
||||
|
||||
prover:
|
||||
name: openshell-prover (${{ matrix.triple }})
|
||||
env:
|
||||
|
||||
@@ -25,10 +25,6 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
conformance-artifact-prefix:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
suite-matrix:
|
||||
required: false
|
||||
type: string
|
||||
@@ -71,8 +67,6 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ./.github/actions/setup-e2e-cli
|
||||
with:
|
||||
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
|
||||
|
||||
- uses: ./.github/actions/setup-e2e-gateway
|
||||
with:
|
||||
|
||||
@@ -7,11 +7,6 @@ on:
|
||||
description: "Image tag to test (typically the commit SHA)"
|
||||
required: true
|
||||
type: string
|
||||
conformance-artifact-prefix:
|
||||
description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is <triple>)"
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
@@ -66,8 +61,6 @@ jobs:
|
||||
|
||||
- name: Use prebuilt OpenShell CLI
|
||||
uses: ./.github/actions/setup-e2e-cli
|
||||
with:
|
||||
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
|
||||
|
||||
- name: Use prebuilt OpenShell gateway
|
||||
uses: ./.github/actions/setup-e2e-gateway
|
||||
|
||||
@@ -67,11 +67,6 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
conformance-artifact-prefix:
|
||||
description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is <triple>)"
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
external-driver-binary:
|
||||
description: "Optional standalone compute driver binary used to compose a local test image"
|
||||
required: false
|
||||
@@ -112,8 +107,6 @@ jobs:
|
||||
|
||||
- name: Use prebuilt OpenShell CLI
|
||||
uses: ./.github/actions/setup-e2e-cli
|
||||
with:
|
||||
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
|
||||
|
||||
- name: Use prebuilt OpenShell gateway
|
||||
if: inputs.gateway-artifact != ''
|
||||
|
||||
@@ -21,10 +21,6 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
conformance-artifact-prefix:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
suite-matrix:
|
||||
required: true
|
||||
type: string
|
||||
@@ -56,8 +52,6 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ./.github/actions/setup-e2e-cli
|
||||
with:
|
||||
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
|
||||
|
||||
- uses: ./.github/actions/setup-e2e-gateway
|
||||
with:
|
||||
|
||||
@@ -22,10 +22,6 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: e2e:vm
|
||||
conformance-artifact-prefix:
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
@@ -47,8 +43,6 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ./.github/actions/setup-e2e-cli
|
||||
with:
|
||||
conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
|
||||
|
||||
- uses: ./.github/actions/setup-e2e-gateway
|
||||
with:
|
||||
|
||||
@@ -159,7 +159,6 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
checkout-ref: ${{ github.sha }}
|
||||
runner: linux-arm64-cpu8
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
vm-e2e:
|
||||
needs: [build-binaries, build-vm-driver]
|
||||
@@ -170,7 +169,6 @@ jobs:
|
||||
uses: ./.github/workflows/e2e-vm-test.yml
|
||||
with:
|
||||
checkout-ref: ${{ github.sha }}
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
tag-ghcr-dev:
|
||||
name: Tag GHCR Images as Dev
|
||||
|
||||
@@ -210,7 +210,6 @@ jobs:
|
||||
image-tag: ${{ needs.compute-versions.outputs.source_sha }}
|
||||
checkout-ref: ${{ needs.compute-versions.outputs.source_sha }}
|
||||
runner: linux-arm64-cpu8
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
vm-e2e:
|
||||
needs: [compute-versions, build-binaries, build-vm-driver]
|
||||
@@ -221,7 +220,6 @@ jobs:
|
||||
uses: ./.github/workflows/e2e-vm-test.yml
|
||||
with:
|
||||
checkout-ref: ${{ needs.compute-versions.outputs.source_sha }}
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
protobuf-compatibility:
|
||||
name: Protobuf Compatibility
|
||||
|
||||
@@ -23,7 +23,6 @@ Do not rely on this file for a full inventory. The detailed public and contribut
|
||||
|------|-----------|---------|
|
||||
| `crates/openshell-cli/` | CLI binary | User-facing command-line interface |
|
||||
| `crates/openshell-conformance/` | CLI conformance library | Reusable driver-agnostic scenarios and command runner |
|
||||
| `crates/openshell-conformance-cli/` | Conformance CLI | Legacy local `list` and `run` entrypoint pending follow-up cleanup |
|
||||
| `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary |
|
||||
| `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O |
|
||||
| `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking |
|
||||
|
||||
Generated
-11
@@ -4306,17 +4306,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openshell-conformance-cli"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"openshell-conformance",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openshell-core"
|
||||
version = "0.0.0"
|
||||
|
||||
+24
-15
@@ -213,11 +213,12 @@ HTTP `Host`, TLS SNI, and mTLS handling.
|
||||
Suites:
|
||||
|
||||
- Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests.
|
||||
- CLI conformance (`openshell-conformance`) - named scenarios for lifecycle,
|
||||
mechanistic drafts, and the sandbox-local API, including agent-authored
|
||||
permission requests. Driver E2E wrappers run every scenario. The
|
||||
installed-artifact conformance suite runs all scenarios and offers a focused
|
||||
`policy-advisor` testsuite for manual integration runs.
|
||||
- CLI conformance (`tests/suites/conformance`) - portable Cargo tests for
|
||||
lifecycle, mechanistic drafts, file transfer, and the sandbox-local API,
|
||||
including agent-authored permission requests. Driver E2E runs the complete
|
||||
Cargo test package. The installed-artifact conformance suite runs the same
|
||||
tests from a nextest archive and offers a focused `policy-advisor` testsuite
|
||||
for manual integration runs.
|
||||
- Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or
|
||||
`e2e-vm`) - CLI conformance plus the common and driver-specific coverage for
|
||||
the selected deployment.
|
||||
@@ -242,17 +243,25 @@ Run the Docker-backed Rust CLI e2e suite:
|
||||
mise run e2e:docker
|
||||
```
|
||||
|
||||
Run the minimal portable CLI conformance profile against the gateway selected
|
||||
in your OpenShell CLI configuration:
|
||||
Run the portable CLI conformance suite against the gateway selected in your
|
||||
OpenShell CLI configuration:
|
||||
|
||||
```shell
|
||||
mise run e2e:cli-conformance
|
||||
cargo build --package openshell-cli
|
||||
OPENSHELL_BIN="$PWD/target/debug/openshell" \
|
||||
cargo test \
|
||||
--locked \
|
||||
--manifest-path tests/suites/conformance/Cargo.toml \
|
||||
--package openshell-test-conformance-cli \
|
||||
--no-fail-fast \
|
||||
-- \
|
||||
--test-threads=1 \
|
||||
--nocapture
|
||||
```
|
||||
|
||||
The gateway must already be installed, reachable, and selected before the task
|
||||
starts. The task does not provision a gateway or select a compute driver. Set
|
||||
`OPENSHELL_BIN` to test a prebuilt CLI; otherwise, the task builds the CLI from
|
||||
the current checkout.
|
||||
The gateway must already be installed, reachable, and selected before the tests
|
||||
start. The test suite does not provision a gateway or select a compute driver.
|
||||
Set `OPENSHELL_BIN` to another executable to test a different prebuilt CLI.
|
||||
|
||||
The phase-1 scenario verifies the complete CLI-to-gateway-to-driver path without
|
||||
depending on how the gateway was installed or which driver is configured. It
|
||||
@@ -278,9 +287,9 @@ openshell sandbox list --output json
|
||||
openshell sandbox delete <sandbox-name>
|
||||
```
|
||||
|
||||
Gateway-backed Rust E2E tasks build the standalone conformance CLI, run its
|
||||
registered scenarios against the configured gateway, then run any lane-specific
|
||||
Rust tests that still apply. Run the Podman-backed Rust CLI e2e suite:
|
||||
Gateway-backed Rust E2E tasks run the CLI conformance Cargo tests against the
|
||||
configured gateway, then run any lane-specific Rust tests that still apply.
|
||||
Run the Podman-backed Rust CLI e2e suite:
|
||||
|
||||
```shell
|
||||
mise run e2e:podman
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
[package]
|
||||
name = "openshell-conformance-cli"
|
||||
description = "Standalone OpenShell CLI conformance test runner"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-conformance"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
clap.workspace = true
|
||||
openshell-conformance = { path = "../openshell-conformance" }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -1,273 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Standalone runner for `OpenShell` CLI conformance scenarios.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::{Parser, Subcommand, ValueEnum};
|
||||
use openshell_conformance::{OpenShellRunner, Scenario, scenario, scenarios};
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(
|
||||
name = "openshell-conformance",
|
||||
about = "Run OpenShell CLI conformance scenarios",
|
||||
version
|
||||
)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Command {
|
||||
/// List registered scenarios.
|
||||
List {
|
||||
#[arg(long, value_enum, default_value_t = OutputFormat::Text)]
|
||||
output: OutputFormat,
|
||||
},
|
||||
/// Run all registered scenarios, or named scenarios.
|
||||
Run {
|
||||
/// Scenario names. Omit to run every registered scenario.
|
||||
scenarios: Vec<String>,
|
||||
/// Explicit path to the `OpenShell` CLI. Defaults to `openshell` on PATH.
|
||||
#[arg(long)]
|
||||
openshell_bin: Option<PathBuf>,
|
||||
#[arg(long, value_enum, default_value_t = OutputFormat::Text)]
|
||||
output: OutputFormat,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
|
||||
enum OutputFormat {
|
||||
Text,
|
||||
Json,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ScenarioDescription<'a> {
|
||||
name: &'a str,
|
||||
description: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ScenarioResult<'a> {
|
||||
name: &'a str,
|
||||
passed: bool,
|
||||
diagnostic: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct RunReport<'a> {
|
||||
scenarios: Vec<ScenarioResult<'a>>,
|
||||
passed: bool,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> ExitCode {
|
||||
match execute(Cli::parse()).await {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
Err(error) => {
|
||||
eprintln!("openshell-conformance: {error}");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute(cli: Cli) -> Result<(), String> {
|
||||
match cli.command {
|
||||
Command::List { output } => list(output),
|
||||
Command::Run {
|
||||
scenarios: requested,
|
||||
openshell_bin,
|
||||
output,
|
||||
} => run(&requested, openshell_bin, output).await,
|
||||
}
|
||||
}
|
||||
|
||||
fn list(output: OutputFormat) -> Result<(), String> {
|
||||
match output {
|
||||
OutputFormat::Text => {
|
||||
for candidate in scenarios() {
|
||||
println!("{:<24} {}", candidate.name, candidate.description);
|
||||
}
|
||||
}
|
||||
OutputFormat::Json => {
|
||||
let result = scenarios()
|
||||
.iter()
|
||||
.map(|candidate| ScenarioDescription {
|
||||
name: candidate.name,
|
||||
description: candidate.description,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&result).map_err(|error| error.to_string())?
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run(
|
||||
requested: &[String],
|
||||
binary: Option<PathBuf>,
|
||||
output: OutputFormat,
|
||||
) -> Result<(), String> {
|
||||
let selected = select_scenarios(requested)?;
|
||||
let mut results = Vec::with_capacity(selected.len());
|
||||
for candidate in selected {
|
||||
results.push(run_scenario(candidate, binary.as_ref()).await);
|
||||
}
|
||||
|
||||
render_results(results, output)
|
||||
}
|
||||
|
||||
async fn run_scenario(
|
||||
candidate: &'static Scenario,
|
||||
binary: Option<&PathBuf>,
|
||||
) -> ScenarioResult<'static> {
|
||||
let runner = binary.map_or_else(
|
||||
|| OpenShellRunner::new(candidate.name),
|
||||
|path| OpenShellRunner::with_binary(path.clone(), candidate.name),
|
||||
);
|
||||
let mut runner = match runner {
|
||||
Ok(runner) => runner,
|
||||
Err(error) => {
|
||||
return ScenarioResult {
|
||||
name: candidate.name,
|
||||
passed: false,
|
||||
diagnostic: Some(error.to_string()),
|
||||
};
|
||||
}
|
||||
};
|
||||
eprintln!("CLI conformance run ID: {}", runner.id());
|
||||
let scenario_result = match runner.check_gateway_status().await {
|
||||
Ok(()) => candidate.run(&mut runner).await,
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
let outcome = runner.finish(scenario_result).await;
|
||||
ScenarioResult {
|
||||
name: candidate.name,
|
||||
passed: outcome.is_ok(),
|
||||
diagnostic: outcome.err(),
|
||||
}
|
||||
}
|
||||
|
||||
fn render_results(
|
||||
results: Vec<ScenarioResult<'static>>,
|
||||
output: OutputFormat,
|
||||
) -> Result<(), String> {
|
||||
let passed = results.iter().all(|result| result.passed);
|
||||
match output {
|
||||
OutputFormat::Text => {
|
||||
for result in &results {
|
||||
if result.passed {
|
||||
println!("PASS {}", result.name);
|
||||
} else {
|
||||
println!(
|
||||
"FAIL {}\n{}",
|
||||
result.name,
|
||||
result.diagnostic.as_deref().unwrap_or("unknown failure")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
OutputFormat::Json => println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&RunReport {
|
||||
scenarios: results,
|
||||
passed
|
||||
})
|
||||
.map_err(|error| error.to_string())?
|
||||
),
|
||||
}
|
||||
if passed {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("one or more scenarios failed".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn select_scenarios(requested: &[String]) -> Result<Vec<&'static Scenario>, String> {
|
||||
if requested.is_empty() {
|
||||
return Ok(scenarios().iter().collect());
|
||||
}
|
||||
requested
|
||||
.iter()
|
||||
.map(|name| {
|
||||
scenario(name).ok_or_else(|| {
|
||||
format!("unknown scenario '{name}'; run `openshell-conformance list`")
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use clap::Parser;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn selects_all_scenarios_by_default() {
|
||||
assert_eq!(
|
||||
select_scenarios(&[]).expect("select all").len(),
|
||||
scenarios().len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_named_scenario() {
|
||||
let selected = select_scenarios(&["smoke".to_string()]).expect("select smoke");
|
||||
assert_eq!(selected[0].name, "smoke");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_scenario_has_actionable_diagnostic() {
|
||||
let error = select_scenarios(&["missing".to_string()]).expect_err("unknown scenario");
|
||||
assert!(error.contains("openshell-conformance list"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_named_policy_scenarios() {
|
||||
let selected = select_scenarios(&[
|
||||
"mechanistic-proposal".to_string(),
|
||||
"policy-local".to_string(),
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
selected
|
||||
.iter()
|
||||
.map(|scenario| scenario.name)
|
||||
.collect::<Vec<_>>(),
|
||||
["mechanistic-proposal", "policy-local"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_binary_override_and_json_output() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"openshell-conformance",
|
||||
"run",
|
||||
"smoke",
|
||||
"--openshell-bin",
|
||||
"/opt/openshell",
|
||||
"--output",
|
||||
"json",
|
||||
])
|
||||
.expect("parse CLI");
|
||||
let Command::Run {
|
||||
openshell_bin,
|
||||
output,
|
||||
..
|
||||
} = cli.command
|
||||
else {
|
||||
panic!("expected run")
|
||||
};
|
||||
assert_eq!(openshell_bin, Some(PathBuf::from("/opt/openshell")));
|
||||
assert_eq!(output, OutputFormat::Json);
|
||||
}
|
||||
}
|
||||
@@ -25,16 +25,15 @@ use self::executor::{CliExecutionError, CliExecutor, ProcessCli};
|
||||
|
||||
pub use scenarios::{
|
||||
FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO,
|
||||
FILE_TRANSFER_ROUND_TRIP_SCENARIO, FILE_TRANSFER_SCENARIO, MECHANISTIC_PROPOSAL_SCENARIO,
|
||||
FILE_TRANSFER_ROUND_TRIP_SCENARIO, MECHANISTIC_PROPOSAL_SCENARIO,
|
||||
NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO,
|
||||
SMOKE_SCENARIO,
|
||||
};
|
||||
|
||||
/// An installed conformance scenario.
|
||||
/// A portable conformance scenario exercised by the Cargo test suite.
|
||||
#[derive(Debug)]
|
||||
pub struct Scenario {
|
||||
pub name: &'static str,
|
||||
pub description: &'static str,
|
||||
run: for<'a> fn(&'a mut OpenShellRunner) -> ScenarioFuture<'a>,
|
||||
}
|
||||
|
||||
@@ -46,25 +45,6 @@ impl Scenario {
|
||||
}
|
||||
}
|
||||
|
||||
const SCENARIOS: &[Scenario] = &[
|
||||
SMOKE_SCENARIO,
|
||||
SANDBOX_LIFECYCLE_SCENARIO,
|
||||
FILE_TRANSFER_SCENARIO,
|
||||
MECHANISTIC_PROPOSAL_SCENARIO,
|
||||
NEW_HOSTNAME_PROPOSAL_SCENARIO,
|
||||
POLICY_LOCAL_SCENARIO,
|
||||
];
|
||||
|
||||
/// Returns every scenario compiled into this distribution.
|
||||
pub fn scenarios() -> &'static [Scenario] {
|
||||
SCENARIOS
|
||||
}
|
||||
|
||||
/// Finds a scenario by its stable command-line name.
|
||||
pub fn scenario(name: &str) -> Option<&'static Scenario> {
|
||||
scenarios().iter().find(|candidate| candidate.name == name)
|
||||
}
|
||||
|
||||
const CLEANUP_TIMEOUT: Duration = Duration::from_mins(2);
|
||||
pub const STATUS_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const GATEWAY_STATUS_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
@@ -281,13 +261,6 @@ pub struct OpenShellCommand<'a> {
|
||||
}
|
||||
|
||||
impl OpenShellRunner {
|
||||
pub fn new(scenario: &str) -> Result<Self, RunnerError> {
|
||||
Ok(Self::with_executor(
|
||||
Arc::new(ProcessCli::new(PathBuf::from("openshell"))),
|
||||
scenario,
|
||||
))
|
||||
}
|
||||
|
||||
/// Uses an explicit `openshell` binary rather than resolving it on `PATH`.
|
||||
pub fn with_binary(binary: PathBuf, scenario: &str) -> Result<Self, RunnerError> {
|
||||
if !binary.is_file() {
|
||||
@@ -335,10 +308,6 @@ impl OpenShellRunner {
|
||||
&self.run_id
|
||||
}
|
||||
|
||||
pub fn scenario(&self) -> &str {
|
||||
&self.scenario
|
||||
}
|
||||
|
||||
pub fn step(&self, step: impl Into<String>) -> CommandStep<'_> {
|
||||
CommandStep {
|
||||
runner: self,
|
||||
@@ -871,7 +840,6 @@ mod tests {
|
||||
let (mut runner, _cli) = test_runner(Vec::new());
|
||||
|
||||
assert_eq!(runner.id().len(), 10);
|
||||
assert_eq!(runner.scenario(), "smoke");
|
||||
runner.finished = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -17,42 +17,24 @@ const COMMAND_TIMEOUT: Duration = Duration::from_mins(2);
|
||||
const TRANSFER_TIMEOUT: Duration = Duration::from_mins(5);
|
||||
const LARGE_FILE_SIZE: usize = 512 * 1024;
|
||||
|
||||
/// Certify portable upload and download behavior through the public CLI.
|
||||
pub const FILE_TRANSFER_SCENARIO: Scenario = Scenario {
|
||||
name: "file-transfer",
|
||||
description: "Verify sandbox uploads, downloads, Git filtering, and path safety.",
|
||||
run: run_file_transfer,
|
||||
};
|
||||
|
||||
/// Certify basic file and directory upload and download behavior.
|
||||
pub const FILE_TRANSFER_ROUND_TRIP_SCENARIO: Scenario = Scenario {
|
||||
name: "file-transfer/round-trip",
|
||||
description: "Verify file and directory upload and download round trips.",
|
||||
run: run_round_trip,
|
||||
};
|
||||
|
||||
/// Certify Git-aware upload filtering and explicit unfiltered uploads.
|
||||
pub const FILE_TRANSFER_GIT_FILTERING_SCENARIO: Scenario = Scenario {
|
||||
name: "file-transfer/git-filtering",
|
||||
description: "Verify Git-aware upload selection and explicit unfiltered uploads.",
|
||||
run: run_git_filtering,
|
||||
};
|
||||
|
||||
/// Certify file-transfer workspace boundary and filename safety behavior.
|
||||
pub const FILE_TRANSFER_PATH_SAFETY_SCENARIO: Scenario = Scenario {
|
||||
name: "file-transfer/path-safety",
|
||||
description: "Verify workspace boundary enforcement and safe filename handling.",
|
||||
run: run_path_safety,
|
||||
};
|
||||
|
||||
fn run_file_transfer(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
|
||||
Box::pin(async move {
|
||||
FILE_TRANSFER_ROUND_TRIP_SCENARIO.run(runner).await?;
|
||||
FILE_TRANSFER_GIT_FILTERING_SCENARIO.run(runner).await?;
|
||||
FILE_TRANSFER_PATH_SAFETY_SCENARIO.run(runner).await
|
||||
})
|
||||
}
|
||||
|
||||
fn run_round_trip(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
|
||||
Box::pin(async move {
|
||||
let (sandbox_name, remote_root, local) = prepare_sandbox(runner, "round-trip").await?;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Registered, portable conformance scenarios.
|
||||
//! Portable conformance scenarios exercised by the Cargo test suite.
|
||||
|
||||
mod file_transfer;
|
||||
mod policy_behavior;
|
||||
@@ -10,7 +10,7 @@ mod smoke;
|
||||
|
||||
pub use file_transfer::{
|
||||
FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO,
|
||||
FILE_TRANSFER_ROUND_TRIP_SCENARIO, FILE_TRANSFER_SCENARIO,
|
||||
FILE_TRANSFER_ROUND_TRIP_SCENARIO,
|
||||
};
|
||||
pub use policy_behavior::{
|
||||
MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO,
|
||||
|
||||
@@ -27,19 +27,16 @@ struct SandboxState {
|
||||
|
||||
pub const POLICY_LOCAL_SCENARIO: Scenario = Scenario {
|
||||
name: "policy-local",
|
||||
description: "Read and request a rule through the sandbox-local policy HTTP API.",
|
||||
run: run_policy_local,
|
||||
};
|
||||
|
||||
pub const MECHANISTIC_PROPOSAL_SCENARIO: Scenario = Scenario {
|
||||
name: "mechanistic-proposal",
|
||||
description: "Turn a denied transparent TCP open into a scoped policy draft.",
|
||||
run: run_mechanistic_proposal,
|
||||
};
|
||||
|
||||
pub const NEW_HOSTNAME_PROPOSAL_SCENARIO: Scenario = Scenario {
|
||||
name: "new-hostname-proposal",
|
||||
description: "Turn a denied TCP open to a hostname absent from policy into a scoped draft.",
|
||||
run: run_new_hostname_proposal,
|
||||
};
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ struct SandboxListPage {
|
||||
/// Certify sandbox stop, start, and deletion lifecycle behavior.
|
||||
pub const SANDBOX_LIFECYCLE_SCENARIO: Scenario = Scenario {
|
||||
name: "sandbox-lifecycle",
|
||||
description: "Verify sandbox stop, start, and deletion lifecycle behavior.",
|
||||
run: run_sandbox_lifecycle,
|
||||
};
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ struct SandboxListPage {
|
||||
/// Certify status -> create -> list Ready -> exec -> delete -> list empty.
|
||||
pub const SMOKE_SCENARIO: Scenario = Scenario {
|
||||
name: "smoke",
|
||||
description: "Create, inspect, execute in, and delete a base sandbox.",
|
||||
run: run_smoke,
|
||||
};
|
||||
|
||||
|
||||
@@ -55,20 +55,16 @@ contents write on the repository. The test auto-resolves the token from
|
||||
|
||||
## Conformance coverage
|
||||
|
||||
The `mechanistic-proposal` and `new-hostname-proposal` conformance scenarios
|
||||
check draft generation for a denied IP address and for a hostname absent from
|
||||
policy. The `policy-local` scenario uses `policy.local` to inspect policy, submit
|
||||
a narrow permission request, and read the resulting proposal. Run them against a configured gateway
|
||||
with `--openshell-bin` pointing to the CLI under test:
|
||||
The portable CLI conformance suite may include policy-related scenarios. The
|
||||
current coverage checks draft generation for denied IP addresses and hostnames
|
||||
absent from policy, plus sandbox-local policy inspection and permission
|
||||
requests through `policy.local`. See [TESTING.md](../../TESTING.md#rust-cli-e2e)
|
||||
for the supported source and installed-artifact test paths.
|
||||
|
||||
```bash
|
||||
openshell-conformance run mechanistic-proposal new-hostname-proposal policy-local --openshell-bin target/debug/openshell
|
||||
```
|
||||
|
||||
Run `openshell-conformance list` to see all scenario names. A manual
|
||||
`Integration Tests` workflow run can select the `policy-advisor` testsuite to
|
||||
run only these three scenarios against an installed candidate. Set
|
||||
`artifact-run-id` to the candidate build's workflow run ID and `test-matrix` to:
|
||||
A manual `Integration Tests` workflow run can select the `policy-advisor`
|
||||
testsuite to run the policy-related subset against an installed candidate. Set
|
||||
`artifact-run-id` to the candidate build's workflow run ID and `test-matrix`
|
||||
to:
|
||||
|
||||
```json
|
||||
[{"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"policy-advisor"}]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Run standalone CLI conformance, and optionally a focused Rust e2e test,
|
||||
# Run conformance suites, and optionally a focused Rust e2e test,
|
||||
# against a gateway using the bundled Docker compute driver. Set
|
||||
# OPENSHELL_GATEWAY_ENDPOINT=http://host:port to reuse an existing plaintext
|
||||
# gateway instead of starting an ephemeral one.
|
||||
|
||||
@@ -66,7 +66,7 @@ is_operator_workspace_mode() {
|
||||
|
||||
run_conformance() {
|
||||
if is_operator_workspace_mode; then
|
||||
echo "note: skipping standalone CLI conformance in Kubernetes operator workspace mode; default workspace is not operator-allowlisted (see #2971)."
|
||||
echo "note: skipping conformance suites in Kubernetes operator workspace mode; default workspace is not operator-allowlisted (see #2971)."
|
||||
return 0
|
||||
fi
|
||||
|
||||
|
||||
+31
-15
@@ -2,29 +2,45 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Shared helpers for running standalone conformance suites against an already
|
||||
# Shared helpers for running conformance suites against an already
|
||||
# configured e2e gateway.
|
||||
|
||||
e2e_run_openshell_conformance() {
|
||||
local gateway_label=${1:-OpenShell}
|
||||
|
||||
e2e_require_openshell_bin() {
|
||||
if [ -z "${OPENSHELL_BIN:-}" ]; then
|
||||
echo "ERROR: OPENSHELL_BIN must point to the openshell CLI under test" >&2
|
||||
return 2
|
||||
fi
|
||||
|
||||
if [ -z "${OPENSHELL_CONFORMANCE_BIN:-}" ]; then
|
||||
echo "ERROR: OPENSHELL_CONFORMANCE_BIN must point to the openshell-conformance CLI under test" >&2
|
||||
if [ ! -x "${OPENSHELL_BIN}" ]; then
|
||||
echo "ERROR: openshell CLI is not executable: ${OPENSHELL_BIN}" >&2
|
||||
return 2
|
||||
fi
|
||||
|
||||
if [ ! -x "${OPENSHELL_CONFORMANCE_BIN}" ]; then
|
||||
echo "ERROR: openshell conformance binary is not executable: ${OPENSHELL_CONFORMANCE_BIN}" >&2
|
||||
return 2
|
||||
fi
|
||||
|
||||
echo "==> Running standalone CLI conformance against the ${gateway_label} gateway"
|
||||
"${OPENSHELL_CONFORMANCE_BIN}" run \
|
||||
--openshell-bin "${OPENSHELL_BIN}" \
|
||||
--output json
|
||||
# Cargo runs each test from its package directory, so resolve the candidate
|
||||
# relative to the caller's working directory before passing it to the tests.
|
||||
local binary_dir
|
||||
binary_dir="$(cd "$(dirname "${OPENSHELL_BIN}")" && pwd -P)" || return
|
||||
export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")"
|
||||
}
|
||||
|
||||
e2e_run_openshell_conformance() {
|
||||
local gateway_label=${1:-OpenShell}
|
||||
local root
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
e2e_require_openshell_bin || return
|
||||
|
||||
echo "==> Running conformance suites against the ${gateway_label} gateway"
|
||||
cargo test \
|
||||
--locked \
|
||||
--manifest-path "${root}/tests/suites/conformance/Cargo.toml" \
|
||||
--package openshell-test-conformance-cli \
|
||||
--no-fail-fast \
|
||||
-- \
|
||||
--test-threads=1 \
|
||||
--nocapture
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
e2e_run_openshell_conformance
|
||||
fi
|
||||
|
||||
@@ -175,19 +175,18 @@ prepared guest image to a particular build or driver configuration.
|
||||
Compose it with `gateway-podman` after either Podman configuration. The role
|
||||
uses the recorded mode to select the corresponding service account. It
|
||||
generates configuration only for development artifacts; RPM installations
|
||||
retain their packaged service and first-start configuration. For example, run
|
||||
conformance after the rootless provisioners complete:
|
||||
retain their packaged service and first-start configuration. For example,
|
||||
check the development gateway after the rootless provisioners complete:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- \
|
||||
--distro fedora --with podman-rootless --with selinux \
|
||||
--copy ./openshell:/usr/local/bin/openshell \
|
||||
--copy ./openshell-conformance:/usr/local/bin/openshell-conformance \
|
||||
--copy ./openshell-gateway:/usr/local/bin/openshell-gateway \
|
||||
--copy ./openshell-sandbox.tar:/usr/local/lib/openshell-sandbox.tar \
|
||||
--provision openshell-development \
|
||||
--provision gateway-podman \
|
||||
-- /usr/local/bin/openshell-conformance run smoke
|
||||
-- /usr/local/bin/openshell status
|
||||
```
|
||||
|
||||
`openshell-rpm` expects OpenShell to have been installed with `--install`. It
|
||||
|
||||
+21
-54
@@ -129,26 +129,12 @@ hide = true
|
||||
|
||||
["e2e:rust"]
|
||||
description = "Run Rust CLI e2e tests against a Docker-backed gateway"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh"
|
||||
run = "e2e/rust/e2e-docker.sh"
|
||||
|
||||
["e2e:workload:build"]
|
||||
description = "Build the Docker E2E workload fixture"
|
||||
run = "CONTAINER_ENGINE=docker bash tasks/scripts/e2e-build-workload.sh"
|
||||
|
||||
["e2e:conformance:build"]
|
||||
description = "Build the standalone CLI conformance binary"
|
||||
run = "if [ -z \"${OPENSHELL_CONFORMANCE_BIN:-}\" ]; then cargo build -p openshell-conformance-cli; fi"
|
||||
hide = true
|
||||
|
||||
["e2e:cli-conformance"]
|
||||
description = "Build and run the standalone CLI conformance suite against the configured gateway"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = [
|
||||
"if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli; fi",
|
||||
"\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" run --openshell-bin \"${OPENSHELL_BIN:-$PWD/target/debug/openshell}\"",
|
||||
]
|
||||
|
||||
["e2e:websocket-conformance"]
|
||||
description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway"
|
||||
run = [
|
||||
@@ -175,13 +161,11 @@ run = [
|
||||
|
||||
["e2e:podman"]
|
||||
description = "Run Rust CLI e2e tests against a Podman-backed gateway"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh"
|
||||
run = "e2e/rust/e2e-podman.sh"
|
||||
|
||||
["e2e:podman:ci"]
|
||||
description = "Run the Podman Rust e2e targets enabled in required branch CI"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_E2E_PODMAN_TEST_SET=ci OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh"
|
||||
run = "OPENSHELL_E2E_PODMAN_TEST_SET=ci e2e/rust/e2e-podman.sh"
|
||||
|
||||
["e2e:oidc-pkce"]
|
||||
description = "Run Linux browser PKCE and RBAC e2e tests against Keycloak and a Podman gateway"
|
||||
@@ -206,67 +190,56 @@ run = [
|
||||
["e2e:podman:gpu"]
|
||||
description = "Run GPU e2e against a standalone gateway with the Podman compute driver"
|
||||
env = { OPENSHELL_E2E_PODMAN_GPU = "1", OPENSHELL_E2E_PODMAN_TEST = "gpu", OPENSHELL_E2E_PODMAN_FEATURES = "e2e-podman-gpu" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh"
|
||||
run = "e2e/rust/e2e-podman.sh"
|
||||
|
||||
["e2e:kubernetes"]
|
||||
description = "Run Rust CLI e2e tests against an OpenShell gateway deployed on Kubernetes via Helm (set OPENSHELL_E2E_KUBE_CONTEXT to reuse a cluster; otherwise creates a local k3d cluster when k3d is installed; set OPENSHELL_E2E_KUBE_TEST=<name> to scope to one test)"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:v1alpha1"]
|
||||
description = "Run Kubernetes e2e against Agent Sandbox v1alpha1"
|
||||
env = { AGENT_SANDBOX_VERSION = "v0.4.6" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:agent-sandbox-versions"]
|
||||
description = "Run Kubernetes e2e against Agent Sandbox v1beta1 and v1alpha1"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = [
|
||||
"OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh",
|
||||
"OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh",
|
||||
"e2e/rust/e2e-kubernetes.sh",
|
||||
"AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh",
|
||||
]
|
||||
|
||||
["e2e:kubernetes:isolation"]
|
||||
description = "Run Kubernetes e2e with the workload network fence and separate supervisor"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:db"]
|
||||
description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)"
|
||||
env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:ha-rebalancing"]
|
||||
description = "Run the Kubernetes HA rebalancing suite through Envoy against two gateway replicas and external PostgreSQL"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET = "openshell-ha-pg", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-high-availability.yaml", OPENSHELL_E2E_KUBE_TEST = "kubernetes_ha_rebalancing", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha", OPENSHELL_E2E_KUBE_USE_ENVOY = "1" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:credential-drivers"]
|
||||
description = "Run Kubernetes e2e for provider credential storage backed by Kubernetes Secrets and Vault"
|
||||
env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_KUBE_TEST = "credential_drivers", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-credential-drivers" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:workspace-managed"]
|
||||
description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)"
|
||||
env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_CREDENTIAL_DRIVER = "kubernetes-secrets", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:workspace-operator"]
|
||||
description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:vm"]
|
||||
description = "Start openshell-gateway with the VM compute driver and run VM e2e tests"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh"
|
||||
run = "e2e/rust/e2e-vm.sh"
|
||||
|
||||
["e2e:gateway:no-compute-drivers"]
|
||||
description = "Build and launch-check openshell-gateway without compiled compute drivers"
|
||||
@@ -275,31 +248,26 @@ run = "bash e2e/no-compute-driver-gateway.sh"
|
||||
["e2e:docker:external-driver"]
|
||||
description = "Run Docker conformance with a driver-free gateway and external Docker driver binary"
|
||||
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_DOCKER_FEATURES = "" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh"
|
||||
run = "e2e/rust/e2e-docker.sh"
|
||||
|
||||
["e2e:podman:external-driver"]
|
||||
description = "Run Podman conformance with a driver-free gateway and external Podman driver binary"
|
||||
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_FEATURES = "" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh"
|
||||
run = "e2e/rust/e2e-podman.sh"
|
||||
|
||||
["e2e:vm:external-driver"]
|
||||
description = "Run VM E2E with a driver-free gateway and external VM driver binary"
|
||||
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh"
|
||||
run = "e2e/rust/e2e-vm.sh"
|
||||
|
||||
["e2e:kubernetes:external-driver"]
|
||||
description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver"
|
||||
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
run = "e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:docker"]
|
||||
description = "Run Docker conformance and Rust e2e tests against a standalone gateway"
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh"
|
||||
run = "e2e/rust/e2e-docker.sh"
|
||||
|
||||
["e2e:mechanistic-existing-endpoint"]
|
||||
description = "Run #2821 existing inspected-endpoint auto-approval regression"
|
||||
@@ -311,8 +279,7 @@ run = [
|
||||
["e2e:docker:gpu"]
|
||||
description = "Run GPU e2e against a standalone gateway with the Docker compute driver"
|
||||
env = { OPENSHELL_E2E_DOCKER_GPU = "1", OPENSHELL_E2E_DOCKER_TEST = "gpu", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-docker-gpu" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh"
|
||||
run = "e2e/rust/e2e-docker.sh"
|
||||
|
||||
["test:gateway-config"]
|
||||
description = "Test generated local gateway TOML without starting a runtime"
|
||||
|
||||
+3
-3
@@ -79,7 +79,7 @@ let
|
||||
'';
|
||||
};
|
||||
|
||||
conformanceCliArchive = mkTestArchive {
|
||||
conformanceTestArchive = mkTestArchive {
|
||||
name = "openshell-conformance";
|
||||
workspacePath = "tests/suites/conformance";
|
||||
manifestPath = "tests/suites/conformance/Cargo.toml";
|
||||
@@ -196,7 +196,7 @@ let
|
||||
in
|
||||
rec {
|
||||
inherit
|
||||
conformanceCliArchive
|
||||
conformanceTestArchive
|
||||
providerRefreshKeycloakArchive
|
||||
podmanDriverArchive
|
||||
podmanE2eArchive
|
||||
@@ -244,7 +244,7 @@ rec {
|
||||
testArchives = pkgs.writeShellApplication {
|
||||
name = "build-artifacts-test-archives";
|
||||
runtimeInputs = [
|
||||
conformanceCliArchive
|
||||
conformanceTestArchive
|
||||
providerRefreshKeycloakArchive
|
||||
podmanDriverArchive
|
||||
podmanE2eArchive
|
||||
|
||||
Reference in New Issue
Block a user