mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-02 07:34:45 +08:00
feat(e2e): run the Kubernetes e2e suite on cargo-nextest with machine- and human-readable reports (#3344)
* feat(e2e): run kubernetes suite on cargo-nextest with JUnit/HTML reports
Switch e2e:kubernetes (and all its variants) from `cargo test` to
`cargo nextest run` for per-test process isolation and output consistent
with the other nextest-based CI runs.
- Add a dedicated `e2e-kubernetes` nextest profile with a JUnit report and a
generous slow-timeout (60s flag, 5-min terminate) suited to live-cluster
tests; kept separate from `ci` so its JUnit path and timeouts don't affect
the workspace run.
- Pin `--target-dir` for the run so the profile's relative JUnit path resolves
to the repo-root results/ regardless of any inherited CARGO_TARGET_DIR
(nextest ignores absolute JUnit paths).
- Render the JUnit XML to a standalone HTML report via xsltproc and a committed
XSLT stylesheet (best-effort; never masks the test exit code).
- Name each report via `OPENSHELL_E2E_REPORT_NAME` (default `e2e-kubernetes`),
used verbatim for both the `results/<name>.{xml,html}` filenames and the HTML
heading. Tasks that invoke the script multiple times in one run set a distinct
name per invocation so the reports no longer clobber the single fixed path:
the credential-driver runs write results/e2e-kubernetes-secrets.xml and
-vault.xml, and e2e:kubernetes:agent-sandbox-versions writes
results/e2e-kubernetes-agent-sandbox-v1beta1.xml and -v1alpha1.xml.
- Declare cargo-nextest in mise [tools] so the task runs without the Nix shell.
- Ignore the results/ output directory.
The results/ reports do not leak information. They are gitignored and no
workflow uploads them as artifacts, so they stay on the ephemeral CI runner
and are discarded when it is torn down. The HTML template renders only test
names, status, timings, and failure messages (no captured stdout/stderr).
Moving from `cargo test -- --nocapture` to nextest's captured, failure-only
output also reduces what lands in the retained, viewable console logs.
Signed-off-by: Jorge Garcia Oncins <jgarciao@redhat.com>
* chore(e2e): revert per-lane report names for agent-sandbox-versions
The agent-sandbox-versions task runs two lanes sequentially against the
same cluster: v0.5.0 (v1beta1 storage version) then v0.4.6 (v1alpha1).
On a reused cluster the second lane fails when kubectl applies the older
CRD, because Kubernetes refuses to drop v1beta1 from spec.versions while
it remains in status.storedVersions (the storage-version downgrade
guardrail). This is a pre-existing issue with the v0.4.6 lane, unrelated
to the nextest reporting work.
The per-lane OPENSHELL_E2E_REPORT_NAME additions do not address that
downgrade failure, so revert them to keep this PR scoped to the nextest
change. Agent Sandbox 0.4.x is also superseded (1.0.0 is published);
dropping or bumping the v1alpha1 lane is left as a follow-up.
Signed-off-by: Jorge Garcia Oncins <jgarciao@redhat.com>
---------
Signed-off-by: Jorge Garcia Oncins <jgarciao@redhat.com>
This commit is contained in:
Jorge
@@ -8,3 +8,15 @@ final-status-level = "fail"
|
||||
failure-output = "immediate-final"
|
||||
# Print slow test names after 30 seconds and terminate them after two minutes.
|
||||
slow-timeout = { period = "30s", terminate-after = 4 }
|
||||
|
||||
[profile.e2e-kubernetes]
|
||||
fail-fast = false
|
||||
status-level = "slow"
|
||||
final-status-level = "fail"
|
||||
failure-output = "immediate-final"
|
||||
# Flag slow tests after 60s and terminate them after five minutes.
|
||||
slow-timeout = { period = "60s", terminate-after = 5 }
|
||||
|
||||
# Relative to the profile store dir (`e2e/rust/target/nextest/e2e-kubernetes/`).
|
||||
[profile.e2e-kubernetes.junit]
|
||||
path = "../../../../../results/e2e-kubernetes.xml"
|
||||
|
||||
@@ -6,6 +6,9 @@
|
||||
/target/
|
||||
e2e/rust/target/
|
||||
target/
|
||||
|
||||
# Test result artifacts (JUnit XML, etc.)
|
||||
/results/
|
||||
debug/
|
||||
release/
|
||||
|
||||
|
||||
+46
-11
@@ -14,6 +14,9 @@
|
||||
# via `server.hostGatewayIP`. Targeting a cluster where the test host is
|
||||
# unreachable from pods? Set OPENSHELL_E2E_KUBERNETES_FEATURES=e2e to drop the
|
||||
# alias-dependent tests entirely.
|
||||
#
|
||||
# Results: `run_suite` writes a JUnit + HTML report under `results/`. Set
|
||||
# `OPENSHELL_E2E_REPORT_NAME` to name it per run when invoking this script repeatedly.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -24,6 +27,25 @@ source "${ROOT}/e2e/support/conformance.sh"
|
||||
|
||||
E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES-e2e,e2e-host-gateway,e2e-kubernetes}"
|
||||
|
||||
# Fixed output path of the `e2e-kubernetes` nextest profile (`.config/nextest.toml`).
|
||||
JUNIT_XML="${ROOT}/results/e2e-kubernetes.xml"
|
||||
|
||||
# Render a sibling HTML report from a JUnit XML. Best-effort: failures only warn.
|
||||
render_html() {
|
||||
local xml="${1:-${JUNIT_XML}}"
|
||||
local title="${2:-e2e-kubernetes}"
|
||||
[ -f "${xml}" ] || return 0
|
||||
local html="${xml%.xml}.html"
|
||||
if command -v xsltproc >/dev/null 2>&1; then
|
||||
xsltproc --stringparam title "${title}" \
|
||||
"${ROOT}/scripts/junit-to-html.xsl" "${xml}" >"${html}" \
|
||||
&& echo "HTML report: ${html}" \
|
||||
|| echo "WARNING: failed to render HTML report from ${xml}" >&2
|
||||
else
|
||||
echo "WARNING: xsltproc not found; skipping HTML report (${xml} still written)" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
# Docker and Podman build their local gateway and CLI together in the shared
|
||||
# gateway wrapper. Kubernetes consumes published gateway images, so only its
|
||||
# local CLI needs to be built when CI has not supplied a prebuilt one.
|
||||
@@ -50,9 +72,19 @@ run_conformance() {
|
||||
e2e_run_openshell_conformance "Kubernetes"
|
||||
}
|
||||
|
||||
# `OPENSHELL_E2E_REPORT_NAME` (default `e2e-kubernetes`) names the report
|
||||
# `results/<name>.{xml,html}` and its heading, so repeated runs do not clobber.
|
||||
run_suite() {
|
||||
local name="${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}"
|
||||
local report="${ROOT}/results/${name}.xml"
|
||||
local status=0
|
||||
"${ROOT}/e2e/with-kube-gateway.sh" \
|
||||
bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}"
|
||||
bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" || status=$?
|
||||
if [ "${report}" != "${JUNIT_XML}" ]; then
|
||||
mv -f "${JUNIT_XML}" "${report}" 2>/dev/null || true
|
||||
fi
|
||||
render_html "${report}" "${name}"
|
||||
return "${status}"
|
||||
}
|
||||
|
||||
run_e2e() {
|
||||
@@ -61,11 +93,13 @@ run_e2e() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \
|
||||
# Pin `--target-dir` so the profile `junit.path` resolves regardless of `CARGO_TARGET_DIR`.
|
||||
cargo nextest run --profile e2e-kubernetes \
|
||||
--config-file "${ROOT}/.config/nextest.toml" \
|
||||
--target-dir "${ROOT}/e2e/rust/target" \
|
||||
--manifest-path "${ROOT}/e2e/rust/Cargo.toml" \
|
||||
--features "${E2E_FEATURES}" \
|
||||
--no-fail-fast \
|
||||
${test_filter[@]+"${test_filter[@]}"} \
|
||||
-- --nocapture
|
||||
${test_filter[@]+"${test_filter[@]}"}
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then
|
||||
@@ -73,12 +107,13 @@ if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Credential-driver mode: run once per storage backend, each with its own report.
|
||||
if [ "${OPENSHELL_E2E_CREDENTIAL_DRIVERS:-0}" = "1" ] \
|
||||
&& [ -z "${OPENSHELL_E2E_CREDENTIAL_DRIVER:-}" ]; then
|
||||
OPENSHELL_E2E_CREDENTIAL_DRIVER=kubernetes-secrets run_suite
|
||||
OPENSHELL_E2E_CREDENTIAL_DRIVER=vault run_suite
|
||||
exit 0
|
||||
OPENSHELL_E2E_CREDENTIAL_DRIVER=kubernetes-secrets \
|
||||
OPENSHELL_E2E_REPORT_NAME=e2e-kubernetes-secrets run_suite
|
||||
OPENSHELL_E2E_CREDENTIAL_DRIVER=vault \
|
||||
OPENSHELL_E2E_REPORT_NAME=e2e-kubernetes-vault run_suite
|
||||
else
|
||||
run_suite
|
||||
fi
|
||||
|
||||
exec "${ROOT}/e2e/with-kube-gateway.sh" \
|
||||
bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}"
|
||||
|
||||
@@ -136,11 +136,6 @@ checksum = "sha256:f73a5c39f96bb6ebb89cc7915cf182260d4cbf30765322c5e793d0fe8bd80
|
||||
url = "https://github.com/mozilla/sccache/releases/download/v0.16.0/sccache-v0.16.0-aarch64-unknown-linux-musl.tar.gz"
|
||||
url_api = "https://api.github.com/repos/mozilla/sccache/releases/assets/452060468"
|
||||
|
||||
[tools."github:mozilla/sccache"."platforms.linux-x64"]
|
||||
checksum = "sha256:aec995a83ad3dff3d14b6314e08858b7b73d35ca85a5bcf3d3a9ec07dee35588"
|
||||
url = "https://github.com/mozilla/sccache/releases/download/v0.16.0/sccache-v0.16.0-x86_64-unknown-linux-musl.tar.gz"
|
||||
url_api = "https://api.github.com/repos/mozilla/sccache/releases/assets/452060682"
|
||||
|
||||
[tools."github:mozilla/sccache"."platforms.macos-arm64"]
|
||||
checksum = "sha256:ded590cae2c72042c61178632906bef62d635fa20d45f8b22110a2241f430960"
|
||||
url = "https://github.com/mozilla/sccache/releases/download/v0.16.0/sccache-v0.16.0-aarch64-apple-darwin.tar.gz"
|
||||
@@ -169,9 +164,12 @@ url = "https://github.com/mozilla/sccache/releases/download/v0.16.0/sccache-v0.1
|
||||
url_api = "https://api.github.com/repos/mozilla/sccache/releases/assets/452060682"
|
||||
|
||||
[[tools."github:nextest-rs/nextest"]]
|
||||
version = "cargo-nextest-0.9.143"
|
||||
version = "0.9.143"
|
||||
backend = "github:nextest-rs/nextest"
|
||||
|
||||
[tools."github:nextest-rs/nextest".options]
|
||||
version_prefix = "cargo-nextest-"
|
||||
|
||||
[tools."github:nextest-rs/nextest"."platforms.linux-arm64"]
|
||||
checksum = "sha256:2a64b3566a92508550a7ab29c3e8db25472ca37730ecb4d22100b6aa440c2a68"
|
||||
url = "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-0.9.143/cargo-nextest-0.9.143-aarch64-unknown-linux-gnu.tar.gz"
|
||||
@@ -183,6 +181,7 @@ checksum = "sha256:66786b9abe23920d022a182d1416b1bbc8130dd4872a9553d76985a1708dc
|
||||
url = "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-0.9.143/cargo-nextest-0.9.143-x86_64-unknown-linux-gnu.tar.gz"
|
||||
url_api = "https://api.github.com/repos/nextest-rs/nextest/releases/assets/501882467"
|
||||
provenance = "github-attestations"
|
||||
provenance_verified = true
|
||||
|
||||
[tools."github:nextest-rs/nextest"."platforms.macos-arm64"]
|
||||
checksum = "sha256:4830d430411148d17602a75cc880bfb4dc8dac153dea59a48a2ef4cc93577f07"
|
||||
|
||||
@@ -44,8 +44,8 @@ k3d = { version = "5.8.3", os = ["macos"] }
|
||||
"github:EmbarkStudios/cargo-deny" = { version = "0.20.2", version_prefix = "" }
|
||||
zig = "0.14.1"
|
||||
"github:rust-secure-code/cargo-auditable" = "0.7.5"
|
||||
"github:nextest-rs/nextest" = { version = "0.9.143", version_prefix = "cargo-nextest-" }
|
||||
"github:rust-cross/cargo-zigbuild" = "0.22.3"
|
||||
"github:nextest-rs/nextest" = { version = "cargo-nextest-0.9.143", os = ["windows"] }
|
||||
"npm:markdownlint-cli2" = "0.22.0"
|
||||
|
||||
[tools."github:mozilla/sccache"]
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
|
||||
<!-- SPDX-License-Identifier: Apache-2.0 -->
|
||||
<!--
|
||||
Convert a nextest JUnit report into a standalone HTML report (XSLT 1.0).
|
||||
|
||||
Usage:
|
||||
xsltproc junit-to-html.xsl report.xml > report.html
|
||||
xsltproc -stringparam title "my suite report" junit-to-html.xsl report.xml > report.html
|
||||
|
||||
Renders a summary plus a per-suite table with pass/fail/skip rows. The report
|
||||
heading is the `title` parameter (default "Test report").
|
||||
-->
|
||||
<xsl:stylesheet version="1.0"
|
||||
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
|
||||
<xsl:output method="html" indent="yes" encoding="UTF-8"
|
||||
doctype-system="about:legacy-compat"/>
|
||||
|
||||
<!-- Report heading; override via the xsltproc `title` string parameter. -->
|
||||
<xsl:param name="title" select="'Test report'"/>
|
||||
|
||||
<xsl:template match="/testsuites">
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8"/>
|
||||
<title><xsl:value-of select="$title"/></title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #f6f7f9; --card: #fff; --ink: #1f2328; --muted: #656d76;
|
||||
--border: #d7dbe0; --pass: #1a7f37; --fail: #cf222e; --skip: #9a6700;
|
||||
--pass-bg: #eafbe7; --fail-bg: #fbecec; --skip-bg: #fdf6e3;
|
||||
--mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
font-family: system-ui, -apple-system, "Segoe UI", sans-serif;
|
||||
margin: 0; padding: 2rem clamp(1rem, 4vw, 3rem); color: var(--ink);
|
||||
background: var(--bg); line-height: 1.45;
|
||||
}
|
||||
h1 { font-size: 1.5rem; margin: 0 0 1rem; }
|
||||
h2 {
|
||||
font-family: var(--mono); font-size: 0.95rem; font-weight: 600;
|
||||
margin: 2rem 0 0; color: var(--ink);
|
||||
}
|
||||
h2 .count { color: var(--muted); font-weight: 400; }
|
||||
|
||||
.summary { display: flex; flex-wrap: wrap; gap: 0.75rem; margin: 0 0 0.5rem; }
|
||||
.summary .card {
|
||||
background: var(--card); border: 1px solid var(--border); border-radius: 8px;
|
||||
padding: 0.5rem 0.9rem; min-width: 5.5rem;
|
||||
}
|
||||
.summary .card .label {
|
||||
display: block; font-size: 0.7rem; text-transform: uppercase;
|
||||
letter-spacing: 0.04em; color: var(--muted);
|
||||
}
|
||||
.summary .card .value { font-size: 1.35rem; font-weight: 700; }
|
||||
.card.c-fail .value { color: var(--fail); }
|
||||
.card.c-skip .value { color: var(--skip); }
|
||||
.card.c-fail.zero .value, .card.c-skip.zero .value { color: var(--muted); }
|
||||
|
||||
table {
|
||||
border-collapse: collapse; width: 100%; margin-top: 0.6rem; table-layout: fixed;
|
||||
background: var(--card); border: 1px solid var(--border);
|
||||
border-radius: 8px; overflow: hidden;
|
||||
}
|
||||
th, td {
|
||||
text-align: left; padding: 0.5rem 0.75rem;
|
||||
border-bottom: 1px solid var(--border); vertical-align: top;
|
||||
}
|
||||
tbody tr:last-child td { border-bottom: none; }
|
||||
td:first-child { font-family: var(--mono); font-size: 0.85rem; overflow-wrap: anywhere; }
|
||||
th {
|
||||
position: sticky; top: 0; background: #eef1f4; z-index: 1;
|
||||
font-size: 0.72rem; text-transform: uppercase; letter-spacing: 0.04em;
|
||||
color: var(--muted);
|
||||
}
|
||||
tbody tr:nth-child(even) { background: #fafbfc; }
|
||||
tbody tr:hover { background: #eef4ff; }
|
||||
tr.failed, tr.failed:nth-child(even) { background: var(--fail-bg); }
|
||||
tr.skipped, tr.skipped:nth-child(even) { background: var(--skip-bg); }
|
||||
td.time { font-family: var(--mono); font-size: 0.85rem; color: var(--muted); }
|
||||
|
||||
.badge {
|
||||
display: inline-block; padding: 0.1rem 0.5rem; border-radius: 999px;
|
||||
font-size: 0.72rem; font-weight: 700; letter-spacing: 0.03em;
|
||||
}
|
||||
.status-pass { background: var(--pass-bg); color: var(--pass); }
|
||||
.status-fail { background: var(--fail-bg); color: var(--fail); }
|
||||
.status-skip { background: var(--skip-bg); color: var(--skip); }
|
||||
pre {
|
||||
margin: 0.4rem 0 0; padding: 0.5rem 0.6rem; white-space: pre-wrap;
|
||||
color: var(--fail); background: #fff; border: 1px solid var(--border);
|
||||
border-radius: 6px; font-size: 0.8rem;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1><xsl:value-of select="$title"/></h1>
|
||||
<div class="summary">
|
||||
<div class="card">
|
||||
<span class="label">Total</span>
|
||||
<span class="value"><xsl:value-of select="@tests"/></span>
|
||||
</div>
|
||||
<div class="card c-fail">
|
||||
<xsl:if test="@failures = 0"><xsl:attribute name="class">card c-fail zero</xsl:attribute></xsl:if>
|
||||
<span class="label">Failures</span>
|
||||
<span class="value"><xsl:value-of select="@failures"/></span>
|
||||
</div>
|
||||
<div class="card c-fail">
|
||||
<xsl:if test="@errors = 0"><xsl:attribute name="class">card c-fail zero</xsl:attribute></xsl:if>
|
||||
<span class="label">Errors</span>
|
||||
<span class="value"><xsl:value-of select="@errors"/></span>
|
||||
</div>
|
||||
<div class="card c-skip">
|
||||
<xsl:if test="sum(testsuite/@skipped) = 0"><xsl:attribute name="class">card c-skip zero</xsl:attribute></xsl:if>
|
||||
<span class="label">Skipped</span>
|
||||
<span class="value"><xsl:value-of select="sum(testsuite/@skipped)"/></span>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="label">Time</span>
|
||||
<span class="value"><xsl:value-of select="@time"/>s</span>
|
||||
</div>
|
||||
</div>
|
||||
<xsl:for-each select="testsuite">
|
||||
<h2><xsl:value-of select="@name"/>
|
||||
<xsl:text> </xsl:text>
|
||||
<span class="count">(<xsl:value-of select="count(testcase)"/> tests, <xsl:value-of select="@time"/>s)</span>
|
||||
</h2>
|
||||
<table>
|
||||
<colgroup>
|
||||
<col/>
|
||||
<col style="width: 6rem;"/>
|
||||
<col style="width: 6rem;"/>
|
||||
</colgroup>
|
||||
<thead>
|
||||
<tr><th>Test</th><th>Status</th><th>Time</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<xsl:for-each select="testcase">
|
||||
<xsl:choose>
|
||||
<xsl:when test="failure or error">
|
||||
<tr class="failed">
|
||||
<td><xsl:value-of select="@name"/>
|
||||
<pre><xsl:value-of select="failure | error"/></pre>
|
||||
</td>
|
||||
<td><span class="badge status-fail">FAIL</span></td>
|
||||
<td class="time"><xsl:value-of select="@time"/>s</td>
|
||||
</tr>
|
||||
</xsl:when>
|
||||
<xsl:when test="skipped">
|
||||
<tr class="skipped">
|
||||
<td><xsl:value-of select="@name"/></td>
|
||||
<td><span class="badge status-skip">SKIP</span></td>
|
||||
<td class="time"><xsl:value-of select="@time"/>s</td>
|
||||
</tr>
|
||||
</xsl:when>
|
||||
<xsl:otherwise>
|
||||
<tr>
|
||||
<td><xsl:value-of select="@name"/></td>
|
||||
<td><span class="badge status-pass">PASS</span></td>
|
||||
<td class="time"><xsl:value-of select="@time"/>s</td>
|
||||
</tr>
|
||||
</xsl:otherwise>
|
||||
</xsl:choose>
|
||||
</xsl:for-each>
|
||||
</tbody>
|
||||
</table>
|
||||
</xsl:for-each>
|
||||
</body>
|
||||
</html>
|
||||
</xsl:template>
|
||||
</xsl:stylesheet>
|
||||
Reference in New Issue
Block a user