mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-21 06:11:31 +08:00
feat(test-guest): build QEMU images with Bazel
Signed-off-by: Drew Newberry <anewberry@nvidia.com>
This commit is contained in:
Drew Newberry
@@ -88,6 +88,22 @@ These pipelines connect skills into end-to-end workflows. Individual skill files
|
||||
- Store plan documents in `architecture/plans`. This is git ignored so its for easier access for humans. When asked to create Spikes or issues, you can skip to GitHub issues. Only use the plans dir when you aren't writing data somewhere else specific.
|
||||
- When asked to write a plan, write it there without asking for the location.
|
||||
|
||||
## Bazel
|
||||
|
||||
- Use the Bazel version pinned in `.bazelversion` and check its current
|
||||
documentation before introducing a new build-system pattern.
|
||||
- Use Bzlmod (`MODULE.bazel`, module extensions, and `use_repo_rule`) instead of
|
||||
adding `WORKSPACE`-era dependency wiring. Keep the root module concise by
|
||||
moving subsystem-specific setup into `<name>.MODULE.bazel` files included
|
||||
with `include()`.
|
||||
- Model tools and configuration with declared attributes, providers,
|
||||
toolchains, and platform constraints. Actions must declare their inputs,
|
||||
outputs, environment, and execution requirements; do not depend on ambient
|
||||
host tools or environment state.
|
||||
- Prefer stable, non-deprecated Bazel and Starlark APIs, preserve local and
|
||||
remote cacheability unless the operation inherently forbids it, and run
|
||||
Buildifier plus a focused `bazel build` or `bazel test` for changed targets.
|
||||
|
||||
## Sandbox Logging (OCSF)
|
||||
|
||||
When adding or modifying log emissions in `openshell-sandbox`, determine whether the event should use OCSF structured logging or plain `tracing`.
|
||||
|
||||
+4
-1
@@ -1 +1,4 @@
|
||||
exports_files(["deploy/rpm/gateway.toml.default"])
|
||||
exports_files([
|
||||
"deploy/rpm/gateway.toml.default",
|
||||
"flake.lock",
|
||||
])
|
||||
|
||||
@@ -59,6 +59,7 @@ exports_files([
|
||||
include("//bazel/annotations:aws-lc-sys.MODULE.bazel")
|
||||
include("//bazel/annotations:z3-sys.MODULE.bazel")
|
||||
include("//bazel/annotations:zstd-sys.MODULE.bazel")
|
||||
include("//nix/test-guest:test-guest.MODULE.bazel")
|
||||
|
||||
z3_repository = use_repo_rule("//third_party/z3:repositories.bzl", "z3_repository")
|
||||
|
||||
|
||||
Generated
+101
-38
File diff suppressed because one or more lines are too long
+13
-41
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Build the current checkout, run its gateway on the host or in a disposable
|
||||
# Nix test guest, and execute one named host-side E2E suite against that gateway.
|
||||
# Bazel test guest, and execute one named host-side E2E suite against that gateway.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
@@ -23,8 +23,8 @@ Usage:
|
||||
--gateway-config PATH --suite NAME
|
||||
|
||||
Options:
|
||||
--vm DISTRO Run the gateway in a Nix test guest
|
||||
--with CONFIG Apply a Nix test-guest configuration; repeatable
|
||||
--vm DISTRO Run the gateway in a Bazel test guest
|
||||
--with CONFIG Select a Bazel test-guest configuration; repeatable
|
||||
--gateway-config PATH
|
||||
Fully resolved gateway TOML
|
||||
--suite NAME Rust suite at e2e/rust/tests/NAME.rs
|
||||
@@ -68,29 +68,8 @@ print(os.path.realpath(sys.argv[1]))
|
||||
PY
|
||||
}
|
||||
|
||||
catalog_has_entry() {
|
||||
local catalog=$1
|
||||
local section=$2
|
||||
local name=$3
|
||||
|
||||
printf '%s\n' "${catalog}" | awk -v wanted_section="${section}:" -v wanted_name="${name}" '
|
||||
$0 == wanted_section {
|
||||
in_section = 1
|
||||
next
|
||||
}
|
||||
/^[^[:space:]]/ {
|
||||
in_section = 0
|
||||
}
|
||||
in_section && $0 == " " wanted_name {
|
||||
found = 1
|
||||
}
|
||||
END {
|
||||
exit(found ? 0 : 1)
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
vm=
|
||||
vm_bazel_target=
|
||||
gateway_config=
|
||||
suite_name=
|
||||
with_configurations=()
|
||||
@@ -174,23 +153,20 @@ if [ "${mode}" = vm ]; then
|
||||
if [ "${gateway_driver}" = podman ] && [ "${vm}" = ubuntu ]; then
|
||||
die "the Ubuntu 24.04 guest lacks the Podman 5 pasta helper required for sandbox callbacks; use --vm fedora --with podman"
|
||||
fi
|
||||
if ! command -v nix >/dev/null 2>&1; then
|
||||
die "Nix is required for VM mode"
|
||||
if ! command -v bazel >/dev/null 2>&1; then
|
||||
die "Bazel is required for VM mode"
|
||||
fi
|
||||
if ! command -v base64 >/dev/null 2>&1; then
|
||||
die "base64 is required for VM mode"
|
||||
fi
|
||||
if ! vm_catalog="$(cd "${ROOT}" && nix run .#test-guest -- --list)"; then
|
||||
die "failed to read the Nix test-guest catalog"
|
||||
fi
|
||||
if ! catalog_has_entry "${vm_catalog}" Distros "${vm}"; then
|
||||
die "unknown VM distro in the Nix test-guest catalog: ${vm}"
|
||||
fi
|
||||
vm_target_name=${vm}
|
||||
for configuration in "${with_configurations[@]}"; do
|
||||
if ! catalog_has_entry "${vm_catalog}" Configurations "${configuration}"; then
|
||||
die "unknown VM configuration in the Nix test-guest catalog: ${configuration}"
|
||||
fi
|
||||
vm_target_name+=_${configuration}
|
||||
done
|
||||
vm_bazel_target="//nix/test-guest:${vm_target_name}"
|
||||
if ! (cd "${ROOT}" && bazel query "${vm_bazel_target}" >/dev/null); then
|
||||
die "unsupported Bazel test-guest combination: ${vm_target_name}"
|
||||
fi
|
||||
fi
|
||||
|
||||
gateway_ready_timeout=${OPENSHELL_E2E_GATEWAY_READY_TIMEOUT:-600}
|
||||
@@ -484,12 +460,8 @@ EOF
|
||||
chmod 0700 "${guest_launcher}"
|
||||
|
||||
vm_args=(
|
||||
nix run .#test-guest --
|
||||
--distro "${vm}"
|
||||
bazel run "${vm_bazel_target}" --
|
||||
)
|
||||
for configuration in "${with_configurations[@]}"; do
|
||||
vm_args+=(--with "${configuration}")
|
||||
done
|
||||
vm_args+=(
|
||||
--copy "${guest_gateway_bin}:/usr/local/bin/openshell-gateway"
|
||||
--copy "${guest_launcher}:${guest_launcher_path}"
|
||||
|
||||
@@ -38,12 +38,8 @@
|
||||
programs.nixfmt.enable = true;
|
||||
};
|
||||
rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
|
||||
testGuest = import ./nix/test-guest { inherit pkgs; };
|
||||
in
|
||||
{
|
||||
apps.test-guest = testGuest.app;
|
||||
apps.test-guest-cache = testGuest.cacheApp;
|
||||
|
||||
devShells.default = pkgs.mkShell {
|
||||
packages = with pkgs; [
|
||||
rustToolchain
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
load(":catalog.bzl", "declare_test_guest_targets")
|
||||
load(
|
||||
":test_guest.bzl",
|
||||
"test_guest_base_image",
|
||||
"test_guest_host_tools",
|
||||
"test_guest_test",
|
||||
"test_guest_variant",
|
||||
)
|
||||
|
||||
exports_files([
|
||||
"build-image.sh",
|
||||
"catalog.bzl",
|
||||
"configuration/docker.yml",
|
||||
"configuration/podman.yml",
|
||||
"configuration/selinux.yml",
|
||||
"extensions.bzl",
|
||||
"host-tools.BUILD.bazel",
|
||||
"host-tools.nix",
|
||||
"image-seal.sh",
|
||||
"nixpkgs.nix",
|
||||
"run.sh",
|
||||
"test-guest.MODULE.bazel",
|
||||
])
|
||||
|
||||
toolchain_type(name = "host_tools_toolchain_type")
|
||||
|
||||
test_guest_host_tools(
|
||||
name = "nix_host_tools",
|
||||
all_files = "@test_guest_host_tools//:all",
|
||||
ansible_version = "@test_guest_host_tools//:ansible_version",
|
||||
bash = "@test_guest_host_tools//:bash",
|
||||
jq = "@test_guest_host_tools//:jq",
|
||||
qemu_img = "@test_guest_host_tools//:qemu_img",
|
||||
qemu_version = "@test_guest_host_tools//:qemu_version",
|
||||
runner = "@test_guest_host_tools//:runner",
|
||||
sha256sum = "@test_guest_host_tools//:sha256sum",
|
||||
)
|
||||
|
||||
toolchain(
|
||||
name = "nix_host_tools_toolchain",
|
||||
toolchain = ":nix_host_tools",
|
||||
toolchain_type = ":host_tools_toolchain_type",
|
||||
use_target_platform_constraints = True,
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "aarch64",
|
||||
constraint_values = ["@platforms//cpu:aarch64"],
|
||||
)
|
||||
|
||||
config_setting(
|
||||
name = "x86_64",
|
||||
constraint_values = ["@platforms//cpu:x86_64"],
|
||||
)
|
||||
|
||||
declare_test_guest_targets(
|
||||
base_image_rule = test_guest_base_image,
|
||||
variant_macro = test_guest_variant,
|
||||
)
|
||||
|
||||
test_guest_test(
|
||||
name = "fedora_podman_smoke",
|
||||
size = "enormous",
|
||||
command = ["true"],
|
||||
image = ":fedora_podman_image",
|
||||
tags = ["manual"],
|
||||
)
|
||||
+169
-239
@@ -5,262 +5,192 @@ SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Test Guests
|
||||
|
||||
This prototype uses Nix, QEMU, and Ansible to boot and configure disposable Linux VMs for testing OpenShell packages and binaries. It supports HVF on Apple Silicon macOS, KVM on native-architecture Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable.
|
||||
Bazel builds and runs disposable Linux QEMU guests for testing OpenShell
|
||||
packages and binaries. Nix supplies a pinned host-tool closure containing QEMU,
|
||||
OVMF, Ansible, SSH, and supporting utilities. Bazel owns the cloud-image inputs,
|
||||
prepared QCOW2 outputs, cache keys, run targets, and test targets.
|
||||
|
||||
The harness supports HVF on Apple Silicon macOS, KVM on native-architecture
|
||||
Linux hosts, and a slower TCG fallback on Linux when KVM is unavailable.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Nix with flakes enabled.
|
||||
- Apple Silicon macOS with HVF, or a native-architecture Linux host. Linux uses KVM when `/dev/kvm` is available and falls back to QEMU TCG otherwise.
|
||||
- Enough local capacity for a four-vCPU, 4 GiB guest and a disposable disk overlay.
|
||||
- Native-architecture artifacts. TCG emulates the guest CPU on Linux but does not enable cross-architecture guests.
|
||||
- Bazel at the version pinned in `.bazelversion`.
|
||||
- Nix with flakes enabled so `rules_nixpkgs` can realize the host tools.
|
||||
- Apple Silicon macOS with HVF, or a native-architecture Linux host. Linux uses
|
||||
KVM when `/dev/kvm` is available and otherwise falls back to QEMU TCG.
|
||||
- Capacity for a four-vCPU, 4 GiB guest and a disposable disk overlay.
|
||||
- Native-architecture artifacts. TCG does not enable cross-architecture guests.
|
||||
|
||||
The first run downloads the selected cloud image and VM runtime. Nix reuses those immutable inputs on later runs, while each guest starts from a fresh writable overlay.
|
||||
## Build an image
|
||||
|
||||
Build a prepared Fedora image with Podman:
|
||||
|
||||
```shell
|
||||
bazel build //nix/test-guest:fedora_podman_image
|
||||
```
|
||||
|
||||
The target produces cacheable Bazel outputs:
|
||||
|
||||
```text
|
||||
bazel-bin/nix/test-guest/fedora_podman_image.qcow2
|
||||
bazel-bin/nix/test-guest/fedora_podman_image.metadata.json
|
||||
```
|
||||
|
||||
The image action boots the pinned cloud image, applies its declared Ansible
|
||||
playbooks, removes instance-specific state, flattens the disk to a standalone
|
||||
compressed QCOW2, and validates a fresh boot from that output.
|
||||
|
||||
## Run a guest
|
||||
|
||||
Run an interactive guest backed by the prepared image:
|
||||
|
||||
```shell
|
||||
bazel run //nix/test-guest:fedora_podman
|
||||
```
|
||||
|
||||
Arguments after the Bazel separator are passed to the guest runner:
|
||||
|
||||
```shell
|
||||
bazel run //nix/test-guest:ubuntu_docker -- \
|
||||
--copy ./openshell:/usr/local/bin/openshell \
|
||||
-- openshell --version
|
||||
```
|
||||
|
||||
Without a command, the target opens an interactive SSH session. Every run
|
||||
creates a fresh writable overlay; the prepared Bazel output remains unchanged.
|
||||
|
||||
## Run a Bazel test
|
||||
|
||||
The representative smoke test consumes the same prepared image provider:
|
||||
|
||||
```shell
|
||||
bazel test //nix/test-guest:fedora_podman_smoke
|
||||
```
|
||||
|
||||
Image and VM test actions execute locally because they require networking and
|
||||
a host accelerator. They remain eligible for Bazel local and remote caching.
|
||||
The generated launchers use `rules_shell` and Bazel's standard runfiles
|
||||
library.
|
||||
|
||||
## Supported variants
|
||||
|
||||
| Distro | Base | Docker | Podman | SELinux combinations |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Ubuntu 24.04 | `ubuntu` | `ubuntu_docker` | `ubuntu_podman` | None |
|
||||
| CentOS Stream 10 | `centos` | None | `centos_podman` | `centos_selinux`, `centos_podman_selinux` |
|
||||
| Fedora 44 | `fedora` | None | `fedora_podman` | `fedora_selinux`, `fedora_podman_selinux` |
|
||||
| Rocky Linux 9 | `rocky` | `rocky_docker` | `rocky_podman` | `rocky_selinux`, `rocky_docker_selinux`, `rocky_podman_selinux` |
|
||||
|
||||
Append `_image` to a variant to build its QCOW2 directly. Query the package to
|
||||
see every generated target:
|
||||
|
||||
```shell
|
||||
bazel query //nix/test-guest:all
|
||||
```
|
||||
|
||||
Ubuntu 24.04 provides Podman 4 without the `pasta` helper required by OpenShell
|
||||
sandbox callbacks. Podman E2E runs use Fedora, which provides Podman 5 and
|
||||
`pasta`.
|
||||
|
||||
## Cache and reproducibility boundary
|
||||
|
||||
Bazel's action cache is the only prepared-image cache. The harness does not
|
||||
maintain a second local cache or publish OCI image-cache artifacts.
|
||||
|
||||
Base cloud images, host tools, playbooks, and sealing logic are declared inputs.
|
||||
Image metadata contains no wall-clock build time. Guest package repositories
|
||||
are still live, however, so an uncached rebuild is not yet bit-reproducible.
|
||||
Only the trusted image-warming workflow should have remote-cache write
|
||||
credentials; ordinary CI and developer clients should use read-only cache
|
||||
credentials. Bump the image `generation` when intentionally refreshing guest
|
||||
packages. Strict reproducibility requires pinned distro repository snapshots.
|
||||
|
||||
## Install or copy artifacts
|
||||
|
||||
Install a repository-built package:
|
||||
|
||||
```shell
|
||||
bazel run //nix/test-guest:ubuntu_docker -- \
|
||||
--install artifacts/openshell_0.0.0-local_arm64.deb \
|
||||
-- openshell --version
|
||||
```
|
||||
|
||||
Ubuntu accepts `.deb` packages. CentOS, Fedora, and Rocky accept `.rpm`
|
||||
packages. Package architecture must match the host and guest architecture.
|
||||
|
||||
Copy executables without packaging them:
|
||||
|
||||
```shell
|
||||
bazel run //nix/test-guest:fedora_podman -- \
|
||||
--copy ./openshell:/usr/local/bin/openshell \
|
||||
-- openshell --version
|
||||
```
|
||||
|
||||
`--install` and `--copy` are repeatable. Copied files are installed with mode
|
||||
`0755`. These per-run artifacts are never included in the prepared QCOW2.
|
||||
|
||||
Forward a loopback port with `--forward-port HOST_PORT:GUEST_PORT`. Both ports
|
||||
must be between 1024 and 65535, and each host port may appear only once.
|
||||
|
||||
## E2E integration
|
||||
|
||||
The named E2E runner selects the corresponding Bazel guest target when VM mode
|
||||
is requested:
|
||||
|
||||
```shell
|
||||
mise run e2e:test -- \
|
||||
--with podman \
|
||||
--gateway-config path/to/gateway.toml \
|
||||
--suite suite-name
|
||||
```
|
||||
|
||||
Use `--vm DISTRO` to override the inferred distro. The ordered `--with` values
|
||||
must correspond to a supported target suffix, such as `rocky_docker_selinux`.
|
||||
|
||||
## Directory structure
|
||||
|
||||
```text
|
||||
nix/test-guest/
|
||||
├── README.md
|
||||
├── default.nix
|
||||
├── BUILD.bazel
|
||||
├── catalog.bzl
|
||||
├── extensions.bzl
|
||||
├── test-guest.MODULE.bazel
|
||||
├── test_guest.bzl
|
||||
├── host-tools.nix
|
||||
├── host-tools.BUILD.bazel
|
||||
├── nixpkgs.nix
|
||||
├── build-image.sh
|
||||
├── image-seal.sh
|
||||
├── run.sh
|
||||
├── cache.sh
|
||||
├── cache-lib.sh
|
||||
├── cache-seal.sh
|
||||
├── distros/
|
||||
│ ├── ubuntu.nix
|
||||
│ ├── centos.nix
|
||||
│ ├── fedora.nix
|
||||
│ └── rocky.nix
|
||||
└── configuration/
|
||||
├── docker.yml
|
||||
├── podman.yml
|
||||
└── selinux.yml
|
||||
```
|
||||
|
||||
- `default.nix` assembles the guest and cache flake apps. It selects host architecture and acceleration, supplies the runtime tools, and exposes distro profiles and configuration playbooks as Nix-store catalogs.
|
||||
- `run.sh` owns the disposable guest lifecycle: cache lookup, cloud-image realization, cloud-init seed creation, QEMU startup, SSH readiness, Ansible execution, artifact installation, guest command execution, and cleanup.
|
||||
- `cache.sh` ensures an exact prepared disk exists locally. It can pull or explicitly push the disk as an OCI artifact.
|
||||
- `cache-lib.sh` defines deterministic cache identity and validation helpers shared by the runner and cache command.
|
||||
- `cache-seal.sh` removes per-instance state and zeroes free space inside a prepared guest before capture.
|
||||
- `distros/*.nix` define the immutable base-image catalog. Each record pins and exports the image URL and hash and declares the expected OS ID, version, and package family.
|
||||
- `configuration/*.yml` are host-executed Ansible playbooks that layer optional capabilities onto a base guest. Configurations remain independent and run in the order supplied with repeated `--with` arguments.
|
||||
- `README.md` documents the supported combinations and developer interface.
|
||||
|
||||
The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-guest` and `test-guest-cache` apps. Debian artifact creation remains outside the guest harness in [`tasks/scripts/package-deb.sh`](../../tasks/scripts/package-deb.sh); the runner only installs or copies artifacts that already exist.
|
||||
|
||||
## Supported configurations
|
||||
|
||||
| Distro | Docker | Podman | SELinux | Package format |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Ubuntu 24.04 | Yes | Yes | No | `.deb` |
|
||||
| CentOS Stream 10 | No | Yes | Yes | `.rpm` |
|
||||
| Fedora 44 | No | Yes | Yes | `.rpm` |
|
||||
| Rocky Linux 9 | Yes | Yes | Yes | `.rpm` |
|
||||
|
||||
The Ubuntu 24.04 Podman configuration is available for runtime and packaging
|
||||
checks, but its Podman 4 release does not provide the `pasta` rootless network
|
||||
helper required by OpenShell sandbox callbacks. OpenShell Podman E2E runs use
|
||||
the Fedora guest, which provides Podman 5 and `pasta`.
|
||||
|
||||
List the available distros and configurations:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- --list
|
||||
```
|
||||
|
||||
## Open an interactive VM
|
||||
|
||||
Boot a base Ubuntu VM:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- --distro ubuntu
|
||||
```
|
||||
|
||||
Apply the Docker configuration before opening the SSH session:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- --distro ubuntu --with docker
|
||||
```
|
||||
|
||||
Other combinations use the same interface:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- --distro rocky --with docker
|
||||
nix run .#test-guest -- --distro centos --with podman
|
||||
nix run .#test-guest -- --distro fedora --with podman
|
||||
```
|
||||
|
||||
Configurations are repeatable:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- \
|
||||
--distro ubuntu \
|
||||
--with docker \
|
||||
--with podman
|
||||
```
|
||||
|
||||
Ensure SELinux is enforcing on CentOS, Fedora, or Rocky:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- \
|
||||
--distro rocky \
|
||||
--with docker \
|
||||
--with selinux \
|
||||
-- getenforce
|
||||
```
|
||||
|
||||
`--with selinux` installs the required tooling, persists `SELINUX=enforcing`, applies enforcing mode live, and verifies the result. It fails on Ubuntu and on guests where SELinux is fully disabled and would require a reboot to enable.
|
||||
|
||||
## Ansible configurations
|
||||
|
||||
Configurations are Ansible playbooks stored under `nix/test-guest/configuration/`. Ansible runs on the host using the VM's ephemeral SSH key and loopback port. The guest does not install Ansible.
|
||||
|
||||
Configurations run in the order provided on the command line. OpenShell packages and copied binaries are installed after all configurations succeed.
|
||||
|
||||
`--install` packages and `--copy` executables are applied by a dedicated per-run Ansible playbook. They are not stored in prepared VM cache entries.
|
||||
|
||||
## Prepared VM cache
|
||||
|
||||
The `test-guest-cache` app ensures a prepared disk exists for one exact distro, host architecture, and ordered configuration list. It checks the local cache first, optionally pulls a matching OCI artifact, or builds and validates a new local entry on a miss:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest-cache -- \
|
||||
--distro ubuntu \
|
||||
--with docker
|
||||
```
|
||||
|
||||
Configure an OCI repository and a trusted manifest digest to use it as a shared
|
||||
backing cache:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest-cache -- \
|
||||
--distro ubuntu \
|
||||
--with docker \
|
||||
--repository ghcr.io/nvidia/openshell/test-guest-cache \
|
||||
--digest sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
||||
```
|
||||
|
||||
The command never publishes implicitly. Add `--push` after authenticating ORAS through its Docker-compatible credential configuration:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest-cache -- \
|
||||
--distro ubuntu \
|
||||
--with docker \
|
||||
--repository ghcr.io/nvidia/openshell/test-guest-cache \
|
||||
--push
|
||||
```
|
||||
|
||||
A successful push prints the immutable `repository@sha256:...` reference. Supply
|
||||
that digest to consumers through trusted CI configuration. Pulls by mutable tag
|
||||
are not allowed. A pulled local entry records its manifest digest and is reused
|
||||
only when it matches the requested trusted digest.
|
||||
|
||||
A cache build boots and configures a disposable VM, runs the internal sealing script, flattens the overlay into a standalone QCOW2 disk, and validates a fresh boot before committing the entry. The OCI artifact contains metadata and a `disk.qcow2.zst` layer.
|
||||
|
||||
The key includes the pinned base-image identity, guest architecture, ordered configuration file digests, Ansible version, cache generation, and sealing script digest. Installed packages, copied binaries, forwarded ports, and guest commands are never cached.
|
||||
|
||||
Normal `test-guest` runs automatically use an exact valid local entry after
|
||||
rechecking its disk checksum and QCOW2 structure. On a local miss, the runner
|
||||
invokes the cache builder and stores the prepared disk before continuing. It
|
||||
then creates a fresh writable overlay, cloud-init instance, machine ID, and SSH
|
||||
identity from that entry. Set `OPENSHELL_TEST_GUEST_CACHE_DISABLE=1` to bypass
|
||||
both local lookup and automatic population.
|
||||
|
||||
The default cache directory is `${XDG_CACHE_HOME:-$HOME/.cache}/openshell/test-guest`. Override it with `--cache-dir` on the cache command or `OPENSHELL_TEST_GUEST_CACHE_DIR` for either app.
|
||||
|
||||
Cache command options:
|
||||
|
||||
```text
|
||||
--distro NAME Base distro: ubuntu, centos, fedora, or rocky
|
||||
--with NAME Apply docker, podman, or selinux; repeatable
|
||||
--repository REF OCI repository without a tag
|
||||
--digest DIGEST Trusted OCI manifest digest required for pulls
|
||||
--cache-dir PATH Override the local prepared-disk cache directory
|
||||
--push Publish the ensured entry to the repository
|
||||
```
|
||||
|
||||
## Install an OpenShell package
|
||||
|
||||
Package existing ARM64 Linux binaries with the repository's `package:deb:arm64` mise task:
|
||||
|
||||
```shell
|
||||
OPENSHELL_CLI_BINARY="$PWD/target/aarch64-unknown-linux-musl/release/openshell" \
|
||||
OPENSHELL_GATEWAY_BINARY="$PWD/target/aarch64-unknown-linux-gnu/release/openshell-gateway" \
|
||||
OPENSHELL_DRIVER_VM_BINARY="$PWD/target/aarch64-unknown-linux-gnu/release/openshell-driver-vm" \
|
||||
OPENSHELL_DEB_VERSION=0.0.0-local \
|
||||
OPENSHELL_OUTPUT_DIR="$PWD/artifacts" \
|
||||
nix develop --command mise run package:deb:arm64
|
||||
```
|
||||
|
||||
Install the package in an Ubuntu VM and run a command:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- \
|
||||
--distro ubuntu \
|
||||
--with docker \
|
||||
--install artifacts/openshell_0.0.0-local_arm64.deb \
|
||||
-- openshell --version
|
||||
```
|
||||
|
||||
For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. The package architecture must match the host and guest architecture.
|
||||
|
||||
`--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS, Fedora, and Rocky Linux. This prototype can install an existing RPM but does not build one.
|
||||
|
||||
## Copy binaries directly
|
||||
|
||||
Use `--copy SOURCE:DEST` to install an executable without creating a package:
|
||||
|
||||
```shell
|
||||
nix run .#test-guest -- \
|
||||
--distro ubuntu \
|
||||
--copy ./openshell:/usr/local/bin/openshell \
|
||||
-- openshell --version
|
||||
```
|
||||
|
||||
The destination must be an absolute guest path. Copied files are installed with mode `0755`.
|
||||
|
||||
## Runner options
|
||||
|
||||
```text
|
||||
--distro NAME Base distro: ubuntu, centos, fedora, or rocky
|
||||
--with NAME Apply docker, podman, or selinux; repeatable
|
||||
--install PATH Install a .deb or .rpm package; repeatable
|
||||
--copy SRC:DEST Copy an executable into the guest; repeatable
|
||||
--ssh-port PORT Use a specific loopback SSH forwarding port
|
||||
--forward-port HOST_PORT:GUEST_PORT
|
||||
Forward a loopback host port to a guest port; repeatable
|
||||
--keep Preserve the disk overlay and logs after shutdown
|
||||
--list List distros and configurations
|
||||
```
|
||||
|
||||
Each `--forward-port` binds only `127.0.0.1` on the host. Both ports must be unprivileged values from 1024 through 65535, and each host port may appear only once.
|
||||
|
||||
Arguments after `--` are executed inside the guest. Without a command, the runner opens an interactive SSH session.
|
||||
|
||||
## Lifecycle
|
||||
|
||||
Each invocation ensures an exact prepared local cache entry exists. On a miss,
|
||||
the cache builder realizes the hash-pinned cloud image, applies the selected
|
||||
configurations, seals and validates the prepared disk, and stores it locally.
|
||||
The runner then:
|
||||
|
||||
1. Creates a temporary QCOW2 overlay backed by the prepared cache disk or pinned cloud image.
|
||||
2. Boots QEMU with HVF, KVM, or the Linux TCG fallback.
|
||||
3. Creates a fresh cloud-init instance and ephemeral SSH key.
|
||||
4. Applies the selected Ansible configurations only when the base is not prepared.
|
||||
5. Installs or copies the supplied artifacts.
|
||||
6. Opens SSH or executes the requested guest command.
|
||||
7. Powers off QEMU and deletes the writable overlay.
|
||||
|
||||
Prepared cache disks remain read-only. Test-specific state exists only in the disposable overlay.
|
||||
|
||||
Use `--keep` to preserve the overlay, cloud-init seed, SSH key, and serial log for debugging. The retained directory is printed when the runner exits.
|
||||
- `catalog.bzl` is the single source for Bazel cloud-image provenance, guest
|
||||
metadata, and supported variants.
|
||||
- `extensions.bzl` creates pinned cloud-image repositories from the catalog.
|
||||
- `host-tools.nix` builds the Nix host-tool closure imported through
|
||||
`rules_nixpkgs`; `nixpkgs.nix` ties it to the repository `flake.lock`.
|
||||
- `test_guest.bzl` defines the providers, host-tools toolchain, image action,
|
||||
and `rules_shell` run and test launchers.
|
||||
- `build-image.sh` adapts the QEMU lifecycle to explicit Bazel inputs and
|
||||
outputs.
|
||||
- `image-seal.sh` removes per-instance state and zeroes free space before image
|
||||
capture.
|
||||
- `run.sh` owns cloud-init generation, QEMU startup, SSH readiness, Ansible
|
||||
execution, artifact installation, guest commands, and cleanup.
|
||||
- `configuration/*.yml` are host-executed Ansible playbooks applied in their
|
||||
declared order.
|
||||
|
||||
## Current limitations
|
||||
|
||||
- Host and guest architectures must match.
|
||||
- TCG is slower than hardware virtualization and uses a longer SSH readiness timeout.
|
||||
- Prepared cache entries are architecture-specific and match the exact ordered configuration list.
|
||||
- OCI pulls transfer a complete compressed standalone disk; incremental disk layers are not implemented.
|
||||
- Guest ports are reachable from the host only when explicitly exposed with loopback-only `--forward-port`.
|
||||
- The runner does not build OpenShell, configure a gateway, or select an E2E test suite.
|
||||
- Guest architecture must match the host architecture.
|
||||
- Image preparation requires local execution, networking, and a working host
|
||||
accelerator or the slower TCG fallback.
|
||||
- The image rule does not yet pin distro package repositories.
|
||||
- The runner supports QEMU/HVF on Apple Silicon macOS and QEMU/KVM or TCG on
|
||||
Linux; Intel macOS is not supported.
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Build one standalone prepared QCOW2 from explicit Bazel inputs and outputs.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: build-image.sh OPTIONS
|
||||
|
||||
Required options:
|
||||
--runner PATH Nix-wrapped test-guest runner
|
||||
--qemu-img PATH qemu-img executable
|
||||
--jq PATH jq executable
|
||||
--sha256sum PATH sha256sum executable
|
||||
--seal PATH guest sealing script
|
||||
--base-image PATH pinned input cloud image
|
||||
--output PATH standalone QCOW2 output
|
||||
--metadata-output PATH JSON metadata output
|
||||
--distro NAME distro catalog name
|
||||
--os-id ID expected guest OS ID
|
||||
--os-version VERSION expected guest OS version
|
||||
--package-family FAMILY guest package family: deb or rpm
|
||||
--architecture ARCH guest architecture
|
||||
--base-image-url URL provenance URL
|
||||
--base-image-hash HASH pinned source hash
|
||||
--generation NUMBER explicit image generation
|
||||
--qemu-version-file PATH Nix-provided QEMU version
|
||||
--ansible-version-file PATH
|
||||
Nix-provided Ansible version
|
||||
|
||||
Optional:
|
||||
--configuration NAME PATH
|
||||
ordered image configuration and playbook; repeatable
|
||||
EOF
|
||||
}
|
||||
|
||||
require_value() {
|
||||
if [ "$#" -lt 2 ] || [ -z "${2:-}" ]; then
|
||||
echo "$1 requires a value" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
runner=
|
||||
qemu_img=
|
||||
jq_bin=
|
||||
sha256sum_bin=
|
||||
seal=
|
||||
base_image=
|
||||
output=
|
||||
metadata_output=
|
||||
distro=
|
||||
os_id=
|
||||
os_version=
|
||||
package_family=
|
||||
architecture=
|
||||
base_image_url=
|
||||
base_image_hash=
|
||||
generation=
|
||||
qemu_version_file=
|
||||
ansible_version_file=
|
||||
configurations=()
|
||||
configuration_paths=()
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--runner) require_value "$@"; runner=$2; shift 2 ;;
|
||||
--qemu-img) require_value "$@"; qemu_img=$2; shift 2 ;;
|
||||
--jq) require_value "$@"; jq_bin=$2; shift 2 ;;
|
||||
--sha256sum) require_value "$@"; sha256sum_bin=$2; shift 2 ;;
|
||||
--seal) require_value "$@"; seal=$2; shift 2 ;;
|
||||
--base-image) require_value "$@"; base_image=$2; shift 2 ;;
|
||||
--output) require_value "$@"; output=$2; shift 2 ;;
|
||||
--metadata-output) require_value "$@"; metadata_output=$2; shift 2 ;;
|
||||
--distro) require_value "$@"; distro=$2; shift 2 ;;
|
||||
--os-id) require_value "$@"; os_id=$2; shift 2 ;;
|
||||
--os-version) require_value "$@"; os_version=$2; shift 2 ;;
|
||||
--package-family) require_value "$@"; package_family=$2; shift 2 ;;
|
||||
--architecture) require_value "$@"; architecture=$2; shift 2 ;;
|
||||
--base-image-url) require_value "$@"; base_image_url=$2; shift 2 ;;
|
||||
--base-image-hash) require_value "$@"; base_image_hash=$2; shift 2 ;;
|
||||
--generation) require_value "$@"; generation=$2; shift 2 ;;
|
||||
--qemu-version-file) require_value "$@"; qemu_version_file=$2; shift 2 ;;
|
||||
--ansible-version-file) require_value "$@"; ansible_version_file=$2; shift 2 ;;
|
||||
--configuration)
|
||||
if [ "$#" -lt 3 ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then
|
||||
echo "--configuration requires NAME PATH" >&2
|
||||
exit 2
|
||||
fi
|
||||
configurations+=("$2")
|
||||
configuration_paths+=("$3")
|
||||
shift 3
|
||||
;;
|
||||
-h | --help) usage; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
required=(
|
||||
runner qemu_img jq_bin sha256sum_bin seal base_image output
|
||||
metadata_output distro os_id os_version package_family architecture
|
||||
base_image_url base_image_hash generation qemu_version_file
|
||||
ansible_version_file
|
||||
)
|
||||
for variable in "${required[@]}"; do
|
||||
if [ -z "${!variable}" ]; then
|
||||
echo "missing required option for ${variable}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# qemu-img records backing paths relative to the overlay location. Bazel passes
|
||||
# execroot-relative paths, so canonicalize files that cross into the runner.
|
||||
# Keep executable symlinks intact: Nix coreutils uses the symlink basename to
|
||||
# select the command implemented by its multicall binary.
|
||||
seal=$(realpath "${seal}")
|
||||
base_image=$(realpath "${base_image}")
|
||||
qemu_version_file=$(realpath "${qemu_version_file}")
|
||||
ansible_version_file=$(realpath "${ansible_version_file}")
|
||||
for index in "${!configuration_paths[@]}"; do
|
||||
configuration_paths[index]=$(realpath "${configuration_paths[index]}")
|
||||
done
|
||||
output="$(realpath "$(dirname "${output}")")/$(basename "${output}")"
|
||||
metadata_output="$(realpath "$(dirname "${metadata_output}")")/$(basename "${metadata_output}")"
|
||||
|
||||
scratch_root=${TEST_TMPDIR:-${TMPDIR:-/tmp}}
|
||||
scratch=$(mktemp -d "${scratch_root%/}/test-guest-image.XXXXXX")
|
||||
preserve_scratch=0
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
trap - EXIT INT TERM
|
||||
if [ "${status}" -ne 0 ]; then
|
||||
echo "test-guest image build failed; scratch directory: ${scratch}" >&2
|
||||
preserve_scratch=1
|
||||
fi
|
||||
if [ "${preserve_scratch}" -eq 0 ]; then
|
||||
rm -rf "${scratch}"
|
||||
fi
|
||||
exit "${status}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
mkdir -p "${scratch}/prepare" "${scratch}/validate"
|
||||
prepare_args=(
|
||||
--distro "${distro}"
|
||||
--os-id "${os_id}"
|
||||
--os-version "${os_version}"
|
||||
--package-family "${package_family}"
|
||||
--keep
|
||||
)
|
||||
for index in "${!configurations[@]}"; do
|
||||
prepare_args+=(
|
||||
--configuration-file
|
||||
"${configurations[index]}"
|
||||
"${configuration_paths[index]}"
|
||||
)
|
||||
done
|
||||
prepare_args+=(
|
||||
--copy
|
||||
"${seal}:/usr/local/sbin/openshell-test-guest-image-seal"
|
||||
--
|
||||
sudo
|
||||
/usr/local/sbin/openshell-test-guest-image-seal
|
||||
)
|
||||
|
||||
echo "==> Building Bazel test-guest image: ${distro} (${architecture})"
|
||||
TMPDIR="${scratch}/prepare" \
|
||||
OPENSHELL_TEST_GUEST_BASE_IMAGE_OVERRIDE="${base_image}" \
|
||||
"${runner}" "${prepare_args[@]}"
|
||||
|
||||
shopt -s nullglob
|
||||
run_dirs=("${scratch}/prepare/openshell-test-guest"/run.*)
|
||||
shopt -u nullglob
|
||||
if [ "${#run_dirs[@]}" -ne 1 ] || [ ! -f "${run_dirs[0]}/disk.qcow2" ]; then
|
||||
echo "image preparation did not retain exactly one guest disk" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"${qemu_img}" convert -q -c -f qcow2 -O qcow2 \
|
||||
"${run_dirs[0]}/disk.qcow2" "${output}"
|
||||
|
||||
image_info=$("${qemu_img}" info --output=json "${output}")
|
||||
"${jq_bin}" -e '
|
||||
.format == "qcow2" and
|
||||
(. ["backing-filename"]? == null) and
|
||||
(. ["data-file"]? == null) and
|
||||
((.snapshots? // []) | length == 0) and
|
||||
(. ["virtual-size"] > 0) and
|
||||
(. ["virtual-size"] <= 68719476736)
|
||||
' <<<"${image_info}" >/dev/null
|
||||
"${qemu_img}" check "${output}" >/dev/null
|
||||
|
||||
validation='test -s /etc/machine-id; sudo test -s /etc/ssh/ssh_host_ed25519_key'
|
||||
for configuration in "${configurations[@]}"; do
|
||||
case "${configuration}" in
|
||||
docker) validation+='; docker info >/dev/null' ;;
|
||||
podman) validation+='; podman info >/dev/null' ;;
|
||||
selinux) validation+='; test "$(getenforce)" = Enforcing' ;;
|
||||
esac
|
||||
done
|
||||
|
||||
validate_args=(
|
||||
--distro "${distro}"
|
||||
--os-id "${os_id}"
|
||||
--os-version "${os_version}"
|
||||
--package-family "${package_family}"
|
||||
)
|
||||
for configuration in "${configurations[@]}"; do
|
||||
validate_args+=(--with "${configuration}")
|
||||
done
|
||||
validate_args+=(-- bash -lc "${validation}")
|
||||
|
||||
echo "==> Validating fresh boot from Bazel image output"
|
||||
TMPDIR="${scratch}/validate" \
|
||||
OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE="${output}" \
|
||||
"${runner}" "${validate_args[@]}"
|
||||
|
||||
configuration_json=$("${jq_bin}" -cn --args '$ARGS.positional' "${configurations[@]}")
|
||||
disk_sha=$("${sha256sum_bin}" "${output}")
|
||||
disk_sha=${disk_sha%% *}
|
||||
virtual_size=$("${jq_bin}" -r '.["virtual-size"]' <<<"${image_info}")
|
||||
qemu_version=$(<"${qemu_version_file}")
|
||||
ansible_version=$(<"${ansible_version_file}")
|
||||
|
||||
"${jq_bin}" -n \
|
||||
--argjson schema 2 \
|
||||
--arg distro "${distro}" \
|
||||
--arg os_id "${os_id}" \
|
||||
--arg os_version "${os_version}" \
|
||||
--arg package_family "${package_family}" \
|
||||
--arg architecture "${architecture}" \
|
||||
--arg base_image_url "${base_image_url}" \
|
||||
--arg base_image_hash "${base_image_hash}" \
|
||||
--arg disk_sha256 "${disk_sha}" \
|
||||
--arg qemu_version "${qemu_version}" \
|
||||
--arg ansible_version "${ansible_version}" \
|
||||
--argjson generation "${generation}" \
|
||||
--argjson virtual_size "${virtual_size}" \
|
||||
--argjson configurations "${configuration_json}" \
|
||||
'{
|
||||
schema: $schema,
|
||||
generation: $generation,
|
||||
disk_layout: "standalone-qcow2-zlib-v2",
|
||||
distro: $distro,
|
||||
os_id: $os_id,
|
||||
os_version: $os_version,
|
||||
package_family: $package_family,
|
||||
architecture: $architecture,
|
||||
base_image_url: $base_image_url,
|
||||
base_image_hash: $base_image_hash,
|
||||
configurations: $configurations,
|
||||
qemu_version: $qemu_version,
|
||||
ansible_version: $ansible_version,
|
||||
disk_sha256: $disk_sha256,
|
||||
virtual_size: $virtual_size
|
||||
}' >"${metadata_output}"
|
||||
|
||||
chmod 0444 "${output}" "${metadata_output}"
|
||||
preserve_scratch=0
|
||||
echo "==> Bazel test-guest image complete: ${output}"
|
||||
@@ -1,158 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Shared cache identity and validation helpers for the test guest runner.
|
||||
|
||||
TEST_GUEST_CACHE_SCHEMA_VERSION=1
|
||||
TEST_GUEST_CACHE_DISK_LAYOUT=standalone-qcow2-zstd-v1
|
||||
TEST_GUEST_CACHE_ARTIFACT_TYPE=application/vnd.nvidia.openshell.test-guest.cache.v1
|
||||
TEST_GUEST_CACHE_METADATA_TYPE=application/vnd.nvidia.openshell.test-guest.cache.metadata.v1+json
|
||||
TEST_GUEST_CACHE_DISK_TYPE=application/vnd.nvidia.openshell.test-guest.cache.disk.qcow2.v1+zstd
|
||||
|
||||
test_vm_cache_root() {
|
||||
if [ -n "${OPENSHELL_TEST_GUEST_CACHE_DIR:-}" ]; then
|
||||
printf '%s\n' "${OPENSHELL_TEST_GUEST_CACHE_DIR}"
|
||||
else
|
||||
printf '%s\n' "${XDG_CACHE_HOME:-${HOME}/.cache}/openshell/test-guest"
|
||||
fi
|
||||
}
|
||||
|
||||
test_vm_cache_oci_architecture() {
|
||||
case "${TEST_GUEST_ARCHITECTURE}" in
|
||||
x86_64) printf '%s\n' amd64 ;;
|
||||
aarch64) printf '%s\n' arm64 ;;
|
||||
*)
|
||||
echo "unsupported cache architecture: ${TEST_GUEST_ARCHITECTURE}" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
test_vm_cache_sha256() {
|
||||
if [ ! -r "$1" ]; then
|
||||
echo "cache identity input is not readable: $1" >&2
|
||||
return 1
|
||||
fi
|
||||
sha256sum "$1" | cut -d ' ' -f 1
|
||||
}
|
||||
|
||||
test_vm_cache_key() {
|
||||
local distro_name=$1
|
||||
shift
|
||||
local configuration
|
||||
local configuration_hash
|
||||
local architecture
|
||||
local seal_hash
|
||||
local material
|
||||
local configuration_line
|
||||
|
||||
architecture=$(test_vm_cache_oci_architecture) || return 1
|
||||
seal_hash=$(test_vm_cache_sha256 "${OPENSHELL_TEST_GUEST_CACHE_SEAL}") || return 1
|
||||
printf -v material \
|
||||
'schema=%s\ngeneration=%s\ndisk_layout=%s\ndistro=%s\nos_version=%s\narchitecture=%s\nbase_url=%s\nbase_hash=%s\noverlay_growth=%s\nansible_version=%s\nseal_sha256=%s\n' \
|
||||
"${TEST_GUEST_CACHE_SCHEMA_VERSION}" \
|
||||
"${TEST_GUEST_CACHE_GENERATION}" \
|
||||
"${TEST_GUEST_CACHE_DISK_LAYOUT}" \
|
||||
"${distro_name}" \
|
||||
"${TEST_GUEST_OS_VERSION}" \
|
||||
"${architecture}" \
|
||||
"${TEST_GUEST_IMAGE_URL}" \
|
||||
"${TEST_GUEST_IMAGE_HASH}" \
|
||||
16G \
|
||||
"${TEST_GUEST_ANSIBLE_VERSION}" \
|
||||
"${seal_hash}"
|
||||
|
||||
for configuration in "$@"; do
|
||||
configuration_hash=$(
|
||||
test_vm_cache_sha256 \
|
||||
"${OPENSHELL_TEST_GUEST_CONFIGURATIONS}/${configuration}"
|
||||
) || return 1
|
||||
printf -v configuration_line \
|
||||
'configuration=%s:%s\n' \
|
||||
"${configuration}" \
|
||||
"${configuration_hash}"
|
||||
material+=${configuration_line}
|
||||
done
|
||||
|
||||
printf '%s' "${material}" | sha256sum | cut -d ' ' -f 1
|
||||
}
|
||||
|
||||
test_vm_cache_tag() {
|
||||
local distro_name=$1
|
||||
local key=$2
|
||||
printf 'v%s-%s-%s-%s\n' \
|
||||
"${TEST_GUEST_CACHE_SCHEMA_VERSION}" \
|
||||
"${distro_name}" \
|
||||
"$(test_vm_cache_oci_architecture)" \
|
||||
"${key}"
|
||||
}
|
||||
|
||||
test_vm_cache_entry_dir() {
|
||||
local root=$1
|
||||
local key=$2
|
||||
printf '%s/entries/%s\n' "${root}" "${key}"
|
||||
}
|
||||
|
||||
test_vm_cache_metadata_matches() {
|
||||
local metadata=$1
|
||||
local key=$2
|
||||
local distro_name=$3
|
||||
shift 3
|
||||
local architecture
|
||||
local expected_configurations
|
||||
architecture=$(test_vm_cache_oci_architecture) || return 1
|
||||
expected_configurations=$(jq -cn --args '$ARGS.positional' "$@") || return 1
|
||||
|
||||
jq -e \
|
||||
--argjson schema "${TEST_GUEST_CACHE_SCHEMA_VERSION}" \
|
||||
--arg key "${key}" \
|
||||
--arg distro "${distro_name}" \
|
||||
--arg os_version "${TEST_GUEST_OS_VERSION}" \
|
||||
--arg architecture "$(test_vm_cache_oci_architecture)" \
|
||||
--arg base_hash "${TEST_GUEST_IMAGE_HASH}" \
|
||||
--argjson configurations "${expected_configurations}" \
|
||||
'
|
||||
.schema == $schema and
|
||||
.key == $key and
|
||||
.distro == $distro and
|
||||
.os_version == $os_version and
|
||||
.architecture == $architecture and
|
||||
.base_image_hash == $base_hash and
|
||||
.configurations == $configurations
|
||||
' "${metadata}" >/dev/null
|
||||
}
|
||||
|
||||
test_vm_cache_local_entry_valid() {
|
||||
local root=$1
|
||||
local key=$2
|
||||
local distro_name=$3
|
||||
shift 3
|
||||
local entry
|
||||
entry=$(test_vm_cache_entry_dir "${root}" "${key}")
|
||||
|
||||
# Cache builders and OCI pulls verify the disk before atomically installing
|
||||
# the complete entry. Keep cache-hit validation metadata-only so launching a
|
||||
# guest does not hash and inspect the entire QCOW2 on every run.
|
||||
[ -f "${entry}/complete" ] &&
|
||||
[ -s "${entry}/disk.qcow2" ] &&
|
||||
[ -f "${entry}/metadata.json" ] &&
|
||||
test_vm_cache_metadata_matches \
|
||||
"${entry}/metadata.json" "${key}" "${distro_name}" "$@"
|
||||
}
|
||||
|
||||
test_vm_cache_validate_disk() {
|
||||
local disk=$1
|
||||
local info
|
||||
info=$(qemu-img info --output=json "${disk}")
|
||||
|
||||
jq -e '
|
||||
.format == "qcow2" and
|
||||
(.["backing-filename"]? == null) and
|
||||
(.["data-file"]? == null) and
|
||||
((.snapshots? // []) | length == 0) and
|
||||
(.["virtual-size"] > 0) and
|
||||
(.["virtual-size"] <= 68719476736)
|
||||
' <<<"${info}" >/dev/null
|
||||
qemu-img check "${disk}" >/dev/null
|
||||
}
|
||||
@@ -1,522 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Ensure a prepared test guest disk exists locally, optionally backed by OCI.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
nix run .#test-guest-cache -- --distro DISTRO [OPTIONS]
|
||||
|
||||
Options:
|
||||
--distro NAME Base distro: ubuntu, centos, fedora, or rocky
|
||||
--with NAME Apply a configuration; repeatable (docker, podman, selinux)
|
||||
--repository REF OCI repository without a tag
|
||||
--digest DIGEST Trusted OCI manifest digest required for pulls
|
||||
--cache-dir PATH Override the local prepared-disk cache directory
|
||||
--push Publish a newly built or local entry to the repository
|
||||
-h, --help Show this help
|
||||
|
||||
The command ensures one exact distro, architecture, and ordered configuration
|
||||
combination exists locally. OCI pulls require a trusted manifest digest.
|
||||
Otherwise, the command builds and validates a prepared disk on a miss.
|
||||
Publishing is explicit and requires both --repository and --push.
|
||||
EOF
|
||||
}
|
||||
|
||||
if [ "${OPENSHELL_TEST_GUEST_RUNTIME:-}" != 1 ] ||
|
||||
[ ! -d "${OPENSHELL_TEST_GUEST_DISTROS:-}" ] ||
|
||||
[ ! -d "${OPENSHELL_TEST_GUEST_CONFIGURATIONS:-}" ] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_CACHE_LIB:-}" ] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_CACHE_SEAL:-}" ] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_RUNNER:-}" ]; then
|
||||
echo "run this script through 'nix run .#test-guest-cache -- ...'" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
. "${OPENSHELL_TEST_GUEST_CACHE_LIB}"
|
||||
|
||||
require_value() {
|
||||
if [ "$#" -lt 2 ] || [ -z "${2:-}" ]; then
|
||||
echo "$1 requires a value" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
distro=
|
||||
repository=${OPENSHELL_TEST_GUEST_CACHE_REPOSITORY:-}
|
||||
pull_digest=${OPENSHELL_TEST_GUEST_CACHE_DIGEST:-}
|
||||
cache_dir=
|
||||
push=0
|
||||
configurations=()
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--distro)
|
||||
require_value "$@"
|
||||
distro=$2
|
||||
shift 2
|
||||
;;
|
||||
--with)
|
||||
require_value "$@"
|
||||
configurations+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
--repository)
|
||||
require_value "$@"
|
||||
repository=$2
|
||||
shift 2
|
||||
;;
|
||||
--digest)
|
||||
require_value "$@"
|
||||
pull_digest=$2
|
||||
shift 2
|
||||
;;
|
||||
--cache-dir)
|
||||
require_value "$@"
|
||||
cache_dir=$2
|
||||
shift 2
|
||||
;;
|
||||
--push)
|
||||
push=1
|
||||
shift
|
||||
;;
|
||||
-h | --help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown test guest cache argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "${distro}" ]; then
|
||||
echo "--distro is required" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}" ]; then
|
||||
echo "unknown distro: ${distro}" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Distro profiles contain only trusted values generated into the Nix store.
|
||||
# shellcheck disable=SC1090
|
||||
. "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}"
|
||||
|
||||
for item in "${configurations[@]}"; do
|
||||
if [[ ! ${item} =~ ^[a-z0-9][a-z0-9-]*$ ]] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}/${item}" ]; then
|
||||
echo "unknown configuration: ${item:-<empty>}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${push}" -eq 1 ] && [ -z "${repository}" ]; then
|
||||
echo "--push requires --repository" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ ${repository} == *[[:space:]]* ]] || [[ ${repository} == *@* ]]; then
|
||||
echo "--repository must be an untagged OCI repository reference" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ -n "${repository}" ] && [ "${push}" -eq 0 ] && [ -z "${pull_digest}" ]; then
|
||||
echo "--repository requires --digest for OCI pulls" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ -n "${pull_digest}" ] && [ -z "${repository}" ]; then
|
||||
echo "--digest requires --repository" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ -n "${pull_digest}" ] &&
|
||||
[[ ! ${pull_digest} =~ ^sha256:[a-f0-9]{64}$ ]]; then
|
||||
echo "--digest must be a lowercase sha256 OCI manifest digest" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ -n "${cache_dir}" ]; then
|
||||
OPENSHELL_TEST_GUEST_CACHE_DIR=${cache_dir}
|
||||
export OPENSHELL_TEST_GUEST_CACHE_DIR
|
||||
fi
|
||||
|
||||
umask 077
|
||||
cache_root=$(test_vm_cache_root)
|
||||
cache_key=$(test_vm_cache_key "${distro}" "${configurations[@]}")
|
||||
cache_tag=$(test_vm_cache_tag "${distro}" "${cache_key}")
|
||||
entry_dir=$(test_vm_cache_entry_dir "${cache_root}" "${cache_key}")
|
||||
remote_ref=
|
||||
pull_ref=
|
||||
if [ -n "${repository}" ]; then
|
||||
remote_ref="${repository}:${cache_tag}"
|
||||
fi
|
||||
if [ -n "${pull_digest}" ]; then
|
||||
pull_ref="${repository}@${pull_digest}"
|
||||
fi
|
||||
|
||||
mkdir -p "${cache_root}/entries" "${cache_root}/locks" "${cache_root}/staging"
|
||||
|
||||
lock_dir=
|
||||
build_stage=
|
||||
preserve_build_stage=0
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT INT TERM
|
||||
if [ -n "${lock_dir}" ]; then
|
||||
rmdir "${lock_dir}" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${build_stage}" ] && [ -d "${build_stage}" ]; then
|
||||
if [ "${status}" -ne 0 ] && [ "${preserve_build_stage}" -eq 1 ]; then
|
||||
echo "Kept failed cache build state at ${build_stage}" >&2
|
||||
else
|
||||
rm -rf "${build_stage}"
|
||||
fi
|
||||
fi
|
||||
exit "${status}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
for _ in $(seq 1 120); do
|
||||
if mkdir "${cache_root}/locks/${cache_key}.lock" 2>/dev/null; then
|
||||
lock_dir="${cache_root}/locks/${cache_key}.lock"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ -z "${lock_dir}" ]; then
|
||||
echo "timed out waiting for cache lock: ${cache_key}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install_entry() {
|
||||
local disk=$1
|
||||
local metadata=$2
|
||||
local manifest_digest=${3:-}
|
||||
local temporary_entry
|
||||
temporary_entry=$(mktemp -d "${cache_root}/entries/.${cache_key}.XXXXXX")
|
||||
cp --sparse=always "${disk}" "${temporary_entry}/disk.qcow2"
|
||||
chmod 0444 "${temporary_entry}/disk.qcow2"
|
||||
install -m 0444 "${metadata}" "${temporary_entry}/metadata.json"
|
||||
if [ -n "${manifest_digest}" ]; then
|
||||
printf '%s\n' "${manifest_digest}" >"${temporary_entry}/manifest.digest"
|
||||
chmod 0444 "${temporary_entry}/manifest.digest"
|
||||
fi
|
||||
: >"${temporary_entry}/complete"
|
||||
chmod 0444 "${temporary_entry}/complete"
|
||||
|
||||
if [ -e "${entry_dir}" ]; then
|
||||
if local_entry_valid; then
|
||||
rm -rf "${temporary_entry}"
|
||||
return
|
||||
fi
|
||||
echo "cache entry appeared concurrently but is invalid: ${entry_dir}" >&2
|
||||
rm -rf "${temporary_entry}"
|
||||
return 1
|
||||
fi
|
||||
mv "${temporary_entry}" "${entry_dir}"
|
||||
}
|
||||
|
||||
local_entry_valid() {
|
||||
test_vm_cache_local_entry_valid \
|
||||
"${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}" ||
|
||||
return 1
|
||||
if [ -n "${pull_digest}" ]; then
|
||||
[ -f "${entry_dir}/manifest.digest" ] &&
|
||||
[ "$(<"${entry_dir}/manifest.digest")" = "${pull_digest}" ]
|
||||
fi
|
||||
}
|
||||
|
||||
is_remote_miss() {
|
||||
grep -Eqi '404|MANIFEST_UNKNOWN|manifest unknown|not found' "$1"
|
||||
}
|
||||
|
||||
pull_remote() {
|
||||
local requested_ref=${1:-${pull_ref}}
|
||||
local trusted_digest=${2:-${pull_digest}}
|
||||
local install_local=${3:-1}
|
||||
local expected_local_sha=${4:-}
|
||||
local pull_dir
|
||||
local pull_log
|
||||
local compressed_size
|
||||
local expected_sha
|
||||
local actual_sha
|
||||
local resolved_digest
|
||||
pull_dir=$(mktemp -d "${cache_root}/staging/pull.${cache_key}.XXXXXX")
|
||||
pull_log="${pull_dir}/oras.log"
|
||||
|
||||
resolved_digest=$(oras resolve "${requested_ref}") || {
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
}
|
||||
if [ "${resolved_digest}" != "${trusted_digest}" ]; then
|
||||
echo "OCI cache manifest digest does not match trusted digest" >&2
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
fi
|
||||
|
||||
if ! oras pull --output "${pull_dir}" "${requested_ref}" >"${pull_log}" 2>&1; then
|
||||
if is_remote_miss "${pull_log}"; then
|
||||
rm -rf "${pull_dir}"
|
||||
return 1
|
||||
fi
|
||||
cat "${pull_log}" >&2
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
fi
|
||||
|
||||
if [ ! -f "${pull_dir}/metadata.json" ] ||
|
||||
[ ! -f "${pull_dir}/disk.qcow2.zst" ]; then
|
||||
echo "OCI cache artifact is missing metadata.json or disk.qcow2.zst" >&2
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
fi
|
||||
if ! test_vm_cache_metadata_matches \
|
||||
"${pull_dir}/metadata.json" "${cache_key}" "${distro}" "${configurations[@]}"; then
|
||||
echo "OCI cache metadata does not match the requested VM" >&2
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
fi
|
||||
|
||||
compressed_size=$(wc -c <"${pull_dir}/disk.qcow2.zst")
|
||||
if [ "${compressed_size}" -gt 17179869184 ]; then
|
||||
echo "OCI cache disk exceeds the 16 GiB compressed size limit" >&2
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
fi
|
||||
|
||||
(
|
||||
# Limit the decompressed file to 32 GiB before qemu-img parses it.
|
||||
ulimit -f 67108864
|
||||
zstd -d --sparse -f \
|
||||
"${pull_dir}/disk.qcow2.zst" \
|
||||
-o "${pull_dir}/disk.qcow2"
|
||||
)
|
||||
expected_sha=$(jq -r '.disk_sha256' "${pull_dir}/metadata.json")
|
||||
if [ -n "${expected_local_sha}" ] && [ "${expected_sha}" != "${expected_local_sha}" ]; then
|
||||
echo "OCI cache disk checksum does not match the local cache entry" >&2
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
fi
|
||||
actual_sha=$(test_vm_cache_sha256 "${pull_dir}/disk.qcow2")
|
||||
if [ "${expected_sha}" != "${actual_sha}" ]; then
|
||||
echo "OCI cache disk checksum does not match metadata" >&2
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
fi
|
||||
if ! test_vm_cache_validate_disk "${pull_dir}/disk.qcow2"; then
|
||||
echo "OCI cache disk failed QCOW2 validation" >&2
|
||||
rm -rf "${pull_dir}"
|
||||
return 2
|
||||
fi
|
||||
|
||||
if [ "${install_local}" -eq 1 ]; then
|
||||
install_entry \
|
||||
"${pull_dir}/disk.qcow2" "${pull_dir}/metadata.json" "${resolved_digest}"
|
||||
fi
|
||||
rm -rf "${pull_dir}"
|
||||
if [ "${install_local}" -eq 1 ]; then
|
||||
echo "==> Cache remote hit: ${requested_ref}"
|
||||
fi
|
||||
}
|
||||
|
||||
build_local() {
|
||||
local -a prepare_args
|
||||
local -a validate_args
|
||||
local -a run_dirs
|
||||
local configuration
|
||||
local prepared_disk
|
||||
local validation
|
||||
local configuration_json
|
||||
local disk_sha
|
||||
local virtual_size
|
||||
local created
|
||||
|
||||
build_stage=$(mktemp -d "${cache_root}/staging/build.${cache_key}.XXXXXX")
|
||||
preserve_build_stage=1
|
||||
mkdir -p "${build_stage}/prepare-tmp" "${build_stage}/validate-tmp"
|
||||
|
||||
prepare_args=(--distro "${distro}" --keep)
|
||||
for configuration in "${configurations[@]}"; do
|
||||
prepare_args+=(--with "${configuration}")
|
||||
done
|
||||
prepare_args+=(
|
||||
--copy
|
||||
"${OPENSHELL_TEST_GUEST_CACHE_SEAL}:/usr/local/sbin/openshell-test-guest-cache-seal"
|
||||
--
|
||||
sudo
|
||||
/usr/local/sbin/openshell-test-guest-cache-seal
|
||||
)
|
||||
|
||||
echo "==> Cache miss: preparing ${distro} ($(test_vm_cache_oci_architecture))"
|
||||
TMPDIR="${build_stage}/prepare-tmp" \
|
||||
OPENSHELL_TEST_GUEST_CACHE_DISABLE=1 \
|
||||
"${TEST_GUEST_BASH}" "${OPENSHELL_TEST_GUEST_RUNNER}" "${prepare_args[@]}"
|
||||
|
||||
shopt -s nullglob
|
||||
run_dirs=("${build_stage}/prepare-tmp/openshell-test-guest"/run.*)
|
||||
shopt -u nullglob
|
||||
if [ "${#run_dirs[@]}" -ne 1 ] ||
|
||||
[ ! -f "${run_dirs[0]}/disk.qcow2" ]; then
|
||||
echo "cache preparation did not retain exactly one guest disk" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
prepared_disk="${build_stage}/disk.qcow2"
|
||||
qemu-img convert -q -f qcow2 -O qcow2 \
|
||||
"${run_dirs[0]}/disk.qcow2" "${prepared_disk}"
|
||||
test_vm_cache_validate_disk "${prepared_disk}"
|
||||
|
||||
validation='test -s /etc/machine-id; test -n "$(find /etc/ssh -name "ssh_host_*_key" -type f -print -quit)"'
|
||||
for configuration in "${configurations[@]}"; do
|
||||
case "${configuration}" in
|
||||
docker) validation+='; docker info >/dev/null' ;;
|
||||
podman) validation+='; podman info >/dev/null' ;;
|
||||
selinux) validation+='; test "$(getenforce)" = Enforcing' ;;
|
||||
esac
|
||||
done
|
||||
|
||||
validate_args=(--distro "${distro}")
|
||||
for configuration in "${configurations[@]}"; do
|
||||
validate_args+=(--with "${configuration}")
|
||||
done
|
||||
validate_args+=(-- bash -lc "${validation}")
|
||||
|
||||
echo "==> Validating fresh boot from prepared cache disk"
|
||||
TMPDIR="${build_stage}/validate-tmp" \
|
||||
OPENSHELL_TEST_GUEST_CACHE_DISABLE=1 \
|
||||
OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE="${prepared_disk}" \
|
||||
"${TEST_GUEST_BASH}" "${OPENSHELL_TEST_GUEST_RUNNER}" "${validate_args[@]}"
|
||||
|
||||
configuration_json=$(jq -cn --args '$ARGS.positional' "${configurations[@]}")
|
||||
disk_sha=$(test_vm_cache_sha256 "${prepared_disk}")
|
||||
virtual_size=$(
|
||||
qemu-img info --output=json "${prepared_disk}" |
|
||||
jq -r '.["virtual-size"]'
|
||||
)
|
||||
created=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
|
||||
jq -n \
|
||||
--argjson schema "${TEST_GUEST_CACHE_SCHEMA_VERSION}" \
|
||||
--arg key "${cache_key}" \
|
||||
--arg distro "${distro}" \
|
||||
--arg os_version "${TEST_GUEST_OS_VERSION}" \
|
||||
--arg architecture "$(test_vm_cache_oci_architecture)" \
|
||||
--arg base_image_url "${TEST_GUEST_IMAGE_URL}" \
|
||||
--arg base_image_hash "${TEST_GUEST_IMAGE_HASH}" \
|
||||
--arg disk_layout "${TEST_GUEST_CACHE_DISK_LAYOUT}" \
|
||||
--arg disk_sha256 "${disk_sha}" \
|
||||
--arg created "${created}" \
|
||||
--argjson virtual_size "${virtual_size}" \
|
||||
--argjson configurations "${configuration_json}" \
|
||||
'{
|
||||
schema: $schema,
|
||||
key: $key,
|
||||
distro: $distro,
|
||||
os_version: $os_version,
|
||||
architecture: $architecture,
|
||||
base_image_url: $base_image_url,
|
||||
base_image_hash: $base_image_hash,
|
||||
configurations: $configurations,
|
||||
disk_layout: $disk_layout,
|
||||
disk_sha256: $disk_sha256,
|
||||
virtual_size: $virtual_size,
|
||||
created: $created
|
||||
}' >"${build_stage}/metadata.json"
|
||||
|
||||
install_entry "${prepared_disk}" "${build_stage}/metadata.json"
|
||||
preserve_build_stage=0
|
||||
rm -rf "${build_stage}"
|
||||
build_stage=
|
||||
echo "==> Cache build complete: ${entry_dir}"
|
||||
}
|
||||
|
||||
push_remote() {
|
||||
local existing_ref
|
||||
local local_disk_sha
|
||||
local push_dir
|
||||
local manifest_log
|
||||
local published_digest
|
||||
push_dir=$(mktemp -d "${cache_root}/staging/push.${cache_key}.XXXXXX")
|
||||
manifest_log="${push_dir}/manifest.log"
|
||||
|
||||
if oras manifest fetch "${remote_ref}" >"${manifest_log}" 2>&1; then
|
||||
published_digest=$(oras resolve "${remote_ref}")
|
||||
existing_ref="${repository}@${published_digest}"
|
||||
local_disk_sha=$(jq -r '.disk_sha256' "${entry_dir}/metadata.json")
|
||||
if ! pull_remote "${existing_ref}" "${published_digest}" 0 "${local_disk_sha}"; then
|
||||
echo "existing OCI cache artifact failed validation: ${remote_ref}" >&2
|
||||
rm -rf "${push_dir}"
|
||||
return 1
|
||||
fi
|
||||
echo "==> Cache already published and validated: ${remote_ref}"
|
||||
echo "==> Cache immutable reference: ${repository}@${published_digest}"
|
||||
rm -rf "${push_dir}"
|
||||
return
|
||||
fi
|
||||
if ! is_remote_miss "${manifest_log}"; then
|
||||
cat "${manifest_log}" >&2
|
||||
rm -rf "${push_dir}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
install -m 0644 "${entry_dir}/metadata.json" "${push_dir}/metadata.json"
|
||||
zstd -T0 -3 -f \
|
||||
"${entry_dir}/disk.qcow2" \
|
||||
-o "${push_dir}/disk.qcow2.zst"
|
||||
|
||||
(
|
||||
cd "${push_dir}"
|
||||
oras push \
|
||||
--artifact-type "${TEST_GUEST_CACHE_ARTIFACT_TYPE}" \
|
||||
"${remote_ref}" \
|
||||
"metadata.json:${TEST_GUEST_CACHE_METADATA_TYPE}" \
|
||||
"disk.qcow2.zst:${TEST_GUEST_CACHE_DISK_TYPE}"
|
||||
)
|
||||
published_digest=$(oras resolve "${remote_ref}")
|
||||
rm -rf "${push_dir}"
|
||||
echo "==> Cache push complete: ${remote_ref}"
|
||||
echo "==> Cache immutable reference: ${repository}@${published_digest}"
|
||||
}
|
||||
|
||||
echo "==> Cache key: ${cache_key}"
|
||||
if local_entry_valid; then
|
||||
echo "==> Cache local hit: ${entry_dir}"
|
||||
else
|
||||
if [ -e "${entry_dir}" ]; then
|
||||
rejected="${cache_root}/staging/rejected.${cache_key}.$(date +%s)"
|
||||
mv "${entry_dir}" "${rejected}"
|
||||
echo "Moved invalid cache entry to ${rejected}" >&2
|
||||
fi
|
||||
|
||||
pulled=0
|
||||
if [ -n "${pull_ref}" ]; then
|
||||
if pull_remote; then
|
||||
pulled=1
|
||||
else
|
||||
pull_status=$?
|
||||
if [ "${pull_status}" -ne 1 ]; then
|
||||
exit "${pull_status}"
|
||||
fi
|
||||
echo "==> Cache remote miss: ${pull_ref}"
|
||||
fi
|
||||
fi
|
||||
if [ "${pulled}" -eq 0 ]; then
|
||||
build_local
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${push}" -eq 1 ]; then
|
||||
push_remote
|
||||
fi
|
||||
|
||||
echo "==> Cache ready: ${entry_dir}"
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Pinned cloud images and supported test-guest variants."""
|
||||
|
||||
TEST_GUEST_BASE_IMAGES = [
|
||||
struct(
|
||||
architecture = "aarch64",
|
||||
distro = "ubuntu",
|
||||
downloaded_file_path = "ubuntu-24.04-aarch64.qcow2",
|
||||
integrity = "sha256-meHUgrlY5r/QGDpMSM5twzTgmj4ppFYPb1/4VZPQnR0=",
|
||||
os_id = "ubuntu",
|
||||
os_version = "24.04",
|
||||
package_family = "deb",
|
||||
repo_name = "test_guest_ubuntu_aarch64",
|
||||
url = "https://cloud-images.ubuntu.com/releases/noble/release-20260225/ubuntu-24.04-server-cloudimg-arm64.img",
|
||||
),
|
||||
struct(
|
||||
architecture = "x86_64",
|
||||
distro = "ubuntu",
|
||||
downloaded_file_path = "ubuntu-24.04-x86_64.qcow2",
|
||||
integrity = "sha256-eqbZ9eijpVx0RbE40xpz0Rh4cSEbK32p2i4abL8WmyE=",
|
||||
os_id = "ubuntu",
|
||||
os_version = "24.04",
|
||||
package_family = "deb",
|
||||
repo_name = "test_guest_ubuntu_x86_64",
|
||||
url = "https://cloud-images.ubuntu.com/releases/noble/release-20260225/ubuntu-24.04-server-cloudimg-amd64.img",
|
||||
),
|
||||
struct(
|
||||
architecture = "aarch64",
|
||||
distro = "centos",
|
||||
downloaded_file_path = "centos-stream-10-aarch64.qcow2",
|
||||
integrity = "sha256-55IuyMUvsbpvqgug2S7w6JpLCSIpR4HJVkuMch60Rag=",
|
||||
os_id = "centos",
|
||||
os_version = "10",
|
||||
package_family = "rpm",
|
||||
repo_name = "test_guest_centos_aarch64",
|
||||
url = "https://cloud.centos.org/centos/10-stream/aarch64/images/CentOS-Stream-GenericCloud-10-20260720.0.aarch64.qcow2",
|
||||
),
|
||||
struct(
|
||||
architecture = "x86_64",
|
||||
distro = "centos",
|
||||
downloaded_file_path = "centos-stream-10-x86_64.qcow2",
|
||||
integrity = "sha256-k3lpRd9eVJUr4hyoUGfwyfOxGi3W7iFjFU/ZdIxMQdc=",
|
||||
os_id = "centos",
|
||||
os_version = "10",
|
||||
package_family = "rpm",
|
||||
repo_name = "test_guest_centos_x86_64",
|
||||
url = "https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-20260720.0.x86_64.qcow2",
|
||||
),
|
||||
struct(
|
||||
architecture = "aarch64",
|
||||
distro = "fedora",
|
||||
downloaded_file_path = "fedora-44-aarch64.qcow2",
|
||||
integrity = "sha256-VcYKO4DTYWoIcFr9BFnnX+nwPFSrp6RuQAKkGnL6DVs=",
|
||||
os_id = "fedora",
|
||||
os_version = "44",
|
||||
package_family = "rpm",
|
||||
repo_name = "test_guest_fedora_aarch64",
|
||||
url = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2",
|
||||
),
|
||||
struct(
|
||||
architecture = "x86_64",
|
||||
distro = "fedora",
|
||||
downloaded_file_path = "fedora-44-x86_64.qcow2",
|
||||
integrity = "sha256-KGgP5bNxpaguv0OjGSbghqFo5ZlJ0DlpxQk+cHH5C38=",
|
||||
os_id = "fedora",
|
||||
os_version = "44",
|
||||
package_family = "rpm",
|
||||
repo_name = "test_guest_fedora_x86_64",
|
||||
url = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2",
|
||||
),
|
||||
struct(
|
||||
architecture = "aarch64",
|
||||
distro = "rocky",
|
||||
downloaded_file_path = "rocky-9-aarch64.qcow2",
|
||||
integrity = "sha256-JGkqRE8fC4u5U3XDjItD+AmaEVNHYjaRviwzC0DIof4=",
|
||||
os_id = "rocky",
|
||||
os_version = "9",
|
||||
package_family = "rpm",
|
||||
repo_name = "test_guest_rocky_aarch64",
|
||||
url = "https://download.rockylinux.org/pub/rocky/9/images/aarch64/Rocky-9-GenericCloud-Base-9.8-20260525.0.aarch64.qcow2",
|
||||
),
|
||||
struct(
|
||||
architecture = "x86_64",
|
||||
distro = "rocky",
|
||||
downloaded_file_path = "rocky-9-x86_64.qcow2",
|
||||
integrity = "sha256-ksIGzG95DGFYMkfu/oeJD4goQgZiwXys8kfOx4q07sg=",
|
||||
os_id = "rocky",
|
||||
os_version = "9",
|
||||
package_family = "rpm",
|
||||
repo_name = "test_guest_rocky_x86_64",
|
||||
url = "https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base-9.8-20260525.0.x86_64.qcow2",
|
||||
),
|
||||
]
|
||||
|
||||
TEST_GUEST_VARIANTS = [
|
||||
struct(configurations = [], distro = "ubuntu", name = "ubuntu"),
|
||||
struct(configurations = ["docker"], distro = "ubuntu", name = "ubuntu_docker"),
|
||||
struct(configurations = ["podman"], distro = "ubuntu", name = "ubuntu_podman"),
|
||||
struct(configurations = [], distro = "centos", name = "centos"),
|
||||
struct(configurations = ["podman"], distro = "centos", name = "centos_podman"),
|
||||
struct(configurations = ["selinux"], distro = "centos", name = "centos_selinux"),
|
||||
struct(configurations = ["podman", "selinux"], distro = "centos", name = "centos_podman_selinux"),
|
||||
struct(configurations = [], distro = "fedora", name = "fedora"),
|
||||
struct(configurations = ["podman"], distro = "fedora", name = "fedora_podman"),
|
||||
struct(configurations = ["selinux"], distro = "fedora", name = "fedora_selinux"),
|
||||
struct(configurations = ["podman", "selinux"], distro = "fedora", name = "fedora_podman_selinux"),
|
||||
struct(configurations = [], distro = "rocky", name = "rocky"),
|
||||
struct(configurations = ["docker"], distro = "rocky", name = "rocky_docker"),
|
||||
struct(configurations = ["podman"], distro = "rocky", name = "rocky_podman"),
|
||||
struct(configurations = ["selinux"], distro = "rocky", name = "rocky_selinux"),
|
||||
struct(configurations = ["docker", "selinux"], distro = "rocky", name = "rocky_docker_selinux"),
|
||||
struct(configurations = ["podman", "selinux"], distro = "rocky", name = "rocky_podman_selinux"),
|
||||
]
|
||||
|
||||
def declare_test_guest_targets(base_image_rule, variant_macro):
|
||||
"""Declare base-image providers and the supported prepared-image matrix."""
|
||||
by_distro = {}
|
||||
for image in TEST_GUEST_BASE_IMAGES:
|
||||
target_name = "base_{}_{}".format(image.distro, image.architecture)
|
||||
base_image_rule(
|
||||
name = target_name,
|
||||
architecture = image.architecture,
|
||||
distro = image.distro,
|
||||
image = "@{}//file".format(image.repo_name),
|
||||
integrity = image.integrity,
|
||||
os_id = image.os_id,
|
||||
os_version = image.os_version,
|
||||
package_family = image.package_family,
|
||||
url = image.url,
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
by_distro.setdefault(image.distro, {})[image.architecture] = ":" + target_name
|
||||
|
||||
for variant in TEST_GUEST_VARIANTS:
|
||||
variant_macro(
|
||||
name = variant.name,
|
||||
base_image = select({
|
||||
":aarch64": by_distro[variant.distro]["aarch64"],
|
||||
":x86_64": by_distro[variant.distro]["x86_64"],
|
||||
}),
|
||||
configurations = [":configuration/{}.yml".format(name) for name in variant.configurations],
|
||||
)
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# PROTOTYPE: Composable distro VMs for installing and exercising artifacts.
|
||||
|
||||
{ pkgs }:
|
||||
|
||||
let
|
||||
isAarch64 = pkgs.stdenv.hostPlatform.isAarch64;
|
||||
isDarwin = pkgs.stdenv.hostPlatform.isDarwin;
|
||||
architecture = if isAarch64 then "aarch64" else "x86_64";
|
||||
qemu = pkgs.qemu.override { hostCpuOnly = true; };
|
||||
qemuBinary =
|
||||
if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64";
|
||||
|
||||
distros = {
|
||||
ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; };
|
||||
centos = import ./distros/centos.nix { inherit pkgs architecture; };
|
||||
fedora = import ./distros/fedora.nix { inherit pkgs architecture; };
|
||||
rocky = import ./distros/rocky.nix { inherit pkgs architecture; };
|
||||
};
|
||||
|
||||
configurations = {
|
||||
docker = ./configuration/docker.yml;
|
||||
podman = ./configuration/podman.yml;
|
||||
selinux = ./configuration/selinux.yml;
|
||||
};
|
||||
|
||||
mkDistroProfile =
|
||||
name: distro:
|
||||
pkgs.writeText "openshell-test-guest-${name}" ''
|
||||
TEST_GUEST_IMAGE_DRV=${builtins.unsafeDiscardStringContext distro.image.drvPath}
|
||||
TEST_GUEST_IMAGE_URL=${pkgs.lib.escapeShellArg distro.imageUrl}
|
||||
TEST_GUEST_IMAGE_HASH=${pkgs.lib.escapeShellArg distro.imageHash}
|
||||
TEST_GUEST_OS_ID=${pkgs.lib.escapeShellArg distro.osId}
|
||||
TEST_GUEST_OS_VERSION=${pkgs.lib.escapeShellArg distro.osVersion}
|
||||
TEST_GUEST_PACKAGE_FAMILY=${pkgs.lib.escapeShellArg distro.packageFamily}
|
||||
export TEST_GUEST_IMAGE_DRV TEST_GUEST_IMAGE_URL TEST_GUEST_IMAGE_HASH
|
||||
export TEST_GUEST_OS_ID TEST_GUEST_OS_VERSION TEST_GUEST_PACKAGE_FAMILY
|
||||
'';
|
||||
|
||||
distroCatalog = pkgs.linkFarm "openshell-test-guest-distros" (
|
||||
pkgs.lib.mapAttrsToList (name: distro: {
|
||||
inherit name;
|
||||
path = mkDistroProfile name distro;
|
||||
}) distros
|
||||
);
|
||||
|
||||
configurationCatalog = pkgs.linkFarm "openshell-test-guest-configurations" (
|
||||
pkgs.lib.mapAttrsToList (name: path: { inherit name path; }) configurations
|
||||
);
|
||||
|
||||
runtimeInputs = [
|
||||
qemu
|
||||
pkgs.python3Packages.ansible-core
|
||||
pkgs.python3Packages.virt-firmware
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
pkgs.nix
|
||||
pkgs.openssh
|
||||
pkgs.oras
|
||||
pkgs.python3
|
||||
pkgs.xorriso
|
||||
pkgs.zstd
|
||||
];
|
||||
|
||||
runtimeEnvironment = ''
|
||||
export OPENSHELL_TEST_GUEST_RUNTIME=1
|
||||
export OPENSHELL_TEST_GUEST_DISTROS=${distroCatalog}
|
||||
export OPENSHELL_TEST_GUEST_CONFIGURATIONS=${configurationCatalog}
|
||||
export OPENSHELL_TEST_GUEST_CACHE_LIB=${./cache-lib.sh}
|
||||
export OPENSHELL_TEST_GUEST_CACHE_RUNNER=${./cache.sh}
|
||||
export OPENSHELL_TEST_GUEST_CACHE_SEAL=${./cache-seal.sh}
|
||||
export OPENSHELL_TEST_GUEST_RUNNER=${./run.sh}
|
||||
export TEST_GUEST_BASH=${pkgs.bash}/bin/bash
|
||||
export TEST_GUEST_QEMU=${qemuBinary}
|
||||
export TEST_GUEST_FIRMWARE_CODE=${pkgs.OVMF.firmware}
|
||||
export TEST_GUEST_FIRMWARE_VARS=${pkgs.OVMF.variables}
|
||||
export TEST_GUEST_MACHINE=${if isAarch64 then "virt" else "q35"}
|
||||
export TEST_GUEST_ACCELERATOR=${if isDarwin then "hvf" else "kvm"}
|
||||
export TEST_GUEST_ARCHITECTURE=${architecture}
|
||||
export TEST_GUEST_ANSIBLE_VERSION=${pkgs.python3Packages.ansible-core.version}
|
||||
export TEST_GUEST_CACHE_GENERATION=1
|
||||
'';
|
||||
|
||||
runner = pkgs.writeShellApplication {
|
||||
name = "openshell-test-guest";
|
||||
inherit runtimeInputs;
|
||||
text = runtimeEnvironment + ''
|
||||
exec ${pkgs.bash}/bin/bash ${./run.sh} "$@"
|
||||
'';
|
||||
};
|
||||
|
||||
cacheRunner = pkgs.writeShellApplication {
|
||||
name = "openshell-test-guest-cache";
|
||||
inherit runtimeInputs;
|
||||
text = runtimeEnvironment + ''
|
||||
exec ${pkgs.bash}/bin/bash ${./cache.sh} "$@"
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
app = {
|
||||
type = "app";
|
||||
program = "${runner}/bin/openshell-test-guest";
|
||||
meta.description = "Boot and configure a disposable distro guest";
|
||||
};
|
||||
|
||||
cacheApp = {
|
||||
type = "app";
|
||||
program = "${cacheRunner}/bin/openshell-test-guest-cache";
|
||||
meta.description = "Ensure a prepared test guest disk is available locally or in OCI";
|
||||
};
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ pkgs, architecture }:
|
||||
|
||||
let
|
||||
imageUrl = "https://cloud.centos.org/centos/10-stream/${architecture}/images/CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2";
|
||||
imageHash =
|
||||
if architecture == "aarch64" then
|
||||
"sha256-55IuyMUvsbpvqgug2S7w6JpLCSIpR4HJVkuMch60Rag="
|
||||
else
|
||||
"sha256-k3lpRd9eVJUr4hyoUGfwyfOxGi3W7iFjFU/ZdIxMQdc=";
|
||||
in
|
||||
{
|
||||
osId = "centos";
|
||||
osVersion = "10";
|
||||
packageFamily = "rpm";
|
||||
inherit imageUrl imageHash;
|
||||
image = pkgs.fetchurl {
|
||||
name = "CentOS-Stream-GenericCloud-10-20260720.0.${architecture}.qcow2";
|
||||
url = imageUrl;
|
||||
hash = imageHash;
|
||||
};
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ pkgs, architecture }:
|
||||
|
||||
let
|
||||
imageUrl = "https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/${architecture}/images/Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2";
|
||||
imageHash =
|
||||
if architecture == "aarch64" then
|
||||
"sha256-VcYKO4DTYWoIcFr9BFnnX+nwPFSrp6RuQAKkGnL6DVs="
|
||||
else
|
||||
"sha256-KGgP5bNxpaguv0OjGSbghqFo5ZlJ0DlpxQk+cHH5C38=";
|
||||
in
|
||||
{
|
||||
osId = "fedora";
|
||||
osVersion = "44";
|
||||
packageFamily = "rpm";
|
||||
inherit imageUrl imageHash;
|
||||
image = pkgs.fetchurl {
|
||||
name = "Fedora-Cloud-Base-Generic-44-1.7.${architecture}.qcow2";
|
||||
url = imageUrl;
|
||||
hash = imageHash;
|
||||
};
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ pkgs, architecture }:
|
||||
|
||||
let
|
||||
imageUrl = "https://download.rockylinux.org/pub/rocky/9/images/${architecture}/Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2";
|
||||
imageHash =
|
||||
if architecture == "aarch64" then
|
||||
"sha256-JGkqRE8fC4u5U3XDjItD+AmaEVNHYjaRviwzC0DIof4="
|
||||
else
|
||||
"sha256-ksIGzG95DGFYMkfu/oeJD4goQgZiwXys8kfOx4q07sg=";
|
||||
in
|
||||
{
|
||||
osId = "rocky";
|
||||
osVersion = "9";
|
||||
packageFamily = "rpm";
|
||||
inherit imageUrl imageHash;
|
||||
image = pkgs.fetchurl {
|
||||
name = "Rocky-9-GenericCloud-Base-9.8-20260525.0.${architecture}.qcow2";
|
||||
url = imageUrl;
|
||||
hash = imageHash;
|
||||
};
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ pkgs, architecture }:
|
||||
|
||||
let
|
||||
imageArchitecture = if architecture == "aarch64" then "arm64" else "amd64";
|
||||
imageUrl = "https://cloud-images.ubuntu.com/releases/noble/release-20260225/ubuntu-24.04-server-cloudimg-${imageArchitecture}.img";
|
||||
imageHash =
|
||||
if architecture == "aarch64" then
|
||||
"sha256-meHUgrlY5r/QGDpMSM5twzTgmj4ppFYPb1/4VZPQnR0="
|
||||
else
|
||||
"sha256-eqbZ9eijpVx0RbE40xpz0Rh4cSEbK32p2i4abL8WmyE=";
|
||||
in
|
||||
{
|
||||
osId = "ubuntu";
|
||||
osVersion = "24.04";
|
||||
packageFamily = "deb";
|
||||
inherit imageUrl imageHash;
|
||||
image = pkgs.fetchurl {
|
||||
name = "ubuntu-24.04-server-cloudimg-${imageArchitecture}.img";
|
||||
url = imageUrl;
|
||||
hash = imageHash;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Bzlmod extensions for pinned test-guest inputs."""
|
||||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
|
||||
load(":catalog.bzl", "TEST_GUEST_BASE_IMAGES")
|
||||
|
||||
def _test_guest_images_impl(module_ctx):
|
||||
for image in TEST_GUEST_BASE_IMAGES:
|
||||
http_file(
|
||||
name = image.repo_name,
|
||||
downloaded_file_path = image.downloaded_file_path,
|
||||
integrity = image.integrity,
|
||||
urls = [image.url],
|
||||
)
|
||||
|
||||
return module_ctx.extension_metadata(
|
||||
reproducible = True,
|
||||
root_module_direct_deps = "all",
|
||||
root_module_direct_dev_deps = [],
|
||||
)
|
||||
|
||||
test_guest_images = module_extension(implementation = _test_guest_images_impl)
|
||||
@@ -0,0 +1,57 @@
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
exports_files([
|
||||
"bin/bash",
|
||||
"bin/jq",
|
||||
"bin/openshell-test-guest",
|
||||
"bin/qemu-img",
|
||||
"bin/sha256sum",
|
||||
"versions/ansible",
|
||||
"versions/qemu",
|
||||
])
|
||||
|
||||
filegroup(
|
||||
name = "all",
|
||||
srcs = glob(
|
||||
["**"],
|
||||
exclude = [
|
||||
"BUILD",
|
||||
"bazel-support/**",
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "ansible_version",
|
||||
srcs = ["versions/ansible"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "bash",
|
||||
srcs = ["bin/bash"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "jq",
|
||||
srcs = ["bin/jq"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "qemu_img",
|
||||
srcs = ["bin/qemu-img"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "qemu_version",
|
||||
srcs = ["versions/qemu"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "runner",
|
||||
srcs = ["bin/openshell-test-guest"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "sha256sum",
|
||||
srcs = ["bin/sha256sum"],
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
|
||||
let
|
||||
isAarch64 = pkgs.stdenv.hostPlatform.isAarch64;
|
||||
isDarwin = pkgs.stdenv.hostPlatform.isDarwin;
|
||||
architecture = if isAarch64 then "aarch64" else "x86_64";
|
||||
qemu = pkgs.qemu.override { hostCpuOnly = true; };
|
||||
qemuBinary =
|
||||
if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64";
|
||||
|
||||
runner = pkgs.writeShellApplication {
|
||||
name = "openshell-test-guest";
|
||||
runtimeInputs = [
|
||||
qemu
|
||||
pkgs.python3Packages.ansible-core
|
||||
pkgs.python3Packages.virt-firmware
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
pkgs.openssh
|
||||
pkgs.python3
|
||||
pkgs.xorriso
|
||||
];
|
||||
text = ''
|
||||
export OPENSHELL_TEST_GUEST_RUNTIME=1
|
||||
export TEST_GUEST_QEMU=${qemuBinary}
|
||||
export TEST_GUEST_FIRMWARE_CODE=${pkgs.OVMF.firmware}
|
||||
export TEST_GUEST_FIRMWARE_VARS=${pkgs.OVMF.variables}
|
||||
export TEST_GUEST_MACHINE=${if isAarch64 then "virt" else "q35"}
|
||||
export TEST_GUEST_ACCELERATOR=${if isDarwin then "hvf" else "kvm"}
|
||||
export TEST_GUEST_ARCHITECTURE=${architecture}
|
||||
exec ${pkgs.bash}/bin/bash ${./run.sh} "$@"
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
bazelHostTools = pkgs.runCommand "openshell-test-guest-bazel-host-tools" { } ''
|
||||
mkdir -p "$out/bin" "$out/firmware" "$out/share" "$out/versions"
|
||||
for tool in ${pkgs.coreutils}/bin/*; do
|
||||
ln -s "$tool" "$out/bin/''${tool##*/}"
|
||||
done
|
||||
ln -s ${runner}/bin/openshell-test-guest "$out/bin/openshell-test-guest"
|
||||
ln -s ${pkgs.bash}/bin/bash "$out/bin/bash"
|
||||
ln -s ${pkgs.jq}/bin/jq "$out/bin/jq"
|
||||
ln -s ${qemu}/bin/qemu-img "$out/bin/qemu-img"
|
||||
ln -s ${qemuBinary} "$out/bin/qemu-system"
|
||||
ln -s ${qemu}/share/qemu "$out/share/qemu"
|
||||
ln -s ${pkgs.OVMF.firmware} "$out/firmware/code.fd"
|
||||
ln -s ${pkgs.OVMF.variables} "$out/firmware/vars.fd"
|
||||
printf '%s\n' ${pkgs.lib.escapeShellArg qemu.version} > "$out/versions/qemu"
|
||||
printf '%s\n' ${pkgs.lib.escapeShellArg pkgs.python3Packages.ansible-core.version} > "$out/versions/ansible"
|
||||
'';
|
||||
}
|
||||
@@ -2,12 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Sanitize a configured test guest before its disk is published as a cache base.
|
||||
# Sanitize a configured test guest before capturing its prepared image.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "cache sealing must run as root" >&2
|
||||
echo "image sealing must run as root" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -47,8 +47,8 @@ rm -f -- "$0"
|
||||
sync
|
||||
|
||||
# Deleted credentials can remain in allocated blocks. Fill free space with
|
||||
# zeroes so qemu-img convert can safely omit those blocks from the cache disk.
|
||||
zero_file=/var/tmp/openshell-cache-zero
|
||||
# zeroes so qemu-img convert can omit those blocks from the prepared disk.
|
||||
zero_file=/var/tmp/openshell-image-zero
|
||||
dd if=/dev/zero of="${zero_file}" bs=64M status=none 2>/dev/null || true
|
||||
rm -f "${zero_file}"
|
||||
sync
|
||||
@@ -0,0 +1,18 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Reconstruct the repository's pinned Nixpkgs input without evaluating the root
|
||||
# flake. rules_nixpkgs copies this file and flake.lock together.
|
||||
let
|
||||
lock = builtins.fromJSON (builtins.readFile ../../flake.lock);
|
||||
root = lock.nodes.${lock.root};
|
||||
inputName = root.inputs.nixpkgs;
|
||||
source = lock.nodes.${inputName}.locked;
|
||||
nixpkgs =
|
||||
assert source.type == "github";
|
||||
builtins.fetchTarball {
|
||||
url = "https://github.com/${source.owner}/${source.repo}/archive/${source.rev}.tar.gz";
|
||||
sha256 = source.narHash;
|
||||
};
|
||||
in
|
||||
import nixpkgs
|
||||
+76
-73
@@ -9,30 +9,23 @@ set -Eeuo pipefail
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
nix run .#test-guest -- --distro DISTRO [OPTIONS] [-- COMMAND...]
|
||||
bazel run //nix/test-guest:VARIANT -- [OPTIONS] [-- COMMAND...]
|
||||
|
||||
Options:
|
||||
--distro NAME Base distro: ubuntu, centos, fedora, or rocky
|
||||
--with NAME Apply a configuration; repeatable (docker, podman, selinux)
|
||||
--install PATH Install a .deb or .rpm package; repeatable
|
||||
--copy SRC:DEST Copy an executable to an absolute guest path; repeatable
|
||||
--ssh-port PORT Use a specific loopback SSH forwarding port
|
||||
--forward-port HOST_PORT:GUEST_PORT
|
||||
Forward a loopback host port to a guest port; repeatable
|
||||
--keep Keep the disposable disk and logs after shutdown
|
||||
--list List distros and configurations
|
||||
-h, --help Show this help
|
||||
|
||||
With no COMMAND, the runner opens an interactive SSH session.
|
||||
EOF
|
||||
}
|
||||
|
||||
if [ "${OPENSHELL_TEST_GUEST_RUNTIME:-}" != 1 ] ||
|
||||
[ ! -d "${OPENSHELL_TEST_GUEST_DISTROS:-}" ] ||
|
||||
[ ! -d "${OPENSHELL_TEST_GUEST_CONFIGURATIONS:-}" ] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_CACHE_LIB:-}" ] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_CACHE_RUNNER:-}" ]; then
|
||||
echo "run this script through 'nix run .#test-guest -- ...'" >&2
|
||||
if [ "${OPENSHELL_TEST_GUEST_RUNTIME:-}" != 1 ]; then
|
||||
echo "run this script through a //nix/test-guest Bazel target" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -44,10 +37,13 @@ require_value() {
|
||||
}
|
||||
|
||||
distro=
|
||||
os_id=
|
||||
os_version=
|
||||
package_family=
|
||||
requested_ssh_port=
|
||||
keep=0
|
||||
list=0
|
||||
configurations=()
|
||||
configuration_paths=()
|
||||
packages=()
|
||||
copies=()
|
||||
forward_ports=()
|
||||
@@ -60,11 +56,36 @@ while [ "$#" -gt 0 ]; do
|
||||
distro=$2
|
||||
shift 2
|
||||
;;
|
||||
--os-id)
|
||||
require_value "$@"
|
||||
os_id=$2
|
||||
shift 2
|
||||
;;
|
||||
--os-version)
|
||||
require_value "$@"
|
||||
os_version=$2
|
||||
shift 2
|
||||
;;
|
||||
--package-family)
|
||||
require_value "$@"
|
||||
package_family=$2
|
||||
shift 2
|
||||
;;
|
||||
--with)
|
||||
require_value "$@"
|
||||
configurations+=("$2")
|
||||
configuration_paths+=("")
|
||||
shift 2
|
||||
;;
|
||||
--configuration-file)
|
||||
if [ "$#" -lt 3 ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then
|
||||
echo "--configuration-file requires NAME PATH" >&2
|
||||
exit 2
|
||||
fi
|
||||
configurations+=("$2")
|
||||
configuration_paths+=("$3")
|
||||
shift 3
|
||||
;;
|
||||
--install)
|
||||
require_value "$@"
|
||||
packages+=("$2")
|
||||
@@ -92,10 +113,6 @@ while [ "$#" -gt 0 ]; do
|
||||
keep=1
|
||||
shift
|
||||
;;
|
||||
--list)
|
||||
list=1
|
||||
shift
|
||||
;;
|
||||
-h | --help)
|
||||
usage
|
||||
exit 0
|
||||
@@ -113,35 +130,34 @@ while [ "$#" -gt 0 ]; do
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${list}" -eq 1 ]; then
|
||||
echo "Distros:"
|
||||
for entry in "${OPENSHELL_TEST_GUEST_DISTROS}"/*; do
|
||||
printf ' %s\n' "${entry##*/}"
|
||||
done
|
||||
echo "Configurations:"
|
||||
for entry in "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}"/*; do
|
||||
printf ' %s\n' "${entry##*/}"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -z "${distro}" ]; then
|
||||
echo "--distro is required" >&2
|
||||
usage >&2
|
||||
echo "Bazel launcher did not supply --distro" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}" ]; then
|
||||
if [[ ! ${distro} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then
|
||||
echo "unknown distro: ${distro}" >&2
|
||||
exit 2
|
||||
fi
|
||||
# Distro profiles contain only trusted values generated into the Nix store.
|
||||
# shellcheck disable=SC1090
|
||||
. "${OPENSHELL_TEST_GUEST_DISTROS}/${distro}"
|
||||
if [[ ! ${os_id} =~ ^[a-z0-9][a-z0-9._-]*$ ]]; then
|
||||
echo "Bazel launcher supplied an invalid OS ID: ${os_id:-<empty>}" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! ${os_version} =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
|
||||
echo "Bazel launcher supplied an invalid OS version: ${os_version:-<empty>}" >&2
|
||||
exit 2
|
||||
fi
|
||||
case "${package_family}" in
|
||||
deb | rpm) ;;
|
||||
*)
|
||||
echo "Bazel launcher supplied an invalid package family: ${package_family:-<empty>}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
for item in "${configurations[@]}"; do
|
||||
for index in "${!configurations[@]}"; do
|
||||
item=${configurations[index]}
|
||||
if [[ ! ${item} =~ ^[a-z0-9][a-z0-9-]*$ ]] ||
|
||||
[ ! -r "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}/${item}" ]; then
|
||||
{ [ -n "${configuration_paths[index]}" ] && [ ! -r "${configuration_paths[index]}" ]; }; then
|
||||
echo "unknown configuration: ${item:-<empty>}" >&2
|
||||
exit 2
|
||||
fi
|
||||
@@ -210,10 +226,10 @@ for package in "${packages[@]}"; do
|
||||
echo "package does not exist: ${package_input}" >&2
|
||||
exit 2
|
||||
fi
|
||||
case "${TEST_GUEST_PACKAGE_FAMILY}:${package}" in
|
||||
case "${package_family}:${package}" in
|
||||
deb:*.deb | rpm:*.rpm) ;;
|
||||
*)
|
||||
echo "${package} does not match the ${TEST_GUEST_PACKAGE_FAMILY} package family" >&2
|
||||
echo "${package} does not match the ${package_family} package family" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -261,9 +277,6 @@ if [ "${TEST_GUEST_ACCELERATOR}" = kvm ] &&
|
||||
ssh_wait_seconds=600
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
. "${OPENSHELL_TEST_GUEST_CACHE_LIB}"
|
||||
|
||||
report_timing() {
|
||||
local label=$1
|
||||
local started_at=$2
|
||||
@@ -282,36 +295,25 @@ if [ -n "${OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE:-}" ]; then
|
||||
TEST_GUEST_IMAGE=${OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE}
|
||||
prepared_image=1
|
||||
echo "==> Using explicit prepared guest image"
|
||||
elif [ "${OPENSHELL_TEST_GUEST_CACHE_DISABLE:-0}" -ne 1 ]; then
|
||||
cache_root=$(test_vm_cache_root)
|
||||
cache_key=$(test_vm_cache_key "${distro}" "${configurations[@]}")
|
||||
cache_entry=$(test_vm_cache_entry_dir "${cache_root}" "${cache_key}")
|
||||
if ! test_vm_cache_local_entry_valid \
|
||||
"${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}"; then
|
||||
cache_args=(--distro "${distro}")
|
||||
for item in "${configurations[@]}"; do
|
||||
cache_args+=(--with "${item}")
|
||||
done
|
||||
echo "==> Cache local miss: populating ${cache_entry}"
|
||||
OPENSHELL_TEST_GUEST_CACHE_DISABLE=1 \
|
||||
"${TEST_GUEST_BASH}" "${OPENSHELL_TEST_GUEST_CACHE_RUNNER}" \
|
||||
"${cache_args[@]}"
|
||||
if ! test_vm_cache_local_entry_valid \
|
||||
"${cache_root}" "${cache_key}" "${distro}" "${configurations[@]}"; then
|
||||
echo "cache builder did not produce a valid entry: ${cache_entry}" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "==> Cache populated: ${cache_entry}"
|
||||
else
|
||||
echo "==> Cache local hit: ${cache_entry}"
|
||||
elif [ -n "${OPENSHELL_TEST_GUEST_BASE_IMAGE_OVERRIDE:-}" ]; then
|
||||
if [ ! -f "${OPENSHELL_TEST_GUEST_BASE_IMAGE_OVERRIDE}" ]; then
|
||||
echo "base guest image does not exist: ${OPENSHELL_TEST_GUEST_BASE_IMAGE_OVERRIDE}" >&2
|
||||
exit 2
|
||||
fi
|
||||
TEST_GUEST_IMAGE="${cache_entry}/disk.qcow2"
|
||||
prepared_image=1
|
||||
TEST_GUEST_IMAGE=${OPENSHELL_TEST_GUEST_BASE_IMAGE_OVERRIDE}
|
||||
echo "==> Using explicit base guest image"
|
||||
else
|
||||
echo "Bazel launcher did not supply a prepared or base guest image" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ "${prepared_image}" -eq 0 ]; then
|
||||
echo "==> Realizing the pinned ${distro} cloud image"
|
||||
TEST_GUEST_IMAGE=$(nix build --no-link --print-out-paths "${TEST_GUEST_IMAGE_DRV}^out")
|
||||
for index in "${!configurations[@]}"; do
|
||||
if [ -z "${configuration_paths[index]}" ]; then
|
||||
echo "missing declared playbook for configuration: ${configurations[index]}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
fi
|
||||
report_timing "guest image resolution" "${phase_started_at}"
|
||||
|
||||
@@ -549,10 +551,10 @@ report_timing "VM boot and SSH" "${phase_started_at}"
|
||||
|
||||
phase_started_at=${SECONDS}
|
||||
echo "==> Validating ${distro}"
|
||||
# Profile values come from the trusted Nix-generated catalog.
|
||||
# Profile values come from the Bazel base-image provider.
|
||||
# shellcheck disable=SC2029
|
||||
ssh "${ssh_args[@]}" openshell@127.0.0.1 \
|
||||
"set -eu; set +e; sudo cloud-init status --wait >/dev/null; status=\$?; set -e; [ \"\${status}\" -eq 0 ] || [ \"\${status}\" -eq 2 ]; . /etc/os-release; test \"\${ID}\" = '${TEST_GUEST_OS_ID}'; case \"\${VERSION_ID}\" in '${TEST_GUEST_OS_VERSION}'*) ;; *) exit 1 ;; esac; test \"\$(uname -m)\" = '${TEST_GUEST_ARCHITECTURE}'"
|
||||
"set -eu; set +e; sudo cloud-init status --wait >/dev/null; status=\$?; set -e; [ \"\${status}\" -eq 0 ] || [ \"\${status}\" -eq 2 ]; . /etc/os-release; test \"\${ID}\" = '${os_id}'; case \"\${VERSION_ID}\" in '${os_version}'*) ;; *) exit 1 ;; esac; test \"\$(uname -m)\" = '${TEST_GUEST_ARCHITECTURE}'"
|
||||
# cloud-init returns 2 when it completes with recoverable errors. Fedora can
|
||||
# report that status for an initial transient-hostname warning even though the
|
||||
# requested user and SSH configuration were applied successfully.
|
||||
@@ -574,10 +576,11 @@ guest ansible_host=127.0.0.1 ansible_port=${ssh_port} ansible_user=openshell ans
|
||||
EOF
|
||||
|
||||
if [ "${prepared_image}" -eq 0 ]; then
|
||||
for item in "${configurations[@]}"; do
|
||||
for index in "${!configurations[@]}"; do
|
||||
item=${configurations[index]}
|
||||
echo "==> Applying configuration: ${item}"
|
||||
ANSIBLE_CONFIG="${ansible_config}" ANSIBLE_NOCOLOR=1 \
|
||||
ansible-playbook "${OPENSHELL_TEST_GUEST_CONFIGURATIONS}/${item}"
|
||||
ansible-playbook "${configuration_paths[index]}"
|
||||
done
|
||||
else
|
||||
echo "==> Reusing cached configuration: ${configurations[*]:-base image}"
|
||||
@@ -592,7 +595,7 @@ if [ "${#packages[@]}" -gt 0 ] || [ "${#copies[@]}" -gt 0 ]; then
|
||||
remote_packages=()
|
||||
artifact_index=0
|
||||
for package in "${packages[@]}"; do
|
||||
remote_path=${artifact_staging_dir}/package-${artifact_index}.${TEST_GUEST_PACKAGE_FAMILY}
|
||||
remote_path=${artifact_staging_dir}/package-${artifact_index}.${package_family}
|
||||
echo "==> Copying package: ${package##*/}"
|
||||
scp -q "${scp_args[@]}" \
|
||||
"${package}" "openshell@127.0.0.1:${remote_path}"
|
||||
@@ -602,7 +605,7 @@ if [ "${#packages[@]}" -gt 0 ] || [ "${#copies[@]}" -gt 0 ]; then
|
||||
|
||||
if [ "${#remote_packages[@]}" -gt 0 ]; then
|
||||
printf -v quoted_packages ' %q' "${remote_packages[@]}"
|
||||
case "${TEST_GUEST_PACKAGE_FAMILY}" in
|
||||
case "${package_family}" in
|
||||
deb)
|
||||
ssh "${ssh_args[@]}" openshell@127.0.0.1 \
|
||||
"sudo apt-get update >/dev/null && sudo apt-get install -y --${quoted_packages}"
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Bzlmod dependencies and repositories for test-guest."""
|
||||
|
||||
bazel_dep(name = "rules_nixpkgs_core", version = "0.13.0")
|
||||
bazel_dep(name = "rules_shell", version = "0.6.1")
|
||||
|
||||
nix_repo = use_extension(
|
||||
"@rules_nixpkgs_core//extensions:repository.bzl",
|
||||
"nix_repo",
|
||||
)
|
||||
nix_repo.file(
|
||||
name = "openshell_nixpkgs",
|
||||
file = "//nix/test-guest:nixpkgs.nix",
|
||||
file_deps = ["//:flake.lock"],
|
||||
)
|
||||
use_repo(nix_repo, "openshell_nixpkgs")
|
||||
|
||||
nix_pkg = use_extension(
|
||||
"@rules_nixpkgs_core//extensions:package.bzl",
|
||||
"nix_pkg",
|
||||
)
|
||||
nix_pkg.file(
|
||||
name = "test_guest_host_tools",
|
||||
attr = "bazelHostTools",
|
||||
build_file = "//nix/test-guest:host-tools.BUILD.bazel",
|
||||
file = "//nix/test-guest:host-tools.nix",
|
||||
file_deps = [
|
||||
"//nix/test-guest:run.sh",
|
||||
],
|
||||
repo = "@openshell_nixpkgs",
|
||||
)
|
||||
use_repo(nix_pkg, "test_guest_host_tools")
|
||||
|
||||
test_guest_images = use_extension("//nix/test-guest:extensions.bzl", "test_guest_images")
|
||||
use_repo(
|
||||
test_guest_images,
|
||||
"test_guest_centos_aarch64",
|
||||
"test_guest_centos_x86_64",
|
||||
"test_guest_fedora_aarch64",
|
||||
"test_guest_fedora_x86_64",
|
||||
"test_guest_rocky_aarch64",
|
||||
"test_guest_rocky_x86_64",
|
||||
"test_guest_ubuntu_aarch64",
|
||||
"test_guest_ubuntu_x86_64",
|
||||
)
|
||||
|
||||
register_toolchains("//nix/test-guest:nix_host_tools_toolchain")
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Bazel rules for building and consuming prepared test-guest QCOW2 images."""
|
||||
|
||||
load("@rules_shell//shell:sh_binary.bzl", "sh_binary")
|
||||
load("@rules_shell//shell:sh_test.bzl", "sh_test")
|
||||
|
||||
_HOST_TOOLS_TOOLCHAIN = "//nix/test-guest:host_tools_toolchain_type"
|
||||
|
||||
TestGuestBaseImageInfo = provider(
|
||||
fields = {
|
||||
"architecture": "Guest architecture.",
|
||||
"distro": "Guest distro catalog name.",
|
||||
"image": "Pinned base cloud image.",
|
||||
"integrity": "Subresource integrity of the base image.",
|
||||
"os_id": "Expected guest operating-system ID.",
|
||||
"os_version": "Expected guest operating-system version.",
|
||||
"package_family": "Guest package family: deb or rpm.",
|
||||
"url": "Provenance URL of the base image.",
|
||||
},
|
||||
)
|
||||
|
||||
TestGuestImageInfo = provider(
|
||||
fields = {
|
||||
"architecture": "Guest architecture.",
|
||||
"configurations": "Ordered image configuration names.",
|
||||
"disk": "Flattened standalone QCOW2 image.",
|
||||
"distro": "Guest distro catalog name.",
|
||||
"metadata": "Image compatibility and provenance metadata.",
|
||||
"os_id": "Expected guest operating-system ID.",
|
||||
"os_version": "Expected guest operating-system version.",
|
||||
"package_family": "Guest package family: deb or rpm.",
|
||||
},
|
||||
)
|
||||
|
||||
def _test_guest_base_image_impl(ctx):
|
||||
return [
|
||||
DefaultInfo(files = depset([ctx.file.image])),
|
||||
TestGuestBaseImageInfo(
|
||||
architecture = ctx.attr.architecture,
|
||||
distro = ctx.attr.distro,
|
||||
image = ctx.file.image,
|
||||
integrity = ctx.attr.integrity,
|
||||
os_id = ctx.attr.os_id,
|
||||
os_version = ctx.attr.os_version,
|
||||
package_family = ctx.attr.package_family,
|
||||
url = ctx.attr.url,
|
||||
),
|
||||
]
|
||||
|
||||
test_guest_base_image = rule(
|
||||
implementation = _test_guest_base_image_impl,
|
||||
attrs = {
|
||||
"architecture": attr.string(mandatory = True, values = ["aarch64", "x86_64"]),
|
||||
"distro": attr.string(mandatory = True),
|
||||
"image": attr.label(allow_single_file = True, mandatory = True),
|
||||
"integrity": attr.string(mandatory = True),
|
||||
"os_id": attr.string(mandatory = True),
|
||||
"os_version": attr.string(mandatory = True),
|
||||
"package_family": attr.string(mandatory = True, values = ["deb", "rpm"]),
|
||||
"url": attr.string(mandatory = True),
|
||||
},
|
||||
)
|
||||
|
||||
def _test_guest_host_tools_impl(ctx):
|
||||
return [platform_common.ToolchainInfo(
|
||||
all_files = ctx.attr.all_files[DefaultInfo].files,
|
||||
ansible_version = ctx.file.ansible_version,
|
||||
bash = ctx.file.bash,
|
||||
jq = ctx.file.jq,
|
||||
qemu_img = ctx.file.qemu_img,
|
||||
qemu_version = ctx.file.qemu_version,
|
||||
runner = ctx.file.runner,
|
||||
sha256sum = ctx.file.sha256sum,
|
||||
)]
|
||||
|
||||
test_guest_host_tools = rule(
|
||||
implementation = _test_guest_host_tools_impl,
|
||||
attrs = {
|
||||
"all_files": attr.label(mandatory = True),
|
||||
"ansible_version": attr.label(allow_single_file = True, mandatory = True),
|
||||
"bash": attr.label(allow_single_file = True, mandatory = True),
|
||||
"jq": attr.label(allow_single_file = True, mandatory = True),
|
||||
"qemu_img": attr.label(allow_single_file = True, mandatory = True),
|
||||
"qemu_version": attr.label(allow_single_file = True, mandatory = True),
|
||||
"runner": attr.label(allow_single_file = True, mandatory = True),
|
||||
"sha256sum": attr.label(allow_single_file = True, mandatory = True),
|
||||
},
|
||||
)
|
||||
|
||||
def _test_guest_image_impl(ctx):
|
||||
if len(ctx.attr.configuration_names) != len(ctx.files.configurations):
|
||||
fail("configuration_names and configurations must have the same length")
|
||||
|
||||
tools = ctx.toolchains[_HOST_TOOLS_TOOLCHAIN]
|
||||
base = ctx.attr.base_image[TestGuestBaseImageInfo]
|
||||
disk = ctx.actions.declare_file(ctx.label.name + ".qcow2")
|
||||
metadata = ctx.actions.declare_file(ctx.label.name + ".metadata.json")
|
||||
|
||||
args = ctx.actions.args()
|
||||
args.add(ctx.file._builder.path)
|
||||
args.add("--runner", tools.runner.path)
|
||||
args.add("--qemu-img", tools.qemu_img.path)
|
||||
args.add("--jq", tools.jq.path)
|
||||
args.add("--sha256sum", tools.sha256sum.path)
|
||||
args.add("--seal", ctx.file._seal.path)
|
||||
args.add("--base-image", base.image.path)
|
||||
args.add("--output", disk.path)
|
||||
args.add("--metadata-output", metadata.path)
|
||||
args.add("--distro", base.distro)
|
||||
args.add("--os-id", base.os_id)
|
||||
args.add("--os-version", base.os_version)
|
||||
args.add("--package-family", base.package_family)
|
||||
args.add("--architecture", base.architecture)
|
||||
args.add("--base-image-url", base.url)
|
||||
args.add("--base-image-hash", base.integrity)
|
||||
args.add("--generation", ctx.attr.generation)
|
||||
args.add("--qemu-version-file", tools.qemu_version.path)
|
||||
args.add("--ansible-version-file", tools.ansible_version.path)
|
||||
for index in range(len(ctx.attr.configuration_names)):
|
||||
args.add("--configuration")
|
||||
args.add(ctx.attr.configuration_names[index])
|
||||
args.add(ctx.files.configurations[index].path)
|
||||
|
||||
ctx.actions.run(
|
||||
arguments = [args],
|
||||
executable = tools.bash,
|
||||
env = {"PATH": tools.bash.dirname},
|
||||
execution_requirements = {
|
||||
"no-remote-exec": "1",
|
||||
"no-sandbox": "1",
|
||||
"requires-network": "1",
|
||||
},
|
||||
inputs = depset(
|
||||
direct = [
|
||||
ctx.file._builder,
|
||||
ctx.file._seal,
|
||||
base.image,
|
||||
tools.ansible_version,
|
||||
tools.qemu_version,
|
||||
] + ctx.files.configurations,
|
||||
transitive = [tools.all_files],
|
||||
),
|
||||
mnemonic = "TestGuestImage",
|
||||
outputs = [disk, metadata],
|
||||
progress_message = "Building prepared test guest image %{label}",
|
||||
tools = tools.all_files,
|
||||
)
|
||||
|
||||
return [
|
||||
DefaultInfo(files = depset([disk, metadata])),
|
||||
TestGuestImageInfo(
|
||||
architecture = base.architecture,
|
||||
configurations = ctx.attr.configuration_names,
|
||||
disk = disk,
|
||||
distro = base.distro,
|
||||
metadata = metadata,
|
||||
os_id = base.os_id,
|
||||
os_version = base.os_version,
|
||||
package_family = base.package_family,
|
||||
),
|
||||
]
|
||||
|
||||
test_guest_image = rule(
|
||||
implementation = _test_guest_image_impl,
|
||||
attrs = {
|
||||
"base_image": attr.label(mandatory = True, providers = [TestGuestBaseImageInfo]),
|
||||
"configuration_names": attr.string_list(),
|
||||
"configurations": attr.label_list(allow_files = [".yml"]),
|
||||
"generation": attr.int(default = 1),
|
||||
"_builder": attr.label(
|
||||
allow_single_file = True,
|
||||
default = "//nix/test-guest:build-image.sh",
|
||||
),
|
||||
"_seal": attr.label(
|
||||
allow_single_file = True,
|
||||
default = "//nix/test-guest:image-seal.sh",
|
||||
),
|
||||
},
|
||||
toolchains = [_HOST_TOOLS_TOOLCHAIN],
|
||||
)
|
||||
|
||||
def _shell_quote(value):
|
||||
return "'{}'".format(value.replace("'", "'\"'\"'"))
|
||||
|
||||
def _rlocation_path(ctx, file):
|
||||
if file.short_path.startswith("../"):
|
||||
return file.short_path[3:]
|
||||
return ctx.workspace_name + "/" + file.short_path
|
||||
|
||||
def _launcher_content(ctx, image, tools, fixed_args, is_test):
|
||||
base_args = [
|
||||
"--distro",
|
||||
image.distro,
|
||||
"--os-id",
|
||||
image.os_id,
|
||||
"--os-version",
|
||||
image.os_version,
|
||||
"--package-family",
|
||||
image.package_family,
|
||||
]
|
||||
for configuration in image.configurations:
|
||||
base_args.extend(["--with", configuration])
|
||||
|
||||
package_files = []
|
||||
for target in ctx.attr.packages:
|
||||
files = target[DefaultInfo].files.to_list()
|
||||
if len(files) != 1:
|
||||
fail("package targets must provide exactly one file: {}".format(target.label))
|
||||
package_files.append(files[0])
|
||||
|
||||
copy_files = []
|
||||
for target, destination in ctx.attr.copies.items():
|
||||
files = target[DefaultInfo].files.to_list()
|
||||
if len(files) != 1:
|
||||
fail("copy targets must provide exactly one file: {}".format(target.label))
|
||||
copy_files.append((files[0], destination))
|
||||
|
||||
lines = [
|
||||
"#!/usr/bin/env bash",
|
||||
"set -Eeuo pipefail",
|
||||
"runner=$(rlocation {})".format(_shell_quote(_rlocation_path(ctx, tools.runner))),
|
||||
"export OPENSHELL_TEST_GUEST_IMAGE_OVERRIDE=$(rlocation {})".format(_shell_quote(_rlocation_path(ctx, image.disk))),
|
||||
]
|
||||
if is_test:
|
||||
lines.append("export TMPDIR=${TEST_TMPDIR:-${TMPDIR:-/tmp}}")
|
||||
lines.append("args=({})".format(" ".join([_shell_quote(arg) for arg in base_args])))
|
||||
for package in package_files:
|
||||
lines.append("args+=(--install \"$(rlocation {})\")".format(_shell_quote(_rlocation_path(ctx, package))))
|
||||
for file, destination in copy_files:
|
||||
lines.append("args+=(--copy \"$(rlocation {}):{}\")".format(
|
||||
_shell_quote(_rlocation_path(ctx, file)),
|
||||
destination,
|
||||
))
|
||||
for forward_port in ctx.attr.forward_ports:
|
||||
lines.append("args+=(--forward-port {})".format(_shell_quote(forward_port)))
|
||||
for arg in fixed_args:
|
||||
lines.append("args+=({})".format(_shell_quote(arg)))
|
||||
lines.append("exec \"${runner}\" \"${args[@]}\" \"$@\"")
|
||||
return "\n".join(lines) + "\n", package_files, [item[0] for item in copy_files]
|
||||
|
||||
def _test_guest_launcher_impl(ctx):
|
||||
image = ctx.attr.image[TestGuestImageInfo]
|
||||
tools = ctx.toolchains[_HOST_TOOLS_TOOLCHAIN]
|
||||
launcher = ctx.actions.declare_file(ctx.label.name + ".sh")
|
||||
fixed_args = (["--"] + ctx.attr.command) if ctx.attr.is_test else []
|
||||
content, packages, copies = _launcher_content(ctx, image, tools, fixed_args, ctx.attr.is_test)
|
||||
ctx.actions.write(launcher, content, is_executable = True)
|
||||
|
||||
runfiles = ctx.runfiles(
|
||||
files = [image.disk, tools.runner] + packages + copies,
|
||||
transitive_files = tools.all_files,
|
||||
)
|
||||
dependency_runfiles = [ctx.attr.image[DefaultInfo].default_runfiles]
|
||||
for target in ctx.attr.data:
|
||||
dependency_runfiles.append(target[DefaultInfo].default_runfiles)
|
||||
dependency_runfiles.append(ctx.runfiles(transitive_files = target[DefaultInfo].files))
|
||||
runfiles = runfiles.merge_all(dependency_runfiles)
|
||||
return [DefaultInfo(files = depset([launcher]), runfiles = runfiles)]
|
||||
|
||||
def _test_guest_launcher_validate(ctx):
|
||||
if ctx.attr.is_test and not ctx.attr.command:
|
||||
fail("command must not be empty")
|
||||
return _test_guest_launcher_impl(ctx)
|
||||
|
||||
_CONSUMER_ATTRS = {
|
||||
"command": attr.string_list(),
|
||||
"copies": attr.label_keyed_string_dict(allow_files = True),
|
||||
"data": attr.label_list(allow_files = True),
|
||||
"forward_ports": attr.string_list(),
|
||||
"image": attr.label(mandatory = True, providers = [TestGuestImageInfo]),
|
||||
"is_test": attr.bool(),
|
||||
"packages": attr.label_list(allow_files = True),
|
||||
}
|
||||
|
||||
_test_guest_launcher = rule(
|
||||
implementation = _test_guest_launcher_validate,
|
||||
attrs = _CONSUMER_ATTRS,
|
||||
toolchains = [_HOST_TOOLS_TOOLCHAIN],
|
||||
)
|
||||
|
||||
def test_guest_binary(name, image, copies = {}, data = [], forward_ports = [], packages = [], tags = [], **kwargs):
|
||||
"""Create an interactive test-guest executable backed by an image target."""
|
||||
launcher = name + "_launcher"
|
||||
_test_guest_launcher(
|
||||
name = launcher,
|
||||
copies = copies,
|
||||
data = data,
|
||||
forward_ports = forward_ports,
|
||||
image = image,
|
||||
packages = packages,
|
||||
testonly = kwargs.get("testonly", False),
|
||||
)
|
||||
sh_binary(
|
||||
name = name,
|
||||
srcs = [":" + launcher],
|
||||
tags = tags,
|
||||
use_bash_launcher = True,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
def test_guest_test(name, image, command, copies = {}, data = [], forward_ports = [], packages = [], tags = [], **kwargs):
|
||||
"""Create a locally executed, remotely cacheable test-guest test."""
|
||||
launcher = name + "_launcher"
|
||||
_test_guest_launcher(
|
||||
name = launcher,
|
||||
command = command,
|
||||
copies = copies,
|
||||
data = data,
|
||||
forward_ports = forward_ports,
|
||||
image = image,
|
||||
is_test = True,
|
||||
packages = packages,
|
||||
testonly = True,
|
||||
)
|
||||
sh_test(
|
||||
name = name,
|
||||
srcs = [":" + launcher],
|
||||
tags = tags + [
|
||||
"no-remote-exec",
|
||||
"no-sandbox",
|
||||
"requires-network",
|
||||
],
|
||||
use_bash_launcher = True,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
def test_guest_variant(name, configurations, **kwargs):
|
||||
"""Create a prepared image and its interactive runner."""
|
||||
image_name = name + "_image"
|
||||
test_guest_image(
|
||||
name = image_name,
|
||||
configuration_names = [configuration.split("/")[-1].split(".")[0] for configuration in configurations],
|
||||
configurations = configurations,
|
||||
tags = ["manual"],
|
||||
**kwargs
|
||||
)
|
||||
test_guest_binary(
|
||||
name = name,
|
||||
image = ":" + image_name,
|
||||
tags = ["manual"],
|
||||
)
|
||||
+1
-1
@@ -38,7 +38,7 @@ description = "Run all end-to-end tests (Rust + Python + MCP)"
|
||||
depends = ["e2e:rust", "e2e:python", "e2e:mcp"]
|
||||
|
||||
["e2e:test"]
|
||||
description = "Build the current checkout and run a named host or Nix test-guest E2E suite"
|
||||
description = "Build the current checkout and run a named host or Bazel test-guest E2E suite"
|
||||
run = "e2e/run.sh"
|
||||
|
||||
["e2e:gpu"]
|
||||
|
||||
Reference in New Issue
Block a user