ci: pin CI images by digest and add native architecture smoke checks

Signed-off-by: Adrien Langou <alangou@nvidia.com>
This commit is contained in:
Adrien Langou
2026-10-01 18:56:43 +02:00
parent 5acaaba192
commit 3362eb0acd
13 changed files with 334 additions and 18 deletions
@@ -125,6 +125,14 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
## View Job Logs
For CI image pin changes, check both `CI image smoke` jobs in `Branch Checks`.
They validate the multiarchitecture index and exercise the baked tools on native
amd64/arm64 runners. A registry denial or missing platform fails these jobs;
do not replace a digest with `latest` to work around it. Smoke checks do not
replace packaging, SDK, kernel, or E2E validation. Follow `CI.md` under
"CI container image pins" for the update and rollback procedure, including the
separate image retained by Docker E2E.
For `Trivy Changes`, inspect the `Resolve PR baseline` step for the base and head
SHAs. PR runs compare the tested merge commit with its
first parent; change detection and scans must use the same pair. On reruns, do
+42 -6
View File
@@ -36,6 +36,42 @@ jobs:
- id: gate
uses: ./.github/actions/pr-gate
ci-image-smoke:
name: CI image smoke (${{ matrix.arch }})
needs: pr_metadata
if: needs.pr_metadata.outputs.should_run == 'true'
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
runner: linux-amd64-cpu8
- arch: arm64
runner: linux-arm64-cpu8
runs-on: ${{ matrix.runner }}
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Validate CI image pins
run: |
bash tasks/scripts/test-check-ci-images.sh
bash tasks/scripts/check-ci-images.sh --check
- name: Log in to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Check indexes and smoke test pinned images
env:
CI_IMAGE_ARCH: ${{ matrix.arch }}
run: bash tasks/scripts/check-ci-images.sh --smoke "$CI_IMAGE_ARCH"
mise-lockfile:
name: mise Lockfile
needs: pr_metadata
@@ -43,7 +79,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 30
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -68,7 +104,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 30
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -360,7 +396,7 @@ jobs:
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -396,7 +432,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 30
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -416,7 +452,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 30
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -436,7 +472,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 30
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
name: Cargo Deny
runs-on: linux-amd64-cpu8
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
runs-on: ${{ matrix.runner }}
timeout-minutes: 20
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+1 -1
View File
@@ -50,7 +50,7 @@ jobs:
matrix:
include: ${{ fromJSON(inputs.suite-matrix) }}
container:
image: ghcr.io/nvidia/openshell/ci:37072ee81cd7b294c714bfa5ecc829b6927b3d70
image: ghcr.io/nvidia/openshell/ci:37072ee81cd7b294c714bfa5ecc829b6927b3d70@sha256:ffa96b8009de6e28bbf157060440c3abb312d3bcda444e9571c539c8c6115615
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
runner: wsl-amd64-gpu-rtxpro6000-latest-1
experimental: true
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+1 -1
View File
@@ -81,7 +81,7 @@ jobs:
if: needs.pr_metadata.outputs.should_run == 'true' && needs.helm_changes.outputs.should_run == 'true'
runs-on: linux-amd64-cpu8
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 5
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -193,7 +193,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 20
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+3 -3
View File
@@ -34,7 +34,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 5
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -295,7 +295,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 20
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -777,7 +777,7 @@ jobs:
contents: read
packages: write
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+2 -2
View File
@@ -39,7 +39,7 @@ jobs:
runs-on: linux-arm64-cpu8
timeout-minutes: 60
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
@@ -89,7 +89,7 @@ jobs:
runs-on: linux-amd64-cpu8
timeout-minutes: 60
container:
image: ghcr.io/nvidia/openshell/ci:latest
image: ghcr.io/nvidia/openshell/ci:9cb72baa2e61a1b5f12407e6e82da7fdba0aa722@sha256:67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+65
View File
@@ -54,6 +54,71 @@ This runs the `mechanistic-proposal`, `new-hostname-proposal`, and
gateway binaries and runtime images. This manual run does not replace the
required PR E2E gate.
## CI container image pins
Workflow job containers use a source commit tag plus the SHA-256 digest of the
multiarchitecture index. The digest selects the content; the tag records the
build source for review. Keep references literal in workflow YAML so the runner
can pull the container before checkout and Zizmor can audit the pin.
The initial common pin comes from the successful
[CI image build for `9cb72baa`](https://github.com/NVIDIA/OpenShell/actions/runs/36526031686).
Docker E2E retains its separate image from the successful
[build for `37072ee8`](https://github.com/NVIDIA/OpenShell/actions/runs/33085793291).
Its digest must be updated independently when adopting a newer image.
`Branch Checks` smoke-tests every distinct pinned CI image on native amd64 and
arm64 runners, after the existing PR admission gate. It rejects mutable or
malformed references, requires an index containing both Linux architectures,
pulls by digest, checks the baked tools, and compiles and runs small C, Rust, and
Go programs. The smoke containers have no network or host mounts. This checks
image availability and basic tool operation; the existing language, packaging,
and E2E jobs remain responsible for testing OpenShell itself.
To update an image:
1. Wait for `Build CI Image` to finish successfully, including both architecture
builds, their smoke checks, and `Merge manifest`. Image publication still runs
on host runners and does not depend on the consumer pin. It publishes commit
tags and `latest`; publication alone no longer changes consumers.
2. With GHCR read access, inspect the **commit tag**, then inspect the reported
index by digest. Check that it contains both `linux/amd64` and `linux/arm64`.
Do not use one of the architecture-specific child digests.
```shell
docker buildx imagetools inspect ghcr.io/nvidia/openshell/ci:<source-commit>
docker buildx imagetools inspect ghcr.io/nvidia/openshell/ci@sha256:<index-digest>
```
3. Replace the matching `image:` references with
`ghcr.io/nvidia/openshell/ci:<source-commit>@sha256:<index-digest>` in one reviewed
change. Keep any intentionally different consumer version separate. Review
image package/security scan results and tool changes when adopting a new
image; an immutable reference does not establish that its contents are safe.
4. Run the local definition checks and the native smoke command on each
architecture, or wait for both `CI image smoke` jobs in `Branch Checks`:
```shell
bash tasks/scripts/test-check-ci-images.sh
bash tasks/scripts/check-ci-images.sh --check
bash tasks/scripts/check-ci-images.sh --smoke amd64
# On a native arm64 machine:
bash tasks/scripts/check-ci-images.sh --smoke arm64
```
5. Require successful Branch Checks (including the TypeScript publish dry-run),
Helm checks, DEB/wheel packaging, and the affected Docker/GPU E2E lanes before
merging. For a toolchain/image update, also validate the Linux VM kernel builds
without invoking their publication job. A smoke pass alone is insufficient.
Do not dispatch production release workflows merely to test a pin.
Review pins whenever `Dockerfile.ci`, `mise.toml`, or `mise.lock` changes and
during regular dependency/security maintenance. A new tool may require building
the candidate image first and then updating its consumer pins; do not restore
`latest` to bypass this ordering. Keep previous digests available in GHCR for
rollback and reproducible reruns. Roll back by reverting the affected pins
together, preserving the separate Docker E2E version.
## Informational security reports
Security workflow compute runs directly on GitHub-hosted runners instead of
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
ci_image_refs() {
# Consumer references stay literal so GitHub can pull the job container before
# checkout, and Zizmor can audit it. The producer's bare repository is excluded.
grep -rhE '^[[:space:]]*image:' "$1" |
grep -oE "ghcr.io/nvidia/openshell/ci([:@][^[:space:]\"']+)?" |
sort -u
}
ci_image_ref_valid() {
[[ "$1" =~ ^ghcr\.io/nvidia/openshell/ci:[0-9a-f]{40}@sha256:[0-9a-f]{64}$ ]]
}
ci_image_index_valid() {
# BuildKit provenance descriptors (unknown/unknown) may coexist with images.
jq -e '
(.mediaType == "application/vnd.oci.image.index.v1+json" or
.mediaType == "application/vnd.docker.distribution.manifest.list.v2+json") and
([.manifests[] | select(.platform.os == "linux" and
.platform.architecture == "amd64")] | length == 1) and
([.manifests[] | select(.platform.os == "linux" and
.platform.architecture == "arm64")] | length == 1)
' >/dev/null
}
ci_image_smoke() {
local image=$1 arch=$2
printf 'Checking multiarchitecture index: %s\n' "$image"
if ! docker buildx imagetools inspect --raw "$image" | ci_image_index_valid; then
echo "Cannot verify a Linux amd64/arm64 index for $image" >&2
return 1
fi
# No host mounts, Docker socket, registry credentials, or network inside the
# container. Exercise the baked tools without installing replacements.
docker run --rm --pull=always --platform "linux/$arch" --network none \
--env "CI_IMAGE_ARCH=$arch" --workdir /opt/mise --entrypoint bash -i "$image" -s <<'SMOKE'
set -euo pipefail
case "$CI_IMAGE_ARCH:$(uname -m)" in
amd64:x86_64|arm64:aarch64) ;;
*) echo "Unexpected container architecture" >&2; exit 1 ;;
esac
mise --version
gh --version
docker buildx version
rustc --version
cargo --version
go version
node --version
npm --version
uv --version
protoc --version
buf --version
helm version --short
zig version
cc --version
pkg-config --exists openssl
smoke_dir=$(mktemp -d)
trap 'rm -rf "$smoke_dir"' EXIT
printf 'int main(void) { return 0; }\n' > "$smoke_dir/main.c"
cc "$smoke_dir/main.c" -o "$smoke_dir/c-smoke"
"$smoke_dir/c-smoke"
printf 'fn main() { assert_eq!(2 + 2, 4); }\n' > "$smoke_dir/main.rs"
rustc "$smoke_dir/main.rs" -o "$smoke_dir/rust-smoke"
"$smoke_dir/rust-smoke"
printf 'package main\nfunc main() {}\n' > "$smoke_dir/main.go"
GOTOOLCHAIN=local go build -o "$smoke_dir/go-smoke" "$smoke_dir/main.go"
"$smoke_dir/go-smoke"
uv run --no-project --offline --python "$(mise which python)" \
python -c 'import ssl; assert ssl.OPENSSL_VERSION'
node -e 'if (2 + 2 !== 4) process.exit(1)'
SMOKE
}
main() {
local mode=${1:---check} arch=${2:-} native_arch refs image
local root
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
case "$mode" in
--check) ;;
--smoke)
case "$(uname -m)" in
x86_64) native_arch=amd64 ;;
aarch64|arm64) native_arch=arm64 ;;
*) echo "Unsupported native architecture" >&2; return 1 ;;
esac
[[ "$arch" == "$native_arch" ]] || {
echo "Run smoke tests on a native $arch runner (host is $native_arch)" >&2
return 1
}
;;
*) echo "Usage: $0 [--check | --smoke amd64|arm64]" >&2; return 1 ;;
esac
refs=$(ci_image_refs "$root/.github/workflows") || {
echo "No CI image references found" >&2
return 1
}
# Validate all references before executing any container.
while IFS= read -r image; do
ci_image_ref_valid "$image" || {
printf 'CI image requires a source commit tag and SHA-256 digest: %s\n' "$image" >&2
return 1
}
printf '%s\n' "$image"
done <<< "$refs"
if [[ "$mode" == --smoke ]]; then
while IFS= read -r image; do
ci_image_smoke "$image" "$arch"
done <<< "$refs"
fi
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
main "$@"
fi
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
# shellcheck source=tasks/scripts/check-ci-images.sh
source "$(dirname "${BASH_SOURCE[0]}")/check-ci-images.sh"
fixture_dir=$(mktemp -d)
trap 'rm -rf "$fixture_dir"' EXIT
source_tag=9cb72baa2e61a1b5f12407e6e82da7fdba0aa722
digest=67a9a0c32cb99825e6d3e9d9eec45d67149ea1ff7c11a1b1e1b3480d2d3df684
ref="ghcr.io/nvidia/openshell/ci:$source_tag@sha256:$digest"
ci_image_ref_valid "$ref"
for invalid in \
ghcr.io/nvidia/openshell/ci \
ghcr.io/nvidia/openshell/ci:latest \
"ghcr.io/nvidia/openshell/ci:$source_tag" \
"ghcr.io/nvidia/openshell/ci:$source_tag@sha256:1234" \
"ghcr.io/nvidia/openshell/ci:latest@sha256:$digest"; do
if ci_image_ref_valid "$invalid"; then
echo "Unexpectedly accepted $invalid" >&2
exit 1
fi
done
# Quoted/unquoted consumers, duplicates, and the producer's bare repository.
printf 'image: %s\nimage: "%s"\nimage: '\''%s'\''\nCI_IMAGE: ghcr.io/nvidia/openshell/ci\n' \
"$ref" "$ref" "$ref" > "$fixture_dir/workflow.yml"
[[ "$(ci_image_refs "$fixture_dir")" == "$ref" ]]
printf 'image: ghcr.io/nvidia/openshell/ci:latest\n' >> "$fixture_dir/workflow.yml"
[[ "$(ci_image_refs "$fixture_dir" | wc -l)" -eq 2 ]]
printf 'image: ghcr.io/nvidia/openshell/ci\n' >> "$fixture_dir/workflow.yml"
[[ "$(ci_image_refs "$fixture_dir" | wc -l)" -eq 3 ]]
index='{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[
{"platform":{"os":"linux","architecture":"amd64"}},
{"platform":{"os":"linux","architecture":"arm64"}},
{"platform":{"os":"unknown","architecture":"unknown"}}
]}'
ci_image_index_valid <<< "$index"
jq '.mediaType = "application/vnd.docker.distribution.manifest.list.v2+json"' \
<<< "$index" | ci_image_index_valid
for mutation in \
'.mediaType = "application/vnd.oci.image.manifest.v1+json"' \
'.manifests = [.manifests[0]]' \
'.manifests = [.manifests[1]]' \
'.manifests[1].platform.os = "windows"' \
'.manifests += [.manifests[0]]' \
'del(.manifests)' ; do
if jq "$mutation" <<< "$index" | ci_image_index_valid 2>/dev/null; then
echo "Unexpectedly accepted index mutation: $mutation" >&2
exit 1
fi
done
if ci_image_index_valid <<< 'not JSON' 2>/dev/null; then
echo "Unexpectedly accepted invalid JSON" >&2
exit 1
fi
# A failed registry lookup or invalid index must stop before running an image.
docker() {
if [[ "$1" == buildx ]]; then
if [[ "$registry_failure" == true ]]; then
return 1
fi
printf '%s\n' "$smoke_index"
else
touch "$fixture_dir/container-ran"
cat >/dev/null
fi
}
for registry_failure in true false; do
smoke_index=$(jq '.manifests = [.manifests[0]]' <<< "$index")
if ci_image_smoke "$ref" amd64 >/dev/null 2>&1; then
echo "Unexpectedly accepted an unavailable or incomplete index" >&2
exit 1
fi
[[ ! -e "$fixture_dir/container-ran" ]]
done
smoke_index=$index
ci_image_smoke "$ref" amd64 >/dev/null
[[ -e "$fixture_dir/container-ran" ]]
echo "CI image pin and multiarchitecture validation tests passed"