mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-18 05:39:50 +08:00
Drew Newberry
59335ec852
cluster-deploy-fast.sh ran helm and kubectl directly on the host, which requires a local kubeconfig pointing at the k3s API server. Since the cluster container does not expose port 6443, these commands always fail with 'connection refused'. Install helm in the cluster container image and run all helm/kubectl commands via docker exec with KUBECONFIG pre-configured. The local chart directory is copied into the container before helm upgrade.
186 lines
9.3 KiB
Docker
186 lines
9.3 KiB
Docker
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# k3s cluster image with OpenShell Helm charts and manifests
|
|
#
|
|
# Multi-stage build: extracts k3s artifacts from the upstream rancher/k3s
|
|
# Alpine image and layers them onto the NVIDIA Ubuntu base image.
|
|
#
|
|
# This image includes:
|
|
# - k3s binary and all supporting binaries (containerd-shim, runc, CNI, etc.)
|
|
# - k9s for interactive cluster debugging (via `openshell doctor exec -- k9s`)
|
|
# - Packaged OpenShell Helm chart
|
|
# - HelmChart CR for auto-deploying OpenShell
|
|
# - Custom entrypoint for DNS configuration in Docker environments
|
|
#
|
|
# Component images (openshell/gateway, openshell/sandbox) are
|
|
# pulled at runtime from the distribution registry rather than bundled as tarballs.
|
|
# Registry credentials are generated by the entrypoint script at container start.
|
|
#
|
|
# The helm charts are built by the docker:build:cluster mise task
|
|
# and placed in deploy/docker/.build/ before this Dockerfile is built.
|
|
|
|
# Tracked upstream vulns in rancher/k3s:v1.35.2-k3s1 (bundled Go dependencies):
|
|
# GHSA-pwhc-rpq9-4c8w containerd v2.1.5-k3s1 (local privesc via CRI dir perms;
|
|
# upstream patched in 2.1.5 -- may be scanner false positive
|
|
# from the -k3s1 suffix)
|
|
# GHSA-p436-gjf2-799p docker/cli v28.3.2 (Windows-only plugin path hijack; N/A)
|
|
# GHSA-9h8m-3fm2-qjrq otel/sdk v1.39.0 (macOS-only PATH hijack; N/A for Linux)
|
|
# CVE-2024-36623 docker/docker v25.0.8 (streamformatter race condition)
|
|
# Bump K3S_VERSION when a release with updated dependencies ships.
|
|
|
|
ARG K3S_VERSION=v1.35.2-k3s1
|
|
ARG K9S_VERSION=v0.50.18
|
|
ARG HELM_VERSION=v3.17.3
|
|
ARG NVIDIA_CONTAINER_TOOLKIT_VERSION=1.18.2-1
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stage 1: Extract k3s artifacts from upstream rancher image (Alpine-based)
|
|
# ---------------------------------------------------------------------------
|
|
FROM rancher/k3s:${K3S_VERSION} AS k3s
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stage 1b: Download k9s binary for interactive cluster debugging
|
|
# ---------------------------------------------------------------------------
|
|
FROM ubuntu:24.04 AS k9s
|
|
ARG K9S_VERSION
|
|
ARG TARGETARCH
|
|
RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \
|
|
curl -fsSL "https://github.com/derailed/k9s/releases/download/${K9S_VERSION}/k9s_Linux_${TARGETARCH}.tar.gz" \
|
|
| tar xz -C /tmp k9s && \
|
|
chmod +x /tmp/k9s && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stage 1c: Download helm binary for in-container chart upgrades
|
|
# ---------------------------------------------------------------------------
|
|
FROM ubuntu:24.04 AS helm
|
|
ARG HELM_VERSION
|
|
ARG TARGETARCH
|
|
RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \
|
|
curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" \
|
|
| tar xz --strip-components=1 -C /tmp "linux-${TARGETARCH}/helm" && \
|
|
chmod +x /tmp/helm && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stage 2: Install NVIDIA container toolkit on Ubuntu
|
|
# ---------------------------------------------------------------------------
|
|
FROM ubuntu:24.04 AS nvidia-toolkit
|
|
|
|
ARG NVIDIA_CONTAINER_TOOLKIT_VERSION
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
gpg curl ca-certificates && \
|
|
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
|
|
| gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg && \
|
|
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
|
|
| sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
|
|
| tee /etc/apt/sources.list.d/nvidia-container-toolkit.list && \
|
|
apt-get update && \
|
|
apt-get install -y --no-install-recommends \
|
|
"nvidia-container-toolkit=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
|
|
"nvidia-container-toolkit-base=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
|
|
"libnvidia-container-tools=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
|
|
"libnvidia-container1=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stage 3: Runtime on NVIDIA hardened Ubuntu base
|
|
# ---------------------------------------------------------------------------
|
|
FROM nvcr.io/nvidia/base/ubuntu:noble-20251013
|
|
|
|
# Install runtime dependencies that k3s expects from the host OS.
|
|
# - iptables: used by flannel/kube-proxy for network policy and NAT rules
|
|
# - mount/umount: needed by kubelet for volume mounts (provided by mount package)
|
|
# - ca-certificates: TLS verification for registry pulls
|
|
# - conntrack: k3s/kube-proxy uses conntrack for connection tracking
|
|
# - dnsutils: nslookup used by entrypoint/healthcheck for DNS probe
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ca-certificates \
|
|
iptables \
|
|
mount \
|
|
dnsutils \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Copy the full /bin directory from k3s (contains all statically-linked
|
|
# binaries and their symlinks: k3s, kubectl, crictl, ctr, containerd,
|
|
# containerd-shim-runc-v2, runc, cni plugins, busybox, coreutils,
|
|
# ip, ipset, conntrack, nsenter, pigz, etc.)
|
|
COPY --from=k3s /bin/ /bin/
|
|
|
|
# Copy k9s binary for interactive cluster debugging via `openshell doctor exec -- k9s`
|
|
COPY --from=k9s /tmp/k9s /usr/local/bin/k9s
|
|
|
|
# Copy helm binary for in-container chart upgrades (used by cluster-deploy-fast.sh)
|
|
COPY --from=helm /tmp/helm /usr/local/bin/helm
|
|
|
|
# Copy iptables/nftables tooling (xtables-nft-multi, iptables-detect.sh, etc.)
|
|
# These are in /bin/aux/ in the k3s image and must be on PATH.
|
|
# Note: the Ubuntu iptables package provides /usr/sbin/iptables, but k3s
|
|
# expects its own bundled version at /bin/aux/iptables. Both are on PATH;
|
|
# k3s finds its copy via /bin/aux in PATH.
|
|
|
|
# Copy CA certificates from k3s (bundled Alpine CA bundle).
|
|
# The Ubuntu ca-certificates package also installs certs; having both is fine.
|
|
COPY --from=k3s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/k3s-ca-certificates.crt
|
|
|
|
# Copy timezone data used by k3s/Go for time.LoadLocation
|
|
COPY --from=k3s /usr/share/zoneinfo/ /usr/share/zoneinfo/
|
|
|
|
# Set environment variables matching the upstream k3s image.
|
|
# PATH includes /bin/aux for iptables tooling and /var/lib/rancher/k3s/data/cni
|
|
# for runtime-extracted CNI binaries.
|
|
ENV PATH="/var/lib/rancher/k3s/data/cni:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux" \
|
|
CRI_CONFIG_FILE="/var/lib/rancher/k3s/agent/etc/crictl.yaml"
|
|
|
|
# Copy NVIDIA Container Toolkit files from the build stage.
|
|
# k3s auto-detects nvidia-container-runtime on PATH and registers it as a
|
|
# containerd runtime + creates the "nvidia" RuntimeClass automatically.
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-cdi-hook /usr/bin/
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-runtime /usr/bin/
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-runtime-hook /usr/bin/
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-cli /usr/bin/
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-ctk /usr/bin/
|
|
COPY --from=nvidia-toolkit /etc/nvidia-container-runtime /etc/nvidia-container-runtime
|
|
COPY --from=nvidia-toolkit /usr/lib/*-linux-gnu/libnvidia-container*.so* /usr/lib/
|
|
|
|
# Create directories for manifests, charts, and configuration
|
|
RUN mkdir -p /var/lib/rancher/k3s/server/manifests \
|
|
/var/lib/rancher/k3s/server/static/charts \
|
|
/etc/rancher/k3s \
|
|
/opt/openshell/manifests \
|
|
/opt/openshell/charts \
|
|
/opt/openshell/gpu-manifests \
|
|
/run/flannel
|
|
|
|
# Copy entrypoint script that configures DNS for Docker environments
|
|
# This script detects the host gateway IP and configures CoreDNS to use it
|
|
COPY deploy/docker/cluster-entrypoint.sh /usr/local/bin/cluster-entrypoint.sh
|
|
RUN chmod +x /usr/local/bin/cluster-entrypoint.sh
|
|
|
|
# Copy healthcheck script that verifies cluster readiness
|
|
COPY deploy/docker/cluster-healthcheck.sh /usr/local/bin/cluster-healthcheck.sh
|
|
RUN chmod +x /usr/local/bin/cluster-healthcheck.sh
|
|
|
|
# Registry credentials for pulling component images at runtime are generated
|
|
# by the entrypoint script at /etc/rancher/k3s/registries.yaml.
|
|
|
|
# Copy packaged helm charts to a staging directory that won't be
|
|
# overwritten by the /var/lib/rancher/k3s volume mount. The entrypoint
|
|
# script copies them into the k3s static charts directory at container start.
|
|
COPY deploy/docker/.build/charts/*.tgz /opt/openshell/charts/
|
|
|
|
# Copy Kubernetes manifests to a persistent location that won't be overwritten by the volume mount.
|
|
# The bootstrap code will copy these to /var/lib/rancher/k3s/server/manifests/ after cluster start.
|
|
COPY deploy/kube/manifests/*.yaml /opt/openshell/manifests/
|
|
|
|
# Copy GPU-specific manifests (deployed conditionally by entrypoint when GPU_ENABLED=true)
|
|
COPY deploy/kube/gpu-manifests/*.yaml /opt/openshell/gpu-manifests/
|
|
|
|
# Use custom entrypoint that configures DNS before starting k3s
|
|
ENTRYPOINT ["/usr/local/bin/cluster-entrypoint.sh"]
|
|
|
|
HEALTHCHECK --interval=5s --timeout=5s --start-period=20s --retries=60 \
|
|
CMD ["/usr/local/bin/cluster-healthcheck.sh"]
|