# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# k3s cluster image with OpenShell Helm charts and manifests
#
# Multi-stage build: extracts k3s artifacts from the upstream rancher/k3s
# Alpine image and layers them onto the NVIDIA Ubuntu base image.
#
# This image includes:
# - k3s binary and all supporting binaries (containerd-shim, runc, CNI, etc.)
# - k9s for interactive cluster debugging (via `openshell doctor exec -- k9s`)
# - Packaged OpenShell Helm chart
# - HelmChart CR for auto-deploying OpenShell
# - Custom entrypoint for DNS configuration in Docker environments
#
# Component images (openshell/gateway, openshell/sandbox) are
# pulled at runtime from the distribution registry rather than bundled as tarballs.
# Registry credentials are generated by the entrypoint script at container start.
#
# The helm charts are built by the docker:build:cluster mise task
# and placed in deploy/docker/.build/ before this Dockerfile is built.

# Tracked upstream vulns in rancher/k3s:v1.35.2-k3s1 (bundled Go dependencies):
#   GHSA-pwhc-rpq9-4c8w  containerd v2.1.5-k3s1 (local privesc via CRI dir perms;
#                         upstream patched in 2.1.5 -- may be scanner false positive
#                         from the -k3s1 suffix)
#   GHSA-p436-gjf2-799p  docker/cli v28.3.2 (Windows-only plugin path hijack; N/A)
#   GHSA-9h8m-3fm2-qjrq  otel/sdk v1.39.0 (macOS-only PATH hijack; N/A for Linux)
#   CVE-2024-36623       docker/docker v25.0.8 (streamformatter race condition)
# Bump K3S_VERSION when a release with updated dependencies ships.

ARG K3S_VERSION=v1.35.2-k3s1
ARG K9S_VERSION=v0.50.18
ARG HELM_VERSION=v3.17.3
ARG NVIDIA_CONTAINER_TOOLKIT_VERSION=1.18.2-1

# ---------------------------------------------------------------------------
# Stage 1: Extract k3s artifacts from upstream rancher image (Alpine-based)
# ---------------------------------------------------------------------------
FROM rancher/k3s:${K3S_VERSION} AS k3s

# ---------------------------------------------------------------------------
# Stage 1b: Download k9s binary for interactive cluster debugging
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS k9s
ARG K9S_VERSION
ARG TARGETARCH
RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \
    curl -fsSL "https://github.com/derailed/k9s/releases/download/${K9S_VERSION}/k9s_Linux_${TARGETARCH}.tar.gz" \
      | tar xz -C /tmp k9s && \
    chmod +x /tmp/k9s && \
    rm -rf /var/lib/apt/lists/*

# ---------------------------------------------------------------------------
# Stage 1c: Download helm binary for in-container chart upgrades
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS helm
ARG HELM_VERSION
ARG TARGETARCH
RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \
    curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" \
      | tar xz --strip-components=1 -C /tmp "linux-${TARGETARCH}/helm" && \
    chmod +x /tmp/helm && \
    rm -rf /var/lib/apt/lists/*

# ---------------------------------------------------------------------------
# Stage 2: Install NVIDIA container toolkit on Ubuntu
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS nvidia-toolkit

ARG NVIDIA_CONTAINER_TOOLKIT_VERSION

RUN apt-get update && apt-get install -y --no-install-recommends \
        gpg curl ca-certificates && \
    curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
        | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg && \
    curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
        | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
        | tee /etc/apt/sources.list.d/nvidia-container-toolkit.list && \
    apt-get update && \
    apt-get install -y --no-install-recommends \
        "nvidia-container-toolkit=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
        "nvidia-container-toolkit-base=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
        "libnvidia-container-tools=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
        "libnvidia-container1=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" && \
    rm -rf /var/lib/apt/lists/*

# ---------------------------------------------------------------------------
# Stage 3: Runtime on NVIDIA hardened Ubuntu base
# ---------------------------------------------------------------------------
FROM nvcr.io/nvidia/base/ubuntu:noble-20251013

# Install runtime dependencies that k3s expects from the host OS.
# - iptables: used by flannel/kube-proxy for network policy and NAT rules
# - mount/umount: needed by kubelet for volume mounts (provided by mount package)
# - ca-certificates: TLS verification for registry pulls
# - conntrack: k3s/kube-proxy uses conntrack for connection tracking
# - dnsutils: nslookup used by entrypoint/healthcheck for DNS probe
RUN apt-get update && apt-get install -y --no-install-recommends \
    ca-certificates \
    iptables \
    mount \
    dnsutils \
    && rm -rf /var/lib/apt/lists/*

# Copy the full /bin directory from k3s (contains all statically-linked
# binaries and their symlinks: k3s, kubectl, crictl, ctr, containerd,
# containerd-shim-runc-v2, runc, cni plugins, busybox, coreutils,
# ip, ipset, conntrack, nsenter, pigz, etc.)
COPY --from=k3s /bin/ /bin/

# Copy k9s binary for interactive cluster debugging via `openshell doctor exec -- k9s`
COPY --from=k9s /tmp/k9s /usr/local/bin/k9s

# Copy helm binary for in-container chart upgrades (used by cluster-deploy-fast.sh)
COPY --from=helm /tmp/helm /usr/local/bin/helm

# Copy iptables/nftables tooling (xtables-nft-multi, iptables-detect.sh, etc.)
# These are in /bin/aux/ in the k3s image and must be on PATH.
# Note: the Ubuntu iptables package provides /usr/sbin/iptables, but k3s
# expects its own bundled version at /bin/aux/iptables. Both are on PATH;
# k3s finds its copy via /bin/aux in PATH.

# Copy CA certificates from k3s (bundled Alpine CA bundle).
# The Ubuntu ca-certificates package also installs certs; having both is fine.
COPY --from=k3s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/k3s-ca-certificates.crt

# Copy timezone data used by k3s/Go for time.LoadLocation
COPY --from=k3s /usr/share/zoneinfo/ /usr/share/zoneinfo/

# Set environment variables matching the upstream k3s image.
# PATH includes /bin/aux for iptables tooling and /var/lib/rancher/k3s/data/cni
# for runtime-extracted CNI binaries.
ENV PATH="/var/lib/rancher/k3s/data/cni:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux" \
    CRI_CONFIG_FILE="/var/lib/rancher/k3s/agent/etc/crictl.yaml"

# Copy NVIDIA Container Toolkit files from the build stage.
# k3s auto-detects nvidia-container-runtime on PATH and registers it as a
# containerd runtime + creates the "nvidia" RuntimeClass automatically.
COPY --from=nvidia-toolkit /usr/bin/nvidia-cdi-hook /usr/bin/
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-runtime /usr/bin/
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-runtime-hook /usr/bin/
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-cli /usr/bin/
COPY --from=nvidia-toolkit /usr/bin/nvidia-ctk /usr/bin/
COPY --from=nvidia-toolkit /etc/nvidia-container-runtime /etc/nvidia-container-runtime
COPY --from=nvidia-toolkit /usr/lib/*-linux-gnu/libnvidia-container*.so* /usr/lib/

# Create directories for manifests, charts, and configuration
RUN mkdir -p /var/lib/rancher/k3s/server/manifests \
             /var/lib/rancher/k3s/server/static/charts \
             /etc/rancher/k3s \
             /opt/openshell/manifests \
             /opt/openshell/charts \
             /opt/openshell/gpu-manifests \
             /run/flannel

# Copy entrypoint script that configures DNS for Docker environments
# This script detects the host gateway IP and configures CoreDNS to use it
COPY deploy/docker/cluster-entrypoint.sh /usr/local/bin/cluster-entrypoint.sh
RUN chmod +x /usr/local/bin/cluster-entrypoint.sh

# Copy healthcheck script that verifies cluster readiness
COPY deploy/docker/cluster-healthcheck.sh /usr/local/bin/cluster-healthcheck.sh
RUN chmod +x /usr/local/bin/cluster-healthcheck.sh

# Registry credentials for pulling component images at runtime are generated
# by the entrypoint script at /etc/rancher/k3s/registries.yaml.

# Copy packaged helm charts to a staging directory that won't be
# overwritten by the /var/lib/rancher/k3s volume mount. The entrypoint
# script copies them into the k3s static charts directory at container start.
COPY deploy/docker/.build/charts/*.tgz /opt/openshell/charts/

# Copy Kubernetes manifests to a persistent location that won't be overwritten by the volume mount.
# The bootstrap code will copy these to /var/lib/rancher/k3s/server/manifests/ after cluster start.
COPY deploy/kube/manifests/*.yaml /opt/openshell/manifests/

# Copy GPU-specific manifests (deployed conditionally by entrypoint when GPU_ENABLED=true)
COPY deploy/kube/gpu-manifests/*.yaml /opt/openshell/gpu-manifests/

# Use custom entrypoint that configures DNS before starting k3s
ENTRYPOINT ["/usr/local/bin/cluster-entrypoint.sh"]

HEALTHCHECK --interval=5s --timeout=5s --start-period=20s --retries=60 \
  CMD ["/usr/local/bin/cluster-healthcheck.sh"]