mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-17 05:31:16 +08:00
274 lines
12 KiB
Docker
274 lines
12 KiB
Docker
# syntax=docker/dockerfile:1.4
|
|
|
|
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Shared OpenShell image build graph.
|
|
#
|
|
# Targets:
|
|
# gateway Final gateway image
|
|
# cluster Final cluster image
|
|
# gateway-builder Release openshell-server binary
|
|
# supervisor-builder Release openshell-sandbox binary
|
|
# supervisor-output Minimal stage exporting only the supervisor binary
|
|
|
|
ARG K3S_VERSION=v1.35.2-k3s1
|
|
ARG K9S_VERSION=v0.50.18
|
|
ARG HELM_VERSION=v3.17.3
|
|
ARG NVIDIA_CONTAINER_TOOLKIT_VERSION=1.18.2-1
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared Rust build stages
|
|
# ---------------------------------------------------------------------------
|
|
FROM --platform=$BUILDPLATFORM rust:1.88-slim AS rust-builder-base
|
|
ARG TARGETARCH
|
|
ARG BUILDARCH
|
|
ARG CARGO_TARGET_CACHE_SCOPE=default
|
|
ARG SCCACHE_MEMCACHED_ENDPOINT
|
|
# CI sets this to 1 for maximum optimization; local builds leave it unset
|
|
# so cargo uses the Cargo.toml default (parallel codegen for fast linking).
|
|
ARG CARGO_CODEGEN_UNITS
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
cmake g++ make protobuf-compiler curl && rm -rf /var/lib/apt/lists/*
|
|
|
|
COPY deploy/docker/cross-build.sh /usr/local/bin/
|
|
RUN . cross-build.sh && install_cross_toolchain && install_sccache && add_rust_target
|
|
|
|
WORKDIR /build
|
|
|
|
FROM rust-builder-base AS rust-builder-skeleton
|
|
|
|
COPY Cargo.toml Cargo.lock ./
|
|
COPY crates/openshell-bootstrap/Cargo.toml crates/openshell-bootstrap/Cargo.toml
|
|
COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml
|
|
COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml
|
|
COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml
|
|
COPY crates/openshell-providers/Cargo.toml crates/openshell-providers/Cargo.toml
|
|
COPY crates/openshell-router/Cargo.toml crates/openshell-router/Cargo.toml
|
|
COPY crates/openshell-sandbox/Cargo.toml crates/openshell-sandbox/Cargo.toml
|
|
COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml
|
|
COPY crates/openshell-tui/Cargo.toml crates/openshell-tui/Cargo.toml
|
|
COPY crates/openshell-core/build.rs crates/openshell-core/build.rs
|
|
COPY proto/ proto/
|
|
|
|
RUN mkdir -p \
|
|
crates/openshell-bootstrap/src \
|
|
crates/openshell-cli/src \
|
|
crates/openshell-core/src \
|
|
crates/openshell-policy/src \
|
|
crates/openshell-providers/src \
|
|
crates/openshell-router/src \
|
|
crates/openshell-sandbox/src \
|
|
crates/openshell-server/src \
|
|
crates/openshell-tui/src && \
|
|
touch crates/openshell-bootstrap/src/lib.rs && \
|
|
printf 'fn main() {}\n' > crates/openshell-cli/src/main.rs && \
|
|
touch crates/openshell-core/src/lib.rs && \
|
|
touch crates/openshell-policy/src/lib.rs && \
|
|
touch crates/openshell-providers/src/lib.rs && \
|
|
touch crates/openshell-router/src/lib.rs && \
|
|
touch crates/openshell-sandbox/src/lib.rs && \
|
|
printf 'fn main() {}\n' > crates/openshell-sandbox/src/main.rs && \
|
|
touch crates/openshell-server/src/lib.rs && \
|
|
printf 'fn main() {}\n' > crates/openshell-server/src/main.rs && \
|
|
touch crates/openshell-tui/src/lib.rs
|
|
|
|
FROM rust-builder-skeleton AS rust-deps
|
|
|
|
RUN --mount=type=cache,id=cargo-registry-${TARGETARCH},sharing=locked,target=/usr/local/cargo/registry \
|
|
--mount=type=cache,id=cargo-git-${TARGETARCH},sharing=locked,target=/usr/local/cargo/git \
|
|
--mount=type=cache,id=cargo-target-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \
|
|
--mount=type=cache,id=sccache-${TARGETARCH},sharing=locked,target=/tmp/sccache \
|
|
. cross-build.sh && cargo_cross_build --release -p openshell-server -p openshell-sandbox
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-target workspace stages
|
|
# ---------------------------------------------------------------------------
|
|
# Copy only the crates needed for each target so that a change to
|
|
# openshell-sandbox does not invalidate the gateway build and vice versa.
|
|
# The skeleton stage already has stub Cargo.toml + src/ for every crate,
|
|
# so cargo workspace resolution continues to work — we just overwrite the
|
|
# crates whose real source is needed for compilation.
|
|
|
|
FROM rust-deps AS gateway-workspace
|
|
ARG OPENSHELL_CARGO_VERSION
|
|
|
|
COPY crates/openshell-core/ crates/openshell-core/
|
|
COPY crates/openshell-policy/ crates/openshell-policy/
|
|
COPY crates/openshell-providers/ crates/openshell-providers/
|
|
COPY crates/openshell-router/ crates/openshell-router/
|
|
COPY crates/openshell-server/ crates/openshell-server/
|
|
|
|
RUN touch \
|
|
crates/openshell-core/build.rs \
|
|
crates/openshell-server/src/main.rs \
|
|
proto/*.proto && \
|
|
if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \
|
|
sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \
|
|
fi
|
|
|
|
FROM gateway-workspace AS gateway-builder
|
|
ARG CARGO_CODEGEN_UNITS
|
|
ARG EXTRA_CARGO_FEATURES=""
|
|
|
|
RUN --mount=type=cache,id=cargo-registry-${TARGETARCH},sharing=locked,target=/usr/local/cargo/registry \
|
|
--mount=type=cache,id=cargo-git-${TARGETARCH},sharing=locked,target=/usr/local/cargo/git \
|
|
--mount=type=cache,id=cargo-target-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \
|
|
--mount=type=cache,id=sccache-${TARGETARCH},sharing=locked,target=/tmp/sccache \
|
|
. cross-build.sh && \
|
|
cargo_cross_build --release -p openshell-server ${EXTRA_CARGO_FEATURES:+--features "$EXTRA_CARGO_FEATURES"} && \
|
|
mkdir -p /build/out && \
|
|
cp "$(cross_output_dir release)/openshell-server" /build/out/
|
|
|
|
FROM rust-deps AS supervisor-workspace
|
|
ARG OPENSHELL_CARGO_VERSION
|
|
|
|
COPY crates/openshell-core/ crates/openshell-core/
|
|
COPY crates/openshell-policy/ crates/openshell-policy/
|
|
COPY crates/openshell-router/ crates/openshell-router/
|
|
COPY crates/openshell-sandbox/ crates/openshell-sandbox/
|
|
|
|
RUN touch \
|
|
crates/openshell-core/build.rs \
|
|
crates/openshell-sandbox/src/main.rs \
|
|
proto/*.proto && \
|
|
if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \
|
|
sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \
|
|
fi
|
|
|
|
FROM supervisor-workspace AS supervisor-builder
|
|
ARG CARGO_CODEGEN_UNITS
|
|
ARG EXTRA_CARGO_FEATURES=""
|
|
|
|
RUN --mount=type=cache,id=cargo-registry-${TARGETARCH},sharing=locked,target=/usr/local/cargo/registry \
|
|
--mount=type=cache,id=cargo-git-${TARGETARCH},sharing=locked,target=/usr/local/cargo/git \
|
|
--mount=type=cache,id=cargo-target-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \
|
|
--mount=type=cache,id=sccache-${TARGETARCH},sharing=locked,target=/tmp/sccache \
|
|
. cross-build.sh && \
|
|
cargo_cross_build --release -p openshell-sandbox ${EXTRA_CARGO_FEATURES:+--features "$EXTRA_CARGO_FEATURES"} && \
|
|
mkdir -p /build/out && \
|
|
cp "$(cross_output_dir release)/openshell-sandbox" /build/out/
|
|
|
|
# Minimal extraction stage for fast-deploy: exports only the supervisor
|
|
# binary (~20-40 MB) instead of the entire build environment (~968 MB).
|
|
FROM scratch AS supervisor-output
|
|
COPY --from=supervisor-builder /build/out/openshell-sandbox /openshell-sandbox
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Final gateway image
|
|
# ---------------------------------------------------------------------------
|
|
FROM nvcr.io/nvidia/base/ubuntu:noble-20251013 AS gateway
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ca-certificates && rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN useradd --create-home --user-group openshell
|
|
|
|
WORKDIR /app
|
|
|
|
COPY --from=gateway-builder /build/out/openshell-server /usr/local/bin/
|
|
|
|
RUN mkdir -p /build/crates/openshell-server
|
|
COPY --chmod=755 crates/openshell-server/migrations /build/crates/openshell-server/migrations
|
|
|
|
USER openshell
|
|
EXPOSE 8080
|
|
|
|
ENTRYPOINT ["openshell-server"]
|
|
CMD ["--port", "8080"]
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cluster asset stages
|
|
# ---------------------------------------------------------------------------
|
|
FROM rancher/k3s:${K3S_VERSION} AS k3s
|
|
|
|
FROM ubuntu:24.04 AS k9s
|
|
ARG K9S_VERSION
|
|
ARG TARGETARCH
|
|
RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \
|
|
curl -fsSL "https://github.com/derailed/k9s/releases/download/${K9S_VERSION}/k9s_Linux_${TARGETARCH}.tar.gz" \
|
|
| tar xz -C /tmp k9s && \
|
|
chmod +x /tmp/k9s && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
FROM ubuntu:24.04 AS helm
|
|
ARG HELM_VERSION
|
|
ARG TARGETARCH
|
|
RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \
|
|
curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" \
|
|
| tar xz --strip-components=1 -C /tmp "linux-${TARGETARCH}/helm" && \
|
|
chmod +x /tmp/helm && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
FROM ubuntu:24.04 AS nvidia-toolkit
|
|
ARG NVIDIA_CONTAINER_TOOLKIT_VERSION
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
gpg curl ca-certificates && \
|
|
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
|
|
| gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg && \
|
|
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
|
|
| sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
|
|
| tee /etc/apt/sources.list.d/nvidia-container-toolkit.list && \
|
|
apt-get update && \
|
|
apt-get install -y --no-install-recommends \
|
|
"nvidia-container-toolkit=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
|
|
"nvidia-container-toolkit-base=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
|
|
"libnvidia-container-tools=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
|
|
"libnvidia-container1=${NVIDIA_CONTAINER_TOOLKIT_VERSION}" && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Final cluster image
|
|
# ---------------------------------------------------------------------------
|
|
FROM nvcr.io/nvidia/base/ubuntu:noble-20251013 AS cluster
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ca-certificates \
|
|
iptables \
|
|
mount \
|
|
dnsutils \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
COPY --from=k3s /bin/ /bin/
|
|
COPY --from=k9s /tmp/k9s /usr/local/bin/k9s
|
|
COPY --from=helm /tmp/helm /usr/local/bin/helm
|
|
COPY --from=k3s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/k3s-ca-certificates.crt
|
|
COPY --from=k3s /usr/share/zoneinfo/ /usr/share/zoneinfo/
|
|
|
|
ENV PATH="/var/lib/rancher/k3s/data/cni:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/bin/aux" \
|
|
CRI_CONFIG_FILE="/var/lib/rancher/k3s/agent/etc/crictl.yaml"
|
|
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-cdi-hook /usr/bin/
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-runtime /usr/bin/
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-runtime-hook /usr/bin/
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-container-cli /usr/bin/
|
|
COPY --from=nvidia-toolkit /usr/bin/nvidia-ctk /usr/bin/
|
|
COPY --from=nvidia-toolkit /etc/nvidia-container-runtime /etc/nvidia-container-runtime
|
|
COPY --from=nvidia-toolkit /usr/lib/*-linux-gnu/libnvidia-container*.so* /usr/lib/
|
|
COPY --from=supervisor-builder /build/out/openshell-sandbox /opt/openshell/bin/openshell-sandbox
|
|
|
|
RUN mkdir -p /var/lib/rancher/k3s/server/manifests \
|
|
/var/lib/rancher/k3s/server/static/charts \
|
|
/etc/rancher/k3s \
|
|
/opt/openshell/manifests \
|
|
/opt/openshell/charts \
|
|
/opt/openshell/gpu-manifests \
|
|
/run/flannel
|
|
|
|
COPY deploy/docker/cluster-entrypoint.sh /usr/local/bin/cluster-entrypoint.sh
|
|
RUN chmod +x /usr/local/bin/cluster-entrypoint.sh
|
|
|
|
COPY deploy/docker/cluster-healthcheck.sh /usr/local/bin/cluster-healthcheck.sh
|
|
RUN chmod +x /usr/local/bin/cluster-healthcheck.sh
|
|
|
|
COPY deploy/docker/.build/charts/*.tgz /opt/openshell/charts/
|
|
COPY deploy/kube/manifests/*.yaml /opt/openshell/manifests/
|
|
COPY deploy/kube/gpu-manifests/*.yaml /opt/openshell/gpu-manifests/
|
|
|
|
ENTRYPOINT ["/usr/local/bin/cluster-entrypoint.sh"]
|
|
|
|
HEALTHCHECK --interval=5s --timeout=5s --start-period=20s --retries=60 \
|
|
CMD ["/usr/local/bin/cluster-healthcheck.sh"]
|