mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-11-11 10:23:17 +08:00
api: move the pause image from ActorTemplate to SandboxConfig (#848)
The pause image holds the sandbox's namespaces and runs no workload code. It is an implementation detail of the sandbox, not something actor authors pick, so it belongs with the sandbox binaries that already moved off the ActorTemplate onto the cluster-scoped SandboxConfig. It now travels with those binaries end to end: resolved from the pool's SandboxConfig, carried on ateletpb.SandboxAssets rather than WorkloadSpec, and recorded in the per-actor sandbox record so Checkpoint pins it into the snapshot manifest and Restore rebuilds the sandbox from the image the snapshot was taken with (the golden's on a DATA_ON_GOLDEN restore). A record without one is rejected outright rather than pulling an empty image. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR
This commit is contained in:
@@ -46,7 +46,6 @@ metadata:
|
|||||||
name: sleep
|
name: sleep
|
||||||
namespace: benchmark-workloads
|
namespace: benchmark-workloads
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
|
|
||||||
containers:
|
containers:
|
||||||
- name: sleep
|
- name: sleep
|
||||||
image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
|
image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
|
||||||
@@ -65,7 +64,6 @@ metadata:
|
|||||||
name: usermem
|
name: usermem
|
||||||
namespace: benchmark-workloads
|
namespace: benchmark-workloads
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
|
|
||||||
containers:
|
containers:
|
||||||
- name: usermem
|
- name: usermem
|
||||||
image: containerstack/alpine-stress:latest
|
image: containerstack/alpine-stress:latest
|
||||||
@@ -84,7 +82,6 @@ metadata:
|
|||||||
name: kernelmem
|
name: kernelmem
|
||||||
namespace: benchmark-workloads
|
namespace: benchmark-workloads
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
|
|
||||||
containers:
|
containers:
|
||||||
- name: kernelmem
|
- name: kernelmem
|
||||||
image: containerstack/alpine-stress:latest
|
image: containerstack/alpine-stress:latest
|
||||||
@@ -103,7 +100,6 @@ metadata:
|
|||||||
name: glutton
|
name: glutton
|
||||||
namespace: benchmark-workloads
|
namespace: benchmark-workloads
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
|
|
||||||
containers:
|
containers:
|
||||||
- name: glutton
|
- name: glutton
|
||||||
image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton
|
image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton
|
||||||
|
|||||||
@@ -42,7 +42,6 @@ metadata:
|
|||||||
spec:
|
spec:
|
||||||
# Must match the WorkerPool's sandboxClass so snapshots stay within a class.
|
# Must match the WorkerPool's sandboxClass so snapshots stay within a class.
|
||||||
sandboxClass: ${SANDBOX_CLASS}
|
sandboxClass: ${SANDBOX_CLASS}
|
||||||
pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
|
|
||||||
containers:
|
containers:
|
||||||
- name: sleep
|
- name: sleep
|
||||||
image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
|
image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e
|
||||||
@@ -62,7 +61,6 @@ metadata:
|
|||||||
namespace: benchmark-workloads
|
namespace: benchmark-workloads
|
||||||
spec:
|
spec:
|
||||||
sandboxClass: ${SANDBOX_CLASS}
|
sandboxClass: ${SANDBOX_CLASS}
|
||||||
pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
|
|
||||||
containers:
|
containers:
|
||||||
- name: glutton
|
- name: glutton
|
||||||
image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton
|
image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton
|
||||||
|
|||||||
@@ -213,7 +213,6 @@ func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) {
|
|||||||
template, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), &atev1alpha1.ActorTemplate{
|
template, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), &atev1alpha1.ActorTemplate{
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: ns},
|
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: ns},
|
||||||
Spec: atev1alpha1.ActorTemplateSpec{
|
Spec: atev1alpha1.ActorTemplateSpec{
|
||||||
PauseImage: "pause@sha256:abc",
|
|
||||||
SnapshotsConfig: atev1alpha1.SnapshotsConfig{Location: "gs://snapshots"},
|
SnapshotsConfig: atev1alpha1.SnapshotsConfig{Location: "gs://snapshots"},
|
||||||
Containers: []atev1alpha1.Container{{
|
Containers: []atev1alpha1.Container{{
|
||||||
Name: "main", Image: "main@sha256:abc", VolumeMounts: []atev1alpha1.VolumeMount{{Name: "data", MountPath: "/data"}},
|
Name: "main", Image: "main@sha256:abc", VolumeMounts: []atev1alpha1.VolumeMount{{Name: "data", MountPath: "/data"}},
|
||||||
|
|||||||
@@ -545,7 +545,6 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st
|
|||||||
Namespace: ns,
|
Namespace: ns,
|
||||||
},
|
},
|
||||||
Spec: atev1alpha1.ActorTemplateSpec{
|
Spec: atev1alpha1.ActorTemplateSpec{
|
||||||
PauseImage: "pause@sha256:abc",
|
|
||||||
SnapshotsConfig: atev1alpha1.SnapshotsConfig{
|
SnapshotsConfig: atev1alpha1.SnapshotsConfig{
|
||||||
Location: "gs://fake-fake-fake",
|
Location: "gs://fake-fake-fake",
|
||||||
},
|
},
|
||||||
@@ -594,6 +593,10 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// testPauseImage is the pause image the default test SandboxConfig carries;
|
||||||
|
// it is what a resolved WorkloadSpec's sandbox assets should name.
|
||||||
|
const testPauseImage = "pause@sha256:abc"
|
||||||
|
|
||||||
// ensureDefaultGvisorSandboxConfig creates the cluster-scoped default gvisor
|
// ensureDefaultGvisorSandboxConfig creates the cluster-scoped default gvisor
|
||||||
// SandboxConfig (idempotently) and waits for it to appear in the lister.
|
// SandboxConfig (idempotently) and waits for it to appear in the lister.
|
||||||
func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) {
|
func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) {
|
||||||
@@ -604,6 +607,7 @@ func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) {
|
|||||||
Spec: atev1alpha1.SandboxConfigSpec{
|
Spec: atev1alpha1.SandboxConfigSpec{
|
||||||
SandboxClass: atev1alpha1.SandboxClassGvisor,
|
SandboxClass: atev1alpha1.SandboxClassGvisor,
|
||||||
Default: true,
|
Default: true,
|
||||||
|
PauseImage: testPauseImage,
|
||||||
Assets: map[string]map[string]atev1alpha1.AssetFile{
|
Assets: map[string]map[string]atev1alpha1.AssetFile{
|
||||||
"amd64": {"runsc": {
|
"amd64": {"runsc": {
|
||||||
URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc",
|
URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc",
|
||||||
@@ -663,7 +667,6 @@ func createTemplateWithSelector(t *testing.T, tc *testContext, ns string, name s
|
|||||||
Namespace: ns,
|
Namespace: ns,
|
||||||
},
|
},
|
||||||
Spec: atev1alpha1.ActorTemplateSpec{
|
Spec: atev1alpha1.ActorTemplateSpec{
|
||||||
PauseImage: "pause@sha256:abc",
|
|
||||||
SnapshotsConfig: atev1alpha1.SnapshotsConfig{
|
SnapshotsConfig: atev1alpha1.SnapshotsConfig{
|
||||||
Location: "gs://fake-fake-fake",
|
Location: "gs://fake-fake-fake",
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -23,10 +23,11 @@ import (
|
|||||||
"k8s.io/apimachinery/pkg/labels"
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
)
|
)
|
||||||
|
|
||||||
// resolveSandboxAssets determines the sandbox binaries an actor should boot with
|
// resolveSandboxAssets determines the sandbox binaries and pause image an actor
|
||||||
// and projects them onto the ateletpb.SandboxAssets atelet fetches. It takes the
|
// should boot with and projects them onto the ateletpb.SandboxAssets atelet
|
||||||
// SandboxClass (default gvisor) of a given worker pool, then picks the SandboxConfig
|
// fetches. It takes the SandboxClass (default gvisor) of a given worker pool,
|
||||||
// named by the pool — or, if none is named, the cluster default SandboxConfig for that class.
|
// then picks the SandboxConfig named by the pool — or, if none is named, the
|
||||||
|
// cluster default SandboxConfig for that class.
|
||||||
func resolveSandboxAssets(
|
func resolveSandboxAssets(
|
||||||
workerPoolLister listersv1alpha1.WorkerPoolLister,
|
workerPoolLister listersv1alpha1.WorkerPoolLister,
|
||||||
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
|
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
|
||||||
@@ -89,6 +90,7 @@ func defaultSandboxConfig(lister listersv1alpha1.SandboxConfigLister, class atev
|
|||||||
func sandboxAssetsProto(class atev1alpha1.SandboxClass, sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets {
|
func sandboxAssetsProto(class atev1alpha1.SandboxClass, sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets {
|
||||||
out := &ateletpb.SandboxAssets{
|
out := &ateletpb.SandboxAssets{
|
||||||
SandboxClass: string(class),
|
SandboxClass: string(class),
|
||||||
|
PauseImage: sc.Spec.PauseImage,
|
||||||
Assets: make(map[string]*ateletpb.ArchAssets, len(sc.Spec.Assets)),
|
Assets: make(map[string]*ateletpb.ArchAssets, len(sc.Spec.Assets)),
|
||||||
}
|
}
|
||||||
for arch, files := range sc.Spec.Assets {
|
for arch, files := range sc.Spec.Assets {
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
// Copyright 2026 Google LLC
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package controlapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
|
||||||
|
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/client-go/tools/cache"
|
||||||
|
)
|
||||||
|
|
||||||
|
// listerFor builds listers over the given objects, using the same key
|
||||||
|
// functions the informers use.
|
||||||
|
func listersFor(t *testing.T, pools []*atev1alpha1.WorkerPool, configs []*atev1alpha1.SandboxConfig) (listersv1alpha1.WorkerPoolLister, listersv1alpha1.SandboxConfigLister) {
|
||||||
|
t.Helper()
|
||||||
|
poolIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
|
||||||
|
for _, p := range pools {
|
||||||
|
if err := poolIdx.Add(p); err != nil {
|
||||||
|
t.Fatalf("adding WorkerPool: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
configIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{})
|
||||||
|
for _, c := range configs {
|
||||||
|
if err := configIdx.Add(c); err != nil {
|
||||||
|
t.Fatalf("adding SandboxConfig: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return listersv1alpha1.NewWorkerPoolLister(poolIdx), listersv1alpha1.NewSandboxConfigLister(configIdx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAssets() map[string]map[string]atev1alpha1.AssetFile {
|
||||||
|
return map[string]map[string]atev1alpha1.AssetFile{
|
||||||
|
"amd64": {"gvisor": {URL: "gs://bucket/gvisor.tar.bz2", SHA256: "abc"}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResolveSandboxAssetsCarriesPauseImage pins that the pause image travels
|
||||||
|
// with the sandbox binaries — it is resolved from the pool's SandboxConfig, not
|
||||||
|
// from the ActorTemplate — for both the named and the class-default config.
|
||||||
|
func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) {
|
||||||
|
const (
|
||||||
|
defaultPause = "registry.k8s.io/pause@sha256:default"
|
||||||
|
namedPause = "gcr.io/gke-release/pause@sha256:named"
|
||||||
|
)
|
||||||
|
defaultConfig := &atev1alpha1.SandboxConfig{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"},
|
||||||
|
Spec: atev1alpha1.SandboxConfigSpec{
|
||||||
|
SandboxClass: atev1alpha1.SandboxClassGvisor,
|
||||||
|
Default: true,
|
||||||
|
PauseImage: defaultPause,
|
||||||
|
Assets: testAssets(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
namedConfig := &atev1alpha1.SandboxConfig{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: "gvisor-custom"},
|
||||||
|
Spec: atev1alpha1.SandboxConfigSpec{
|
||||||
|
SandboxClass: atev1alpha1.SandboxClassGvisor,
|
||||||
|
PauseImage: namedPause,
|
||||||
|
Assets: testAssets(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
configName string
|
||||||
|
wantPauseImage string
|
||||||
|
}{
|
||||||
|
{name: "class default", wantPauseImage: defaultPause},
|
||||||
|
{name: "named config", configName: "gvisor-custom", wantPauseImage: namedPause},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pool := &atev1alpha1.WorkerPool{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: "pool1", Namespace: "worker-ns"},
|
||||||
|
Spec: atev1alpha1.WorkerPoolSpec{SandboxConfigName: tt.configName},
|
||||||
|
}
|
||||||
|
poolLister, configLister := listersFor(t, []*atev1alpha1.WorkerPool{pool},
|
||||||
|
[]*atev1alpha1.SandboxConfig{defaultConfig, namedConfig})
|
||||||
|
|
||||||
|
got, err := resolveSandboxAssets(poolLister, configLister, "worker-ns", "pool1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveSandboxAssets() error: %v", err)
|
||||||
|
}
|
||||||
|
if got.GetPauseImage() != tt.wantPauseImage {
|
||||||
|
t.Errorf("pause image = %q, want %q", got.GetPauseImage(), tt.wantPauseImage)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -37,9 +37,7 @@ const envSecretCacheTTL = 30 * time.Second
|
|||||||
// container env vars. Use this when downstream consumers (e.g. checkpoint
|
// container env vars. Use this when downstream consumers (e.g. checkpoint
|
||||||
// requests) don't need env entries materialized.
|
// requests) don't need env entries materialized.
|
||||||
func workloadSpecFromActorTemplate(actorTemplate *atev1alpha1.ActorTemplate, actor *ateapipb.Actor) (*ateletpb.WorkloadSpec, error) {
|
func workloadSpecFromActorTemplate(actorTemplate *atev1alpha1.ActorTemplate, actor *ateapipb.Actor) (*ateletpb.WorkloadSpec, error) {
|
||||||
workloadSpec := &ateletpb.WorkloadSpec{
|
workloadSpec := &ateletpb.WorkloadSpec{}
|
||||||
PauseImage: actorTemplate.Spec.PauseImage,
|
|
||||||
}
|
|
||||||
|
|
||||||
// add volumes
|
// add volumes
|
||||||
for _, vol := range actorTemplate.Spec.Volumes {
|
for _, vol := range actorTemplate.Spec.Volumes {
|
||||||
|
|||||||
@@ -43,7 +43,6 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) {
|
|||||||
template: &atev1alpha1.ActorTemplate{
|
template: &atev1alpha1.ActorTemplate{
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
|
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
|
||||||
Spec: atev1alpha1.ActorTemplateSpec{
|
Spec: atev1alpha1.ActorTemplateSpec{
|
||||||
PauseImage: "pause",
|
|
||||||
Volumes: []atev1alpha1.Volume{
|
Volumes: []atev1alpha1.Volume{
|
||||||
{Name: "home", VolumeSource: atev1alpha1.VolumeSource{DurableDir: &atev1alpha1.DurableDirVolumeSource{}}},
|
{Name: "home", VolumeSource: atev1alpha1.VolumeSource{DurableDir: &atev1alpha1.DurableDirVolumeSource{}}},
|
||||||
},
|
},
|
||||||
@@ -60,7 +59,6 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
want: &ateletpb.WorkloadSpec{
|
want: &ateletpb.WorkloadSpec{
|
||||||
PauseImage: "pause",
|
|
||||||
Volumes: []*ateletpb.Volume{
|
Volumes: []*ateletpb.Volume{
|
||||||
{
|
{
|
||||||
Name: "home",
|
Name: "home",
|
||||||
@@ -227,7 +225,6 @@ func TestWorkloadSpecFromActorTemplateWithEnv(t *testing.T) {
|
|||||||
template: &atev1alpha1.ActorTemplate{
|
template: &atev1alpha1.ActorTemplate{
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
|
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
|
||||||
Spec: atev1alpha1.ActorTemplateSpec{
|
Spec: atev1alpha1.ActorTemplateSpec{
|
||||||
PauseImage: "pause",
|
|
||||||
Containers: []atev1alpha1.Container{
|
Containers: []atev1alpha1.Container{
|
||||||
{
|
{
|
||||||
Name: "main",
|
Name: "main",
|
||||||
@@ -247,7 +244,6 @@ func TestWorkloadSpecFromActorTemplateWithEnv(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
want: &ateletpb.WorkloadSpec{
|
want: &ateletpb.WorkloadSpec{
|
||||||
PauseImage: "pause",
|
|
||||||
Containers: []*ateletpb.Container{
|
Containers: []*ateletpb.Container{
|
||||||
{
|
{
|
||||||
Name: "main",
|
Name: "main",
|
||||||
|
|||||||
+14
-10
@@ -422,7 +422,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := s.prepareOCIBundles(ctx, actorUID, actorRef.Name,
|
if err := s.prepareOCIBundles(ctx, actorUID, actorRef.Name,
|
||||||
req.GetSpec(), req.GetTargetAteomUid(),
|
req.GetSpec(), sandboxRec.PauseImage, req.GetTargetAteomUid(),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonInvalidContainerConfig)
|
return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonInvalidContainerConfig)
|
||||||
}
|
}
|
||||||
@@ -963,12 +963,12 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest)
|
|||||||
op.kind = restoreSnapshotKind(req, sandboxRec)
|
op.kind = restoreSnapshotKind(req, sandboxRec)
|
||||||
op.sandboxClass = sandboxRec.SandboxClass
|
op.sandboxClass = sandboxRec.SandboxClass
|
||||||
|
|
||||||
// The record whose pinned binaries run the restored workload: the golden's
|
// The record whose pinned sandbox (binaries + pause image) runs the restored
|
||||||
// for a DATA_ON_GOLDEN restore, the snapshot's own otherwise. The golden's
|
// workload: the golden's for a DATA_ON_GOLDEN restore, the snapshot's own
|
||||||
// set wins because the guest state being resumed is the golden snapshot's
|
// otherwise. The golden's set wins because the guest state being resumed is
|
||||||
// memory image, and a memory image must be resumed by the exact binary
|
// the golden snapshot's memory image, and a memory image must be resumed by
|
||||||
// versions that produced it; the actor's snapshot contributes only durable
|
// the exact sandbox that produced it; the actor's snapshot contributes only
|
||||||
// data (a plain tar), which no binary version reads back.
|
// durable data (a plain tar), which no sandbox version reads back.
|
||||||
runtimeRec := sandboxRec
|
runtimeRec := sandboxRec
|
||||||
if goldenRec != nil {
|
if goldenRec != nil {
|
||||||
runtimeRec = goldenRec
|
runtimeRec = goldenRec
|
||||||
@@ -1043,7 +1043,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest)
|
|||||||
return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError, ateerrors.ReasonInvalidSandboxAsset)
|
return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError, ateerrors.ReasonInvalidSandboxAsset)
|
||||||
}
|
}
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
err = s.prepareOCIBundles(gctx, actorUID, actorRef.Name, req.GetSpec(), req.GetTargetAteomUid())
|
err = s.prepareOCIBundles(gctx, actorUID, actorRef.Name, req.GetSpec(), runtimeRec.PauseImage, req.GetTargetAteomUid())
|
||||||
dBundles = time.Since(t)
|
dBundles = time.Since(t)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack
|
prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack
|
||||||
@@ -1339,12 +1339,16 @@ func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotUR
|
|||||||
}
|
}
|
||||||
|
|
||||||
// prepareOCIBundles pulls images and assembles OCI bundles for the pause
|
// prepareOCIBundles pulls images and assembles OCI bundles for the pause
|
||||||
// container and every application container in spec, in parallel.
|
// container and every application container in spec, in parallel. pauseImage
|
||||||
|
// comes from the sandbox record, not the workload spec: it is sandbox
|
||||||
|
// configuration, and on a restore it must be the image the snapshot was taken
|
||||||
|
// with.
|
||||||
func (s *AteomHerder) prepareOCIBundles(
|
func (s *AteomHerder) prepareOCIBundles(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
actorUID string,
|
actorUID string,
|
||||||
actorName string,
|
actorName string,
|
||||||
spec *ateletpb.WorkloadSpec,
|
spec *ateletpb.WorkloadSpec,
|
||||||
|
pauseImage string,
|
||||||
targetAteomUid string,
|
targetAteomUid string,
|
||||||
) error {
|
) error {
|
||||||
// Populate the per-actor identity directory that gets bind-mounted into
|
// Populate the per-actor identity directory that gets bind-mounted into
|
||||||
@@ -1390,7 +1394,7 @@ func (s *AteomHerder) prepareOCIBundles(
|
|||||||
s.imageCache,
|
s.imageCache,
|
||||||
actorUID,
|
actorUID,
|
||||||
"pause",
|
"pause",
|
||||||
spec.GetPauseImage(),
|
pauseImage,
|
||||||
[]string{"/pause"},
|
[]string{"/pause"},
|
||||||
nil,
|
nil,
|
||||||
nil,
|
nil,
|
||||||
|
|||||||
+13
-2
@@ -73,7 +73,7 @@ func TestSnapshotManifestActorMetadata(t *testing.T) {
|
|||||||
// written before the scope field existed must still parse, reporting an empty
|
// written before the scope field existed must still parse, reporting an empty
|
||||||
// scope, and a scope-less record must not serialize a scope key at all.
|
// scope, and a scope-less record must not serialize a scope key at all.
|
||||||
func TestSnapshotManifestScopeAbsent(t *testing.T) {
|
func TestSnapshotManifestScopeAbsent(t *testing.T) {
|
||||||
legacy := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`)
|
legacy := []byte(`{"sandboxClass":"gvisor","pauseImage":"pause@sha256:abc","snapshotFiles":["checkpoint.img"]}`)
|
||||||
rec, err := unmarshalSandboxRecord(legacy)
|
rec, err := unmarshalSandboxRecord(legacy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err)
|
t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err)
|
||||||
@@ -91,6 +91,18 @@ func TestSnapshotManifestScopeAbsent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSnapshotManifestRequiresPauseImage pins that a manifest without a pause
|
||||||
|
// image is rejected outright rather than yielding an empty image that would
|
||||||
|
// fail later, deep in the image pull.
|
||||||
|
func TestSnapshotManifestRequiresPauseImage(t *testing.T) {
|
||||||
|
noPause := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`)
|
||||||
|
if _, err := unmarshalSandboxRecord(noPause); err == nil {
|
||||||
|
t.Fatal("unmarshalSandboxRecord accepted a manifest with no pauseImage")
|
||||||
|
} else if !strings.Contains(err.Error(), "pauseImage") {
|
||||||
|
t.Errorf("error = %v, want it to name pauseImage", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWriteFileAtomic(t *testing.T) {
|
func TestWriteFileAtomic(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
target := filepath.Join(dir, "actor-id")
|
target := filepath.Join(dir, "actor-id")
|
||||||
@@ -639,7 +651,6 @@ func TestRPCBoundariesReject(t *testing.T) {
|
|||||||
|
|
||||||
func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) {
|
func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) {
|
||||||
in := &ateletpb.WorkloadSpec{
|
in := &ateletpb.WorkloadSpec{
|
||||||
PauseImage: "pause",
|
|
||||||
Containers: []*ateletpb.Container{
|
Containers: []*ateletpb.Container{
|
||||||
{
|
{
|
||||||
Name: "with-probe",
|
Name: "with-probe",
|
||||||
|
|||||||
@@ -67,15 +67,20 @@ type assetEntry struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto
|
// sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto
|
||||||
// the local node's architecture: the sandbox class plus the asset set keyed by
|
// the local node's architecture: the sandbox class and pause image plus the
|
||||||
// asset name (gVisor uses a single "gvisor" release-tarball asset; records
|
// asset set keyed by asset name (gVisor uses a single "gvisor" release-tarball
|
||||||
// written before the tarball release mechanism use a bare "runsc" asset).
|
// asset; records written before the tarball release mechanism use a bare
|
||||||
|
// "runsc" asset).
|
||||||
// It is both the per-actor on-node record (written at Run/Restore, read at
|
// It is both the per-actor on-node record (written at Run/Restore, read at
|
||||||
// Checkpoint) and the snapshot manifest (written at Checkpoint, read at
|
// Checkpoint) and the snapshot manifest (written at Checkpoint, read at
|
||||||
// Restore).
|
// Restore).
|
||||||
type sandboxAssetsRecord struct {
|
type sandboxAssetsRecord struct {
|
||||||
SandboxClass string `json:"sandboxClass"`
|
SandboxClass string `json:"sandboxClass"`
|
||||||
Assets map[string]assetEntry `json:"assets"`
|
Assets map[string]assetEntry `json:"assets"`
|
||||||
|
// PauseImage is the root sandbox container's image. It is recorded here
|
||||||
|
// rather than taken from the request at Restore so a snapshot is rebuilt
|
||||||
|
// with the same sandbox it was captured from.
|
||||||
|
PauseImage string `json:"pauseImage"`
|
||||||
// Actor identity makes a flat snapshot self-identifying if control-plane
|
// Actor identity makes a flat snapshot self-identifying if control-plane
|
||||||
// persistence is unavailable.
|
// persistence is unavailable.
|
||||||
Atespace string `json:"atespace,omitempty"`
|
Atespace string `json:"atespace,omitempty"`
|
||||||
@@ -107,8 +112,12 @@ func recordFromRequest(sa *ateletpb.SandboxAssets) (*sandboxAssetsRecord, error)
|
|||||||
if archAssets == nil || len(archAssets.GetFiles()) == 0 {
|
if archAssets == nil || len(archAssets.GetFiles()) == 0 {
|
||||||
return nil, fmt.Errorf("sandbox_assets has no assets for architecture %q", arch)
|
return nil, fmt.Errorf("sandbox_assets has no assets for architecture %q", arch)
|
||||||
}
|
}
|
||||||
|
if sa.GetPauseImage() == "" {
|
||||||
|
return nil, fmt.Errorf("sandbox_assets has no pause_image")
|
||||||
|
}
|
||||||
rec := &sandboxAssetsRecord{
|
rec := &sandboxAssetsRecord{
|
||||||
SandboxClass: sa.GetSandboxClass(),
|
SandboxClass: sa.GetSandboxClass(),
|
||||||
|
PauseImage: sa.GetPauseImage(),
|
||||||
Assets: make(map[string]assetEntry, len(archAssets.GetFiles())),
|
Assets: make(map[string]assetEntry, len(archAssets.GetFiles())),
|
||||||
}
|
}
|
||||||
for name, f := range archAssets.GetFiles() {
|
for name, f := range archAssets.GetFiles() {
|
||||||
@@ -416,6 +425,12 @@ func unmarshalSandboxRecord(data []byte) (*sandboxAssetsRecord, error) {
|
|||||||
if err := json.Unmarshal(data, rec); err != nil {
|
if err := json.Unmarshal(data, rec); err != nil {
|
||||||
return nil, fmt.Errorf("%w: while parsing sandbox record/manifest: %w", ateerrors.ReasonInvalidSandboxAsset, err)
|
return nil, fmt.Errorf("%w: while parsing sandbox record/manifest: %w", ateerrors.ReasonInvalidSandboxAsset, err)
|
||||||
}
|
}
|
||||||
|
// Fail loudly rather than let an empty image reach the image pull: a record
|
||||||
|
// without one predates the pause image moving into the sandbox config, and
|
||||||
|
// its snapshot cannot be rebuilt with a known-matching sandbox.
|
||||||
|
if rec.PauseImage == "" {
|
||||||
|
return nil, fmt.Errorf("%w: sandbox record/manifest has no pauseImage", ateerrors.ReasonInvalidSandboxAsset)
|
||||||
|
}
|
||||||
return rec, nil
|
return rec, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -45,7 +45,6 @@ metadata:
|
|||||||
name: counter
|
name: counter
|
||||||
namespace: ate-demo-autoscaled-workerpool
|
namespace: ate-demo-autoscaled-workerpool
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: counter
|
- name: counter
|
||||||
image: ko://github.com/agent-substrate/substrate/demos/counter
|
image: ko://github.com/agent-substrate/substrate/demos/counter
|
||||||
|
|||||||
@@ -65,7 +65,6 @@ metadata:
|
|||||||
name: agent-luna
|
name: agent-luna
|
||||||
namespace: claude-multiplex-demo
|
namespace: claude-multiplex-demo
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: claude
|
- name: claude
|
||||||
image: ${WORKLOAD_IMAGE}
|
image: ${WORKLOAD_IMAGE}
|
||||||
@@ -95,7 +94,6 @@ metadata:
|
|||||||
name: agent-mars
|
name: agent-mars
|
||||||
namespace: claude-multiplex-demo
|
namespace: claude-multiplex-demo
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: claude
|
- name: claude
|
||||||
image: ${WORKLOAD_IMAGE}
|
image: ${WORKLOAD_IMAGE}
|
||||||
@@ -125,7 +123,6 @@ metadata:
|
|||||||
name: agent-orion
|
name: agent-orion
|
||||||
namespace: claude-multiplex-demo
|
namespace: claude-multiplex-demo
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: claude
|
- name: claude
|
||||||
image: ${WORKLOAD_IMAGE}
|
image: ${WORKLOAD_IMAGE}
|
||||||
|
|||||||
@@ -51,7 +51,6 @@ spec:
|
|||||||
# sandbox classes, so only pools of the same class are eligible to run this
|
# sandbox classes, so only pools of the same class are eligible to run this
|
||||||
# template's actors.
|
# template's actors.
|
||||||
sandboxClass: microvm
|
sandboxClass: microvm
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: counter
|
- name: counter
|
||||||
image: ko://github.com/agent-substrate/substrate/demos/counter
|
image: ko://github.com/agent-substrate/substrate/demos/counter
|
||||||
|
|||||||
@@ -38,7 +38,6 @@ metadata:
|
|||||||
name: counter
|
name: counter
|
||||||
namespace: ate-demo-counter
|
namespace: ate-demo-counter
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: counter
|
- name: counter
|
||||||
image: ko://github.com/agent-substrate/substrate/demos/counter
|
image: ko://github.com/agent-substrate/substrate/demos/counter
|
||||||
|
|||||||
@@ -54,7 +54,6 @@ metadata:
|
|||||||
name: counter
|
name: counter
|
||||||
namespace: ate-demo-multi-template-counter
|
namespace: ate-demo-multi-template-counter
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: counter
|
- name: counter
|
||||||
image: ko://github.com/agent-substrate/substrate/demos/counter
|
image: ko://github.com/agent-substrate/substrate/demos/counter
|
||||||
@@ -70,7 +69,6 @@ metadata:
|
|||||||
name: fspersist
|
name: fspersist
|
||||||
namespace: ate-demo-multi-template-fspersist
|
namespace: ate-demo-multi-template-fspersist
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: fspersist
|
- name: fspersist
|
||||||
image: ko://github.com/agent-substrate/substrate/demos/multi-template/fspersist
|
image: ko://github.com/agent-substrate/substrate/demos/multi-template/fspersist
|
||||||
|
|||||||
@@ -50,7 +50,6 @@ metadata:
|
|||||||
name: parking
|
name: parking
|
||||||
namespace: ate-demo-parking
|
namespace: ate-demo-parking
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: counter
|
- name: counter
|
||||||
image: ko://github.com/agent-substrate/substrate/demos/counter
|
image: ko://github.com/agent-substrate/substrate/demos/counter
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ spec:
|
|||||||
workerSelector:
|
workerSelector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
workload: sandbox
|
workload: sandbox
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: sandbox
|
- name: sandbox
|
||||||
image: ko://github.com/agent-substrate/substrate/demos/sandbox
|
image: ko://github.com/agent-substrate/substrate/demos/sandbox
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ spec:
|
|||||||
workerSelector:
|
workerSelector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
workload: sandbox
|
workload: sandbox
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: sandbox
|
- name: sandbox
|
||||||
image: ko://github.com/agent-substrate/substrate/demos/sandbox
|
image: ko://github.com/agent-substrate/substrate/demos/sandbox
|
||||||
|
|||||||
+8
-9
@@ -13,7 +13,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage
|
|||||||
| `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. |
|
| `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. |
|
||||||
| `ateomImage` | `string` | **Required.** The container image for the `ateom` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor`). |
|
| `ateomImage` | `string` | **Required.** The container image for the `ateom` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor`). |
|
||||||
| `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement) and which `SandboxConfig`s are eligible. |
|
| `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement) and which `SandboxConfig`s are eligible. |
|
||||||
| `sandboxConfigName` | `string` | Optional. Name of a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-sandbox-binaries) providing the sandbox binaries. If empty, the cluster default `SandboxConfig` for the pool's `sandboxClass` is used. |
|
| `sandboxConfigName` | `string` | Optional. Name of a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) providing the sandbox binaries and pause image. If empty, the cluster default `SandboxConfig` for the pool's `sandboxClass` is used. |
|
||||||
| `template` | `WorkerPoolPodTemplate` | **Optional.** Pod scheduling and resource settings for worker pods. |
|
| `template` | `WorkerPoolPodTemplate` | **Optional.** Pod scheduling and resource settings for worker pods. |
|
||||||
|
|
||||||
#### `WorkerPoolPodTemplate` (`spec.template`)
|
#### `WorkerPoolPodTemplate` (`spec.template`)
|
||||||
@@ -144,10 +144,9 @@ The `ActorTemplate` defines the code, environment, and state-management policies
|
|||||||
| `sandboxClass` | `string` | Optional. The sandbox runtime family this template's actors require: `gvisor` (default) or `microvm`. Only `WorkerPool`s whose `sandboxClass` matches are eligible. |
|
| `sandboxClass` | `string` | Optional. The sandbox runtime family this template's actors require: `gvisor` (default) or `microvm`. Only `WorkerPool`s whose `sandboxClass` matches are eligible. |
|
||||||
| `workerSelector` | `*LabelSelector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels. If unset, all pools are eligible (subject to the actor's own `worker_selector`). |
|
| `workerSelector` | `*LabelSelector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels. If unset, all pools are eligible (subject to the actor's own `worker_selector`). |
|
||||||
| `snapshotsConfig` | `SnapshotsConfig` | **Required.** The base object-storage location snapshots are written under, plus the pause/commit/resume scopes. See [Snapshot Storage Layout](#snapshot-storage-layout). |
|
| `snapshotsConfig` | `SnapshotsConfig` | **Required.** The base object-storage location snapshots are written under, plus the pause/commit/resume scopes. See [Snapshot Storage Layout](#snapshot-storage-layout). |
|
||||||
| `pauseImage` | `string` | **Required.** The image used for the sandbox root (e.g. `gcr.io/gke-release/pause`). |
|
|
||||||
| `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each either a `durableDir` or an `externalVolumeTemplate`. Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one, and `externalVolumeTemplate` is `gvisor`-only. |
|
| `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each either a `durableDir` or an `externalVolumeTemplate`. Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one, and `externalVolumeTemplate` is `gvisor`-only. |
|
||||||
|
|
||||||
The sandbox binaries (e.g. the gVisor `runsc` binary) are **no longer configured on the `ActorTemplate`**. They are resolved from the referenced `WorkerPool`'s [`SandboxConfig`](#3-sandboxconfig-sandbox-binaries) — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`.
|
The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — is **not configured on the `ActorTemplate`**. It is resolved from the referenced `WorkerPool`'s [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`.
|
||||||
|
|
||||||
Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It defaults to `gvisor` and, like the rest of the spec, is immutable, so each template's class is fixed at creation.
|
Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It defaults to `gvisor` and, like the rest of the spec, is immutable, so each template's class is fixed at creation.
|
||||||
|
|
||||||
@@ -207,10 +206,8 @@ metadata:
|
|||||||
name: secret-agent
|
name: secret-agent
|
||||||
namespace: ate-demo
|
namespace: ate-demo
|
||||||
spec:
|
spec:
|
||||||
# No sandbox/runsc config here — the binaries come from the WorkerPool's
|
# No sandbox config here — the binaries and pause image come from the
|
||||||
# SandboxConfig (see section 3).
|
# WorkerPool's SandboxConfig (see section 3).
|
||||||
# GKE clusters: use gcr.io/gke-release/pause. Other clusters: registry.k8s.io/pause:3.9
|
|
||||||
pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
|
|
||||||
containers:
|
containers:
|
||||||
- name: agent
|
- name: agent
|
||||||
image: gcr.io/my-project/my-agent:latest
|
image: gcr.io/my-project/my-agent:latest
|
||||||
@@ -261,9 +258,9 @@ Two consequences worth planning for:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. SandboxConfig: Sandbox Binaries
|
## 3. SandboxConfig: The Sandbox Itself
|
||||||
|
|
||||||
`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) from the `ActorTemplate`. A `WorkerPool` resolves its binaries from a `SandboxConfig` — either the one named by `spec.sandboxConfigName`, or the cluster default for the pool's `sandboxClass`.
|
`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the `ActorTemplate`. A `WorkerPool` resolves its sandbox from a `SandboxConfig` — either the one named by `spec.sandboxConfigName`, or the cluster default for the pool's `sandboxClass`.
|
||||||
|
|
||||||
This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place.
|
This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place.
|
||||||
|
|
||||||
@@ -272,6 +269,7 @@ This means a single, cluster-managed config pins the sandbox runtime version for
|
|||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| :--- | :--- | :--- |
|
| :--- | :--- | :--- |
|
||||||
| `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. A `WorkerPool` only uses `SandboxConfig`s whose `sandboxClass` matches its own. |
|
| `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. A `WorkerPool` only uses `SandboxConfig`s whose `sandboxClass` matches its own. |
|
||||||
|
| `pauseImage` | `string` | **Required.** The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must be pinned by digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. |
|
||||||
| `default` | `bool` | Optional. Marks this as the cluster default for its `sandboxClass`. A `WorkerPool` with no `sandboxConfigName` resolves to the default for its class. At most one default per class. |
|
| `default` | `bool` | Optional. Marks this as the cluster default for its `sandboxClass`. A `WorkerPool` with no `sandboxConfigName` resolves to the default for its class. At most one default per class. |
|
||||||
| `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.bz2`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. |
|
| `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.bz2`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. |
|
||||||
|
|
||||||
@@ -287,6 +285,7 @@ metadata:
|
|||||||
spec:
|
spec:
|
||||||
sandboxClass: gvisor
|
sandboxClass: gvisor
|
||||||
default: true
|
default: true
|
||||||
|
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
||||||
assets:
|
assets:
|
||||||
amd64:
|
amd64:
|
||||||
gvisor:
|
gvisor:
|
||||||
|
|||||||
@@ -324,7 +324,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov
|
|||||||
|
|
||||||
### Sandbox Classes
|
### Sandbox Classes
|
||||||
|
|
||||||
A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they are fetched at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-sandbox-binaries) and pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades.
|
A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades.
|
||||||
|
|
||||||
* **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree.
|
* **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree.
|
||||||
|
|
||||||
|
|||||||
+4
-3
@@ -19,9 +19,10 @@ For how the pieces fit together, see the [Architecture](architecture.md) and
|
|||||||
|
|
||||||
- **SandboxConfig**: a cluster-scoped resource holding the sandbox binaries for
|
- **SandboxConfig**: a cluster-scoped resource holding the sandbox binaries for
|
||||||
one runtime family (the gVisor `runsc` binary, or a micro-VM
|
one runtime family (the gVisor `runsc` binary, or a micro-VM
|
||||||
kernel/firmware/config). A `WorkerPool` resolves its binaries from the config
|
kernel/firmware/config), plus the pause image for the sandbox's root
|
||||||
it names, or from the cluster default for its class, so one config pins the
|
container. A `WorkerPool` resolves its sandbox from the config it names, or
|
||||||
runtime version for many templates.
|
from the cluster default for its class, so one config pins the runtime version
|
||||||
|
for many templates.
|
||||||
|
|
||||||
## Records (dynamic state, in the control-plane store)
|
## Records (dynamic state, in the control-plane store)
|
||||||
|
|
||||||
|
|||||||
@@ -38,7 +38,6 @@ metadata:
|
|||||||
name: probe
|
name: probe
|
||||||
namespace: ate-e2e-probe
|
namespace: ate-e2e-probe
|
||||||
spec:
|
spec:
|
||||||
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
|
||||||
containers:
|
containers:
|
||||||
- name: probe
|
- name: probe
|
||||||
image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe
|
image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe
|
||||||
|
|||||||
@@ -934,7 +934,6 @@ func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e
|
|||||||
// ActorTemplate↔WorkerPool match succeeds. The micro-VM source sets
|
// ActorTemplate↔WorkerPool match succeeds. The micro-VM source sets
|
||||||
// "microvm"; the gVisor source leaves it "" — copying keeps both correct.
|
// "microvm"; the gVisor source leaves it "" — copying keeps both correct.
|
||||||
SandboxClass: existingAt.Spec.SandboxClass,
|
SandboxClass: existingAt.Spec.SandboxClass,
|
||||||
PauseImage: existingAt.Spec.PauseImage,
|
|
||||||
Containers: existingAt.Spec.Containers,
|
Containers: existingAt.Spec.Containers,
|
||||||
SnapshotsConfig: v1alpha1.SnapshotsConfig{
|
SnapshotsConfig: v1alpha1.SnapshotsConfig{
|
||||||
Location: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name,
|
Location: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name,
|
||||||
|
|||||||
@@ -326,7 +326,6 @@ func setupDemoCounterTemplate(ctx context.Context, t *testing.T, clients *e2e.Cl
|
|||||||
MatchLabels: map[string]string{"netpol-test": ns},
|
MatchLabels: map[string]string{"netpol-test": ns},
|
||||||
},
|
},
|
||||||
SandboxClass: existingAt.Spec.SandboxClass,
|
SandboxClass: existingAt.Spec.SandboxClass,
|
||||||
PauseImage: existingAt.Spec.PauseImage,
|
|
||||||
Containers: existingAt.Spec.Containers,
|
Containers: existingAt.Spec.Containers,
|
||||||
SnapshotsConfig: existingAt.Spec.SnapshotsConfig,
|
SnapshotsConfig: existingAt.Spec.SnapshotsConfig,
|
||||||
Volumes: existingAt.Spec.Volumes,
|
Volumes: existingAt.Spec.Volumes,
|
||||||
|
|||||||
@@ -210,7 +210,6 @@ func createParkingFixture(ctx context.Context, t *testing.T, clients *e2e.Client
|
|||||||
MatchLabels: map[string]string{"demo": nsObj.Name},
|
MatchLabels: map[string]string{"demo": nsObj.Name},
|
||||||
},
|
},
|
||||||
SandboxClass: existingAt.Spec.SandboxClass,
|
SandboxClass: existingAt.Spec.SandboxClass,
|
||||||
PauseImage: existingAt.Spec.PauseImage,
|
|
||||||
Containers: existingAt.Spec.Containers,
|
Containers: existingAt.Spec.Containers,
|
||||||
SnapshotsConfig: v1alpha1.SnapshotsConfig{
|
SnapshotsConfig: v1alpha1.SnapshotsConfig{
|
||||||
Location: "gs://" + env["BUCKET_NAME"] + "/e2e-parking-" + nsObj.Name,
|
Location: "gs://" + env["BUCKET_NAME"] + "/e2e-parking-" + nsObj.Name,
|
||||||
|
|||||||
@@ -563,14 +563,19 @@ func (x *ArchAssets) GetFiles() map[string]*AssetFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// SandboxAssets is the generic, backend-agnostic description of the sandbox
|
// SandboxAssets is the generic, backend-agnostic description of the sandbox
|
||||||
// binaries for an actor: a sandbox class plus assets keyed first by
|
// an actor runs in: a sandbox class, the pause image holding its namespaces,
|
||||||
// architecture (GOARCH) and then by asset name. atelet's backend code
|
// and binaries keyed first by architecture (GOARCH) and then by asset name.
|
||||||
// interprets the asset names (gVisor expects "gvisor", the release tarball;
|
// atelet's backend code interprets the asset names (gVisor expects "gvisor",
|
||||||
// legacy "runsc", a bare binary, is still accepted).
|
// the release tarball; legacy "runsc", a bare binary, is still accepted).
|
||||||
type SandboxAssets struct {
|
type SandboxAssets struct {
|
||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
state protoimpl.MessageState `protogen:"open.v1"`
|
||||||
SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` // e.g. "gvisor"
|
SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` // e.g. "gvisor"
|
||||||
Assets map[string]*ArchAssets `protobuf:"bytes,2,rep,name=assets,proto3" json:"assets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // arch -> {name -> file}
|
Assets map[string]*ArchAssets `protobuf:"bytes,2,rep,name=assets,proto3" json:"assets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // arch -> {name -> file}
|
||||||
|
// pause_image is the image for the sandbox's root container. Like the
|
||||||
|
// binaries above it is sandbox configuration, not workload configuration,
|
||||||
|
// and atelet pins it into the snapshot manifest so a restore rebuilds the
|
||||||
|
// sandbox from the same image.
|
||||||
|
PauseImage string `protobuf:"bytes,3,opt,name=pause_image,json=pauseImage,proto3" json:"pause_image,omitempty"`
|
||||||
unknownFields protoimpl.UnknownFields
|
unknownFields protoimpl.UnknownFields
|
||||||
sizeCache protoimpl.SizeCache
|
sizeCache protoimpl.SizeCache
|
||||||
}
|
}
|
||||||
@@ -619,11 +624,17 @@ func (x *SandboxAssets) GetAssets() map[string]*ArchAssets {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (x *SandboxAssets) GetPauseImage() string {
|
||||||
|
if x != nil {
|
||||||
|
return x.PauseImage
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// WorkloadSpec parallels Pod, but with far fewer configurable fields.
|
// WorkloadSpec parallels Pod, but with far fewer configurable fields.
|
||||||
type WorkloadSpec struct {
|
type WorkloadSpec struct {
|
||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
state protoimpl.MessageState `protogen:"open.v1"`
|
||||||
Containers []*Container `protobuf:"bytes,1,rep,name=containers,proto3" json:"containers,omitempty"`
|
Containers []*Container `protobuf:"bytes,1,rep,name=containers,proto3" json:"containers,omitempty"`
|
||||||
PauseImage string `protobuf:"bytes,2,opt,name=pause_image,json=pauseImage,proto3" json:"pause_image,omitempty"`
|
|
||||||
Volumes []*Volume `protobuf:"bytes,3,rep,name=volumes,proto3" json:"volumes,omitempty"`
|
Volumes []*Volume `protobuf:"bytes,3,rep,name=volumes,proto3" json:"volumes,omitempty"`
|
||||||
unknownFields protoimpl.UnknownFields
|
unknownFields protoimpl.UnknownFields
|
||||||
sizeCache protoimpl.SizeCache
|
sizeCache protoimpl.SizeCache
|
||||||
@@ -666,13 +677,6 @@ func (x *WorkloadSpec) GetContainers() []*Container {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *WorkloadSpec) GetPauseImage() string {
|
|
||||||
if x != nil {
|
|
||||||
return x.PauseImage
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *WorkloadSpec) GetVolumes() []*Volume {
|
func (x *WorkloadSpec) GetVolumes() []*Volume {
|
||||||
if x != nil {
|
if x != nil {
|
||||||
return x.Volumes
|
return x.Volumes
|
||||||
@@ -1905,20 +1909,20 @@ const file_atelet_proto_rawDesc = "" +
|
|||||||
"\n" +
|
"\n" +
|
||||||
"FilesEntry\x12\x10\n" +
|
"FilesEntry\x12\x10\n" +
|
||||||
"\x03key\x18\x01 \x01(\tR\x03key\x12'\n" +
|
"\x03key\x18\x01 \x01(\tR\x03key\x12'\n" +
|
||||||
"\x05value\x18\x02 \x01(\v2\x11.atelet.AssetFileR\x05value:\x028\x01\"\xbe\x01\n" +
|
"\x05value\x18\x02 \x01(\v2\x11.atelet.AssetFileR\x05value:\x028\x01\"\xdf\x01\n" +
|
||||||
"\rSandboxAssets\x12#\n" +
|
"\rSandboxAssets\x12#\n" +
|
||||||
"\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x129\n" +
|
"\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x129\n" +
|
||||||
"\x06assets\x18\x02 \x03(\v2!.atelet.SandboxAssets.AssetsEntryR\x06assets\x1aM\n" +
|
"\x06assets\x18\x02 \x03(\v2!.atelet.SandboxAssets.AssetsEntryR\x06assets\x12\x1f\n" +
|
||||||
|
"\vpause_image\x18\x03 \x01(\tR\n" +
|
||||||
|
"pauseImage\x1aM\n" +
|
||||||
"\vAssetsEntry\x12\x10\n" +
|
"\vAssetsEntry\x12\x10\n" +
|
||||||
"\x03key\x18\x01 \x01(\tR\x03key\x12(\n" +
|
"\x03key\x18\x01 \x01(\tR\x03key\x12(\n" +
|
||||||
"\x05value\x18\x02 \x01(\v2\x12.atelet.ArchAssetsR\x05value:\x028\x01\"\x8c\x01\n" +
|
"\x05value\x18\x02 \x01(\v2\x12.atelet.ArchAssetsR\x05value:\x028\x01\"~\n" +
|
||||||
"\fWorkloadSpec\x121\n" +
|
"\fWorkloadSpec\x121\n" +
|
||||||
"\n" +
|
"\n" +
|
||||||
"containers\x18\x01 \x03(\v2\x11.atelet.ContainerR\n" +
|
"containers\x18\x01 \x03(\v2\x11.atelet.ContainerR\n" +
|
||||||
"containers\x12\x1f\n" +
|
"containers\x12(\n" +
|
||||||
"\vpause_image\x18\x02 \x01(\tR\n" +
|
"\avolumes\x18\x03 \x03(\v2\x0e.atelet.VolumeR\avolumesJ\x04\b\x02\x10\x03R\vpause_image\"\x12\n" +
|
||||||
"pauseImage\x12(\n" +
|
|
||||||
"\avolumes\x18\x03 \x03(\v2\x0e.atelet.VolumeR\avolumes\"\x12\n" +
|
|
||||||
"\x10DurableDirVolume\"\xfd\x01\n" +
|
"\x10DurableDirVolume\"\xfd\x01\n" +
|
||||||
"\x14ExternalVolumeSource\x12*\n" +
|
"\x14ExternalVolumeSource\x12*\n" +
|
||||||
"\x11storage_volume_id\x18\x01 \x01(\tR\x0fstorageVolumeId\x12\x1f\n" +
|
"\x11storage_volume_id\x18\x01 \x01(\tR\x0fstorageVolumeId\x12\x1f\n" +
|
||||||
|
|||||||
@@ -102,20 +102,28 @@ message ArchAssets {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// SandboxAssets is the generic, backend-agnostic description of the sandbox
|
// SandboxAssets is the generic, backend-agnostic description of the sandbox
|
||||||
// binaries for an actor: a sandbox class plus assets keyed first by
|
// an actor runs in: a sandbox class, the pause image holding its namespaces,
|
||||||
// architecture (GOARCH) and then by asset name. atelet's backend code
|
// and binaries keyed first by architecture (GOARCH) and then by asset name.
|
||||||
// interprets the asset names (gVisor expects "gvisor", the release tarball;
|
// atelet's backend code interprets the asset names (gVisor expects "gvisor",
|
||||||
// legacy "runsc", a bare binary, is still accepted).
|
// the release tarball; legacy "runsc", a bare binary, is still accepted).
|
||||||
message SandboxAssets {
|
message SandboxAssets {
|
||||||
string sandbox_class = 1; // e.g. "gvisor"
|
string sandbox_class = 1; // e.g. "gvisor"
|
||||||
map<string, ArchAssets> assets = 2; // arch -> {name -> file}
|
map<string, ArchAssets> assets = 2; // arch -> {name -> file}
|
||||||
|
|
||||||
|
// pause_image is the image for the sandbox's root container. Like the
|
||||||
|
// binaries above it is sandbox configuration, not workload configuration,
|
||||||
|
// and atelet pins it into the snapshot manifest so a restore rebuilds the
|
||||||
|
// sandbox from the same image.
|
||||||
|
string pause_image = 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
// WorkloadSpec parallels Pod, but with far fewer configurable fields.
|
// WorkloadSpec parallels Pod, but with far fewer configurable fields.
|
||||||
message WorkloadSpec {
|
message WorkloadSpec {
|
||||||
repeated Container containers = 1;
|
repeated Container containers = 1;
|
||||||
string pause_image = 2;
|
|
||||||
repeated Volume volumes = 3;
|
repeated Volume volumes = 3;
|
||||||
|
|
||||||
|
reserved 2;
|
||||||
|
reserved "pause_image"; // moved to SandboxAssets
|
||||||
}
|
}
|
||||||
|
|
||||||
enum VolumeType {
|
enum VolumeType {
|
||||||
|
|||||||
@@ -264,19 +264,6 @@ spec:
|
|||||||
type: object
|
type: object
|
||||||
maxItems: 10
|
maxItems: 10
|
||||||
type: array
|
type: array
|
||||||
pauseImage:
|
|
||||||
description: |-
|
|
||||||
PauseImage is the container to use as the root sandbox container.
|
|
||||||
|
|
||||||
Typically, set it to [1] for on-gcp, and [2] for off-gcp
|
|
||||||
|
|
||||||
- [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da
|
|
||||||
- [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4
|
|
||||||
type: string
|
|
||||||
x-kubernetes-validations:
|
|
||||||
- message: All images must be pinned (changing the image invalidates
|
|
||||||
snapshots)
|
|
||||||
rule: self.contains('@')
|
|
||||||
sandboxClass:
|
sandboxClass:
|
||||||
default: gvisor
|
default: gvisor
|
||||||
description: |-
|
description: |-
|
||||||
@@ -456,7 +443,6 @@ spec:
|
|||||||
type: object
|
type: object
|
||||||
x-kubernetes-map-type: atomic
|
x-kubernetes-map-type: atomic
|
||||||
required:
|
required:
|
||||||
- pauseImage
|
|
||||||
- snapshotsConfig
|
- snapshotsConfig
|
||||||
type: object
|
type: object
|
||||||
x-kubernetes-validations:
|
x-kubernetes-validations:
|
||||||
|
|||||||
@@ -111,6 +111,24 @@ spec:
|
|||||||
the default config for its SandboxClass. At most one default is expected
|
the default config for its SandboxClass. At most one default is expected
|
||||||
per SandboxClass.
|
per SandboxClass.
|
||||||
type: boolean
|
type: boolean
|
||||||
|
pauseImage:
|
||||||
|
description: |-
|
||||||
|
PauseImage is the container image used as the root sandbox container.
|
||||||
|
It holds the sandbox's namespaces and runs no workload code, so it is an
|
||||||
|
implementation detail of the sandbox rather than something actor authors
|
||||||
|
choose. It is captured in the snapshot manifest alongside the sandbox
|
||||||
|
binaries, so a restore always re-creates the sandbox from the same image
|
||||||
|
the snapshot was taken with.
|
||||||
|
|
||||||
|
Typically, set it to [1] for on-gcp, and [2] for off-gcp
|
||||||
|
|
||||||
|
- [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da
|
||||||
|
- [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4
|
||||||
|
type: string
|
||||||
|
x-kubernetes-validations:
|
||||||
|
- message: All images must be pinned (changing the image invalidates
|
||||||
|
snapshots)
|
||||||
|
rule: self.contains('@')
|
||||||
sandboxClass:
|
sandboxClass:
|
||||||
default: gvisor
|
default: gvisor
|
||||||
description: |-
|
description: |-
|
||||||
@@ -121,6 +139,7 @@ spec:
|
|||||||
- microvm
|
- microvm
|
||||||
type: string
|
type: string
|
||||||
required:
|
required:
|
||||||
|
- pauseImage
|
||||||
- sandboxClass
|
- sandboxClass
|
||||||
type: object
|
type: object
|
||||||
required:
|
required:
|
||||||
|
|||||||
@@ -26,6 +26,9 @@ metadata:
|
|||||||
spec:
|
spec:
|
||||||
sandboxClass: gvisor
|
sandboxClass: gvisor
|
||||||
default: true
|
default: true
|
||||||
|
# The root sandbox container's image. On GCP, prefer the in-project mirror
|
||||||
|
# gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da.
|
||||||
|
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
||||||
assets:
|
assets:
|
||||||
amd64:
|
amd64:
|
||||||
gvisor:
|
gvisor:
|
||||||
|
|||||||
@@ -45,6 +45,9 @@ metadata:
|
|||||||
name: microvm
|
name: microvm
|
||||||
spec:
|
spec:
|
||||||
sandboxClass: microvm
|
sandboxClass: microvm
|
||||||
|
# The root sandbox container's image. On GCP, prefer the in-project mirror
|
||||||
|
# gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da.
|
||||||
|
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
||||||
assets:
|
assets:
|
||||||
arm64:
|
arm64:
|
||||||
cloud-hypervisor:
|
cloud-hypervisor:
|
||||||
|
|||||||
@@ -358,17 +358,6 @@ type SnapshotsConfig struct {
|
|||||||
// +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'"
|
// +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'"
|
||||||
// +kubebuilder:validation:XValidation:rule="(has(self.sandboxClass) && self.sandboxClass == 'microvm') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : 'ColdBoot') != 'Golden'",message="onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'"
|
// +kubebuilder:validation:XValidation:rule="(has(self.sandboxClass) && self.sandboxClass == 'microvm') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : 'ColdBoot') != 'Golden'",message="onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'"
|
||||||
type ActorTemplateSpec struct {
|
type ActorTemplateSpec struct {
|
||||||
// PauseImage is the container to use as the root sandbox container.
|
|
||||||
//
|
|
||||||
// Typically, set it to [1] for on-gcp, and [2] for off-gcp
|
|
||||||
//
|
|
||||||
// - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da
|
|
||||||
// - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4
|
|
||||||
//
|
|
||||||
// +required
|
|
||||||
// +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must be pinned (changing the image invalidates snapshots)"
|
|
||||||
PauseImage string `json:"pauseImage,omitempty"`
|
|
||||||
|
|
||||||
// Containers is the workload definition.
|
// Containers is the workload definition.
|
||||||
//
|
//
|
||||||
// +optional
|
// +optional
|
||||||
|
|||||||
@@ -70,7 +70,6 @@ func TestActorTemplateValidation(t *testing.T) {
|
|||||||
Namespace: "default",
|
Namespace: "default",
|
||||||
},
|
},
|
||||||
Spec: ActorTemplateSpec{
|
Spec: ActorTemplateSpec{
|
||||||
PauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da",
|
|
||||||
Containers: []Container{
|
Containers: []Container{
|
||||||
{
|
{
|
||||||
Name: "main",
|
Name: "main",
|
||||||
@@ -98,20 +97,6 @@ func TestActorTemplateValidation(t *testing.T) {
|
|||||||
name: "base template",
|
name: "base template",
|
||||||
mutate: func(at *ActorTemplate) {},
|
mutate: func(at *ActorTemplate) {},
|
||||||
wantErr: false,
|
wantErr: false,
|
||||||
}, {
|
|
||||||
name: "missing PauseImage",
|
|
||||||
mutate: func(at *ActorTemplate) {
|
|
||||||
at.Spec.PauseImage = ""
|
|
||||||
},
|
|
||||||
wantErr: true,
|
|
||||||
errMsg: "Required value",
|
|
||||||
}, {
|
|
||||||
name: "unpinned PauseImage",
|
|
||||||
mutate: func(at *ActorTemplate) {
|
|
||||||
at.Spec.PauseImage = "pause"
|
|
||||||
},
|
|
||||||
wantErr: true,
|
|
||||||
errMsg: "All images must be pinned",
|
|
||||||
}, {
|
}, {
|
||||||
name: "missing SnapshotsConfig.Location",
|
name: "missing SnapshotsConfig.Location",
|
||||||
mutate: func(at *ActorTemplate) {
|
mutate: func(at *ActorTemplate) {
|
||||||
@@ -1269,7 +1254,6 @@ func TestActorTemplateSpecImmutability(t *testing.T) {
|
|||||||
|
|
||||||
baseTemplate := &ActorTemplate{
|
baseTemplate := &ActorTemplate{
|
||||||
Spec: ActorTemplateSpec{
|
Spec: ActorTemplateSpec{
|
||||||
PauseImage: "pause@hash",
|
|
||||||
Containers: []Container{
|
Containers: []Container{
|
||||||
{
|
{
|
||||||
Name: "main",
|
Name: "main",
|
||||||
@@ -1290,9 +1274,9 @@ func TestActorTemplateSpecImmutability(t *testing.T) {
|
|||||||
mutate func(*ActorTemplate)
|
mutate func(*ActorTemplate)
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "update-pause-image",
|
name: "update-container-image",
|
||||||
mutate: func(at *ActorTemplate) {
|
mutate: func(at *ActorTemplate) {
|
||||||
at.Spec.PauseImage = "pause@new"
|
at.Spec.Containers[0].Image = "busybox@new"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -67,6 +67,22 @@ type SandboxConfigSpec struct {
|
|||||||
// +optional
|
// +optional
|
||||||
Default bool `json:"default,omitempty"`
|
Default bool `json:"default,omitempty"`
|
||||||
|
|
||||||
|
// PauseImage is the container image used as the root sandbox container.
|
||||||
|
// It holds the sandbox's namespaces and runs no workload code, so it is an
|
||||||
|
// implementation detail of the sandbox rather than something actor authors
|
||||||
|
// choose. It is captured in the snapshot manifest alongside the sandbox
|
||||||
|
// binaries, so a restore always re-creates the sandbox from the same image
|
||||||
|
// the snapshot was taken with.
|
||||||
|
//
|
||||||
|
// Typically, set it to [1] for on-gcp, and [2] for off-gcp
|
||||||
|
//
|
||||||
|
// - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da
|
||||||
|
// - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4
|
||||||
|
//
|
||||||
|
// +required
|
||||||
|
// +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must be pinned (changing the image invalidates snapshots)"
|
||||||
|
PauseImage string `json:"pauseImage"`
|
||||||
|
|
||||||
// Assets is the set of files atelet fetches for this runtime, keyed first by
|
// Assets is the set of files atelet fetches for this runtime, keyed first by
|
||||||
// architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The
|
// architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The
|
||||||
// asset names are interpreted by the sandbox backend: gVisor expects a
|
// asset names are interpreted by the sandbox backend: gVisor expects a
|
||||||
|
|||||||
@@ -35,16 +35,25 @@ const validSHA256 = "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea8
|
|||||||
// the install — loaded here so the test guards the policy we actually ship.
|
// the install — loaded here so the test guards the policy we actually ship.
|
||||||
const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation.yaml"
|
const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation.yaml"
|
||||||
|
|
||||||
|
const validPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
|
||||||
|
|
||||||
func sandboxConfig(name string, class SandboxClass, assets map[string]map[string]AssetFile) *SandboxConfig {
|
func sandboxConfig(name string, class SandboxClass, assets map[string]map[string]AssetFile) *SandboxConfig {
|
||||||
return &SandboxConfig{
|
return &SandboxConfig{
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: name},
|
ObjectMeta: metav1.ObjectMeta{Name: name},
|
||||||
Spec: SandboxConfigSpec{
|
Spec: SandboxConfigSpec{
|
||||||
SandboxClass: class,
|
SandboxClass: class,
|
||||||
|
PauseImage: validPauseImage,
|
||||||
Assets: assets,
|
Assets: assets,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withPauseImage overrides the pause image on an otherwise-valid config.
|
||||||
|
func withPauseImage(sc *SandboxConfig, image string) *SandboxConfig {
|
||||||
|
sc.Spec.PauseImage = image
|
||||||
|
return sc
|
||||||
|
}
|
||||||
|
|
||||||
func runscAsset() AssetFile { return AssetFile{URL: "gs://bucket/runsc", SHA256: validSHA256} }
|
func runscAsset() AssetFile { return AssetFile{URL: "gs://bucket/runsc", SHA256: validSHA256} }
|
||||||
|
|
||||||
func gvisorAsset() AssetFile {
|
func gvisorAsset() AssetFile {
|
||||||
@@ -184,6 +193,16 @@ func TestSandboxConfigValidation(t *testing.T) {
|
|||||||
sc: sandboxConfig("bad-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": {URL: "gs://bucket/runsc", SHA256: "deadbeef"}}}),
|
sc: sandboxConfig("bad-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": {URL: "gs://bucket/runsc", SHA256: "deadbeef"}}}),
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
errMsg: "sha256",
|
errMsg: "sha256",
|
||||||
|
}, {
|
||||||
|
name: "missing pauseImage",
|
||||||
|
sc: withPauseImage(sandboxConfig("bad-no-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), ""),
|
||||||
|
wantErr: true,
|
||||||
|
errMsg: "pauseImage",
|
||||||
|
}, {
|
||||||
|
name: "unpinned pauseImage",
|
||||||
|
sc: withPauseImage(sandboxConfig("bad-unpinned-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), "registry.k8s.io/pause:3.10.2"),
|
||||||
|
wantErr: true,
|
||||||
|
errMsg: "All images must be pinned",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
Reference in New Issue
Block a user