trending/substrate
1
0
Fork 0
mirror of https://github.com/agent-substrate/substrate.git synced 2026-10-02 03:24:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Resolve SandboxConfig from the ActorTemplate instead of the WorkerPool (#1446)

Browse Source 
This PR moves sandbox config selection from the WorkerPool to the
ActorTemplate.

The existing behavior is preserved while we are designing the upgrade:
sandbox config still cannot be updated once set (ActorTemplates are
create-only and `sandbox_config` is immutable).

For now the ActorTemplate still *requires* `sandbox_config.config_name`
— there is no resolution of the cluster default (`spec.default`). This
is temporary while we figure out the defaulting design.

- [ ] Tests pass
- [x] Appropriate changes to documentation are included in the PR
This commit is contained in:
Zoe Zhao
2026-09-03 16:13:07 -07:00
committed by GitHub
parent f11d3b2bb8
commit 6d3afdd63b
42 changed files with 369 additions and 361 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.golangci-kal.yaml
+2 -2
View File
@@ -96,7 +96,7 @@ linters:
- path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
text: '^requiredfields: .*\bAssetFile\.(SHA256|URL)\b'
- path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
text: '^(nomaps|optionalfields|requiredfields): .*\bSandboxConfigSpec\.(Assets|Default|PauseImage|SandboxClass)\b'
text: '^(nomaps|optionalfields|requiredfields): .*\bSandboxConfigSpec\.(Assets|PauseImage|SandboxClass)\b'
- path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b'
- path: 'pkg/api/v1alpha1/csidriverconfig_types\.go'
@@ -112,7 +112,7 @@ linters:
- path: 'pkg/api/v1alpha1/workerpool_types\.go'
text: '^optionalfields: .*\bWorkerPoolPodTemplate\.PriorityClassName\b'
- path: 'pkg/api/v1alpha1/workerpool_types\.go'
text: '^(optionalfields|requiredfields): .*\bWorkerPoolSpec\.(Replicas|SandboxClass|SandboxConfigName|WorkerImage)\b'
text: '^(optionalfields|requiredfields): .*\bWorkerPoolSpec\.(Replicas|SandboxClass|WorkerImage)\b'
- path: 'pkg/api/v1alpha1/workerpool_types\.go'
text: '^optionalfields: .*\bWorkerPoolStatus\.(ReadyReplicas|Replicas|Selector)\b'
- path: 'pkg/api/v1alpha1/workerpool_types\.go'
benchmarking/automation/orchestrator.py
+2 -2
View File
@@ -315,8 +315,8 @@ def teardown_substrate() -> None:
def install_microvm_deps() -> None:
"""Stage kata/cloud-hypervisor assets and apply the cluster-wide
microvm SandboxConfig. Required before a microvm WorkerPool can
schedule; must run after deploy_substrate() (which installs the CRDs)."""
microvm SandboxConfig. Required before a microvm ActorTemplate can
boot; must run after deploy_substrate() (which installs the CRDs)."""
run(["hack/install-microvm-deps.sh", "--install"])
benchmarking/workloads/deploy.sh
+5 -5
View File
@@ -115,11 +115,11 @@ run_kubectl_ate() {
}
substitute() {
# SandboxConfig names are pinned per class (rather than defaulted) so a stale
# config from a dirty teardown fails loudly instead of silently binding this
# pool. gvisor-default is applied by hack/install-ate.sh; microvm is applied
# by hack/install-microvm-deps.sh. The protojson templates take the sandbox
# class as its proto enum spelling.
# SandboxConfig names are pinned per class in the ActorTemplates (rather
# than defaulted) so a stale config from a dirty teardown fails loudly
# instead of silently binding these workloads. gvisor-default is applied by
# hack/install-ate.sh; microvm is applied by hack/install-microvm-deps.sh.
# The protojson templates take the sandbox class as its proto enum spelling.
local manifest="$1"
local sandbox_config_name sandbox_class_enum
case "${SANDBOX_CLASS}" in
benchmarking/workloads/manifests/workloads.yaml.tmpl
-1
View File
@@ -34,5 +34,4 @@ metadata:
spec:
replicas: ${WORKER_COUNT}
sandboxClass: ${SANDBOX_CLASS}
sandboxConfigName: ${SANDBOX_CONFIG_NAME}
workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-${SANDBOX_CLASS}
cmd/ate-setup/internal/steps/deploy.go
+5 -5
View File
@@ -101,15 +101,15 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error {
}
// Enforce per-class SandboxConfig asset requirements. This is applied
// before any SandboxConfig so the default below is validated too.
// before any SandboxConfig so the config below is validated too.
if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-validation.yaml")); err != nil {
return err
}
// Install the cluster-wide default sandbox config. Sandbox binaries live
// on cluster-scoped SandboxConfigs resolved via each WorkerPool's
// SandboxClass, decoupled from ActorTemplate; gVisor pools resolve to this
// default unless they name their own SandboxConfig.
// Install the cluster-wide sandbox config. Sandbox binaries live on
// cluster-scoped SandboxConfigs each ActorTemplate names via
// sandboxConfig.configName; gVisor templates name this one unless they
// create their own SandboxConfig.
if err := e.Kube.ApplyPath(ctx, e.Cfg.Manifest("sandboxconfig-gvisor.yaml")); err != nil {
return err
}
cmd/ateapi/internal/controlapi/actor.go
+16 -21
View File
@@ -281,8 +281,8 @@ func (s *ServiceImpl) UpdateActor(ctx context.Context, actorRef resources.ActorR
// Update actor template is only allowed while the actor is suspended.
// The repointed ref must also resolve, mirroring CreateActor's
// check (same non-atomicity caveat; resume re-resolves and fails
// cleanly), and the replacement's sandbox class, volumes, and volume
// mounts must match the old template's.
// cleanly), and the replacement's sandbox config, volumes, and
// volume mounts must match the old template's.
if !proto.Equal(oldVal.GetActorTemplate(), newVal.GetActorTemplate()) {
if state := oldVal.GetStatus().GetState(); state != ateapipb.ActorState_ACTOR_STATE_SUSPENDED {
return status.Errorf(codes.FailedPrecondition,
@@ -293,15 +293,24 @@ func (s *ServiceImpl) UpdateActor(ctx context.Context, actorRef resources.ActorR
return err
}
oldTemplate, err := resolveActorTemplate(ctx, s.store, oldVal)
if err == nil {
if err := validateTemplateSandboxClassUnchanged(oldTemplate, newTemplate); err != nil {
return err
switch {
case err == nil:
// Snapshots are not portable across sandbox runtime
// families, so the replacement template must name the same
// SandboxConfig.
if !proto.Equal(oldTemplate.GetSandboxConfig(), newTemplate.GetSandboxConfig()) {
oldSC, newSC := oldTemplate.GetSandboxConfig(), newTemplate.GetSandboxConfig()
return status.Errorf(codes.FailedPrecondition,
"the current actor template names SandboxConfig %q (class %s) but the new one names %q (class %s); the sandbox config must be identical to repoint an actor",
oldSC.GetConfigName(), oldSC.GetSandboxClass(), newSC.GetConfigName(), newSC.GetSandboxClass())
}
if err := validateTemplateVolumesUnchanged(oldTemplate, newTemplate); err != nil {
return err
}
} else if !errors.Is(err, errActorTemplateNotFound) {
// Skip the validation if old template is not found
case errors.Is(err, errActorTemplateNotFound):
// The old template is gone, so there is nothing left to
// compare the sandbox config or volume layout against.
default:
return err
}
}
@@ -331,20 +340,6 @@ func (s *ServiceImpl) UpdateActor(ctx context.Context, actorRef resources.ActorR
return storedActor, nil
}
// validateTemplateSandboxClassUnchanged rejects a template repoint that
// changes the sandbox class: snapshots are not portable across sandbox
// runtime families, so the actor's saved state could not be restored under
// the new template.
func validateTemplateSandboxClassUnchanged(oldTemplate, newTemplate *ateapipb.ActorTemplate) error {
oldClass := oldTemplate.GetSandboxConfig().GetSandboxClass()
newClass := newTemplate.GetSandboxConfig().GetSandboxClass()
if oldClass != newClass {
return status.Errorf(codes.FailedPrecondition,
"sandbox class differs between the current (%s) and the new (%s) actor template; the sandbox class must be identical to repoint an actor", oldClass, newClass)
}
return nil
}
// validateTemplateVolumesUnchanged rejects a template repoint that changes
// the template's volumes or any container's volume mounts: an actor's
// snapshot data is laid out per the volumes and mount paths it was captured
cmd/ateapi/internal/controlapi/actor_template.go
+6 -1
View File
@@ -45,6 +45,12 @@ func (s *RPCService) CreateActorTemplate(ctx context.Context, req *ateapipb.Crea
return nil, toGRPCStatusError(errs)
}
// config_name is required; the declarative validation has already
// rejected an empty one.
if _, err := resolveTemplateSandboxConfig(s.sandboxConfigLister, in.GetSandboxConfig()); err != nil {
return nil, err
}
templateRef := resources.ActorTemplateRefFromActorTemplate(in)
stored, err := s.impl.CreateActorTemplate(ctx, in)
@@ -63,7 +69,6 @@ func (s *RPCService) CreateActorTemplate(ctx context.Context, req *ateapipb.Crea
func (s *ServiceImpl) CreateActorTemplate(ctx context.Context, inTemplate *ateapipb.ActorTemplate) (*ateapipb.ActorTemplate, error) {
// Build the stored object: status is server-owned and starts empty.
// TODO: check that sandbox_config.config_name matches sandbox_class.
outTemplate := proto.Clone(inTemplate).(*ateapipb.ActorTemplate)
outTemplate.Status = &ateapipb.ActorTemplateStatus{}
cmd/ateapi/internal/controlapi/actor_template_test.go
+66 -2
View File
@@ -23,12 +23,15 @@ import (
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
"github.com/agent-substrate/substrate/internal/resources"
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"github.com/google/go-cmp/cmp"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/testing/protocmp"
"k8s.io/apimachinery/pkg/api/operation"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/validation/field"
)
@@ -174,11 +177,72 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) {
}
}
// gvisorDefaultLister returns a SandboxConfig lister seeded with the
// "gvisor-default" config that validActorTemplate names.
func gvisorDefaultLister(t *testing.T) listersv1alpha1.SandboxConfigLister {
t.Helper()
return sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{{
ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"},
Spec: atev1alpha1.SandboxConfigSpec{
SandboxClass: atev1alpha1.SandboxClassGvisor,
PauseImage: "registry.k8s.io/pause@sha256:x",
Assets: testAssets(),
},
}})
}
// TestCreateActorTemplate_SandboxConfigChecks pins the create-time checks on
// the template's named SandboxConfig: it must exist and match the template's
// class, both FailedPrecondition — they depend on cluster state, and the
// lister may briefly lag a just-created config, so the error is retryable.
func TestCreateActorTemplate_SandboxConfigChecks(t *testing.T) {
persistence := newTestPersistence(t)
s := &RPCService{impl: newServiceImpl(persistence, nil), sandboxConfigLister: gvisorDefaultLister(t)}
ctx := context.Background()
if _, err := persistence.CreateAtespace(ctx, &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "ns1"}}); err != nil {
t.Fatalf("CreateAtespace failed: %v", err)
}
tests := []struct {
name string
sandbox *ateapipb.SandboxConfig
wantCode codes.Code
}{{
name: "named config exists and matches",
sandbox: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"},
wantCode: codes.OK,
}, {
name: "empty config_name is rejected",
sandbox: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_MICROVM},
wantCode: codes.InvalidArgument,
}, {
name: "named config missing",
sandbox: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "does-not-exist"},
wantCode: codes.FailedPrecondition,
}, {
name: "named config class mismatch",
sandbox: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_MICROVM, ConfigName: "gvisor-default"},
wantCode: codes.FailedPrecondition,
}}
for i, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) {
tmpl.Metadata = &ateapipb.ResourceMetadata{Atespace: "ns1", Name: fmt.Sprintf("tmpl-%d", i)}
tmpl.SandboxConfig = tt.sandbox
})}
_, err := s.CreateActorTemplate(ctx, req)
if status.Code(err) != tt.wantCode {
t.Errorf("CreateActorTemplate error = %v, want code %v", err, tt.wantCode)
}
})
}
}
// TestCreateActorTemplate covers the atespace precondition: creation fails
// while the atespace is missing, and succeeds once the atespace exists.
func TestCreateActorTemplate(t *testing.T) {
persistence := newTestPersistence(t)
s := &RPCService{impl: newServiceImpl(persistence, nil)}
s := &RPCService{impl: newServiceImpl(persistence, nil), sandboxConfigLister: gvisorDefaultLister(t)}
ctx := context.Background()
req := func(atespace, name string) *ateapipb.CreateActorTemplateRequest {
return &ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) {
@@ -208,7 +272,7 @@ func TestCreateActorTemplate(t *testing.T) {
// the only guard.
func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) {
persistence := newTestPersistence(t)
s := &RPCService{impl: newServiceImpl(persistence, nil)}
s := &RPCService{impl: newServiceImpl(persistence, nil), sandboxConfigLister: gvisorDefaultLister(t)}
ctx := context.Background()
if _, err := persistence.CreateAtespace(ctx, &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "ns1"}}); err != nil {
cmd/ateapi/internal/controlapi/actor_test.go
+23 -52
View File
@@ -870,8 +870,8 @@ func TestUpdateActor(t *testing.T) {
// TestUpdateActor_RepointTemplate covers the mutable actor_template ref: an
// update may point a suspended actor at a different template (it takes effect
// on the next ResumeActor), but the actor must be suspended, the new ref must
// resolve, and the replacement's volumes and volume mounts must match the old
// template's.
// resolve, and the replacement's sandbox config, volumes, and volume mounts
// must match the old template's.
func TestUpdateActor_RepointTemplate(t *testing.T) {
ctx := context.Background()
persistence, cleanup := storetest.SetupTestStore(t)
@@ -946,13 +946,13 @@ func TestUpdateActor_RepointTemplate(t *testing.T) {
t.Fatalf("UpdateActor to a template with different volumes = %v, want FailedPrecondition (err: %v)", got, err)
}
// Repointing at a template with a different sandbox class is rejected.
// Repointing at a template naming a different SandboxConfig is rejected.
_, err = svc.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: &ateapipb.Actor{
Metadata: created.GetMetadata(),
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-e"},
}})
if got := status.Code(err); got != codes.FailedPrecondition {
t.Fatalf("UpdateActor to a template with a different sandbox class = %v, want FailedPrecondition (err: %v)", got, err)
t.Fatalf("UpdateActor to a template with a different sandbox config = %v, want FailedPrecondition (err: %v)", got, err)
}
// Repointing at an existing template with identical volumes and mounts
@@ -968,6 +968,25 @@ func TestUpdateActor_RepointTemplate(t *testing.T) {
t.Errorf("updated actor_template.name = %q, want %q", got, want)
}
// When the old template no longer exists there is nothing left to
// compare the sandbox config or volume layout against, so the repoint
// only requires the new ref to resolve.
orphan := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "orphan-actor"},
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-gone"},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
})
repointed, err := svc.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: &ateapipb.Actor{
Metadata: orphan.GetMetadata(),
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-e"},
}})
if err != nil {
t.Fatalf("UpdateActor from a deleted template failed: %v", err)
}
if got, want := repointed.GetActorTemplate().GetName(), "tmpl-e"; got != want {
t.Errorf("updated actor_template.name = %q, want %q", got, want)
}
// Repointing an actor that is not suspended is rejected, even at a
// compatible template.
running := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{
@@ -998,54 +1017,6 @@ func TestUpdateActor_RepointTemplate(t *testing.T) {
}
}
// TestValidateTemplateSandboxClassUnchanged exercises the sandbox class
// comparison applied when an actor is repointed at a replacement template.
func TestValidateTemplateSandboxClassUnchanged(t *testing.T) {
template := func(config *ateapipb.SandboxConfig) *ateapipb.ActorTemplate {
return &ateapipb.ActorTemplate{SandboxConfig: config}
}
gvisorDefault := &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}
gvisorNightly := &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-nightly"}
microvm := &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_MICROVM, ConfigName: "microvm"}
tests := []struct {
name string
oldTmpl, newTmpl *ateapipb.ActorTemplate
wantErr bool
}{{
name: "same sandbox class",
oldTmpl: template(gvisorDefault),
newTmpl: template(gvisorDefault),
}, {
name: "same class with a different config name",
oldTmpl: template(gvisorDefault),
newTmpl: template(gvisorNightly),
}, {
name: "class changed",
oldTmpl: template(gvisorDefault),
newTmpl: template(microvm),
wantErr: true,
}, {
name: "class set on the new template only",
oldTmpl: template(nil),
newTmpl: template(microvm),
wantErr: true,
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateTemplateSandboxClassUnchanged(tt.oldTmpl, tt.newTmpl)
if gotErr := err != nil; gotErr != tt.wantErr {
t.Fatalf("validateTemplateSandboxClassUnchanged() error = %v, wantErr %v", err, tt.wantErr)
}
if err != nil {
if got := status.Code(err); got != codes.FailedPrecondition {
t.Errorf("status code = %v, want FailedPrecondition", got)
}
}
})
}
}
// TestValidateTemplateVolumesUnchanged exercises the volumes and
// per-container mount comparison applied when an actor is repointed at a
// replacement template.
cmd/ateapi/internal/controlapi/functionaltest/actor_template_test.go
+12
View File
@@ -29,6 +29,7 @@ func TestActorTemplateCRUD(t *testing.T) {
tc := setupTest(t, ns)
defer tc.cleanup()
ctx := context.Background()
ensureDefaultGvisorSandboxConfig(t, tc)
created, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{
ActorTemplate: &ateapipb.ActorTemplate{
@@ -120,4 +121,15 @@ func TestActorTemplateCRUD(t *testing.T) {
assertGrpcError(t, err, codes.NotFound, "ActorTemplate "+testAtespace+"/tmpl-a not found")
_, err = tc.client.DeleteActorTemplate(ctx, &ateapipb.DeleteActorTemplateRequest{ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl-a"}})
assertGrpcError(t, err, codes.NotFound, "ActorTemplate "+testAtespace+"/tmpl-a not found")
// config_name is required: a template must name its SandboxConfig.
_, err = tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{
ActorTemplate: &ateapipb.ActorTemplate{
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl-unnamed-config"},
Containers: []*ateapipb.Container{{Name: "main", Image: "example.com/app:v1"}},
SnapshotsConfig: &ateapipb.SnapshotsConfig{StorageLocation: "gs://my-bucket/snapshots"},
SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR},
},
})
assertGrpcErrorRegex(t, err, codes.InvalidArgument, `sandbox_config\.config_name`)
}
cmd/ateapi/internal/controlapi/functionaltest/actor_test.go
+16 -8
View File
@@ -180,6 +180,7 @@ func TestCreateActor_SubstrateTemplateRef(t *testing.T) {
defer tc.cleanup()
ctx := context.Background()
ensureDefaultGvisorSandboxConfig(t, tc)
if _, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{
ActorTemplate: &ateapipb.ActorTemplate{
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "sub-tmpl"},
@@ -313,6 +314,7 @@ func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) {
ns := namespaceForTest("ns-snapshot-external-volume")
tc := setupTest(t, ns)
defer tc.cleanup()
ensureDefaultGvisorSandboxConfig(t, tc)
template, err := tc.client.CreateActorTemplate(context.Background(), &ateapipb.CreateActorTemplateRequest{
ActorTemplate: &ateapipb.ActorTemplate{
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "tmpl1"},
@@ -667,8 +669,8 @@ func TestUpdateActor_Success(t *testing.T) {
// TestUpdateActor_RepointTemplate verifies UpdateActor can point an actor at
// a different substrate ActorTemplate (effective on the next ResumeActor),
// and that a ref to an absent template, or to one with different volumes or
// volume mounts, is rejected.
// and that a ref to an absent template, or to one with a different sandbox
// config, volumes, or volume mounts, is rejected.
func TestUpdateActor_RepointTemplate(t *testing.T) {
tests := []struct {
name string
@@ -678,6 +680,7 @@ func TestUpdateActor_RepointTemplate(t *testing.T) {
{name: "absent-template", template: "absent", wantCode: codes.FailedPrecondition},
{name: "different-mounts", template: "tmpl-c", wantCode: codes.FailedPrecondition},
{name: "different-volumes", template: "tmpl-d", wantCode: codes.FailedPrecondition},
{name: "different-sandbox-config", template: "tmpl-e", wantCode: codes.FailedPrecondition},
{name: "same-volumes", template: "tmpl-b", wantCode: codes.OK},
}
for _, tt := range tests {
@@ -687,18 +690,23 @@ func TestUpdateActor_RepointTemplate(t *testing.T) {
defer tc.cleanup()
ctx := context.Background()
ensureDefaultGvisorSandboxConfig(t, tc)
ensureGvisorSandboxConfig(t, tc, "gvisor-nightly")
// tmpl-a and tmpl-b are volume-compatible; tmpl-c mounts the data
// volume elsewhere and tmpl-d declares an extra volume.
// volume elsewhere, tmpl-d declares an extra volume, and tmpl-e
// names a different SandboxConfig.
dataVolume := &ateapipb.Volume{Name: "data", DurableDir: &ateapipb.DurableDirVolumeSource{}}
scratchVolume := &ateapipb.Volume{Name: "scratch", DurableDir: &ateapipb.DurableDirVolumeSource{}}
templates := map[string]struct {
mountPath string
volumes []*ateapipb.Volume
configName string
}{
"tmpl-a": {"/data", []*ateapipb.Volume{dataVolume}},
"tmpl-b": {"/data", []*ateapipb.Volume{dataVolume}},
"tmpl-c": {"/mnt/data", []*ateapipb.Volume{dataVolume}},
"tmpl-d": {"/data", []*ateapipb.Volume{dataVolume, scratchVolume}},
"tmpl-a": {"/data", []*ateapipb.Volume{dataVolume}, "gvisor-default"},
"tmpl-b": {"/data", []*ateapipb.Volume{dataVolume}, "gvisor-default"},
"tmpl-c": {"/mnt/data", []*ateapipb.Volume{dataVolume}, "gvisor-default"},
"tmpl-d": {"/data", []*ateapipb.Volume{dataVolume, scratchVolume}, "gvisor-default"},
"tmpl-e": {"/data", []*ateapipb.Volume{dataVolume}, "gvisor-nightly"},
}
for name, tmpl := range templates {
if _, err := tc.client.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{
@@ -711,7 +719,7 @@ func TestUpdateActor_RepointTemplate(t *testing.T) {
}},
Volumes: tmpl.volumes,
SnapshotsConfig: &ateapipb.SnapshotsConfig{StorageLocation: "gs://my-bucket/snapshots"},
SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"},
SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: tmpl.configName},
},
}); err != nil {
t.Fatalf("CreateActorTemplate %s failed: %v", name, err)
cmd/ateapi/internal/controlapi/functionaltest/common_test.go
+13 -8
View File
@@ -176,7 +176,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu
mockDriverName: mockPlugin,
}
}
service := controlapi.NewRPCService(persistence, wc, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", volPlugins)
service := controlapi.NewRPCService(persistence, wc, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", volPlugins)
// 5. Start REAL gRPC Server for ATE API
grpcServer := grpc.NewServer(grpc.ChainUnaryInterceptor(
@@ -305,8 +305,8 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st
t.Helper()
// Sandbox binaries live on a (cluster-scoped) SandboxConfig the template
// names. Create a default gvisor SandboxConfig so a boot-from-spec Run can
// resolve its assets.
// names. Create the gvisor-default SandboxConfig so a boot-from-spec Run
// can resolve its assets.
ensureDefaultGvisorSandboxConfig(t, tc)
createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns})
@@ -367,16 +367,21 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st
// it is what a resolved WorkloadSpec's sandbox assets should name.
const testPauseImage = "pause@sha256:abc"
// ensureDefaultGvisorSandboxConfig creates the cluster-scoped default gvisor
// ensureDefaultGvisorSandboxConfig creates the cluster-scoped "gvisor-default"
// SandboxConfig (idempotently) and waits for it to appear in the lister.
func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) {
t.Helper()
const name = "gvisor-default"
ensureGvisorSandboxConfig(t, tc, "gvisor-default")
}
// ensureGvisorSandboxConfig creates a cluster-scoped gvisor SandboxConfig
// (idempotently) and waits for it to appear in the lister.
func ensureGvisorSandboxConfig(t *testing.T, tc *testContext, name string) {
t.Helper()
sc := &atev1alpha1.SandboxConfig{
ObjectMeta: metav1.ObjectMeta{Name: name},
Spec: atev1alpha1.SandboxConfigSpec{
SandboxClass: atev1alpha1.SandboxClassGvisor,
Default: true,
PauseImage: testPauseImage,
Assets: map[string]map[string]atev1alpha1.AssetFile{
"amd64": {"runsc": {
@@ -391,13 +396,13 @@ func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) {
},
}
if _, err := tc.substrateClient.ApiV1alpha1().SandboxConfigs().Create(context.Background(), sc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
t.Fatalf("failed to create default SandboxConfig: %v", err)
t.Fatalf("failed to create SandboxConfig %s: %v", name, err)
}
if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
_, err := tc.sandboxConfigLister.Get(name)
return err == nil, nil
}); err != nil {
t.Fatalf("default SandboxConfig not synced into lister: %v", err)
t.Fatalf("SandboxConfig %s not synced into lister: %v", name, err)
}
}
cmd/ateapi/internal/controlapi/sandbox_assets.go
+40 -54
View File
@@ -20,76 +20,62 @@ import (
"github.com/agent-substrate/substrate/internal/proto/ateletpb"
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
"k8s.io/apimachinery/pkg/labels"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
)
// resolveSandboxAssets determines the sandbox binaries and pause image an actor
// should boot with and projects them onto the ateletpb.SandboxAssets atelet
// fetches. It takes the SandboxClass (default gvisor) of a given worker pool,
// then picks the SandboxConfig named by the pool — or, if none is named, the
// cluster default SandboxConfig for that class.
func resolveSandboxAssets(
workerPoolLister listersv1alpha1.WorkerPoolLister,
// resolveTemplateSandboxConfig resolves the SandboxConfig the ActorTemplate
// names via sandbox_config.config_name and checks that its class matches the
// template's sandbox_class.
func resolveTemplateSandboxConfig(
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
poolNamespace, poolName string,
) (*ateletpb.SandboxAssets, error) {
wp, err := workerPoolLister.WorkerPools(poolNamespace).Get(poolName)
if err != nil {
return nil, fmt.Errorf("while getting WorkerPool %s/%s: %w", poolNamespace, poolName, err)
templateSandbox *ateapipb.SandboxConfig,
) (*atev1alpha1.SandboxConfig, error) {
name := templateSandbox.GetConfigName()
sc, err := sandboxConfigLister.Get(name)
if k8serrors.IsNotFound(err) {
return nil, status.Errorf(codes.FailedPrecondition, "SandboxConfig %q not found", name)
}
class := wp.Spec.SandboxClass
if class == "" {
class = atev1alpha1.SandboxClassGvisor
}
var sc *atev1alpha1.SandboxConfig
if name := wp.Spec.SandboxConfigName; name != "" {
sc, err = sandboxConfigLister.Get(name)
if err != nil {
return nil, fmt.Errorf("while getting SandboxConfig %q: %w", name, err)
}
if sc.Spec.SandboxClass != class {
return nil, fmt.Errorf("SandboxConfig %q has class %q but WorkerPool %s/%s is class %q",
name, sc.Spec.SandboxClass, poolNamespace, poolName, class)
if class := sandboxClassString(templateSandbox.GetSandboxClass()); string(sc.Spec.SandboxClass) != class {
return nil, status.Errorf(codes.FailedPrecondition,
"SandboxConfig %q has class %q but sandbox_config.sandbox_class is %q", name, sc.Spec.SandboxClass, class)
}
} else {
sc, err = defaultSandboxConfig(sandboxConfigLister, class)
return sc, nil
}
// resolveSandboxAssets determines the sandbox binaries and pause image an actor
// should boot with and projects them onto the ateletpb.SandboxAssets atelet
// fetches: the SandboxConfig the ActorTemplate names via
// sandbox_config.config_name (required; enforced by CreateActorTemplate),
// checked against the template's sandbox_class.
func resolveSandboxAssets(
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
templateSandbox *ateapipb.SandboxConfig,
) (*ateletpb.SandboxAssets, error) {
if sandboxClassString(templateSandbox.GetSandboxClass()) == "" {
return nil, fmt.Errorf("ActorTemplate names unrecognized sandbox_class %v", templateSandbox.GetSandboxClass())
}
if templateSandbox.GetConfigName() == "" {
return nil, fmt.Errorf("ActorTemplate names no sandbox_config.config_name")
}
sc, err := resolveTemplateSandboxConfig(sandboxConfigLister, templateSandbox)
if err != nil {
return nil, err
}
}
return sandboxAssetsProto(class, sc), nil
}
// defaultSandboxConfig returns the single SandboxConfig marked Default for the
// given class, erroring if there are zero or more than one.
func defaultSandboxConfig(lister listersv1alpha1.SandboxConfigLister, class atev1alpha1.SandboxClass) (*atev1alpha1.SandboxConfig, error) {
all, err := lister.List(labels.Everything())
if err != nil {
return nil, fmt.Errorf("while listing SandboxConfigs: %w", err)
}
var match *atev1alpha1.SandboxConfig
for _, sc := range all {
if sc.Spec.SandboxClass == class && sc.Spec.Default {
if match != nil {
return nil, fmt.Errorf("multiple default SandboxConfigs for class %q (%q and %q)", class, match.Name, sc.Name)
}
match = sc
}
}
if match == nil {
return nil, fmt.Errorf("no default SandboxConfig for class %q; set one with spec.default=true or name one via WorkerPool.spec.sandboxConfigName", class)
}
return match, nil
return sandboxAssetsProto(sc), nil
}
// sandboxAssetsProto converts a resolved SandboxConfig into the proto atelet
// consumes.
func sandboxAssetsProto(class atev1alpha1.SandboxClass, sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets {
func sandboxAssetsProto(sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets {
out := &ateletpb.SandboxAssets{
SandboxClass: string(class),
SandboxClass: string(sc.Spec.SandboxClass),
PauseImage: sc.Spec.PauseImage,
Assets: make(map[string]*ateletpb.ArchAssets, len(sc.Spec.Assets)),
}
cmd/ateapi/internal/controlapi/sandbox_assets_test.go
+55 -39
View File
@@ -15,31 +15,27 @@
package controlapi
import (
"strings"
"testing"
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/cache"
)
// listerFor builds listers over the given objects, using the same key
// functions the informers use.
func listersFor(t *testing.T, pools []*atev1alpha1.WorkerPool, configs []*atev1alpha1.SandboxConfig) (listersv1alpha1.WorkerPoolLister, listersv1alpha1.SandboxConfigLister) {
// sandboxConfigListerFor builds a lister over the given SandboxConfigs, using
// the same key function the informers use.
func sandboxConfigListerFor(t *testing.T, configs []*atev1alpha1.SandboxConfig) listersv1alpha1.SandboxConfigLister {
t.Helper()
poolIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
for _, p := range pools {
if err := poolIdx.Add(p); err != nil {
t.Fatalf("adding WorkerPool: %v", err)
}
}
configIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{})
for _, c := range configs {
if err := configIdx.Add(c); err != nil {
t.Fatalf("adding SandboxConfig: %v", err)
}
}
return listersv1alpha1.NewWorkerPoolLister(poolIdx), listersv1alpha1.NewSandboxConfigLister(configIdx)
return listersv1alpha1.NewSandboxConfigLister(configIdx)
}
func testAssets() map[string]map[string]atev1alpha1.AssetFile {
@@ -48,23 +44,12 @@ func testAssets() map[string]map[string]atev1alpha1.AssetFile {
}
}
// TestResolveSandboxAssetsCarriesPauseImage pins that the pause image travels
// with the sandbox binaries — it is resolved from the pool's SandboxConfig, not
// from the ActorTemplate — for both the named and the class-default config.
func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) {
const (
defaultPause = "registry.k8s.io/pause@sha256:default"
namedPause = "gcr.io/gke-release/pause@sha256:named"
)
defaultConfig := &atev1alpha1.SandboxConfig{
ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"},
Spec: atev1alpha1.SandboxConfigSpec{
SandboxClass: atev1alpha1.SandboxClassGvisor,
Default: true,
PauseImage: defaultPause,
Assets: testAssets(),
},
}
// TestResolveSandboxAssets pins the template-side resolution: the config the
// template names is resolved (with its class checked), an empty name or an
// unrecognized class is an error, and the pause image travels with the
// sandbox binaries.
func TestResolveSandboxAssets(t *testing.T) {
const namedPause = "gcr.io/gke-release/pause@sha256:named"
namedConfig := &atev1alpha1.SandboxConfig{
ObjectMeta: metav1.ObjectMeta{Name: "gvisor-custom"},
Spec: atev1alpha1.SandboxConfigSpec{
@@ -76,22 +61,53 @@ func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) {
tests := []struct {
name string
configName string
sandbox *ateapipb.SandboxConfig
wantPauseImage string
}{
{name: "class default", wantPauseImage: defaultPause},
{name: "named config", configName: "gvisor-custom", wantPauseImage: namedPause},
}
wantErr string
}{{
name: "named config",
sandbox: &ateapipb.SandboxConfig{
SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR,
ConfigName: "gvisor-custom",
},
wantPauseImage: namedPause,
}, {
name: "named config class mismatch",
sandbox: &ateapipb.SandboxConfig{
SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_MICROVM,
ConfigName: "gvisor-custom",
},
wantErr: `has class "gvisor"`,
}, {
name: "missing named config",
sandbox: &ateapipb.SandboxConfig{
SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR,
ConfigName: "does-not-exist",
},
wantErr: `SandboxConfig "does-not-exist" not found`,
}, {
name: "unrecognized sandbox class",
sandbox: &ateapipb.SandboxConfig{
SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_UNSPECIFIED,
ConfigName: "gvisor-custom",
},
wantErr: "unrecognized sandbox_class",
}, {
name: "empty config name",
sandbox: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR},
wantErr: "names no sandbox_config.config_name",
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pool := &atev1alpha1.WorkerPool{
ObjectMeta: metav1.ObjectMeta{Name: "pool1", Namespace: "worker-ns"},
Spec: atev1alpha1.WorkerPoolSpec{SandboxConfigName: tt.configName},
}
poolLister, configLister := listersFor(t, []*atev1alpha1.WorkerPool{pool},
[]*atev1alpha1.SandboxConfig{defaultConfig, namedConfig})
configLister := sandboxConfigListerFor(t, []*atev1alpha1.SandboxConfig{namedConfig})
got, err := resolveSandboxAssets(poolLister, configLister, "worker-ns", "pool1")
got, err := resolveSandboxAssets(configLister, tt.sandbox)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("resolveSandboxAssets() error = %v, want it to contain %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("resolveSandboxAssets() error: %v", err)
}
cmd/ateapi/internal/controlapi/service.go
+3 -4
View File
@@ -39,7 +39,7 @@ type RPCService struct {
persistence serviceStore
workerCache *workercache.Cache
dialer *AteletDialer
workerPoolLister listersv1alpha1.WorkerPoolLister
sandboxConfigLister listersv1alpha1.SandboxConfigLister
csiDriverConfigLister listersv1alpha1.CSIDriverConfigLister
actorWorkflow *ActorWorkflow
workerWorkflow *WorkerWorkflow
@@ -62,7 +62,6 @@ type VolumePluginRegistry interface {
func NewRPCService(
persistence store.Interface,
workerCache *workercache.Cache,
workerPoolLister listersv1alpha1.WorkerPoolLister,
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
csiDriverConfigLister listersv1alpha1.CSIDriverConfigLister,
storageClassLister storagev1listers.StorageClassLister,
@@ -76,13 +75,13 @@ func NewRPCService(
impl: impl,
persistence: persistence,
workerCache: workerCache,
workerPoolLister: workerPoolLister,
sandboxConfigLister: sandboxConfigLister,
csiDriverConfigLister: csiDriverConfigLister,
dialer: dialer,
instruments: instruments,
volumePlugins: volumePlugins,
}
s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, workerPoolLister, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s)
s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s)
s.workerWorkflow = NewWorkerWorkflow(impl)
return s
}
cmd/ateapi/internal/controlapi/template_reconciler.go
+1 -5
View File
@@ -23,7 +23,6 @@ import (
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
"github.com/agent-substrate/substrate/internal/resources"
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -76,15 +75,13 @@ type goldenActorControl interface {
type ActorTemplateReconciler struct {
persistence templateReconcilerStore
control goldenActorControl
sandboxConfigs listersv1alpha1.SandboxConfigLister
queue workqueue.TypedRateLimitingInterface[resources.ActorTemplateRef]
}
func NewActorTemplateReconciler(persistence templateReconcilerStore, control goldenActorControl, sandboxConfigs listersv1alpha1.SandboxConfigLister) *ActorTemplateReconciler {
func NewActorTemplateReconciler(persistence templateReconcilerStore, control goldenActorControl) *ActorTemplateReconciler {
return &ActorTemplateReconciler{
persistence: persistence,
control: control,
sandboxConfigs: sandboxConfigs,
// Create rate-limiting queue with exponential backoff
queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[resources.ActorTemplateRef]()),
}
@@ -206,7 +203,6 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
// The golden snapshot exists already.
return 0, nil
}
// TODO: Freeze sandbox assets before creating the golden actor.
actor, err := r.ensureActorExists(ctx, tmpl, goldenActorRef)
if err != nil {
cmd/ateapi/internal/controlapi/template_reconciler_test.go
+1 -2
View File
@@ -299,8 +299,7 @@ func withFailed(reason string) func(*ateapipb.ActorTemplate) {
}
func newTestTemplateReconciler(persistence templateReconcilerStore, control goldenActorControl) *ActorTemplateReconciler {
// The SandboxConfig lister is not used by reconcileOne or resync.
return NewActorTemplateReconciler(persistence, control, nil)
return NewActorTemplateReconciler(persistence, control)
}
func TestGoldenSnapshotWarmupFor(t *testing.T) {
cmd/ateapi/internal/controlapi/workflow.go
-3
View File
@@ -72,7 +72,6 @@ type ActorWorkflow struct {
workerCache *workercache.Cache
scheduler scheduling.Scheduler
dialer *AteletDialer
workerPoolLister listersv1alpha1.WorkerPoolLister
sandboxConfigLister listersv1alpha1.SandboxConfigLister
storageClassLister storagev1listers.StorageClassLister
instruments *Instruments
@@ -85,7 +84,6 @@ func NewActorWorkflow(
store actorWorkflowStore,
workerCache *workercache.Cache,
dialer *AteletDialer,
workerPoolLister listersv1alpha1.WorkerPoolLister,
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
storageClassLister storagev1listers.StorageClassLister,
instruments *Instruments,
@@ -97,7 +95,6 @@ func NewActorWorkflow(
workerCache: workerCache,
scheduler: scheduling.New(workerCache, scheduling.WithMeter(otel.Meter("ateapi"))),
dialer: dialer,
workerPoolLister: workerPoolLister,
sandboxConfigLister: sandboxConfigLister,
storageClassLister: storageClassLister,
instruments: instruments,
cmd/ateapi/internal/controlapi/workflow_resume.go
+5 -4
View File
@@ -762,10 +762,11 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou
slog.InfoContext(ctx, "Actor has no snapshot; ActorTemplate has no golden snapshot; Booting from ActorTemplate spec")
tele.SnapshotKind = ateattr.SnapshotKindBoot
// Booting from scratch: resolve the sandbox binaries from the pool's
// SandboxConfig and send them so atelet can fetch and record them.
// (Restores above are self-describing via the snapshot manifest.)
sandboxAssets, err := resolveSandboxAssets(w.workerPoolLister, w.sandboxConfigLister, assignment.GetWorkerNamespace(), assignment.GetWorkerPool())
// Booting from scratch: resolve the sandbox binaries from the
// template's SandboxConfig and send them so atelet can fetch and
// record them. (Restores above are self-describing via the snapshot
// manifest.)
sandboxAssets, err := resolveSandboxAssets(w.sandboxConfigLister, actorTemplate.GetSandboxConfig())
if err != nil {
return tele, fmt.Errorf("while resolving sandbox assets: %w", err)
}
cmd/ateapi/internal/controlapi/workflow_testutil_test.go
+1 -1
View File
@@ -54,7 +54,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa
}); err != nil && !errors.Is(err, store.ErrAlreadyExists) {
t.Fatalf("create test ActorTemplate: %v", err)
}
return NewActorWorkflow(st, nil, nil, nil, nil, nil, nil, "", nil)
return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil)
}
// seedWorkflowActor stores an actor with the given state, bound to the given
cmd/ateapi/main.go
+2 -2
View File
@@ -188,10 +188,10 @@ func main() {
volPlugins := make(map[string]volume.VolumePluginControlPlane)
ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts)
controlSrv := controlapi.NewRPCService(persistence, workerCache, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, volPlugins)
controlSrv := controlapi.NewRPCService(persistence, workerCache, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, volPlugins)
// Drive stored ActorTemplates through the golden actor flow.
templateReconciler := controlapi.NewActorTemplateReconciler(persistence, controlSrv, sandboxConfigLister)
templateReconciler := controlapi.NewActorTemplateReconciler(persistence, controlSrv)
templateReconciler.Start(shutdownCtx)
actorIDCAPool, err := localca.NewRefreshingPool(*actorIDCAPoolFile)
demos/counter/counter-microvm-csi-test.yaml
-1
View File
@@ -33,5 +33,4 @@ metadata:
spec:
replicas: 1
sandboxClass: microvm
sandboxConfigName: microvm
workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm
demos/counter/counter-microvm.yaml.tmpl
-1
View File
@@ -40,7 +40,6 @@ spec:
# template reconciler runs while building the golden snapshot.
replicas: 2
sandboxClass: microvm
sandboxConfigName: microvm
workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm
template:
nodeSelector:
demos/egress/egress-microvm-mitm-template.yaml.tmpl
+2 -2
View File
@@ -76,8 +76,8 @@ resources:
quantity: 512Mi
sandboxConfig:
sandboxClass: SANDBOX_CLASS_MICROVM
# Deliberately not the class default; installed cluster-wide by
# hack/install-microvm-deps.sh, so a missing or stale install fails loudly.
# Installed cluster-wide by hack/install-microvm-deps.sh; naming it
# explicitly makes a missing or stale install fail loudly.
configName: microvm
snapshotsConfig:
onPause: SNAPSHOT_CONTENT_SCOPE_FULL
demos/egress/egress-microvm-mitm.yaml.tmpl
-1
View File
@@ -36,7 +36,6 @@ metadata:
spec:
replicas: 2
sandboxClass: microvm
sandboxConfigName: microvm
workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm
# No template.resources, unlike the counter demo's micro-VM variant: an
# unlimited ateom container reports no capacity, which the scheduler reads as
demos/egress/egress-microvm-template.yaml.tmpl
+2 -2
View File
@@ -49,8 +49,8 @@ resources:
quantity: 512Mi
sandboxConfig:
sandboxClass: SANDBOX_CLASS_MICROVM
# Deliberately not the class default; installed cluster-wide by
# hack/install-microvm-deps.sh, so a missing or stale install fails loudly.
# Installed cluster-wide by hack/install-microvm-deps.sh; naming it
# explicitly makes a missing or stale install fail loudly.
configName: microvm
snapshotsConfig:
onPause: SNAPSHOT_CONTENT_SCOPE_FULL
demos/egress/egress-microvm.yaml.tmpl
-1
View File
@@ -37,7 +37,6 @@ metadata:
spec:
replicas: 2
sandboxClass: microvm
sandboxConfigName: microvm
workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm
# No template.resources, unlike the counter demo's micro-VM variant: an
# unlimited ateom container reports no capacity, which the scheduler reads as
docs/api-guide.md
+14 -17
View File
@@ -12,8 +12,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage
| :--- | :--- | :--- |
| `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. |
| `workerImage` | `string` | **Required.** The container image for the `ateom` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor`). |
| `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement) and which `SandboxConfig`s are eligible. |
| `sandboxConfigName` | `string` | Optional. Name of a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) providing the sandbox binaries and pause image. If empty, the cluster default `SandboxConfig` for the pool's `sandboxClass` is used. |
| `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement). The sandbox binaries themselves come from the [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) each `ActorTemplate` selects. |
| `template` | `WorkerPoolPodTemplate` | **Optional.** Metadata, scheduling, and resource settings for worker workloads. |
#### `WorkerPoolPodTemplate` (`spec.template`)
@@ -85,8 +84,8 @@ spec:
project: agent-platform
annotations:
policy.example.com/exemption: sandbox-host
# sandboxClass defaults to gvisor; the pool resolves to the cluster's default
# gvisor SandboxConfig unless sandboxConfigName is set.
# sandboxClass defaults to gvisor. The sandbox binaries come from the
# SandboxConfig each ActorTemplate selects, not from the pool.
```
### Devices (GPUs) — temporarily unsupported
@@ -118,15 +117,15 @@ The `ActorTemplate` defines the code, environment, and state-management policies
| Field | Type | Description |
| :--- | :--- | :--- |
| `containers` | `[]Container` | **Required.** The workload definition — see [Container Fields](#container-fields) below. Each container may also declare an optional `readyz` HTTP probe — see [Container Readiness Probe](#container-readiness-probe-readyz). |
| `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClass` matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. |
| `sandboxConfig` | `SandboxConfig` | **Required.** The sandbox runtime selection: `sandboxClass` (**required**, `SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`) picks the runtime family this template's actors require — only `WorkerPool`s whose `sandboxClass` matches are eligible — and `configName` (**required**) names the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object supplying the sandbox binaries. It must reference an existing config of the matching class; `CreateActorTemplate` rejects the template otherwise. |
| `workerSelector` | `*LabelSelector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels. If unset, all pools are eligible (subject to the actor's own `worker_selector`). |
| `snapshotsConfig` | `SnapshotsConfig` | **Required.** The base object-storage location snapshots are written under, plus the pause/commit/resume scopes. See [Snapshot Storage Layout](#snapshot-storage-layout). |
| `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `externalVolumeTemplate` (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one. |
| `resources` | `*ResourceRequirements` | Optional. Declares each actor's compute size via `limits` — see [Sandbox Right-Sizing](#sandbox-right-sizing-specresources). Immutable, like the rest of the spec. |
The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — comes from the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object named by `sandboxConfig.configName`. At runtime the workers resolve it from the `WorkerPool` side — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`.
The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — comes from the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) object the template names via `sandboxConfig.configName`. An actor always resolves the config from its current template — repointing the actor at another template requires the same config.
Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It has no default — `sandboxConfig` is required — and, like the rest of the spec, is immutable, so each template's class is fixed at creation.
Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It has no default — `sandboxConfig` is required and its `sandboxClass` must be set — and, like the rest of the spec, is immutable, so each template's class is fixed at creation.
### Sandbox Right-Sizing (`spec.resources`)
@@ -266,7 +265,7 @@ A container that exceeds its memory limit is OOM-killed on its own; the actor's
Per-container limits are micro-VM only today. gVisor applies cgroup limits at the sandbox level: one sentry backs every container in the actor, so a per-container cgroup is created and then stays empty ([google/gvisor#190](https://github.com/google/gvisor/issues/190)). A template that sets `resources` with `sandboxClass: gvisor` is rejected.
These limits subdivide the sandbox that [`spec.resources`](#sandbox-right-sizing-specresources) already sized; a container that declares none is bounded by the guest as a whole, not by a copy of the actor's total. A micro-VM guest is sized from `spec.resources.limits.memory` minus the VMM reserve, or from the pool's [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) when the template declares no actor-level limit. The CPU ceiling is the guest's vCPU count, which falls back to the pool's `default_vcpus` (1 unless the `SandboxConfig` raises it), so a template that declares no `spec.resources.limits.cpu` caps each container, and their sum, at `1000m`. A limit above either ceiling can never bind, so the actor fails to start with an error naming both the limit and the ceiling.
These limits subdivide the sandbox that [`spec.resources`](#sandbox-right-sizing-specresources) already sized; a container that declares none is bounded by the guest as a whole, not by a copy of the actor's total. A micro-VM guest is sized from `spec.resources.limits.memory` minus the VMM reserve, or from the template's [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) when the template declares no actor-level limit. The CPU ceiling is the guest's vCPU count, which falls back to the config's `default_vcpus` (1 unless the `SandboxConfig` raises it), so a template that declares no `spec.resources.limits.cpu` caps each container, and their sum, at `1000m`. A limit above either ceiling can never bind, so the actor fails to start with an error naming both the limit and the ceiling.
Each limit is validated on its own at apply, but the sum across the actor's containers is only checked when the actor first runs, against the real guest size. A template whose limits do not fit is accepted by the API server and fails on its first actor.
@@ -311,10 +310,10 @@ containers:
workerSelector:
matchLabels:
workload: secret-agent
# Both fields are required: sandboxClass picks the runtime family (set
# SANDBOX_CLASS_MICROVM to require micro-VM pools) and configName names the
# cluster-scoped SandboxConfig supplying the sandbox binaries (see section 3);
# gvisor-default is the cluster-wide default that manifests/ate-install ships.
# sandboxClass (required) picks the runtime family (set SANDBOX_CLASS_MICROVM
# to require micro-VM pools); configName (required) names the cluster-scoped
# SandboxConfig supplying the sandbox binaries (see section 3).
# gvisor-default is the SandboxConfig that manifests/ate-install ships.
sandboxConfig:
sandboxClass: SANDBOX_CLASS_GVISOR
configName: gvisor-default
@@ -356,7 +355,7 @@ Two consequences worth planning for:
## 3. SandboxConfig: The Sandbox Itself
`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the `ActorTemplate`. A `WorkerPool` resolves its sandbox from a `SandboxConfig` — either the one named by `spec.sandboxConfigName`, or the cluster default for the pool's `sandboxClass`.
`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. An actor's cold boot resolves the sandbox binaries from the config its `ActorTemplate` names via `sandboxConfig.configName`.
This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place.
@@ -364,12 +363,11 @@ This means a single, cluster-managed config pins the sandbox runtime version for
| Field | Type | Description |
| :--- | :--- | :--- |
| `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. A `WorkerPool` only uses `SandboxConfig`s whose `sandboxClass` matches its own. |
| `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. An `ActorTemplate` only uses `SandboxConfig`s whose `sandboxClass` matches its own. |
| `pauseImage` | `string` | **Required.** The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must be pinned by digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. |
| `default` | `bool` | Optional. Marks this as the cluster default for its `sandboxClass`. A `WorkerPool` with no `sandboxConfigName` resolves to the default for its class. At most one default per class. |
| `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. |
A default cluster-wide gVisor `SandboxConfig` (`gvisor-default`) is installed with the platform, so gVisor pools work out of the box.
A cluster-wide gVisor `SandboxConfig` (`gvisor-default`) is installed with the platform, so gVisor templates can name it via `sandboxConfig.configName` without any extra setup.
### Example
@@ -380,7 +378,6 @@ metadata:
name: gvisor-default
spec:
sandboxClass: gvisor
default: true
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
assets:
amd64:
docs/architecture.md
+1 -1
View File
@@ -324,7 +324,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov
### Sandbox Classes
A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades.
A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `ActorTemplate` names in its sandbox config (naming one is currently required; per-class cluster defaults are planned) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades.
* **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree.
docs/glossary.md
+4 -3
View File
@@ -20,9 +20,10 @@ For how the pieces fit together, see the [Architecture](architecture.md) and
- **SandboxConfig**: a cluster-scoped resource holding the sandbox binaries for
one runtime family (the gVisor `runsc` binary, or a micro-VM
kernel/firmware/config), plus the pause image for the sandbox's root
container. A `WorkerPool` resolves its sandbox from the config it names, or
from the cluster default for its class, so one config pins the runtime version
for many templates.
container. An actor resolves its sandbox at first cold boot from the config its
`ActorTemplate` names (naming one is currently required; per-class cluster
defaults are planned), so one config pins the runtime version for many
templates.
## Records (dynamic state, in the control-plane store)
hack/install-ate.sh
+5 -5
View File
@@ -887,13 +887,13 @@ deploy_ate_system() {
deploy_crds
# Enforce per-class SandboxConfig asset requirements (applied before any
# SandboxConfig so the defaults below are validated too).
# SandboxConfig so the configs below are validated too).
run_kubectl apply -f manifests/ate-install/sandboxconfig-validation.yaml
# Install the cluster-wide default sandbox config(s). Sandbox binaries live on
# cluster-scoped SandboxConfigs resolved via each WorkerPool's SandboxClass
# (decoupled from ActorTemplate). gVisor pools resolve to this default unless
# they name their own SandboxConfig.
# Install the cluster-wide sandbox config(s). Sandbox binaries live on
# cluster-scoped SandboxConfigs each ActorTemplate names via
# sandboxConfig.configName; gVisor templates name this one unless they
# create their own SandboxConfig.
run_kubectl apply -f manifests/ate-install/sandboxconfig-gvisor.yaml
# Ahead of the bundle below, for the same reason as the namespace: every
hack/install-microvm-deps.sh
+4 -4
View File
@@ -29,9 +29,9 @@
# arm64 the v1.14.0 binary is built from source, so its bytes vary per
# toolchain and cannot be pinned in the manifest).
#
# WorkerPools must reference the SandboxConfig explicitly via
# sandboxConfigName: microvm. This avoids a dirty teardown silently binding
# new pools to a stale config.
# ActorTemplates must reference the SandboxConfig explicitly via
# sandboxConfig.configName: microvm. This avoids a dirty teardown silently
# binding new templates to a stale config.
#
# On --delete: removes the SandboxConfig from the cluster. Bucket contents
# are left alone (they're inert until a SandboxConfig points at them, and
@@ -195,4 +195,4 @@ sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \
"${MANIFEST_TEMPLATE}" \
| run_kubectl apply -f -
log "Done. WorkerPools must reference this SandboxConfig by name (sandboxConfigName: microvm)."
log "Done. ActorTemplates must reference this SandboxConfig by name (sandboxConfig.configName: microvm)."
internal/e2e/fixture.go
+4 -6
View File
@@ -38,10 +38,9 @@ func substrateTemplateSubstitutions(bucket, name string, trustBundle bool) (inli
"${FIXTURE_SUFFIX}": "-" + name,
}
blocks = map[string]string{
// gvisor-default is the cluster-wide default SandboxConfig
// gvisor-default is the cluster-wide SandboxConfig
// manifests/ate-install ships; config_name is required, so the
// templates name it explicitly even though the gVisor WorkerPools
// leave sandboxConfigName empty and resolve to the same object.
// fixtures name it explicitly.
"${TEMPLATE_SANDBOX_CONFIG}": "sandboxConfig:\n sandboxClass: SANDBOX_CLASS_GVISOR\n configName: gvisor-default",
"${TEMPLATE_RESOURCES}": "",
// Off unless the caller opts in; see WithTrustBundle.
@@ -57,9 +56,8 @@ func substrateTemplateSubstitutions(bucket, name string, trustBundle bool) (inli
}
inline["${FIXTURE_SUFFIX}"] = "-" + SandboxClassMicroVM + "-" + name
// The cluster-wide SandboxConfig hack/install-microvm-deps.sh installs.
// It is deliberately not the class default, so a missing or stale one
// fails loudly.
// The cluster-wide SandboxConfig hack/install-microvm-deps.sh installs;
// a missing or stale one fails loudly at template creation.
blocks["${TEMPLATE_SANDBOX_CONFIG}"] = "sandboxConfig:\n sandboxClass: SANDBOX_CLASS_MICROVM\n configName: microvm"
// Only for fixtures that declare no limits of their own. Without them the
// guest boots at the kata config's default (2GiB), and several of those
internal/e2e/sandbox.go
+4 -4
View File
@@ -195,10 +195,10 @@ func fixtureSubstitutions(bucket, name string) (inline, blocks map[string]string
inline["${ATEOM_IMAGE}"] = "ko://github.com/agent-substrate/substrate/cmd/ateom-microvm"
inline["${FIXTURE_SUFFIX}"] = "-" + SandboxClassMicroVM + "-" + name
// The cluster-wide SandboxConfig hack/install-microvm-deps.sh installs. A
// micro-VM WorkerPool has to name it: it is deliberately not the class
// default, so a missing or stale one fails loudly.
blocks["${WORKERPOOL_RUNTIME}"] = " sandboxClass: microvm\n sandboxConfigName: microvm"
// The micro-VM ActorTemplates name the cluster-wide SandboxConfig
// hack/install-microvm-deps.sh installs (configName: microvm), so a
// missing or stale one fails loudly. The pool only selects the class.
blocks["${WORKERPOOL_RUNTIME}"] = " sandboxClass: microvm"
// Must match the WorkerPool's: a snapshot is not portable across sandbox
// classes, so only same-class pools are eligible to run these actors.
blocks["${TEMPLATE_SANDBOX_CLASS}"] = " sandboxClass: microvm"
internal/e2e/sandbox_test.go
+12 -14
View File
@@ -119,7 +119,7 @@ func memoryLimit(tmpl *ateapipb.ActorTemplate) string {
// TestRenderSubstrateFixtures_GVisor pins the default rendering: every
// micro-VM block is gone, no placeholder survives, and the templates name the
// cluster-wide default SandboxConfig.
// cluster-wide gvisor-default SandboxConfig.
func TestRenderSubstrateFixtures_GVisor(t *testing.T) {
t.Setenv(sandboxClassEnv, "")
for _, fixture := range substrateFixtures {
@@ -128,9 +128,8 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) {
if !strings.HasSuffix(pool.Spec.WorkerImage, "/cmd/ateom-gvisor") {
t.Errorf("WorkerPool workerImage = %q, want the gVisor ateom", pool.Spec.WorkerImage)
}
if pool.Spec.SandboxClass != "" || pool.Spec.SandboxConfigName != "" {
t.Errorf("WorkerPool carries micro-VM runtime fields: class=%q config=%q",
pool.Spec.SandboxClass, pool.Spec.SandboxConfigName)
if pool.Spec.SandboxClass != "" {
t.Errorf("WorkerPool carries micro-VM runtime fields: class=%q", pool.Spec.SandboxClass)
}
templates := renderTemplates(t, fixture.manifests.Template)
@@ -142,8 +141,8 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) {
if got := tmpl.GetSandboxConfig().GetSandboxClass(); got != ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR {
t.Errorf("template %s sandboxClass = %v, want GVISOR", name, got)
}
// The templates name the cluster-wide default SandboxConfig
// explicitly: config_name is required.
// The templates name the cluster-wide gvisor-default
// SandboxConfig explicitly: config_name is required.
if got := tmpl.GetSandboxConfig().GetConfigName(); got != "gvisor-default" {
t.Errorf("template %s configName = %q, want gvisor-default", name, got)
}
@@ -167,9 +166,9 @@ func TestRenderSubstrateFixtures_GVisor(t *testing.T) {
}
}
// TestRenderSubstrateFixtures_MicroVM pins the micro-VM rendering: the pool
// names the cluster-wide SandboxConfig, the templates match its class and
// carry limits, and the snapshots land under their own prefix.
// TestRenderSubstrateFixtures_MicroVM pins the micro-VM rendering: the
// templates name the cluster-wide SandboxConfig and match the pool's class
// and carry limits, and the snapshots land under their own prefix.
func TestRenderSubstrateFixtures_MicroVM(t *testing.T) {
t.Setenv(sandboxClassEnv, SandboxClassMicroVM)
for _, fixture := range substrateFixtures {
@@ -178,9 +177,8 @@ func TestRenderSubstrateFixtures_MicroVM(t *testing.T) {
if !strings.HasSuffix(pool.Spec.WorkerImage, "/cmd/ateom-microvm") {
t.Errorf("WorkerPool workerImage = %q, want the micro-VM ateom", pool.Spec.WorkerImage)
}
if pool.Spec.SandboxClass != SandboxClassMicroVM || pool.Spec.SandboxConfigName != "microvm" {
t.Errorf("WorkerPool runtime = class %q / config %q, want microvm / microvm",
pool.Spec.SandboxClass, pool.Spec.SandboxConfigName)
if pool.Spec.SandboxClass != SandboxClassMicroVM {
t.Errorf("WorkerPool runtime = class %q, want microvm", pool.Spec.SandboxClass)
}
templates := renderTemplates(t, fixture.manifests.Template)
@@ -192,8 +190,8 @@ func TestRenderSubstrateFixtures_MicroVM(t *testing.T) {
if got := tmpl.GetSandboxConfig().GetSandboxClass(); got != ateapipb.SandboxClass_SANDBOX_CLASS_MICROVM {
t.Errorf("template %s sandboxClass = %v, want MICROVM — it must match the pool's or no worker is eligible", name, got)
}
// Deliberately not the class default (see fixture.go), so a
// missing or stale microvm install fails loudly.
// Named explicitly (see fixture.go), so a missing or stale
// microvm install fails loudly.
if got := tmpl.GetSandboxConfig().GetConfigName(); got != "microvm" {
t.Errorf("template %s configName = %q, want microvm", name, got)
}
internal/e2e/template.go
+2 -3
View File
@@ -88,7 +88,6 @@ func CreateSubstrateTemplateFrom(ctx context.Context, t *testing.T, clients *Cli
Replicas: opts.PoolReplicas,
WorkerImage: existingWp.Spec.WorkerImage,
SandboxClass: existingWp.Spec.SandboxClass,
SandboxConfigName: existingWp.Spec.SandboxConfigName,
},
}
if _, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(namespace).Create(ctx, wp, metav1.CreateOptions{}); err != nil {
@@ -112,8 +111,8 @@ func CreateSubstrateTemplateFrom(ctx context.Context, t *testing.T, clients *Cli
// micro-VM, where an ActorTemplate that declares none boots the guest
// at the kata config default (2GiB) instead of the demo's 512Mi.
Resources: srcTmpl.GetResources(),
// Both sandbox_class and config_name are required; the source carries
// the pair for the class under test.
// The source carries the sandbox_class/config_name pair for the
// class under test.
SandboxConfig: srcTmpl.GetSandboxConfig(),
SnapshotsConfig: snapshots,
Volumes: srcTmpl.GetVolumes(),
manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml
+6 -14
View File
@@ -34,9 +34,6 @@ spec:
- jsonPath: .spec.sandboxClass
name: Class
type: string
- jsonPath: .spec.default
name: Default
type: boolean
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
@@ -45,8 +42,9 @@ spec:
openAPIV3Schema:
description: |-
SandboxConfig is cluster-scoped configuration describing the sandbox binaries
for a sandbox runtime family. It is referenced (or defaulted) by WorkerPools
and decouples sandbox binary selection from ActorTemplate.
for a sandbox runtime family. It is referenced by an ActorTemplate's
sandbox_config.config_name (required) and decouples
sandbox binary selection from the workload definition.
properties:
apiVersion:
description: |-
@@ -104,13 +102,6 @@ spec:
intentionally generic; per-class requirements are enforced by a
ValidatingAdmissionPolicy.
type: object
default:
description: |-
Default marks this SandboxConfig as the cluster-wide default for its
SandboxClass. A WorkerPool with no explicit SandboxConfigName resolves to
the default config for its SandboxClass. At most one default is expected
per SandboxClass.
type: boolean
pauseImage:
description: |-
PauseImage is the container image used as the root sandbox container.
@@ -132,8 +123,9 @@ spec:
sandboxClass:
default: gvisor
description: |-
SandboxClass is the sandbox runtime family this config applies to. A
WorkerPool only uses SandboxConfigs whose SandboxClass matches its own.
SandboxClass is the sandbox runtime family this config applies to. An
ActorTemplate only uses SandboxConfigs whose SandboxClass matches its
sandbox_config.sandbox_class.
enum:
- gvisor
- microvm
manifests/ate-install/generated/ate.dev_workerpools.yaml
+4 -11
View File
@@ -77,23 +77,16 @@ spec:
default: gvisor
description: |-
SandboxClass selects the sandbox runtime family for this pool, which drives
the worker pod shape (KVM/vhost device mounts and node placement) and which
SandboxConfigs are eligible. The concrete binary is still selected by
WorkerImage. Defaults to gvisor.
the worker pod shape (KVM/vhost device mounts and node placement). The
concrete binary is still selected by WorkerImage. Defaults to gvisor.
The sandbox binaries themselves come from the SandboxConfig each
ActorTemplate names (required).
See Also: TODOs in ActorTemplate SandboxClass
enum:
- gvisor
- microvm
type: string
sandboxConfigName:
description: |-
SandboxConfigName names a cluster-scoped SandboxConfig to use for fetching
sandbox binaries. It overrides the cluster-wide default SandboxConfig for
this pool's SandboxClass. The referenced config's SandboxClass must match
this pool's SandboxClass. If empty, the default SandboxConfig for the
SandboxClass is used.
type: string
template:
description: Template holds optional metadata, scheduling, and resource
settings for worker workloads.
manifests/ate-install/sandboxconfig-gvisor.yaml
+4 -5
View File
@@ -12,20 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Cluster-wide default SandboxConfig for the gVisor (runsc) sandbox class. A
# WorkerPool with sandboxClass gvisor (the default) and no explicit
# sandboxConfigName resolves to this. atelet fetches the gVisor release tarball
# Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with
# the platform so gVisor ActorTemplates have a config to name via
# sandboxConfig.configName. atelet fetches the gVisor release tarball
# (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to
# it) matching the worker node's architecture and extracts it locally. To pin a
# different release, edit the assets below or create another SandboxConfig and
# name it from the WorkerPool.
# name it from the ActorTemplate.
apiVersion: ate.dev/v1alpha1
kind: SandboxConfig
metadata:
name: gvisor-default
spec:
sandboxClass: gvisor
default: true
# The root sandbox container's image. On GCP, prefer the in-project mirror
# gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da.
pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
manifests/microvm/sandboxconfig-microvm.yaml.tmpl
+5 -9
View File
@@ -14,15 +14,11 @@
# Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox
# class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by
# hack/install-ate.sh --deploy-ate-system and marked default:true), this is
# opt-in: apply via hack/install-microvm-deps.sh --install after staging the
# asset set (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh).
#
# It is deliberately NOT marked default:true. A dirty teardown could leave this
# CR behind, and if it were the class default a subsequent WorkerPool that
# omitted sandboxConfigName would silently resolve to the stale config. Making
# every microvm WorkerPool name it explicitly (sandboxConfigName: microvm) makes
# a missing/stale config fail loudly instead.
# hack/install-ate.sh --deploy-ate-system), this is opt-in: apply via
# hack/install-microvm-deps.sh --install after staging the asset set
# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). Every microvm
# ActorTemplate names it explicitly (configName: microvm), so a missing/stale
# config fails loudly at template creation.
#
# The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs,
# base configuration.toml) are FETCHED at runtime from the cluster object store
pkg/api/v1alpha1/sandboxconfig_types.go
+6 -13
View File
@@ -51,22 +51,15 @@ type AssetFile struct {
// SandboxConfigSpec is the desired state of a SandboxConfig.
type SandboxConfigSpec struct {
// SandboxClass is the sandbox runtime family this config applies to. A
// WorkerPool only uses SandboxConfigs whose SandboxClass matches its own.
// SandboxClass is the sandbox runtime family this config applies to. An
// ActorTemplate only uses SandboxConfigs whose SandboxClass matches its
// sandbox_config.sandbox_class.
//
// +required
// +kubebuilder:validation:Enum=gvisor;microvm
// +kubebuilder:default=gvisor
SandboxClass SandboxClass `json:"sandboxClass"`
// Default marks this SandboxConfig as the cluster-wide default for its
// SandboxClass. A WorkerPool with no explicit SandboxConfigName resolves to
// the default config for its SandboxClass. At most one default is expected
// per SandboxClass.
//
// +optional
Default bool `json:"default,omitempty"`
// PauseImage is the container image used as the root sandbox container.
// It holds the sandbox's namespaces and runs no workload code, so it is an
// implementation detail of the sandbox rather than something actor authors
@@ -98,8 +91,9 @@ type SandboxConfigSpec struct {
}
// SandboxConfig is cluster-scoped configuration describing the sandbox binaries
// for a sandbox runtime family. It is referenced (or defaulted) by WorkerPools
// and decouples sandbox binary selection from ActorTemplate.
// for a sandbox runtime family. It is referenced by an ActorTemplate's
// sandbox_config.config_name (required) and decouples
// sandbox binary selection from the workload definition.
//
// +genclient
// +genclient:nonNamespaced
@@ -107,7 +101,6 @@ type SandboxConfigSpec struct {
// +kubebuilder:object:root=true
// +kubebuilder:resource:scope=Cluster,shortName=sandboxconfig
// +kubebuilder:printcolumn:name="Class",type=string,JSONPath=`.spec.sandboxClass`
// +kubebuilder:printcolumn:name="Default",type=boolean,JSONPath=`.spec.default`
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
type SandboxConfig struct {
metav1.TypeMeta `json:",inline"`
pkg/api/v1alpha1/workerpool_types.go
+4 -11
View File
@@ -94,9 +94,10 @@ type WorkerPoolSpec struct {
Template *WorkerPoolPodTemplate `json:"template,omitempty"`
// SandboxClass selects the sandbox runtime family for this pool, which drives
// the worker pod shape (KVM/vhost device mounts and node placement) and which
// SandboxConfigs are eligible. The concrete binary is still selected by
// WorkerImage. Defaults to gvisor.
// the worker pod shape (KVM/vhost device mounts and node placement). The
// concrete binary is still selected by WorkerImage. Defaults to gvisor.
// The sandbox binaries themselves come from the SandboxConfig each
// ActorTemplate names (required).
//
// See Also: TODOs in ActorTemplate SandboxClass
//
@@ -104,14 +105,6 @@ type WorkerPoolSpec struct {
// +kubebuilder:validation:Enum=gvisor;microvm
// +kubebuilder:default=gvisor
SandboxClass SandboxClass `json:"sandboxClass,omitempty"`
// SandboxConfigName names a cluster-scoped SandboxConfig to use for fetching
// sandbox binaries. It overrides the cluster-wide default SandboxConfig for
// this pool's SandboxClass. The referenced config's SandboxClass must match
// this pool's SandboxClass. If empty, the default SandboxConfig for the
// SandboxClass is used.
// +optional
SandboxConfigName string `json:"sandboxConfigName,omitempty"`
}
type WorkerPoolStatus struct {