Files
substrate/cmd/ateapi/internal/controlapi/workload_spec_test.go
T
Dmitry Berkovich 37b5006d92 DurableDir support (#295)
Implement a layer allowing to take durableDir snapshot only, without
taking snapshot of entire memory. This is a part of the #119 feature.

The "durableDir" support is introduced via adding Volumes support to
ActorTemplate.

## Knowns issues / limitations


1. No MicroVM support for homedir yet. Will be implemented in following
PR.

2. Resume from SNAPSHOT_SCOPE_DATA temporary behavior
Current behavior of resume from durableStorage returns immediately after
gVisor process is started, it means container's bootstrap is not
completed by the time resume API returned to the caller. It causes the
issue that first curl request for suspended actor throws an HTTP
exception, since webserver listening on the port inside container is not
running yet.
PR #330 fixes the issue, for a meantime the workaround is to call `
kubectl ate resume actor` and after it call to the curl itself.

- [X ] Unit tests  pass
- [X ] E2E tests are updated and pass
2026-06-26 20:05:03 -07:00

442 lines
12 KiB
Go

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controlapi
import (
"context"
"testing"
"github.com/agent-substrate/substrate/internal/proto/ateletpb"
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
"github.com/google/go-cmp/cmp"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/testing/protocmp"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/utils/ptr"
)
func TestWorkloadSpecFromActorTemplate(t *testing.T) {
tests := []struct {
name string
template *atev1alpha1.ActorTemplate
want *ateletpb.WorkloadSpec
}{
{
name: "converts DurableDir volume and mounts",
template: &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
PauseImage: "pause",
Volumes: []atev1alpha1.Volume{
{Name: "home", VolumeSource: atev1alpha1.VolumeSource{DurableDir: &atev1alpha1.DurableDirVolumeSource{}}},
},
Containers: []atev1alpha1.Container{
{
Name: "main",
Image: "main",
VolumeMounts: []atev1alpha1.VolumeMount{
{Name: "home", MountPath: "/home/user"},
{Name: "home", MountPath: "/workspace"},
},
},
},
},
},
want: &ateletpb.WorkloadSpec{
PauseImage: "pause",
Volumes: []*ateletpb.Volume{
{
Name: "home",
Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR,
Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}},
},
},
Containers: []*ateletpb.Container{
{
Name: "main",
Image: "main",
VolumeMounts: []*ateletpb.VolumeMount{
{Name: "home", MountPath: "/home/user"},
{Name: "home", MountPath: "/workspace"},
},
},
},
},
},
{
name: "skips non-DurableDir volumes",
template: &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
Volumes: []atev1alpha1.Volume{
{Name: "unsupported", VolumeSource: atev1alpha1.VolumeSource{}},
{Name: "home", VolumeSource: atev1alpha1.VolumeSource{DurableDir: &atev1alpha1.DurableDirVolumeSource{}}},
},
Containers: []atev1alpha1.Container{
{
Name: "main",
Image: "main",
VolumeMounts: []atev1alpha1.VolumeMount{
{Name: "home", MountPath: "/workspace"},
},
},
},
},
},
want: &ateletpb.WorkloadSpec{
Volumes: []*ateletpb.Volume{
{
Name: "home",
Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR,
Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}},
},
},
Containers: []*ateletpb.Container{
{
Name: "main",
Image: "main",
VolumeMounts: []*ateletpb.VolumeMount{
{Name: "home", MountPath: "/workspace"},
},
},
},
},
},
{
name: "container without volume mounts has none",
template: &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
Volumes: []atev1alpha1.Volume{
{Name: "home", VolumeSource: atev1alpha1.VolumeSource{DurableDir: &atev1alpha1.DurableDirVolumeSource{}}},
},
Containers: []atev1alpha1.Container{
{Name: "main", Image: "main"},
},
},
},
want: &ateletpb.WorkloadSpec{
Volumes: []*ateletpb.Volume{
{
Name: "home",
Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR,
Source: &ateletpb.Volume_DurableDir{DurableDir: &ateletpb.DurableDirVolume{}},
},
},
Containers: []*ateletpb.Container{{Name: "main", Image: "main"}},
},
},
{
name: "ignores container env",
template: &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
Containers: []atev1alpha1.Container{
{
Name: "main",
Image: "main",
Env: []atev1alpha1.EnvVar{
{Name: "LITERAL", Value: ptr.To("plain")},
{
Name: "SECRET",
ValueFrom: &atev1alpha1.EnvVarSource{
SecretKeyRef: &atev1alpha1.SecretKeySelector{Name: "any", Key: "any"},
},
},
},
},
},
},
},
want: &ateletpb.WorkloadSpec{
Containers: []*ateletpb.Container{{Name: "main", Image: "main"}},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := workloadSpecFromActorTemplate(tt.template)
if diff := cmp.Diff(tt.want, got, protocmp.Transform()); diff != "" {
t.Errorf("WorkloadSpec mismatch (-want +got):\n%s", diff)
}
})
}
}
func TestWorkloadSpecFromActorTemplateWithEnv(t *testing.T) {
tests := []struct {
name string
secrets []runtime.Object
template *atev1alpha1.ActorTemplate
want *ateletpb.WorkloadSpec
wantErrCode codes.Code
}{
{
name: "resolves literal and secretKeyRef env",
secrets: []runtime.Object{
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "some-secret", Namespace: "agent-ns"},
Data: map[string][]byte{"some-key": []byte("some-value")},
},
},
template: &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
PauseImage: "pause",
Containers: []atev1alpha1.Container{
{
Name: "main",
Image: "main",
Command: []string{"/main"},
Env: []atev1alpha1.EnvVar{
{Name: "LITERAL", Value: ptr.To("plain")},
{
Name: "SOME_KEY",
ValueFrom: &atev1alpha1.EnvVarSource{
SecretKeyRef: &atev1alpha1.SecretKeySelector{Name: "some-secret", Key: "some-key"},
},
},
},
},
},
},
},
want: &ateletpb.WorkloadSpec{
PauseImage: "pause",
Containers: []*ateletpb.Container{
{
Name: "main",
Image: "main",
Command: []string{"/main"},
Env: []*ateletpb.EnvEntry{
{Name: "LITERAL", Value: "plain"},
{Name: "SOME_KEY", Value: "some-value"},
},
},
},
},
},
{
name: "skips optional missing secret",
template: &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
Containers: []atev1alpha1.Container{
{
Name: "main",
Image: "main",
Env: []atev1alpha1.EnvVar{
{
Name: "OPTIONAL",
ValueFrom: &atev1alpha1.EnvVarSource{
SecretKeyRef: &atev1alpha1.SecretKeySelector{Name: "missing", Key: "key", Optional: ptr.To(true)},
},
},
},
},
},
},
},
want: &ateletpb.WorkloadSpec{
Containers: []*ateletpb.Container{{Name: "main", Image: "main"}},
},
},
{
name: "required missing secret fails",
template: &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
Containers: []atev1alpha1.Container{
{
Name: "main",
Image: "main",
Env: []atev1alpha1.EnvVar{
{
Name: "REQUIRED",
ValueFrom: &atev1alpha1.EnvVarSource{
SecretKeyRef: &atev1alpha1.SecretKeySelector{Name: "missing", Key: "key"},
},
},
},
},
},
},
},
wantErrCode: codes.FailedPrecondition,
},
{
name: "empty valueFrom fails",
template: &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
Containers: []atev1alpha1.Container{
{
Name: "main",
Image: "main",
Env: []atev1alpha1.EnvVar{
{Name: "EMPTY", ValueFrom: &atev1alpha1.EnvVarSource{}},
},
},
},
},
},
wantErrCode: codes.FailedPrecondition,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
kubeClient := fake.NewSimpleClientset(tt.secrets...)
got, err := workloadSpecFromActorTemplateWithEnv(context.Background(), kubeClient, nil, tt.template)
if tt.wantErrCode != codes.OK {
if status.Code(err) != tt.wantErrCode {
t.Fatalf("error code = %v, want %v: %v", status.Code(err), tt.wantErrCode, err)
}
return
}
if err != nil {
t.Fatalf("workloadSpecFromActorTemplateWithEnv failed: %v", err)
}
if diff := cmp.Diff(tt.want, got, protocmp.Transform()); diff != "" {
t.Errorf("WorkloadSpec mismatch (-want +got):\n%s", diff)
}
})
}
}
func TestWorkloadSpecFromActorTemplatePropagatesReadyz(t *testing.T) {
got := workloadSpecFromActorTemplate(&atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl-readyz", Namespace: "agent-ns"},
Spec: atev1alpha1.ActorTemplateSpec{
Containers: []atev1alpha1.Container{
{
Name: "with-probe",
Image: "main",
Readyz: &atev1alpha1.ContainerReadyz{
HTTPGet: &atev1alpha1.HTTPGetAction{Path: "/health", Port: 8080},
},
},
{
Name: "without-probe",
Image: "side",
},
},
},
})
want := &ateletpb.WorkloadSpec{
Containers: []*ateletpb.Container{
{
Name: "with-probe",
Image: "main",
Readyz: &ateletpb.Readyz{
HttpGet: &ateletpb.HTTPGetAction{Path: "/health", Port: 8080},
},
},
{
Name: "without-probe",
Image: "side",
},
},
}
if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" {
t.Errorf("WorkloadSpec mismatch (-want +got):\n%s", diff)
}
}
func TestWorkloadSpecFromActorTemplateWithEnvCachesSecretsAcrossCalls(t *testing.T) {
ctx := context.Background()
secretCache := newEnvSecretCache(envSecretCacheTTL)
kubeClient := fake.NewSimpleClientset(
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "some-secret",
Namespace: "agent-ns",
},
Data: map[string][]byte{
"some-key": []byte("some-value"),
},
},
)
actorTemplate := &atev1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: "tmpl1",
Namespace: "agent-ns",
},
Spec: atev1alpha1.ActorTemplateSpec{
Containers: []atev1alpha1.Container{
{
Name: "main",
Image: "main",
Env: []atev1alpha1.EnvVar{
{
Name: "SOME_KEY",
ValueFrom: &atev1alpha1.EnvVarSource{
SecretKeyRef: &atev1alpha1.SecretKeySelector{
Name: "some-secret",
Key: "some-key",
},
},
},
},
},
},
},
}
if _, err := workloadSpecFromActorTemplateWithEnv(ctx, kubeClient, secretCache, actorTemplate); err != nil {
t.Fatalf("first workloadSpecFromActorTemplateWithEnv failed: %v", err)
}
if _, err := workloadSpecFromActorTemplateWithEnv(ctx, kubeClient, secretCache, actorTemplate); err != nil {
t.Fatalf("second workloadSpecFromActorTemplateWithEnv failed: %v", err)
}
if got := secretGetCount(kubeClient); got != 1 {
t.Fatalf("secret gets before TTL expiry = %d, want 1", got)
}
expireSecretCache(secretCache)
if _, err := workloadSpecFromActorTemplateWithEnv(ctx, kubeClient, secretCache, actorTemplate); err != nil {
t.Fatalf("third workloadSpecFromActorTemplateWithEnv failed: %v", err)
}
if got := secretGetCount(kubeClient); got != 2 {
t.Fatalf("secret gets after TTL expiry = %d, want 2", got)
}
}
func expireSecretCache(secretCache *envSecretCache) {
secretCache.mu.Lock()
defer secretCache.mu.Unlock()
for key, entry := range secretCache.entries {
entry.expiresAt = entry.expiresAt.Add(-envSecretCacheTTL)
secretCache.entries[key] = entry
}
}
func secretGetCount(kubeClient *fake.Clientset) int {
count := 0
for _, action := range kubeClient.Actions() {
if action.GetVerb() == "get" && action.GetResource().Resource == "secrets" {
count++
}
}
return count
}