Publish ActorTemplate golden snapshots as tags (#1523)

Fixes #1507

Golden snapshots currently remain owned by the temporary golden actor,
so another resume/suspend cycle or actor deletion can collect a snapshot
still referenced by its template. The controller now copies the warmed
snapshot into a published tag, deletes the golden actor, and records the
tag reference on the template. Interrupted tag creation and cleanup
remain retryable; template deletion cleans up both resources.

`CreateActor` resolves an explicit `sourceTag` or the template's golden
tag into the actor's initial snapshot. Actors created before the golden
tag is ready retain their cold-boot behavior. The golden tag uses the
template UID as its name in `ate-golden`. The proto replaces
`golden_snapshot` with `golden_tag` at field 1, without backward
compatibility.

This PR is based directly on `main` and does not depend on #1521.

Follow-up recommendation: move the create → resume → wait → suspend →
tag → delete sequence into a golden-template workflow using the existing
workflow conventions. The reconciler now coordinates multiple
recoverable steps; it could retain scheduling and retries while
delegating that sequence to the workflow. This refactor is outside this
PR.

- [x] Tests pass
- [x] Appropriate changes to documentation are included in the PR

Validation: full `env -u NO_COLOR make verify` passed after rebasing
onto `main`. After the final proto field-number change, bindings were
regenerated and the control API unit/functional tests plus proto-format
and Go-format checks passed.

---------

Signed-off-by: Eitan Yarmush <eitan.yarmush@solo.io>
This commit is contained in:
Eitan Yarmush
2026-09-16 09:24:42 -04:00
committed by GitHub
parent 7592eef420
commit a58481a18e
27 changed files with 944 additions and 369 deletions
File diff suppressed because one or more lines are too long
@@ -473,7 +473,7 @@ class ControlServicer:
def DeleteActorTemplate(self, request, context):
"""Delete an ActorTemplate together with its golden actor and golden
snapshot in the ActorTemplate's namespace.
tag in the reserved ate-golden atespace.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
+1 -1
View File
@@ -149,7 +149,7 @@ wait_actortemplate_ready() {
while ((SECONDS < deadline)); do
if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then
snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenSnapshot.snapshotUri // empty' <<<"${json}")
snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}")
if [[ -n "${snapshot}" ]]; then
return 0
fi
+1 -1
View File
@@ -97,7 +97,7 @@ func WaitActorTemplateGolden(ctx context.Context, client *ateclient.Client, ref
lastErr = err
if err == nil {
goldenStatus := template.GetStatus().GetGoldenSnapshotStatus()
if goldenStatus.GetGoldenSnapshot().GetSnapshotUri() != "" {
if goldenStatus.GetGoldenTag().GetName() != "" {
return nil
}
if msg := goldenStatus.GetErrorMessage(); msg != "" {
+18 -9
View File
@@ -88,14 +88,29 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor)
return nil, err
}
// If a source tag is requested, resolve it to the external
// snapshot the new Actor starts from.
// Resolve the explicit tag, or freeze the template's current golden default.
tagRef := inActor.GetSourceTag()
if tagRef == nil {
tagRef = template.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()
} else {
for _, volume := range template.GetVolumes() {
if volume.GetExternalVolumeTemplate() != nil {
// TODO: Permit cloning after CSI volume snapshots are supported.
return nil, status.Error(codes.FailedPrecondition, "Tag cloning does not support ActorTemplates with external volumes")
}
}
}
var sourceTag *ateapipb.Tag
if tagRef := inActor.GetSourceTag(); tagRef != nil {
if tagRef != nil {
sourceTag, err = s.resolveTagSource(ctx, inActor.GetMetadata().GetAtespace(), tagRef, template)
if err != nil {
return nil, err
}
if inActor.GetSourceTag() == nil {
if err := validateGoldenSnapshotScope(sourceTag.GetStatus().GetSnapshot()); err != nil {
return nil, err
}
}
}
atespace := inActor.GetMetadata().GetAtespace()
@@ -178,12 +193,6 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string
if tag.GetStatus().GetActorTemplateUid() != template.GetMetadata().GetUid() {
return nil, status.Errorf(codes.FailedPrecondition, "source Tag must be taken from an actor with ActorTemplate uid %q", tag.GetStatus().GetActorTemplateUid())
}
for _, volume := range template.GetVolumes() {
if volume.GetExternalVolumeTemplate() != nil {
// TODO: Permit cloning after CSI volume snapshots are supported.
return nil, status.Error(codes.FailedPrecondition, "Tag cloning does not support ActorTemplates with external volumes")
}
}
return tag, nil
}
@@ -157,6 +157,26 @@ func (s *RPCService) DeleteActorTemplate(ctx context.Context, req *ateapipb.Dele
}
templateRef := resources.ActorTemplateRefFromObjectRef(req.GetActorTemplate())
// Serialize cleanup against golden actor/tag creation by the reconciler.
ctx, lease, err := acquireLease(ctx, s.impl, "lease:actortemplate:"+templateRef.Atespace+":"+templateRef.Name, "ActorTemplate "+templateRef.String())
if err != nil {
return nil, err
}
defer lease.Close()
tmpl, err := s.impl.GetActorTemplate(ctx, templateRef)
if errors.Is(err, store.ErrNotFound) {
return nil, status.Errorf(codes.NotFound, "ActorTemplate %s not found", templateRef)
}
if err != nil {
return nil, err
}
goldenRef := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()}
if _, err := s.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: goldenRef, AnyState: true}); err != nil && status.Code(err) != codes.NotFound {
return nil, fmt.Errorf("while deleting golden actor: %w", err)
}
if _, err := s.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: goldenRef}); err != nil && status.Code(err) != codes.NotFound {
return nil, fmt.Errorf("while deleting golden tag: %w", err)
}
deleted, err := s.impl.DeleteActorTemplate(ctx, templateRef)
if err != nil {
if errors.Is(err, store.ErrNotFound) {
@@ -22,6 +22,7 @@ import (
"testing"
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest"
"github.com/agent-substrate/substrate/internal/resources"
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
@@ -325,7 +326,7 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) {
// Server-owned status a client must not be able to set.
tmpl.Status = &ateapipb.ActorTemplateStatus{
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://my-bucket/snapshots/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/sneaky"},
GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
},
}
})
@@ -349,6 +350,125 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) {
}
}
func TestDeleteActorTemplate(t *testing.T) {
tests := []struct {
name string
actorDeleted bool
tagDeleted bool
pendingTag bool
// failPrefix makes object storage fail cleanup for this resource kind.
failPrefix string
wantActorAfterFailure bool
}{
{name: "golden actor and tag"},
{name: "golden actor already deleted", actorDeleted: true},
{name: "golden tag absent", tagDeleted: true},
{name: "no golden resources", actorDeleted: true, tagDeleted: true},
{name: "incomplete golden tag", pendingTag: true},
{name: "actor cleanup failure", failPrefix: "/actors/", wantActorAfterFailure: true},
{name: "tag cleanup failure", failPrefix: "/tags/"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := t.Context()
persistence := newTestPersistence(t)
tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl")
templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl)
goldenRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()}
actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: goldenRef.Atespace, Name: goldenRef.Name},
ActorTemplate: templateRef.ToObjectRef(),
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
})
workflow, objects := newFinalizeWorkflow(persistence)
actorURI := mustActorSnapshotURI(t, tmpl, actor, "snapshot")
objects.PutSnapshot(t, actorURI, "manifest.json")
actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) {
s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: actorURI.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}
s.CurrentActorTemplateUid = tmpl.GetMetadata().GetUid()
})
var tag *ateapipb.Tag
if tt.pendingTag {
tag = storetest.MustCreateTag(t, ctx, persistence, newPendingTestTag(t, goldenRef.Name, actor))
} else {
var err error
tag, err = workflow.TagActorSnapshot(ctx, tagToCreate(goldenRef, goldenRef.Name))
if err != nil {
t.Fatal(err)
}
}
tagRef := resources.TagRefFromTag(tag)
tagURI := mustReservedTagSnapshotURI(t, tag)
objects.PutSnapshot(t, tagURI, "manifest.json")
svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, objectStore: objects}
// The handler must request AnyState to clean up an active golden actor.
mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) {
s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING
})
if tt.actorDeleted {
if _, err := workflow.DeleteActor(ctx, goldenRef, true); err != nil {
t.Fatal(err)
}
}
if tt.tagDeleted {
if _, err := svc.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: tagRef.ToObjectRef()}); err != nil {
t.Fatal(err)
}
}
if tt.failPrefix != "" {
objects.OnDelete = func(_, key string) error {
if strings.Contains(key, tt.failPrefix) {
return errObjectStore
}
return nil
}
}
req := &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()}
deleted, err := svc.DeleteActorTemplate(ctx, req)
if tt.failPrefix != "" {
if !errors.Is(err, errObjectStore) {
t.Fatalf("DeleteActorTemplate = %v, want object storage error", err)
}
if _, err := persistence.GetActorTemplate(ctx, templateRef); err != nil {
t.Fatalf("template lost after cleanup failure: %v", err)
}
if _, err := persistence.GetTag(ctx, tagRef); err != nil {
t.Fatalf("tag lost after cleanup failure: %v", err)
}
_, actorErr := persistence.GetActor(ctx, goldenRef)
if tt.wantActorAfterFailure && actorErr != nil || !tt.wantActorAfterFailure && !errors.Is(actorErr, store.ErrNotFound) {
t.Fatalf("GetActor after failure = %v, want present %v", actorErr, tt.wantActorAfterFailure)
}
objects.OnDelete = nil
deleted, err = svc.DeleteActorTemplate(ctx, req)
}
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(tmpl, deleted, protocmp.Transform()); diff != "" {
t.Fatalf("deleted template mismatch (-want +got):\n%s", diff)
}
if _, err := persistence.GetActorTemplate(ctx, templateRef); !errors.Is(err, store.ErrNotFound) {
t.Fatalf("GetActorTemplate after delete = %v, want NotFound", err)
}
if _, err := persistence.GetActor(ctx, goldenRef); !errors.Is(err, store.ErrNotFound) {
t.Fatalf("GetActor after delete = %v, want NotFound", err)
}
if _, err := persistence.GetTag(ctx, tagRef); !errors.Is(err, store.ErrNotFound) {
t.Fatalf("GetTag after delete = %v, want NotFound", err)
}
for _, uri := range []resources.SnapshotURI{actorURI, tagURI} {
if got := objects.Snapshot(t, uri); len(got) != 0 {
t.Errorf("snapshot %s still holds %v", uri, got)
}
}
if _, err := svc.DeleteActorTemplate(ctx, req); status.Code(err) != codes.NotFound {
t.Fatalf("delete missing template = %v, want NotFound", err)
}
})
}
}
func TestValidateGetActorTemplateRequest(t *testing.T) {
tests := []struct {
name string
@@ -1275,7 +1395,7 @@ func TestUpdateActorTemplateMetadata(t *testing.T) {
// A server-owned status write passes validation and bumps the version.
updated, err := persistence.UpdateActorTemplate(ctx, ref, store.PreconditionFrom(created), func(tmpl *ateapipb.ActorTemplate) error {
tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1"},
GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
}}
return nil
})
@@ -1436,3 +1436,90 @@ func TestValidateSuspendActorRequest(t *testing.T) {
})
}
}
func TestCreateActor_GoldenTagDefault(t *testing.T) {
for _, scenario := range []string{"default", "explicit tag", "own snapshot", "missing", "pending", "wrong template", "data scope"} {
t.Run(scenario, func(t *testing.T) {
ctx := t.Context()
persistence := newTestPersistence(t)
storetest.MustCreateAtespace(t, ctx, persistence, "team-a")
storetest.MustCreateAtespace(t, ctx, persistence, resources.GoldenActorAtespace)
tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl")
ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "golden"}
tag := &ateapipb.Tag{
Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name},
SourceActor: ref,
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
Status: &ateapipb.TagStatus{
ActorTemplateUid: tmpl.GetMetadata().GetUid(),
Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/ate-golden/tags/" + someActorUID, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
},
}
wantCode := codes.OK
switch scenario {
case "missing":
wantCode = codes.NotFound
case "pending":
tag.Status.Snapshot = nil
wantCode = codes.FailedPrecondition
case "wrong template":
tag.Status.ActorTemplateUid = "other"
wantCode = codes.FailedPrecondition
case "data scope":
tag.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA
wantCode = codes.FailedPrecondition
}
if scenario != "missing" {
if _, err := persistence.CreateTag(ctx, tag); err != nil {
t.Fatal(err)
}
}
if _, err := persistence.UpdateActorTemplate(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl), store.PreconditionFrom(tmpl), func(db *ateapipb.ActorTemplate) error {
db.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: ref}}
return nil
}); err != nil {
t.Fatal(err)
}
actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()}
if scenario == "explicit tag" {
tag.Metadata.Name = "explicit"
tag.Status.Snapshot.SnapshotUri = "gs://bucket/atespaces/ate-golden/tags/explicit"
if _, err := persistence.CreateTag(ctx, tag); err != nil {
t.Fatal(err)
}
actor.SourceTag = &ateapipb.ObjectRef{Atespace: ref.Atespace, Name: "explicit"}
}
svc := &ServiceImpl{store: persistence}
created, err := svc.CreateActor(ctx, actor)
if status.Code(err) != wantCode {
t.Fatalf("CreateActor = %v, want %v", err, wantCode)
}
if err != nil {
return
}
if got := created.GetStatus(); got.GetExternalSnapshot().GetSnapshotUri() != tag.GetStatus().GetSnapshot().GetSnapshotUri() || got.GetCurrentActorTemplateUid() != tmpl.GetMetadata().GetUid() {
t.Fatalf("incorrect initial status: %v", got)
}
if scenario == "own snapshot" {
uri, err := resources.NewActorSnapshotURI(tmpl.GetSnapshotsConfig().GetStorageLocation(), "team-a", created.GetMetadata().GetUid(), "snapshot")
if err != nil {
t.Fatal(err)
}
if _, err := persistence.UpdateActor(ctx, resources.ActorRefFromActor(created), store.PreconditionFrom(created), func(db *ateapipb.Actor) error {
db.Status.ExternalSnapshot.SnapshotUri = uri.String()
return nil
}); err != nil {
t.Fatal(err)
}
}
workflow := &ActorWorkflow{store: persistence}
_, _, src, err := workflow.loadActorForResume(ctx, resources.ActorRefFromActor(created))
if err != nil {
t.Fatal(err)
}
if src.SnapshotURI.IsZero() {
t.Fatalf("missing snapshot source for %s", scenario)
}
})
}
}
@@ -16,7 +16,12 @@ package functionaltest
import (
"context"
"github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi"
"github.com/agent-substrate/substrate/internal/resources"
"google.golang.org/grpc/status"
"k8s.io/apimachinery/pkg/util/wait"
"testing"
"time"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"github.com/google/go-cmp/cmp"
@@ -44,7 +49,7 @@ func TestActorTemplateCRUD(t *testing.T) {
// Server-owned status on the request is ignored.
Status: &ateapipb.ActorTemplateStatus{
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://my-bucket/snapshots/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/sneaky"},
GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
},
},
},
@@ -139,6 +144,98 @@ func TestActorTemplateCRUD(t *testing.T) {
assertGrpcErrorRegex(t, err, codes.InvalidArgument, `sandbox_config\.config_name`)
}
func TestGoldenTagLifecycle(t *testing.T) {
ns := namespaceForTest("golden-tag")
tc := setupTest(t, ns)
defer tc.cleanup()
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns})
tmpl := createTemplateWithSelector(t, tc, "golden-template", &ateapipb.Selector{MatchLabels: map[string]string{poolLabelKey: ns}})
workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1")
templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl)
// Created before readiness: a later golden tag must not change its source.
early, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "early"}, ActorTemplate: templateRef.ToObjectRef(),
}})
if err != nil {
t.Fatal(err)
}
controlapi.NewActorTemplateReconciler(tc.persistence, tc.service).Start(ctx)
var goldenRef *ateapipb.ObjectRef
err = wait.PollUntilContextTimeout(ctx, 50*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (bool, error) {
current, err := tc.client.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()})
goldenRef = current.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()
return goldenRef != nil, err
})
if err != nil {
t.Fatalf("waiting for golden tag: %v", err)
}
golden, err := tc.client.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenRef})
if err != nil {
t.Fatal(err)
}
uri := golden.GetStatus().GetSnapshot().GetSnapshotUri()
parsed, err := resources.ParseSnapshotURI(uri)
if err != nil {
t.Fatal(err)
}
if !parsed.OwnedBy(resources.TagSnapshotOwner(resources.GoldenActorAtespace, parsed.Name())) {
t.Fatal("golden snapshot is not tag-owned")
}
assertSnapshotPresent(t, tc, uri)
_, err = tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: goldenRef})
if status.Code(err) != codes.NotFound {
t.Fatalf("golden actor was not deleted: %v", err)
}
late, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "late"}, ActorTemplate: templateRef.ToObjectRef(),
}})
if err != nil {
t.Fatal(err)
}
if late.GetStatus().GetExternalSnapshot().GetSnapshotUri() != uri || late.GetStatus().GetCurrentActorTemplateUid() != tmpl.GetMetadata().GetUid() {
t.Fatal("actor did not inherit golden tag snapshot and template UID")
}
waitForWorkerAvailable(t, tc, workerName)
tc.fakeAtelet.Lock.Lock()
tc.fakeAtelet.RunCalled = false
tc.fakeAtelet.Lock.Unlock()
if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: resources.ActorRefFromActor(early).ToObjectRef()}); err != nil {
t.Fatal(err)
}
if !tc.fakeAtelet.RunCalled {
t.Fatal("actor created before golden readiness did not cold boot")
}
if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: resources.ActorRefFromActor(early).ToObjectRef()}); err != nil {
t.Fatal(err)
}
waitForWorkerAvailable(t, tc, workerName)
if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: resources.ActorRefFromActor(late).ToObjectRef()}); err != nil {
t.Fatal(err)
}
if !tc.fakeAtelet.RestoreCalled {
t.Fatal("actor did not restore golden tag")
}
if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: resources.ActorRefFromActor(late).ToObjectRef()}); err != nil {
t.Fatal(err)
}
assertSnapshotPresent(t, tc, uri)
for _, actor := range []*ateapipb.Actor{early, late} {
if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: resources.ActorRefFromActor(actor).ToObjectRef(), AnyState: true}); err != nil {
t.Fatal(err)
}
}
if _, err := tc.client.DeleteActorTemplate(ctx, &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()}); err != nil {
t.Fatal(err)
}
assertSnapshotCollected(t, tc, uri)
_, err = tc.client.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenRef})
if status.Code(err) != codes.NotFound {
t.Fatalf("golden tag was not deleted: %v", err)
}
}
func TestListActorTemplates_InvalidPageToken(t *testing.T) {
ns := namespaceForTest("ns-template-invalid-token")
tc := setupTest(t, ns)
@@ -51,7 +51,7 @@ func TestCreateActor_Success(t *testing.T) {
tc := setupTest(t, ns)
defer tc.cleanup()
createTemplate(t, tc, ns)
tmpl := createTemplate(t, tc, ns)
createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{
@@ -67,9 +67,13 @@ func TestCreateActor_Success(t *testing.T) {
}
want := &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1},
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1},
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
Status: &ateapipb.ActorStatus{
State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
},
WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}},
}
@@ -663,7 +667,7 @@ func TestUpdateActor_Success(t *testing.T) {
tc := setupTest(t, ns)
defer tc.cleanup()
createTemplate(t, tc, ns)
tmpl := createTemplate(t, tc, ns)
toUpdate, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"},
@@ -687,7 +691,11 @@ func TestUpdateActor_Success(t *testing.T) {
wantActor := &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2},
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
Status: &ateapipb.ActorStatus{
State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
},
WorkerSelector: &ateapipb.Selector{
MatchLabels: map[string]string{"tier": "paid"},
},
@@ -830,7 +838,11 @@ func TestUpdateActor(t *testing.T) {
wantActor := &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2},
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
Status: &ateapipb.ActorStatus{
State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
},
WorkerSelector: &ateapipb.Selector{
MatchLabels: map[string]string{"tier": "paid"},
},
@@ -1800,6 +1812,7 @@ func TestResumeActor(t *testing.T) {
Status: &ateapipb.ActorStatus{
State: ateapipb.ActorState_ACTOR_STATE_RUNNING,
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
WorkerAssignment: &ateapipb.WorkerAssignment{
Worker: &ateapipb.ObjectRef{Name: podUID},
WorkerNamespace: ns,
@@ -2544,6 +2557,7 @@ func TestPauseActor(t *testing.T) {
ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
},
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
},
}
@@ -341,15 +341,13 @@ func assertSnapshotCollected(t *testing.T, tc *testContext, snapshotURI string)
}
}
// goldenSnapshotURI is the golden snapshot the test templates record: the
// golden Actor owns it under its own prefix in the reserved atespace, the way
// the ActorTemplateReconciler's checkpoint would leave it.
// goldenSnapshotURI is the snapshot owned by the test template's golden tag.
func goldenSnapshotURI(t *testing.T) string {
t.Helper()
const goldenActorUID = "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94"
uri, err := resources.NewActorSnapshotURI(testStorageLocation, resources.GoldenActorAtespace, goldenActorUID, "golden")
const goldenSnapshotName = "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94"
uri, err := resources.NewTagSnapshotURI(testStorageLocation, resources.GoldenActorAtespace, goldenSnapshotName)
if err != nil {
t.Fatalf("NewActorSnapshotURI: %v", err)
t.Fatalf("NewTagSnapshotURI: %v", err)
}
return uri.String()
}
@@ -455,6 +453,21 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st
t.Fatalf("failed to create actor template: %v", err)
}
createAtespace(t, tc, resources.GoldenActorAtespace)
tag, err := tc.persistence.CreateTag(context.Background(), &ateapipb.Tag{
Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()},
SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()},
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
Status: &ateapipb.TagStatus{
Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
ActorTemplateUid: created.GetMetadata().GetUid(),
SourceActorUid: "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94",
},
})
if err != nil {
t.Fatalf("create golden tag: %v", err)
}
// Record the golden snapshot on the template's status directly in the
// store, as the ActorTemplateReconciler's checkpoint would: there is no
// status RPC, and the reconciler does not run in this test environment.
@@ -463,7 +476,7 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st
func(dbTemplate *ateapipb.ActorTemplate) error {
dbTemplate.Status = &ateapipb.ActorTemplateStatus{
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
GoldenTag: resources.TagRefFromTag(tag).ToObjectRef(),
},
}
return nil
@@ -26,7 +26,6 @@ import (
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/workqueue"
@@ -46,6 +45,7 @@ const (
)
const (
reasonGoldenTagConflict = "GoldenTagConflict"
reasonGoldenActorInvalid = "GoldenActorInvalid"
reasonGoldenActorCrashed = "GoldenActorCrashed"
reasonUnexpectedState = "GoldenActorUnexpectedState"
@@ -63,6 +63,10 @@ type templateReconcilerStore interface {
// goldenActorControl is the in-process slice of the Control service the
// reconciler drives golden actors through. *RPCService satisfies it.
type goldenActorControl interface {
GetTag(ctx context.Context, req *ateapipb.GetTagRequest) (*ateapipb.Tag, error)
CreateTag(ctx context.Context, req *ateapipb.CreateTagRequest) (*ateapipb.Tag, error)
DeleteTag(ctx context.Context, req *ateapipb.DeleteTagRequest) (*ateapipb.Tag, error)
DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error)
CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error)
CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error)
GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)
@@ -201,11 +205,29 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
// The snapshot has already failed.
return 0, nil
}
if goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri() != "" {
if goldenSnapshotStatus.GetGoldenTag() != nil {
// The golden snapshot exists already.
return 0, nil
}
// A completed tag survives a crash during actor deletion or checkpointing.
tag, err := r.control.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenActorRef})
if err != nil && status.Code(err) != codes.NotFound {
return 0, fmt.Errorf("while getting golden tag: %w", err)
}
if err == nil {
if tag.GetStatus().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() || resources.ActorRefFromObjectRef(tag.GetSourceActor()) != resources.ActorRefFromObjectRef(goldenActorRef) {
return 0, r.fail(ctx, tmpl, reasonGoldenTagConflict, "golden tag belongs to another actor or template")
}
if tag.GetStatus().GetSnapshot().GetSnapshotUri() != "" {
return 0, r.saveGoldenTag(ctx, tmpl, goldenActorRef)
}
// CreateTag cannot resume an incomplete copy. Delete it before retrying.
if _, err := r.control.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: goldenActorRef}); err != nil && status.Code(err) != codes.NotFound {
return 0, fmt.Errorf("while deleting incomplete golden tag: %w", err)
}
}
actor, err := r.ensureActorExists(ctx, tmpl, goldenActorRef)
if err != nil {
if status.Code(err) == codes.InvalidArgument {
@@ -239,19 +261,19 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
return rem, nil
}
// Warmup done: suspend the golden actor and record its snapshot.
snapshot, err := r.suspendActor(ctx, goldenActorRef)
err := r.suspendActor(ctx, goldenActorRef)
if err != nil {
return 0, err
}
return 0, r.saveGoldenSnapshot(ctx, tmpl, snapshot)
return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef)
case ateapipb.ActorState_ACTOR_STATE_SUSPENDING:
// A previous pass died mid-suspend; retry suspend.
snapshot, err := r.suspendActor(ctx, goldenActorRef)
err := r.suspendActor(ctx, goldenActorRef)
if err != nil {
return 0, err
}
return 0, r.saveGoldenSnapshot(ctx, tmpl, snapshot)
return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef)
case ateapipb.ActorState_ACTOR_STATE_RESUMING,
ateapipb.ActorState_ACTOR_STATE_SUSPENDED:
@@ -261,7 +283,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
// Golden actors never start from a source snapshot, so an
// existing snapshot means an earlier suspend completed
// without being recorded.
return 0, r.saveGoldenSnapshot(ctx, tmpl, actor.GetStatus().GetExternalSnapshot())
return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef)
}
if _, err := r.control.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: goldenActorRef}); err != nil {
// A crash during resume is observed as CRASHED on the retry.
@@ -284,29 +306,42 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
}
}
// suspendActor suspends the golden actor and returns the external snapshot it
// wrote. Reentrant: SuspendActor completes an in-flight suspend and is a no-op
// on an already-suspended actor, returning the existing snapshot either way.
func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef) (*ateapipb.ExternalSnapshot, error) {
// suspendActor waits for the golden actor to produce an external snapshot.
// SuspendActor completes an in-flight suspend and is a no-op if already suspended.
func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef) error {
resp, err := r.control.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: goldenRef})
if err != nil {
// A crash during suspend is observed as CRASHED on the retry.
return nil, fmt.Errorf("while suspending golden actor: %w", err)
return fmt.Errorf("while suspending golden actor: %w", err)
}
suspended := resp.GetActor().GetStatus().GetExternalSnapshot()
if suspended.GetSnapshotUri() == "" {
return nil, fmt.Errorf("suspending golden actor produced no external snapshot")
return fmt.Errorf("suspending golden actor produced no external snapshot")
}
return suspended, nil
return nil
}
// saveGoldenSnapshot records the golden actor's external snapshot, the
// terminal success state that marks the template ready for use, ending the
// reconcile pass. The golden actor keeps owning those objects; the template
// only points at them.
func (r *ActorTemplateReconciler) saveGoldenSnapshot(ctx context.Context, observed *ateapipb.ActorTemplate, golden *ateapipb.ExternalSnapshot) error {
_, err := r.checkpoint(ctx, observed, func(snapshotStatus *ateapipb.GoldenSnapshotStatus) {
snapshotStatus.GoldenSnapshot = proto.CloneOf(golden)
// tagGoldenActor copies the snapshot into a tag before releasing the actor's copy.
func (r *ActorTemplateReconciler) tagGoldenActor(ctx context.Context, tmpl *ateapipb.ActorTemplate, ref *ateapipb.ObjectRef) error {
_, err := r.control.CreateTag(ctx, &ateapipb.CreateTagRequest{Tag: &ateapipb.Tag{
Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()},
SourceActor: ref,
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
}})
if err != nil {
return fmt.Errorf("while creating golden tag: %w", err)
}
return r.saveGoldenTag(ctx, tmpl, ref)
}
// saveGoldenTag finishes cleanup before recording terminal success, so retries
// can rediscover the tag even if deletion or the status write fails.
func (r *ActorTemplateReconciler) saveGoldenTag(ctx context.Context, tmpl *ateapipb.ActorTemplate, ref *ateapipb.ObjectRef) error {
if _, err := r.control.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}); err != nil && status.Code(err) != codes.NotFound {
return fmt.Errorf("while deleting golden actor: %w", err)
}
_, err := r.checkpoint(ctx, tmpl, func(snapshotStatus *ateapipb.GoldenSnapshotStatus) {
snapshotStatus.GoldenTag = ref
})
return err
}
@@ -346,7 +381,7 @@ func (r *ActorTemplateReconciler) fail(ctx context.Context, observed *ateapipb.A
// goldenSnapshotDone reports whether the golden snapshot build reached a
// terminal state: the snapshot was recorded, or the build failed.
func goldenSnapshotDone(snapshotStatus *ateapipb.GoldenSnapshotStatus) bool {
return snapshotStatus.GetGoldenSnapshot().GetSnapshotUri() != "" || snapshotStatus.GetErrorMessage() != ""
return snapshotStatus.GetGoldenTag() != nil || snapshotStatus.GetErrorMessage() != ""
}
// goldenSnapshotWarmupFor returns 0 when every container has a readyz probe
@@ -143,7 +143,13 @@ func (s *fakeTemplateStore) storedStatus(t *testing.T, ref resources.ActorTempla
// from that observation. Tests seed mid-lifecycle states via exists /
// goldenState / goldenSnapshot.
type fakeGoldenControl struct {
mu sync.Mutex
mu sync.Mutex
tag *ateapipb.Tag
tagErr error
deleteErr error
deleteTagErr error
tagReqs []*ateapipb.CreateTagRequest
deleteReqs []*ateapipb.DeleteActorRequest
createErr error
resumeErr error
@@ -235,6 +241,52 @@ func (c *fakeGoldenControl) SuspendActor(_ context.Context, req *ateapipb.Suspen
}, nil
}
func (c *fakeGoldenControl) GetTag(_ context.Context, _ *ateapipb.GetTagRequest) (*ateapipb.Tag, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.tag == nil {
return nil, status.Error(codes.NotFound, "no tag")
}
return proto.CloneOf(c.tag), nil
}
func (c *fakeGoldenControl) CreateTag(_ context.Context, req *ateapipb.CreateTagRequest) (*ateapipb.Tag, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.tagReqs = append(c.tagReqs, req)
if c.tagErr != nil {
return nil, c.tagErr
}
c.tag = proto.CloneOf(req.GetTag())
c.tag.Status = &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}}
return proto.CloneOf(c.tag), nil
}
func (c *fakeGoldenControl) DeleteTag(_ context.Context, _ *ateapipb.DeleteTagRequest) (*ateapipb.Tag, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.deleteTagErr != nil {
return nil, c.deleteTagErr
}
tag := c.tag
c.tag = nil
return tag, nil
}
func (c *fakeGoldenControl) DeleteActor(_ context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.deleteReqs = append(c.deleteReqs, req)
if c.deleteErr != nil {
return nil, c.deleteErr
}
if !c.exists {
return nil, status.Error(codes.NotFound, "no actor")
}
c.exists = false
return &ateapipb.Actor{}, nil
}
func (c *fakeGoldenControl) callCounts() (creates, resumes, suspends int) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -286,9 +338,9 @@ func withSnapshotDeadline(at time.Time) func(*ateapipb.ActorTemplate) {
}
}
func withGoldenSnapshot(snapshotURI string) func(*ateapipb.ActorTemplate) {
func withGoldenTag() func(*ateapipb.ActorTemplate) {
return func(tmpl *ateapipb.ActorTemplate) {
seededGoldenStatus(tmpl).GoldenSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: snapshotURI}
seededGoldenStatus(tmpl).GoldenTag = &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}
}
}
@@ -340,9 +392,8 @@ func TestReconcileOne(t *testing.T) {
// stored error message must be empty. Checked when template is seeded.
wantFailedReason string
wantMessage string
// wantSnapshot must equal the stored golden snapshot URI; empty means
// the snapshot must not be recorded.
wantSnapshot string
// wantTag indicates that the golden tag should be recorded.
wantTag bool
// wantDeadline asserts whether take_golden_snapshot_at is set.
wantDeadline bool
wantCreates int
@@ -353,7 +404,7 @@ func TestReconcileOne(t *testing.T) {
name: "happy path creates, resumes, and snapshots the golden actor",
template: testTemplate(),
control: &fakeGoldenControl{snapshot: goldenSnapshot},
wantSnapshot: goldenSnapshot,
wantTag: true,
wantDeadline: true,
wantCreates: 1,
wantResumes: 1,
@@ -382,7 +433,7 @@ func TestReconcileOne(t *testing.T) {
template: testTemplate(
withSnapshotDeadline(time.Now().Add(-time.Minute))),
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_RUNNING, snapshot: goldenSnapshot},
wantSnapshot: goldenSnapshot,
wantTag: true,
wantSuspends: 1,
},
{
@@ -405,14 +456,14 @@ func TestReconcileOne(t *testing.T) {
name: "suspending golden actor is completed and recorded",
template: testTemplate(),
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, snapshot: goldenSnapshot},
wantSnapshot: goldenSnapshot,
wantTag: true,
wantSuspends: 1,
},
{
name: "suspended golden actor with a snapshot is recorded without more control calls",
template: testTemplate(),
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: goldenSnapshot},
wantSnapshot: goldenSnapshot,
name: "suspended golden actor with a snapshot is recorded without more control calls",
template: testTemplate(),
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: goldenSnapshot},
wantTag: true,
},
{
name: "create AlreadyExists requeues for the retry to observe",
@@ -488,10 +539,10 @@ func TestReconcileOne(t *testing.T) {
control: &fakeGoldenControl{},
},
{
name: "terminal golden snapshot is a noop",
template: testTemplate(withGoldenSnapshot(goldenSnapshot)),
control: &fakeGoldenControl{},
wantSnapshot: goldenSnapshot,
name: "terminal golden snapshot is a noop",
template: testTemplate(withGoldenTag()),
control: &fakeGoldenControl{},
wantTag: true,
},
{
name: "terminal error message is a noop",
@@ -538,8 +589,8 @@ func TestReconcileOne(t *testing.T) {
t.Errorf("stored error message = %q, want it to contain %q", errorMessage, tt.wantMessage)
}
}
if got := snapshotStatus.GetGoldenSnapshot().GetSnapshotUri(); got != tt.wantSnapshot {
t.Errorf("stored golden snapshot uri = %q, want %q", got, tt.wantSnapshot)
if got := snapshotStatus.GetGoldenTag(); (got != nil) != tt.wantTag {
t.Errorf("stored golden tag = %v, want tag %v", got, tt.wantTag)
}
if tt.wantDeadline && snapshotStatus.GetTakeGoldenSnapshotAt() == nil {
t.Error("stored take_golden_snapshot_at is nil, want set")
@@ -591,13 +642,12 @@ func TestReconcileOne_GoldenActorRequests(t *testing.T) {
func TestCheckpoint_TerminalStateErrors(t *testing.T) {
ctx := context.Background()
goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1"
for _, seed := range []struct {
name string
opt func(*ateapipb.ActorTemplate)
}{
{"golden snapshot taken", withGoldenSnapshot(goldenSnapshot)},
{"golden snapshot taken", withGoldenTag()},
{"failed", withFailed(reasonGoldenActorCrashed)},
} {
t.Run(seed.name, func(t *testing.T) {
@@ -658,7 +708,6 @@ func drainQueue(r *ActorTemplateReconciler) []resources.ActorTemplateRef {
}
func TestResync_QueuesOnlyActionableTemplates(t *testing.T) {
goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1"
tests := []struct {
name string
@@ -667,7 +716,7 @@ func TestResync_QueuesOnlyActionableTemplates(t *testing.T) {
}{
{"empty status", nil, true},
{"mid warmup", []func(*ateapipb.ActorTemplate){withSnapshotDeadline(time.Now().Add(time.Hour))}, true},
{"golden snapshot taken", []func(*ateapipb.ActorTemplate){withGoldenSnapshot(goldenSnapshot)}, false},
{"golden snapshot taken", []func(*ateapipb.ActorTemplate){withGoldenTag()}, false},
{"failed", []func(*ateapipb.ActorTemplate){withFailed(reasonGoldenActorCrashed)}, false},
}
@@ -714,3 +763,125 @@ func TestResync_FollowsPagination(t *testing.T) {
t.Errorf("queued %d templates, want 3 (all pages walked)", got)
}
}
func TestReconcileOne_GoldenTagRecovery(t *testing.T) {
ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}
completed := &ateapipb.Tag{
Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name},
SourceActor: ref,
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/tag-snapshot"}},
}
incomplete := proto.CloneOf(completed)
incomplete.Status.Snapshot = nil
tests := []struct {
name string
// tag is the golden tag an earlier pass left behind, if any.
tag *ateapipb.Tag
// actorDeleted seeds a pass that died after deleting the golden actor.
actorDeleted bool
// These errors fail one step of the first pass; the retry succeeds.
createTagErr error
deleteActorErr error
deleteTagErr error
// wantCreateTags counts copy attempts across both passes.
wantCreateTags int
}{
{name: "completed tag", tag: completed, wantCreateTags: 0},
{name: "actor already deleted", tag: completed, actorDeleted: true, wantCreateTags: 0},
{name: "incomplete tag", tag: incomplete, wantCreateTags: 1},
{name: "copy failure", createTagErr: errors.New("copy interrupted"), wantCreateTags: 2},
{name: "actor deletion failure", deleteActorErr: errors.New("storage unavailable"), wantCreateTags: 1},
{name: "tag deletion failure", tag: incomplete, deleteTagErr: errors.New("storage unavailable"), wantCreateTags: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
control := &fakeGoldenControl{
tag: proto.CloneOf(tt.tag), exists: !tt.actorDeleted,
goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: "gs://bucket/actor-snapshot",
tagErr: tt.createTagErr, deleteErr: tt.deleteActorErr, deleteTagErr: tt.deleteTagErr,
}
st := newFakeTemplateStore(testTemplate())
r := newTestTemplateReconciler(st, control)
defer r.queue.ShutDown()
_, err := r.reconcileOne(t.Context(), testTemplateRef)
wantErr := tt.createTagErr != nil || tt.deleteActorErr != nil || tt.deleteTagErr != nil
if (err != nil) != wantErr {
t.Fatalf("reconcile = %v, want error %v", err, wantErr)
}
if wantErr {
if st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag() != nil {
t.Fatal("marked ready before cleanup completed")
}
if !control.exists {
t.Fatal("deleted actor after tag failure")
}
control.tagErr, control.deleteErr, control.deleteTagErr = nil, nil, nil
if _, err := r.reconcileOne(t.Context(), testTemplateRef); err != nil {
t.Fatal(err)
}
}
if control.exists {
t.Fatal("golden actor still exists")
}
if !proto.Equal(st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag(), ref) {
t.Fatal("golden tag not recorded")
}
if control.tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" {
t.Fatal("golden tag has no snapshot")
}
if len(control.createReqs) != 0 || len(control.resumeReqs) != 0 || len(control.suspendReqs) != 0 {
t.Fatal("repeated golden actor warmup")
}
if got := len(control.tagReqs); got != tt.wantCreateTags {
t.Fatalf("CreateTag calls = %d, want %d", got, tt.wantCreateTags)
}
for _, req := range control.tagReqs {
if !proto.Equal(req.Tag.SourceActor, ref) || req.Tag.Scope != ateapipb.TagScope_TAG_SCOPE_PUBLISHED || req.Tag.Metadata.Name != ref.Name {
t.Fatalf("incorrect golden tag request: %v", req)
}
}
})
}
}
func TestReconcileOne_GoldenTagConflict(t *testing.T) {
tests := []struct {
name string
templateUID string
sourceActor *ateapipb.ObjectRef
}{
{name: "template", templateUID: "another-template", sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}},
{name: "actor name", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "another-actor"}},
{name: "actor atespace", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: "another-atespace", Name: testTemplateUID}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tag := &ateapipb.Tag{
SourceActor: tt.sourceActor,
Status: &ateapipb.TagStatus{ActorTemplateUid: tt.templateUID},
}
control := &fakeGoldenControl{tag: proto.CloneOf(tag), exists: true}
st := newFakeTemplateStore(testTemplate())
r := newTestTemplateReconciler(st, control)
defer r.queue.ShutDown()
for range 2 {
after, err := r.reconcileOne(t.Context(), testTemplateRef)
if err != nil || after != 0 {
t.Fatalf("reconcile = (%v, %v), want terminal failure without retry", after, err)
}
snapshotStatus := st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus()
if snapshotStatus.GetErrorMessage() != reasonGoldenTagConflict+": golden tag belongs to another actor or template" || snapshotStatus.GetGoldenTag() != nil {
t.Fatalf("unexpected golden snapshot status: %v", snapshotStatus)
}
if !proto.Equal(control.tag, tag) || !control.exists || len(control.tagReqs) != 0 || len(control.deleteReqs) != 0 {
t.Fatal("modified golden resources after ownership conflict")
}
}
r.resync(t.Context())
if r.queue.Len() != 0 {
t.Fatal("resync queued a terminally failed template")
}
})
}
}
@@ -1314,19 +1314,6 @@ func TestValidateNestedExternalSnapshot(t *testing.T) {
return Validate_Tag(ctx, op, nil, obj, nil)
},
},
{
name: "actor_template.status.golden_snapshot_status.golden_snapshot",
path: field.NewPath("golden_snapshot_status", "golden_snapshot"),
validate: func(ctx context.Context) field.ErrorList {
op := operation.Operation{Type: operation.Create}
obj := &ateapipb.ActorTemplateStatus{
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: badExternalSnapshot(),
},
}
return Validate_ActorTemplateStatus(ctx, op, nil, obj, nil)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -38,9 +38,9 @@ import (
// and passed by value to the restore step — never mutated after resolution.
type resumeSnapshotSource struct {
// SnapshotURI is the storage location of the durable snapshot to restore
// from: the actor's own latest snapshot when one exists, the template's
// golden snapshot otherwise. Zero means cold boot from the spec (unless
// the actor holds a local snapshot, which takes precedence at restore).
// from: the actor's latest snapshot, including a tag borrowed at creation.
// Zero means cold boot from the spec (unless the actor holds a local
// snapshot, which takes precedence at restore).
SnapshotURI resources.SnapshotURI
Scope ateapipb.SnapshotContentScope
// GoldenSnapshotURI is the storage location of the ActorTemplate's golden
@@ -179,7 +179,6 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc
if err != nil {
return nil, nil, src, err
}
goldenSnapshotStatus := actorTemplate.GetStatus().GetGoldenSnapshotStatus()
if uri := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); uri != "" {
if src.SnapshotURI, err = resources.ParseSnapshotURI(uri); err != nil {
return nil, nil, src, status.Errorf(codes.DataLoss, "Actor %s external snapshot: %v", actorRef, err)
@@ -192,14 +191,6 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc
// as well; it is already disallowed at admission time.
builtOnTemplateUID := actor.GetStatus().GetCurrentActorTemplateUid()
src.TemplateReplaced = builtOnTemplateUID != "" && builtOnTemplateUID != actorTemplate.GetMetadata().GetUid()
} else if goldenURI := goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri(); goldenURI != "" {
if err := validateGoldenSnapshotScope(goldenSnapshotStatus.GetGoldenSnapshot()); err != nil {
return nil, nil, src, err
}
if src.SnapshotURI, err = resources.ParseSnapshotURI(goldenURI); err != nil {
return nil, nil, src, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", goldenURI, err)
}
src.Scope = goldenSnapshotStatus.GetGoldenSnapshot().GetContentScope()
}
// The template's onResume configuration selects the boot source for the
@@ -218,13 +209,25 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc
dataOnly = src.Scope == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA
}
if dataOnly {
goldenURI := goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri()
if goldenURI == "" {
return nil, nil, src, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden snapshot, which is not available")
ref := actorTemplate.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()
if ref == nil {
return nil, nil, src, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden tag, which is not available")
}
if err := validateGoldenSnapshotScope(goldenSnapshotStatus.GetGoldenSnapshot()); err != nil {
tag, err := w.store.GetTag(ctx, resources.TagRefFromObjectRef(ref))
if errors.Is(err, store.ErrNotFound) {
return nil, nil, src, status.Error(codes.FailedPrecondition, "ActorTemplate golden tag is not available")
}
if err != nil {
return nil, nil, src, fmt.Errorf("while getting golden tag: %w", err)
}
golden := tag.GetStatus().GetSnapshot()
if golden.GetSnapshotUri() == "" || tag.GetStatus().GetActorTemplateUid() != actorTemplate.GetMetadata().GetUid() {
return nil, nil, src, status.Error(codes.FailedPrecondition, "ActorTemplate golden tag is incomplete or belongs to another template")
}
if err := validateGoldenSnapshotScope(golden); err != nil {
return nil, nil, src, err
}
goldenURI := golden.GetSnapshotUri()
if src.GoldenSnapshotURI, err = resources.ParseSnapshotURI(goldenURI); err != nil {
return nil, nil, src, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", goldenURI, err)
}
@@ -753,7 +756,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou
_, err = client.Restore(ctx, req)
return tele, maybeCrashActor(ctx, w.store, actorRef, err, "while restoring durable snapshot", ateattr.OperationResume)
} else {
slog.InfoContext(ctx, "Actor has no snapshot; ActorTemplate has no golden snapshot; Booting from ActorTemplate spec")
slog.InfoContext(ctx, "Actor has no snapshot; Booting from ActorTemplate spec")
tele.SnapshotKind = ateattr.SnapshotKindBoot
// Booting from scratch: resolve the sandbox binaries from the
@@ -19,7 +19,6 @@ import (
"errors"
"fmt"
"net"
"strings"
"sync"
"testing"
"time"
@@ -1028,12 +1027,27 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) {
}
if tt.goldenURI != "" {
tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.goldenURI, ContentScope: tt.goldenScope},
GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
}}
}
if _, err := persistence.CreateActorTemplate(ctx, tmpl); err != nil {
stored, err := persistence.CreateActorTemplate(ctx, tmpl)
if err != nil {
t.Fatalf("create template: %v", err)
}
if tt.goldenURI != "" {
_, err := persistence.CreateTag(ctx, &ateapipb.Tag{
Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "golden"},
SourceActor: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
Status: &ateapipb.TagStatus{
ActorTemplateUid: stored.GetMetadata().GetUid(),
Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.goldenURI, ContentScope: tt.goldenScope},
},
})
if err != nil {
t.Fatalf("create golden tag: %v", err)
}
}
w := &ActorWorkflow{store: persistence}
_, _, src, err := w.loadActorForResume(ctx, actorRef)
@@ -1053,41 +1067,25 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) {
}
}
// TestLoadActorForResume_GoldenFallbackRejectsNonFullGolden covers the
// golden-fallback branch (actor with no snapshot of its own): a golden
// snapshot recorded with a non-Full scope holds no guest state, so the resume
// must fail with a clear error instead of forwarding its scope to atelet
// with no golden location (which atelet rejects with a confusing
// "missing bucket" validation error).
func TestLoadActorForResume_GoldenFallbackRejectsNonFullGolden(t *testing.T) {
// A golden tag becoming ready after creation does not change an actor's source.
func TestLoadActorForResume_DoesNotDefaultGolden(t *testing.T) {
ctx := context.Background()
persistence := newTestPersistence(t)
actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"}
seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED)
storetest.MustCreateAtespace(t, ctx, persistence, "ns")
if _, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{
Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"},
Status: &ateapipb.ActorTemplateStatus{
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: &ateapipb.ExternalSnapshot{
SnapshotUri: someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1"),
ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
},
},
},
Status: &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
}},
}); err != nil {
t.Fatalf("create template: %v", err)
t.Fatal(err)
}
w := &ActorWorkflow{store: persistence}
_, _, _, err := w.loadActorForResume(ctx, actorRef)
if got := status.Code(err); got != codes.FailedPrecondition {
t.Fatalf("status.Code(err) = %v, want FailedPrecondition (err: %v)", got, err)
}
if !strings.Contains(err.Error(), "regenerate the golden snapshot") {
t.Errorf("error %q does not tell the operator to regenerate the golden snapshot", err)
_, _, src, err := w.loadActorForResume(ctx, actorRef)
if err != nil || !src.SnapshotURI.IsZero() {
t.Fatalf("source = %+v, err = %v; want cold boot", src, err)
}
}
@@ -1326,7 +1324,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
type templateSeed struct {
// onPause is the template's pause scope.
onPause ateapipb.SnapshotContentScope
// golden seeds Status.GoldenSnapshotStatus.GoldenSnapshot.
// golden seeds the template's golden tag snapshot.
golden *ateapipb.ExternalSnapshot
// fromData is the template's onResume boot-source policy.
fromData ateapipb.ResumeSource
@@ -1366,8 +1364,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
want: restoreWant{run: true},
},
{
name: "03 golden fallback restores the golden snapshot in Full",
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}},
name: "03 inherited golden snapshot restores in Full",
actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, tmplUID: "current"},
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}},
want: restoreWant{
checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL,
snapshotURI: goldenURI,
@@ -1375,9 +1374,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
},
},
{
// With no actor snapshot the restore is not data-only, so the
// golden rides in ExternalConfig and GoldenSnapshotUri stays empty.
name: "04 golden fallback under Golden fromData is a plain Full restore",
// An inherited Full golden snapshot needs no data-only overlay.
name: "04 inherited golden under Golden fromData is a plain Full restore",
actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, tmplUID: "current"},
tmpl: templateSeed{
golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope},
fromData: fromGolden,
@@ -1389,27 +1388,21 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
},
},
{
name: "05 golden fallback rejects a non-Full golden",
name: "05 late non-Full golden does not change a cold boot",
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: dataScope}},
want: restoreWant{code: codes.FailedPrecondition},
want: restoreWant{run: true},
},
{
name: "06 golden fallback rejects a malformed golden URI",
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}},
want: restoreWant{code: codes.DataLoss},
name: "06 inherited golden snapshot rejects a malformed URI",
actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}, tmplUID: "current"},
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}},
want: restoreWant{code: codes.DataLoss},
},
{
// TemplateReplaced is derived from the actor's own durable
// snapshot; without one, a repoint cannot downgrade the golden
// fallback.
name: "07 template repoint does not affect the golden fallback",
actor: actorSeed{tmplUID: "mismatch"},
name: "07 template repoint with a late golden still cold-boots",
actor: actorSeed{tmplUID: "old-template-uid"},
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}},
want: restoreWant{
checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL,
snapshotURI: goldenURI,
scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL,
},
want: restoreWant{run: true},
},
{
name: "08 Full durable snapshot restores itself in Full",
@@ -1732,13 +1725,23 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
}
if tt.tmpl.golden != nil {
tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: tt.tmpl.golden,
GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
}}
}
createdTmpl, err := persistence.CreateActorTemplate(ctx, tmpl)
if err != nil {
t.Fatalf("create template: %v", err)
}
if tt.tmpl.golden != nil {
if _, err := persistence.CreateTag(ctx, &ateapipb.Tag{
Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "golden"},
SourceActor: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
Status: &ateapipb.TagStatus{ActorTemplateUid: createdTmpl.GetMetadata().GetUid(), Snapshot: tt.tmpl.golden},
}); err != nil {
t.Fatalf("create golden tag: %v", err)
}
}
if createdTmpl.GetMetadata().GetUid() == "" {
t.Fatal("created template has no UID; the matching tmplUID case would be vacuous")
}
@@ -3610,10 +3610,10 @@ func Validate_GoldenSnapshotStatus(
ctx context.Context, op operation.Operation, fldPath *field.Path,
obj, oldObj *ateapipb.GoldenSnapshotStatus) (errs field.ErrorList) {
{ // field ateapipb.GoldenSnapshotStatus.GoldenSnapshot
{ // field ateapipb.GoldenSnapshotStatus.GoldenTag
fn := func(
fldPath *field.Path,
obj, oldObj *ateapipb.ExternalSnapshot,
obj, oldObj *ateapipb.ObjectRef,
oldValueCorrelated bool) (errs field.ErrorList) {
// don't revalidate unchanged data
if oldValueCorrelated && op.Type == operation.Update {
@@ -3629,15 +3629,30 @@ func Validate_GoldenSnapshotStatus(
if earlyReturn {
return // do not proceed
}
func() { // cohort = "atespace"
earlyReturn := false
if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace",
func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.RequiredValue).MarkShortCircuit(); len(e) != 0 {
errs = append(errs, e...)
earlyReturn = true
}
if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace",
func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 {
earlyReturn = true
}
if earlyReturn {
return // do not proceed
}
}()
// call the type's validation function
errs = append(errs, Validate_ExternalSnapshot(ctx, op, fldPath, obj, oldObj)...)
errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...)
return
}
oldVal := safe.Field(oldObj,
func(oldObj *ateapipb.GoldenSnapshotStatus) *ateapipb.ExternalSnapshot {
return oldObj.GoldenSnapshot
func(oldObj *ateapipb.GoldenSnapshotStatus) *ateapipb.ObjectRef {
return oldObj.GoldenTag
})
errs = append(errs, fn(fldPath.Child("golden_snapshot"), obj.GoldenSnapshot, oldVal, oldObj != nil)...)
errs = append(errs, fn(fldPath.Child("golden_tag"), obj.GoldenTag, oldVal, oldObj != nil)...)
}
// field ateapipb.GoldenSnapshotStatus.TakeGoldenSnapshotAt has no validation
+2 -2
View File
@@ -107,7 +107,7 @@ kubectl ate get workers -l <label-selector>
| Column | Meaning |
|---|---|
| `ATESPACE` | The atespace the actor belongs to. Part of the actor's identity; folded into the storage key as `actor:<atespace>:<name>`. |
| `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises during `ResumeGoldenActor`. |
| `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises while building its golden tag. |
| `TEMPLATE` | The `ActorTemplate` the actor was created from, displayed as `<atespace>/<name>`. |
| `STATE` | One of `ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`. |
| `WORKER POD` | The worker pod (namespace/name) currently hosting the actor. Empty while suspended. |
@@ -184,7 +184,7 @@ for a complete manifest example.
| `ATESPACE` | The atespace the template belongs to. |
| `NAME` | The template's name. |
| `SANDBOX CLASS` | The sandbox runtime family (`SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`). |
| `GOLDEN SNAPSHOT` | The golden snapshot's name once it exists (actors can be created); empty while the golden build is still running. |
| `GOLDEN TAG` | The golden tag's name once it exists; empty while the golden build is still running. |
| `ERROR` | `ERROR` when the golden build failed; `-o yaml` shows the full message. |
| `AGE` | Time elapsed since the template was created. |
+2 -2
View File
@@ -291,7 +291,7 @@ func PrintActorTemplatesTo(out io.Writer, templates []*ateapipb.ActorTemplate, f
return printProto(out, &ateapipb.ListActorTemplatesResponse{ActorTemplates: templates}, format)
case "table":
w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0)
fmt.Fprintln(w, "ATESPACE\tNAME\tSANDBOX CLASS\tGOLDEN SNAPSHOT\tERROR\tAGE")
fmt.Fprintln(w, "ATESPACE\tNAME\tSANDBOX CLASS\tGOLDEN TAG\tERROR\tAGE")
for _, t := range templates {
gss := t.GetStatus().GetGoldenSnapshotStatus()
// Error messages are too long for a table cell.
@@ -302,7 +302,7 @@ func PrintActorTemplatesTo(out io.Writer, templates []*ateapipb.ActorTemplate, f
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
t.GetMetadata().GetAtespace(), t.GetMetadata().GetName(),
t.GetSandboxConfig().GetSandboxClass(),
gss.GetGoldenSnapshot().GetSnapshotUri(), errFlag,
gss.GetGoldenTag().GetName(), errFlag,
formatAge(t.GetMetadata().GetCreateTime()))
}
return w.Flush()
@@ -440,7 +440,7 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) {
},
Status: &ateapipb.ActorTemplateStatus{
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/snap-1"},
GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
},
},
},
@@ -481,10 +481,10 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) {
// Sorted by atespace, then name. The ERROR column only flags that an
// error message exists; the full text is available via json/yaml.
expected := `ATESPACE NAME SANDBOX CLASS GOLDEN SNAPSHOT ERROR AGE
ate-demo-counter-substrate counter SANDBOX_CLASS_GVISOR gs://private/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/snap-1 5m
ate-demo-counter-substrate counter-2 SANDBOX_CLASS_GVISOR 3d
ate-demo-counter-substrate-microvm counter-microvm SANDBOX_CLASS_MICROVM ERROR 5h
expected := `ATESPACE NAME SANDBOX CLASS GOLDEN TAG ERROR AGE
ate-demo-counter-substrate counter SANDBOX_CLASS_GVISOR golden-tag 5m
ate-demo-counter-substrate counter-2 SANDBOX_CLASS_GVISOR 3d
ate-demo-counter-substrate-microvm counter-microvm SANDBOX_CLASS_MICROVM ERROR 5h
`
if diff := cmp.Diff(expected, buf.String()); diff != "" {
t.Errorf("output mismatch (-want +got):\n%s", diff)
+8 -6
View File
@@ -360,13 +360,13 @@ snapshotsConfig:
<location>/atespaces/<atespace>/tags/<tag uid>
```
The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot — the golden actor lives in the reserved `ate-golden` atespace — under `gs://my-bucket/secret-agent/atespaces/ate-golden/actors/<uid>/snapshots/<name>`.
The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot — the golden tag lives in the reserved `ate-golden` atespace — under `gs://my-bucket/secret-agent/atespaces/ate-golden/tags/<tag uid>`.
An actor takes a series of snapshots over its life, so it gets a prefix of its own and each snapshot sits below it. A tag holds exactly one, so the tag's prefix *is* its snapshot's. Both owners are keyed on their UID, so recreating an actor or tag under the same name never inherits its predecessor's objects. A pending tag records its base location in `status.storageLocation`; together with its atespace and UID, this identifies any partial copy to collect if creation fails.
An owner is collected by deleting everything under its prefix, and it can delete nothing else. That is what makes a borrowed snapshot safe: an actor created from a tag points at a URI under `tags/`, which its own prefix does not cover. See [Snapshot lifetime](#snapshot-lifetime).
An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot`, a `Tag` in `status.snapshot`, and an `ActorTemplate` its golden one in `status.goldenSnapshotStatus.goldenSnapshot` — each an `ExternalSnapshot` carrying `snapshotUri` and the `contentScope` it captured. The URI is recorded when the snapshot is written. All three are server-owned: do not send them on input, and parse a URI only against the scheme above.
An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot` and a `Tag` in `status.snapshot`, each an `ExternalSnapshot` carrying `snapshotUri` and `contentScope`. The URI is recorded when the snapshot is written. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above.
An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor lives in the reserved `ate-golden` atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `<atespace>` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like:
@@ -434,10 +434,12 @@ See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble
### The Golden Snapshot
When an `ActorTemplate` is created:
1. Substrate starts a temporary **Golden Pod**.
2. It executes your workload containers as defined in the template.
3. Once the process is initialized, gVisor takes a **Golden Snapshot** (Version 0).
4. The template enters the `Ready` phase.
1. Substrate creates and resumes a temporary golden actor in `ate-golden`.
2. It waits for readiness (or the warm-up interval), then suspends the actor.
3. It creates a published tag named after the template UID, copying the snapshot into tag-owned storage.
4. It deletes the golden actor and records the tag reference in the template status.
`CreateActor` uses an explicit `sourceTag` when supplied; otherwise it resolves the template's golden tag and records that snapshot on the new actor. If the golden tag is not ready yet, the actor starts without a snapshot and cold-boots even if the tag becomes ready before its first resume. The default does not populate the caller-owned `sourceTag` field. Deleting the template collects its golden tag and any unfinished golden actor.
### Resumption Lifecycle
Once a template is `Ready`, creating an actor logically (via `kubectl ate create actor`) allows it to be resumed instantly on any free worker in the referenced `WorkerPool`. Substrate bypasses the standard container boot and restores the process directly from its last saved state.
+1 -1
View File
@@ -1204,7 +1204,7 @@ wait_actortemplate_ready() {
while ((SECONDS < deadline)); do
if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then
snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenSnapshot.snapshotUri // empty' <<<"${json}")
snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}")
if [[ -n "${snapshot}" ]]; then
return 0
fi
+1 -1
View File
@@ -94,7 +94,7 @@ log_step "preflight"
# Ready means the template's golden snapshot exists; protojson omits empty
# fields, so the key is only present once it is set.
run_kubectl_ate get actor-template "${DEMO_POOL}" -a "${ATESPACE}" -o json 2>/dev/null \
| grep -q '"goldenSnapshot"' \
| grep -q '"goldenTag"' \
|| fail "actor template ${ATESPACE}/${DEMO_POOL} has no golden snapshot; install the counter demo first"
# Column 4 is STATUS; the header row's "ASSIGNED ACTOR" column name must not
# trip the check.
+1 -1
View File
@@ -162,7 +162,7 @@ func WaitForSubstrateTemplateReady(ctx context.Context, t *testing.T, clients *C
})
if err == nil {
lastStatus = at.GetStatus().GetGoldenSnapshotStatus()
if lastStatus.GetGoldenSnapshot().GetSnapshotUri() != "" {
if lastStatus.GetGoldenTag().GetName() != "" {
return
}
if msg := lastStatus.GetErrorMessage(); msg != "" {
+11 -11
View File
@@ -2271,12 +2271,11 @@ func (x *Limits) GetQuantity() string {
type GoldenSnapshotStatus struct {
state protoimpl.MessageState `protogen:"open.v1"`
// golden_snapshot is the external snapshot built for this version by
// ate-api, taken from the golden Actor in the reserved ate-golden system
// atespace. Set once state is READY. The golden Actor owns it.
//
// golden_tag owns the immutable snapshot built by the template controller.
// Set after tagging the snapshot and deleting the temporary golden Actor.
// +k8s:optional
GoldenSnapshot *ExternalSnapshot `protobuf:"bytes,1,opt,name=golden_snapshot,json=goldenSnapshot,proto3" json:"golden_snapshot,omitempty"`
// +k8s:subfield(atespace)=+k8s:required
GoldenTag *ObjectRef `protobuf:"bytes,1,opt,name=golden_tag,json=goldenTag,proto3" json:"golden_tag,omitempty"`
// take_golden_snapshot_at is when the golden-actor warmup ends and the
// golden snapshot may be taken.
TakeGoldenSnapshotAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=take_golden_snapshot_at,json=takeGoldenSnapshotAt,proto3" json:"take_golden_snapshot_at,omitempty"`
@@ -2315,9 +2314,9 @@ func (*GoldenSnapshotStatus) Descriptor() ([]byte, []int) {
return file_ateapi_proto_rawDescGZIP(), []int{21}
}
func (x *GoldenSnapshotStatus) GetGoldenSnapshot() *ExternalSnapshot {
func (x *GoldenSnapshotStatus) GetGoldenTag() *ObjectRef {
if x != nil {
return x.GoldenSnapshot
return x.GoldenTag
}
return nil
}
@@ -6961,9 +6960,10 @@ const file_ateapi_proto_rawDesc = "" +
"\x06limits\x18\x01 \x03(\v2\x0e.ateapi.LimitsR\x06limits\"8\n" +
"\x06Limits\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" +
"\bquantity\x18\x02 \x01(\tR\bquantity\"\xd1\x01\n" +
"\x14GoldenSnapshotStatus\x12A\n" +
"\x0fgolden_snapshot\x18\x01 \x01(\v2\x18.ateapi.ExternalSnapshotR\x0egoldenSnapshot\x12Q\n" +
"\bquantity\x18\x02 \x01(\tR\bquantity\"\xc0\x01\n" +
"\x14GoldenSnapshotStatus\x120\n" +
"\n" +
"golden_tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\tgoldenTag\x12Q\n" +
"\x17take_golden_snapshot_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x14takeGoldenSnapshotAt\x12#\n" +
"\rerror_message\x18\x03 \x01(\tR\ferrorMessage\"i\n" +
"\x13ActorTemplateStatus\x12R\n" +
@@ -7443,7 +7443,7 @@ var file_ateapi_proto_depIdxs = []int32{
28, // 37: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources
31, // 38: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus
29, // 39: ateapi.Resources.limits:type_name -> ateapi.Limits
9, // 40: ateapi.GoldenSnapshotStatus.golden_snapshot:type_name -> ateapi.ExternalSnapshot
26, // 40: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef
108, // 41: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp
30, // 42: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus
3, // 43: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass
+5 -6
View File
@@ -138,7 +138,7 @@ service Control {
rpc ListActorTemplates(ListActorTemplatesRequest) returns (ListActorTemplatesResponse) {}
// Delete an ActorTemplate together with its golden actor and golden
// snapshot in the ActorTemplate's namespace.
// tag in the reserved ate-golden atespace.
rpc DeleteActorTemplate(DeleteActorTemplateRequest) returns (ActorTemplate) {}
}
@@ -807,12 +807,11 @@ message Limits {
}
message GoldenSnapshotStatus {
// golden_snapshot is the external snapshot built for this version by
// ate-api, taken from the golden Actor in the reserved ate-golden system
// atespace. Set once state is READY. The golden Actor owns it.
//
// golden_tag owns the immutable snapshot built by the template controller.
// Set after tagging the snapshot and deleting the temporary golden Actor.
// +k8s:optional
ExternalSnapshot golden_snapshot = 1;
// +k8s:subfield(atespace)=+k8s:required
ObjectRef golden_tag = 1;
// take_golden_snapshot_at is when the golden-actor warmup ends and the
// golden snapshot may be taken.
+2 -2
View File
@@ -157,7 +157,7 @@ type ControlClient interface {
GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error)
ListActorTemplates(ctx context.Context, in *ListActorTemplatesRequest, opts ...grpc.CallOption) (*ListActorTemplatesResponse, error)
// Delete an ActorTemplate together with its golden actor and golden
// snapshot in the ActorTemplate's namespace.
// tag in the reserved ate-golden atespace.
DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error)
}
@@ -597,7 +597,7 @@ type ControlServer interface {
GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error)
ListActorTemplates(context.Context, *ListActorTemplatesRequest) (*ListActorTemplatesResponse, error)
// Delete an ActorTemplate together with its golden actor and golden
// snapshot in the ActorTemplate's namespace.
// tag in the reserved ate-golden atespace.
DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error)
mustEmbedUnimplementedControlServer()
}