mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-10-02 03:24:42 +08:00
Publish ActorTemplate golden snapshots as tags (#1523)
Fixes #1507 Golden snapshots currently remain owned by the temporary golden actor, so another resume/suspend cycle or actor deletion can collect a snapshot still referenced by its template. The controller now copies the warmed snapshot into a published tag, deletes the golden actor, and records the tag reference on the template. Interrupted tag creation and cleanup remain retryable; template deletion cleans up both resources. `CreateActor` resolves an explicit `sourceTag` or the template's golden tag into the actor's initial snapshot. Actors created before the golden tag is ready retain their cold-boot behavior. The golden tag uses the template UID as its name in `ate-golden`. The proto replaces `golden_snapshot` with `golden_tag` at field 1, without backward compatibility. This PR is based directly on `main` and does not depend on #1521. Follow-up recommendation: move the create → resume → wait → suspend → tag → delete sequence into a golden-template workflow using the existing workflow conventions. The reconciler now coordinates multiple recoverable steps; it could retain scheduling and retries while delegating that sequence to the workflow. This refactor is outside this PR. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR Validation: full `env -u NO_COLOR make verify` passed after rebasing onto `main`. After the final proto field-number change, bindings were regenerated and the control API unit/functional tests plus proto-format and Go-format checks passed. --------- Signed-off-by: Eitan Yarmush <eitan.yarmush@solo.io>
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -473,7 +473,7 @@ class ControlServicer:
|
||||
|
||||
def DeleteActorTemplate(self, request, context):
|
||||
"""Delete an ActorTemplate together with its golden actor and golden
|
||||
snapshot in the ActorTemplate's namespace.
|
||||
tag in the reserved ate-golden atespace.
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
|
||||
@@ -149,7 +149,7 @@ wait_actortemplate_ready() {
|
||||
|
||||
while ((SECONDS < deadline)); do
|
||||
if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then
|
||||
snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenSnapshot.snapshotUri // empty' <<<"${json}")
|
||||
snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}")
|
||||
if [[ -n "${snapshot}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -97,7 +97,7 @@ func WaitActorTemplateGolden(ctx context.Context, client *ateclient.Client, ref
|
||||
lastErr = err
|
||||
if err == nil {
|
||||
goldenStatus := template.GetStatus().GetGoldenSnapshotStatus()
|
||||
if goldenStatus.GetGoldenSnapshot().GetSnapshotUri() != "" {
|
||||
if goldenStatus.GetGoldenTag().GetName() != "" {
|
||||
return nil
|
||||
}
|
||||
if msg := goldenStatus.GetErrorMessage(); msg != "" {
|
||||
|
||||
@@ -88,14 +88,29 @@ func (s *ServiceImpl) CreateActor(ctx context.Context, inActor *ateapipb.Actor)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If a source tag is requested, resolve it to the external
|
||||
// snapshot the new Actor starts from.
|
||||
// Resolve the explicit tag, or freeze the template's current golden default.
|
||||
tagRef := inActor.GetSourceTag()
|
||||
if tagRef == nil {
|
||||
tagRef = template.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()
|
||||
} else {
|
||||
for _, volume := range template.GetVolumes() {
|
||||
if volume.GetExternalVolumeTemplate() != nil {
|
||||
// TODO: Permit cloning after CSI volume snapshots are supported.
|
||||
return nil, status.Error(codes.FailedPrecondition, "Tag cloning does not support ActorTemplates with external volumes")
|
||||
}
|
||||
}
|
||||
}
|
||||
var sourceTag *ateapipb.Tag
|
||||
if tagRef := inActor.GetSourceTag(); tagRef != nil {
|
||||
if tagRef != nil {
|
||||
sourceTag, err = s.resolveTagSource(ctx, inActor.GetMetadata().GetAtespace(), tagRef, template)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if inActor.GetSourceTag() == nil {
|
||||
if err := validateGoldenSnapshotScope(sourceTag.GetStatus().GetSnapshot()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
atespace := inActor.GetMetadata().GetAtespace()
|
||||
@@ -178,12 +193,6 @@ func (s *ServiceImpl) resolveTagSource(ctx context.Context, actorAtespace string
|
||||
if tag.GetStatus().GetActorTemplateUid() != template.GetMetadata().GetUid() {
|
||||
return nil, status.Errorf(codes.FailedPrecondition, "source Tag must be taken from an actor with ActorTemplate uid %q", tag.GetStatus().GetActorTemplateUid())
|
||||
}
|
||||
for _, volume := range template.GetVolumes() {
|
||||
if volume.GetExternalVolumeTemplate() != nil {
|
||||
// TODO: Permit cloning after CSI volume snapshots are supported.
|
||||
return nil, status.Error(codes.FailedPrecondition, "Tag cloning does not support ActorTemplates with external volumes")
|
||||
}
|
||||
}
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -157,6 +157,26 @@ func (s *RPCService) DeleteActorTemplate(ctx context.Context, req *ateapipb.Dele
|
||||
}
|
||||
|
||||
templateRef := resources.ActorTemplateRefFromObjectRef(req.GetActorTemplate())
|
||||
// Serialize cleanup against golden actor/tag creation by the reconciler.
|
||||
ctx, lease, err := acquireLease(ctx, s.impl, "lease:actortemplate:"+templateRef.Atespace+":"+templateRef.Name, "ActorTemplate "+templateRef.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer lease.Close()
|
||||
tmpl, err := s.impl.GetActorTemplate(ctx, templateRef)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
return nil, status.Errorf(codes.NotFound, "ActorTemplate %s not found", templateRef)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
goldenRef := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()}
|
||||
if _, err := s.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: goldenRef, AnyState: true}); err != nil && status.Code(err) != codes.NotFound {
|
||||
return nil, fmt.Errorf("while deleting golden actor: %w", err)
|
||||
}
|
||||
if _, err := s.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: goldenRef}); err != nil && status.Code(err) != codes.NotFound {
|
||||
return nil, fmt.Errorf("while deleting golden tag: %w", err)
|
||||
}
|
||||
deleted, err := s.impl.DeleteActorTemplate(ctx, templateRef)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store"
|
||||
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest"
|
||||
"github.com/agent-substrate/substrate/internal/resources"
|
||||
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
|
||||
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
|
||||
@@ -325,7 +326,7 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) {
|
||||
// Server-owned status a client must not be able to set.
|
||||
tmpl.Status = &ateapipb.ActorTemplateStatus{
|
||||
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://my-bucket/snapshots/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/sneaky"},
|
||||
GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
|
||||
},
|
||||
}
|
||||
})
|
||||
@@ -349,6 +350,125 @@ func TestCreateActorTemplateIgnoresServerOwnedFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteActorTemplate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
actorDeleted bool
|
||||
tagDeleted bool
|
||||
pendingTag bool
|
||||
// failPrefix makes object storage fail cleanup for this resource kind.
|
||||
failPrefix string
|
||||
wantActorAfterFailure bool
|
||||
}{
|
||||
{name: "golden actor and tag"},
|
||||
{name: "golden actor already deleted", actorDeleted: true},
|
||||
{name: "golden tag absent", tagDeleted: true},
|
||||
{name: "no golden resources", actorDeleted: true, tagDeleted: true},
|
||||
{name: "incomplete golden tag", pendingTag: true},
|
||||
{name: "actor cleanup failure", failPrefix: "/actors/", wantActorAfterFailure: true},
|
||||
{name: "tag cleanup failure", failPrefix: "/tags/"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
persistence := newTestPersistence(t)
|
||||
tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl")
|
||||
templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl)
|
||||
goldenRef := resources.ActorRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()}
|
||||
actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: goldenRef.Atespace, Name: goldenRef.Name},
|
||||
ActorTemplate: templateRef.ToObjectRef(),
|
||||
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
|
||||
})
|
||||
workflow, objects := newFinalizeWorkflow(persistence)
|
||||
actorURI := mustActorSnapshotURI(t, tmpl, actor, "snapshot")
|
||||
objects.PutSnapshot(t, actorURI, "manifest.json")
|
||||
actor = mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) {
|
||||
s.ExternalSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: actorURI.String(), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL}
|
||||
s.CurrentActorTemplateUid = tmpl.GetMetadata().GetUid()
|
||||
})
|
||||
var tag *ateapipb.Tag
|
||||
if tt.pendingTag {
|
||||
tag = storetest.MustCreateTag(t, ctx, persistence, newPendingTestTag(t, goldenRef.Name, actor))
|
||||
} else {
|
||||
var err error
|
||||
tag, err = workflow.TagActorSnapshot(ctx, tagToCreate(goldenRef, goldenRef.Name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
tagRef := resources.TagRefFromTag(tag)
|
||||
tagURI := mustReservedTagSnapshotURI(t, tag)
|
||||
objects.PutSnapshot(t, tagURI, "manifest.json")
|
||||
svc := &RPCService{impl: newServiceImpl(persistence, nil), actorWorkflow: workflow, objectStore: objects}
|
||||
// The handler must request AnyState to clean up an active golden actor.
|
||||
mustUpdateActorStatus(t, ctx, persistence, actor, func(s *ateapipb.ActorStatus) {
|
||||
s.State = ateapipb.ActorState_ACTOR_STATE_RUNNING
|
||||
})
|
||||
if tt.actorDeleted {
|
||||
if _, err := workflow.DeleteActor(ctx, goldenRef, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tt.tagDeleted {
|
||||
if _, err := svc.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: tagRef.ToObjectRef()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tt.failPrefix != "" {
|
||||
objects.OnDelete = func(_, key string) error {
|
||||
if strings.Contains(key, tt.failPrefix) {
|
||||
return errObjectStore
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
req := &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()}
|
||||
deleted, err := svc.DeleteActorTemplate(ctx, req)
|
||||
if tt.failPrefix != "" {
|
||||
if !errors.Is(err, errObjectStore) {
|
||||
t.Fatalf("DeleteActorTemplate = %v, want object storage error", err)
|
||||
}
|
||||
if _, err := persistence.GetActorTemplate(ctx, templateRef); err != nil {
|
||||
t.Fatalf("template lost after cleanup failure: %v", err)
|
||||
}
|
||||
if _, err := persistence.GetTag(ctx, tagRef); err != nil {
|
||||
t.Fatalf("tag lost after cleanup failure: %v", err)
|
||||
}
|
||||
_, actorErr := persistence.GetActor(ctx, goldenRef)
|
||||
if tt.wantActorAfterFailure && actorErr != nil || !tt.wantActorAfterFailure && !errors.Is(actorErr, store.ErrNotFound) {
|
||||
t.Fatalf("GetActor after failure = %v, want present %v", actorErr, tt.wantActorAfterFailure)
|
||||
}
|
||||
objects.OnDelete = nil
|
||||
deleted, err = svc.DeleteActorTemplate(ctx, req)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diff := cmp.Diff(tmpl, deleted, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("deleted template mismatch (-want +got):\n%s", diff)
|
||||
}
|
||||
if _, err := persistence.GetActorTemplate(ctx, templateRef); !errors.Is(err, store.ErrNotFound) {
|
||||
t.Fatalf("GetActorTemplate after delete = %v, want NotFound", err)
|
||||
}
|
||||
if _, err := persistence.GetActor(ctx, goldenRef); !errors.Is(err, store.ErrNotFound) {
|
||||
t.Fatalf("GetActor after delete = %v, want NotFound", err)
|
||||
}
|
||||
if _, err := persistence.GetTag(ctx, tagRef); !errors.Is(err, store.ErrNotFound) {
|
||||
t.Fatalf("GetTag after delete = %v, want NotFound", err)
|
||||
}
|
||||
for _, uri := range []resources.SnapshotURI{actorURI, tagURI} {
|
||||
if got := objects.Snapshot(t, uri); len(got) != 0 {
|
||||
t.Errorf("snapshot %s still holds %v", uri, got)
|
||||
}
|
||||
}
|
||||
if _, err := svc.DeleteActorTemplate(ctx, req); status.Code(err) != codes.NotFound {
|
||||
t.Fatalf("delete missing template = %v, want NotFound", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateGetActorTemplateRequest(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -1275,7 +1395,7 @@ func TestUpdateActorTemplateMetadata(t *testing.T) {
|
||||
// A server-owned status write passes validation and bumps the version.
|
||||
updated, err := persistence.UpdateActorTemplate(ctx, ref, store.PreconditionFrom(created), func(tmpl *ateapipb.ActorTemplate) error {
|
||||
tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1"},
|
||||
GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
|
||||
}}
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -1436,3 +1436,90 @@ func TestValidateSuspendActorRequest(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateActor_GoldenTagDefault(t *testing.T) {
|
||||
for _, scenario := range []string{"default", "explicit tag", "own snapshot", "missing", "pending", "wrong template", "data scope"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
persistence := newTestPersistence(t)
|
||||
storetest.MustCreateAtespace(t, ctx, persistence, "team-a")
|
||||
storetest.MustCreateAtespace(t, ctx, persistence, resources.GoldenActorAtespace)
|
||||
tmpl := seedSubstrateTemplate(t, ctx, persistence, "tmpl")
|
||||
ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "golden"}
|
||||
tag := &ateapipb.Tag{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name},
|
||||
SourceActor: ref,
|
||||
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
|
||||
Status: &ateapipb.TagStatus{
|
||||
ActorTemplateUid: tmpl.GetMetadata().GetUid(),
|
||||
Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/atespaces/ate-golden/tags/" + someActorUID, ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
|
||||
},
|
||||
}
|
||||
wantCode := codes.OK
|
||||
switch scenario {
|
||||
case "missing":
|
||||
wantCode = codes.NotFound
|
||||
case "pending":
|
||||
tag.Status.Snapshot = nil
|
||||
wantCode = codes.FailedPrecondition
|
||||
case "wrong template":
|
||||
tag.Status.ActorTemplateUid = "other"
|
||||
wantCode = codes.FailedPrecondition
|
||||
case "data scope":
|
||||
tag.Status.Snapshot.ContentScope = ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA
|
||||
wantCode = codes.FailedPrecondition
|
||||
}
|
||||
if scenario != "missing" {
|
||||
if _, err := persistence.CreateTag(ctx, tag); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := persistence.UpdateActorTemplate(ctx, resources.ActorTemplateRefFromActorTemplate(tmpl), store.PreconditionFrom(tmpl), func(db *ateapipb.ActorTemplate) error {
|
||||
db.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{GoldenTag: ref}}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
actor := &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor"}, ActorTemplate: resources.ActorTemplateRefFromActorTemplate(tmpl).ToObjectRef()}
|
||||
if scenario == "explicit tag" {
|
||||
tag.Metadata.Name = "explicit"
|
||||
tag.Status.Snapshot.SnapshotUri = "gs://bucket/atespaces/ate-golden/tags/explicit"
|
||||
if _, err := persistence.CreateTag(ctx, tag); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
actor.SourceTag = &ateapipb.ObjectRef{Atespace: ref.Atespace, Name: "explicit"}
|
||||
}
|
||||
svc := &ServiceImpl{store: persistence}
|
||||
created, err := svc.CreateActor(ctx, actor)
|
||||
if status.Code(err) != wantCode {
|
||||
t.Fatalf("CreateActor = %v, want %v", err, wantCode)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if got := created.GetStatus(); got.GetExternalSnapshot().GetSnapshotUri() != tag.GetStatus().GetSnapshot().GetSnapshotUri() || got.GetCurrentActorTemplateUid() != tmpl.GetMetadata().GetUid() {
|
||||
t.Fatalf("incorrect initial status: %v", got)
|
||||
}
|
||||
if scenario == "own snapshot" {
|
||||
uri, err := resources.NewActorSnapshotURI(tmpl.GetSnapshotsConfig().GetStorageLocation(), "team-a", created.GetMetadata().GetUid(), "snapshot")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := persistence.UpdateActor(ctx, resources.ActorRefFromActor(created), store.PreconditionFrom(created), func(db *ateapipb.Actor) error {
|
||||
db.Status.ExternalSnapshot.SnapshotUri = uri.String()
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
workflow := &ActorWorkflow{store: persistence}
|
||||
_, _, src, err := workflow.loadActorForResume(ctx, resources.ActorRefFromActor(created))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.SnapshotURI.IsZero() {
|
||||
t.Fatalf("missing snapshot source for %s", scenario)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,12 @@ package functionaltest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi"
|
||||
"github.com/agent-substrate/substrate/internal/resources"
|
||||
"google.golang.org/grpc/status"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
@@ -44,7 +49,7 @@ func TestActorTemplateCRUD(t *testing.T) {
|
||||
// Server-owned status on the request is ignored.
|
||||
Status: &ateapipb.ActorTemplateStatus{
|
||||
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://my-bucket/snapshots/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/sneaky"},
|
||||
GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -139,6 +144,98 @@ func TestActorTemplateCRUD(t *testing.T) {
|
||||
assertGrpcErrorRegex(t, err, codes.InvalidArgument, `sandbox_config\.config_name`)
|
||||
}
|
||||
|
||||
func TestGoldenTagLifecycle(t *testing.T) {
|
||||
ns := namespaceForTest("golden-tag")
|
||||
tc := setupTest(t, ns)
|
||||
defer tc.cleanup()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns})
|
||||
tmpl := createTemplateWithSelector(t, tc, "golden-template", &ateapipb.Selector{MatchLabels: map[string]string{poolLabelKey: ns}})
|
||||
workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1")
|
||||
templateRef := resources.ActorTemplateRefFromActorTemplate(tmpl)
|
||||
// Created before readiness: a later golden tag must not change its source.
|
||||
early, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "early"}, ActorTemplate: templateRef.ToObjectRef(),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
controlapi.NewActorTemplateReconciler(tc.persistence, tc.service).Start(ctx)
|
||||
var goldenRef *ateapipb.ObjectRef
|
||||
err = wait.PollUntilContextTimeout(ctx, 50*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (bool, error) {
|
||||
current, err := tc.client.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()})
|
||||
goldenRef = current.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()
|
||||
return goldenRef != nil, err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("waiting for golden tag: %v", err)
|
||||
}
|
||||
golden, err := tc.client.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenRef})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
uri := golden.GetStatus().GetSnapshot().GetSnapshotUri()
|
||||
parsed, err := resources.ParseSnapshotURI(uri)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !parsed.OwnedBy(resources.TagSnapshotOwner(resources.GoldenActorAtespace, parsed.Name())) {
|
||||
t.Fatal("golden snapshot is not tag-owned")
|
||||
}
|
||||
assertSnapshotPresent(t, tc, uri)
|
||||
_, err = tc.client.GetActor(ctx, &ateapipb.GetActorRequest{Actor: goldenRef})
|
||||
if status.Code(err) != codes.NotFound {
|
||||
t.Fatalf("golden actor was not deleted: %v", err)
|
||||
}
|
||||
late, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "late"}, ActorTemplate: templateRef.ToObjectRef(),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if late.GetStatus().GetExternalSnapshot().GetSnapshotUri() != uri || late.GetStatus().GetCurrentActorTemplateUid() != tmpl.GetMetadata().GetUid() {
|
||||
t.Fatal("actor did not inherit golden tag snapshot and template UID")
|
||||
}
|
||||
waitForWorkerAvailable(t, tc, workerName)
|
||||
tc.fakeAtelet.Lock.Lock()
|
||||
tc.fakeAtelet.RunCalled = false
|
||||
tc.fakeAtelet.Lock.Unlock()
|
||||
if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: resources.ActorRefFromActor(early).ToObjectRef()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !tc.fakeAtelet.RunCalled {
|
||||
t.Fatal("actor created before golden readiness did not cold boot")
|
||||
}
|
||||
if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: resources.ActorRefFromActor(early).ToObjectRef()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForWorkerAvailable(t, tc, workerName)
|
||||
if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: resources.ActorRefFromActor(late).ToObjectRef()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !tc.fakeAtelet.RestoreCalled {
|
||||
t.Fatal("actor did not restore golden tag")
|
||||
}
|
||||
if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: resources.ActorRefFromActor(late).ToObjectRef()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertSnapshotPresent(t, tc, uri)
|
||||
for _, actor := range []*ateapipb.Actor{early, late} {
|
||||
if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: resources.ActorRefFromActor(actor).ToObjectRef(), AnyState: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := tc.client.DeleteActorTemplate(ctx, &ateapipb.DeleteActorTemplateRequest{ActorTemplate: templateRef.ToObjectRef()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertSnapshotCollected(t, tc, uri)
|
||||
_, err = tc.client.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenRef})
|
||||
if status.Code(err) != codes.NotFound {
|
||||
t.Fatalf("golden tag was not deleted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListActorTemplates_InvalidPageToken(t *testing.T) {
|
||||
ns := namespaceForTest("ns-template-invalid-token")
|
||||
tc := setupTest(t, ns)
|
||||
|
||||
@@ -51,7 +51,7 @@ func TestCreateActor_Success(t *testing.T) {
|
||||
tc := setupTest(t, ns)
|
||||
defer tc.cleanup()
|
||||
|
||||
createTemplate(t, tc, ns)
|
||||
tmpl := createTemplate(t, tc, ns)
|
||||
|
||||
createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{
|
||||
@@ -67,9 +67,13 @@ func TestCreateActor_Success(t *testing.T) {
|
||||
}
|
||||
|
||||
want := &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1},
|
||||
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
|
||||
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
|
||||
Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1},
|
||||
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
|
||||
Status: &ateapipb.ActorStatus{
|
||||
State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
|
||||
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
|
||||
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
|
||||
},
|
||||
WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}},
|
||||
}
|
||||
|
||||
@@ -663,7 +667,7 @@ func TestUpdateActor_Success(t *testing.T) {
|
||||
tc := setupTest(t, ns)
|
||||
defer tc.cleanup()
|
||||
|
||||
createTemplate(t, tc, ns)
|
||||
tmpl := createTemplate(t, tc, ns)
|
||||
|
||||
toUpdate, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"},
|
||||
@@ -687,7 +691,11 @@ func TestUpdateActor_Success(t *testing.T) {
|
||||
wantActor := &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2},
|
||||
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
|
||||
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
|
||||
Status: &ateapipb.ActorStatus{
|
||||
State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
|
||||
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
|
||||
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
|
||||
},
|
||||
WorkerSelector: &ateapipb.Selector{
|
||||
MatchLabels: map[string]string{"tier": "paid"},
|
||||
},
|
||||
@@ -830,7 +838,11 @@ func TestUpdateActor(t *testing.T) {
|
||||
wantActor := &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2},
|
||||
ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
|
||||
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
|
||||
Status: &ateapipb.ActorStatus{
|
||||
State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
|
||||
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
|
||||
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
|
||||
},
|
||||
WorkerSelector: &ateapipb.Selector{
|
||||
MatchLabels: map[string]string{"tier": "paid"},
|
||||
},
|
||||
@@ -1800,6 +1812,7 @@ func TestResumeActor(t *testing.T) {
|
||||
Status: &ateapipb.ActorStatus{
|
||||
State: ateapipb.ActorState_ACTOR_STATE_RUNNING,
|
||||
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
|
||||
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
|
||||
WorkerAssignment: &ateapipb.WorkerAssignment{
|
||||
Worker: &ateapipb.ObjectRef{Name: podUID},
|
||||
WorkerNamespace: ns,
|
||||
@@ -2544,6 +2557,7 @@ func TestPauseActor(t *testing.T) {
|
||||
ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
|
||||
},
|
||||
CurrentActorTemplateUid: tmpl.GetMetadata().GetUid(),
|
||||
ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -341,15 +341,13 @@ func assertSnapshotCollected(t *testing.T, tc *testContext, snapshotURI string)
|
||||
}
|
||||
}
|
||||
|
||||
// goldenSnapshotURI is the golden snapshot the test templates record: the
|
||||
// golden Actor owns it under its own prefix in the reserved atespace, the way
|
||||
// the ActorTemplateReconciler's checkpoint would leave it.
|
||||
// goldenSnapshotURI is the snapshot owned by the test template's golden tag.
|
||||
func goldenSnapshotURI(t *testing.T) string {
|
||||
t.Helper()
|
||||
const goldenActorUID = "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94"
|
||||
uri, err := resources.NewActorSnapshotURI(testStorageLocation, resources.GoldenActorAtespace, goldenActorUID, "golden")
|
||||
const goldenSnapshotName = "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94"
|
||||
uri, err := resources.NewTagSnapshotURI(testStorageLocation, resources.GoldenActorAtespace, goldenSnapshotName)
|
||||
if err != nil {
|
||||
t.Fatalf("NewActorSnapshotURI: %v", err)
|
||||
t.Fatalf("NewTagSnapshotURI: %v", err)
|
||||
}
|
||||
return uri.String()
|
||||
}
|
||||
@@ -455,6 +453,21 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st
|
||||
t.Fatalf("failed to create actor template: %v", err)
|
||||
}
|
||||
|
||||
createAtespace(t, tc, resources.GoldenActorAtespace)
|
||||
tag, err := tc.persistence.CreateTag(context.Background(), &ateapipb.Tag{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()},
|
||||
SourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: created.GetMetadata().GetUid()},
|
||||
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
|
||||
Status: &ateapipb.TagStatus{
|
||||
Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
|
||||
ActorTemplateUid: created.GetMetadata().GetUid(),
|
||||
SourceActorUid: "9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create golden tag: %v", err)
|
||||
}
|
||||
|
||||
// Record the golden snapshot on the template's status directly in the
|
||||
// store, as the ActorTemplateReconciler's checkpoint would: there is no
|
||||
// status RPC, and the reconciler does not run in this test environment.
|
||||
@@ -463,7 +476,7 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st
|
||||
func(dbTemplate *ateapipb.ActorTemplate) error {
|
||||
dbTemplate.Status = &ateapipb.ActorTemplateStatus{
|
||||
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
|
||||
GoldenTag: resources.TagRefFromTag(tag).ToObjectRef(),
|
||||
},
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -26,7 +26,6 @@ import (
|
||||
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
@@ -46,6 +45,7 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
reasonGoldenTagConflict = "GoldenTagConflict"
|
||||
reasonGoldenActorInvalid = "GoldenActorInvalid"
|
||||
reasonGoldenActorCrashed = "GoldenActorCrashed"
|
||||
reasonUnexpectedState = "GoldenActorUnexpectedState"
|
||||
@@ -63,6 +63,10 @@ type templateReconcilerStore interface {
|
||||
// goldenActorControl is the in-process slice of the Control service the
|
||||
// reconciler drives golden actors through. *RPCService satisfies it.
|
||||
type goldenActorControl interface {
|
||||
GetTag(ctx context.Context, req *ateapipb.GetTagRequest) (*ateapipb.Tag, error)
|
||||
CreateTag(ctx context.Context, req *ateapipb.CreateTagRequest) (*ateapipb.Tag, error)
|
||||
DeleteTag(ctx context.Context, req *ateapipb.DeleteTagRequest) (*ateapipb.Tag, error)
|
||||
DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error)
|
||||
CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error)
|
||||
CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error)
|
||||
GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)
|
||||
@@ -201,11 +205,29 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
|
||||
// The snapshot has already failed.
|
||||
return 0, nil
|
||||
}
|
||||
if goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri() != "" {
|
||||
if goldenSnapshotStatus.GetGoldenTag() != nil {
|
||||
// The golden snapshot exists already.
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// A completed tag survives a crash during actor deletion or checkpointing.
|
||||
tag, err := r.control.GetTag(ctx, &ateapipb.GetTagRequest{Tag: goldenActorRef})
|
||||
if err != nil && status.Code(err) != codes.NotFound {
|
||||
return 0, fmt.Errorf("while getting golden tag: %w", err)
|
||||
}
|
||||
if err == nil {
|
||||
if tag.GetStatus().GetActorTemplateUid() != tmpl.GetMetadata().GetUid() || resources.ActorRefFromObjectRef(tag.GetSourceActor()) != resources.ActorRefFromObjectRef(goldenActorRef) {
|
||||
return 0, r.fail(ctx, tmpl, reasonGoldenTagConflict, "golden tag belongs to another actor or template")
|
||||
}
|
||||
if tag.GetStatus().GetSnapshot().GetSnapshotUri() != "" {
|
||||
return 0, r.saveGoldenTag(ctx, tmpl, goldenActorRef)
|
||||
}
|
||||
// CreateTag cannot resume an incomplete copy. Delete it before retrying.
|
||||
if _, err := r.control.DeleteTag(ctx, &ateapipb.DeleteTagRequest{Tag: goldenActorRef}); err != nil && status.Code(err) != codes.NotFound {
|
||||
return 0, fmt.Errorf("while deleting incomplete golden tag: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
actor, err := r.ensureActorExists(ctx, tmpl, goldenActorRef)
|
||||
if err != nil {
|
||||
if status.Code(err) == codes.InvalidArgument {
|
||||
@@ -239,19 +261,19 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
|
||||
return rem, nil
|
||||
}
|
||||
// Warmup done: suspend the golden actor and record its snapshot.
|
||||
snapshot, err := r.suspendActor(ctx, goldenActorRef)
|
||||
err := r.suspendActor(ctx, goldenActorRef)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 0, r.saveGoldenSnapshot(ctx, tmpl, snapshot)
|
||||
return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef)
|
||||
|
||||
case ateapipb.ActorState_ACTOR_STATE_SUSPENDING:
|
||||
// A previous pass died mid-suspend; retry suspend.
|
||||
snapshot, err := r.suspendActor(ctx, goldenActorRef)
|
||||
err := r.suspendActor(ctx, goldenActorRef)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 0, r.saveGoldenSnapshot(ctx, tmpl, snapshot)
|
||||
return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef)
|
||||
|
||||
case ateapipb.ActorState_ACTOR_STATE_RESUMING,
|
||||
ateapipb.ActorState_ACTOR_STATE_SUSPENDED:
|
||||
@@ -261,7 +283,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
|
||||
// Golden actors never start from a source snapshot, so an
|
||||
// existing snapshot means an earlier suspend completed
|
||||
// without being recorded.
|
||||
return 0, r.saveGoldenSnapshot(ctx, tmpl, actor.GetStatus().GetExternalSnapshot())
|
||||
return 0, r.tagGoldenActor(ctx, tmpl, goldenActorRef)
|
||||
}
|
||||
if _, err := r.control.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: goldenActorRef}); err != nil {
|
||||
// A crash during resume is observed as CRASHED on the retry.
|
||||
@@ -284,29 +306,42 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
|
||||
}
|
||||
}
|
||||
|
||||
// suspendActor suspends the golden actor and returns the external snapshot it
|
||||
// wrote. Reentrant: SuspendActor completes an in-flight suspend and is a no-op
|
||||
// on an already-suspended actor, returning the existing snapshot either way.
|
||||
func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef) (*ateapipb.ExternalSnapshot, error) {
|
||||
// suspendActor waits for the golden actor to produce an external snapshot.
|
||||
// SuspendActor completes an in-flight suspend and is a no-op if already suspended.
|
||||
func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef) error {
|
||||
resp, err := r.control.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: goldenRef})
|
||||
if err != nil {
|
||||
// A crash during suspend is observed as CRASHED on the retry.
|
||||
return nil, fmt.Errorf("while suspending golden actor: %w", err)
|
||||
return fmt.Errorf("while suspending golden actor: %w", err)
|
||||
}
|
||||
suspended := resp.GetActor().GetStatus().GetExternalSnapshot()
|
||||
if suspended.GetSnapshotUri() == "" {
|
||||
return nil, fmt.Errorf("suspending golden actor produced no external snapshot")
|
||||
return fmt.Errorf("suspending golden actor produced no external snapshot")
|
||||
}
|
||||
return suspended, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveGoldenSnapshot records the golden actor's external snapshot, the
|
||||
// terminal success state that marks the template ready for use, ending the
|
||||
// reconcile pass. The golden actor keeps owning those objects; the template
|
||||
// only points at them.
|
||||
func (r *ActorTemplateReconciler) saveGoldenSnapshot(ctx context.Context, observed *ateapipb.ActorTemplate, golden *ateapipb.ExternalSnapshot) error {
|
||||
_, err := r.checkpoint(ctx, observed, func(snapshotStatus *ateapipb.GoldenSnapshotStatus) {
|
||||
snapshotStatus.GoldenSnapshot = proto.CloneOf(golden)
|
||||
// tagGoldenActor copies the snapshot into a tag before releasing the actor's copy.
|
||||
func (r *ActorTemplateReconciler) tagGoldenActor(ctx context.Context, tmpl *ateapipb.ActorTemplate, ref *ateapipb.ObjectRef) error {
|
||||
_, err := r.control.CreateTag(ctx, &ateapipb.CreateTagRequest{Tag: &ateapipb.Tag{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: ref.GetAtespace(), Name: ref.GetName()},
|
||||
SourceActor: ref,
|
||||
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
|
||||
}})
|
||||
if err != nil {
|
||||
return fmt.Errorf("while creating golden tag: %w", err)
|
||||
}
|
||||
return r.saveGoldenTag(ctx, tmpl, ref)
|
||||
}
|
||||
|
||||
// saveGoldenTag finishes cleanup before recording terminal success, so retries
|
||||
// can rediscover the tag even if deletion or the status write fails.
|
||||
func (r *ActorTemplateReconciler) saveGoldenTag(ctx context.Context, tmpl *ateapipb.ActorTemplate, ref *ateapipb.ObjectRef) error {
|
||||
if _, err := r.control.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: ref}); err != nil && status.Code(err) != codes.NotFound {
|
||||
return fmt.Errorf("while deleting golden actor: %w", err)
|
||||
}
|
||||
_, err := r.checkpoint(ctx, tmpl, func(snapshotStatus *ateapipb.GoldenSnapshotStatus) {
|
||||
snapshotStatus.GoldenTag = ref
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -346,7 +381,7 @@ func (r *ActorTemplateReconciler) fail(ctx context.Context, observed *ateapipb.A
|
||||
// goldenSnapshotDone reports whether the golden snapshot build reached a
|
||||
// terminal state: the snapshot was recorded, or the build failed.
|
||||
func goldenSnapshotDone(snapshotStatus *ateapipb.GoldenSnapshotStatus) bool {
|
||||
return snapshotStatus.GetGoldenSnapshot().GetSnapshotUri() != "" || snapshotStatus.GetErrorMessage() != ""
|
||||
return snapshotStatus.GetGoldenTag() != nil || snapshotStatus.GetErrorMessage() != ""
|
||||
}
|
||||
|
||||
// goldenSnapshotWarmupFor returns 0 when every container has a readyz probe
|
||||
|
||||
@@ -143,7 +143,13 @@ func (s *fakeTemplateStore) storedStatus(t *testing.T, ref resources.ActorTempla
|
||||
// from that observation. Tests seed mid-lifecycle states via exists /
|
||||
// goldenState / goldenSnapshot.
|
||||
type fakeGoldenControl struct {
|
||||
mu sync.Mutex
|
||||
mu sync.Mutex
|
||||
tag *ateapipb.Tag
|
||||
tagErr error
|
||||
deleteErr error
|
||||
deleteTagErr error
|
||||
tagReqs []*ateapipb.CreateTagRequest
|
||||
deleteReqs []*ateapipb.DeleteActorRequest
|
||||
|
||||
createErr error
|
||||
resumeErr error
|
||||
@@ -235,6 +241,52 @@ func (c *fakeGoldenControl) SuspendActor(_ context.Context, req *ateapipb.Suspen
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *fakeGoldenControl) GetTag(_ context.Context, _ *ateapipb.GetTagRequest) (*ateapipb.Tag, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.tag == nil {
|
||||
return nil, status.Error(codes.NotFound, "no tag")
|
||||
}
|
||||
return proto.CloneOf(c.tag), nil
|
||||
}
|
||||
|
||||
func (c *fakeGoldenControl) CreateTag(_ context.Context, req *ateapipb.CreateTagRequest) (*ateapipb.Tag, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.tagReqs = append(c.tagReqs, req)
|
||||
if c.tagErr != nil {
|
||||
return nil, c.tagErr
|
||||
}
|
||||
c.tag = proto.CloneOf(req.GetTag())
|
||||
c.tag.Status = &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: c.goldenSnapshot}}
|
||||
return proto.CloneOf(c.tag), nil
|
||||
}
|
||||
|
||||
func (c *fakeGoldenControl) DeleteTag(_ context.Context, _ *ateapipb.DeleteTagRequest) (*ateapipb.Tag, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.deleteTagErr != nil {
|
||||
return nil, c.deleteTagErr
|
||||
}
|
||||
tag := c.tag
|
||||
c.tag = nil
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
func (c *fakeGoldenControl) DeleteActor(_ context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.deleteReqs = append(c.deleteReqs, req)
|
||||
if c.deleteErr != nil {
|
||||
return nil, c.deleteErr
|
||||
}
|
||||
if !c.exists {
|
||||
return nil, status.Error(codes.NotFound, "no actor")
|
||||
}
|
||||
c.exists = false
|
||||
return &ateapipb.Actor{}, nil
|
||||
}
|
||||
|
||||
func (c *fakeGoldenControl) callCounts() (creates, resumes, suspends int) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
@@ -286,9 +338,9 @@ func withSnapshotDeadline(at time.Time) func(*ateapipb.ActorTemplate) {
|
||||
}
|
||||
}
|
||||
|
||||
func withGoldenSnapshot(snapshotURI string) func(*ateapipb.ActorTemplate) {
|
||||
func withGoldenTag() func(*ateapipb.ActorTemplate) {
|
||||
return func(tmpl *ateapipb.ActorTemplate) {
|
||||
seededGoldenStatus(tmpl).GoldenSnapshot = &ateapipb.ExternalSnapshot{SnapshotUri: snapshotURI}
|
||||
seededGoldenStatus(tmpl).GoldenTag = &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -340,9 +392,8 @@ func TestReconcileOne(t *testing.T) {
|
||||
// stored error message must be empty. Checked when template is seeded.
|
||||
wantFailedReason string
|
||||
wantMessage string
|
||||
// wantSnapshot must equal the stored golden snapshot URI; empty means
|
||||
// the snapshot must not be recorded.
|
||||
wantSnapshot string
|
||||
// wantTag indicates that the golden tag should be recorded.
|
||||
wantTag bool
|
||||
// wantDeadline asserts whether take_golden_snapshot_at is set.
|
||||
wantDeadline bool
|
||||
wantCreates int
|
||||
@@ -353,7 +404,7 @@ func TestReconcileOne(t *testing.T) {
|
||||
name: "happy path creates, resumes, and snapshots the golden actor",
|
||||
template: testTemplate(),
|
||||
control: &fakeGoldenControl{snapshot: goldenSnapshot},
|
||||
wantSnapshot: goldenSnapshot,
|
||||
wantTag: true,
|
||||
wantDeadline: true,
|
||||
wantCreates: 1,
|
||||
wantResumes: 1,
|
||||
@@ -382,7 +433,7 @@ func TestReconcileOne(t *testing.T) {
|
||||
template: testTemplate(
|
||||
withSnapshotDeadline(time.Now().Add(-time.Minute))),
|
||||
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_RUNNING, snapshot: goldenSnapshot},
|
||||
wantSnapshot: goldenSnapshot,
|
||||
wantTag: true,
|
||||
wantSuspends: 1,
|
||||
},
|
||||
{
|
||||
@@ -405,14 +456,14 @@ func TestReconcileOne(t *testing.T) {
|
||||
name: "suspending golden actor is completed and recorded",
|
||||
template: testTemplate(),
|
||||
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, snapshot: goldenSnapshot},
|
||||
wantSnapshot: goldenSnapshot,
|
||||
wantTag: true,
|
||||
wantSuspends: 1,
|
||||
},
|
||||
{
|
||||
name: "suspended golden actor with a snapshot is recorded without more control calls",
|
||||
template: testTemplate(),
|
||||
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: goldenSnapshot},
|
||||
wantSnapshot: goldenSnapshot,
|
||||
name: "suspended golden actor with a snapshot is recorded without more control calls",
|
||||
template: testTemplate(),
|
||||
control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: goldenSnapshot},
|
||||
wantTag: true,
|
||||
},
|
||||
{
|
||||
name: "create AlreadyExists requeues for the retry to observe",
|
||||
@@ -488,10 +539,10 @@ func TestReconcileOne(t *testing.T) {
|
||||
control: &fakeGoldenControl{},
|
||||
},
|
||||
{
|
||||
name: "terminal golden snapshot is a noop",
|
||||
template: testTemplate(withGoldenSnapshot(goldenSnapshot)),
|
||||
control: &fakeGoldenControl{},
|
||||
wantSnapshot: goldenSnapshot,
|
||||
name: "terminal golden snapshot is a noop",
|
||||
template: testTemplate(withGoldenTag()),
|
||||
control: &fakeGoldenControl{},
|
||||
wantTag: true,
|
||||
},
|
||||
{
|
||||
name: "terminal error message is a noop",
|
||||
@@ -538,8 +589,8 @@ func TestReconcileOne(t *testing.T) {
|
||||
t.Errorf("stored error message = %q, want it to contain %q", errorMessage, tt.wantMessage)
|
||||
}
|
||||
}
|
||||
if got := snapshotStatus.GetGoldenSnapshot().GetSnapshotUri(); got != tt.wantSnapshot {
|
||||
t.Errorf("stored golden snapshot uri = %q, want %q", got, tt.wantSnapshot)
|
||||
if got := snapshotStatus.GetGoldenTag(); (got != nil) != tt.wantTag {
|
||||
t.Errorf("stored golden tag = %v, want tag %v", got, tt.wantTag)
|
||||
}
|
||||
if tt.wantDeadline && snapshotStatus.GetTakeGoldenSnapshotAt() == nil {
|
||||
t.Error("stored take_golden_snapshot_at is nil, want set")
|
||||
@@ -591,13 +642,12 @@ func TestReconcileOne_GoldenActorRequests(t *testing.T) {
|
||||
|
||||
func TestCheckpoint_TerminalStateErrors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1"
|
||||
|
||||
for _, seed := range []struct {
|
||||
name string
|
||||
opt func(*ateapipb.ActorTemplate)
|
||||
}{
|
||||
{"golden snapshot taken", withGoldenSnapshot(goldenSnapshot)},
|
||||
{"golden snapshot taken", withGoldenTag()},
|
||||
{"failed", withFailed(reasonGoldenActorCrashed)},
|
||||
} {
|
||||
t.Run(seed.name, func(t *testing.T) {
|
||||
@@ -658,7 +708,6 @@ func drainQueue(r *ActorTemplateReconciler) []resources.ActorTemplateRef {
|
||||
}
|
||||
|
||||
func TestResync_QueuesOnlyActionableTemplates(t *testing.T) {
|
||||
goldenSnapshot := "gs://bucket/root/atespaces/ate-golden/actors/" + someActorUID + "/snapshots/snap-1"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -667,7 +716,7 @@ func TestResync_QueuesOnlyActionableTemplates(t *testing.T) {
|
||||
}{
|
||||
{"empty status", nil, true},
|
||||
{"mid warmup", []func(*ateapipb.ActorTemplate){withSnapshotDeadline(time.Now().Add(time.Hour))}, true},
|
||||
{"golden snapshot taken", []func(*ateapipb.ActorTemplate){withGoldenSnapshot(goldenSnapshot)}, false},
|
||||
{"golden snapshot taken", []func(*ateapipb.ActorTemplate){withGoldenTag()}, false},
|
||||
{"failed", []func(*ateapipb.ActorTemplate){withFailed(reasonGoldenActorCrashed)}, false},
|
||||
}
|
||||
|
||||
@@ -714,3 +763,125 @@ func TestResync_FollowsPagination(t *testing.T) {
|
||||
t.Errorf("queued %d templates, want 3 (all pages walked)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileOne_GoldenTagRecovery(t *testing.T) {
|
||||
ref := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}
|
||||
completed := &ateapipb.Tag{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name},
|
||||
SourceActor: ref,
|
||||
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
|
||||
Status: &ateapipb.TagStatus{ActorTemplateUid: testTemplateUID, Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/tag-snapshot"}},
|
||||
}
|
||||
incomplete := proto.CloneOf(completed)
|
||||
incomplete.Status.Snapshot = nil
|
||||
tests := []struct {
|
||||
name string
|
||||
// tag is the golden tag an earlier pass left behind, if any.
|
||||
tag *ateapipb.Tag
|
||||
// actorDeleted seeds a pass that died after deleting the golden actor.
|
||||
actorDeleted bool
|
||||
// These errors fail one step of the first pass; the retry succeeds.
|
||||
createTagErr error
|
||||
deleteActorErr error
|
||||
deleteTagErr error
|
||||
// wantCreateTags counts copy attempts across both passes.
|
||||
wantCreateTags int
|
||||
}{
|
||||
{name: "completed tag", tag: completed, wantCreateTags: 0},
|
||||
{name: "actor already deleted", tag: completed, actorDeleted: true, wantCreateTags: 0},
|
||||
{name: "incomplete tag", tag: incomplete, wantCreateTags: 1},
|
||||
{name: "copy failure", createTagErr: errors.New("copy interrupted"), wantCreateTags: 2},
|
||||
{name: "actor deletion failure", deleteActorErr: errors.New("storage unavailable"), wantCreateTags: 1},
|
||||
{name: "tag deletion failure", tag: incomplete, deleteTagErr: errors.New("storage unavailable"), wantCreateTags: 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
control := &fakeGoldenControl{
|
||||
tag: proto.CloneOf(tt.tag), exists: !tt.actorDeleted,
|
||||
goldenState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED, goldenSnapshot: "gs://bucket/actor-snapshot",
|
||||
tagErr: tt.createTagErr, deleteErr: tt.deleteActorErr, deleteTagErr: tt.deleteTagErr,
|
||||
}
|
||||
st := newFakeTemplateStore(testTemplate())
|
||||
r := newTestTemplateReconciler(st, control)
|
||||
defer r.queue.ShutDown()
|
||||
_, err := r.reconcileOne(t.Context(), testTemplateRef)
|
||||
wantErr := tt.createTagErr != nil || tt.deleteActorErr != nil || tt.deleteTagErr != nil
|
||||
if (err != nil) != wantErr {
|
||||
t.Fatalf("reconcile = %v, want error %v", err, wantErr)
|
||||
}
|
||||
if wantErr {
|
||||
if st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag() != nil {
|
||||
t.Fatal("marked ready before cleanup completed")
|
||||
}
|
||||
if !control.exists {
|
||||
t.Fatal("deleted actor after tag failure")
|
||||
}
|
||||
control.tagErr, control.deleteErr, control.deleteTagErr = nil, nil, nil
|
||||
if _, err := r.reconcileOne(t.Context(), testTemplateRef); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if control.exists {
|
||||
t.Fatal("golden actor still exists")
|
||||
}
|
||||
if !proto.Equal(st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus().GetGoldenTag(), ref) {
|
||||
t.Fatal("golden tag not recorded")
|
||||
}
|
||||
if control.tag.GetStatus().GetSnapshot().GetSnapshotUri() == "" {
|
||||
t.Fatal("golden tag has no snapshot")
|
||||
}
|
||||
if len(control.createReqs) != 0 || len(control.resumeReqs) != 0 || len(control.suspendReqs) != 0 {
|
||||
t.Fatal("repeated golden actor warmup")
|
||||
}
|
||||
if got := len(control.tagReqs); got != tt.wantCreateTags {
|
||||
t.Fatalf("CreateTag calls = %d, want %d", got, tt.wantCreateTags)
|
||||
}
|
||||
for _, req := range control.tagReqs {
|
||||
if !proto.Equal(req.Tag.SourceActor, ref) || req.Tag.Scope != ateapipb.TagScope_TAG_SCOPE_PUBLISHED || req.Tag.Metadata.Name != ref.Name {
|
||||
t.Fatalf("incorrect golden tag request: %v", req)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileOne_GoldenTagConflict(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
templateUID string
|
||||
sourceActor *ateapipb.ObjectRef
|
||||
}{
|
||||
{name: "template", templateUID: "another-template", sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: testTemplateUID}},
|
||||
{name: "actor name", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: "another-actor"}},
|
||||
{name: "actor atespace", templateUID: testTemplateUID, sourceActor: &ateapipb.ObjectRef{Atespace: "another-atespace", Name: testTemplateUID}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tag := &ateapipb.Tag{
|
||||
SourceActor: tt.sourceActor,
|
||||
Status: &ateapipb.TagStatus{ActorTemplateUid: tt.templateUID},
|
||||
}
|
||||
control := &fakeGoldenControl{tag: proto.CloneOf(tag), exists: true}
|
||||
st := newFakeTemplateStore(testTemplate())
|
||||
r := newTestTemplateReconciler(st, control)
|
||||
defer r.queue.ShutDown()
|
||||
for range 2 {
|
||||
after, err := r.reconcileOne(t.Context(), testTemplateRef)
|
||||
if err != nil || after != 0 {
|
||||
t.Fatalf("reconcile = (%v, %v), want terminal failure without retry", after, err)
|
||||
}
|
||||
snapshotStatus := st.storedStatus(t, testTemplateRef).GetGoldenSnapshotStatus()
|
||||
if snapshotStatus.GetErrorMessage() != reasonGoldenTagConflict+": golden tag belongs to another actor or template" || snapshotStatus.GetGoldenTag() != nil {
|
||||
t.Fatalf("unexpected golden snapshot status: %v", snapshotStatus)
|
||||
}
|
||||
if !proto.Equal(control.tag, tag) || !control.exists || len(control.tagReqs) != 0 || len(control.deleteReqs) != 0 {
|
||||
t.Fatal("modified golden resources after ownership conflict")
|
||||
}
|
||||
}
|
||||
r.resync(t.Context())
|
||||
if r.queue.Len() != 0 {
|
||||
t.Fatal("resync queued a terminally failed template")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1314,19 +1314,6 @@ func TestValidateNestedExternalSnapshot(t *testing.T) {
|
||||
return Validate_Tag(ctx, op, nil, obj, nil)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "actor_template.status.golden_snapshot_status.golden_snapshot",
|
||||
path: field.NewPath("golden_snapshot_status", "golden_snapshot"),
|
||||
validate: func(ctx context.Context) field.ErrorList {
|
||||
op := operation.Operation{Type: operation.Create}
|
||||
obj := &ateapipb.ActorTemplateStatus{
|
||||
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: badExternalSnapshot(),
|
||||
},
|
||||
}
|
||||
return Validate_ActorTemplateStatus(ctx, op, nil, obj, nil)
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -38,9 +38,9 @@ import (
|
||||
// and passed by value to the restore step — never mutated after resolution.
|
||||
type resumeSnapshotSource struct {
|
||||
// SnapshotURI is the storage location of the durable snapshot to restore
|
||||
// from: the actor's own latest snapshot when one exists, the template's
|
||||
// golden snapshot otherwise. Zero means cold boot from the spec (unless
|
||||
// the actor holds a local snapshot, which takes precedence at restore).
|
||||
// from: the actor's latest snapshot, including a tag borrowed at creation.
|
||||
// Zero means cold boot from the spec (unless the actor holds a local
|
||||
// snapshot, which takes precedence at restore).
|
||||
SnapshotURI resources.SnapshotURI
|
||||
Scope ateapipb.SnapshotContentScope
|
||||
// GoldenSnapshotURI is the storage location of the ActorTemplate's golden
|
||||
@@ -179,7 +179,6 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc
|
||||
if err != nil {
|
||||
return nil, nil, src, err
|
||||
}
|
||||
goldenSnapshotStatus := actorTemplate.GetStatus().GetGoldenSnapshotStatus()
|
||||
if uri := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); uri != "" {
|
||||
if src.SnapshotURI, err = resources.ParseSnapshotURI(uri); err != nil {
|
||||
return nil, nil, src, status.Errorf(codes.DataLoss, "Actor %s external snapshot: %v", actorRef, err)
|
||||
@@ -192,14 +191,6 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc
|
||||
// as well; it is already disallowed at admission time.
|
||||
builtOnTemplateUID := actor.GetStatus().GetCurrentActorTemplateUid()
|
||||
src.TemplateReplaced = builtOnTemplateUID != "" && builtOnTemplateUID != actorTemplate.GetMetadata().GetUid()
|
||||
} else if goldenURI := goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri(); goldenURI != "" {
|
||||
if err := validateGoldenSnapshotScope(goldenSnapshotStatus.GetGoldenSnapshot()); err != nil {
|
||||
return nil, nil, src, err
|
||||
}
|
||||
if src.SnapshotURI, err = resources.ParseSnapshotURI(goldenURI); err != nil {
|
||||
return nil, nil, src, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", goldenURI, err)
|
||||
}
|
||||
src.Scope = goldenSnapshotStatus.GetGoldenSnapshot().GetContentScope()
|
||||
}
|
||||
|
||||
// The template's onResume configuration selects the boot source for the
|
||||
@@ -218,13 +209,25 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc
|
||||
dataOnly = src.Scope == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA
|
||||
}
|
||||
if dataOnly {
|
||||
goldenURI := goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri()
|
||||
if goldenURI == "" {
|
||||
return nil, nil, src, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden snapshot, which is not available")
|
||||
ref := actorTemplate.GetStatus().GetGoldenSnapshotStatus().GetGoldenTag()
|
||||
if ref == nil {
|
||||
return nil, nil, src, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden tag, which is not available")
|
||||
}
|
||||
if err := validateGoldenSnapshotScope(goldenSnapshotStatus.GetGoldenSnapshot()); err != nil {
|
||||
tag, err := w.store.GetTag(ctx, resources.TagRefFromObjectRef(ref))
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
return nil, nil, src, status.Error(codes.FailedPrecondition, "ActorTemplate golden tag is not available")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, src, fmt.Errorf("while getting golden tag: %w", err)
|
||||
}
|
||||
golden := tag.GetStatus().GetSnapshot()
|
||||
if golden.GetSnapshotUri() == "" || tag.GetStatus().GetActorTemplateUid() != actorTemplate.GetMetadata().GetUid() {
|
||||
return nil, nil, src, status.Error(codes.FailedPrecondition, "ActorTemplate golden tag is incomplete or belongs to another template")
|
||||
}
|
||||
if err := validateGoldenSnapshotScope(golden); err != nil {
|
||||
return nil, nil, src, err
|
||||
}
|
||||
goldenURI := golden.GetSnapshotUri()
|
||||
if src.GoldenSnapshotURI, err = resources.ParseSnapshotURI(goldenURI); err != nil {
|
||||
return nil, nil, src, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", goldenURI, err)
|
||||
}
|
||||
@@ -753,7 +756,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou
|
||||
_, err = client.Restore(ctx, req)
|
||||
return tele, maybeCrashActor(ctx, w.store, actorRef, err, "while restoring durable snapshot", ateattr.OperationResume)
|
||||
} else {
|
||||
slog.InfoContext(ctx, "Actor has no snapshot; ActorTemplate has no golden snapshot; Booting from ActorTemplate spec")
|
||||
slog.InfoContext(ctx, "Actor has no snapshot; Booting from ActorTemplate spec")
|
||||
tele.SnapshotKind = ateattr.SnapshotKindBoot
|
||||
|
||||
// Booting from scratch: resolve the sandbox binaries from the
|
||||
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -1028,12 +1027,27 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) {
|
||||
}
|
||||
if tt.goldenURI != "" {
|
||||
tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.goldenURI, ContentScope: tt.goldenScope},
|
||||
GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
|
||||
}}
|
||||
}
|
||||
if _, err := persistence.CreateActorTemplate(ctx, tmpl); err != nil {
|
||||
stored, err := persistence.CreateActorTemplate(ctx, tmpl)
|
||||
if err != nil {
|
||||
t.Fatalf("create template: %v", err)
|
||||
}
|
||||
if tt.goldenURI != "" {
|
||||
_, err := persistence.CreateTag(ctx, &ateapipb.Tag{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "golden"},
|
||||
SourceActor: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
|
||||
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
|
||||
Status: &ateapipb.TagStatus{
|
||||
ActorTemplateUid: stored.GetMetadata().GetUid(),
|
||||
Snapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.goldenURI, ContentScope: tt.goldenScope},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create golden tag: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
w := &ActorWorkflow{store: persistence}
|
||||
_, _, src, err := w.loadActorForResume(ctx, actorRef)
|
||||
@@ -1053,41 +1067,25 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadActorForResume_GoldenFallbackRejectsNonFullGolden covers the
|
||||
// golden-fallback branch (actor with no snapshot of its own): a golden
|
||||
// snapshot recorded with a non-Full scope holds no guest state, so the resume
|
||||
// must fail with a clear error instead of forwarding its scope to atelet
|
||||
// with no golden location (which atelet rejects with a confusing
|
||||
// "missing bucket" validation error).
|
||||
func TestLoadActorForResume_GoldenFallbackRejectsNonFullGolden(t *testing.T) {
|
||||
// A golden tag becoming ready after creation does not change an actor's source.
|
||||
func TestLoadActorForResume_DoesNotDefaultGolden(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
persistence := newTestPersistence(t)
|
||||
actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"}
|
||||
|
||||
seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED)
|
||||
|
||||
storetest.MustCreateAtespace(t, ctx, persistence, "ns")
|
||||
if _, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"},
|
||||
Status: &ateapipb.ActorTemplateStatus{
|
||||
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: &ateapipb.ExternalSnapshot{
|
||||
SnapshotUri: someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1"),
|
||||
ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
|
||||
},
|
||||
},
|
||||
},
|
||||
Status: &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("create template: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
w := &ActorWorkflow{store: persistence}
|
||||
_, _, _, err := w.loadActorForResume(ctx, actorRef)
|
||||
if got := status.Code(err); got != codes.FailedPrecondition {
|
||||
t.Fatalf("status.Code(err) = %v, want FailedPrecondition (err: %v)", got, err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "regenerate the golden snapshot") {
|
||||
t.Errorf("error %q does not tell the operator to regenerate the golden snapshot", err)
|
||||
_, _, src, err := w.loadActorForResume(ctx, actorRef)
|
||||
if err != nil || !src.SnapshotURI.IsZero() {
|
||||
t.Fatalf("source = %+v, err = %v; want cold boot", src, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1326,7 +1324,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
|
||||
type templateSeed struct {
|
||||
// onPause is the template's pause scope.
|
||||
onPause ateapipb.SnapshotContentScope
|
||||
// golden seeds Status.GoldenSnapshotStatus.GoldenSnapshot.
|
||||
// golden seeds the template's golden tag snapshot.
|
||||
golden *ateapipb.ExternalSnapshot
|
||||
// fromData is the template's onResume boot-source policy.
|
||||
fromData ateapipb.ResumeSource
|
||||
@@ -1366,8 +1364,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
|
||||
want: restoreWant{run: true},
|
||||
},
|
||||
{
|
||||
name: "03 golden fallback restores the golden snapshot in Full",
|
||||
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}},
|
||||
name: "03 inherited golden snapshot restores in Full",
|
||||
actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, tmplUID: "current"},
|
||||
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}},
|
||||
want: restoreWant{
|
||||
checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL,
|
||||
snapshotURI: goldenURI,
|
||||
@@ -1375,9 +1374,9 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
// With no actor snapshot the restore is not data-only, so the
|
||||
// golden rides in ExternalConfig and GoldenSnapshotUri stays empty.
|
||||
name: "04 golden fallback under Golden fromData is a plain Full restore",
|
||||
// An inherited Full golden snapshot needs no data-only overlay.
|
||||
name: "04 inherited golden under Golden fromData is a plain Full restore",
|
||||
actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}, tmplUID: "current"},
|
||||
tmpl: templateSeed{
|
||||
golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope},
|
||||
fromData: fromGolden,
|
||||
@@ -1389,27 +1388,21 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "05 golden fallback rejects a non-Full golden",
|
||||
name: "05 late non-Full golden does not change a cold boot",
|
||||
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: dataScope}},
|
||||
want: restoreWant{code: codes.FailedPrecondition},
|
||||
want: restoreWant{run: true},
|
||||
},
|
||||
{
|
||||
name: "06 golden fallback rejects a malformed golden URI",
|
||||
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}},
|
||||
want: restoreWant{code: codes.DataLoss},
|
||||
name: "06 inherited golden snapshot rejects a malformed URI",
|
||||
actor: actorSeed{externalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}, tmplUID: "current"},
|
||||
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: malformedURI, ContentScope: fullScope}},
|
||||
want: restoreWant{code: codes.DataLoss},
|
||||
},
|
||||
{
|
||||
// TemplateReplaced is derived from the actor's own durable
|
||||
// snapshot; without one, a repoint cannot downgrade the golden
|
||||
// fallback.
|
||||
name: "07 template repoint does not affect the golden fallback",
|
||||
actor: actorSeed{tmplUID: "mismatch"},
|
||||
name: "07 template repoint with a late golden still cold-boots",
|
||||
actor: actorSeed{tmplUID: "old-template-uid"},
|
||||
tmpl: templateSeed{golden: &ateapipb.ExternalSnapshot{SnapshotUri: goldenURI, ContentScope: fullScope}},
|
||||
want: restoreWant{
|
||||
checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL,
|
||||
snapshotURI: goldenURI,
|
||||
scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL,
|
||||
},
|
||||
want: restoreWant{run: true},
|
||||
},
|
||||
{
|
||||
name: "08 Full durable snapshot restores itself in Full",
|
||||
@@ -1732,13 +1725,23 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
|
||||
}
|
||||
if tt.tmpl.golden != nil {
|
||||
tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: tt.tmpl.golden,
|
||||
GoldenTag: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
|
||||
}}
|
||||
}
|
||||
createdTmpl, err := persistence.CreateActorTemplate(ctx, tmpl)
|
||||
if err != nil {
|
||||
t.Fatalf("create template: %v", err)
|
||||
}
|
||||
if tt.tmpl.golden != nil {
|
||||
if _, err := persistence.CreateTag(ctx, &ateapipb.Tag{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "golden"},
|
||||
SourceActor: &ateapipb.ObjectRef{Atespace: "ns", Name: "golden"},
|
||||
Scope: ateapipb.TagScope_TAG_SCOPE_PUBLISHED,
|
||||
Status: &ateapipb.TagStatus{ActorTemplateUid: createdTmpl.GetMetadata().GetUid(), Snapshot: tt.tmpl.golden},
|
||||
}); err != nil {
|
||||
t.Fatalf("create golden tag: %v", err)
|
||||
}
|
||||
}
|
||||
if createdTmpl.GetMetadata().GetUid() == "" {
|
||||
t.Fatal("created template has no UID; the matching tmplUID case would be vacuous")
|
||||
}
|
||||
|
||||
@@ -3610,10 +3610,10 @@ func Validate_GoldenSnapshotStatus(
|
||||
ctx context.Context, op operation.Operation, fldPath *field.Path,
|
||||
obj, oldObj *ateapipb.GoldenSnapshotStatus) (errs field.ErrorList) {
|
||||
|
||||
{ // field ateapipb.GoldenSnapshotStatus.GoldenSnapshot
|
||||
{ // field ateapipb.GoldenSnapshotStatus.GoldenTag
|
||||
fn := func(
|
||||
fldPath *field.Path,
|
||||
obj, oldObj *ateapipb.ExternalSnapshot,
|
||||
obj, oldObj *ateapipb.ObjectRef,
|
||||
oldValueCorrelated bool) (errs field.ErrorList) {
|
||||
// don't revalidate unchanged data
|
||||
if oldValueCorrelated && op.Type == operation.Update {
|
||||
@@ -3629,15 +3629,30 @@ func Validate_GoldenSnapshotStatus(
|
||||
if earlyReturn {
|
||||
return // do not proceed
|
||||
}
|
||||
func() { // cohort = "atespace"
|
||||
earlyReturn := false
|
||||
if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace",
|
||||
func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.RequiredValue).MarkShortCircuit(); len(e) != 0 {
|
||||
errs = append(errs, e...)
|
||||
earlyReturn = true
|
||||
}
|
||||
if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace",
|
||||
func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 {
|
||||
earlyReturn = true
|
||||
}
|
||||
if earlyReturn {
|
||||
return // do not proceed
|
||||
}
|
||||
}()
|
||||
// call the type's validation function
|
||||
errs = append(errs, Validate_ExternalSnapshot(ctx, op, fldPath, obj, oldObj)...)
|
||||
errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...)
|
||||
return
|
||||
}
|
||||
oldVal := safe.Field(oldObj,
|
||||
func(oldObj *ateapipb.GoldenSnapshotStatus) *ateapipb.ExternalSnapshot {
|
||||
return oldObj.GoldenSnapshot
|
||||
func(oldObj *ateapipb.GoldenSnapshotStatus) *ateapipb.ObjectRef {
|
||||
return oldObj.GoldenTag
|
||||
})
|
||||
errs = append(errs, fn(fldPath.Child("golden_snapshot"), obj.GoldenSnapshot, oldVal, oldObj != nil)...)
|
||||
errs = append(errs, fn(fldPath.Child("golden_tag"), obj.GoldenTag, oldVal, oldObj != nil)...)
|
||||
}
|
||||
|
||||
// field ateapipb.GoldenSnapshotStatus.TakeGoldenSnapshotAt has no validation
|
||||
|
||||
@@ -107,7 +107,7 @@ kubectl ate get workers -l <label-selector>
|
||||
| Column | Meaning |
|
||||
|---|---|
|
||||
| `ATESPACE` | The atespace the actor belongs to. Part of the actor's identity; folded into the storage key as `actor:<atespace>:<name>`. |
|
||||
| `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises during `ResumeGoldenActor`. |
|
||||
| `NAME` | The actor's name. User-provided for application actors; UUID for the golden actor that each template materialises while building its golden tag. |
|
||||
| `TEMPLATE` | The `ActorTemplate` the actor was created from, displayed as `<atespace>/<name>`. |
|
||||
| `STATE` | One of `ACTOR_STATE_RESUMING`, `ACTOR_STATE_RUNNING`, `ACTOR_STATE_SUSPENDING`, `ACTOR_STATE_SUSPENDED`. |
|
||||
| `WORKER POD` | The worker pod (namespace/name) currently hosting the actor. Empty while suspended. |
|
||||
@@ -184,7 +184,7 @@ for a complete manifest example.
|
||||
| `ATESPACE` | The atespace the template belongs to. |
|
||||
| `NAME` | The template's name. |
|
||||
| `SANDBOX CLASS` | The sandbox runtime family (`SANDBOX_CLASS_GVISOR` or `SANDBOX_CLASS_MICROVM`). |
|
||||
| `GOLDEN SNAPSHOT` | The golden snapshot's name once it exists (actors can be created); empty while the golden build is still running. |
|
||||
| `GOLDEN TAG` | The golden tag's name once it exists; empty while the golden build is still running. |
|
||||
| `ERROR` | `ERROR` when the golden build failed; `-o yaml` shows the full message. |
|
||||
| `AGE` | Time elapsed since the template was created. |
|
||||
|
||||
|
||||
@@ -291,7 +291,7 @@ func PrintActorTemplatesTo(out io.Writer, templates []*ateapipb.ActorTemplate, f
|
||||
return printProto(out, &ateapipb.ListActorTemplatesResponse{ActorTemplates: templates}, format)
|
||||
case "table":
|
||||
w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0)
|
||||
fmt.Fprintln(w, "ATESPACE\tNAME\tSANDBOX CLASS\tGOLDEN SNAPSHOT\tERROR\tAGE")
|
||||
fmt.Fprintln(w, "ATESPACE\tNAME\tSANDBOX CLASS\tGOLDEN TAG\tERROR\tAGE")
|
||||
for _, t := range templates {
|
||||
gss := t.GetStatus().GetGoldenSnapshotStatus()
|
||||
// Error messages are too long for a table cell.
|
||||
@@ -302,7 +302,7 @@ func PrintActorTemplatesTo(out io.Writer, templates []*ateapipb.ActorTemplate, f
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
t.GetMetadata().GetAtespace(), t.GetMetadata().GetName(),
|
||||
t.GetSandboxConfig().GetSandboxClass(),
|
||||
gss.GetGoldenSnapshot().GetSnapshotUri(), errFlag,
|
||||
gss.GetGoldenTag().GetName(), errFlag,
|
||||
formatAge(t.GetMetadata().GetCreateTime()))
|
||||
}
|
||||
return w.Flush()
|
||||
|
||||
@@ -440,7 +440,7 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) {
|
||||
},
|
||||
Status: &ateapipb.ActorTemplateStatus{
|
||||
GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
|
||||
GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://private/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/snap-1"},
|
||||
GoldenTag: &ateapipb.ObjectRef{Atespace: "ate-golden", Name: "golden-tag"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -481,10 +481,10 @@ func TestPrintActorTemplatesTo_Table(t *testing.T) {
|
||||
|
||||
// Sorted by atespace, then name. The ERROR column only flags that an
|
||||
// error message exists; the full text is available via json/yaml.
|
||||
expected := `ATESPACE NAME SANDBOX CLASS GOLDEN SNAPSHOT ERROR AGE
|
||||
ate-demo-counter-substrate counter SANDBOX_CLASS_GVISOR gs://private/atespaces/ate-golden/actors/9c2f7b41-6d05-4e83-a1f7-3b8c0d5e2a94/snapshots/snap-1 5m
|
||||
ate-demo-counter-substrate counter-2 SANDBOX_CLASS_GVISOR 3d
|
||||
ate-demo-counter-substrate-microvm counter-microvm SANDBOX_CLASS_MICROVM ERROR 5h
|
||||
expected := `ATESPACE NAME SANDBOX CLASS GOLDEN TAG ERROR AGE
|
||||
ate-demo-counter-substrate counter SANDBOX_CLASS_GVISOR golden-tag 5m
|
||||
ate-demo-counter-substrate counter-2 SANDBOX_CLASS_GVISOR 3d
|
||||
ate-demo-counter-substrate-microvm counter-microvm SANDBOX_CLASS_MICROVM ERROR 5h
|
||||
`
|
||||
if diff := cmp.Diff(expected, buf.String()); diff != "" {
|
||||
t.Errorf("output mismatch (-want +got):\n%s", diff)
|
||||
|
||||
+8
-6
@@ -360,13 +360,13 @@ snapshotsConfig:
|
||||
<location>/atespaces/<atespace>/tags/<tag uid>
|
||||
```
|
||||
|
||||
The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot — the golden actor lives in the reserved `ate-golden` atespace — under `gs://my-bucket/secret-agent/atespaces/ate-golden/actors/<uid>/snapshots/<name>`.
|
||||
The objects of a snapshot (its manifest, memory image, durable-data tar) are named below it. So for the template above, a snapshot of an actor in atespace `team-a` is stored at `gs://my-bucket/secret-agent/atespaces/team-a/actors/3f8b…/snapshots/f47ac10b-…`, and the template's golden snapshot — the golden tag lives in the reserved `ate-golden` atespace — under `gs://my-bucket/secret-agent/atespaces/ate-golden/tags/<tag uid>`.
|
||||
|
||||
An actor takes a series of snapshots over its life, so it gets a prefix of its own and each snapshot sits below it. A tag holds exactly one, so the tag's prefix *is* its snapshot's. Both owners are keyed on their UID, so recreating an actor or tag under the same name never inherits its predecessor's objects. A pending tag records its base location in `status.storageLocation`; together with its atespace and UID, this identifies any partial copy to collect if creation fails.
|
||||
|
||||
An owner is collected by deleting everything under its prefix, and it can delete nothing else. That is what makes a borrowed snapshot safe: an actor created from a tag points at a URI under `tags/`, which its own prefix does not cover. See [Snapshot lifetime](#snapshot-lifetime).
|
||||
|
||||
An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot`, a `Tag` in `status.snapshot`, and an `ActorTemplate` its golden one in `status.goldenSnapshotStatus.goldenSnapshot` — each an `ExternalSnapshot` carrying `snapshotUri` and the `contentScope` it captured. The URI is recorded when the snapshot is written. All three are server-owned: do not send them on input, and parse a URI only against the scheme above.
|
||||
An `Actor` reports its current snapshot in the server-managed `status.externalSnapshot` and a `Tag` in `status.snapshot`, each an `ExternalSnapshot` carrying `snapshotUri` and `contentScope`. The URI is recorded when the snapshot is written. An `ActorTemplate` references its golden tag with the `ObjectRef` in `status.goldenSnapshotStatus.goldenTag`. These status fields are server-owned and ignored on input. Parse a URI only against the scheme above.
|
||||
|
||||
An `ActorTemplate` belongs to one atespace, but one `storageLocation` still holds snapshots for many atespaces: the golden actor lives in the reserved `ate-golden` atespace, and a `PUBLISHED` snapshot may be cloned from other atespaces. The `<atespace>` level exists so that access can be granted per tenant: an object-storage policy can only condition on an **object-name prefix**, and cannot read the identity recorded inside a snapshot's manifest. Binding a per-atespace grant on GCS looks like:
|
||||
|
||||
@@ -434,10 +434,12 @@ See [`hack/microvm-assets/`](../hack/microvm-assets/) for scripts that assemble
|
||||
|
||||
### The Golden Snapshot
|
||||
When an `ActorTemplate` is created:
|
||||
1. Substrate starts a temporary **Golden Pod**.
|
||||
2. It executes your workload containers as defined in the template.
|
||||
3. Once the process is initialized, gVisor takes a **Golden Snapshot** (Version 0).
|
||||
4. The template enters the `Ready` phase.
|
||||
1. Substrate creates and resumes a temporary golden actor in `ate-golden`.
|
||||
2. It waits for readiness (or the warm-up interval), then suspends the actor.
|
||||
3. It creates a published tag named after the template UID, copying the snapshot into tag-owned storage.
|
||||
4. It deletes the golden actor and records the tag reference in the template status.
|
||||
|
||||
`CreateActor` uses an explicit `sourceTag` when supplied; otherwise it resolves the template's golden tag and records that snapshot on the new actor. If the golden tag is not ready yet, the actor starts without a snapshot and cold-boots even if the tag becomes ready before its first resume. The default does not populate the caller-owned `sourceTag` field. Deleting the template collects its golden tag and any unfinished golden actor.
|
||||
|
||||
### Resumption Lifecycle
|
||||
Once a template is `Ready`, creating an actor logically (via `kubectl ate create actor`) allows it to be resumed instantly on any free worker in the referenced `WorkerPool`. Substrate bypasses the standard container boot and restores the process directly from its last saved state.
|
||||
|
||||
+1
-1
@@ -1204,7 +1204,7 @@ wait_actortemplate_ready() {
|
||||
|
||||
while ((SECONDS < deadline)); do
|
||||
if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then
|
||||
snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenSnapshot.snapshotUri // empty' <<<"${json}")
|
||||
snapshot=$(jq -r '.status.goldenSnapshotStatus.goldenTag.name // empty' <<<"${json}")
|
||||
if [[ -n "${snapshot}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -94,7 +94,7 @@ log_step "preflight"
|
||||
# Ready means the template's golden snapshot exists; protojson omits empty
|
||||
# fields, so the key is only present once it is set.
|
||||
run_kubectl_ate get actor-template "${DEMO_POOL}" -a "${ATESPACE}" -o json 2>/dev/null \
|
||||
| grep -q '"goldenSnapshot"' \
|
||||
| grep -q '"goldenTag"' \
|
||||
|| fail "actor template ${ATESPACE}/${DEMO_POOL} has no golden snapshot; install the counter demo first"
|
||||
# Column 4 is STATUS; the header row's "ASSIGNED ACTOR" column name must not
|
||||
# trip the check.
|
||||
|
||||
@@ -162,7 +162,7 @@ func WaitForSubstrateTemplateReady(ctx context.Context, t *testing.T, clients *C
|
||||
})
|
||||
if err == nil {
|
||||
lastStatus = at.GetStatus().GetGoldenSnapshotStatus()
|
||||
if lastStatus.GetGoldenSnapshot().GetSnapshotUri() != "" {
|
||||
if lastStatus.GetGoldenTag().GetName() != "" {
|
||||
return
|
||||
}
|
||||
if msg := lastStatus.GetErrorMessage(); msg != "" {
|
||||
|
||||
@@ -2271,12 +2271,11 @@ func (x *Limits) GetQuantity() string {
|
||||
|
||||
type GoldenSnapshotStatus struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// golden_snapshot is the external snapshot built for this version by
|
||||
// ate-api, taken from the golden Actor in the reserved ate-golden system
|
||||
// atespace. Set once state is READY. The golden Actor owns it.
|
||||
//
|
||||
// golden_tag owns the immutable snapshot built by the template controller.
|
||||
// Set after tagging the snapshot and deleting the temporary golden Actor.
|
||||
// +k8s:optional
|
||||
GoldenSnapshot *ExternalSnapshot `protobuf:"bytes,1,opt,name=golden_snapshot,json=goldenSnapshot,proto3" json:"golden_snapshot,omitempty"`
|
||||
// +k8s:subfield(atespace)=+k8s:required
|
||||
GoldenTag *ObjectRef `protobuf:"bytes,1,opt,name=golden_tag,json=goldenTag,proto3" json:"golden_tag,omitempty"`
|
||||
// take_golden_snapshot_at is when the golden-actor warmup ends and the
|
||||
// golden snapshot may be taken.
|
||||
TakeGoldenSnapshotAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=take_golden_snapshot_at,json=takeGoldenSnapshotAt,proto3" json:"take_golden_snapshot_at,omitempty"`
|
||||
@@ -2315,9 +2314,9 @@ func (*GoldenSnapshotStatus) Descriptor() ([]byte, []int) {
|
||||
return file_ateapi_proto_rawDescGZIP(), []int{21}
|
||||
}
|
||||
|
||||
func (x *GoldenSnapshotStatus) GetGoldenSnapshot() *ExternalSnapshot {
|
||||
func (x *GoldenSnapshotStatus) GetGoldenTag() *ObjectRef {
|
||||
if x != nil {
|
||||
return x.GoldenSnapshot
|
||||
return x.GoldenTag
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -6961,9 +6960,10 @@ const file_ateapi_proto_rawDesc = "" +
|
||||
"\x06limits\x18\x01 \x03(\v2\x0e.ateapi.LimitsR\x06limits\"8\n" +
|
||||
"\x06Limits\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" +
|
||||
"\bquantity\x18\x02 \x01(\tR\bquantity\"\xd1\x01\n" +
|
||||
"\x14GoldenSnapshotStatus\x12A\n" +
|
||||
"\x0fgolden_snapshot\x18\x01 \x01(\v2\x18.ateapi.ExternalSnapshotR\x0egoldenSnapshot\x12Q\n" +
|
||||
"\bquantity\x18\x02 \x01(\tR\bquantity\"\xc0\x01\n" +
|
||||
"\x14GoldenSnapshotStatus\x120\n" +
|
||||
"\n" +
|
||||
"golden_tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\tgoldenTag\x12Q\n" +
|
||||
"\x17take_golden_snapshot_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x14takeGoldenSnapshotAt\x12#\n" +
|
||||
"\rerror_message\x18\x03 \x01(\tR\ferrorMessage\"i\n" +
|
||||
"\x13ActorTemplateStatus\x12R\n" +
|
||||
@@ -7443,7 +7443,7 @@ var file_ateapi_proto_depIdxs = []int32{
|
||||
28, // 37: ateapi.ActorTemplate.resources:type_name -> ateapi.Resources
|
||||
31, // 38: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus
|
||||
29, // 39: ateapi.Resources.limits:type_name -> ateapi.Limits
|
||||
9, // 40: ateapi.GoldenSnapshotStatus.golden_snapshot:type_name -> ateapi.ExternalSnapshot
|
||||
26, // 40: ateapi.GoldenSnapshotStatus.golden_tag:type_name -> ateapi.ObjectRef
|
||||
108, // 41: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp
|
||||
30, // 42: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus
|
||||
3, // 43: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass
|
||||
|
||||
@@ -138,7 +138,7 @@ service Control {
|
||||
rpc ListActorTemplates(ListActorTemplatesRequest) returns (ListActorTemplatesResponse) {}
|
||||
|
||||
// Delete an ActorTemplate together with its golden actor and golden
|
||||
// snapshot in the ActorTemplate's namespace.
|
||||
// tag in the reserved ate-golden atespace.
|
||||
rpc DeleteActorTemplate(DeleteActorTemplateRequest) returns (ActorTemplate) {}
|
||||
}
|
||||
|
||||
@@ -807,12 +807,11 @@ message Limits {
|
||||
}
|
||||
|
||||
message GoldenSnapshotStatus {
|
||||
// golden_snapshot is the external snapshot built for this version by
|
||||
// ate-api, taken from the golden Actor in the reserved ate-golden system
|
||||
// atespace. Set once state is READY. The golden Actor owns it.
|
||||
//
|
||||
// golden_tag owns the immutable snapshot built by the template controller.
|
||||
// Set after tagging the snapshot and deleting the temporary golden Actor.
|
||||
// +k8s:optional
|
||||
ExternalSnapshot golden_snapshot = 1;
|
||||
// +k8s:subfield(atespace)=+k8s:required
|
||||
ObjectRef golden_tag = 1;
|
||||
|
||||
// take_golden_snapshot_at is when the golden-actor warmup ends and the
|
||||
// golden snapshot may be taken.
|
||||
|
||||
@@ -157,7 +157,7 @@ type ControlClient interface {
|
||||
GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error)
|
||||
ListActorTemplates(ctx context.Context, in *ListActorTemplatesRequest, opts ...grpc.CallOption) (*ListActorTemplatesResponse, error)
|
||||
// Delete an ActorTemplate together with its golden actor and golden
|
||||
// snapshot in the ActorTemplate's namespace.
|
||||
// tag in the reserved ate-golden atespace.
|
||||
DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error)
|
||||
}
|
||||
|
||||
@@ -597,7 +597,7 @@ type ControlServer interface {
|
||||
GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error)
|
||||
ListActorTemplates(context.Context, *ListActorTemplatesRequest) (*ListActorTemplatesResponse, error)
|
||||
// Delete an ActorTemplate together with its golden actor and golden
|
||||
// snapshot in the ActorTemplate's namespace.
|
||||
// tag in the reserved ate-golden atespace.
|
||||
DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error)
|
||||
mustEmbedUnimplementedControlServer()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user