Files
substrate/cmd/ateapi/internal/controlapi/workflow_delete_test.go
T
Sneha-at b7080602c6 Allow deleting an actor from any state. (#788)
Fixes #643 
This feature introduces the ability to delete Substrate actors from any
lifecycle state (e.g., RUNNING, PAUSED, PENDING), rather than requiring
them to be hibernated into SUSPENDED (or CRASHED) beforehand. This is
crucial for cleaning up stuck or failed actors.
#### 1. Control Plane & gRPC API (pkg/proto/ateapipb, ateapi)
• DeleteActorRequest.any_state: Added a new boolean field any_state to
DeleteActorRequest.
• When false (default): enforces the existing behavior where only actors
in ACTOR_STATE_SUSPENDED or ACTOR_STATE_CRASHED (or already
ACTOR_STATE_DELETING) can be deleted.
• When true: allows deleting an actor in any state (e.g., RUNNING,
PAUSED).
• Orchestrated Cleanup Workflow (workflow_delete.go):
1. Transitions actor state to ACTOR_STATE_DELETING.
2. Calls atelet.Terminate to stop live workloads on the node.
3. Detaches external volumes from the worker node (with fallback logic
using actor status if the ActorTemplate was deleted).
4. Releases the assigned physical worker Pod back to the WorkerPool.
5. Deletes external storage volumes through the storage plugins.
6. Finalizes removal of the actor from the persistence store.
#### 2. Node Herder Agent (atelet)
• Terminate RPC (main.go): Added a new Terminate method to AteomHerder:
• Calls ateom.TerminateWorkload on the target worker pod.
• Unmounts external volumes mounted for the actor on the node.
• Cleans up and resets actor runtime host directories (OCI bundles,
checkpoints, pid files).
#### 3. In-Pod Sandboxes (ateom-gvisor & ateom-microvm)
• TerminateWorkload RPC:
• gVisor (main.go): Stops and deletes active runsc containers, unmounts
bundle rootfs overlays, and tears down the actor's interior network
namespace.
• Micro-VM (checkpoint.go): Shuts down the Cloud Hypervisor VMM,
unmounts bundle rootfs overlays, and cleans up the network namespace.
• Emits an "Actor terminated" lifecycle log and clears the active actor
attribution so the worker can host new workloads.
#### 4. CLI (kubectl-ate)
• --any-state Flag (delete_actor.go):
kubectl-ate delete actor <actor-name> -a <atespace> --any-state

──────
  ## Verification 
  [x] Changes tested in local with 
 ```
• go build ./...: Passed
go test ./...: Passed (all unit and functional tests pass)
    make lint: Passed (no lint errors) 
```
Detailed manual tests                                                                                                                                                                                                                                                                                                                                           
 ```                                                                                                                                                                                                                                                                                                                                                
  ### Scenario 1: Force Delete a Running Actor with --any-state                                                                                                                                                                                                                                                                                                      
                                                                                                                                                                                                                                                                                                                                                                     
  1. Create the actor:                                                                                                                                                                                                                                                                                                                                               
    kubectl ate create actor manual-test-1 --template=ate-demo-counter-microvm/counter-microvm -a demo                                                                                                                                                                                                                                                               
                                                                                                                                                                                                                                                                                                                                                                     
  2. Resume the actor:                                                                                                                                                                                                                                                                                                                                               
    kubectl ate resume actor manual-test-1 -a demo                                                                                                                                                                                                                                                                                                                   
                                                                                                                                                                                                                                                                                                                                                                     
  3. Force delete the running actor:                                                                                                                                                                                                                                                                                                                                 
    kubectl ate delete actor manual-test-1 -a demo --any-state                                                                                                                                                                                                                                                                                                       
    actor "manual-test-1" deleted                                                                                                                                                                                                                                                                                                                                    
```
──────
### Scenario 2: Standard Delete a Suspended Actor
```                                                                                                                                                                                                                                                                                                                                  
  1. Create the actor:                                                                                                                                                                                                                                                                                                                                               
    kubectl ate create actor manual-test-2 --template=ate-demo-counter-microvm/counter-microvm -a demo                                                                                                                                                                                                                                                               
                                                                                                                                                                                                                                                                                                                                                                     
  2. Resume the actor:                                                                                                                                                                                                                                                                                                                                               
    kubectl ate resume actor manual-test-2 -a demo                                                                                                                                                                                                                                                                                                                   
                                                                                                                                                                                                                                                                                                                                                                     
  3. Suspend the actor:                                                                                                                                                                                                                                                                                                                                              
    kubectl ate suspend actor manual-test-2 -a demo                                                                                                                                                                                                                                                                                                                  
                                                                                                                                                                                                                                                                                                                                                                     
  4. Standard delete the suspended actor:                                                                                                                                                                                                                                                                                                                            
    kubectl ate delete actor manual-test-2 -a demo                                                                                                                                                                                                                                                                                                                   
    actor "manual-test-2" deleted                                                                                                                                                                                                                                                                                                                                    
```
──────
### Unit & E2E Tests
# Unit & Functional Tests
go test ./cmd/ateapi/internal/controlapi -run
"TestDeleteActorWorkflow|TestEnsureMarkedDeleting"
# E2E Tests
E2E_TEMPLATE_NAMESPACE=ate-demo-counter-microvm
E2E_TEMPLATE_NAME=counter-microvm ./hack/run-e2e-kind.sh
./internal/e2e/suites/demo -run TestActorLifecycle
./hack/run-e2e-kind.sh ./internal/e2e/suites/demo -run
TestForceDeleteActorWithExternalVolume
```
- [ x] Appropriate changes to documentation are included in the PR
2026-08-20 13:17:59 -07:00

201 lines
6.0 KiB
Go

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controlapi
import (
"context"
"testing"
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest"
"github.com/agent-substrate/substrate/internal/resources"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) {
tests := []struct {
name string
seedState ateapipb.ActorState
anyState bool
missingTmpl bool
wantErr bool
wantCode codes.Code
}{
{
name: "delete suspended actor succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
anyState: false,
wantErr: false,
},
{
name: "delete crashed actor succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_CRASHED,
anyState: false,
wantErr: false,
},
{
name: "delete deleting actor succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_DELETING,
anyState: false,
wantErr: false,
},
{
name: "delete running actor rejected when not any_state",
seedState: ateapipb.ActorState_ACTOR_STATE_RUNNING,
anyState: false,
wantErr: true,
wantCode: codes.FailedPrecondition,
},
{
name: "delete paused actor rejected when not any_state",
seedState: ateapipb.ActorState_ACTOR_STATE_PAUSED,
anyState: false,
wantErr: true,
wantCode: codes.FailedPrecondition,
},
{
name: "any_state delete suspended actor succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
anyState: true,
wantErr: false,
},
{
name: "any_state delete running actor succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_RUNNING,
anyState: true,
wantErr: false,
},
{
name: "any_state delete paused actor succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_PAUSED,
anyState: true,
wantErr: false,
},
{
name: "any_state delete crashed actor succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_CRASHED,
anyState: true,
wantErr: false,
},
{
name: "delete suspended actor with missing template succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
anyState: false,
missingTmpl: true,
wantErr: false,
},
{
name: "any_state delete suspended actor with missing template succeeds",
seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
anyState: true,
missingTmpl: true,
wantErr: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
st, cleanup := storetest.SetupTestStore(t)
defer cleanup()
w := newTestActorWorkflow(t, st, "ns", "tmpl1")
actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"}
tmplName := "tmpl1"
if tc.missingTmpl {
tmplName = "missing-tmpl"
}
seedWorkflowActor(t, ctx, st, actorRef, "ns", tmplName, tc.seedState)
deleted, err := w.DeleteActor(ctx, actorRef, tc.anyState)
if tc.wantErr {
if got := status.Code(err); got != tc.wantCode {
t.Fatalf("status.Code(err) = %v, want %v (err: %v)", got, tc.wantCode, err)
}
} else {
if err != nil {
t.Fatalf("DeleteActor failed: %v", err)
}
if deleted == nil {
t.Fatalf("expected non-nil deleted actor")
}
if _, err := st.GetActor(ctx, actorRef); err == nil {
t.Errorf("expected actor to be deleted from store, but it still exists")
}
}
})
}
}
func TestEnsureMarkedDeleting_StateMatrix(t *testing.T) {
tests := []struct {
name string
anyState bool
allowed map[ateapipb.ActorState]bool
}{
{
name: "standard delete",
anyState: false,
allowed: map[ateapipb.ActorState]bool{
ateapipb.ActorState_ACTOR_STATE_SUSPENDED: true,
ateapipb.ActorState_ACTOR_STATE_CRASHED: true,
ateapipb.ActorState_ACTOR_STATE_DELETING: true, // skipped
},
},
{
name: "any_state delete",
anyState: true,
allowed: map[ateapipb.ActorState]bool{
ateapipb.ActorState_ACTOR_STATE_UNSPECIFIED: true,
ateapipb.ActorState_ACTOR_STATE_RUNNING: true,
ateapipb.ActorState_ACTOR_STATE_RESUMING: true,
ateapipb.ActorState_ACTOR_STATE_SUSPENDING: true,
ateapipb.ActorState_ACTOR_STATE_PAUSING: true,
ateapipb.ActorState_ACTOR_STATE_PAUSED: true,
ateapipb.ActorState_ACTOR_STATE_SUSPENDED: true,
ateapipb.ActorState_ACTOR_STATE_CRASHED: true,
ateapipb.ActorState_ACTOR_STATE_DELETING: true, // skipped
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
for _, seedState := range allActorStates {
ctx := context.Background()
st, cleanup := storetest.SetupTestStore(t)
w := newTestActorWorkflow(t, st, "ns", "tmpl1")
actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"}
seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", seedState)
actor, err := st.GetActor(ctx, actorRef)
if err != nil {
t.Fatalf("state %v: get seeded actor: %v", seedState, err)
}
updated, err := w.ensureMarkedDeleting(ctx, actorRef, actor, tc.anyState)
assertPrerequisiteResult(t, seedState, err, tc.allowed[seedState])
if err == nil && seedState != ateapipb.ActorState_ACTOR_STATE_DELETING {
if updated.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_DELETING {
t.Errorf("state %v: ensureMarkedDeleting returned actor in %v, want DELETING", seedState, updated.GetStatus().GetState())
}
}
cleanup()
}
})
}
}