mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-11-11 10:23:17 +08:00
Fixes #643 This feature introduces the ability to delete Substrate actors from any lifecycle state (e.g., RUNNING, PAUSED, PENDING), rather than requiring them to be hibernated into SUSPENDED (or CRASHED) beforehand. This is crucial for cleaning up stuck or failed actors. #### 1. Control Plane & gRPC API (pkg/proto/ateapipb, ateapi) • DeleteActorRequest.any_state: Added a new boolean field any_state to DeleteActorRequest. • When false (default): enforces the existing behavior where only actors in ACTOR_STATE_SUSPENDED or ACTOR_STATE_CRASHED (or already ACTOR_STATE_DELETING) can be deleted. • When true: allows deleting an actor in any state (e.g., RUNNING, PAUSED). • Orchestrated Cleanup Workflow (workflow_delete.go): 1. Transitions actor state to ACTOR_STATE_DELETING. 2. Calls atelet.Terminate to stop live workloads on the node. 3. Detaches external volumes from the worker node (with fallback logic using actor status if the ActorTemplate was deleted). 4. Releases the assigned physical worker Pod back to the WorkerPool. 5. Deletes external storage volumes through the storage plugins. 6. Finalizes removal of the actor from the persistence store. #### 2. Node Herder Agent (atelet) • Terminate RPC (main.go): Added a new Terminate method to AteomHerder: • Calls ateom.TerminateWorkload on the target worker pod. • Unmounts external volumes mounted for the actor on the node. • Cleans up and resets actor runtime host directories (OCI bundles, checkpoints, pid files). #### 3. In-Pod Sandboxes (ateom-gvisor & ateom-microvm) • TerminateWorkload RPC: • gVisor (main.go): Stops and deletes active runsc containers, unmounts bundle rootfs overlays, and tears down the actor's interior network namespace. • Micro-VM (checkpoint.go): Shuts down the Cloud Hypervisor VMM, unmounts bundle rootfs overlays, and cleans up the network namespace. • Emits an "Actor terminated" lifecycle log and clears the active actor attribution so the worker can host new workloads. #### 4. CLI (kubectl-ate) • --any-state Flag (delete_actor.go): kubectl-ate delete actor <actor-name> -a <atespace> --any-state ────── ## Verification [x] Changes tested in local with ``` • go build ./...: Passed go test ./...: Passed (all unit and functional tests pass) make lint: Passed (no lint errors) ``` Detailed manual tests ``` ### Scenario 1: Force Delete a Running Actor with --any-state 1. Create the actor: kubectl ate create actor manual-test-1 --template=ate-demo-counter-microvm/counter-microvm -a demo 2. Resume the actor: kubectl ate resume actor manual-test-1 -a demo 3. Force delete the running actor: kubectl ate delete actor manual-test-1 -a demo --any-state actor "manual-test-1" deleted ``` ────── ### Scenario 2: Standard Delete a Suspended Actor ``` 1. Create the actor: kubectl ate create actor manual-test-2 --template=ate-demo-counter-microvm/counter-microvm -a demo 2. Resume the actor: kubectl ate resume actor manual-test-2 -a demo 3. Suspend the actor: kubectl ate suspend actor manual-test-2 -a demo 4. Standard delete the suspended actor: kubectl ate delete actor manual-test-2 -a demo actor "manual-test-2" deleted ``` ────── ### Unit & E2E Tests # Unit & Functional Tests go test ./cmd/ateapi/internal/controlapi -run "TestDeleteActorWorkflow|TestEnsureMarkedDeleting" # E2E Tests E2E_TEMPLATE_NAMESPACE=ate-demo-counter-microvm E2E_TEMPLATE_NAME=counter-microvm ./hack/run-e2e-kind.sh ./internal/e2e/suites/demo -run TestActorLifecycle ./hack/run-e2e-kind.sh ./internal/e2e/suites/demo -run TestForceDeleteActorWithExternalVolume ``` - [ x] Appropriate changes to documentation are included in the PR
201 lines
6.0 KiB
Go
201 lines
6.0 KiB
Go
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controlapi
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest"
|
|
"github.com/agent-substrate/substrate/internal/resources"
|
|
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
func TestDeleteActorWorkflow_ExecutionPaths(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
seedState ateapipb.ActorState
|
|
anyState bool
|
|
missingTmpl bool
|
|
wantErr bool
|
|
wantCode codes.Code
|
|
}{
|
|
{
|
|
name: "delete suspended actor succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
|
|
anyState: false,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "delete crashed actor succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_CRASHED,
|
|
anyState: false,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "delete deleting actor succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_DELETING,
|
|
anyState: false,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "delete running actor rejected when not any_state",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_RUNNING,
|
|
anyState: false,
|
|
wantErr: true,
|
|
wantCode: codes.FailedPrecondition,
|
|
},
|
|
{
|
|
name: "delete paused actor rejected when not any_state",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_PAUSED,
|
|
anyState: false,
|
|
wantErr: true,
|
|
wantCode: codes.FailedPrecondition,
|
|
},
|
|
{
|
|
name: "any_state delete suspended actor succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
|
|
anyState: true,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "any_state delete running actor succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_RUNNING,
|
|
anyState: true,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "any_state delete paused actor succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_PAUSED,
|
|
anyState: true,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "any_state delete crashed actor succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_CRASHED,
|
|
anyState: true,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "delete suspended actor with missing template succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
|
|
anyState: false,
|
|
missingTmpl: true,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "any_state delete suspended actor with missing template succeeds",
|
|
seedState: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
|
|
anyState: true,
|
|
missingTmpl: true,
|
|
wantErr: false,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ctx := context.Background()
|
|
st, cleanup := storetest.SetupTestStore(t)
|
|
defer cleanup()
|
|
w := newTestActorWorkflow(t, st, "ns", "tmpl1")
|
|
|
|
actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"}
|
|
tmplName := "tmpl1"
|
|
if tc.missingTmpl {
|
|
tmplName = "missing-tmpl"
|
|
}
|
|
seedWorkflowActor(t, ctx, st, actorRef, "ns", tmplName, tc.seedState)
|
|
|
|
deleted, err := w.DeleteActor(ctx, actorRef, tc.anyState)
|
|
if tc.wantErr {
|
|
if got := status.Code(err); got != tc.wantCode {
|
|
t.Fatalf("status.Code(err) = %v, want %v (err: %v)", got, tc.wantCode, err)
|
|
}
|
|
} else {
|
|
if err != nil {
|
|
t.Fatalf("DeleteActor failed: %v", err)
|
|
}
|
|
if deleted == nil {
|
|
t.Fatalf("expected non-nil deleted actor")
|
|
}
|
|
if _, err := st.GetActor(ctx, actorRef); err == nil {
|
|
t.Errorf("expected actor to be deleted from store, but it still exists")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEnsureMarkedDeleting_StateMatrix(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
anyState bool
|
|
allowed map[ateapipb.ActorState]bool
|
|
}{
|
|
{
|
|
name: "standard delete",
|
|
anyState: false,
|
|
allowed: map[ateapipb.ActorState]bool{
|
|
ateapipb.ActorState_ACTOR_STATE_SUSPENDED: true,
|
|
ateapipb.ActorState_ACTOR_STATE_CRASHED: true,
|
|
ateapipb.ActorState_ACTOR_STATE_DELETING: true, // skipped
|
|
},
|
|
},
|
|
{
|
|
name: "any_state delete",
|
|
anyState: true,
|
|
allowed: map[ateapipb.ActorState]bool{
|
|
ateapipb.ActorState_ACTOR_STATE_UNSPECIFIED: true,
|
|
ateapipb.ActorState_ACTOR_STATE_RUNNING: true,
|
|
ateapipb.ActorState_ACTOR_STATE_RESUMING: true,
|
|
ateapipb.ActorState_ACTOR_STATE_SUSPENDING: true,
|
|
ateapipb.ActorState_ACTOR_STATE_PAUSING: true,
|
|
ateapipb.ActorState_ACTOR_STATE_PAUSED: true,
|
|
ateapipb.ActorState_ACTOR_STATE_SUSPENDED: true,
|
|
ateapipb.ActorState_ACTOR_STATE_CRASHED: true,
|
|
ateapipb.ActorState_ACTOR_STATE_DELETING: true, // skipped
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
for _, seedState := range allActorStates {
|
|
ctx := context.Background()
|
|
st, cleanup := storetest.SetupTestStore(t)
|
|
w := newTestActorWorkflow(t, st, "ns", "tmpl1")
|
|
|
|
actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"}
|
|
seedWorkflowActor(t, ctx, st, actorRef, "ns", "tmpl1", seedState)
|
|
actor, err := st.GetActor(ctx, actorRef)
|
|
if err != nil {
|
|
t.Fatalf("state %v: get seeded actor: %v", seedState, err)
|
|
}
|
|
|
|
updated, err := w.ensureMarkedDeleting(ctx, actorRef, actor, tc.anyState)
|
|
assertPrerequisiteResult(t, seedState, err, tc.allowed[seedState])
|
|
if err == nil && seedState != ateapipb.ActorState_ACTOR_STATE_DELETING {
|
|
if updated.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_DELETING {
|
|
t.Errorf("state %v: ensureMarkedDeleting returned actor in %v, want DELETING", seedState, updated.GetStatus().GetState())
|
|
}
|
|
}
|
|
cleanup()
|
|
}
|
|
})
|
|
}
|
|
}
|