trending/substrate
1
0
Fork 0
mirror of https://github.com/agent-substrate/substrate.git synced 2026-10-02 03:24:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Switching e2e tests to use the new ActorTemplate substrate proto (#1290)

Browse Source 
Switches all e2e tests that uses counter demo to validate the new
substrate proto ActorTemplate.

Made some changes to make the e2e test pass:
* cmd/ateapi/internal/controlapi/template_reconciler.go - golden actors
still need the golden atespace, because the logic in suspend actor
relies on it to always take FULL snapshot regardless of the config.
(filed https://github.com/agent-substrate/substrate/issues/1299)
* internal/ateattr/ateattr.go - Added placeholder label for now, will
fix in a follow up PR to wire the metric reporting when using new
ActorTemplate substrate object.
This commit is contained in:
Zoe Zhao
2026-08-28 18:52:13 -07:00
committed by GitHub
parent 9ff766279e
commit 77e69a1b9d
17 changed files with 525 additions and 511 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-workflow.yaml
+4 -8
View File
@@ -83,12 +83,12 @@ jobs:
run: hack/create-kind-cluster.sh
- name: Install Agent Substrate
run: hack/install-ate-kind.sh --deploy-ate-system
- name: Deploy micro-VM counter demo
- name: Deploy substrate micro-VM counter demo
# Stages the (cached) assets into the cluster's rustfs and applies the
# counter-microvm demo onto the control plane installed above.
run: hack/run-microvm-demo-kind.sh
- name: Deploy gVisor counter demo
run: hack/install-ate-kind.sh --deploy-demo-counter
run: hack/run-microvm-demo-kind.sh --substrate
- name: Deploy substrate gVisor counter demo
run: hack/install-ate-kind.sh --deploy-demo-counter-substrate
- name: Deploy egress demos
# TestActorEgress in the networking suite builds its Actor from the egress
# ActorTemplate for the class under test, so both fixtures have to exist
@@ -96,10 +96,6 @@ jobs:
run: |
hack/install-ate-kind.sh --deploy-demo-egress
hack/install-ate-kind.sh --deploy-demo-egress-microvm
- name: Wait for micro-VM golden snapshot
run: |
kubectl --context kind-kind wait --for=condition=Ready \
actortemplate/counter-microvm -n ate-demo-counter-microvm --timeout=600s
- name: Run E2E tests (gVisor)
run: hack/run-e2e-kind.sh -v -args --no-color
- name: Run E2E tests (micro-VM)
cmd/ateapi/internal/controlapi/template_reconciler.go
+13 -2
View File
@@ -65,6 +65,7 @@ type templateReconcilerStore interface {
// goldenActorControl is the in-process slice of the Control service the
// reconciler drives golden actors through. *RPCService satisfies it.
type goldenActorControl interface {
CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error)
CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error)
GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)
ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error)
@@ -183,7 +184,11 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource
}
goldenActorRef := &ateapipb.ObjectRef{
Atespace: tmpl.GetMetadata().GetAtespace(),
// Golden actors live in the reserved ate-golden atespace, because
// the suspend workflow relies on the ate-golden system atespace to
// always take a full snapshot of the golden actor.
// https://github.com/agent-substrate/substrate/blob/cb7c8385ef2bb489c3d5f7bfa71820fd33935d91/cmd/ateapi/internal/controlapi/workflow_suspend.go#L170-L173
Atespace: resources.GoldenActorAtespace,
// Use the template's UID as golden actor's name to prevent collision
// when templates are recreated with the same name.
Name: tmpl.GetMetadata().GetUid(),
@@ -371,7 +376,13 @@ func (r *ActorTemplateReconciler) ensureActorExists(ctx context.Context, tmpl *a
if status.Code(err) != codes.NotFound {
return nil, fmt.Errorf("while getting golden actor: %w", err)
}
// Golden actor has not yet been created.
// Golden actor has not yet been created. Its reserved atespace is
// system-owned, so ensure it exists rather than assuming bootstrap did.
if _, err := r.control.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{
Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: goldenActorRef.GetAtespace()}},
}); err != nil && status.Code(err) != codes.AlreadyExists {
return nil, fmt.Errorf("while ensuring atespace %q: %w", goldenActorRef.GetAtespace(), err)
}
actor, err = r.control.CreateActor(ctx, &ateapipb.CreateActorRequest{
Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{
cmd/ateapi/internal/controlapi/template_reconciler_test.go
+19 -6
View File
@@ -160,9 +160,17 @@ type fakeGoldenControl struct {
// nil simulates a suspend that produced no ActorSnapshot.
snapshot *ateapipb.ObjectRef
createReqs []*ateapipb.CreateActorRequest
resumeReqs []*ateapipb.ResumeActorRequest
suspendReqs []*ateapipb.SuspendActorRequest
createReqs []*ateapipb.CreateActorRequest
resumeReqs []*ateapipb.ResumeActorRequest
suspendReqs []*ateapipb.SuspendActorRequest
atespaceReqs []*ateapipb.CreateAtespaceRequest
}
func (c *fakeGoldenControl) CreateAtespace(_ context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.atespaceReqs = append(c.atespaceReqs, req)
return req.GetAtespace(), nil
}
func (c *fakeGoldenControl) CreateActor(_ context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) {
@@ -543,7 +551,9 @@ func TestReconcileOne(t *testing.T) {
// TestReconcileOne_GoldenActorRequests pins the shape of the control-plane
// requests the happy path issues: the golden actor is named after the
// template UID so recreated templates with the same name never collide.
// template UID so recreated templates with the same name never collide, and
// lives in the reserved ate-golden atespace so the suspend workflow commits
// it Full regardless of the template's onCommit scope.
func TestReconcileOne_GoldenActorRequests(t *testing.T) {
ctx := context.Background()
st := newFakeTemplateStore(testTemplate())
@@ -558,12 +568,15 @@ func TestReconcileOne_GoldenActorRequests(t *testing.T) {
if got := created.GetMetadata().GetName(); got != testTemplateUID {
t.Errorf("golden actor name = %q, want template UID %q", got, testTemplateUID)
}
if got := created.GetMetadata().GetAtespace(); got != testAtespace {
t.Errorf("golden actor atespace = %q, want %q", got, testAtespace)
if got := created.GetMetadata().GetAtespace(); got != resources.GoldenActorAtespace {
t.Errorf("golden actor atespace = %q, want %q", got, resources.GoldenActorAtespace)
}
if got := created.GetActorTemplate().GetName(); got != testTemplateName {
t.Errorf("golden actor template ref = %q, want %q", got, testTemplateName)
}
if len(control.atespaceReqs) != 1 || control.atespaceReqs[0].GetAtespace().GetMetadata().GetName() != resources.GoldenActorAtespace {
t.Errorf("atespace ensure requests = %v, want one for %q", control.atespaceReqs, resources.GoldenActorAtespace)
}
if got := control.resumeReqs[0].GetActor().GetName(); got != testTemplateUID {
t.Errorf("resumed actor = %q, want %q", got, testTemplateUID)
}
hack/install-demo-counter-substrate.sh
+7 -3
View File
@@ -45,7 +45,7 @@ demo-counter-substrate_cmdline() {
demo-counter-substrate_deploy_variant \
demos/counter/counter-substrate.yaml.tmpl \
demos/counter/counter-substrate-template.yaml.tmpl \
ate-demo-counter-substrate counter-substrate counter
ate-demo-counter-substrate counter-substrate counter 300
;;
--delete-demo-counter-substrate)
demo-counter-substrate_delete_variant \
@@ -53,10 +53,13 @@ demo-counter-substrate_cmdline() {
ate-demo-counter-substrate counter
;;
--deploy-demo-counter-substrate-microvm)
# 600s golden budget: a micro-VM golden is a cloud-hypervisor cold boot
# plus checkpoint, on nested KVM in CI — the same budget the CRD demo's
# `kubectl wait` gets there.
demo-counter-substrate_deploy_variant \
demos/counter/counter-substrate-microvm.yaml.tmpl \
demos/counter/counter-substrate-microvm-template.yaml.tmpl \
ate-demo-counter-substrate-microvm counter-substrate-microvm counter-microvm
ate-demo-counter-substrate-microvm counter-substrate-microvm counter-microvm 600
;;
--delete-demo-counter-substrate-microvm)
demo-counter-substrate_delete_variant \
@@ -79,6 +82,7 @@ demo-counter-substrate_deploy_variant() {
local atespace="$3" # also the pool's k8s namespace
local pool="$4"
local template="$5"
local golden_timeout="${6:-300}"
log_step "demo-counter-substrate_deploy (${atespace}/${template})"
ensure_crds
@@ -114,7 +118,7 @@ demo-counter-substrate_deploy_variant() {
# `kubectl wait --for=condition=Ready actortemplate/...` (there is no
# kubectl wait for substrate resources).
log_step "Waiting for the ${atespace}/${template} golden snapshot..."
if ! wait_actortemplate_ready "${atespace}" "${template}" 300; then
if ! wait_actortemplate_ready "${atespace}" "${template}" "${golden_timeout}"; then
exit 1
fi
}
internal/ateattr/ateattr.go
+17 -2
View File
@@ -179,6 +179,10 @@ const (
OperationUnknown = "unknown"
)
// TemplateUnknown is the placeholder for the template labels when the Actor
// record does not carry its template ref.
const TemplateUnknown = "unknown"
// AllOperations lists all registered bounded actor lifecycle operations.
var AllOperations = []string{
OperationCreate,
@@ -363,10 +367,21 @@ func ActorMetricAttributes(a *ateapipb.Actor, sandboxClass, operationName, reaso
}
operationName = NormalizeOperationName(operationName)
// TODO(zoez7): actors created via the ActorTemplate resource path do not carry
// the template ref yet, so report "unknown" rather than an empty label.
templateNamespace := a.GetActorTemplateNamespace()
if templateNamespace == "" {
templateNamespace = TemplateUnknown
}
templateName := a.GetActorTemplateName()
if templateName == "" {
templateName = TemplateUnknown
}
ass := a.GetStatus().GetWorkerAssignment()
attrs := []attribute.KeyValue{
TemplateNamespaceKey.String(a.GetActorTemplateNamespace()),
TemplateNameKey.String(a.GetActorTemplateName()),
TemplateNamespaceKey.String(templateNamespace),
TemplateNameKey.String(templateName),
SandboxClassKey.String(sandboxClass),
ActorOperationNameKey.String(operationName),
FailureReasonKey.String(reason),
internal/ateattr/ateattr_test.go
+23
View File
@@ -391,6 +391,29 @@ func TestActorMetricAttributes(t *testing.T) {
assertAttrs(t, got, want)
})
t.Run("empty template ref reports unknown", func(t *testing.T) {
noTemplate := &ateapipb.Actor{
Status: &ateapipb.ActorStatus{
WorkerAssignment: &ateapipb.WorkerAssignment{
WorkerNamespace: "ate-workers",
WorkerPool: "default-pool",
},
},
}
got := toMap(ActorMetricAttributes(noTemplate, "gvisor", OperationResume, ReasonUnknown))
want := map[attribute.Key]any{
TemplateNamespaceKey: TemplateUnknown,
TemplateNameKey: TemplateUnknown,
WorkerPoolNamespaceKey: "ate-workers",
WorkerPoolNameKey: "default-pool",
SandboxClassKey: "gvisor",
ActorOperationNameKey: OperationResume,
FailureReasonKey: ReasonUnknown,
}
assertAttrs(t, got, want)
})
// An actor that crashed before it reached a worker has no pool. Reporting
// one key of the pair, or an empty-string name, would put that crash in a
// series that looks like a real pool.
internal/e2e/sandbox.go
+44 -15
View File
@@ -49,28 +49,57 @@ type Fixture struct {
DeployWith string
}
// CounterFixture returns the counter demo for the sandbox class under test.
// E2E_TEMPLATE_NAMESPACE / E2E_TEMPLATE_NAME override it, for a cluster that
// installs the fixture somewhere else.
func CounterFixture() Fixture {
f := Fixture{
Namespace: "ate-demo-counter",
Name: "counter",
DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter",
// SubstrateFixture identifies an installed substrate ActorTemplate (the proto
// resource created through the ate API, not the CRD) plus the CRD WorkerPool
// backing it. Suites copy the resolved runtime — container images, sandbox
// config, sandbox size — out of the template, and the ateom image and sandbox
// class out of the pool.
type SubstrateFixture struct {
// Atespace and Name locate the ActorTemplate for GetActorTemplate.
Atespace string
Name string
// PoolNamespace and PoolName locate the WorkerPool CRD.
PoolNamespace string
PoolName string
// DeployWith is the install flag or script that creates the fixture, so a
// missing one reports how to fix it rather than just failing.
DeployWith string
}
// SubstrateCounterFixture returns the substrate-resource counter demo for the
// sandbox class under test. E2E_SUBSTRATE_TEMPLATE_ATESPACE /
// E2E_SUBSTRATE_TEMPLATE_NAME / E2E_SUBSTRATE_POOL_NAMESPACE /
// E2E_SUBSTRATE_POOL_NAME override it, for a cluster that installs the
// fixture somewhere else.
func SubstrateCounterFixture() SubstrateFixture {
f := SubstrateFixture{
Atespace: "ate-demo-counter-substrate",
Name: "counter",
PoolNamespace: "ate-demo-counter-substrate",
PoolName: "counter-substrate",
DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter-substrate",
}
if IsMicroVM() {
f = Fixture{
Namespace: "ate-demo-counter-microvm",
Name: "counter-microvm",
DeployWith: "hack/run-microvm-demo-kind.sh",
f = SubstrateFixture{
Atespace: "ate-demo-counter-substrate-microvm",
Name: "counter-microvm",
PoolNamespace: "ate-demo-counter-substrate-microvm",
PoolName: "counter-substrate-microvm",
DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter-substrate-microvm",
}
}
if v := os.Getenv("E2E_TEMPLATE_NAMESPACE"); v != "" {
f.Namespace = v
if v := os.Getenv("E2E_SUBSTRATE_TEMPLATE_ATESPACE"); v != "" {
f.Atespace = v
}
if v := os.Getenv("E2E_TEMPLATE_NAME"); v != "" {
if v := os.Getenv("E2E_SUBSTRATE_TEMPLATE_NAME"); v != "" {
f.Name = v
}
if v := os.Getenv("E2E_SUBSTRATE_POOL_NAMESPACE"); v != "" {
f.PoolNamespace = v
}
if v := os.Getenv("E2E_SUBSTRATE_POOL_NAME"); v != "" {
f.PoolName = v
}
return f
}
internal/e2e/sandbox_test.go
+46 -12
View File
@@ -152,30 +152,64 @@ func TestRenderFixtureManifest_MicroVM(t *testing.T) {
}
}
// TestCounterFixture covers the knob every suite reads: the class picks the
// fixture, and the explicit environment overrides still win.
func TestCounterFixture(t *testing.T) {
// TestEgressFixture covers the knob the networking suite reads: the class
// picks the fixture.
func TestEgressFixture(t *testing.T) {
t.Run("gvisor", func(t *testing.T) {
t.Setenv(sandboxClassEnv, "")
if got := CounterFixture(); got.Namespace != "ate-demo-counter" || got.Name != "counter" {
t.Errorf("CounterFixture() = %+v, want the gVisor counter demo", got)
if got := EgressFixture(); got.Namespace != "ate-demo-egress" || got.Name != "egress" {
t.Errorf("EgressFixture() = %+v, want the gVisor egress demo", got)
}
})
t.Run("microvm", func(t *testing.T) {
t.Setenv(sandboxClassEnv, SandboxClassMicroVM)
if got := CounterFixture(); got.Namespace != "ate-demo-counter-microvm" || got.Name != "counter-microvm" {
t.Errorf("CounterFixture() = %+v, want the micro-VM counter demo", got)
}
if got := EgressFixture(); got.Namespace != "ate-demo-egress-microvm" || got.Name != "egress-microvm" {
t.Errorf("EgressFixture() = %+v, want the micro-VM egress demo", got)
}
})
}
// TestSubstrateCounterFixture covers the knob every counter-based suite reads:
// the class picks the fixture, and the explicit environment overrides still
// win.
func TestSubstrateCounterFixture(t *testing.T) {
t.Run("gvisor", func(t *testing.T) {
t.Setenv(sandboxClassEnv, "")
got := SubstrateCounterFixture()
want := SubstrateFixture{
Atespace: "ate-demo-counter-substrate",
Name: "counter",
PoolNamespace: "ate-demo-counter-substrate",
PoolName: "counter-substrate",
DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter-substrate",
}
if got != want {
t.Errorf("SubstrateCounterFixture() = %+v, want %+v", got, want)
}
})
t.Run("microvm", func(t *testing.T) {
t.Setenv(sandboxClassEnv, SandboxClassMicroVM)
got := SubstrateCounterFixture()
want := SubstrateFixture{
Atespace: "ate-demo-counter-substrate-microvm",
Name: "counter-microvm",
PoolNamespace: "ate-demo-counter-substrate-microvm",
PoolName: "counter-substrate-microvm",
DeployWith: "hack/install-ate-kind.sh --deploy-demo-counter-substrate-microvm",
}
if got != want {
t.Errorf("SubstrateCounterFixture() = %+v, want %+v", got, want)
}
})
t.Run("explicit override wins", func(t *testing.T) {
t.Setenv(sandboxClassEnv, SandboxClassMicroVM)
t.Setenv("E2E_TEMPLATE_NAMESPACE", "elsewhere")
t.Setenv("E2E_TEMPLATE_NAME", "other")
if got := CounterFixture(); got.Namespace != "elsewhere" || got.Name != "other" {
t.Errorf("CounterFixture() = %+v, want the environment override", got)
t.Setenv("E2E_SUBSTRATE_TEMPLATE_ATESPACE", "elsewhere")
t.Setenv("E2E_SUBSTRATE_TEMPLATE_NAME", "other")
t.Setenv("E2E_SUBSTRATE_POOL_NAMESPACE", "pool-ns")
t.Setenv("E2E_SUBSTRATE_POOL_NAME", "pool")
got := SubstrateCounterFixture()
if got.Atespace != "elsewhere" || got.Name != "other" || got.PoolNamespace != "pool-ns" || got.PoolName != "pool" {
t.Errorf("SubstrateCounterFixture() = %+v, want the environment overrides", got)
}
})
}
internal/e2e/suites/demo/demo_test.go
+125 -244
View File
@@ -26,11 +26,9 @@ import (
"github.com/agent-substrate/substrate/internal/ateclient"
"github.com/agent-substrate/substrate/internal/e2e"
"github.com/agent-substrate/substrate/internal/resources"
"github.com/agent-substrate/substrate/pkg/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/tools/portforward"
@@ -46,18 +44,15 @@ func TestActorLifecycle(t *testing.T) {
ctx := context.Background()
clients := e2e.GetClients()
// CreateActor requires the atespace to exist first.
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: demoAtespace}}})
// Create actor template.
at, err := createActorTemplate(ctx, t, clients, nsObj, v1alpha1.SnapshotScopeFull, v1alpha1.SnapshotScopeFull, v1alpha1.ResumeSourceColdBoot)
at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT)
if err != nil {
t.Fatalf("failed to initialize ActorTemplate: %v", err)
}
tests := []struct {
name string
f func(ctx context.Context, t *testing.T, clients *e2e.Clients, ns *e2e.Namespace, at *v1alpha1.ActorTemplate) error
f func(ctx context.Context, t *testing.T, clients *e2e.Clients, ns *e2e.Namespace, at *ateapipb.ActorTemplate) error
}{
{
name: "CreateActor",
@@ -96,10 +91,7 @@ func TestActorSnapshotLifecycle(t *testing.T) {
clients := e2e.GetClients()
nsObj := e2e.CreateNamespace(t)
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{
Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: demoAtespace}},
})
at, err := createActorTemplate(ctx, t, clients, nsObj, v1alpha1.SnapshotScopeFull, v1alpha1.SnapshotScopeFull, v1alpha1.ResumeSourceColdBoot)
at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT)
if err != nil {
t.Fatalf("failed to initialize ActorTemplate: %v", err)
}
@@ -107,7 +99,6 @@ func TestActorSnapshotLifecycle(t *testing.T) {
sourceName := "snapshot-source-" + nsObj.Name
cloneName := "snapshot-clone-" + nsObj.Name
for _, name := range []string{sourceName, cloneName} {
name := name
t.Cleanup(func() {
cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
@@ -117,9 +108,8 @@ func TestActorSnapshotLifecycle(t *testing.T) {
}
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: sourceName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: sourceName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create source Actor: %v", err)
}
@@ -183,10 +173,9 @@ func TestActorSnapshotLifecycle(t *testing.T) {
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{
Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: cloneName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
SourceSnapshotTag: tagRef,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: cloneName},
ActorTemplate: e2e.TemplateRef(at),
SourceSnapshotTag: tagRef,
},
}); err != nil {
t.Fatalf("failed to create Actor from snapshot tag: %v", err)
@@ -214,8 +203,8 @@ func TestDurableDirLifecycle(t *testing.T) {
{
name: "onCommit:Full, onPause:Full",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeFull,
onPause: v1alpha1.SnapshotScopeFull,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
wantMemoryAfterPause: 2,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 3,
@@ -225,8 +214,8 @@ func TestDurableDirLifecycle(t *testing.T) {
{
name: "onCommit:Data, onPause:Full",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeFull,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
wantMemoryAfterPause: 2,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -236,8 +225,8 @@ func TestDurableDirLifecycle(t *testing.T) {
{
name: "onCommit:Data, onPause:Data",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeData,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
wantMemoryAfterPause: 1,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -254,9 +243,9 @@ func TestDurableDirLifecycle(t *testing.T) {
// (the golden's own durable tar would read 0).
name: "onCommit:Data, onPause:Full, onResume.fromData:Golden",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeFull,
fromData: v1alpha1.ResumeSourceGolden,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN,
wantMemoryAfterPause: 2,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -271,9 +260,9 @@ func TestDurableDirLifecycle(t *testing.T) {
// snapshot (local checkpoint + external golden).
name: "onCommit:Data, onPause:Data, onResume.fromData:Golden",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeData,
fromData: v1alpha1.ResumeSourceGolden,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN,
wantMemoryAfterPause: 1,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -286,8 +275,8 @@ func TestDurableDirLifecycle(t *testing.T) {
// Suspend from PAUSED with matching Full scopes.
name: "onCommit:Full, onPause:Full, suspend from PAUSED",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeFull,
onPause: v1alpha1.SnapshotScopeFull,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
wantMemoryAfterPause: 2,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 3,
@@ -300,8 +289,8 @@ func TestDurableDirLifecycle(t *testing.T) {
// Suspend from PAUSED with matching Data scopes.
name: "onCommit:Data, onPause:Data, suspend from PAUSED",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeData,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
wantMemoryAfterPause: 1,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -315,8 +304,8 @@ func TestDurableDirLifecycle(t *testing.T) {
// mircoVM already implemnted, while gVisor is blocked by #790:
name: "onCommit:Data, onPause:Full, suspend from PAUSED",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeFull,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
wantMemoryAfterPause: 2,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -351,8 +340,8 @@ func TestMultipleDurableDirLifecycle(t *testing.T) {
{
name: "onCommit:Full, onPause:Full",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeFull,
onPause: v1alpha1.SnapshotScopeFull,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
wantMemoryAfterPause: 2,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 3,
@@ -363,8 +352,8 @@ func TestMultipleDurableDirLifecycle(t *testing.T) {
{
name: "onCommit:Data, onPause:Data",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeData,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
wantMemoryAfterPause: 1,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -379,9 +368,9 @@ func TestMultipleDurableDirLifecycle(t *testing.T) {
// (or read 0 — the golden guest was never called).
name: "onCommit:Data, onPause:Full, onResume.fromData:Golden",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeFull,
fromData: v1alpha1.ResumeSourceGolden,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN,
wantMemoryAfterPause: 2,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -413,8 +402,8 @@ func TestExternalVolumeLifecycle(t *testing.T) {
{
name: "onCommit:Data, onPause:Data",
tc: actorLifecycleTestCase{
onCommit: v1alpha1.SnapshotScopeData,
onPause: v1alpha1.SnapshotScopeData,
onCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
wantMemoryAfterPause: 1,
wantFileAfterPause: 2,
wantMemoryAfterSuspend: 1,
@@ -440,11 +429,7 @@ func TestDeleteActorAnyStateWithExternalVolume(t *testing.T) {
clients := e2e.GetClients()
nsObj := e2e.CreateNamespace(t)
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{
Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: demoAtespace}},
})
at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, v1alpha1.SnapshotScopeData, v1alpha1.SnapshotScopeData, v1alpha1.ResumeSourceColdBoot)
at, err := createActorTemplateWithExternalVolume(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT)
if err != nil {
t.Fatalf("failed to initialize ActorTemplate: %v", err)
}
@@ -453,9 +438,8 @@ func TestDeleteActorAnyStateWithExternalVolume(t *testing.T) {
t.Logf("Creating Actor %q...", actorName)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
}
@@ -511,9 +495,9 @@ func TestDeleteActorAnyStateWithExternalVolume(t *testing.T) {
// 5. Suspend & Resume actor.
// 6. Call to actor and validate memory and file counters.
type actorLifecycleTestCase struct {
onCommit v1alpha1.SnapshotScope
onPause v1alpha1.SnapshotScope
fromData v1alpha1.ResumeSource
onCommit ateapipb.SnapshotContentScope
onPause ateapipb.SnapshotContentScope
fromData ateapipb.ResumeSource
wantMemoryAfterPause int
wantFileAfterPause int
wantMemoryAfterSuspend int
@@ -543,16 +527,13 @@ type actorLifecycleTestCase struct {
suspendWhilePaused bool
}
func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(context.Context, *testing.T, *e2e.Clients, *e2e.Namespace, v1alpha1.SnapshotScope, v1alpha1.SnapshotScope, v1alpha1.ResumeSource) (*v1alpha1.ActorTemplate, error), tc actorLifecycleTestCase) {
func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(context.Context, *testing.T, *e2e.Clients, *e2e.Namespace, ateapipb.SnapshotContentScope, ateapipb.SnapshotContentScope, ateapipb.ResumeSource) (*ateapipb.ActorTemplate, error), tc actorLifecycleTestCase) {
// Create namespace
nsObj := e2e.CreateNamespace(t)
ctx := context.Background()
clients := e2e.GetClients()
// CreateActor requires the atespace to exist first.
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: demoAtespace}}})
// Create actor template.
at, err := createTemplate(ctx, t, clients, nsObj, tc.onCommit, tc.onPause, tc.fromData)
if err != nil {
@@ -566,9 +547,8 @@ func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(
t.Logf("Creating Actor %q using Substrate API...", actorID)
createResp, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID},
ActorTemplate: e2e.TemplateRef(at),
}})
if err != nil {
t.Fatalf("failed to create Actor: %v", err)
@@ -731,15 +711,14 @@ func validateCounterResponse(t *testing.T, resp string, stage string, wantMemory
}
}
func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *v1alpha1.ActorTemplate) error {
func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *ateapipb.ActorTemplate) error {
// Create an Actor using the ATE API.
actorName := "demo-actor-1-" + nsObj.Name
t.Logf("Creating Actor %q using Substrate API...", actorName)
createResp, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}})
if err != nil {
t.Fatalf("failed to create Actor: %v", err)
@@ -758,42 +737,41 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj
var myActors []*ateapipb.Actor
for _, actor := range listResp.GetActors() {
if actor.GetActorTemplateNamespace() == nsObj.Name && actor.GetMetadata().GetName() == actorName {
if actor.GetActorTemplate().GetName() == at.GetMetadata().GetName() && actor.GetMetadata().GetName() == actorName {
myActors = append(myActors, actor)
}
}
// Check that we have our Actor created.
if len(myActors) != 1 {
t.Fatalf("expected actor %s in namespace %s, got %d actors: %v", actorName, nsObj.Name, len(myActors), myActors)
t.Fatalf("expected actor %s from template %s, got %d actors: %v", actorName, at.GetMetadata().GetName(), len(myActors), myActors)
}
actor := myActors[0]
if actor.GetMetadata().GetName() != actorName {
t.Errorf("expected actor name %s, got %s", actorName, actor.GetMetadata().GetName())
}
if actor.GetActorTemplateName() != at.Name {
t.Errorf("expected actor template name %s, got %s", at.Name, actor.GetActorTemplateName())
if actor.GetActorTemplate().GetName() != at.GetMetadata().GetName() {
t.Errorf("expected actor template name %s, got %s", at.GetMetadata().GetName(), actor.GetActorTemplate().GetName())
}
if actor.Status.State != ateapipb.ActorState_ACTOR_STATE_SUSPENDED {
t.Errorf("expected actor state to be SUSPENDED, got %v", actor.Status.State)
}
t.Logf("Successfully queried Substrate API. Found %d active actors total, %d in our namespace %s.",
len(listResp.GetActors()), len(myActors), nsObj.Name)
t.Logf("Successfully queried Substrate API. Found %d active actors total, %d from our template %s.",
len(listResp.GetActors()), len(myActors), at.GetMetadata().GetName())
return nil
}
func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *v1alpha1.ActorTemplate) error {
func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *ateapipb.ActorTemplate) error {
actorName := "pause-actor-" + nsObj.Name
// Creating an actor
t.Logf("Creating Actor %q...", actorName)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
}
@@ -865,15 +843,14 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *
return nil
}
func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *v1alpha1.ActorTemplate) error {
func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *ateapipb.ActorTemplate) error {
actorName := "suspend-actor-" + nsObj.Name
// Creating an actor
t.Logf("Creating Actor %q...", actorName)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
}
@@ -944,15 +921,14 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj
return nil
}
func deleteActorAnyState(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *v1alpha1.ActorTemplate) error {
func deleteActorAnyState(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *ateapipb.ActorTemplate) error {
actorName := "anystate-delete-actor-" + nsObj.Name
// 1. Creating an actor
t.Logf("Creating Actor %q...", actorName)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
}
@@ -970,9 +946,8 @@ func deleteActorAnyState(ctx context.Context, t *testing.T, clients *e2e.Clients
// 2. Re-creating and Resuming the actor
t.Logf("Re-creating Actor %q...", actorName)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
}
@@ -1021,9 +996,8 @@ func deleteActorAnyState(ctx context.Context, t *testing.T, clients *e2e.Clients
// 7. Verify actor name can be immediately reused
t.Logf("Re-creating Actor %q to verify name reuse...", actorName)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to recreate Actor: %v", err)
}
@@ -1037,15 +1011,14 @@ func deleteActorAnyState(ctx context.Context, t *testing.T, clients *e2e.Clients
return nil
}
func deletePausedActorAnyState(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *v1alpha1.ActorTemplate) error {
func deletePausedActorAnyState(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *ateapipb.ActorTemplate) error {
actorName := "anystate-delete-paused-actor-" + nsObj.Name
// 1. Creating an actor
t.Logf("Creating Actor %q...", actorName)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
}
@@ -1088,122 +1061,37 @@ func deletePausedActorAnyState(ctx context.Context, t *testing.T, clients *e2e.C
return nil
}
func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, name string, onCommit, onPause v1alpha1.SnapshotScope, fromData v1alpha1.ResumeSource, modifyTemplate func(*v1alpha1.ActorTemplate)) (*v1alpha1.ActorTemplate, error) {
env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO")
func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, base string, onCommit, onPause ateapipb.SnapshotContentScope, fromData ateapipb.ResumeSource, modifyTemplate func(*ateapipb.ActorTemplate)) (*ateapipb.ActorTemplate, error) {
env, err := e2e.CheckEnv("BUCKET_NAME")
if err != nil {
t.Fatalf("CheckEnv failed: %v", err)
}
// The source WorkerPool+ActorTemplate to copy the resolved runtime (sandbox
// class, ateom image, container images, sandbox size) from: the counter demo
// for the sandbox class under test, so this one lifecycle test covers both.
src := e2e.CounterFixture()
srcNS, srcName := src.Namespace, src.Name
// Query existing WorkerPool and ActorTemplate to get the resolved container images
existingWp, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(srcNS).Get(ctx, srcName, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get existing WorkerPool %s/%s: %v", srcNS, srcName, err)
}
existingAt, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(srcNS).Get(ctx, srcName, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get existing ActorTemplate %s/%s: %v", srcNS, srcName, err)
}
// Create WorkerPool. Labeled uniquely to this test's namespace so the
// cluster-wide scheduler doesn't make this pool's workers eligible for
// (or eligible to receive) any other namespace's actors.
wp := &v1alpha1.WorkerPool{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: nsObj.Name,
Labels: map[string]string{"demo": nsObj.Name},
// The whole suite shares demoAtespace, so the per-test suffix keeps
// template names unique.
name := base + "-" + nsObj.Name
at := e2e.CreateSubstrateCounterTemplate(ctx, t, clients, nsObj.Name, e2e.SubstrateCounterTemplateOptions{
Atespace: demoAtespace,
Name: name,
PoolName: base,
PoolReplicas: 5,
Labels: map[string]string{"demo": nsObj.Name},
SnapshotsConfig: &ateapipb.SnapshotsConfig{
StorageLocation: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name,
OnPause: onPause,
OnCommit: onCommit,
OnResume: &ateapipb.OnResumeConfig{FromData: fromData},
},
Spec: v1alpha1.WorkerPoolSpec{
Replicas: 5,
WorkerImage: existingWp.Spec.WorkerImage,
SandboxClass: existingWp.Spec.SandboxClass,
SandboxConfigName: existingWp.Spec.SandboxConfigName,
},
}
_, err = clients.SubstrateK8s.ApiV1alpha1().WorkerPools(nsObj.Name).Create(ctx, wp, metav1.CreateOptions{})
if err != nil {
t.Fatalf("failed to create WorkerPool: %v", err)
}
// Create ActorTemplate
at := &v1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: nsObj.Name,
},
Spec: v1alpha1.ActorTemplateSpec{
WorkerSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"demo": nsObj.Name},
},
// SandboxClass must match the per-test WorkerPool's (copied above) so the
// ActorTemplate↔WorkerPool match succeeds. The micro-VM source sets
// "microvm"; the gVisor source leaves it "" — copying keeps both correct.
SandboxClass: existingAt.Spec.SandboxClass,
Containers: existingAt.Spec.Containers,
// The source's limits size the sandbox. Copying them matters most on
// micro-VM, where an ActorTemplate that declares none boots the guest
// at the kata config default (2GiB) instead of the demo's 512Mi.
Resources: existingAt.Spec.Resources,
SnapshotsConfig: v1alpha1.SnapshotsConfig{
Location: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name,
OnPause: onPause,
OnCommit: onCommit,
OnResume: v1alpha1.OnResumeConfig{FromData: fromData},
},
Volumes: existingAt.Spec.Volumes,
},
}
if modifyTemplate != nil {
modifyTemplate(at)
}
_, err = clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(nsObj.Name).Create(ctx, at, metav1.CreateOptions{})
if err != nil {
t.Fatalf("failed to create ActorTemplate: %v", err)
}
// Wait for ActorTemplate to be Ready (golden snapshot created) before creating
// an actor. TemplateReadyTimeout budgets for the micro-VM golden (a CH cold
// boot plus checkpoint on nested KVM) being slower than the gVisor one.
t.Logf("Waiting for ActorTemplate %s to be Ready...", at.Name)
tmplTimeout := e2e.TemplateReadyTimeout(t)
tmplCtx, tmplCancel := context.WithTimeout(ctx, tmplTimeout)
defer tmplCancel()
var lastPhase v1alpha1.PhaseType
for {
curAt, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(nsObj.Name).Get(tmplCtx, at.Name, metav1.GetOptions{})
if err == nil {
lastPhase = curAt.Status.Phase
if lastPhase == v1alpha1.PhaseReady {
t.Logf("ActorTemplate %s is Ready with golden snapshot %q", at.Name, curAt.Status.GoldenSnapshot)
break
}
if lastPhase == v1alpha1.PhaseFailed {
t.Fatalf("ActorTemplate %s transitioned to PhaseFailed!", at.Name)
}
}
select {
case <-tmplCtx.Done():
t.Fatalf("Timed out waiting for ActorTemplate %q to be Ready after %v (last phase: %s, err: %v)", at.Name, tmplTimeout, lastPhase, err)
case <-time.After(1 * time.Second):
// Keep polling.
}
}
Modify: modifyTemplate,
})
return at, nil
}
func createActorTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit, onPause v1alpha1.SnapshotScope, fromData v1alpha1.ResumeSource) (*v1alpha1.ActorTemplate, error) {
func createActorTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit, onPause ateapipb.SnapshotContentScope, fromData ateapipb.ResumeSource) (*ateapipb.ActorTemplate, error) {
return createActorTemplateInternal(ctx, t, clients, nsObj, "counter", onCommit, onPause, fromData, nil)
}
func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit, onPause v1alpha1.SnapshotScope, fromData v1alpha1.ResumeSource) (*v1alpha1.ActorTemplate, error) {
func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit, onPause ateapipb.SnapshotContentScope, fromData ateapipb.ResumeSource) (*ateapipb.ActorTemplate, error) {
var scName string
switch {
// TODO: add support for other storage classes in e2e environment (e.g. csi-nfs-sc)
@@ -1213,45 +1101,42 @@ func createActorTemplateWithExternalVolume(ctx context.Context, t *testing.T, cl
t.Skip("Skipping TestExternalVolumeLifecycle: neither csi-hostpath-sc nor csi-nfs-sc StorageClass found")
}
modify := func(at *v1alpha1.ActorTemplate) {
var res []v1alpha1.Container
for _, c := range at.Spec.Containers {
if c.Name == "counter" {
c.Command = []string{"/ko-app/counter", "--file-counter-directory=/external-data"}
modify := func(at *ateapipb.ActorTemplate) {
for _, c := range at.GetContainers() {
if c.GetName() != "counter" {
continue
}
c.Command = []string{"/ko-app/counter", "--file-counter-directory=/external-data"}
hasExtMount := false
for _, vm := range c.VolumeMounts {
if vm.Name == "external-data" {
hasExtMount = true
break
}
}
if !hasExtMount {
c.VolumeMounts = append(c.VolumeMounts, v1alpha1.VolumeMount{
Name: "external-data",
MountPath: "/external-data",
})
hasExtMount := false
for _, vm := range c.GetVolumeMounts() {
if vm.GetName() == "external-data" {
hasExtMount = true
break
}
}
res = append(res, c)
if !hasExtMount {
c.VolumeMounts = append(c.VolumeMounts, &ateapipb.VolumeMount{
Name: "external-data",
MountPath: "/external-data",
})
}
}
at.Spec.Containers = res
hasExtVol := false
for _, v := range at.Spec.Volumes {
if v.Name == "external-data" {
for _, v := range at.GetVolumes() {
if v.GetName() == "external-data" {
hasExtVol = true
break
}
}
if !hasExtVol {
at.Spec.Volumes = append(at.Spec.Volumes, v1alpha1.Volume{
at.Volumes = append(at.Volumes, &ateapipb.Volume{
Name: "external-data",
VolumeSource: v1alpha1.VolumeSource{
ExternalVolumeTemplate: &v1alpha1.ExternalVolumeTemplate{
Capacity: resource.MustParse("1Gi"),
StorageClassName: scName,
},
Type: "ExternalVolumeTemplate",
ExternalVolumeTemplate: &ateapipb.ExternalVolumeTemplate{
Capacity: "1Gi",
StorageClassName: scName,
},
})
}
@@ -1271,22 +1156,22 @@ const (
// counter's second file counter at it, so both volumes are written on every
// request. Only the micro-VM runtime accepts this: gVisor templates are still
// capped at one durable-dir volume by the ActorTemplate CEL rules.
func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit, onPause v1alpha1.SnapshotScope, fromData v1alpha1.ResumeSource) (*v1alpha1.ActorTemplate, error) {
modify := func(at *v1alpha1.ActorTemplate) {
for i, c := range at.Spec.Containers {
if c.Name != "counter" {
func createActorTemplateWithTwoDurableDirs(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, onCommit, onPause ateapipb.SnapshotContentScope, fromData ateapipb.ResumeSource) (*ateapipb.ActorTemplate, error) {
modify := func(at *ateapipb.ActorTemplate) {
for _, c := range at.GetContainers() {
if c.GetName() != "counter" {
continue
}
c.Command = []string{"/ko-app/counter", "--second-file-counter-directory=" + secondDurableDirMountPath}
c.VolumeMounts = append(c.VolumeMounts, v1alpha1.VolumeMount{
c.VolumeMounts = append(c.VolumeMounts, &ateapipb.VolumeMount{
Name: secondDurableDirVolume,
MountPath: secondDurableDirMountPath,
})
at.Spec.Containers[i] = c
}
at.Spec.Volumes = append(at.Spec.Volumes, v1alpha1.Volume{
Name: secondDurableDirVolume,
VolumeSource: v1alpha1.VolumeSource{DurableDir: &v1alpha1.DurableDirVolumeSource{}},
at.Volumes = append(at.Volumes, &ateapipb.Volume{
Name: secondDurableDirVolume,
Type: "DurableDir",
DurableDir: &ateapipb.DurableDirVolumeSource{},
})
}
return createActorTemplateInternal(ctx, t, clients, nsObj, "counter-two-durabledirs", onCommit, onPause, fromData, modify)
@@ -1440,11 +1325,8 @@ func TestWorkerPodDeletion(t *testing.T) {
ctx := context.Background()
clients := e2e.GetClients()
// CreateActor requires the atespace to exist first.
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: demoAtespace}}})
// Create actor template.
at, err := createActorTemplate(ctx, t, clients, nsObj, v1alpha1.SnapshotScopeFull, v1alpha1.SnapshotScopeFull, v1alpha1.ResumeSourceColdBoot)
at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT)
if err != nil {
t.Fatalf("failed to initialize ActorTemplate: %v", err)
}
@@ -1454,9 +1336,8 @@ func TestWorkerPodDeletion(t *testing.T) {
// Creating an actor
t.Logf("Creating Actor %q...", actorName)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
}
internal/e2e/suites/demo/termination_test.go
+9 -19
View File
@@ -21,7 +21,6 @@ import (
"github.com/agent-substrate/substrate/internal/e2e"
"github.com/agent-substrate/substrate/internal/resources"
"github.com/agent-substrate/substrate/pkg/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -42,9 +41,7 @@ func TestGracefulWorkerTermination(t *testing.T) {
ctx := context.Background()
clients := e2e.GetClients()
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: demoAtespace}}})
at, err := createActorTemplate(ctx, t, clients, nsObj, v1alpha1.SnapshotScopeFull, v1alpha1.SnapshotScopeFull, v1alpha1.ResumeSourceColdBoot)
at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT)
if err != nil {
t.Fatalf("failed to initialize ActorTemplate: %v", err)
}
@@ -52,9 +49,8 @@ func TestGracefulWorkerTermination(t *testing.T) {
actorID := "graceful-term-" + nsObj.Name
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{
Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID},
ActorTemplate: e2e.TemplateRef(at),
},
}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
@@ -153,9 +149,7 @@ func TestGracefulWorkerTerminationTimeout(t *testing.T) {
ctx := context.Background()
clients := e2e.GetClients()
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: demoAtespace}}})
at, err := createActorTemplate(ctx, t, clients, nsObj, v1alpha1.SnapshotScopeFull, v1alpha1.SnapshotScopeFull, v1alpha1.ResumeSourceColdBoot)
at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT)
if err != nil {
t.Fatalf("failed to initialize ActorTemplate: %v", err)
}
@@ -163,9 +157,8 @@ func TestGracefulWorkerTerminationTimeout(t *testing.T) {
actorID := "graceful-term-timeout-" + nsObj.Name
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{
Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID},
ActorTemplate: e2e.TemplateRef(at),
},
}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
@@ -240,9 +233,7 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) {
ctx := context.Background()
clients := e2e.GetClients()
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: demoAtespace}}})
at, err := createActorTemplate(ctx, t, clients, nsObj, v1alpha1.SnapshotScopeFull, v1alpha1.SnapshotScopeFull, v1alpha1.ResumeSourceColdBoot)
at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT)
if err != nil {
t.Fatalf("failed to initialize ActorTemplate: %v", err)
}
@@ -250,9 +241,8 @@ func TestGracefulWorkerTerminationSuspend(t *testing.T) {
actorID := "graceful-term-suspend-" + nsObj.Name
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{
Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID},
ActorTemplate: e2e.TemplateRef(at),
},
}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
internal/e2e/suites/metrics/metrics_test.go
+5 -6
View File
@@ -16,8 +16,8 @@
// the platform metrics in e2e.PlatformMetricPrefixes reach the kind stack's OTel
// Collector. It closes the "silent regression" gap: a renamed or dropped
// instrument fails here rather than surfacing as an empty dashboard. The prefix
// set grows as each metric slice lands. Requires the demo counter template for
// the sandbox class under test to be installed (see e2e.CounterFixture).
// set grows as each metric slice lands. Requires the substrate counter demo for
// the sandbox class under test to be installed (see e2e.SubstrateCounterFixture).
package metrics
import (
@@ -40,7 +40,7 @@ const metricsAtespace = "ate-metrics-e2e"
func TestPlatformMetricsEmitted(t *testing.T) {
ctx := context.Background()
clients := e2e.GetClients()
tmpl := e2e.CounterFixture()
tmpl := e2e.SubstrateCounterFixture()
actorID := fmt.Sprintf("metrics-probe-%d", time.Now().UnixNano())
// CreateActor requires the atespace to exist first; ignore AlreadyExists.
@@ -49,9 +49,8 @@ func TestPlatformMetricsEmitted(t *testing.T) {
})
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: metricsAtespace, Name: actorID},
ActorTemplateNamespace: tmpl.Namespace,
ActorTemplateName: tmpl.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: metricsAtespace, Name: actorID},
ActorTemplate: &ateapipb.ObjectRef{Atespace: tmpl.Atespace, Name: tmpl.Name},
}}); err != nil {
t.Fatalf("CreateActor: %v", err)
}
internal/e2e/suites/networking/arbitraryport_test.go
+1 -1
View File
@@ -50,7 +50,7 @@ const counterExtraPort = 9090
// resolution and atunnel's dial to the actor's pod actually happen.
func TestActorArbitraryPortAccess(t *testing.T) {
ctx := context.Background()
actorName, _ := createAndResumeActor(t, ctx, "arbitraryport", e2e.CounterFixture())
actorName, _ := createAndResumeSubstrateActor(t, ctx, "arbitraryport", e2e.SubstrateCounterFixture())
actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName}
router := mustRouterClient(t, ctx)
defer router.Close()
internal/e2e/suites/networking/grpcingress_test.go
+1 -1
View File
@@ -54,7 +54,7 @@ const grpcEchoFixtureManifest = "internal/e2e/fixtures/testserver/grpcecho.yaml.
// actor that really does speak gRPC.
func TestIngressProtocolDowngrade(t *testing.T) {
ctx := context.Background()
actorName, _ := createAndResumeActor(t, ctx, "protodowngrade", e2e.CounterFixture())
actorName, _ := createAndResumeSubstrateActor(t, ctx, "protodowngrade", e2e.SubstrateCounterFixture())
actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName}
base := "http://" + routerAddress(t, ctx)
internal/e2e/suites/networking/networking_test.go
+18 -7
View File
@@ -66,7 +66,7 @@ func egressFixture() e2e.Fixture {
func TestActorDirectAccess(t *testing.T) {
ctx := context.Background()
actorName, actor := createAndResumeActor(t, ctx, "direct", e2e.CounterFixture())
actorName, actor := createAndResumeSubstrateActor(t, ctx, "direct", e2e.SubstrateCounterFixture())
router := mustRouterClient(t, ctx)
defer router.Close()
@@ -309,6 +309,20 @@ func accessLogField(line, key string) (string, bool) {
}
func createAndResumeActor(t *testing.T, ctx context.Context, prefix string, template e2e.Fixture) (string, *ateapipb.Actor) {
t.Helper()
actor := &ateapipb.Actor{ActorTemplateNamespace: template.Namespace, ActorTemplateName: template.Name}
return createAndResume(t, ctx, prefix, actor, template.Namespace+"/"+template.Name, template.DeployWith)
}
// createAndResumeSubstrateActor is createAndResumeActor for a substrate
// ActorTemplate fixture, referenced by atespace/name instead of the CRD pair.
func createAndResumeSubstrateActor(t *testing.T, ctx context.Context, prefix string, template e2e.SubstrateFixture) (string, *ateapipb.Actor) {
t.Helper()
actor := &ateapipb.Actor{ActorTemplate: &ateapipb.ObjectRef{Atespace: template.Atespace, Name: template.Name}}
return createAndResume(t, ctx, prefix, actor, template.Atespace+"/"+template.Name, template.DeployWith)
}
func createAndResume(t *testing.T, ctx context.Context, prefix string, actor *ateapipb.Actor, source, deployWith string) (string, *ateapipb.Actor) {
t.Helper()
clients := e2e.GetClients()
actorName := fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())
@@ -318,12 +332,9 @@ func createAndResumeActor(t *testing.T, ctx context.Context, prefix string, temp
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{
Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: networkingAtespace}},
})
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: networkingAtespace, Name: actorName},
ActorTemplateNamespace: template.Namespace,
ActorTemplateName: template.Name,
}}); err != nil {
t.Fatalf("CreateActor from %s/%s: %v (deploy the fixture with %s)", template.Namespace, template.Name, err, template.DeployWith)
actor.Metadata = &ateapipb.ResourceMetadata{Atespace: networkingAtespace, Name: actorName}
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: actor}); err != nil {
t.Fatalf("CreateActor from %s: %v (deploy the fixture with %s)", source, err, deployWith)
}
t.Cleanup(func() {
_, _ = clients.SubstrateAPI.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef})
internal/e2e/suites/networkpolicy/networkpolicy_test.go
+23 -94
View File
@@ -143,23 +143,16 @@ func TestNetworkPolicyDataPlaneEnforcement(t *testing.T) {
ctx := context.Background()
clients := e2e.GetClients()
// Setup WorkerPool and ActorTemplate from the standard counter demo
wp, at := setupDemoCounterTemplate(ctx, t, clients, nsObj.Name)
// Create Atespace
if _, err := clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{
Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: nsObj.Name}},
}); err != nil {
t.Fatalf("failed to create atespace: %v", err)
}
// Setup WorkerPool and ActorTemplate from the substrate counter demo (the
// template's atespace, named after the test namespace, is created there).
poolName, at := setupDemoCounterTemplate(ctx, t, clients, nsObj.Name)
// Create and Resume Actor
actorName := "netpol-dataplane-" + nsObj.Name
t.Logf("Creating Actor %q in Atespace %q...", actorName, nsObj.Name)
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: nsObj.Name, Name: actorName},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: nsObj.Name, Name: actorName},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create Actor: %v", err)
}
@@ -218,10 +211,10 @@ func TestNetworkPolicyDataPlaneEnforcement(t *testing.T) {
// Find the IP address of the worker pod backing our WorkerPool
pods, err := clients.K8s.CoreV1().Pods(nsObj.Name).List(ctx, metav1.ListOptions{
LabelSelector: fmt.Sprintf("ate.dev/worker-pool=%s", wp.Name),
LabelSelector: fmt.Sprintf("ate.dev/worker-pool=%s", poolName),
})
if err != nil || len(pods.Items) == 0 {
t.Fatalf("failed to list worker pods for worker pool %q: %v", wp.Name, err)
t.Fatalf("failed to list worker pods for worker pool %q: %v", poolName, err)
}
var workerIP string
for i := range pods.Items {
@@ -232,7 +225,7 @@ func TestNetworkPolicyDataPlaneEnforcement(t *testing.T) {
}
}
if workerIP == "" {
t.Fatalf("no running worker pod with IP found for worker pool %q", wp.Name)
t.Fatalf("no running worker pod with IP found for worker pool %q", poolName)
}
// Deploy an unauthorized probe pod in an external test namespace
@@ -277,86 +270,22 @@ func TestNetworkPolicyDataPlaneEnforcement(t *testing.T) {
t.Logf("Negative Data Plane Verification PASSED: Unauthorized connection attempt from %s/%s to %s:8080 was blocked as expected (err: %v)", rogueNsObj.Name, probePod.Name, workerIP, err)
}
func setupDemoCounterTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, ns string) (*v1alpha1.WorkerPool, *v1alpha1.ActorTemplate) {
// setupDemoCounterTemplate provisions the per-test WorkerPool and substrate
// ActorTemplate from the substrate counter demo, returning the pool name and
// the template. The template lives in an atespace named after the test's k8s
// namespace, so its name needs no per-test suffix. SnapshotsConfig is copied
// from the source, as the CRD-era setup did.
func setupDemoCounterTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, ns string) (string, *ateapipb.ActorTemplate) {
t.Helper()
src := e2e.CounterFixture()
srcNS, srcName := src.Namespace, src.Name
existingWp, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(srcNS).Get(ctx, srcName, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get existing WorkerPool %s/%s: %v", srcNS, srcName, err)
}
existingAt, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(srcNS).Get(ctx, srcName, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get existing ActorTemplate %s/%s: %v", srcNS, srcName, err)
}
wp := &v1alpha1.WorkerPool{
ObjectMeta: metav1.ObjectMeta{
Name: "counter",
Namespace: ns,
Labels: map[string]string{"netpol-test": ns},
},
Spec: v1alpha1.WorkerPoolSpec{
Replicas: 1,
WorkerImage: existingWp.Spec.WorkerImage,
SandboxClass: existingWp.Spec.SandboxClass,
SandboxConfigName: existingWp.Spec.SandboxConfigName,
},
}
if _, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(ns).Create(ctx, wp, metav1.CreateOptions{}); err != nil {
t.Fatalf("failed to create WorkerPool: %v", err)
}
at := &v1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: "counter",
Namespace: ns,
},
Spec: v1alpha1.ActorTemplateSpec{
WorkerSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"netpol-test": ns},
},
SandboxClass: existingAt.Spec.SandboxClass,
Containers: existingAt.Spec.Containers,
// The source's limits size the sandbox. Copying them matters most on
// micro-VM, where an ActorTemplate that declares none boots the guest
// at the kata config default (2GiB) instead of the demo's 512Mi.
Resources: existingAt.Spec.Resources,
SnapshotsConfig: existingAt.Spec.SnapshotsConfig,
Volumes: existingAt.Spec.Volumes,
},
}
if _, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(ns).Create(ctx, at, metav1.CreateOptions{}); err != nil {
t.Fatalf("failed to create ActorTemplate: %v", err)
}
t.Logf("Waiting for ActorTemplate %s/%s to be Ready...", ns, at.Name)
tmplTimeout := e2e.TemplateReadyTimeout(t)
tmplCtx, tmplCancel := context.WithTimeout(ctx, tmplTimeout)
defer tmplCancel()
var lastPhase v1alpha1.PhaseType
for {
curAt, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(ns).Get(tmplCtx, at.Name, metav1.GetOptions{})
if err == nil {
lastPhase = curAt.Status.Phase
if lastPhase == v1alpha1.PhaseReady {
t.Logf("ActorTemplate %s/%s is Ready with golden snapshot %q", ns, at.Name, curAt.Status.GoldenSnapshot)
break
}
if lastPhase == v1alpha1.PhaseFailed {
t.Fatalf("ActorTemplate %s/%s transitioned to PhaseFailed!", ns, at.Name)
}
}
select {
case <-tmplCtx.Done():
t.Fatalf("Timed out waiting for ActorTemplate %q to be Ready after %v (last phase: %s, err: %v)", at.Name, tmplTimeout, lastPhase, err)
case <-time.After(1 * time.Second):
}
}
return wp, at
const poolName = "counter"
at := e2e.CreateSubstrateCounterTemplate(ctx, t, clients, ns, e2e.SubstrateCounterTemplateOptions{
Atespace: ns,
Name: "counter",
PoolName: poolName,
PoolReplicas: 1,
Labels: map[string]string{"netpol-test": ns},
})
return poolName, at
}
func waitForActorRunning(ctx context.Context, t *testing.T, clients *e2e.Clients, atespace, actorName string) {
internal/e2e/suites/parking/parking_test.go
+20 -91
View File
@@ -30,11 +30,8 @@ import (
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/agent-substrate/substrate/internal/e2e"
"github.com/agent-substrate/substrate/internal/resources"
v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
)
@@ -54,14 +51,10 @@ func TestRequestParking(t *testing.T) {
// One worker, two actors: the minimal deterministic oversubscription.
at := createParkingFixture(ctx, t, clients, nsObj)
_, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{
Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: parkingAtespace}},
})
actorA := "parked-a-" + nsObj.Name
actorB := "parked-b-" + nsObj.Name
for _, name := range []string{actorA, actorB} {
createActor(ctx, t, clients, nsObj, at, name)
createActor(ctx, t, clients, at, name)
}
router, err := e2e.NewRouterClient(ctx)
@@ -192,100 +185,36 @@ func TestRequestParking(t *testing.T) {
})
}
// createParkingFixture provisions a 1-worker pool and an ActorTemplate in the
// test namespace, copying the resolved runtime (sandbox class, ateom image,
// container images) from the installed counter demo — the same source and
// isolation pattern as the demo suite: the unique pool label keeps this pool's
// worker invisible to other namespaces' actors.
func createParkingFixture(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace) *v1alpha1.ActorTemplate {
// createParkingFixture provisions a 1-worker pool and a substrate
// ActorTemplate, copying the resolved runtime (sandbox config, ateom image,
// container images) from the installed substrate counter demo — the same
// source and isolation pattern as the demo suite: the unique pool label keeps
// this pool's worker invisible to other namespaces' actors.
func createParkingFixture(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace) *ateapipb.ActorTemplate {
t.Helper()
env, err := e2e.CheckEnv("BUCKET_NAME")
if err != nil {
t.Fatalf("CheckEnv failed: %v", err)
}
src := e2e.CounterFixture()
srcNS, srcName := src.Namespace, src.Name
existingWp, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(srcNS).Get(ctx, srcName, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get source WorkerPool %s/%s: %v", srcNS, srcName, err)
}
existingAt, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(srcNS).Get(ctx, srcName, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get source ActorTemplate %s/%s: %v", srcNS, srcName, err)
}
wp := &v1alpha1.WorkerPool{
ObjectMeta: metav1.ObjectMeta{
Name: "parking",
Namespace: nsObj.Name,
Labels: map[string]string{"demo": nsObj.Name},
return e2e.CreateSubstrateCounterTemplate(ctx, t, clients, nsObj.Name, e2e.SubstrateCounterTemplateOptions{
Atespace: parkingAtespace,
// Unique within the suite-shared atespace.
Name: "parking-" + nsObj.Name,
PoolName: "parking",
PoolReplicas: 1, // deliberately undersized: 2 actors will contend for it
Labels: map[string]string{"demo": nsObj.Name},
SnapshotsConfig: &ateapipb.SnapshotsConfig{
StorageLocation: "gs://" + env["BUCKET_NAME"] + "/e2e-parking-" + nsObj.Name,
},
Spec: v1alpha1.WorkerPoolSpec{
Replicas: 1, // deliberately undersized: 2 actors will contend for it
WorkerImage: existingWp.Spec.WorkerImage,
SandboxClass: existingWp.Spec.SandboxClass,
SandboxConfigName: existingWp.Spec.SandboxConfigName,
},
}
if _, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(nsObj.Name).Create(ctx, wp, metav1.CreateOptions{}); err != nil {
t.Fatalf("failed to create WorkerPool: %v", err)
}
at := &v1alpha1.ActorTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: "parking",
Namespace: nsObj.Name,
},
Spec: v1alpha1.ActorTemplateSpec{
WorkerSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"demo": nsObj.Name},
},
SandboxClass: existingAt.Spec.SandboxClass,
Containers: existingAt.Spec.Containers,
// The source's limits size the sandbox. Copying them matters most on
// micro-VM, where an ActorTemplate that declares none boots the guest
// at the kata config default (2GiB) instead of the demo's 512Mi.
Resources: existingAt.Spec.Resources,
SnapshotsConfig: v1alpha1.SnapshotsConfig{
Location: "gs://" + env["BUCKET_NAME"] + "/e2e-parking-" + nsObj.Name,
},
Volumes: existingAt.Spec.Volumes,
},
}
if _, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(nsObj.Name).Create(ctx, at, metav1.CreateOptions{}); err != nil {
t.Fatalf("failed to create ActorTemplate: %v", err)
}
t.Logf("Waiting for ActorTemplate %s to be Ready...", at.Name)
tmplCtx, tmplCancel := context.WithTimeout(ctx, e2e.TemplateReadyTimeout(t))
defer tmplCancel()
var lastPhase v1alpha1.PhaseType
for {
curAt, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(nsObj.Name).Get(tmplCtx, at.Name, metav1.GetOptions{})
if err == nil {
lastPhase = curAt.Status.Phase
if lastPhase == v1alpha1.PhaseReady {
return at
}
if lastPhase == v1alpha1.PhaseFailed {
t.Fatalf("ActorTemplate %s transitioned to PhaseFailed", at.Name)
}
}
select {
case <-tmplCtx.Done():
t.Fatalf("timed out waiting for ActorTemplate %q to be Ready (last phase: %s, err: %v)", at.Name, lastPhase, err)
case <-time.After(1 * time.Second):
}
}
})
}
func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, at *v1alpha1.ActorTemplate, name string) {
func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, at *ateapipb.ActorTemplate, name string) {
t.Helper()
if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Atespace: parkingAtespace, Name: name},
ActorTemplateNamespace: nsObj.Name,
ActorTemplateName: at.Name,
Metadata: &ateapipb.ResourceMetadata{Atespace: parkingAtespace, Name: name},
ActorTemplate: e2e.TemplateRef(at),
}}); err != nil {
t.Fatalf("failed to create actor %q: %v", name, err)
}
internal/e2e/template.go
+150
View File
@@ -21,6 +21,9 @@ import (
"time"
"github.com/agent-substrate/substrate/pkg/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -60,3 +63,150 @@ func WaitForTemplateReady(ctx context.Context, t *testing.T, clients *Clients, n
}
}
}
// TemplateRef builds the substrate template reference an Actor carries.
func TemplateRef(at *ateapipb.ActorTemplate) *ateapipb.ObjectRef {
return &ateapipb.ObjectRef{Atespace: at.GetMetadata().GetAtespace(), Name: at.GetMetadata().GetName()}
}
// SubstrateCounterTemplateOptions shapes CreateSubstrateCounterTemplate.
type SubstrateCounterTemplateOptions struct {
// Atespace and Name locate the new template. The atespace is created if
// missing; Name must be unique within it (atespaces are shared across a
// suite's tests, unlike the k8s namespaces the CRD templates lived in).
Atespace string
Name string
// PoolName and PoolReplicas shape the WorkerPool CRD created in the test's
// k8s namespace.
PoolName string
PoolReplicas int32
// Labels tie the template's workerSelector to the pool, keeping this
// pool's workers invisible to other namespaces' actors.
Labels map[string]string
// SnapshotsConfig for the new template; nil copies the source's.
SnapshotsConfig *ateapipb.SnapshotsConfig
// Modify, when set, edits the template before it is created.
Modify func(*ateapipb.ActorTemplate)
}
// CreateSubstrateCounterTemplate creates a per-test WorkerPool CRD plus a
// substrate ActorTemplate copying the resolved runtime (sandbox config, ateom
// image, container images, sandbox size) from the substrate counter demo for
// the sandbox class under test. It registers cleanup of the template (which
// does not ride the k8s namespace GC the CRD templates did) and blocks until
// the golden snapshot exists.
func CreateSubstrateCounterTemplate(ctx context.Context, t *testing.T, clients *Clients, namespace string, opts SubstrateCounterTemplateOptions) *ateapipb.ActorTemplate {
t.Helper()
src := SubstrateCounterFixture()
existingWp, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(src.PoolNamespace).Get(ctx, src.PoolName, metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get WorkerPool %s/%s (deploy with: %s): %v", src.PoolNamespace, src.PoolName, src.DeployWith, err)
}
srcTmpl, err := clients.SubstrateAPI.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{
ActorTemplate: &ateapipb.ObjectRef{Atespace: src.Atespace, Name: src.Name},
})
if err != nil {
t.Fatalf("failed to get ActorTemplate %s/%s (deploy with: %s): %v", src.Atespace, src.Name, src.DeployWith, err)
}
wp := &v1alpha1.WorkerPool{
ObjectMeta: metav1.ObjectMeta{
Name: opts.PoolName,
Namespace: namespace,
Labels: opts.Labels,
},
Spec: v1alpha1.WorkerPoolSpec{
Replicas: opts.PoolReplicas,
WorkerImage: existingWp.Spec.WorkerImage,
SandboxClass: existingWp.Spec.SandboxClass,
SandboxConfigName: existingWp.Spec.SandboxConfigName,
},
}
if _, err := clients.SubstrateK8s.ApiV1alpha1().WorkerPools(namespace).Create(ctx, wp, metav1.CreateOptions{}); err != nil {
t.Fatalf("failed to create WorkerPool: %v", err)
}
// CreateActorTemplate requires the atespace to exist first.
if _, err := clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: opts.Atespace}}}); err != nil && status.Code(err) != codes.AlreadyExists {
t.Fatalf("failed to create atespace %q: %v", opts.Atespace, err)
}
snapshots := opts.SnapshotsConfig
if snapshots == nil {
snapshots = srcTmpl.GetSnapshotsConfig()
}
tmpl := &ateapipb.ActorTemplate{
Metadata: &ateapipb.ResourceMetadata{Atespace: opts.Atespace, Name: opts.Name},
WorkerSelector: &ateapipb.Selector{MatchLabels: opts.Labels},
Containers: srcTmpl.GetContainers(),
// The source's limits size the sandbox. Copying them matters most on
// micro-VM, where an ActorTemplate that declares none boots the guest
// at the kata config default (2GiB) instead of the demo's 512Mi.
Resources: srcTmpl.GetResources(),
// Both sandbox_class and config_name are required; the source carries
// the pair for the class under test.
SandboxConfig: srcTmpl.GetSandboxConfig(),
SnapshotsConfig: snapshots,
Volumes: srcTmpl.GetVolumes(),
}
if opts.Modify != nil {
opts.Modify(tmpl)
}
created, err := clients.SubstrateAPI.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: tmpl})
if err != nil {
t.Fatalf("failed to create ActorTemplate %s/%s: %v", opts.Atespace, opts.Name, err)
}
// Registered before the golden wait so a template whose golden never
// builds still gets cleaned up; the actors are gone by then (test-body
// defers run before cleanups).
t.Cleanup(func() {
cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
if _, err := clients.SubstrateAPI.DeleteActorTemplate(cleanupCtx, &ateapipb.DeleteActorTemplateRequest{
ActorTemplate: &ateapipb.ObjectRef{Atespace: opts.Atespace, Name: opts.Name},
}); err != nil && status.Code(err) != codes.NotFound {
t.Logf("failed to delete ActorTemplate %s/%s: %v", opts.Atespace, opts.Name, err)
}
})
// The timeout budgets for the micro-VM golden (a CH cold boot plus
// checkpoint on nested KVM) being slower than the gVisor one.
t.Logf("Waiting for ActorTemplate %s/%s golden snapshot...", opts.Atespace, opts.Name)
WaitForSubstrateTemplateReady(ctx, t, clients, opts.Atespace, opts.Name)
return created
}
// WaitForSubstrateTemplateReady blocks until the substrate ActorTemplate's
// golden snapshot exists, the same readiness the CRD phase poll above proves.
// The timeout follows the sandbox class under test (see TemplateReadyTimeout).
func WaitForSubstrateTemplateReady(ctx context.Context, t *testing.T, clients *Clients, atespace, name string) {
t.Helper()
timeout := TemplateReadyTimeout(t)
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
var lastStatus *ateapipb.GoldenSnapshotStatus
for {
at, err := clients.SubstrateAPI.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{
ActorTemplate: &ateapipb.ObjectRef{Atespace: atespace, Name: name},
})
if err == nil {
lastStatus = at.GetStatus().GetGoldenSnapshotStatus()
if lastStatus.GetGoldenSnapshot() != nil {
return
}
if msg := lastStatus.GetErrorMessage(); msg != "" {
t.Fatalf("ActorTemplate %s/%s failed to build its golden snapshot: %s", atespace, name, msg)
}
}
select {
case <-ctx.Done():
t.Fatalf("timed out after %v waiting for ActorTemplate %s/%s golden snapshot (last status %v, err %v)", timeout, atespace, name, lastStatus, err)
case <-time.After(time.Second):
}
}
}