// Copyright 2026 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package controlapi import ( "context" "errors" "fmt" "log/slog" "sync" "time" "go.ate.dev/substrate/api/ateapi/v1" "go.ate.dev/substrate/api/objectstore/v1" "go.ate.dev/substrate/cmd/ateapi/internal/scheduling" "go.ate.dev/substrate/cmd/ateapi/internal/store" "go.ate.dev/substrate/cmd/ateapi/internal/workercache" "go.ate.dev/substrate/internal/actorevent" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/pluginsocket" "go.ate.dev/substrate/internal/resources" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" storagev1listers "k8s.io/client-go/listers/storage/v1" ) // stepSpan opens the per-step trace span ("step." on the controlapi // tracer) and returns the step context plus a finish func the step defers: // it records a non-nil error on the span and wraps it with the step name. // // Workflow steps follow the ensure pattern: each step derives whether its // work is already done from persisted state alone (calling markSkipped when // so), validates the state-machine edge it is about to take, and persists // what it changed before returning — so a re-entered workflow fast-forwards // to wherever the previous attempt stopped. func stepSpan(ctx context.Context, name string) (context.Context, func(error) error) { ctx, span := otel.Tracer("controlapi").Start(ctx, "step."+name) return ctx, func(err error) error { defer span.End() if err == nil { return nil } span.RecordError(err) span.SetStatus(codes.Error, err.Error()) return fmt.Errorf("workflow failed at step %s: %w", name, err) } } // markSkipped annotates the current step's span when its postcondition // already holds, so a re-entered workflow's trace shows which steps // fast-forwarded and where real work restarted. func markSkipped(ctx context.Context, reason string) { trace.SpanFromContext(ctx).SetAttributes( attribute.Bool("step.skipped", true), attribute.String("step.skip_reason", reason), ) } // logActorStateChanged records an actor state change. The last record for an // actor's uid is the state it is in now. // // The state is read off the committed record, never passed in, so a record // cannot claim a state the store did not hold. Pass the actor the store returned // and call it after UpdateActor returns, not inside the mutate closure, which // can be retried. Every state commit carries a version precondition, so a call // that returns is the one that made the change. // // Crashes go through logActorCrashed instead, so read the state off // ate.actor.state rather than off the message. func logActorStateChanged(ctx context.Context, actor *ateapi.Actor, opName string) { logActorState(ctx, actor, opName, ateattr.ActorStateValue(actor.GetStatus().GetState())) } // logActorDeleted records the terminal transition. The row is gone, so there is // no committed state left to read and this is the one state named by hand. func logActorDeleted(ctx context.Context, actor *ateapi.Actor, opName string) { logActorState(ctx, actor, opName, ateattr.ActorStateDeleted) } func logActorState(ctx context.Context, actor *ateapi.Actor, opName, state string) { attrs := ateattr.ActorLogAttrs(resources.ActorAttributionFromActor(actor)) attrs = append(attrs, slog.String(string(ateattr.ActorOperationNameKey), ateattr.NormalizeOperationName(opName)), slog.String(string(ateattr.ActorStateKey), state)) actorevent.Log(ctx, actorevent.StateChanged, attrs) } // ActorWorkflow handles the workflows for actor's resume / suspend operations. // // Workflows act for the system, not the user whose request started them, so // each exported method begins with preview.ContextWithOptIn: the request was // already validated, and what the workflow writes is the system's own state. type ActorWorkflow struct { store actorWorkflowStore workerCache *workercache.Cache scheduler scheduling.Scheduler dialer *AteletDialer storageClassLister storagev1listers.StorageClassLister instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry objectStorePlugin objectstore.ControlProviderClient clusterSnapshotLocation string } // NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. // // objectStorePlugin may be nil, which leaves external snapshots in place instead // of copying and releasing them. Only tests that never reach those steps pass // nil; ate-api always builds one. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, dialer *AteletDialer, storageClassLister storagev1listers.StorageClassLister, instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, objectStorePlugin objectstore.ControlProviderClient, ) *ActorWorkflow { return &ActorWorkflow{ store: store, workerCache: workerCache, scheduler: scheduling.New(workerCache), dialer: dialer, storageClassLister: storageClassLister, instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, objectStorePlugin: objectStorePlugin, } } // snapshotLocation returns the base object-storage URI actorTemplate's // snapshots are stored under; see resolveSnapshotLocation. func (w *ActorWorkflow) snapshotLocation(actorTemplate *ateapi.ActorTemplate) (string, error) { return resolveSnapshotLocation(w.clusterSnapshotLocation, actorTemplate) } // cleanupSnapshot deletes every object under prefix through the control // object-store plugin. func (w *ActorWorkflow) cleanupSnapshot(ctx context.Context, prefix resources.StoragePrefix) error { _, err := w.objectStorePlugin.CleanupSnapshot(ctx, &objectstore.CleanupSnapshotRequest{SnapshotUri: prefix.String()}) return pluginsocket.CallError(err) } // copySnapshot copies every object under src to dst through the control // object-store plugin. func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.StoragePrefix) error { _, err := w.objectStorePlugin.CopySnapshot(ctx, &objectstore.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()}) return pluginsocket.CallError(err) } // actorWorkflowStore enumerates the exact storage methods needed by // ActorWorkflow and nothing more. type actorWorkflowStore interface { GetActor(ctx context.Context, actorRef resources.ActorRef) (*ateapi.Actor, error) UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapi.Actor) error) (*ateapi.Actor, error) MarkActorForDeletion(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions, mutate func(toUpdate *ateapi.Actor) error) (*ateapi.Actor, error) DeleteActor(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions) (*ateapi.Actor, error) GetWorker(ctx context.Context, name string) (*ateapi.Worker, error) UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapi.Worker) error) (*ateapi.Worker, error) BindActorToWorker(ctx context.Context, workerName string, assignment *ateapi.ActorAssignment, admit func(*ateapi.Worker) error) error ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapi.Worker, error) GetWorkerAssignment(ctx context.Context, workerName, actorUID string) (*ateapi.ActorAssignment, error) FindWorkerHostingActor(ctx context.Context, actorUID string) (string, error) // Read from the records rather than the Worker's status: only the service // layer attaches assignments on read, and this workflow holds the store. ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapi.ActorAssignment], error) CreateTag(ctx context.Context, tag *ateapi.Tag) (*ateapi.Tag, error) GetTag(ctx context.Context, tagRef resources.TagRef) (*ateapi.Tag, error) UpdateTag(ctx context.Context, tagRef resources.TagRef, precondition store.Precondition, mutate func(toUpdate *ateapi.Tag) error) (*ateapi.Tag, error) MarkTagForDeletion(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions, mutate func(toUpdate *ateapi.Tag) error) (*ateapi.Tag, error) DeleteTag(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions) (*ateapi.Tag, error) GetActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef) (*ateapi.ActorTemplate, error) MarkActorTemplateForDeletion(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions, mutate func(toUpdate *ateapi.ActorTemplate) error) (*ateapi.ActorTemplate, error) DeleteActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions) (*ateapi.ActorTemplate, error) AcquireLease(ctx context.Context, key string) (*store.Lease, error) } // WorkerWorkflow handles the multi-step operations on a Worker. Like // ActorWorkflow, each exported method begins with preview.ContextWithOptIn. // // Its steps reach across the Actor↔Worker binding, which an ActorWorkflow step // does from the other side: releasing the Actor bound to a Worker stays // in-process because there is no bind/release RPC. type WorkerWorkflow struct { store workerWorkflowStore dialer *AteletDialer reclaimTimeout time.Duration now func() time.Time mu sync.Mutex gaveUp map[string]time.Time } // NewWorkerWorkflow creates a new WorkerWorkflow. dialer may be nil when the // caller only runs ReconcileAssignments. func NewWorkerWorkflow(store workerWorkflowStore, dialer *AteletDialer) *WorkerWorkflow { return &WorkerWorkflow{ store: store, dialer: dialer, reclaimTimeout: defaultWorkerReclaimTimeout, now: time.Now, gaveUp: make(map[string]time.Time), } } // workerWorkflowStore enumerates the exact storage methods needed by // WorkerWorkflow and nothing more. type workerWorkflowStore interface { GetWorker(ctx context.Context, name string) (*ateapi.Worker, error) UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapi.Worker) error) (*ateapi.Worker, error) MarkWorkerForDeletion(ctx context.Context, name string, precondition store.DeletePreconditions, mutate func(toUpdate *ateapi.Worker) error) (*ateapi.Worker, error) DeleteWorker(ctx context.Context, name string, precondition store.DeletePreconditions) (*ateapi.Worker, error) ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapi.ActorAssignment], error) ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapi.Worker, error) GetActor(ctx context.Context, actorRef resources.ActorRef) (*ateapi.Actor, error) UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapi.Actor) error) (*ateapi.Actor, error) AcquireLease(ctx context.Context, key string) (*store.Lease, error) } // leaseHolder takes the distributed leases that serialize the operations on one // resource. type leaseHolder interface { AcquireLease(ctx context.Context, key string) (*store.Lease, error) } // acquireLease takes the lease named by key and returns the context to run // under: it is cancelled if the lease is lost. subject names what the lease // covers, for the message a caller that loses the race gets. func acquireLease(ctx context.Context, holder leaseHolder, key, subject string) (context.Context, *store.Lease, error) { lease, err := holder.AcquireLease(ctx, key) if err != nil { if errors.Is(err, store.ErrLeaseConflict) { return nil, nil, apierror.Aborted("another operation is in progress for this %s", subject) } return nil, nil, fmt.Errorf("while acquiring lease: %w", err) } return lease.Context(), lease, nil } // actorLeaseKey names the lease that serializes the operations on an Actor. func actorLeaseKey(actorRef resources.ActorRef) string { return "lease:actor:" + actorRef.Atespace + ":" + actorRef.Name } func (w *ActorWorkflow) acquireActorLease(ctx context.Context, actorRef resources.ActorRef) (context.Context, *store.Lease, error) { return acquireLease(ctx, w.store, actorLeaseKey(actorRef), "actor") } func acquireTagLease(ctx context.Context, holder leaseHolder, tagRef resources.TagRef) (context.Context, *store.Lease, error) { return acquireLease(ctx, holder, "lease:tag:"+tagRef.Atespace+":"+tagRef.Name, "Tag") }