substrate/cmd/ateapi/internal/controlapi/workflow.go

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controlapi

import (
	"context"
	"errors"
	"fmt"
	"log/slog"

	"go.ate.dev/substrate/cmd/ateapi/internal/scheduling"
	"go.ate.dev/substrate/cmd/ateapi/internal/store"
	"go.ate.dev/substrate/cmd/ateapi/internal/workercache"
	"go.ate.dev/substrate/internal/actorevent"
	"go.ate.dev/substrate/internal/apierror"
	"go.ate.dev/substrate/internal/ateattr"
	"go.ate.dev/substrate/internal/pluginsocket"
	"go.ate.dev/substrate/internal/resources"
	"go.ate.dev/substrate/pkg/proto/ateapipb"
	objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/codes"
	"go.opentelemetry.io/otel/trace"
	storagev1listers "k8s.io/client-go/listers/storage/v1"
)

// stepSpan opens the per-step trace span ("step.<name>" on the controlapi
// tracer) and returns the step context plus a finish func the step defers:
// it records a non-nil error on the span and wraps it with the step name.
//
// Workflow steps follow the ensure pattern: each step derives whether its
// work is already done from persisted state alone (calling markSkipped when
// so), validates the state-machine edge it is about to take, and persists
// what it changed before returning — so a re-entered workflow fast-forwards
// to wherever the previous attempt stopped.
func stepSpan(ctx context.Context, name string) (context.Context, func(error) error) {
	ctx, span := otel.Tracer("controlapi").Start(ctx, "step."+name)
	return ctx, func(err error) error {
		defer span.End()
		if err == nil {
			return nil
		}
		span.RecordError(err)
		span.SetStatus(codes.Error, err.Error())
		return fmt.Errorf("workflow failed at step %s: %w", name, err)
	}
}

// markSkipped annotates the current step's span when its postcondition
// already holds, so a re-entered workflow's trace shows which steps
// fast-forwarded and where real work restarted.
func markSkipped(ctx context.Context, reason string) {
	trace.SpanFromContext(ctx).SetAttributes(
		attribute.Bool("step.skipped", true),
		attribute.String("step.skip_reason", reason),
	)
}

// logActorStateChanged records an actor state change. The last record for an
// actor's uid is the state it is in now.
//
// The state is read off the committed record, never passed in, so a record
// cannot claim a state the store did not hold. Pass the actor the store returned
// and call it after UpdateActor returns, not inside the mutate closure, which
// can be retried. Every state commit carries a version precondition, so a call
// that returns is the one that made the change.
//
// Crashes go through logActorCrashed instead, so read the state off
// ate.actor.state rather than off the message.
func logActorStateChanged(ctx context.Context, actor *ateapipb.Actor, opName string) {
	logActorState(ctx, actor, opName, ateattr.ActorStateValue(actor.GetStatus().GetState()))
}

// logActorDeleted records the terminal transition. The row is gone, so there is
// no committed state left to read and this is the one state named by hand.
func logActorDeleted(ctx context.Context, actor *ateapipb.Actor, opName string) {
	logActorState(ctx, actor, opName, ateattr.ActorStateDeleted)
}

func logActorState(ctx context.Context, actor *ateapipb.Actor, opName, state string) {
	attrs := ateattr.ActorLogAttrs(resources.ActorAttributionFromActor(actor))
	attrs = append(attrs,
		slog.String(string(ateattr.ActorOperationNameKey), ateattr.NormalizeOperationName(opName)),
		slog.String(string(ateattr.ActorStateKey), state))
	actorevent.Log(ctx, actorevent.StateChanged, attrs)
}

// ActorWorkflow handles the workflows for actor's resume / suspend operations.
//
// Workflows act for the system, not the user whose request started them, so
// each exported method begins with preview.ContextWithOptIn: the request was
// already validated, and what the workflow writes is the system's own state.
type ActorWorkflow struct {
	store                   actorWorkflowStore
	workerCache             *workercache.Cache
	scheduler               scheduling.Scheduler
	dialer                  *AteletDialer
	storageClassLister      storagev1listers.StorageClassLister
	instruments             *Instruments
	egressGatewayAddress    string
	pluginRegistry          VolumePluginRegistry
	objectStorePlugin       objectstorev1.ControlProviderClient
	clusterSnapshotLocation string
}

// NewActorWorkflow creates a new ActorWorkflow. instruments may be nil.
//
// objectStorePlugin may be nil, which leaves external snapshots in place instead
// of copying and releasing them. Only tests that never reach those steps pass
// nil; ate-api always builds one.
func NewActorWorkflow(
	store actorWorkflowStore,
	workerCache *workercache.Cache,
	dialer *AteletDialer,
	storageClassLister storagev1listers.StorageClassLister,
	instruments *Instruments,
	egressGatewayAddress string,
	pluginRegistry VolumePluginRegistry,
	objectStorePlugin objectstorev1.ControlProviderClient,
) *ActorWorkflow {
	return &ActorWorkflow{
		store:                store,
		workerCache:          workerCache,
		scheduler:            scheduling.New(workerCache),
		dialer:               dialer,
		storageClassLister:   storageClassLister,
		instruments:          instruments,
		egressGatewayAddress: egressGatewayAddress,
		pluginRegistry:       pluginRegistry,
		objectStorePlugin:    objectStorePlugin,
	}
}

// snapshotLocation returns the base object-storage URI actorTemplate's
// snapshots are stored under; see resolveSnapshotLocation.
func (w *ActorWorkflow) snapshotLocation(actorTemplate *ateapipb.ActorTemplate) (string, error) {
	return resolveSnapshotLocation(w.clusterSnapshotLocation, actorTemplate)
}

// cleanupSnapshot deletes every object under prefix through the control
// object-store plugin.
func (w *ActorWorkflow) cleanupSnapshot(ctx context.Context, prefix resources.StoragePrefix) error {
	_, err := w.objectStorePlugin.CleanupSnapshot(ctx, &objectstorev1.CleanupSnapshotRequest{SnapshotUri: prefix.String()})
	return pluginsocket.CallError(err)
}

// copySnapshot copies every object under src to dst through the control
// object-store plugin.
func (w *ActorWorkflow) copySnapshot(ctx context.Context, src, dst resources.StoragePrefix) error {
	_, err := w.objectStorePlugin.CopySnapshot(ctx, &objectstorev1.CopySnapshotRequest{SrcUri: src.String(), DstUri: dst.String()})
	return pluginsocket.CallError(err)
}

// actorWorkflowStore enumerates the exact storage methods needed by
// ActorWorkflow and nothing more.
type actorWorkflowStore interface {
	GetActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error)
	UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapipb.Actor) error) (*ateapipb.Actor, error)
	DeleteActor(ctx context.Context, actorRef resources.ActorRef, precondition store.DeletePreconditions) (*ateapipb.Actor, error)
	GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error)
	UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error)
	BindActorToWorker(ctx context.Context, workerName string, assignment *ateapipb.ActorAssignment, admit func(*ateapipb.Worker) error) error
	ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error)
	GetWorkerAssignment(ctx context.Context, workerName, actorUID string) (*ateapipb.ActorAssignment, error)
	FindWorkerHostingActor(ctx context.Context, actorUID string) (string, error)
	// Read from the records rather than the Worker's status: only the service
	// layer attaches assignments on read, and this workflow holds the store.
	ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error)
	CreateTag(ctx context.Context, tag *ateapipb.Tag) (*ateapipb.Tag, error)
	GetTag(ctx context.Context, tagRef resources.TagRef) (*ateapipb.Tag, error)
	UpdateTag(ctx context.Context, tagRef resources.TagRef, precondition store.Precondition, mutate func(toUpdate *ateapipb.Tag) error) (*ateapipb.Tag, error)
	DeleteTag(ctx context.Context, tagRef resources.TagRef, precondition store.DeletePreconditions) (*ateapipb.Tag, error)
	GetActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef) (*ateapipb.ActorTemplate, error)
	DeleteActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions) (*ateapipb.ActorTemplate, error)
	AcquireLease(ctx context.Context, key string) (*store.Lease, error)
}

// WorkerWorkflow handles the multi-step operations on a Worker. Like
// ActorWorkflow, each exported method begins with preview.ContextWithOptIn.
//
// Its steps reach across the Actor↔Worker binding, which an ActorWorkflow step
// does from the other side: releasing the Actor bound to a Worker stays
// in-process because there is no bind/release RPC.
type WorkerWorkflow struct {
	store workerWorkflowStore
}

// NewWorkerWorkflow creates a new WorkerWorkflow.
func NewWorkerWorkflow(store workerWorkflowStore) *WorkerWorkflow {
	return &WorkerWorkflow{store: store}
}

// workerWorkflowStore enumerates the exact storage methods needed by
// WorkerWorkflow and nothing more.
type workerWorkflowStore interface {
	GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error)
	UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error)
	DeleteWorker(ctx context.Context, name string, precondition store.DeletePreconditions) (*ateapipb.Worker, error)
	ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error)
	ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error)
	GetActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error)
	UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapipb.Actor) error) (*ateapipb.Actor, error)
	AcquireLease(ctx context.Context, key string) (*store.Lease, error)
}

// leaseHolder takes the distributed leases that serialize the operations on one
// resource.
type leaseHolder interface {
	AcquireLease(ctx context.Context, key string) (*store.Lease, error)
}

// acquireLease takes the lease named by key and returns the context to run
// under: it is cancelled if the lease is lost. subject names what the lease
// covers, for the message a caller that loses the race gets.
func acquireLease(ctx context.Context, holder leaseHolder, key, subject string) (context.Context, *store.Lease, error) {
	lease, err := holder.AcquireLease(ctx, key)
	if err != nil {
		if errors.Is(err, store.ErrLeaseConflict) {
			return nil, nil, apierror.Aborted("another operation is in progress for this %s", subject)
		}
		return nil, nil, fmt.Errorf("while acquiring lease: %w", err)
	}

	return lease.Context(), lease, nil
}

// actorLeaseKey names the lease that serializes the operations on an Actor.
func actorLeaseKey(actorRef resources.ActorRef) string {
	return "lease:actor:" + actorRef.Atespace + ":" + actorRef.Name
}

func (w *ActorWorkflow) acquireActorLease(ctx context.Context, actorRef resources.ActorRef) (context.Context, *store.Lease, error) {
	return acquireLease(ctx, w.store, actorLeaseKey(actorRef), "actor")
}

func acquireTagLease(ctx context.Context, holder leaseHolder, tagRef resources.TagRef) (context.Context, *store.Lease, error) {
	return acquireLease(ctx, holder, "lease:tag:"+tagRef.Atespace+":"+tagRef.Name, "Tag")
}