substrate/cmd/ateapi/internal/controlapi/worker.go

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controlapi

import (
	"context"
	"errors"
	"fmt"

	"go.ate.dev/substrate/cmd/ateapi/internal/apivalidation"
	"go.ate.dev/substrate/cmd/ateapi/internal/defaults"
	"go.ate.dev/substrate/cmd/ateapi/internal/store"
	"go.ate.dev/substrate/internal/apierror"
	"go.ate.dev/substrate/internal/resources"
	"go.ate.dev/substrate/pkg/proto/ateapipb"
	"google.golang.org/protobuf/proto"
	"k8s.io/apimachinery/pkg/util/validation/field"
)

// ListWorkerActorAssignments lists the Actors a Worker hosts. The assignments are a
// subresource rather than a field on Worker, so this is the only way to read
// them and neither GetWorker nor ListWorkers grows with occupancy.
func (s *RPCService) ListWorkerActorAssignments(ctx context.Context, req *ateapipb.ListWorkerActorAssignmentsRequest) (*ateapipb.ListWorkerActorAssignmentsResponse, error) {
	if errs := apivalidation.ValidateListWorkerActorAssignmentsRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}
	name := req.GetWorker().GetName()

	// The Worker is read first so a listing against one that does not exist is
	// NOT_FOUND rather than an empty page, which a caller cannot tell from a
	// Worker hosting nothing.
	if _, err := s.impl.GetWorker(ctx, name); err != nil {
		if errors.Is(err, store.ErrNotFound) {
			return nil, apierror.NotFound("Worker %s not found", name)
		}
		return nil, fmt.Errorf("while fetching worker %s: %w", name, err)
	}

	page, err := s.impl.ListWorkerAssignments(ctx, name, toStoreListOptions(req.GetOptions()))
	if err != nil {
		return nil, mapListError(fmt.Errorf("while listing the assignments of worker %s: %w", name, err))
	}
	return &ateapipb.ListWorkerActorAssignmentsResponse{
		ActorAssignments: page.Items,
		NextPageToken:    page.NextPageToken,
	}, nil
}

func (s *RPCService) ListWorkers(ctx context.Context, req *ateapipb.ListWorkersRequest) (*ateapipb.ListWorkersResponse, error) {
	if errs := apivalidation.ValidateListWorkersRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}

	page, err := s.impl.ListWorkers(ctx, toStoreListOptions(req.GetOptions()))
	if err != nil {
		return nil, mapListError(fmt.Errorf("while listing workers in db: %w", err))
	}
	return &ateapipb.ListWorkersResponse{
		Workers:       page.Items,
		NextPageToken: page.NextPageToken,
	}, nil
}

func (s *ServiceImpl) ListWorkers(ctx context.Context, opts store.ListOptions) (store.ListResponse[*ateapipb.Worker], error) {
	return s.store.ListWorkers(ctx, opts)
}

func (s *RPCService) GetWorker(ctx context.Context, req *ateapipb.GetWorkerRequest) (*ateapipb.Worker, error) {
	if errs := apivalidation.ValidateGetWorkerRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}
	name := req.GetWorker().GetName()

	worker, err := s.impl.GetWorker(ctx, name)
	if errors.Is(err, store.ErrNotFound) {
		return nil, apierror.NotFound("Worker %s not found", name)
	}
	if err != nil {
		return nil, fmt.Errorf("while getting worker: %w", err)
	}
	return worker, nil
}

// GetWorker returns the Worker with the Actors it hosts, read separately since
// the assignments are their own records. The one read that pays O(assignments).
// A failed read is an error, not a Worker reported as hosting nothing: the
// caller cannot tell those apart.
func (s *ServiceImpl) GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error) {
	return s.store.GetWorker(ctx, name)
}

func (s *RPCService) CreateWorker(ctx context.Context, req *ateapipb.CreateWorkerRequest) (*ateapipb.Worker, error) {
	// First scrub any fields that callers are not allowed to set. status is
	// output-only, so whatever the request carried there is replaced rather
	// than rejected.
	inWorker := req.Worker
	if inWorker != nil { // otherwise validation will flag it
		scrubResourceMetadataForCreate(inWorker.Metadata)
		inWorker.Status = nil
		defaults.Apply(inWorker)
	}

	// Validate the request, including the object within it.
	if errs := apivalidation.ValidateCreateWorkerRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}

	// Handle the creation, including validation of the final stored object.
	return s.impl.CreateWorker(ctx, inWorker)
}

func (s *ServiceImpl) CreateWorker(ctx context.Context, inWorker *ateapipb.Worker) (*ateapipb.Worker, error) {
	// A Worker takes no Actors until MarkWorkerAvailable.
	outWorker := proto.CloneOf(inWorker)
	// A new Worker hosts no Actors, so none are left from an earlier epoch.
	outWorker.Status = &ateapipb.WorkerStatus{
		State:         ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE,
		ObservedEpoch: inWorker.GetEpoch(),
	}

	// Capacity and runtimes are left unset: a Worker holds nothing and runs no
	// sandbox until its own worker says what it has, through
	// WorkerService.RegisterWorker. Nothing is placed on it in the meantime,
	// which is the point -- the alternative is guessing on the Worker's behalf
	// and placing against the guess.

	// Verify that the result is properly valid before storing it.
	if errs := apivalidation.ValidateWorkerUpdate(ctx, field.NewPath("worker"), outWorker, inWorker, true); len(errs) > 0 {
		return nil, toGRPCInternalError(errs)
	}

	// Save the data in the storage layer.
	created, err := s.store.CreateWorker(ctx, outWorker)
	if err != nil {
		if errors.Is(err, store.ErrAlreadyExists) {
			return nil, apierror.AlreadyExists("Worker %s already exists", inWorker.GetMetadata().GetName())
		}
		return nil, fmt.Errorf("while creating worker: %w", err)
	}
	return created, nil
}

// UpdateWorker replaces the stored Worker with the one the request carries.
// Only labels and epoch are the caller's to change; a request that alters an
// immutable field — including by leaving it unset, which would clear it — is
// rejected.
// The service layer enforces that with declarative validation against the
// stored worker inside the update transaction.
func (s *RPCService) UpdateWorker(ctx context.Context, req *ateapipb.UpdateWorkerRequest) (*ateapipb.Worker, error) {
	// First scrub any fields that callers are not allowed to set.
	inWorker := req.Worker
	if inWorker != nil { // otherwise validation will flag it
		scrubResourceMetadataForUpdate(inWorker.Metadata)
		inWorker.Status = nil
	}

	// Validate the request.
	if errs := apivalidation.ValidateUpdateWorkerRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}

	return s.mutateWorker(ctx, inWorker.GetMetadata().GetName(), store.PreconditionFrom(inWorker), func(toUpdate *ateapipb.Worker) error {
		// Status and metadata are server-owned fields.
		status, metadata := toUpdate.GetStatus(), toUpdate.GetMetadata()
		// Reset + merge from the input worker.
		proto.Reset(toUpdate)
		proto.Merge(toUpdate, inWorker)
		// Restore status and metadata from the server.
		toUpdate.Status = status
		toUpdate.Metadata = metadata
		// Defaults are re-applied to the merged object, so a defaulted field
		// the request left unset is defaulted again rather than cleared.
		defaults.Apply(toUpdate)
		return nil
	})
}

func (s *ServiceImpl) UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) {
	return s.store.UpdateWorker(ctx, name, precondition, func(toUpdate *ateapipb.Worker) error {
		// Apply the mutation function to the stored value.
		oldVal := proto.CloneOf(toUpdate)
		if err := mutate(toUpdate); err != nil {
			return err
		}
		newVal := toUpdate

		// Validate the mutated value before doing any further work. This is
		// what enforces the immutable fields, since only the stored worker
		// gives declarative validation an old value to compare against.
		if errs := apivalidation.ValidateWorkerUpdate(ctx, field.NewPath("worker"), newVal, oldVal, false); len(errs) > 0 {
			return resources.ToAPIError(errs)
		}

		// Do any further work on the resource.

		// Validate the final value before storing it.
		if errs := apivalidation.ValidateWorkerUpdate(ctx, field.NewPath("worker"), newVal, oldVal, true); len(errs) > 0 {
			return toGRPCInternalError(errs)
		}

		return nil
	})
}

// The assignment operations are pass-throughs: an assignment is its own record,
// so binding and releasing are single store calls rather than a read-modify-write
// of the Worker.
func (s *ServiceImpl) BindActorToWorker(ctx context.Context, workerName string, assignment *ateapipb.ActorAssignment, admit func(*ateapipb.Worker) error) error {
	return s.store.BindActorToWorker(ctx, workerName, assignment, admit)
}

func (s *ServiceImpl) ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error) {
	return s.store.ReleaseActorFromWorker(ctx, workerName, actorUID)
}

func (s *ServiceImpl) GetWorkerAssignment(ctx context.Context, workerName, actorUID string) (*ateapipb.ActorAssignment, error) {
	return s.store.GetWorkerAssignment(ctx, workerName, actorUID)
}

func (s *ServiceImpl) ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error) {
	return s.store.ListWorkerAssignments(ctx, workerName, opts)
}

func (s *ServiceImpl) FindWorkerHostingActor(ctx context.Context, actorUID string) (string, error) {
	return s.store.FindWorkerHostingActor(ctx, actorUID)
}

func (s *RPCService) DeleteWorker(ctx context.Context, req *ateapipb.DeleteWorkerRequest) (*ateapipb.Worker, error) {
	if errs := apivalidation.ValidateDeleteWorkerRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}
	// The delete releases the Actor bound to this Worker before removing the
	// record, so it is a workflow rather than a single store call.
	return s.workerWorkflow.DeleteWorker(ctx, req.GetWorker().GetName(), toDeletePreconditions(req.GetOptions()))
}

func (s *ServiceImpl) DeleteWorker(ctx context.Context, name string, precondition store.DeletePreconditions) (*ateapipb.Worker, error) {
	return s.store.DeleteWorker(ctx, name, precondition)
}

func (s *RPCService) DrainWorker(ctx context.Context, req *ateapipb.DrainWorkerRequest) (*ateapipb.DrainWorkerResponse, error) {
	if errs := apivalidation.ValidateDrainWorkerRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}
	drained, err := s.mutateNamedWorker(ctx, req.GetWorker().GetName(), func(toUpdate *ateapipb.Worker) error {
		if toUpdate.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_DRAINING {
			// already draining, do nothing
			return &workerUnchanged{worker: proto.Clone(toUpdate).(*ateapipb.Worker)}
		}
		toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING
		// The assignments are left alone: a draining Worker keeps its Actors
		// until something releases them. Draining only stops new placements.
		return nil
	})
	if err != nil {
		return nil, err
	}
	return &ateapipb.DrainWorkerResponse{Worker: drained}, nil
}

func (s *RPCService) MarkWorkerAvailable(ctx context.Context, req *ateapipb.MarkWorkerAvailableRequest) (*ateapipb.MarkWorkerAvailableResponse, error) {
	if errs := apivalidation.ValidateMarkWorkerAvailableRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}
	marked, err := s.mutateNamedWorker(ctx, req.GetWorker().GetName(), func(toUpdate *ateapipb.Worker) error {
		switch toUpdate.GetStatus().GetState() {
		case ateapipb.WorkerState_WORKER_STATE_ACTIVE:
			return &workerUnchanged{worker: proto.Clone(toUpdate).(*ateapipb.Worker)}
		case ateapipb.WorkerState_WORKER_STATE_DRAINING:
			return errWorkerDraining
		}
		toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_ACTIVE
		return nil
	})
	if err != nil {
		return nil, err
	}
	return &ateapipb.MarkWorkerAvailableResponse{Worker: marked}, nil
}

func (s *RPCService) MarkWorkerUnavailable(ctx context.Context, req *ateapipb.MarkWorkerUnavailableRequest) (*ateapipb.MarkWorkerUnavailableResponse, error) {
	if errs := apivalidation.ValidateMarkWorkerUnavailableRequest(ctx, req); len(errs) > 0 {
		return nil, resources.ToAPIError(errs)
	}
	marked, err := s.mutateNamedWorker(ctx, req.GetWorker().GetName(), func(toUpdate *ateapipb.Worker) error {
		switch toUpdate.GetStatus().GetState() {
		case ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE:
			return &workerUnchanged{worker: proto.Clone(toUpdate).(*ateapipb.Worker)}
		case ateapipb.WorkerState_WORKER_STATE_DRAINING:
			return errWorkerDraining
		}
		toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_UNAVAILABLE
		// The assignments are left alone, as for draining.
		return nil
	})
	if err != nil {
		return nil, err
	}
	return &ateapipb.MarkWorkerUnavailableResponse{Worker: marked}, nil
}

// errWorkerDraining refuses to move a draining Worker out of DRAINING, which
// is final.
var errWorkerDraining = errors.New("worker is draining")

// mutateNamedWorker runs mutate against the named Worker for a request that
// carries no guards, so the ones the store requires come from a read here
// rather than from the client. A write that lands in between is reported as a
// conflict for the caller to retry, the same as any other guarded update.
func (s *RPCService) mutateNamedWorker(ctx context.Context, name string, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) {
	observed, err := s.impl.GetWorker(ctx, name)
	if errors.Is(err, store.ErrNotFound) {
		return nil, apierror.NotFound("Worker %s not found", name)
	}
	if err != nil {
		return nil, fmt.Errorf("while getting worker %s: %w", name, err)
	}
	return s.mutateWorker(ctx, name, store.PreconditionFrom(observed), mutate)
}

// mutateWorker runs mutate against the named Worker and translates what comes
// back into the RPC's result. A mutation that found nothing to do reports the
// Worker it saw; anything else is a store error.
func (s *RPCService) mutateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) {
	worker, err := s.impl.UpdateWorker(ctx, name, precondition, mutate)
	if err == nil {
		return worker, nil
	}

	var unchanged *workerUnchanged
	if errors.As(err, &unchanged) {
		return unchanged.worker, nil
	}
	switch {
	case errors.Is(err, errWorkerDraining):
		return nil, apierror.FailedPrecondition("Worker %s is draining", name)
	case errors.Is(err, store.ErrNotFound):
		return nil, apierror.NotFound("Worker %s not found", name)
	case errors.Is(err, store.ErrUIDConflict):
		return nil, apierror.Aborted("Worker %s is not the one the request describes", name)
	case errors.Is(err, store.ErrVersionConflict):
		return nil, apierror.Aborted("concurrent update conflict, please retry")
	case errors.Is(err, store.ErrPreconditionRequired):
		return nil, apierror.InvalidArgument("while updating worker %s: %v", name, err)
	}
	return nil, fmt.Errorf("while updating worker: %w", err)
}

// workerUnchanged ends an UpdateWorker mutation that found its work already
// done. The store hands a mutation's error straight back and leaves the Worker —
// and its version — untouched, which is what lets DrainWorker be idempotent: a
// call with nothing left to do costs no version bump. worker is a copy, because
// the store is free to reuse or discard the message once mutate returns.
type workerUnchanged struct {
	worker *ateapipb.Worker
}

func (u *workerUnchanged) Error() string { return "worker is already in the requested state" }

func (s *ServiceImpl) WatchWorkers(ctx context.Context) (*store.WorkerWatch, error) {
	return s.store.WatchWorkers(ctx)
}