Files
substrate/cmd/atecontroller/internal/controllers/workerpool_controller.go
T
han2ni3bal-pixel 962ff6b1ae Add WorkerPool scheduling fields (#247)
**Updated Original PR Description**
## Summary

This PR adds a small, controlled set of scheduling fields to WorkerPool
so WorkerPool-managed Pods can be placed onto appropriate Kubernetes
nodes.

Added fields:

- `nodeSelector`
- `tolerations`
- `priorityClassName`
- `nodeAffinity`

## Scope

This intentionally does not expose a full `PodTemplateSpec`, and does
not add support for full `affinity`, `podAffinity`, or
`podAntiAffinity`.

Resource requests/limits are intentionally left out of scope for this CL
while the WorkerPool resource model is still being discussed separately.

The goal is to keep the first version small while addressing the
WorkerPool-to-Kubernetes-Node scheduling gap discussed in
[#212](https://github.com/agent-substrate/substrate/issues/212).

`nodeAffinity` is included to support heterogeneous node pools where
`nodeSelector` is too limited, such as requiring or preferring one of
several equivalent accelerator/local-SSD/cache node pools, or expressing
soft preferences during node pool migration.

`priorityClassName` is included to help preserve warm WorkerPool
capacity under cluster pressure. Since WorkerPool pods are the slots
available for actor resume, higher-priority pools can protect
latency-sensitive or interactive actor workloads from displacement by
lower-priority batch workloads, and allow different WorkerPools to
represent different service classes, e.g. interactive vs. batch.

## Related issue

Part of #212.
Related to #47.
2026-06-17 10:33:00 -04:00

125 lines
3.9 KiB
Go

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"context"
"fmt"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/api/equality"
k8errors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
)
const workerPoolFieldOwner = "workerpool-controller"
type WorkerPoolReconciler struct {
client.Client
Scheme *runtime.Scheme
}
//+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=ate.dev,resources=workerpools/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=ate.dev,resources=workerpools/finalizers,verbs=update
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *WorkerPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := log.FromContext(ctx)
// Fetch worker pool
wp := &atev1alpha1.WorkerPool{}
if err := r.Get(ctx, req.NamespacedName, wp); err != nil {
if k8errors.IsNotFound(err) {
return ctrl.Result{}, nil
}
return ctrl.Result{}, fmt.Errorf("failed to get worker pool %q: %w", req.NamespacedName, err)
}
// Handle deletion
if !wp.GetDeletionTimestamp().IsZero() {
log.Info("WorkerPool is being deleted")
return ctrl.Result{}, nil
}
if err := r.reconcileWorkerPool(ctx, wp); err != nil {
log.Error(err, "Failed to reconcile worker pool")
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
func (r *WorkerPoolReconciler) reconcileWorkerPool(ctx context.Context, wp *atev1alpha1.WorkerPool) error {
log := log.FromContext(ctx)
log.Info("Reconciling worker pool")
if err := r.applyDeployment(ctx, wp); err != nil {
return err
}
dep := &appsv1.Deployment{}
if err := r.Get(ctx, types.NamespacedName{Name: deploymentName(wp.Name), Namespace: wp.Namespace}, dep); err != nil {
if k8errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("failed to get deployment: %w", err)
}
return r.syncStatus(ctx, wp, dep)
}
func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alpha1.WorkerPool) error {
depAC := buildDeploymentApplyConfig(wp)
if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil {
return fmt.Errorf("failed to apply Deployment: %w", err)
}
return nil
}
func (r *WorkerPoolReconciler) syncStatus(ctx context.Context, wp *atev1alpha1.WorkerPool, dep *appsv1.Deployment) error {
want := atev1alpha1.WorkerPoolStatus{Replicas: dep.Status.Replicas}
if equality.Semantic.DeepEqual(wp.Status, want) {
return nil
}
wp.Status = want
if err := r.Status().Update(ctx, wp); err != nil {
return fmt.Errorf("failed to update WorkerPool status: %w", err)
}
return nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *WorkerPoolReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&atev1alpha1.WorkerPool{}).
Owns(&appsv1.Deployment{}).
Complete(r)
}
func deploymentName(wpName string) string {
return wpName + "-deployment"
}