mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-11-11 10:23:17 +08:00
**Updated Original PR Description** ## Summary This PR adds a small, controlled set of scheduling fields to WorkerPool so WorkerPool-managed Pods can be placed onto appropriate Kubernetes nodes. Added fields: - `nodeSelector` - `tolerations` - `priorityClassName` - `nodeAffinity` ## Scope This intentionally does not expose a full `PodTemplateSpec`, and does not add support for full `affinity`, `podAffinity`, or `podAntiAffinity`. Resource requests/limits are intentionally left out of scope for this CL while the WorkerPool resource model is still being discussed separately. The goal is to keep the first version small while addressing the WorkerPool-to-Kubernetes-Node scheduling gap discussed in [#212](https://github.com/agent-substrate/substrate/issues/212). `nodeAffinity` is included to support heterogeneous node pools where `nodeSelector` is too limited, such as requiring or preferring one of several equivalent accelerator/local-SSD/cache node pools, or expressing soft preferences during node pool migration. `priorityClassName` is included to help preserve warm WorkerPool capacity under cluster pressure. Since WorkerPool pods are the slots available for actor resume, higher-priority pools can protect latency-sensitive or interactive actor workloads from displacement by lower-priority batch workloads, and allow different WorkerPools to represent different service classes, e.g. interactive vs. batch. ## Related issue Part of #212. Related to #47.
125 lines
3.9 KiB
Go
125 lines
3.9 KiB
Go
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
"k8s.io/apimachinery/pkg/api/equality"
|
|
k8errors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/log"
|
|
|
|
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
|
|
)
|
|
|
|
const workerPoolFieldOwner = "workerpool-controller"
|
|
|
|
type WorkerPoolReconciler struct {
|
|
client.Client
|
|
Scheme *runtime.Scheme
|
|
}
|
|
|
|
//+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch;create;update;patch;delete
|
|
//+kubebuilder:rbac:groups=ate.dev,resources=workerpools/status,verbs=get;update;patch
|
|
//+kubebuilder:rbac:groups=ate.dev,resources=workerpools/finalizers,verbs=update
|
|
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
|
|
|
|
// Reconcile is part of the main kubernetes reconciliation loop which aims to
|
|
// move the current state of the cluster closer to the desired state.
|
|
func (r *WorkerPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
|
log := log.FromContext(ctx)
|
|
|
|
// Fetch worker pool
|
|
wp := &atev1alpha1.WorkerPool{}
|
|
if err := r.Get(ctx, req.NamespacedName, wp); err != nil {
|
|
if k8errors.IsNotFound(err) {
|
|
return ctrl.Result{}, nil
|
|
}
|
|
return ctrl.Result{}, fmt.Errorf("failed to get worker pool %q: %w", req.NamespacedName, err)
|
|
}
|
|
|
|
// Handle deletion
|
|
if !wp.GetDeletionTimestamp().IsZero() {
|
|
log.Info("WorkerPool is being deleted")
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
if err := r.reconcileWorkerPool(ctx, wp); err != nil {
|
|
log.Error(err, "Failed to reconcile worker pool")
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
func (r *WorkerPoolReconciler) reconcileWorkerPool(ctx context.Context, wp *atev1alpha1.WorkerPool) error {
|
|
log := log.FromContext(ctx)
|
|
log.Info("Reconciling worker pool")
|
|
|
|
if err := r.applyDeployment(ctx, wp); err != nil {
|
|
return err
|
|
}
|
|
|
|
dep := &appsv1.Deployment{}
|
|
if err := r.Get(ctx, types.NamespacedName{Name: deploymentName(wp.Name), Namespace: wp.Namespace}, dep); err != nil {
|
|
if k8errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to get deployment: %w", err)
|
|
}
|
|
|
|
return r.syncStatus(ctx, wp, dep)
|
|
}
|
|
|
|
func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alpha1.WorkerPool) error {
|
|
depAC := buildDeploymentApplyConfig(wp)
|
|
if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil {
|
|
return fmt.Errorf("failed to apply Deployment: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *WorkerPoolReconciler) syncStatus(ctx context.Context, wp *atev1alpha1.WorkerPool, dep *appsv1.Deployment) error {
|
|
want := atev1alpha1.WorkerPoolStatus{Replicas: dep.Status.Replicas}
|
|
if equality.Semantic.DeepEqual(wp.Status, want) {
|
|
return nil
|
|
}
|
|
|
|
wp.Status = want
|
|
if err := r.Status().Update(ctx, wp); err != nil {
|
|
return fmt.Errorf("failed to update WorkerPool status: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// SetupWithManager sets up the controller with the Manager.
|
|
func (r *WorkerPoolReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
|
return ctrl.NewControllerManagedBy(mgr).
|
|
For(&atev1alpha1.WorkerPool{}).
|
|
Owns(&appsv1.Deployment{}).
|
|
Complete(r)
|
|
}
|
|
|
|
func deploymentName(wpName string) string {
|
|
return wpName + "-deployment"
|
|
}
|