substrate/cmd/atecontroller/internal/controllers/workerpool_controller_test.go

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controllers

import (
	"context"
	"fmt"
	"os"
	"reflect"
	"testing"
	"time"

	"go.opentelemetry.io/otel/attribute"
	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/metric/metricdata"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	k8errors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/util/retry"
	"k8s.io/utils/ptr"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"
	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"

	"github.com/agent-substrate/substrate/internal/ateattr"
	"github.com/agent-substrate/substrate/internal/installdefaults"
	"github.com/agent-substrate/substrate/internal/testenv"
	atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
)

var (
	cfg       *rest.Config
	k8sClient client.Client
)

func TestMain(m *testing.M) {
	var stopEnv func()
	cfg, stopEnv = testenv.Start()

	scheme := runtime.NewScheme()
	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
	utilruntime.Must(atev1alpha1.AddToScheme(scheme))

	var err error
	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
	if err != nil {
		fmt.Fprintf(os.Stderr, "k8s client creation failed: %v\n", err)
		os.Exit(1)
	}

	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
		Scheme:                 scheme,
		Metrics:                metricsserver.Options{BindAddress: "0"},
		HealthProbeBindAddress: "0",
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "manager creation failed: %v\n", err)
		os.Exit(1)
	}

	if err := (&WorkerPoolReconciler{
		Client: mgr.GetClient(),
		Scheme: mgr.GetScheme(),
	}).SetupWithManager(mgr); err != nil {
		fmt.Fprintf(os.Stderr, "controller setup failed: %v\n", err)
		os.Exit(1)
	}

	if err := (&NetworkPolicyReconciler{
		Client:          mgr.GetClient(),
		Scheme:          mgr.GetScheme(),
		SystemNamespace: installdefaults.SystemNamespace,
	}).SetupWithManager(mgr); err != nil {
		fmt.Fprintf(os.Stderr, "netpolicy controller setup failed: %v\n", err)
		os.Exit(1)
	}

	mgrCtx, mgrCancel := context.WithCancel(context.Background())
	go func() {
		_ = mgr.Start(mgrCtx)
	}()

	code := m.Run()

	mgrCancel()
	stopEnv()
	os.Exit(code)
}

// TestWorkerPoolCreatesDeployment verifies that creating a WorkerPool causes
// the controller to create a correctly-configured Deployment.
func TestWorkerPoolCreatesDeployment(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-create", "default", 3, "ateom:v1")
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil {
			return false, nil
		}
		if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 3 {
			return false, nil
		}
		if len(dep.Spec.Template.Spec.Containers) == 0 {
			return false, nil
		}
		container := dep.Spec.Template.Spec.Containers[0]
		if container.Image != "ateom:v1" || container.Name != "ateom" {
			return false, nil
		}
		if dep.Spec.Template.Labels["ate.dev/worker-pool"] != wp.Name {
			return false, nil
		}
		if len(dep.OwnerReferences) == 0 || dep.OwnerReferences[0].Name != wp.Name {
			return false, nil
		}
		return len(dep.Spec.Template.Spec.Volumes) == 4 &&
			dep.Spec.Template.Spec.Volumes[0].Name == ateomCapacityVolume &&
			dep.Spec.Template.Spec.Volumes[1].Name == "run-ateom" &&
			dep.Spec.Template.Spec.Volumes[2].Name == atunnelIdentityVolume &&
			dep.Spec.Template.Spec.Volumes[3].Name == atunnelEgressTrustVolume, nil
	})
}

// TestWorkerPoolReplicasUpdate verifies that changing spec.replicas on a
// WorkerPool propagates to the managed Deployment.
func TestWorkerPoolReplicasUpdate(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-replicas", "default", 2, "ateom:v1")
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		_, err := getDeployment(ctx, wp)
		return err == nil, nil
	})

	updateWorkerPoolSpec(t, ctx, wp, "update WorkerPool replicas", func(current *atev1alpha1.WorkerPool) {
		current.Spec.Replicas = 5
	})

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil {
			return false, nil
		}
		return dep.Spec.Replicas != nil && *dep.Spec.Replicas == 5, nil
	})
}

// TestWorkerPoolImageUpdate verifies that changing spec.workerImage on a
// WorkerPool propagates to the managed Deployment.
func TestWorkerPoolImageUpdate(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-image", "default", 1, "ateom:v1")
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		_, err := getDeployment(ctx, wp)
		return err == nil, nil
	})

	updateWorkerPoolSpec(t, ctx, wp, "update WorkerPool image", func(current *atev1alpha1.WorkerPool) {
		current.Spec.WorkerImage = "ateom:v2"
	})

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil || len(dep.Spec.Template.Spec.Containers) == 0 {
			return false, nil
		}
		return dep.Spec.Template.Spec.Containers[0].Image == "ateom:v2", nil
	})
}

// TestSSAPreservesUnownedFields verifies that SSA leaves fields set by other
// field managers untouched during reconciliation.
func TestSSAPreservesUnownedFields(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-ssa-unowned", "default", 2, "ateom:v1")
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		_, err := getDeployment(ctx, wp)
		return err == nil, nil
	})

	// An external manager sets revisionHistoryLimit — a field the controller
	// never declares in its apply config.
	revisionHistoryLimit := int32(7)
	updateDeploymentSpec(t, ctx, wp, "set revisionHistoryLimit", func(dep *appsv1.Deployment) {
		dep.Spec.RevisionHistoryLimit = &revisionHistoryLimit
	})

	// The Deployment update triggers a reconcile via Owns(). Wait until the
	// reconcile has run (replicas still correct) and the field is still present.
	eventually(t, func(ctx context.Context) (bool, error) {
		d, err := getDeployment(ctx, wp)
		if err != nil {
			return false, nil
		}
		return d.Spec.Replicas != nil && *d.Spec.Replicas == 2 &&
			d.Spec.RevisionHistoryLimit != nil && *d.Spec.RevisionHistoryLimit == 7, nil
	})
}

// TestSSARevertsOwnedFields verifies that if an external actor changes a field
// owned by the workerpool-controller (e.g. replicas on the Deployment), the
// controller reverts it on the next reconcile.
func TestSSARevertsOwnedFields(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-ssa-owned", "default", 2, "ateom:v1")
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		return err == nil && dep.Spec.Replicas != nil && *dep.Spec.Replicas == 2, nil
	})

	rogueReplicas := int32(99)
	updateDeploymentSpec(t, ctx, wp, "rogue update", func(dep *appsv1.Deployment) {
		dep.Spec.Replicas = &rogueReplicas
	})

	// The controller re-applies with ForceOwnership, reclaiming replicas.
	eventually(t, func(ctx context.Context) (bool, error) {
		d, err := getDeployment(ctx, wp)
		if err != nil {
			return false, nil
		}
		return d.Spec.Replicas != nil && *d.Spec.Replicas == 2, nil
	})
}

// TestDeletedDeploymentRecreated verifies that if the managed Deployment is
// deleted externally, the controller recreates it.
func TestDeletedDeploymentRecreated(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-recreate", "default", 2, "ateom:v1")
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		_, err := getDeployment(ctx, wp)
		return err == nil, nil
	})

	dep, err := getDeployment(ctx, wp)
	if err != nil {
		t.Fatalf("get Deployment: %v", err)
	}
	if err := k8sClient.Delete(ctx, dep); err != nil {
		t.Fatalf("delete Deployment: %v", err)
	}

	eventually(t, func(ctx context.Context) (bool, error) {
		_, err := getDeployment(ctx, wp)
		return err == nil, nil
	})
}

// TestStatusReplicasPropagation verifies that the controller syncs the
// Deployment's status.replicas into WorkerPool.status.replicas, and publishes
// the Deployment's pod selector as WorkerPool.status.selector.
func TestStatusReplicasPropagation(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-status", "default", 3, "ateom:v1")
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		_, err := getDeployment(ctx, wp)
		return err == nil, nil
	})

	// Simulate the deployment controller reporting 3 running pods.
	updateDeploymentStatus(t, ctx, wp, "patch Deployment status", func(dep *appsv1.Deployment) {
		dep.Status.Replicas = 3
	})

	eventually(t, func(ctx context.Context) (bool, error) {
		current := &atev1alpha1.WorkerPool{}
		if err := k8sClient.Get(ctx, types.NamespacedName{Name: wp.Name, Namespace: wp.Namespace}, current); err != nil {
			return false, nil
		}
		if current.Status.Selector != "ate.dev/worker-pool="+wp.Name {
			return false, nil
		}
		return current.Status.Replicas == 3, nil
	})
}

func sampleWorkerPoolPodTemplate() *atev1alpha1.WorkerPoolPodTemplate {
	return &atev1alpha1.WorkerPoolPodTemplate{
		NodeSelector: map[string]string{
			"workload": "substrate",
		},
		Tolerations: []corev1.Toleration{{
			Key:      "nvidia.com/gpu",
			Operator: corev1.TolerationOpExists,
			Effect:   corev1.TaintEffectNoSchedule,
		}},
		PriorityClassName:  "substrate-workers",
		ServiceAccountName: ptr.To("substrate-worker"),
		NodeAffinity: &corev1.NodeAffinity{
			RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
				NodeSelectorTerms: []corev1.NodeSelectorTerm{{
					MatchExpressions: []corev1.NodeSelectorRequirement{{
						Key:      "workload",
						Operator: corev1.NodeSelectorOpIn,
						Values:   []string{"substrate"},
					}},
				}},
			},
		},
		Resources: &corev1.ResourceRequirements{
			Requests: corev1.ResourceList{
				corev1.ResourceCPU:    resource.MustParse("500m"),
				corev1.ResourceMemory: resource.MustParse("1Gi"),
			},
			Limits: corev1.ResourceList{
				corev1.ResourceCPU:    resource.MustParse("1"),
				corev1.ResourceMemory: resource.MustParse("2Gi"),
			},
		},
	}
}

// TestWorkerPoolPodTemplatePropagation verifies that template fields propagate
// to the managed Deployment pod template.
func TestWorkerPoolPodTemplatePropagation(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-template-propagate", "default", 1, "ateom:v1")
	wp.Spec.Template = sampleWorkerPoolPodTemplate()
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil || len(dep.Spec.Template.Spec.Containers) == 0 {
			return false, nil
		}
		podSpec := dep.Spec.Template.Spec
		container := podSpec.Containers[0]

		if podSpec.NodeSelector["workload"] != "substrate" {
			return false, nil
		}
		// The controller appends its own sandbox class toleration, so look
		// for the template's rather than expecting it to be the only one.
		hasTemplateToleration := false
		for _, tol := range podSpec.Tolerations {
			if tol.Key == "nvidia.com/gpu" {
				hasTemplateToleration = true
			}
		}
		if !hasTemplateToleration {
			return false, nil
		}
		if podSpec.PriorityClassName != "substrate-workers" {
			return false, nil
		}
		if podSpec.ServiceAccountName != "substrate-worker" {
			return false, nil
		}
		if podSpec.Affinity == nil || podSpec.Affinity.NodeAffinity == nil {
			return false, nil
		}
		return container.Resources.Requests.Cpu().String() == "500m" &&
			container.Resources.Requests.Memory().String() == "1Gi" &&
			container.Resources.Limits.Cpu().String() == "1" &&
			container.Resources.Limits.Memory().String() == "2Gi", nil
	})
}

// TestWorkerPoolPodTemplateUpdate verifies that changing template fields on a
// WorkerPool propagates to the managed Deployment.
func TestWorkerPoolPodTemplateUpdate(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-template-update", "default", 1, "ateom:v1")
	wp.Spec.Template = sampleWorkerPoolPodTemplate()
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		return err == nil && dep.Spec.Template.Spec.NodeSelector["workload"] == "substrate", nil
	})

	updateWorkerPoolSpec(t, ctx, wp, "update WorkerPool template", func(current *atev1alpha1.WorkerPool) {
		current.Spec.Template.NodeSelector = map[string]string{"workload": "updated"}
	})

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil || len(dep.Spec.Template.Spec.Containers) == 0 {
			return false, nil
		}
		podSpec := dep.Spec.Template.Spec
		return podSpec.NodeSelector["workload"] == "updated" &&
			podSpec.Containers[0].Resources.Requests.Cpu().String() == "500m", nil
	})
}

// TestWorkerPoolPodTemplateClear verifies that clearing template.nodeSelector
// removes it from the managed Deployment.
func TestWorkerPoolPodTemplateClear(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-template-clear", "default", 1, "ateom:v1")
	wp.Spec.Template = sampleWorkerPoolPodTemplate()
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		return err == nil && dep.Spec.Template.Spec.NodeSelector["workload"] == "substrate", nil
	})

	updateWorkerPoolSpec(t, ctx, wp, "clear WorkerPool nodeSelector", func(current *atev1alpha1.WorkerPool) {
		current.Spec.Template.NodeSelector = nil
	})

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil {
			return false, nil
		}
		return len(dep.Spec.Template.Spec.NodeSelector) == 0, nil
	})
}

// TestWorkerPoolPodTemplateClearAll verifies that removing spec.template clears
// all pod template fields owned by the workerpool-controller.
func TestWorkerPoolPodTemplateClearAll(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-template-clear-all", "default", 1, "ateom:v1")
	wp.Spec.Template = sampleWorkerPoolPodTemplate()
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil || len(dep.Spec.Template.Spec.Containers) == 0 {
			return false, nil
		}
		podSpec := dep.Spec.Template.Spec
		container := podSpec.Containers[0]
		// The template's toleration plus the controller's sandbox class one.
		return podSpec.NodeSelector["workload"] == "substrate" &&
			len(podSpec.Tolerations) == 2 &&
			podSpec.PriorityClassName == "substrate-workers" &&
			podSpec.ServiceAccountName == "substrate-worker" &&
			podSpec.Affinity != nil &&
			podSpec.Affinity.NodeAffinity != nil &&
			container.Resources.Requests.Cpu().String() == "500m", nil
	})

	updateWorkerPoolSpec(t, ctx, wp, "clear WorkerPool template", func(current *atev1alpha1.WorkerPool) {
		current.Spec.Template = nil
	})

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil || len(dep.Spec.Template.Spec.Containers) == 0 {
			return false, nil
		}
		podSpec := dep.Spec.Template.Spec
		container := podSpec.Containers[0]
		// Only the controller's sandbox class toleration survives the clear.
		return len(podSpec.NodeSelector) == 0 &&
			len(podSpec.Tolerations) == 1 &&
			podSpec.Tolerations[0].Key == sandboxClassTaintKey &&
			podSpec.PriorityClassName == "" &&
			podSpec.ServiceAccountName == "default" &&
			(podSpec.Affinity == nil || podSpec.Affinity.NodeAffinity == nil) &&
			len(container.Resources.Limits) == 0 &&
			len(container.Resources.Requests) == 0, nil
	})
}

// TestSSARevertsOwnedPodTemplateFields verifies that if an external actor
// changes pod template fields owned by the workerpool-controller, the
// controller reverts them on the next reconcile.
func TestSSARevertsOwnedPodTemplateFields(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-ssa-template", "default", 1, "ateom:v1")
	wp.Spec.Template = sampleWorkerPoolPodTemplate()
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		return err == nil && dep.Spec.Template.Spec.NodeSelector["workload"] == "substrate", nil
	})

	updateDeploymentSpec(t, ctx, wp, "rogue update", func(dep *appsv1.Deployment) {
		dep.Spec.Template.Spec.NodeSelector = map[string]string{"workload": "rogue"}
	})

	eventually(t, func(ctx context.Context) (bool, error) {
		dep, err := getDeployment(ctx, wp)
		if err != nil {
			return false, nil
		}
		return dep.Spec.Template.Spec.NodeSelector["workload"] == "substrate", nil
	})
}

// TestReplicasValidationRejectsNegative verifies that the API server rejects a
// WorkerPool whose spec.replicas is negative.
func TestReplicasValidationRejectsNegative(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-neg-replicas", "default", -1, "ateom:v1")
	err := k8sClient.Create(ctx, wp)
	if err == nil {
		deleteOnCleanup(t, wp)
		t.Fatal("expected creation with negative replicas to fail, but it succeeded")
	}
	if !k8errors.IsInvalid(err) {
		t.Fatalf("expected Invalid error, got: %v", err)
	}
}

// --- helpers ---

func makeWorkerPool(name, ns string, replicas int32, image string) *atev1alpha1.WorkerPool {
	return &atev1alpha1.WorkerPool{
		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
		Spec: atev1alpha1.WorkerPoolSpec{
			Replicas:       replicas,
			WorkerImage:    image,
			SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{Name: atev1alpha1.SandboxClassGvisor}},
		},
	}
}

// deleteOnCleanup registers a cleanup that deletes obj. It uses its own
// context because t.Context() is already canceled when cleanups run.
func deleteOnCleanup(t *testing.T, obj client.Object) {
	t.Helper()
	t.Cleanup(func() {
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		if err := client.IgnoreNotFound(k8sClient.Delete(ctx, obj)); err != nil {
			t.Errorf("cleanup: delete %s: %v", obj.GetName(), err)
		}
	})
}

func getDeployment(ctx context.Context, wp *atev1alpha1.WorkerPool) (*appsv1.Deployment, error) {
	dep := &appsv1.Deployment{}
	err := k8sClient.Get(ctx, types.NamespacedName{
		Name:      wp.Name,
		Namespace: wp.Namespace,
	}, dep)
	return dep, err
}

// updateDeployment re-fetches the managed Deployment, applies mutate, and
// persists it via update, retrying on optimistic-concurrency conflicts. The
// controller reconciles the Deployment concurrently (bumping its
// resourceVersion on every SSA apply), so a plain get-then-update flakes with
// "the object has been modified" under load.
func updateDeployment(t *testing.T, ctx context.Context, wp *atev1alpha1.WorkerPool, action string, mutate func(*appsv1.Deployment), update func(*appsv1.Deployment) error) {
	t.Helper()
	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		dep, err := getDeployment(ctx, wp)
		if err != nil {
			return err
		}
		mutate(dep)
		return update(dep)
	})
	if err != nil {
		t.Fatalf("%s: %v", action, err)
	}
}

func updateDeploymentSpec(t *testing.T, ctx context.Context, wp *atev1alpha1.WorkerPool, action string, mutate func(*appsv1.Deployment)) {
	t.Helper()
	updateDeployment(t, ctx, wp, action, mutate, func(dep *appsv1.Deployment) error {
		return k8sClient.Update(ctx, dep)
	})
}

// updateDeploymentStatus is updateDeploymentSpec for the status subresource.
func updateDeploymentStatus(t *testing.T, ctx context.Context, wp *atev1alpha1.WorkerPool, action string, mutate func(*appsv1.Deployment)) {
	t.Helper()
	updateDeployment(t, ctx, wp, action, mutate, func(dep *appsv1.Deployment) error {
		return k8sClient.Status().Update(ctx, dep)
	})
}

func updateWorkerPoolSpec(t *testing.T, ctx context.Context, wp *atev1alpha1.WorkerPool, action string, mutate func(*atev1alpha1.WorkerPool)) {
	t.Helper()
	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		current := &atev1alpha1.WorkerPool{}
		if err := k8sClient.Get(ctx, types.NamespacedName{Name: wp.Name, Namespace: wp.Namespace}, current); err != nil {
			return err
		}
		mutate(current)
		return k8sClient.Update(ctx, current)
	})
	if err != nil {
		t.Fatalf("%s: %v", action, err)
	}
}

// eventually polls condition every 100ms until it returns true or 15s elapses.
func eventually(t *testing.T, condition func(ctx context.Context) (bool, error)) {
	t.Helper()
	if err := wait.PollUntilContextTimeout(t.Context(), 100*time.Millisecond, 15*time.Second, true, condition); err != nil {
		t.Fatalf("condition not met within timeout: %v", err)
	}
}

// TestSyncStatus_ReadyReplicas verifies that a Deployment reporting 3 replicas
// with 2 ready propagates to WorkerPool.status.
func TestSyncStatus_ReadyReplicas(t *testing.T) {
	t.Parallel()
	ctx := t.Context()
	wp := makeWorkerPool("test-sync-ready", "default", 3, "ateom:v1")
	if err := k8sClient.Create(ctx, wp); err != nil {
		t.Fatalf("create WorkerPool: %v", err)
	}
	deleteOnCleanup(t, wp)
	eventually(t, func(ctx context.Context) (bool, error) {
		_, err := getDeployment(ctx, wp)
		return err == nil, nil
	})
	// Simulate the deployment controller reporting 3 pods, 2 of them ready.
	updateDeploymentStatus(t, ctx, wp, "patch Deployment status", func(dep *appsv1.Deployment) {
		dep.Status.Replicas = 3
		dep.Status.ReadyReplicas = 2
	})
	eventually(t, func(ctx context.Context) (bool, error) {
		current := &atev1alpha1.WorkerPool{}
		if err := k8sClient.Get(ctx, types.NamespacedName{Name: wp.Name, Namespace: wp.Namespace}, current); err != nil {
			return false, nil
		}
		return current.Status.Replicas == 3 && current.Status.ReadyReplicas == 2, nil
	})
}

// TestWorkerPoolMetrics verifies that the registered callback observes
// spec.replicas and status.readyReplicas per WorkerPool, labeled with the pool
// namespace and name. A fake client keeps this off the envtest reconciler,
// which would otherwise resync status out from under the assertion.
func TestWorkerPoolMetrics(t *testing.T) {
	t.Parallel()
	scheme := runtime.NewScheme()
	if err := atev1alpha1.AddToScheme(scheme); err != nil {
		t.Fatalf("add scheme: %v", err)
	}
	wp := makeWorkerPool("test-metrics", "default", 4, "ateom:v1")
	wp.Status = atev1alpha1.WorkerPoolStatus{Replicas: 4, ReadyReplicas: 2}
	reader := sdkmetric.NewManualReader()
	r := &WorkerPoolReconciler{
		Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(wp).Build(),
	}
	if err := r.InitMetrics(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)).Meter("test")); err != nil {
		t.Fatalf("InitMetrics: %v", err)
	}
	var rm metricdata.ResourceMetrics
	if err := reader.Collect(t.Context(), &rm); err != nil {
		t.Fatalf("collect metrics: %v", err)
	}
	wantAttrs := attribute.NewSet(
		ateattr.WorkerPoolNamespaceKey.String(wp.Namespace),
		ateattr.WorkerPoolNameKey.String(wp.Name),
	)
	got := map[string]int64{}
	for _, sm := range rm.ScopeMetrics {
		for _, m := range sm.Metrics {
			sum, ok := m.Data.(metricdata.Sum[int64])
			if !ok {
				continue
			}
			for _, dp := range sum.DataPoints {
				if dp.Attributes.Equals(&wantAttrs) {
					got[m.Name] = dp.Value
				}
			}
		}
	}
	want := map[string]int64{
		"ate.workerpool.desired_workers": 4,
		"ate.workerpool.ready_workers":   2,
	}
	if !reflect.DeepEqual(got, want) {
		t.Errorf("observed %v, want %v", got, want)
	}
}