workerpool: propagate template metadata to workers (#1058)

Part of #212

> It's a good idea to open an issue first for discussion.

- [ ] Tests pass
- [x] Appropriate changes to documentation are included in the PR

## Summary

- add bounded, validated labels and annotations to
`WorkerPoolPodTemplate`
- propagate that metadata to the generated Deployment and worker pod
template
- reserve the controller-owned `ate.dev/worker-pool` label
- regenerate the WorkerPool CRD and deepcopy code
- add API validation, controller tests, and documentation

I chose intentionally to avoid exposing a complete `PodTemplateSpec` as
per the discussion in the #212.

cmd/atecontroller/internal/controllers/workerpool_apply.go

@@ -72,6 +72,18 @@ const (
 // are declared here. otel, when it carries an endpoint, is propagated to the
 // ateom container so it pushes telemetry to that collector.
 func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettings) *appsv1ac.DeploymentApplyConfiguration {
+	labels := map[string]string{}
+	annotations := map[string]string{}
+	if wp.Spec.Template != nil {
+		for key, value := range wp.Spec.Template.Labels {
+			labels[key] = string(value)
+		}
+		for key, value := range wp.Spec.Template.Annotations {
+			annotations[key] = value
+		}
+	}
+	labels["ate.dev/worker-pool"] = wp.Name
+
 	containerAC := corev1ac.Container().
 		WithName("ateom").
 		WithImage(wp.Spec.AteomImage).
@@ -157,6 +169,8 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin
 	podSpecAC.WithTerminationGracePeriodSeconds(workerTerminationGracePeriodSeconds)
 
 	return appsv1ac.Deployment(wp.Name, wp.Namespace).
+		WithLabels(labels).
+		WithAnnotations(annotations).
 		WithOwnerReferences(metav1ac.OwnerReference().
 			WithAPIVersion(atev1alpha1.GroupVersion.String()).
 			WithKind("WorkerPool").
@@ -169,9 +183,8 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin
 			WithSelector(metav1ac.LabelSelector().
 				WithMatchLabels(map[string]string{"ate.dev/worker-pool": wp.Name})).
 			WithTemplate(corev1ac.PodTemplateSpec().
-				WithLabels(map[string]string{
-					"ate.dev/worker-pool": wp.Name,
-				}).
+				WithLabels(labels).
+				WithAnnotations(annotations).
 				WithSpec(podSpecAC)))
 }
 

cmd/atecontroller/internal/controllers/workerpool_apply_test.go

@@ -209,6 +209,42 @@ func TestBuildDeploymentApplyConfig(t *testing.T) {
 	}
 }
 
+func TestBuildDeploymentApplyConfigMetadata(t *testing.T) {
+	wp := testWorkerPoolApplyConfig(&atev1alpha1.WorkerPoolPodTemplate{
+		Labels: map[string]atev1alpha1.WorkerPoolLabelValue{
+			"project":             "agent-substrate",
+			"team":                "compute",
+			"ate.dev/worker-pool": "incorrect",
+		},
+		Annotations: map[string]string{
+			"policy.example.com/exemption": "sandbox-host",
+		},
+	})
+
+	got := buildDeploymentApplyConfig(wp, ateomOTelSettings{})
+	wantLabels := map[string]string{
+		"project":             "agent-substrate",
+		"team":                "compute",
+		"ate.dev/worker-pool": wp.Name,
+	}
+	wantAnnotations := map[string]string{
+		"policy.example.com/exemption": "sandbox-host",
+	}
+
+	if diff := cmp.Diff(wantLabels, got.Labels); diff != "" {
+		t.Errorf("Deployment labels mismatch (-want +got):\n%s", diff)
+	}
+	if diff := cmp.Diff(wantAnnotations, got.Annotations); diff != "" {
+		t.Errorf("Deployment annotations mismatch (-want +got):\n%s", diff)
+	}
+	if diff := cmp.Diff(wantLabels, got.Spec.Template.Labels); diff != "" {
+		t.Errorf("pod-template labels mismatch (-want +got):\n%s", diff)
+	}
+	if diff := cmp.Diff(wantAnnotations, got.Spec.Template.Annotations); diff != "" {
+		t.Errorf("pod-template annotations mismatch (-want +got):\n%s", diff)
+	}
+}
+
 // TestMicroVMPodShape asserts the micro-VM sandbox class adds the /dev/kvm
 // device (volume + container mount) and node placement (nodeSelector +
 // toleration on ate.dev/sandboxClass); other classes get none of it.
@@ -791,6 +827,7 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf
 	}
 
 	return appsv1ac.Deployment(wp.Name, wp.Namespace).
+		WithLabels(map[string]string{"ate.dev/worker-pool": wp.Name}).
 		WithOwnerReferences(metav1ac.OwnerReference().
 			WithAPIVersion(atev1alpha1.GroupVersion.String()).
 			WithKind("WorkerPool").

docs/api-guide.md

@@ -14,18 +14,29 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage
 | `ateomImage` | `string` | **Required.** The container image for the `ateom` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor`). |
 | `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement) and which `SandboxConfig`s are eligible. |
 | `sandboxConfigName` | `string` | Optional. Name of a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) providing the sandbox binaries and pause image. If empty, the cluster default `SandboxConfig` for the pool's `sandboxClass` is used. |
-| `template` | `WorkerPoolPodTemplate` | **Optional.** Pod scheduling and resource settings for worker pods. |
+| `template` | `WorkerPoolPodTemplate` | **Optional.** Metadata, scheduling, and resource settings for worker workloads. |
 
 #### `WorkerPoolPodTemplate` (`spec.template`)
 
-| Field | Type | Pod mapping |
+| Field | Type | Workload mapping |
 | :--- | :--- | :--- |
+| `labels` | `map[string]string` | Generated Deployment and `spec.template.metadata.labels` (max 64) |
+| `annotations` | `map[string]string` | Generated Deployment and `spec.template.metadata.annotations` (max 64) |
 | `nodeSelector` | `map[string]string` | `spec.nodeSelector` |
 | `tolerations` | `[]Toleration` | `spec.tolerations` (max 16) |
 | `priorityClassName` | `string` | `spec.priorityClassName` |
 | `nodeAffinity` | `NodeAffinity` | `spec.affinity.nodeAffinity` |
 | `resources` | `ResourceRequirements` | `spec.containers[].resources` |
 
+Keys in `ate.dev/` and its subdomains (for example, `policy.ate.dev/`) are
+reserved for controllers and cannot be set in `template.labels` or
+`template.annotations`. Metadata keys and label values must follow Kubernetes
+syntax.
+
+`template.labels` and `template.annotations` only configure Kubernetes workload
+metadata; they do not affect actor scheduling. Actor selectors match
+`WorkerPool.metadata.labels`, not `WorkerPool.spec.template.labels`.
+
 #### Worker Capacity (`spec.template.resources`)
 
 Setting `resources.limits` (CPU and Memory) on a `WorkerPool` establishes each worker pod's **capacity** — the envelope available to host an actor sandbox, taken from the `ateom` container's limits. The scheduler only places an actor on a worker whose capacity is `>=` the actor's declared resource limits (see [Sandbox Right-Sizing](#sandbox-right-sizing-specresources) on the `ActorTemplate`).
@@ -46,6 +57,11 @@ metadata:
 spec:
   replicas: 10
   ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+  template:
+    labels:
+      project: agent-platform
+    annotations:
+      policy.example.com/exemption: sandbox-host
   # sandboxClass defaults to gvisor; the pool resolves to the cluster's default
   # gvisor SandboxConfig unless sandboxConfigName is set.
 ```

manifests/ate-install/generated/ate.dev_workerpools.yaml

@@ -100,9 +100,41 @@ spec:
                   SandboxClass is used.
                 type: string
               template:
-                description: Template holds optional pod scheduling and resource settings
-                  for worker pods.
+                description: Template holds optional metadata, scheduling, and resource
+                  settings for worker workloads.
                 properties:
+                  annotations:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      Annotations are added to the generated Deployment and worker pods. Keys
+                      in the ate.dev domain and its subdomains are reserved for controllers.
+                    maxProperties: 64
+                    type: object
+                    x-kubernetes-validations:
+                    - message: ate.dev and its subdomains are reserved
+                      rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/'))
+                    - message: annotation keys must be valid Kubernetes qualified
+                        names
+                      rule: self.all(key, !format.qualifiedName().validate(key).hasValue())
+                  labels:
+                    additionalProperties:
+                      description: |-
+                        WorkerPoolLabelValue is a Kubernetes label value for generated worker
+                        workloads.
+                      maxLength: 63
+                      pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$
+                      type: string
+                    description: |-
+                      Labels are added to the generated Deployment and worker pods. Keys in
+                      the ate.dev domain and its subdomains are reserved for controllers.
+                    maxProperties: 64
+                    type: object
+                    x-kubernetes-validations:
+                    - message: ate.dev and its subdomains are reserved
+                      rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/'))
+                    - message: label keys must be valid Kubernetes qualified names
+                      rule: self.all(key, !format.qualifiedName().validate(key).hasValue())
                   nodeAffinity:
                     description: |-
                       NodeAffinity scheduling rules for the worker pods. Mapped to

pkg/api/v1alpha1/workerpool_types.go

@@ -19,9 +19,35 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// WorkerPoolPodTemplate defines optional scheduling and resource settings for
-// worker pods. NodeAffinity is mapped to spec.affinity.nodeAffinity on the pod.
+// WorkerPoolLabelValue is a Kubernetes label value for generated worker
+// workloads.
+//
+// +kubebuilder:validation:MaxLength=63
+// +kubebuilder:validation:Pattern=`^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$`
+type WorkerPoolLabelValue string
+
+// WorkerPoolPodTemplate defines optional metadata, scheduling, and resource
+// settings for worker workloads. NodeAffinity is mapped to
+// spec.affinity.nodeAffinity on the pod.
 type WorkerPoolPodTemplate struct {
+	// Labels are added to the generated Deployment and worker pods. Keys in
+	// the ate.dev domain and its subdomains are reserved for controllers.
+	//
+	// +optional
+	// +kubebuilder:validation:MaxProperties=64
+	// +kubebuilder:validation:XValidation:rule="self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/'))",message="ate.dev and its subdomains are reserved"
+	// +kubebuilder:validation:XValidation:rule="self.all(key, !format.qualifiedName().validate(key).hasValue())",message="label keys must be valid Kubernetes qualified names"
+	Labels map[string]WorkerPoolLabelValue `json:"labels,omitempty"`
+
+	// Annotations are added to the generated Deployment and worker pods. Keys
+	// in the ate.dev domain and its subdomains are reserved for controllers.
+	//
+	// +optional
+	// +kubebuilder:validation:MaxProperties=64
+	// +kubebuilder:validation:XValidation:rule="self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/'))",message="ate.dev and its subdomains are reserved"
+	// +kubebuilder:validation:XValidation:rule="self.all(key, !format.qualifiedName().validate(key).hasValue())",message="annotation keys must be valid Kubernetes qualified names"
+	Annotations map[string]string `json:"annotations,omitempty"`
+
 	// NodeSelector is a selector which must be true for the pod to fit on a node.
 	//
 	// +optional
@@ -64,7 +90,7 @@ type WorkerPoolSpec struct {
 	// +required
 	AteomImage string `json:"ateomImage"`
 
-	// Template holds optional pod scheduling and resource settings for worker pods.
+	// Template holds optional metadata, scheduling, and resource settings for worker workloads.
 	//
 	// +optional
 	Template *WorkerPoolPodTemplate `json:"template,omitempty"`

pkg/api/v1alpha1/workerpool_validation_test.go

@@ -16,6 +16,7 @@ package v1alpha1
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"testing"
 
@@ -73,6 +74,13 @@ func TestWorkerPoolValidation(t *testing.T) {
 		name: "valid template",
 		mutate: func(wp *WorkerPool) {
 			wp.Spec.Template = &WorkerPoolPodTemplate{
+				Labels: map[string]WorkerPoolLabelValue{
+					"project":                    "agent-substrate",
+					"policy.example.com/profile": "sandbox_host",
+				},
+				Annotations: map[string]string{
+					"policy.example.com/exemption": "sandbox-host",
+				},
 				NodeSelector: map[string]string{"workload": "substrate"},
 				Tolerations: []corev1.Toleration{{
 					Key:      "gpu",
@@ -92,6 +100,77 @@ func TestWorkerPoolValidation(t *testing.T) {
 			}
 		},
 		wantErr: false,
+	}, {
+		name: "invalid worker label key",
+		mutate: func(wp *WorkerPool) {
+			wp.Spec.Template = &WorkerPoolPodTemplate{Labels: map[string]WorkerPoolLabelValue{"bad key": "value"}}
+		},
+		wantErr: true,
+		errMsg:  "label keys must be valid Kubernetes qualified names",
+	}, {
+		name: "invalid worker label value",
+		mutate: func(wp *WorkerPool) {
+			wp.Spec.Template = &WorkerPoolPodTemplate{Labels: map[string]WorkerPoolLabelValue{"project": "bad value"}}
+		},
+		wantErr: true,
+		errMsg:  "spec.template.labels.project in body should match",
+	}, {
+		name: "reserved ate.dev label",
+		mutate: func(wp *WorkerPool) {
+			wp.Spec.Template = &WorkerPoolPodTemplate{Labels: map[string]WorkerPoolLabelValue{"ate.dev/custom": "value"}}
+		},
+		wantErr: true,
+		errMsg:  "ate.dev and its subdomains are reserved",
+	}, {
+		name: "reserved ate.dev subdomain label",
+		mutate: func(wp *WorkerPool) {
+			wp.Spec.Template = &WorkerPoolPodTemplate{Labels: map[string]WorkerPoolLabelValue{"policy.ate.dev/exemption": "value"}}
+		},
+		wantErr: true,
+		errMsg:  "ate.dev and its subdomains are reserved",
+	}, {
+		name: "reserved ate.dev annotation",
+		mutate: func(wp *WorkerPool) {
+			wp.Spec.Template = &WorkerPoolPodTemplate{Annotations: map[string]string{"ate.dev/custom": "value"}}
+		},
+		wantErr: true,
+		errMsg:  "ate.dev and its subdomains are reserved",
+	}, {
+		name: "reserved ate.dev subdomain annotation",
+		mutate: func(wp *WorkerPool) {
+			wp.Spec.Template = &WorkerPoolPodTemplate{Annotations: map[string]string{"policy.ate.dev/exemption": "value"}}
+		},
+		wantErr: true,
+		errMsg:  "ate.dev and its subdomains are reserved",
+	}, {
+		name: "invalid worker annotation key",
+		mutate: func(wp *WorkerPool) {
+			wp.Spec.Template = &WorkerPoolPodTemplate{Annotations: map[string]string{"bad key": "value"}}
+		},
+		wantErr: true,
+		errMsg:  "annotation keys must be valid Kubernetes qualified names",
+	}, {
+		name: "too many worker labels",
+		mutate: func(wp *WorkerPool) {
+			labels := make(map[string]WorkerPoolLabelValue, 65)
+			for i := range 65 {
+				labels[fmt.Sprintf("label-%d", i)] = "value"
+			}
+			wp.Spec.Template = &WorkerPoolPodTemplate{Labels: labels}
+		},
+		wantErr: true,
+		errMsg:  "spec.template.labels: Too many",
+	}, {
+		name: "too many worker annotations",
+		mutate: func(wp *WorkerPool) {
+			annotations := make(map[string]string, 65)
+			for i := range 65 {
+				annotations[fmt.Sprintf("annotation-%d", i)] = "value"
+			}
+			wp.Spec.Template = &WorkerPoolPodTemplate{Annotations: annotations}
+		},
+		wantErr: true,
+		errMsg:  "spec.template.annotations: Too many",
 	}, {
 		name: "too many tolerations",
 		mutate: func(wp *WorkerPool) {
@@ -180,3 +259,32 @@ func TestWorkerPoolValidation(t *testing.T) {
 		})
 	}
 }
+
+func TestWorkerPoolReservedMetadataUpdate(t *testing.T) {
+	ctx := context.Background()
+	wp := &WorkerPool{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-reserved-metadata-update",
+			Namespace: "default",
+		},
+		Spec: WorkerPoolSpec{
+			Replicas:   1,
+			AteomImage: "example.com/ateom:latest",
+		},
+	}
+	if err := k8sClient.Create(ctx, wp); err != nil {
+		t.Fatalf("create WorkerPool: %v", err)
+	}
+	t.Cleanup(func() { _ = k8sClient.Delete(ctx, wp) })
+
+	wp.Spec.Template = &WorkerPoolPodTemplate{
+		Annotations: map[string]string{"security.ate.dev/exemption": "value"},
+	}
+	err := k8sClient.Update(ctx, wp)
+	if err == nil {
+		t.Fatal("update unexpectedly accepted a reserved annotation")
+	}
+	if want := "ate.dev and its subdomains are reserved"; !strings.Contains(err.Error(), want) {
+		t.Errorf("wrong error:\n  wanted: %q\n     got: %q", want, err.Error())
+	}
+}

pkg/api/v1alpha1/zz_generated.deepcopy.go

@@ -727,6 +727,20 @@ func (in *WorkerPoolList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WorkerPoolPodTemplate) DeepCopyInto(out *WorkerPoolPodTemplate) {
 	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]WorkerPoolLabelValue, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.NodeSelector != nil {
 		in, out := &in.NodeSelector, &out.NodeSelector
 		*out = make(map[string]string, len(*in))