ateapi: key the workerpool metric by namespace (#632)

`ate.workerpool.workers` tallies by pool name, state, and sandbox class,
but a WorkerPool is namespaced, so two pools that share a name sum into
a single series. This is reachable today: the counter demo and the
autoscaled-workerpool demo both create a pool named `counter`, and with
both installed the demo HPA scales on the other pool's assigned workers.
Adds `ate.workerpool.namespace` to the tally key and attributes, and
pins it in the demo's adapter query and HPA selector.

cmd/ateapi/internal/controlapi/metrics.go

@@ -36,7 +36,7 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker,
 	counter, err := meter.Int64ObservableUpDownCounter(
 		workerpoolWorkersMetric,
 		metric.WithUnit("{worker}"),
-		metric.WithDescription("Number of workers by pool, worker state, and sandbox class."),
+		metric.WithDescription("Number of workers by pool namespace, pool, worker state, and sandbox class."),
 	)
 	if err != nil {
 		return fmt.Errorf("create %s updowncounter: %w", workerpoolWorkersMetric, err)
@@ -48,7 +48,7 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker,
 			// Worker cache unavailable (warmup/reconnect): skip the whole observation.
 			return nil
 		}
-		type key struct{ pool, state, class string }
+		type key struct{ namespace, pool, state, class string }
 		tally := make(map[key]int64)
 		// Seed both states at 0 for every known pool so a saturated or empty pool
 		// reports 0, not an absent series that breaks idle==0 alerts. A failed
@@ -59,8 +59,8 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker,
 				if class == "" {
 					class = string(atev1alpha1.SandboxClassGvisor)
 				}
-				tally[key{p.Name, ateattr.WorkerStateIdle, class}] = 0
-				tally[key{p.Name, ateattr.WorkerStateAssigned, class}] = 0
+				tally[key{p.Namespace, p.Name, ateattr.WorkerStateIdle, class}] = 0
+				tally[key{p.Namespace, p.Name, ateattr.WorkerStateAssigned, class}] = 0
 			}
 		}
 		for _, w := range ws {
@@ -68,10 +68,11 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker,
 			if w.GetAssignment() != nil {
 				state = ateattr.WorkerStateAssigned
 			}
-			tally[key{w.GetWorkerPool(), state, w.GetSandboxClass()}]++
+			tally[key{w.GetWorkerNamespace(), w.GetWorkerPool(), state, w.GetSandboxClass()}]++
 		}
 		for k, n := range tally {
 			o.ObserveInt64(counter, n, metric.WithAttributes(
+				ateattr.WorkerPoolNamespaceKey.String(k.namespace),
 				ateattr.WorkerPoolNameKey.String(k.pool),
 				ateattr.WorkerStateKey.String(k.state),
 				ateattr.SandboxClassKey.String(k.class),

cmd/ateapi/internal/controlapi/metrics_test.go

@@ -67,21 +67,38 @@ func mustMetric(t *testing.T, reader *sdkmetric.ManualReader, name string) metri
 	return m
 }
 
-func worker(pool, class string, assigned bool) *ateapipb.Worker {
-	w := &ateapipb.Worker{WorkerPool: pool, SandboxClass: class}
+func worker(namespace, pool, class string, assigned bool) *ateapipb.Worker {
+	w := &ateapipb.Worker{WorkerNamespace: namespace, WorkerPool: pool, SandboxClass: class}
 	if assigned {
 		w.Assignment = &ateapipb.Assignment{}
 	}
 	return w
 }
 
+type series struct{ namespace, pool, state, class string }
+
+func seriesCounts(sum metricdata.Sum[int64]) map[series]int64 {
+	got := make(map[series]int64)
+	for _, dp := range sum.DataPoints {
+		namespace, _ := dp.Attributes.Value(ateattr.WorkerPoolNamespaceKey)
+		pool, _ := dp.Attributes.Value(ateattr.WorkerPoolNameKey)
+		state, _ := dp.Attributes.Value(ateattr.WorkerStateKey)
+		class, _ := dp.Attributes.Value(ateattr.SandboxClassKey)
+		got[series{namespace.AsString(), pool.AsString(), state.AsString(), class.AsString()}] = dp.Value
+	}
+	return got
+}
+
+// TestWorkerCountTally covers the tally, including two pools that share a name in
+// different namespaces: a WorkerPool is namespaced, so those must stay two series.
 func TestWorkerCountTally(t *testing.T) {
 	workers := func() ([]*ateapipb.Worker, error) {
 		return []*ateapipb.Worker{
-			worker("pool-a", "gvisor", false),
-			worker("pool-a", "gvisor", false),
-			worker("pool-a", "gvisor", true),
-			worker("pool-b", "microvm", false),
+			worker("ns-1", "pool-a", "gvisor", false),
+			worker("ns-1", "pool-a", "gvisor", false),
+			worker("ns-1", "pool-a", "gvisor", true),
+			worker("ns-1", "pool-b", "microvm", false),
+			worker("ns-2", "pool-a", "gvisor", false),
 		}, nil
 	}
 	reader := newWorkerCountReader(t, workers, noPools)
@@ -98,18 +115,12 @@ func TestWorkerCountTally(t *testing.T) {
 		t.Errorf("IsMonotonic = true, want false (updowncounter, not counter)")
 	}
 
-	type key struct{ pool, state, class string }
-	got := make(map[key]int64)
-	for _, dp := range sum.DataPoints {
-		pool, _ := dp.Attributes.Value(ateattr.WorkerPoolNameKey)
-		state, _ := dp.Attributes.Value(ateattr.WorkerStateKey)
-		class, _ := dp.Attributes.Value(ateattr.SandboxClassKey)
-		got[key{pool.AsString(), state.AsString(), class.AsString()}] = dp.Value
-	}
-	want := map[key]int64{
-		{"pool-a", ateattr.WorkerStateIdle, "gvisor"}:     2,
-		{"pool-a", ateattr.WorkerStateAssigned, "gvisor"}: 1,
-		{"pool-b", ateattr.WorkerStateIdle, "microvm"}:    1,
+	got := seriesCounts(sum)
+	want := map[series]int64{
+		{"ns-1", "pool-a", ateattr.WorkerStateIdle, "gvisor"}:     2,
+		{"ns-1", "pool-a", ateattr.WorkerStateAssigned, "gvisor"}: 1,
+		{"ns-1", "pool-b", ateattr.WorkerStateIdle, "microvm"}:    1,
+		{"ns-2", "pool-a", ateattr.WorkerStateIdle, "gvisor"}:     1,
 	}
 	if len(got) != len(want) {
 		t.Fatalf("got %d series, want %d: %v", len(got), len(want), got)
@@ -134,9 +145,9 @@ func TestWorkerCountSkipsWhenCacheNotReady(t *testing.T) {
 	}
 }
 
-func workerPool(name string, class atev1alpha1.SandboxClass) *atev1alpha1.WorkerPool {
+func workerPool(namespace, name string, class atev1alpha1.SandboxClass) *atev1alpha1.WorkerPool {
 	return &atev1alpha1.WorkerPool{
-		ObjectMeta: metav1.ObjectMeta{Name: name},
+		ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
 		Spec:       atev1alpha1.WorkerPoolSpec{SandboxClass: class},
 	}
 }
@@ -147,29 +158,22 @@ func workerPool(name string, class atev1alpha1.SandboxClass) *atev1alpha1.Worker
 func TestWorkerCountSeedsZeroForKnownPools(t *testing.T) {
 	pools := func(labels.Selector) ([]*atev1alpha1.WorkerPool, error) {
 		return []*atev1alpha1.WorkerPool{
-			workerPool("pool-a", ""),
-			workerPool("pool-c", atev1alpha1.SandboxClassMicroVM),
+			workerPool("ns-1", "pool-a", ""),
+			workerPool("ns-2", "pool-a", atev1alpha1.SandboxClassMicroVM),
 		}, nil
 	}
 	workers := func() ([]*ateapipb.Worker, error) {
-		return []*ateapipb.Worker{worker("pool-a", "gvisor", true)}, nil
+		return []*ateapipb.Worker{worker("ns-1", "pool-a", "gvisor", true)}, nil
 	}
 	reader := newWorkerCountReader(t, workers, pools)
 
 	sum := mustMetric(t, reader, workerpoolWorkersMetric).Data.(metricdata.Sum[int64])
-	type key struct{ pool, state, class string }
-	got := make(map[key]int64)
-	for _, dp := range sum.DataPoints {
-		pool, _ := dp.Attributes.Value(ateattr.WorkerPoolNameKey)
-		state, _ := dp.Attributes.Value(ateattr.WorkerStateKey)
-		class, _ := dp.Attributes.Value(ateattr.SandboxClassKey)
-		got[key{pool.AsString(), state.AsString(), class.AsString()}] = dp.Value
-	}
-	want := map[key]int64{
-		{"pool-a", ateattr.WorkerStateIdle, "gvisor"}:      0,
-		{"pool-a", ateattr.WorkerStateAssigned, "gvisor"}:  1,
-		{"pool-c", ateattr.WorkerStateIdle, "microvm"}:     0,
-		{"pool-c", ateattr.WorkerStateAssigned, "microvm"}: 0,
+	got := seriesCounts(sum)
+	want := map[series]int64{
+		{"ns-1", "pool-a", ateattr.WorkerStateIdle, "gvisor"}:      0,
+		{"ns-1", "pool-a", ateattr.WorkerStateAssigned, "gvisor"}:  1,
+		{"ns-2", "pool-a", ateattr.WorkerStateIdle, "microvm"}:     0,
+		{"ns-2", "pool-a", ateattr.WorkerStateAssigned, "microvm"}: 0,
 	}
 	if len(got) != len(want) {
 		t.Fatalf("got %d series, want %d: %v", len(got), len(want), got)

demos/autoscaled-workerpool/README.md

@@ -52,8 +52,9 @@ The example HPA uses an **External** metric with target type **AverageValue**:
 desiredReplicas = ceil( metricValue / target.averageValue )
 ```
 
-where `metricValue = max(ate_workerpool_workers{name=<pool>, state=assigned})`:
-the pool's assigned-worker count.
+where `metricValue = max(ate_workerpool_workers{namespace=<ns>, name=<pool>, state=assigned})`:
+the pool's assigned-worker count. WorkerPool names are only unique within a
+namespace, so the selector must pin both.
 
 `averageValue` is the target **assigned-workers-per-replica**:
 
@@ -98,7 +99,7 @@ Confirm that `prometheus-adapter` is serving the external metric:
 kubectl get apiservice v1beta1.external.metrics.k8s.io          # Available=True
 
 # 2. The external metric resolves for the counter pool
-kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/ate-demo-autoscaled-workerpool/ate_workerpool_workers?labelSelector=ate_worker_state%3Dassigned,ate_workerpool_name%3Dcounter"
+kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/ate-demo-autoscaled-workerpool/ate_workerpool_workers?labelSelector=ate_worker_state%3Dassigned,ate_workerpool_namespace%3Date-demo-autoscaled-workerpool,ate_workerpool_name%3Dcounter"
 ```
 
 ## How to Use

demos/autoscaled-workerpool/hpa-kind.yaml

@@ -36,6 +36,9 @@ spec:
         name: ate_workerpool_workers
         selector:
           matchLabels:
+            # The namespace is required: pool names are unique per namespace, and
+            # the counter demo ships a WorkerPool named "counter" too.
+            ate_workerpool_namespace: ate-demo-autoscaled-workerpool
             ate_workerpool_name: counter
             ate_worker_state: assigned
       target:

demos/autoscaled-workerpool/prometheus-adapter.yaml

@@ -31,9 +31,10 @@ metadata:
 # The external rule maps the Prometheus series ate_workerpool_workers to an
 # external metric of the same name. An HPA's metric.selector.matchLabels are
 # substituted into <<.LabelMatchers>>, so a selector of
-# {ate_workerpool_name: shared-pool, ate_worker_state: assigned} yields
-# max(ate_workerpool_workers{ate_workerpool_name="shared-pool",ate_worker_state="assigned"})
-# — the pool's assigned-worker count.
+# {ate_workerpool_namespace: demo-ns, ate_workerpool_name: shared-pool, ate_worker_state: assigned}
+# yields max(ate_workerpool_workers{ate_workerpool_namespace="demo-ns",ate_workerpool_name="shared-pool",ate_worker_state="assigned"}),
+# the pool's assigned-worker count. Selecting on the name alone would match
+# every same-named pool in the cluster, so the namespace belongs in the selector.
 
 # We use max() rather than sum() because ate-api-server is sharded;
 # every replica reports the global worker tally.
@@ -53,7 +54,7 @@ data:
         as: "$1"
       # max() across ate-api-server replicas since each replica reports the
       # global count.
-      metricsQuery: 'max(<<.Series>>{<<.LabelMatchers>>}) by (ate_workerpool_name, ate_worker_state)'
+      metricsQuery: 'max(<<.Series>>{<<.LabelMatchers>>}) by (ate_workerpool_namespace, ate_workerpool_name, ate_worker_state)'
 ---
 apiVersion: apps/v1
 kind: Deployment

internal/ateattr/ateattr.go

@@ -49,10 +49,13 @@ const (
 // name/uid, atespace) is absent by design; it belongs on spans and logs.
 // WorkerStateKey stays worker-rooted rather than nesting under the pool so it
 // can grow siblings.
+// WorkerPoolNamespaceKey pairs with WorkerPoolNameKey: a WorkerPool is
+// namespaced, so the name alone does not identify one.
 const (
-	WorkerPoolNameKey = attribute.Key("ate.workerpool.name")
-	WorkerStateKey    = attribute.Key("ate.worker.state")
-	SandboxClassKey   = attribute.Key("ate.sandbox.class")
+	WorkerPoolNamespaceKey = attribute.Key("ate.workerpool.namespace")
+	WorkerPoolNameKey      = attribute.Key("ate.workerpool.name")
+	WorkerStateKey         = attribute.Key("ate.worker.state")
+	SandboxClassKey        = attribute.Key("ate.sandbox.class")
 )
 
 // Values for WorkerStateKey. Only idle and assigned are representable today;

internal/ateattr/ateattr_test.go

@@ -148,6 +148,7 @@ func TestKeySpellings(t *testing.T) {
 		{TemplateNameKey, "ate.template.name"},
 		{TemplateNamespaceKey, "ate.template.namespace"},
 		{ActorVersionKey, "ate.actor.version"},
+		{WorkerPoolNamespaceKey, "ate.workerpool.namespace"},
 		{WorkerPoolNameKey, "ate.workerpool.name"},
 		{WorkerStateKey, "ate.worker.state"},
 		{SandboxClassKey, "ate.sandbox.class"},