mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-11-12 10:35:06 +08:00
Part of #791 (PR 3 of the [implementation plan](https://github.com/agent-substrate/substrate/issues/791#issuecomment-5226674097)). Two behavior-preserving refactors needed before suspending a PAUSED actor, which has no worker assignment: **Dialer** — extract `dialAteletOnNode` from `DialForWorker`'s tail (byNode index lookup → single-atelet check → per-pod-UID connection cache → pinned-mTLS dial) and expose `DialForNode(nodeName)` for actors whose state is pinned to a node without a worker pod (a PAUSED actor's local snapshot). A node with no atelet in the informer cache returns the new sentinel `ErrNoAteletOnNode` — documented as retryable (atelet restarting / informer lag), distinct from the crash-worthy `ErrWorkerPodNotFound`. `DialForWorker` behavior is unchanged apart from error wrapping in the zero-atelets case. **FinalizeSuspendedStep** — hoist finalization (ActorSnapshot record, `LatestSnapshot`, the `SUSPENDED` flip, clearing `InProgressSnapshot`/`WorkerAssignment`/`LocalSnapshotInfo`) out of the `if assignment != nil` worker-freeing branch so it always runs. Previously, a SUSPENDING actor with no assignment was silently skipped — the workflow reported success while the actor stayed SUSPENDING forever. Unreachable today (`CallAteletSuspendStep`'s prerequisite crashes nil-assignment actors first), but load-bearing for the paused-origin suspend, where no worker exists by design. Tests: - `TestDialForNode`: no atelet on node → `ErrNoAteletOnNode`; two atelets → error; happy path asserts pod-UID-pinned credentials and connection caching. - `TestFinalizeSuspendedStep_NoAssignment`: SUSPENDING actor with `LocalSnapshotInfo` and an in-progress URI but no assignment → SUSPENDED, ActorSnapshot created at the URI, `LatestSnapshot` set, in-progress fields and `LocalSnapshotInfo` cleared. `go test -race ./cmd/ateapi/...` (incl. the functional suite covering the running-path suspend end-to-end), `go vet`, `gofmt` clean. Independent of #810 and #812. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
295 lines
9.2 KiB
Go
295 lines
9.2 KiB
Go
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controlapi
|
|
|
|
import (
|
|
"crypto/ecdsa"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"errors"
|
|
"math/big"
|
|
"net/url"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/agent-substrate/substrate/internal/substratex509"
|
|
"github.com/spiffe/go-spiffe/v2/bundle/x509bundle"
|
|
"github.com/spiffe/go-spiffe/v2/spiffeid"
|
|
"google.golang.org/grpc/credentials"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/tools/cache"
|
|
)
|
|
|
|
const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet"
|
|
|
|
// makeTestCA mints a self-signed CA and returns it along with an X.509 bundle
|
|
// containing it as the sole authority for the cluster.local trust domain.
|
|
func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509bundle.Bundle) {
|
|
t.Helper()
|
|
|
|
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
t.Fatalf("generating CA key: %v", err)
|
|
}
|
|
template := &x509.Certificate{
|
|
SerialNumber: big.NewInt(1),
|
|
Subject: pkix.Name{CommonName: "test-ca"},
|
|
NotBefore: time.Now().Add(-time.Hour),
|
|
NotAfter: time.Now().Add(time.Hour),
|
|
IsCA: true,
|
|
BasicConstraintsValid: true,
|
|
KeyUsage: x509.KeyUsageCertSign,
|
|
}
|
|
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
|
if err != nil {
|
|
t.Fatalf("creating CA certificate: %v", err)
|
|
}
|
|
cert, err := x509.ParseCertificate(der)
|
|
if err != nil {
|
|
t.Fatalf("parsing CA certificate: %v", err)
|
|
}
|
|
td := spiffeid.RequireTrustDomainFromString("cluster.local")
|
|
bundle := x509bundle.FromX509Authorities(td, []*x509.Certificate{cert})
|
|
return cert, key, bundle
|
|
}
|
|
|
|
// leafOpts controls the contents of a test leaf certificate.
|
|
type leafOpts struct {
|
|
// podUID, if non-empty, is embedded in a PodIdentity extension.
|
|
podUID string
|
|
// spiffeID, if non-empty, is added as a URI SAN.
|
|
spiffeID string
|
|
// noServerAuth omits the serverAuth EKU.
|
|
noServerAuth bool
|
|
}
|
|
|
|
// makeLeafCert mints a server leaf certificate signed by the given CA.
|
|
func makeLeafCert(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, opts leafOpts) *x509.Certificate {
|
|
t.Helper()
|
|
|
|
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
t.Fatalf("generating leaf key: %v", err)
|
|
}
|
|
template := &x509.Certificate{
|
|
SerialNumber: big.NewInt(2),
|
|
NotBefore: time.Now().Add(-time.Hour),
|
|
NotAfter: time.Now().Add(time.Hour),
|
|
KeyUsage: x509.KeyUsageDigitalSignature,
|
|
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
}
|
|
if opts.noServerAuth {
|
|
template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
|
|
}
|
|
if opts.spiffeID != "" {
|
|
uri, err := url.Parse(opts.spiffeID)
|
|
if err != nil {
|
|
t.Fatalf("parsing SPIFFE ID %q: %v", opts.spiffeID, err)
|
|
}
|
|
template.URIs = []*url.URL{uri}
|
|
}
|
|
if opts.podUID != "" {
|
|
// AddPodIdentityToCertificate requires all fields to be non-empty;
|
|
// only PodUID matters to these tests.
|
|
err := substratex509.AddPodIdentityToCertificate(&substratex509.PodIdentity{
|
|
Namespace: "ate-system",
|
|
ServiceAccountName: "atelet",
|
|
ServiceAccountUID: "sa-uid",
|
|
PodName: "atelet-abc",
|
|
PodUID: opts.podUID,
|
|
NodeName: "node-1",
|
|
NodeUID: "node-uid",
|
|
}, template)
|
|
if err != nil {
|
|
t.Fatalf("adding PodIdentity extension: %v", err)
|
|
}
|
|
}
|
|
der, err := x509.CreateCertificate(rand.Reader, template, ca, &key.PublicKey, caKey)
|
|
if err != nil {
|
|
t.Fatalf("creating leaf certificate: %v", err)
|
|
}
|
|
cert, err := x509.ParseCertificate(der)
|
|
if err != nil {
|
|
t.Fatalf("parsing leaf certificate: %v", err)
|
|
}
|
|
return cert
|
|
}
|
|
|
|
func TestVerifyAteletServerCert(t *testing.T) {
|
|
ca, caKey, bundle := makeTestCA(t)
|
|
otherCA, otherCAKey, _ := makeTestCA(t)
|
|
|
|
expectedID := spiffeid.RequireFromString(testAteletSPIFFEID)
|
|
|
|
const uid = "5a2e1c9f-0b57-4a52-9f6e-2f6d3a1b8c4d"
|
|
|
|
tests := []struct {
|
|
name string
|
|
leaf *x509.Certificate
|
|
expectedUID string
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "matching UID succeeds",
|
|
leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: uid, spiffeID: testAteletSPIFFEID}),
|
|
expectedUID: uid,
|
|
},
|
|
{
|
|
name: "mismatched UID fails",
|
|
leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: "some-other-uid", spiffeID: testAteletSPIFFEID}),
|
|
expectedUID: uid,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "missing pod UID extension fails",
|
|
leaf: makeLeafCert(t, ca, caKey, leafOpts{spiffeID: testAteletSPIFFEID}),
|
|
expectedUID: uid,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "cert from untrusted CA fails",
|
|
leaf: makeLeafCert(t, otherCA, otherCAKey, leafOpts{podUID: uid, spiffeID: testAteletSPIFFEID}),
|
|
expectedUID: uid,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "wrong SPIFFE ID fails",
|
|
leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: uid, spiffeID: "spiffe://cluster.local/ns/other/sa/other"}),
|
|
expectedUID: uid,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "missing URI SAN fails",
|
|
leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: uid}),
|
|
expectedUID: uid,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "missing serverAuth EKU fails",
|
|
leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: uid, spiffeID: testAteletSPIFFEID, noServerAuth: true}),
|
|
expectedUID: uid,
|
|
wantErr: true,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
verify, err := verifyAteletServerCert(bundle, expectedID, tc.expectedUID)
|
|
if err != nil {
|
|
t.Fatalf("constructing verifier: %v", err)
|
|
}
|
|
err = verify(tls.ConnectionState{
|
|
PeerCertificates: []*x509.Certificate{tc.leaf},
|
|
})
|
|
if gotErr := err != nil; gotErr != tc.wantErr {
|
|
t.Fatalf("verify returned error %v, wantErr=%v", err, tc.wantErr)
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("no peer certificate fails", func(t *testing.T) {
|
|
verify, err := verifyAteletServerCert(bundle, expectedID, uid)
|
|
if err != nil {
|
|
t.Fatalf("constructing verifier: %v", err)
|
|
}
|
|
if err := verify(tls.ConnectionState{}); err == nil {
|
|
t.Fatal("verify succeeded, want error")
|
|
}
|
|
})
|
|
|
|
t.Run("empty expected UID fails at construction", func(t *testing.T) {
|
|
if _, err := verifyAteletServerCert(bundle, expectedID, ""); err == nil {
|
|
t.Fatal("verifyAteletServerCert succeeded, want error")
|
|
}
|
|
})
|
|
}
|
|
|
|
// newTestAteletIndexer builds an indexer with the production byNode index
|
|
// shape, holding the given atelet pods.
|
|
func newTestAteletIndexer(t *testing.T, pods ...*corev1.Pod) cache.Indexer {
|
|
t.Helper()
|
|
idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{
|
|
byNode: func(obj any) ([]string, error) {
|
|
return []string{obj.(*corev1.Pod).Spec.NodeName}, nil
|
|
},
|
|
})
|
|
for _, p := range pods {
|
|
if err := idx.Add(p); err != nil {
|
|
t.Fatalf("adding pod to indexer: %v", err)
|
|
}
|
|
}
|
|
return idx
|
|
}
|
|
|
|
func TestDialForAteletOnNode(t *testing.T) {
|
|
ateletPod := func(name, uid, node, ip string) *corev1.Pod {
|
|
return &corev1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: name, UID: types.UID(uid)},
|
|
Spec: corev1.PodSpec{NodeName: node},
|
|
Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: ip}}},
|
|
}
|
|
}
|
|
|
|
t.Run("no atelet on node", func(t *testing.T) {
|
|
d := NewAteletDialer(nil, newTestAteletIndexer(t), "", "")
|
|
if _, err := d.DialForAteletOnNode("node1"); !errors.Is(err, ErrNoAteletOnNode) {
|
|
t.Fatalf("DialForAteletOnNode = %v, want ErrNoAteletOnNode", err)
|
|
}
|
|
})
|
|
|
|
t.Run("more than one atelet on node", func(t *testing.T) {
|
|
d := NewAteletDialer(nil, newTestAteletIndexer(t,
|
|
ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"),
|
|
ateletPod("atelet-2", "uid-2", "node1", "10.0.0.2"),
|
|
), "", "")
|
|
_, err := d.DialForAteletOnNode("node1")
|
|
if err == nil || errors.Is(err, ErrNoAteletOnNode) {
|
|
t.Fatalf("DialForAteletOnNode = %v, want a non-ErrNoAteletOnNode error", err)
|
|
}
|
|
})
|
|
|
|
t.Run("dials and caches the node's atelet", func(t *testing.T) {
|
|
d := NewAteletDialer(nil, newTestAteletIndexer(t,
|
|
ateletPod("atelet-1", "uid-1", "node1", "10.0.0.1"),
|
|
), "", "")
|
|
var credsUID string
|
|
d.dialCredentials = func(expectedPodUID string) (credentials.TransportCredentials, error) {
|
|
credsUID = expectedPodUID
|
|
return insecure.NewCredentials(), nil
|
|
}
|
|
|
|
conn, err := d.DialForAteletOnNode("node1")
|
|
if err != nil {
|
|
t.Fatalf("DialForAteletOnNode: %v", err)
|
|
}
|
|
if credsUID != "uid-1" {
|
|
t.Errorf("credentials pinned to pod UID %q, want %q", credsUID, "uid-1")
|
|
}
|
|
again, err := d.DialForAteletOnNode("node1")
|
|
if err != nil {
|
|
t.Fatalf("DialForAteletOnNode (cached): %v", err)
|
|
}
|
|
if again != conn {
|
|
t.Error("second DialForAteletOnNode returned a different connection, want the cached one")
|
|
}
|
|
})
|
|
}
|