From 98902191516f082bdf2f12f79527da9e34e57331 Mon Sep 17 00:00:00 2001 From: Zoe Zhao <35669350+zoez7@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:43:03 -0700 Subject: [PATCH] Added mTLS between ate system components (#237) Part of [#170](https://github.com/agent-substrate/substrate/issues/170) Establishes mutual TLS between all ate system components, and updates the certificate plumbing it depends on. Main changes: 1. The atenet router now verifies ate apiserver' serving certificate, and presents its client cert to ate apiserver. Previously the connection used `InsecureSkipVerify`. 2. AteApi server verifies atelet's serving cert. Minor bug fixes: 1. Prevent `servicednssigner` from signing a cert with no DNS SANs. Also updated valkey cluster's cert configuration, because it was relying on the cert with empty DNS. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR --- .github/workflows/pr-workflow.yaml | 8 +- cmd/ateapi/internal/controlapi/dialer.go | 115 ++++- cmd/ateapi/internal/controlapi/dialer_test.go | 216 +++++++++ .../internal/controlapi/functional_test.go | 9 +- .../controlapi/workflow_suspend_test.go | 2 +- cmd/ateapi/main.go | 57 ++- cmd/atecontroller/main.go | 22 +- cmd/atelet/main.go | 38 +- cmd/atenet/internal/root.go | 3 +- .../internal/{router.go => router/cmd.go} | 20 +- cmd/atenet/internal/router/config.go | 58 +++ cmd/atenet/internal/router/controller.go | 4 +- cmd/atenet/internal/router/envoyrunner.go | 4 +- cmd/atenet/internal/router/health.go | 4 +- cmd/atenet/internal/router/health_test.go | 12 +- cmd/atenet/internal/router/router.go | 102 +--- cmd/atenet/internal/router/status_test.go | 59 ++- cmd/atenet/internal/router/xds.go | 33 +- cmd/atenet/internal/router/xds_test.go | 2 +- cmd/kubectl-ate/README.md | 4 +- .../podidentitysigner/podidentitysigner.go | 43 +- .../podidentitysigner_test.go | 452 ++++++++++++++++++ .../servicednssigner/servicednssigner.go | 13 +- go.mod | 2 +- hack/install-ate.sh | 77 +-- hack/run-microvm-demo.sh | 20 +- internal/ateapiauth/client.go | 85 ++-- internal/ateapiauth/client_test.go | 275 ++++++++++- internal/ateapiauth/server.go | 156 +++--- internal/ateapiauth/server_test.go | 171 +++++-- internal/ateclient/builder.go | 127 ++--- internal/ateclient/builder_test.go | 147 +++++- internal/ateinterceptors/ateinterceptors.go | 4 + .../credbundle/credbundle.go | 11 + .../credbundle/credbundle_test.go | 0 internal/principal/principal.go | 45 ++ internal/substratex509/substratex509.go | 133 ++++++ internal/substratex509/substratex509_test.go | 232 +++++++++ manifests/ate-install/ate-api-server.yaml | 34 +- manifests/ate-install/ate-controller.yaml | 25 + manifests/ate-install/atelet.yaml | 21 + manifests/ate-install/atenet-router.yaml | 28 +- .../kustomization.yaml | 33 +- .../kind/atelet/kustomization.yaml | 2 + .../{jwt => token-client}/kustomization.yaml | 31 +- .../{jwt => token-client}/patches.yaml | 0 manifests/ate-install/valkey.yaml | 30 +- .../go-spiffe/v2/svid/x509svid/source.go | 7 + .../spiffe/go-spiffe/v2/svid/x509svid/svid.go | 249 ++++++++++ .../go-spiffe/v2/svid/x509svid/verify.go | 116 +++++ .../google.golang.org/grpc/health/client.go | 117 +++++ .../google.golang.org/grpc/health/logging.go | 23 + .../google.golang.org/grpc/health/producer.go | 106 ++++ .../google.golang.org/grpc/health/server.go | 187 ++++++++ vendor/modules.txt | 2 + 55 files changed, 3258 insertions(+), 518 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/dialer_test.go rename cmd/atenet/internal/{router.go => router/cmd.go} (74%) create mode 100644 cmd/atenet/internal/router/config.go create mode 100644 cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go rename {cmd/ateapi/internal => internal}/credbundle/credbundle.go (85%) rename {cmd/ateapi/internal => internal}/credbundle/credbundle_test.go (100%) create mode 100644 internal/principal/principal.go create mode 100644 internal/substratex509/substratex509.go create mode 100644 internal/substratex509/substratex509_test.go rename manifests/ate-install/{kind-jwt => kind-token-client}/kustomization.yaml (56%) rename manifests/ate-install/{jwt => token-client}/kustomization.yaml (60%) rename manifests/ate-install/{jwt => token-client}/patches.yaml (100%) create mode 100644 vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/source.go create mode 100644 vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/svid.go create mode 100644 vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/verify.go create mode 100644 vendor/google.golang.org/grpc/health/client.go create mode 100644 vendor/google.golang.org/grpc/health/logging.go create mode 100644 vendor/google.golang.org/grpc/health/producer.go create mode 100644 vendor/google.golang.org/grpc/health/server.go diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 44fe57b0b..7695db938 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -48,8 +48,8 @@ jobs: strategy: fail-fast: false matrix: - auth-mode: [mtls, jwt] - name: e2e-test (${{ matrix.auth-mode }}) + ateapi-client-auth: [cert, token] + name: e2e-test (${{ matrix.ateapi-client-auth }}) steps: - name: Checkout uses: actions/checkout@v5 @@ -85,11 +85,11 @@ jobs: - name: Create cluster run: hack/create-kind-cluster.sh - name: Install Agent Substrate - run: hack/install-ate-kind.sh --deploy-ate-system --auth-mode=${{ matrix.auth-mode }} + run: hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth=${{ matrix.ateapi-client-auth }} - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and applies the # counter-microvm demo onto the control plane installed above. - run: hack/run-microvm-demo-kind.sh --auth-mode=${{ matrix.auth-mode }} + run: hack/run-microvm-demo-kind.sh --ateapi-client-auth=${{ matrix.ateapi-client-auth }} - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Wait for micro-VM golden snapshot diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 89e8e37a6..a10858491 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -15,12 +15,20 @@ package controlapi import ( + "crypto/tls" + "crypto/x509" "errors" "fmt" + "slices" + "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" + "github.com/spiffe/go-spiffe/v2/spiffeid" + "github.com/spiffe/go-spiffe/v2/svid/x509svid" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -28,19 +36,39 @@ import ( var ErrWorkerPodNotFound = errors.New("worker pod not found") +// The SPIFFE identity that atelet serving certs carry, as minted by the +// podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). +// The namespace part is ateletNamespace, declared in informer.go. +const ( + trustDomainName = "cluster.local" + ateletSA = "atelet" +) + // AteletDialer handles gRPC connections to Atelet pods. type AteletDialer struct { workerIndexer cache.Indexer ateletIndexer cache.Indexer ateletConns *lru.Cache + // dialCredentials builds the transport credentials used to dial a given + // atelet, keyed on the atelet's expected pod UID. Production wires this to + // per-atelet mTLS; tests can override it with insecure credentials. + dialCredentials func(expectedPodUID string) (credentials.TransportCredentials, error) } -// NewAteletDialer creates a new AteletDialer. -func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer) *AteletDialer { +// NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath +// are used to build the per-atelet mTLS credentials used for every atelet connection. +func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string) *AteletDialer { return &AteletDialer{ workerIndexer: workerIndexer, ateletIndexer: ateletIndexer, ateletConns: lru.New(1024), + dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) { + tlsConfig, err := buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID) + if err != nil { + return nil, err + } + return credentials.NewTLS(tlsConfig), nil + }, } } @@ -73,7 +101,7 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( } selectedAtelet := matchingAtelets[0].(*corev1.Pod) - ateletKey := selectedAtelet.ObjectMeta.Namespace + "/" + selectedAtelet.ObjectMeta.Name + ateletKey := string(selectedAtelet.ObjectMeta.UID) ateletConnAny, ok := d.ateletConns.Get(ateletKey) if ok { @@ -81,12 +109,17 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( } if len(selectedAtelet.Status.PodIPs) == 0 { - return nil, fmt.Errorf("selected atelet %q has no assigned IPs: %w", selectedAtelet.ObjectMeta.Namespace+"/"+selectedAtelet.ObjectMeta.Name, err) + return nil, fmt.Errorf("selected atelet %q has no assigned IPs", selectedAtelet.ObjectMeta.Namespace+"/"+selectedAtelet.ObjectMeta.Name) + } + + creds, err := d.dialCredentials(string(selectedAtelet.ObjectMeta.UID)) + if err != nil { + return nil, fmt.Errorf("while building atelet credentials: %w", err) } ateletConn, err := grpc.NewClient( selectedAtelet.Status.PodIPs[0].IP+":8085", - grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithTransportCredentials(creds), grpc.WithStatsHandler(otelgrpc.NewClientHandler()), ) if err != nil { @@ -97,3 +130,73 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( return ateletConn, nil } + +func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { + trustDomain, err := spiffeid.TrustDomainFromString(trustDomainName) + if err != nil { + return nil, fmt.Errorf("while parsing trust domain %q: %w", trustDomainName, err) + } + bundle, err := x509bundle.Load(trustDomain, serverCAPath) + if err != nil { + return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) + } + expectedID, err := spiffeid.FromSegments(trustDomain, "ns", ateletNamespace, "sa", ateletSA) + if err != nil { + return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) + } + + verify, err := verifyAteletServerCert(bundle, expectedID, expectedPodUID) + if err != nil { + return nil, fmt.Errorf("while creating atelet server cert verifier: %w", err) + } + + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS13, + GetClientCertificate: credbundle.ClientLoader(clientBundlePath), + // Skip the default verification because the peer is dialed by IP and its + // certificate has no DNS/IP SAN. + InsecureSkipVerify: true, + VerifyConnection: verify, + } + + return &tlsConfig, nil +} + +func verifyAteletServerCert(bundle *x509bundle.Bundle, expectedID spiffeid.ID, expectedPodUID string) (func(tls.ConnectionState) error, error) { + if expectedPodUID == "" { + return nil, fmt.Errorf("expected pod UID must not be empty") + } + if expectedID.IsZero() { + return nil, fmt.Errorf("expected pod spiffe ID must not be empty") + } + return func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + return fmt.Errorf("server presented no certificate") + } + id, _, err := x509svid.Verify(cs.PeerCertificates, bundle) + if err != nil { + return fmt.Errorf("verifying server certificate chain: %w", err) + } + if id != expectedID { + return fmt.Errorf("server SPIFFE ID %q does not match expected %q", id, expectedID) + } + + leaf := cs.PeerCertificates[0] + if !slices.Contains(leaf.ExtKeyUsage, x509.ExtKeyUsageServerAuth) { + return fmt.Errorf("server certificate lacks the serverAuth extended key usage") + } + + identity, err := substratex509.PodIdentityFromCertificate(leaf) + if err != nil { + return fmt.Errorf("failed to parse PodIdentity extension: %w", err) + } + if identity == nil { + return fmt.Errorf("server certificate has no PodIdentity extension, expected pod UID %q", expectedPodUID) + } + if identity.PodUID != expectedPodUID { + return fmt.Errorf("pod UID %q does not match expected %q", identity.PodUID, expectedPodUID) + } + + return nil + }, nil +} diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go new file mode 100644 index 000000000..cda40dea9 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -0,0 +1,216 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net/url" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" + "github.com/spiffe/go-spiffe/v2/spiffeid" +) + +const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" + +// makeTestCA mints a self-signed CA and returns it along with an X.509 bundle +// containing it as the sole authority for the cluster.local trust domain. +func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509bundle.Bundle) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating CA key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating CA certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing CA certificate: %v", err) + } + td := spiffeid.RequireTrustDomainFromString("cluster.local") + bundle := x509bundle.FromX509Authorities(td, []*x509.Certificate{cert}) + return cert, key, bundle +} + +// leafOpts controls the contents of a test leaf certificate. +type leafOpts struct { + // podUID, if non-empty, is embedded in a PodIdentity extension. + podUID string + // spiffeID, if non-empty, is added as a URI SAN. + spiffeID string + // noServerAuth omits the serverAuth EKU. + noServerAuth bool +} + +// makeLeafCert mints a server leaf certificate signed by the given CA. +func makeLeafCert(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, opts leafOpts) *x509.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating leaf key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if opts.noServerAuth { + template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + } + if opts.spiffeID != "" { + uri, err := url.Parse(opts.spiffeID) + if err != nil { + t.Fatalf("parsing SPIFFE ID %q: %v", opts.spiffeID, err) + } + template.URIs = []*url.URL{uri} + } + if opts.podUID != "" { + // AddPodIdentityToCertificate requires all fields to be non-empty; + // only PodUID matters to these tests. + err := substratex509.AddPodIdentityToCertificate(&substratex509.PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: opts.podUID, + NodeName: "node-1", + NodeUID: "node-uid", + }, template) + if err != nil { + t.Fatalf("adding PodIdentity extension: %v", err) + } + } + der, err := x509.CreateCertificate(rand.Reader, template, ca, &key.PublicKey, caKey) + if err != nil { + t.Fatalf("creating leaf certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing leaf certificate: %v", err) + } + return cert +} + +func TestVerifyAteletServerCert(t *testing.T) { + ca, caKey, bundle := makeTestCA(t) + otherCA, otherCAKey, _ := makeTestCA(t) + + expectedID := spiffeid.RequireFromString(testAteletSPIFFEID) + + const uid = "5a2e1c9f-0b57-4a52-9f6e-2f6d3a1b8c4d" + + tests := []struct { + name string + leaf *x509.Certificate + expectedUID string + wantErr bool + }{ + { + name: "matching UID succeeds", + leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: uid, spiffeID: testAteletSPIFFEID}), + expectedUID: uid, + }, + { + name: "mismatched UID fails", + leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: "some-other-uid", spiffeID: testAteletSPIFFEID}), + expectedUID: uid, + wantErr: true, + }, + { + name: "missing pod UID extension fails", + leaf: makeLeafCert(t, ca, caKey, leafOpts{spiffeID: testAteletSPIFFEID}), + expectedUID: uid, + wantErr: true, + }, + { + name: "cert from untrusted CA fails", + leaf: makeLeafCert(t, otherCA, otherCAKey, leafOpts{podUID: uid, spiffeID: testAteletSPIFFEID}), + expectedUID: uid, + wantErr: true, + }, + { + name: "wrong SPIFFE ID fails", + leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: uid, spiffeID: "spiffe://cluster.local/ns/other/sa/other"}), + expectedUID: uid, + wantErr: true, + }, + { + name: "missing URI SAN fails", + leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: uid}), + expectedUID: uid, + wantErr: true, + }, + { + name: "missing serverAuth EKU fails", + leaf: makeLeafCert(t, ca, caKey, leafOpts{podUID: uid, spiffeID: testAteletSPIFFEID, noServerAuth: true}), + expectedUID: uid, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + verify, err := verifyAteletServerCert(bundle, expectedID, tc.expectedUID) + if err != nil { + t.Fatalf("constructing verifier: %v", err) + } + err = verify(tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{tc.leaf}, + }) + if gotErr := err != nil; gotErr != tc.wantErr { + t.Fatalf("verify returned error %v, wantErr=%v", err, tc.wantErr) + } + }) + } + + t.Run("no peer certificate fails", func(t *testing.T) { + verify, err := verifyAteletServerCert(bundle, expectedID, uid) + if err != nil { + t.Fatalf("constructing verifier: %v", err) + } + if err := verify(tls.ConnectionState{}); err == nil { + t.Fatal("verify succeeded, want error") + } + }) + + t.Run("empty expected UID fails at construction", func(t *testing.T) { + if _, err := verifyAteletServerCert(bundle, expectedID, ""); err == nil { + t.Fatal("verifyAteletServerCert succeeded, want error") + } + }) +} diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functional_test.go index 55429e6aa..e95a65589 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functional_test.go @@ -42,6 +42,7 @@ import ( "github.com/redis/go-redis/v9" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -312,7 +313,13 @@ func setupTest(t *testing.T, ns string) *testContext { t.Fatalf("failed to start worker cache: %v", err) } - dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer()) + dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "") + // Dial the fake atelet over insecure transport instead of per-atelet mTLS, + // so DialForWorker's real lookup/dial/cache path is exercised under test. + dialer.dialCredentials = func(_ string) (credentials.TransportCredentials, error) { + return insecure.NewCredentials(), nil + } + service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient) // 5. Start REAL gRPC Server for ATE API diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 9d1b8e9d2..387087d83 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -195,7 +195,7 @@ func newDanglingDialer() *AteletDialer { empty := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{ byNamespaceAndName: func(obj any) ([]string, error) { return nil, nil }, }) - return NewAteletDialer(empty, empty) + return NewAteletDialer(empty, empty, "", "") } func TestCallAteletSuspendStep_DanglingWorkerDoesNotRecordPhantomSnapshot(t *testing.T) { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 575924b21..b01a8d70f 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -28,7 +28,6 @@ import ( "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/credbundle" "github.com/agent-substrate/substrate/cmd/ateapi/internal/debugapi" "github.com/agent-substrate/substrate/cmd/ateapi/internal/k8sjwt" "github.com/agent-substrate/substrate/cmd/ateapi/internal/sessionidentity" @@ -36,6 +35,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" @@ -72,11 +72,11 @@ var ( clientJWTAudience = pflag.String("client-jwt-audience", "", "The expected audience for client JWTs.") sessionIDJWTPoolFile = pflag.String("session-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing session JWTs") - sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") - workerpoolCACerts = pflag.String("workerpool-ca-certs", "", "The file that contains the CA for verifying workerpool client certificates.") + sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") + podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") + ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") showVersion = pflag.Bool("version", false, "Print version and exit.") - authMode = pflag.String("auth-mode", "mtls", "Auth mode for incoming gRPC: mtls|jwt. 'mtls' (default) relies on transport-level mTLS for client identity. 'jwt' additionally requires a Kubernetes ServiceAccount Bearer token on every RPC. Substrate will drop support for JWT auth mode once the Pod Certificates feature is enabled by default in the minimum supported Kubernetes version.") clientJWTCAFile = pflag.String("client-jwt-ca-cert", ateapiauth.DefaultServiceAccountCAFile, "CA cert file used to verify TLS when fetching the OIDC discovery document and JWKS for JWT authentication. Defaults to the in-cluster service account CA.") ) @@ -107,11 +107,6 @@ func main() { loadFlagsFromEnv() logFlagValues(ctx) - authModeParsed, err := ateapiauth.ParseMode(*authMode) - if err != nil { - serverboot.Fatal(ctx, "Invalid --auth-mode", err) - } - redisClient, err := connectRedis(ctx) if err != nil { serverboot.Fatal(ctx, "Failed to set up Redis/Valkey", err) @@ -159,15 +154,12 @@ func main() { serverboot.Fatal(ctx, "Failed to register worker-count metric", err) } - dialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer()) - sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, clientset) + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) + sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, ateletDialer, clientset) jwtIssuerDiscoveryClient := buildK8sServiceAccountIssuerDiscoveryClient(ctx, *clientJWTCAFile, *clientJWTIssuer) - if authModeParsed == ateapiauth.ModeJWT && jwtIssuerDiscoveryClient == nil { - serverboot.Fatal(ctx, "JWT auth mode requires a Kubernetes ServiceAccount issuer discovery client", fmt.Errorf("client JWT issuer %q is not usable for discovery", *clientJWTIssuer)) - } - sessionIdentitySrv := sessionidentity.New(*clientJWTIssuer, *clientJWTAudience, *sessionIDJWTPoolFile, *sessionIDCAPoolFile, *workerpoolCACerts, jwtIssuerDiscoveryClient) + sessionIdentitySrv := sessionidentity.New(*clientJWTIssuer, *clientJWTAudience, *sessionIDJWTPoolFile, *sessionIDCAPoolFile, *podIdentityCACerts, jwtIssuerDiscoveryClient) debugSrv := debugapi.NewService(redisPersistence) lisCfg := &net.ListenConfig{} @@ -177,10 +169,12 @@ func main() { } authCfg := ateapiauth.ServerConfig{ - Mode: authModeParsed, - VerifyBearerToken: func(ctx context.Context, bearer string) error { - _, err := k8sjwt.Verify(ctx, jwtIssuerDiscoveryClient, bearer, *clientJWTIssuer, *clientJWTAudience, time.Now()) - return err + VerifyBearerToken: func(ctx context.Context, bearer string) (string, error) { + claims, err := k8sjwt.Verify(ctx, jwtIssuerDiscoveryClient, bearer, *clientJWTIssuer, *clientJWTAudience, time.Now()) + if err != nil { + return "", err + } + return claims.Subject, nil }, } if err := ateapiauth.ValidateServerConfig(authCfg); err != nil { @@ -249,8 +243,8 @@ func logFlagValues(ctx context.Context) { slog.String("client-jwt-audience", *clientJWTAudience), slog.String("session-id-jwt-pool", *sessionIDJWTPoolFile), slog.String("session-id-ca-pool", *sessionIDCAPoolFile), - slog.String("workerpool-ca-certs", *workerpoolCACerts), - slog.String("auth-mode", *authMode), + slog.String("pod-identity-ca-certs", *podIdentityCACerts), + slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), ) } @@ -294,7 +288,7 @@ func connectRedis(ctx context.Context) (*redis.ClusterClient, error) { } func buildRedisTLSConfig(ctx context.Context) (*tls.Config, error) { - tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS13} if *redisCACerts != "" { ca, err := os.ReadFile(*redisCACerts) if err != nil { @@ -357,27 +351,30 @@ func newKubeClients() (*kubernetes.Clientset, versioned.Interface, error) { return clientset, ateClient, nil } -// buildServerCreds loads the workerpool CA pool (if configured) and +// buildServerCreds loads the pod-identity CA pool (if configured) and // composes gRPC TransportCredentials over the server bundle + optional // client-cert verification. func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, error) { var clientCAs *x509.CertPool - if *workerpoolCACerts != "" { + if *podIdentityCACerts != "" { // TODO: Periodically reload these to handle rotations. Consult with Tina to see how she did it for client-go. - ca, err := os.ReadFile(*workerpoolCACerts) + ca, err := os.ReadFile(*podIdentityCACerts) if err != nil { - return nil, fmt.Errorf("read workerpool CA: %w", err) + return nil, fmt.Errorf("read pod-identity CA: %w", err) } clientCAs = x509.NewCertPool() if !clientCAs.AppendCertsFromPEM(ca) { - return nil, fmt.Errorf("parse workerpool CA from %s", *workerpoolCACerts) + return nil, fmt.Errorf("parse pod-identity CA from %s", *podIdentityCACerts) } - slog.InfoContext(ctx, "Using custom CA for workerpool clients", slog.String("path", *workerpoolCACerts)) + slog.InfoContext(ctx, "Using pod-identity CA for client-cert verification", slog.String("path", *podIdentityCACerts)) } return credentials.NewTLS(&tls.Config{ GetCertificate: credbundle.Loader(*grpcServerCredBundle), - ClientAuth: tls.VerifyClientCertIfGiven, - ClientCAs: clientCAs, + // Client certs stay optional at the transport level: certless + // clients such as kubectl-ate authenticate with a Bearer token in the + // ateapiauth interceptor. + ClientAuth: tls.VerifyClientCertIfGiven, + ClientCAs: clientCAs, }), nil } diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index e96bdb242..fc9edc098 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -39,10 +39,11 @@ var ( ateAPIConnSpec = pflag.String("ateapi-conn-spec", "dns:///api.ate-system.svc:443", "") - ateapiAuthMode = pflag.String("ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) dials with insecure TLS and relies on pod-projected mTLS credentials for identity. 'jwt' verifies the server cert and sends a Bearer SA token.") - ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.") + ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") - ateapiTokenFile = pflag.String("ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") + ateapiTokenAuth = pflag.Bool("ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") + ateapiTokenFile = pflag.String("ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.") + ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required unless --ateapi-use-token-auth is set, ignored otherwise.") ) func init() { @@ -54,17 +55,12 @@ func main() { pflag.Parse() ctrl.SetLogger(zap.New(zap.UseDevMode(true))) - mode, err := ateapiauth.ParseMode(*ateapiAuthMode) - if err != nil { - setupLog.Error(err, "invalid --ateapi-auth") - os.Exit(1) - } - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: mode, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - TokenFile: *ateapiTokenFile, + UseTokenAuth: *ateapiTokenAuth, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + TokenFile: *ateapiTokenFile, + ClientCredBundle: *ateapiClientCert, }) if err != nil { setupLog.Error(err, "building ateapi dial options") diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index de3ad2bff..4f1e02f63 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -16,6 +16,8 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -33,6 +35,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateerrors" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" @@ -53,6 +56,7 @@ import ( "google.golang.org/api/option" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/reflection" "google.golang.org/grpc/status" @@ -65,6 +69,9 @@ var ( port = pflag.Int("port", 8085, "The port to listen on") metricsListenAddr = pflag.String("metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") + grpcServerCredBundle = pflag.String("grpc-server-cred-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Credential bundle atelet presents as its gRPC serving certificate.") + clientCACerts = pflag.String("client-ca-certs", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify gRPC client certificates.") + gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") imageCacheDir = pflag.String("image-cache-dir", ateompath.ImageCacheDir, "Directory for the node-local OCI image layer cache. Must be on the volume shared with the ateom pods (the cached layers are their overlay lowerdirs), and on a disk sized for both capacity and IOPS: unpack throughput is gated by the volume's IOPS.") @@ -177,7 +184,16 @@ func main() { serverboot.Fatal(ctx, "Failed to listen", err) } - svr := grpc.NewServer(grpc.StatsHandler(otelgrpc.NewServerHandler()), grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor)) + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) + } + + svr := grpc.NewServer( + grpc.Creds(credentials.NewTLS(tlsCfg)), + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), + ) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) @@ -1113,3 +1129,23 @@ func resetActorDirs(actorUID string) error { return nil } + +// ateletServerTLSConfig builds a *tls.Config for a gRPC server that presents the +// credential bundle at servingBundlePath, requires a client certificate +// chaining to a CA in clientCAPath. +func ateletServerTLSConfig(servingBundlePath, clientCAPath string) (*tls.Config, error) { + caBytes, err := os.ReadFile(clientCAPath) + if err != nil { + return nil, fmt.Errorf("read CA bundle %s: %w", clientCAPath, err) + } + clientCAs := x509.NewCertPool() + if !clientCAs.AppendCertsFromPEM(caBytes) { + return nil, fmt.Errorf("parse CA bundle from %s", clientCAPath) + } + return &tls.Config{ + MinVersion: tls.VersionTLS13, + GetCertificate: credbundle.Loader(servingBundlePath), + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: clientCAs, + }, nil +} diff --git a/cmd/atenet/internal/root.go b/cmd/atenet/internal/root.go index 7c7e66806..53ae85d13 100644 --- a/cmd/atenet/internal/root.go +++ b/cmd/atenet/internal/root.go @@ -18,6 +18,7 @@ import ( "fmt" "os" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router" "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/cobra" ) @@ -37,6 +38,6 @@ func Execute() { } func init() { - rootCmd.AddCommand(NewRouterCmd()) + rootCmd.AddCommand(router.NewRouterCmd()) rootCmd.AddCommand(NewDnsCmd()) } diff --git a/cmd/atenet/internal/router.go b/cmd/atenet/internal/router/cmd.go similarity index 74% rename from cmd/atenet/internal/router.go rename to cmd/atenet/internal/router/cmd.go index c58f31a7e..3cddd048d 100644 --- a/cmd/atenet/internal/router.go +++ b/cmd/atenet/internal/router/cmd.go @@ -12,26 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package router import ( "fmt" "time" "github.com/spf13/cobra" - - "github.com/agent-substrate/substrate/cmd/atenet/internal/router" - "github.com/agent-substrate/substrate/internal/ateapiauth" ) func NewRouterCmd() *cobra.Command { - var cfg router.RouterConfig + var cfg routerConfig cmd := &cobra.Command{ Use: "router", Short: "Router components including xDS server and Envoy ExtProc gateway processing server", RunE: func(cmd *cobra.Command, args []string) error { - srv, err := router.NewRouterServer(cfg) + srv, err := NewRouterServer(cfg) if err != nil { return fmt.Errorf("failed to create router server: %w", err) } @@ -56,12 +53,13 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)") cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services") cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") - cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)") + cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)") - cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) dials with insecure TLS and relies on pod-projected mTLS credentials for identity. 'jwt' verifies the server cert and sends a Bearer SA token.") - cmd.Flags().StringVar(&cfg.AteapiCAFile, "ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.") - cmd.Flags().StringVar(&cfg.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") - cmd.Flags().StringVar(&cfg.AteapiTokenFile, "ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") + cmd.Flags().StringVar(&cfg.Auth.AteapiCAFile, "ateapi-ca-file", "", "PEM file with CAs trusted to verify the ateapi server cert. Required.") + cmd.Flags().StringVar(&cfg.Auth.AteapiClientCertPath, "ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required unless --ateapi-use-token-auth is set, ignored otherwise.") + cmd.Flags().StringVar(&cfg.Auth.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") + cmd.Flags().BoolVar(&cfg.Auth.AteapiUseTokenAuth, "ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") + cmd.Flags().StringVar(&cfg.Auth.AteapiTokenFile, "ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.") return cmd } diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go new file mode 100644 index 000000000..83893ea9b --- /dev/null +++ b/cmd/atenet/internal/router/config.go @@ -0,0 +1,58 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "time" +) + +// authConfig holds the router's client-auth settings for dialing ateapi. +// AteapiCAFile always verifies ateapi's serving cert (the servicedns trust +// bundle in-cluster). By default the router presents AteapiClientCertPath +// (the podidentity credential bundle) as its client cert; with +// AteapiUseTokenAuth it sends a Bearer token from AteapiTokenFile instead and +// the cert path is ignored. +type authConfig struct { + AteapiUseTokenAuth bool + AteapiCAFile string + AteapiClientCertPath string + AteapiServerName string + AteapiTokenFile string +} + +// routerConfig holds deployment setup and endpoint options for the router node instance. +type routerConfig struct { + Standalone bool + Namespace string + Kubeconfig string + AteapiAddr string + HttpPort int + XdsPort int + ExtprocPort int + ExtprocAddr string + EnvoyImage string + TemplatesFile string + StatusPort int + HealthInterval time.Duration + HttpsPort int + EnvoyCertPath string + LogLevel string + MetricsAddr string + // OtlpCollectorAddress is the host:port of the OTLP gRPC collector that + // Envoy reports tracing spans to. Empty disables Envoy-side tracing. + OtlpCollectorAddress string + + Auth authConfig +} diff --git a/cmd/atenet/internal/router/controller.go b/cmd/atenet/internal/router/controller.go index 1b0a1ed18..e66243d5b 100644 --- a/cmd/atenet/internal/router/controller.go +++ b/cmd/atenet/internal/router/controller.go @@ -28,7 +28,7 @@ import ( type Controller struct { k8sClient client.Client clientset kubernetes.Interface - cfg RouterConfig + cfg routerConfig xdsSrv *XdsServer extprocSrv *ExtProcServer @@ -39,7 +39,7 @@ type Controller struct { func NewController( k8sClient client.Client, clientset kubernetes.Interface, - cfg RouterConfig, + cfg routerConfig, xdsSrv *XdsServer, extprocSrv *ExtProcServer, ) *Controller { diff --git a/cmd/atenet/internal/router/envoyrunner.go b/cmd/atenet/internal/router/envoyrunner.go index df8765c62..4eee29303 100644 --- a/cmd/atenet/internal/router/envoyrunner.go +++ b/cmd/atenet/internal/router/envoyrunner.go @@ -37,10 +37,10 @@ const ( // Envoy proxy instance running inside Kubernetes. type envoyrunner struct { k8sClient client.Client - cfg RouterConfig + cfg routerConfig } -func newEnvoyRunner(k8sClient client.Client, cfg RouterConfig) *envoyrunner { +func newEnvoyRunner(k8sClient client.Client, cfg routerConfig) *envoyrunner { return &envoyrunner{ k8sClient: k8sClient, cfg: cfg, diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index 21332c409..7d735e2e6 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -64,11 +64,11 @@ type routerHealth struct { interval time.Duration clientset kubernetes.Interface apiClient ateapipb.ControlClient - cfg RouterConfig + cfg routerConfig envoyClient *http.Client } -func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg RouterConfig) *routerHealth { +func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg routerConfig) *routerHealth { if interval <= 0 { interval = time.Second } diff --git a/cmd/atenet/internal/router/health_test.go b/cmd/atenet/internal/router/health_test.go index 29d410f7b..ef91a5158 100644 --- a/cmd/atenet/internal/router/health_test.go +++ b/cmd/atenet/internal/router/health_test.go @@ -74,7 +74,7 @@ func TestCheckK8sTimesOut(t *testing.T) { })) defer server.Close() - rh := newRouterHealth(time.Second, newHealthTestClientset(t, server), nil, RouterConfig{}) + rh := newRouterHealth(time.Second, newHealthTestClientset(t, server), nil, routerConfig{}) startedAt := time.Now() healthy, msg := rh.checkK8s(context.Background()) elapsed := time.Since(startedAt) @@ -91,7 +91,7 @@ func TestCheckK8sTimesOut(t *testing.T) { } func TestCheckK8sWithoutRESTClient(t *testing.T) { - rh := newRouterHealth(time.Second, kubernetesfake.NewSimpleClientset(), nil, RouterConfig{}) + rh := newRouterHealth(time.Second, kubernetesfake.NewSimpleClientset(), nil, routerConfig{}) healthy, msg := rh.checkK8s(context.Background()) if healthy { t.Fatal("checkK8s returned healthy without a discovery REST client") @@ -119,7 +119,7 @@ func TestHealthCheckDoesNotBlockReportOrStatusz(t *testing.T) { })) defer server.Close() - rh := newRouterHealth(time.Second, newHealthTestClientset(t, server), nil, RouterConfig{}) + rh := newRouterHealth(time.Second, newHealthTestClientset(t, server), nil, routerConfig{}) setHealthyEnvoyClient(rh) checkDone := make(chan struct{}) go func() { @@ -144,7 +144,7 @@ func TestHealthCheckDoesNotBlockReportOrStatusz(t *testing.T) { } statusServer := httptest.NewServer(http.HandlerFunc((&RouterServer{ - cfg: RouterConfig{Standalone: true}, + cfg: routerConfig{Standalone: true}, health: rh, }).handleStatusz)) defer statusServer.Close() @@ -208,7 +208,7 @@ func TestHealthChecksRunConcurrently(t *testing.T) { } }, } - rh := newRouterHealth(time.Second, newHealthTestClientset(t, server), apiClient, RouterConfig{}) + rh := newRouterHealth(time.Second, newHealthTestClientset(t, server), apiClient, routerConfig{}) rh.envoyClient = &http.Client{Transport: healthRoundTripFunc(func(*http.Request) (*http.Response, error) { started <- "envoy" <-release @@ -261,7 +261,7 @@ func TestHealthStartStopsWhenK8sCheckIsCanceled(t *testing.T) { })) defer server.Close() - rh := newRouterHealth(time.Hour, newHealthTestClientset(t, server), nil, RouterConfig{}) + rh := newRouterHealth(time.Hour, newHealthTestClientset(t, server), nil, routerConfig{}) setHealthyEnvoyClient(rh) ctx, cancel := context.WithCancel(context.Background()) startDone := make(chan struct{}) diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 7ed77df73..61cfb0cf4 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -16,22 +16,15 @@ package router import ( "context" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "errors" "fmt" "log/slog" - "math/big" "net" "net/http" "os" "os/signal" "strings" "syscall" - "time" "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -62,37 +55,9 @@ func init() { utilruntime.Must(v1alpha1.AddToScheme(scheme)) } -// RouterConfig holds deployment setup and endpoint options for the router node instance. -type RouterConfig struct { - Standalone bool - Namespace string - Kubeconfig string - AteapiAddr string - HttpPort int - XdsPort int - ExtprocPort int - ExtprocAddr string - EnvoyImage string - TemplatesFile string - StatusPort int - HealthInterval time.Duration - HttpsPort int - EnvoyCertPath string - LogLevel string - MetricsAddr string - // OtlpCollectorAddress is the host:port of the OTLP gRPC collector that - // Envoy reports tracing spans to. Empty disables Envoy-side tracing. - OtlpCollectorAddress string - - AteapiAuthMode string - AteapiCAFile string - AteapiServerName string - AteapiTokenFile string -} - // RouterServer instantiates and coordinates runtime threads executing system modules. type RouterServer struct { - cfg RouterConfig + cfg routerConfig Cmd *cobra.Command k8sClient client.Client @@ -103,7 +68,7 @@ type RouterServer struct { atStore atStore } -func NewRouterServer(cfg RouterConfig) (*RouterServer, error) { +func NewRouterServer(cfg routerConfig) (*RouterServer, error) { var k8sClient client.Client var clientset kubernetes.Interface @@ -193,15 +158,12 @@ func (s *RouterServer) Run(ctx context.Context) error { go serverboot.StartMetricsServer(ctx, serverboot.MetricsServerOptions{Addr: s.cfg.MetricsAddr}) - authMode, err := ateapiauth.ParseMode(s.cfg.AteapiAuthMode) - if err != nil { - return fmt.Errorf("invalid --ateapi-auth: %w", err) - } dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: authMode, - CAFile: s.cfg.AteapiCAFile, - ServerName: s.cfg.AteapiServerName, - TokenFile: s.cfg.AteapiTokenFile, + UseTokenAuth: s.cfg.Auth.AteapiUseTokenAuth, + CAFile: s.cfg.Auth.AteapiCAFile, + ServerName: s.cfg.Auth.AteapiServerName, + TokenFile: s.cfg.Auth.AteapiTokenFile, + ClientCredBundle: s.cfg.Auth.AteapiClientCertPath, }) if err != nil { return fmt.Errorf("building ateapi dial options: %w", err) @@ -214,7 +176,7 @@ func (s *RouterServer) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to establish grpc channel to ateapi client: %w", err) } - slog.InfoContext(ctx, "Connecting to ateapi", slog.String("address", s.cfg.AteapiAddr), slog.String("auth", string(authMode))) + slog.InfoContext(ctx, "Connecting to ateapi", slog.String("address", s.cfg.AteapiAddr), slog.Bool("use-api-token-auth", s.cfg.Auth.AteapiUseTokenAuth)) s.apiClient = ateapipb.NewControlClient(conn) slog.InfoContext(ctx, "Starting substrate router subsystem", slog.Bool("standalone", s.cfg.Standalone)) @@ -227,17 +189,7 @@ func (s *RouterServer) Run(ctx context.Context) error { return fmt.Errorf("configure OTLP collector: %w", err) } - var certContent, keyContent string - if s.cfg.EnvoyCertPath == "" { - slog.InfoContext(ctx, "No Envoy certificate path provided, generating self-signed certificate for testing") - var err error - certContent, keyContent, err = generateSelfSignedCert() - if err != nil { - return fmt.Errorf("failed to generate self-signed cert: %w", err) - } - } - - xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath, certContent, keyContent) + xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath) if s.extprocSrv == nil { routeDuration, err := newRouteDurationHistogram() if err != nil { @@ -313,39 +265,3 @@ func (s *RouterServer) Run(ctx context.Context) error { return g.Wait() } - -func generateSelfSignedCert() (string, string, error) { - priv, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return "", "", err - } - - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - Organization: []string{"Substrate Local Test"}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 365), - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - DNSNames: []string{"localhost"}, - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) - if err != nil { - return "", "", err - } - - certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - - privBytes, err := x509.MarshalPKCS8PrivateKey(priv) - if err != nil { - return "", "", err - } - keyPem := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) - - return string(certPem), string(keyPem), nil -} diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index a72011848..116359ce8 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -16,12 +16,20 @@ package router import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "fmt" "io" + "math/big" "net" "net/http" "os" + "path/filepath" "strings" "testing" "time" @@ -47,7 +55,10 @@ func TestStatuszEndpoint(t *testing.T) { defer os.Remove(tmpFile.Name()) tmpFile.Close() - cfg := RouterConfig{ + // Run() dials ateapi in mtls mode, which requires real TLS material; generate it. + caPath, clientCertPath := writeTestTLSMaterial(t) + + cfg := routerConfig{ Standalone: true, Namespace: "default", StatusPort: httpPort, @@ -56,7 +67,10 @@ func TestStatuszEndpoint(t *testing.T) { ExtprocPort: 50051, TemplatesFile: tmpFile.Name(), MetricsAddr: "127.0.0.1:0", - AteapiCAFile: "unused-in-mtls-mode", + Auth: authConfig{ + AteapiCAFile: caPath, + AteapiClientCertPath: clientCertPath, + }, } srv, err := NewRouterServer(cfg) @@ -142,3 +156,44 @@ func TestStatuszEndpoint(t *testing.T) { t.Errorf("Target parameters unassigned inside context payload context properties: found %s", dashboard.Queries[0].Target) } } + +// writeTestTLSMaterial generates a self-signed certificate and writes a CA trust +// bundle and a client credential bundle to temp files, returning their paths. +// Run() (mtls mode) requires both to build its ateapi mTLS credentials. +func writeTestTLSMaterial(t *testing.T) (caPath, clientCertPath string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating certificate: %v", err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshaling key: %v", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + + dir := t.TempDir() + caPath = filepath.Join(dir, "ca.pem") + if err := os.WriteFile(caPath, certPEM, 0o600); err != nil { + t.Fatalf("writing CA file: %v", err) + } + clientCertPath = filepath.Join(dir, "client.pem") + if err := os.WriteFile(clientCertPath, append(certPEM, keyPEM...), 0o600); err != nil { + t.Fatalf("writing client cert file: %v", err) + } + return caPath, clientCertPath +} diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e7e1602a9..3d52657af 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -80,10 +80,8 @@ type XdsServer struct { mu sync.Mutex - httpsPort int - certPath string - certContent string - keyContent string + httpsPort int + certPath string otlpHost string otlpPort uint32 @@ -111,13 +109,11 @@ func (x *XdsServer) SetConfig(ingressPort int, extprocPort int, extprocAddr stri x.extprocAddr = extprocAddr } -func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string, certContent string, keyContent string) { +func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string) { x.mu.Lock() defer x.mu.Unlock() x.httpsPort = httpsPort x.certPath = certPath - x.certContent = certContent - x.keyContent = keyContent } // SetOtlpCollector enables Envoy-side tracing pointed at the OTLP gRPC @@ -576,30 +572,15 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { } func (x *XdsServer) buildTlsCertificate() *tlsv3.TlsCertificate { - if x.certPath != "" { - return &tlsv3.TlsCertificate{ - CertificateChain: &corev3.DataSource{ - Specifier: &corev3.DataSource_Filename{ - Filename: x.certPath, - }, - }, - PrivateKey: &corev3.DataSource{ - Specifier: &corev3.DataSource_Filename{ - Filename: x.certPath, // Assuming combined file - }, - }, - } - } - return &tlsv3.TlsCertificate{ CertificateChain: &corev3.DataSource{ - Specifier: &corev3.DataSource_InlineString{ - InlineString: x.certContent, + Specifier: &corev3.DataSource_Filename{ + Filename: x.certPath, }, }, PrivateKey: &corev3.DataSource{ - Specifier: &corev3.DataSource_InlineString{ - InlineString: x.keyContent, + Specifier: &corev3.DataSource_Filename{ + Filename: x.certPath, // Assuming combined file }, }, } diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 92e347648..9a2c2460f 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -141,7 +141,7 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) { server := NewXdsServer(18000) server.SetConfig(8085, 50053, "127.0.0.1") - server.SetTlsConfig(8443, "", "dummy-cert", "dummy-key") + server.SetTlsConfig(8443, "") err := server.UpdateSnapshot() if err != nil { diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 54712e826..61f75324a 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -180,9 +180,9 @@ Logs are streamable only while the actor is bound to a worker (i.e., `STATUS_RUN Commands for bootstrapping the Substrate control plane and debugging local environments. ```bash -# Generate a new CA pool and push it directly to a Kubernetes Secret +# Generate a new Session ID CA pool and push it directly to a Kubernetes Secret kubectl ate admin make-ca-pool \ - --name workerpool-ca-certs \ + --name session-id-ca-pool \ --secret-namespace ate-system \ --ca-id "1" diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go index 5b63a15e4..815d38b6d 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go @@ -28,6 +28,7 @@ import ( "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/signercontroller" "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -38,6 +39,22 @@ import ( const Name = "podidentity.podcert.ate.dev/identity" const CTBPrefix = "podidentity.podcert.ate.dev:identity:" +// atelet's identity, as installed by manifests/ate-install/atelet.yaml. Pods +// running as atelet serve TLS (e.g. to ate-apiserver), so their certs also +// carry the serverAuth EKU. +const ( + ateletNamespace = "ate-system" + ateletServiceAccount = "atelet" +) + +func extKeyUsages(namespace, serviceAccount string) []x509.ExtKeyUsage { + usages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + if namespace == ateletNamespace && serviceAccount == ateletServiceAccount { + usages = append(usages, x509.ExtKeyUsageServerAuth) + } + return usages +} + type Impl struct { kc kubernetes.Interface caPool *localca.Pool @@ -121,16 +138,38 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq Path: path.Join("ns", pcr.ObjectMeta.Namespace, "sa", pcr.Spec.ServiceAccountName), } + parent := h.caPool.CAs[0].RootCertificate + template := &x509.Certificate{ BasicConstraintsValid: true, NotBefore: notBefore, NotAfter: notAfter, URIs: []*url.URL{spiffeURI}, KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + ExtKeyUsage: extKeyUsages(pcr.ObjectMeta.Namespace, pcr.Spec.ServiceAccountName), + // Link the leaf to its issuing CA by key id so verifiers can disambiguate + // a multi-CA trust bundle (e.g. valkey trusts both the servicedns and + // podidentity CAs). + // https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.1 + AuthorityKeyId: parent.SubjectKeyId, } - subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, h.caPool.CAs[0].RootCertificate, subjectPublicKey, h.caPool.CAs[0].SigningKey) + // Fields are sourced from the PCR spec (attested by kube-apiserver) rather + // than the Pod object, which lacks the ServiceAccount and Node UIDs. + podIdentity := &substratex509.PodIdentity{ + Namespace: pcr.ObjectMeta.Namespace, + ServiceAccountName: pcr.Spec.ServiceAccountName, + ServiceAccountUID: string(pcr.Spec.ServiceAccountUID), + PodName: pcr.Spec.PodName, + PodUID: string(pcr.Spec.PodUID), + NodeName: string(pcr.Spec.NodeName), + NodeUID: string(pcr.Spec.NodeUID), + } + if err := substratex509.AddPodIdentityToCertificate(podIdentity, template); err != nil { + return fmt.Errorf("while adding pod identity to certificate: %w", err) + } + + subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey) if err != nil { return fmt.Errorf("while signing subject cert: %w", err) } diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go new file mode 100644 index 000000000..6afd1d50c --- /dev/null +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go @@ -0,0 +1,452 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podidentitysigner + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "errors" + "slices" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" + certsv1beta1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/utils/ptr" +) + +// fixedClock is a PassiveClock frozen at a fixed instant. +type fixedClock struct { + now time.Time +} + +func (c fixedClock) Now() time.Time { return c.now } +func (c fixedClock) Since(t time.Time) time.Duration { return c.now.Sub(t) } + +// testNow is a whole-second instant so times survive the x509 encoding +// round-trip (certificates carry 1s precision) and compare exactly. It must +// stay near wall-clock time because GenerateED25519CA stamps CA validity +// from time.Now(). +var testNow = time.Now().UTC().Truncate(time.Second) + +// makePodAndPCR returns a pod and a matching PodCertificateRequest with no +// key material set; callers fill in StubPKCS10Request. +func makePodAndPCR(namespace, podName, serviceAccount string, maxExpirationSeconds int32) (*corev1.Pod, *certsv1beta1.PodCertificateRequest) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: podName, + UID: types.UID("pod-uid-1"), + }, + } + pcr := &certsv1beta1.PodCertificateRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "req-1", + }, + Spec: certsv1beta1.PodCertificateRequestSpec{ + SignerName: Name, + PodName: pod.ObjectMeta.Name, + PodUID: pod.ObjectMeta.UID, + ServiceAccountName: serviceAccount, + ServiceAccountUID: types.UID("sa-uid-1"), + NodeName: types.NodeName("node-1"), + NodeUID: types.UID("node-uid-1"), + MaxExpirationSeconds: ptr.To(maxExpirationSeconds), + }, + } + return pod, pcr +} + +// stubCSR returns a stub PKCS#10 request carrying priv's public key. +func stubCSR(t *testing.T, priv ed25519.PrivateKey) []byte { + t.Helper() + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, priv) + if err != nil { + t.Fatalf("while creating stub CSR: %v", err) + } + return csr +} + +func TestMakeCert(t *testing.T) { + testCases := []struct { + name string + namespace string + podName string + serviceAccount string + maxExpirationSeconds int32 + wantLifetime time.Duration + wantURI string + wantEKUs []x509.ExtKeyUsage + wantIdentity *substratex509.PodIdentity + }{ + { + name: "atelet in ate-system", + namespace: "ate-system", + podName: "atelet-abcde", + serviceAccount: "atelet", + maxExpirationSeconds: 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/ate-system/sa/atelet", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid-1", + PodName: "atelet-abcde", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "ordinary workload is client-only", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "requested lifetime capped at 24h", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 7 * 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "shorter requested lifetime honored", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 3600, + wantLifetime: time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + subjectPub, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + + pod, pcr := makePodAndPCR(tc.namespace, tc.podName, tc.serviceAccount, tc.maxExpirationSeconds) + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + + kc := fake.NewSimpleClientset(pod, pcr) + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err != nil { + t.Fatalf("MakeCert: %v", err) + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests(tc.namespace).Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching updated PCR: %v", err) + } + if len(gotPCR.Status.Conditions) != 1 || gotPCR.Status.Conditions[0].Type != certsv1beta1.PodCertificateRequestConditionTypeIssued { + t.Fatalf("PCR status not marked Issued: %+v", gotPCR.Status.Conditions) + } + + block, rest := pem.Decode([]byte(gotPCR.Status.CertificateChain)) + if block == nil { + t.Fatalf("certificate chain contains no PEM block") + } + if len(rest) != 0 { + t.Errorf("expected exactly one certificate in chain (no intermediates), got trailing data") + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("while parsing leaf certificate: %v", err) + } + + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: roots, + CurrentTime: testNow, + KeyUsages: tc.wantEKUs, + }); err != nil { + t.Errorf("leaf does not verify against CA root: %v", err) + } + + leafPub, ok := leaf.PublicKey.(ed25519.PublicKey) + if !ok || !leafPub.Equal(subjectPub) { + t.Errorf("leaf public key %v is not the subject key %v", leaf.PublicKey, subjectPub) + } + + wantNotBefore := testNow.Add(-2 * time.Minute) + wantNotAfter := wantNotBefore.Add(tc.wantLifetime) + wantBeginRefreshAt := wantNotAfter.Add(-30 * time.Minute) + if !leaf.NotBefore.Equal(wantNotBefore) { + t.Errorf("got NotBefore %v, want %v", leaf.NotBefore, wantNotBefore) + } + if !leaf.NotAfter.Equal(wantNotAfter) { + t.Errorf("got NotAfter %v, want %v", leaf.NotAfter, wantNotAfter) + } + if gotPCR.Status.NotBefore == nil || !gotPCR.Status.NotBefore.Time.Equal(wantNotBefore) { + t.Errorf("got status NotBefore %v, want %v", gotPCR.Status.NotBefore, wantNotBefore) + } + if gotPCR.Status.NotAfter == nil || !gotPCR.Status.NotAfter.Time.Equal(wantNotAfter) { + t.Errorf("got status NotAfter %v, want %v", gotPCR.Status.NotAfter, wantNotAfter) + } + if gotPCR.Status.BeginRefreshAt == nil || !gotPCR.Status.BeginRefreshAt.Time.Equal(wantBeginRefreshAt) { + t.Errorf("got status BeginRefreshAt %v, want %v", gotPCR.Status.BeginRefreshAt, wantBeginRefreshAt) + } + + if len(leaf.URIs) != 1 || leaf.URIs[0].String() != tc.wantURI { + t.Errorf("got URIs %v, want [%s]", leaf.URIs, tc.wantURI) + } + if !slices.Equal(leaf.ExtKeyUsage, tc.wantEKUs) { + t.Errorf("got EKUs %v, want %v", leaf.ExtKeyUsage, tc.wantEKUs) + } + if !bytes.Equal(leaf.AuthorityKeyId, ca.RootCertificate.SubjectKeyId) { + t.Errorf("got AuthorityKeyId %x, want CA SubjectKeyId %x", leaf.AuthorityKeyId, ca.RootCertificate.SubjectKeyId) + } + + identity, err := substratex509.PodIdentityFromCertificate(leaf) + if err != nil { + t.Fatalf("while extracting PodIdentity: %v", err) + } + if *identity != *tc.wantIdentity { + t.Errorf("got PodIdentity %+v, want %+v", identity, tc.wantIdentity) + } + }) + } +} + +func TestMakeCertErrors(t *testing.T) { + testCases := []struct { + name string + omitPod bool + podUID types.UID + omitKey bool + failUpdate bool + }{ + { + name: "pod not found", + omitPod: true, + podUID: "pod-uid-1", + }, + { + name: "pod UID mismatch", + podUID: "other-uid", + }, + { + name: "no key material in PCR", + podUID: "pod-uid-1", + omitKey: true, + }, + { + name: "status update fails", + podUID: "pod-uid-1", + failUpdate: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + pod, pcr := makePodAndPCR("ate-system", "atelet-abcde", "atelet", 86400) + pod.ObjectMeta.UID = tc.podUID + if !tc.omitKey { + _, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + } + + objects := []runtime.Object{pcr} + if !tc.omitPod { + objects = append(objects, pod) + } + kc := fake.NewSimpleClientset(objects...) + if tc.failUpdate { + kc.PrependReactor("update", "podcertificaterequests", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("injected update failure") + }) + } + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err == nil { + t.Fatalf("MakeCert: got nil error, want error") + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests("ate-system").Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching PCR: %v", err) + } + if len(gotPCR.Status.Conditions) != 0 || gotPCR.Status.CertificateChain != "" { + t.Errorf("PCR status updated despite error: %+v", gotPCR.Status) + } + }) + } +} + +func TestMakeCertChainIncludesIntermediates(t *testing.T) { + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + intermediateCA, err := localca.GenerateED25519CA("test-intermediate") + if err != nil { + t.Fatalf("while generating intermediate CA: %v", err) + } + ca.IntermediateCertificates = []*x509.Certificate{intermediateCA.RootCertificate} + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + _, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + + pod, pcr := makePodAndPCR("ate-system", "atelet-abcde", "atelet", 86400) + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + + kc := fake.NewSimpleClientset(pod, pcr) + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err != nil { + t.Fatalf("MakeCert: %v", err) + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests("ate-system").Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching updated PCR: %v", err) + } + + var chainDER [][]byte + rest := []byte(gotPCR.Status.CertificateChain) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + chainDER = append(chainDER, block.Bytes) + } + if len(rest) != 0 { + t.Errorf("certificate chain has trailing non-PEM data") + } + if len(chainDER) != 2 { + t.Fatalf("got %d certificates in chain, want 2 (leaf + intermediate)", len(chainDER)) + } + if _, err := x509.ParseCertificate(chainDER[0]); err != nil { + t.Errorf("while parsing leaf certificate: %v", err) + } + if !bytes.Equal(chainDER[1], intermediateCA.RootCertificate.Raw) { + t.Errorf("second chain entry is not the intermediate certificate") + } +} + +func TestDesiredClusterTrustBundles(t *testing.T) { + ca1, err := localca.GenerateED25519CA("test-ca-1") + if err != nil { + t.Fatalf("while generating CA 1: %v", err) + } + ca2, err := localca.GenerateED25519CA("test-ca-2") + if err != nil { + t.Fatalf("while generating CA 2: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca1, ca2}} + impl := NewImpl(nil, caPool, fixedClock{now: testNow}) + + ctbs := impl.DesiredClusterTrustBundles() + if len(ctbs) != 1 { + t.Fatalf("got %d ClusterTrustBundles, want 1", len(ctbs)) + } + ctb := ctbs[0] + + if want := CTBPrefix + "primary-bundle"; ctb.ObjectMeta.Name != want { + t.Errorf("got CTB name %q, want %q", ctb.ObjectMeta.Name, want) + } + if ctb.Spec.SignerName != Name { + t.Errorf("got signer name %q, want %q", ctb.Spec.SignerName, Name) + } + if got := ctb.ObjectMeta.Labels["podcert.ate.dev/canarying"]; got != "live" { + t.Errorf("got canarying label %q, want %q", got, "live") + } + + wantBundle := &bytes.Buffer{} + for _, ca := range caPool.CAs { + wantBundle.Write(pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: ca.RootCertificate.Raw, + })) + } + if ctb.Spec.TrustBundle != wantBundle.String() { + t.Errorf("got trust bundle:\n%s\nwant:\n%s", ctb.Spec.TrustBundle, wantBundle.String()) + } +} diff --git a/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go b/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go index 4258f1921..1a995b20d 100644 --- a/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go +++ b/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go @@ -135,6 +135,13 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq } } + // This is returned as a transient error to allow retries. + // Without this, ate-apiserver can have a servicedns cert without DNS name + // while the covering Service is being created, and cache it for 24 hours. + if len(dnsNames) == 0 { + return fmt.Errorf("pod %s/%s is not (yet) selected by any Service; refusing to issue a serving cert with no DNS SANs", pcr.ObjectMeta.Namespace, pcr.Spec.PodName) + } + // TODO: Encode the OIDC issuer of the cluster into the certificate. subjectPublicKey, err := podcertificate.PublicKey(pcr) @@ -156,6 +163,7 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq notAfter := notBefore.Add(lifetime) beginRefreshAt := notAfter.Add(-30 * time.Minute) + parent := h.caPool.CAs[0].RootCertificate template := &x509.Certificate{ BasicConstraintsValid: true, NotBefore: notBefore, @@ -163,9 +171,12 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq DNSNames: dnsNames, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + // Link the leaf to its issuing CA by key id. Needed this for Valkey + // to understand which CA to use when validating a client cert. + AuthorityKeyId: parent.SubjectKeyId, } - subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, h.caPool.CAs[0].RootCertificate, subjectPublicKey, h.caPool.CAs[0].SigningKey) + subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey) if err != nil { return fmt.Errorf("while signing subject cert: %w", err) } diff --git a/go.mod b/go.mod index 1281dd80f..1a9e61197 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/redis/go-redis/v9 v9.18.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 + github.com/spiffe/go-spiffe/v2 v2.6.0 github.com/vishvananda/netlink v1.3.1 github.com/vishvananda/netns v0.0.5 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 @@ -156,7 +157,6 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect github.com/ugorji/go/codec v1.3.1 // indirect diff --git a/hack/install-ate.sh b/hack/install-ate.sh index f37803bc0..dbc2f0ad0 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -63,7 +63,7 @@ function usage() { echo " --deploy-ate-system Deploy core system (CRDs, atelet, apiserver)" echo " --delete-ate-system Delete core system" echo " --delete-all Delete core system and all registered demos" - echo " --auth-mode=mtls|jwt Select ateapi auth mode for --deploy-ate-system (default: mtls)" + echo " --ateapi-client-auth=cert|token Select how in-cluster clients authenticate to ateapi for --deploy-ate-system (default: cert; the server always accepts both)" echo "" echo "Infrastructure components:" echo "" @@ -132,26 +132,26 @@ run_ko() { esac } -ate_auth_mode() { - case "${ATE_API_AUTH_MODE:-mtls}" in - mtls|jwt) - echo "${ATE_API_AUTH_MODE:-mtls}" +ateapi_client_auth() { + case "${ATE_ATEAPI_CLIENT_AUTH:-cert}" in + cert|token) + echo "${ATE_ATEAPI_CLIENT_AUTH:-cert}" ;; *) - echo "Error: ATE_API_AUTH_MODE must be mtls or jwt, got '${ATE_API_AUTH_MODE}'" >&2 + echo "Error: ATE_ATEAPI_CLIENT_AUTH must be cert or token, got '${ATE_ATEAPI_CLIENT_AUTH}'" >&2 exit 1 ;; esac } render_ate_system_manifests() { - local auth_mode="" - auth_mode="$(ate_auth_mode)" + local client_auth="" + client_auth="$(ateapi_client_auth)" - if [[ "${auth_mode}" == "jwt" ]]; then - local overlay="manifests/ate-install/jwt" + if [[ "${client_auth}" == "token" ]]; then + local overlay="manifests/ate-install/token-client" if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - overlay="manifests/ate-install/kind-jwt" + overlay="manifests/ate-install/kind-token-client" fi kubectl kustomize "${overlay}" --load-restrictor LoadRestrictionsNone | run_ko resolve -f - return @@ -166,17 +166,36 @@ render_ate_system_manifests() { fi } -create_valkey_ca_certs_secret() { - log_step "create_valkey_ca_certs_secret" - local ca_certs="" - # Extract from in-cluster service-dns-ca-pool secret (base64 json) +# Extract a CA pool secret's RootCertificateDER and emit it as a PEM certificate. +ca_pool_root_pem() { + local secret="$1" local pool_json="" - pool_json=$(run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool -o jsonpath='{.data.pool}' | base64 --decode) - # Extract RootCertificateDER base64 string + pool_json=$(run_kubectl get secret -n podcertificate-controller-system "${secret}" -o jsonpath='{.data.pool}' | base64 --decode) local der_base64="" der_base64=$(echo "${pool_json}" | grep -o '"RootCertificateDER":"[^"]*' | sed 's/"RootCertificateDER":"//') - # Convert DER to PEM certificate - ca_certs=$(echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem) + echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem +} + +create_valkey_ca_certs_secret() { + log_step "create_valkey_ca_certs_secret" + # valkey requires a single tls-ca-cert-file to verify client and server certs it sees, + # so it needs both CAs: + # - servicedns CA: verifies valkey peers' server certs. + # - podidentity CA: verifies the client certs that connect to valkey + # (apiserver, the init job, and peers acting as clients). + # Extract each root into its own variable: errexit cannot see a substitution + # failing inside printf's argument list, which would silently produce a CA + # file with a missing root. + local servicedns_root="" + servicedns_root=$(ca_pool_root_pem service-dns-ca-pool) + local podidentity_root="" + podidentity_root=$(ca_pool_root_pem pod-identity-ca-pool) + if [[ -z "${servicedns_root}" || -z "${podidentity_root}" ]]; then + echo "error: failed to extract a CA root for valkey-ca-certs" >&2 + return 1 + fi + local ca_certs="" + ca_certs=$(printf '%s\n%s\n' "${servicedns_root}" "${podidentity_root}") run_kubectl create secret generic valkey-ca-certs \ --from-literal=ca.crt="${ca_certs}" \ @@ -226,7 +245,9 @@ create_api_server_env_vars() { redis_address="valkey-cluster.ate-system.svc:6379" use_iam_auth="false" tls_server_name="valkey-cluster.ate-system.svc" - client_cert="/run/servicedns.podcert.ate.dev/credential-bundle.pem" + # The apiserver dials valkey as a client, so it presents a podidentity + # (SPIFFE) client cert rather than a servicedns serving cert. + client_cert="/run/podidentity.podcert.ate.dev/credential-bundle.pem" echo "REDIS_ADDRESS: ${redis_address}" @@ -529,13 +550,13 @@ BENCHMARK_WORKER_COUNT=1 prescan_args=("$@") for ((i = 0; i < ${#prescan_args[@]}; i++)); do case "${prescan_args[i]}" in - --auth-mode=*) ATE_API_AUTH_MODE="${prescan_args[i]#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${prescan_args[i]#*=}" ;; + --ateapi-client-auth) if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi - ATE_API_AUTH_MODE="${prescan_args[$((i + 1))]}" + ATE_ATEAPI_CLIENT_AUTH="${prescan_args[$((i + 1))]}" ;; --benchmark-worker-count) BENCHMARK_WORKER_COUNT="${prescan_args[i+1]:-1}" @@ -560,14 +581,14 @@ while [[ "$#" -gt 0 ]]; do done case $1 in - --auth-mode=*) ATE_API_AUTH_MODE="${1#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${1#*=}" ;; + --ateapi-client-auth) shift if [[ "$#" -eq 0 ]]; then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi - ATE_API_AUTH_MODE="$1" + ATE_ATEAPI_CLIENT_AUTH="$1" ;; --deploy-ate-system) deploy_ate_system ;; diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 6522318cf..628877f4e 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -48,18 +48,18 @@ KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" -ATE_API_AUTH_MODE="${ATE_API_AUTH_MODE:-mtls}" +ATE_ATEAPI_CLIENT_AUTH="${ATE_ATEAPI_CLIENT_AUTH:-cert}" while [[ $# -gt 0 ]]; do case "$1" in - --auth-mode=*) ATE_API_AUTH_MODE="${1#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${1#*=}" ;; + --ateapi-client-auth) if [[ $# -lt 2 ]]; then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi shift - ATE_API_AUTH_MODE="$1" + ATE_ATEAPI_CLIENT_AUTH="$1" ;; *) echo "Error: unknown argument $1" >&2 @@ -69,10 +69,10 @@ while [[ $# -gt 0 ]]; do shift done -case "${ATE_API_AUTH_MODE}" in - mtls|jwt) ;; +case "${ATE_ATEAPI_CLIENT_AUTH}" in + cert|token) ;; *) - echo "Error: --auth-mode must be mtls or jwt, got '${ATE_API_AUTH_MODE}'" >&2 + echo "Error: --ateapi-client-auth must be cert or token, got '${ATE_ATEAPI_CLIENT_AUTH}'" >&2 exit 1 ;; esac @@ -131,10 +131,10 @@ fi log "Deploying the ate control plane (--deploy-ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" else # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" fi # --- 4. apply the demo ------------------------------------------------------ diff --git a/internal/ateapiauth/client.go b/internal/ateapiauth/client.go index 8ae0807fe..aed3b5d71 100644 --- a/internal/ateapiauth/client.go +++ b/internal/ateapiauth/client.go @@ -22,6 +22,7 @@ import ( "os" "strings" + "github.com/agent-substrate/substrate/internal/credbundle" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) @@ -31,27 +32,33 @@ const ( DefaultServiceAccountTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" ) -// ClientConfig configures how to dial the ateapi gRPC server. -// -// - Mode=ModeMTLS: insecure TLS dial (InsecureSkipVerify=true). Client -// identity is expected to come from mTLS credentials projected into -// the pod (servicedns.podcert.ate.dev). No app-level credentials. -// - Mode=ModeJWT: validates the server cert against CAFile, sends a Bearer -// token from TokenFile as per-RPC credentials. +// ClientConfig configures how to dial the ateapi gRPC server. The server +// cert is always validated against CAFile. UseTokenAuth selects the client +// credential: a client certificate from ClientCredBundle (mutual TLS, +// re-read on every handshake so in-place pod-certificate rotations are +// picked up) by default, or a Bearer token from TokenFile sent as per-RPC +// credentials. The path not selected is ignored. type ClientConfig struct { - Mode Mode + // UseTokenAuth authenticates with the Bearer token from TokenFile instead + // of the client certificate from ClientCredBundle. + UseTokenAuth bool // CAFile is a PEM file containing CA certs that sign the server cert. - // Required in all modes. Ignored for ModeMTLS until mTLS verification is - // fully wired. + // Required. CAFile string // ServerName overrides SNI / hostname verification. Optional. ServerName string // TokenFile is a path to a Kubernetes projected ServiceAccount token used - // as a Bearer credential. Required for ModeJWT. + // as a Bearer credential. Required when UseTokenAuth is set, ignored + // otherwise. TokenFile string + + // ClientCredBundle is a PEM file containing the client certificate chain + // and PKCS8 private key presented to the server. Required unless + // UseTokenAuth is set, ignored otherwise. + ClientCredBundle string } // DialOptions returns the grpc.DialOption set described by cfg, suitable to @@ -60,38 +67,44 @@ func DialOptions(cfg ClientConfig) ([]grpc.DialOption, error) { if cfg.CAFile == "" { return nil, fmt.Errorf("ateapiauth: CAFile is required") } - switch cfg.Mode { - case "", ModeMTLS: - tlsCfg := &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicit opt-in - return []grpc.DialOption{ - grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), - }, nil - - case ModeJWT: + if cfg.UseTokenAuth { if cfg.TokenFile == "" { - return nil, fmt.Errorf("ateapiauth: jwt mode requires TokenFile") - } - caPEM, err := os.ReadFile(cfg.CAFile) - if err != nil { - return nil, fmt.Errorf("ateapiauth: reading CA file: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(caPEM) { - return nil, fmt.Errorf("ateapiauth: no certificates found in CA file %q", cfg.CAFile) - } - tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: pool, - ServerName: cfg.ServerName, + return nil, fmt.Errorf("ateapiauth: token auth requires a token file") } + } else if cfg.ClientCredBundle == "" { + return nil, fmt.Errorf("ateapiauth: a client credential bundle (mTLS) is required unless token auth is enabled") + } + pool, err := loadCAPool(cfg.CAFile) + if err != nil { + return nil, err + } + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS13, + RootCAs: pool, + ServerName: cfg.ServerName, + } + if cfg.UseTokenAuth { return []grpc.DialOption{ grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), grpc.WithPerRPCCredentials(&fileTokenCreds{path: cfg.TokenFile}), }, nil - - default: - return nil, fmt.Errorf("ateapiauth: unknown client mode %q", cfg.Mode) } + tlsCfg.GetClientCertificate = credbundle.ClientLoader(cfg.ClientCredBundle) + return []grpc.DialOption{ + grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), + }, nil +} + +func loadCAPool(caFile string) (*x509.CertPool, error) { + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("ateapiauth: reading CA file: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("ateapiauth: no certificates found in CA file %q", caFile) + } + return pool, nil } // fileTokenCreds reads a Kubernetes projected SA token from disk for every diff --git a/internal/ateapiauth/client_test.go b/internal/ateapiauth/client_test.go index d8f271ee3..9109699d2 100644 --- a/internal/ateapiauth/client_test.go +++ b/internal/ateapiauth/client_test.go @@ -14,18 +14,279 @@ package ateapiauth -import "testing" +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) func TestDialOptionsRequiresCAFile(t *testing.T) { - for _, mode := range []Mode{ModeMTLS, ModeJWT} { - t.Run(string(mode), func(t *testing.T) { - _, err := DialOptions(ClientConfig{ - Mode: mode, - TokenFile: "token", - }) + for name, cfg := range map[string]ClientConfig{ + "token": {UseTokenAuth: true, TokenFile: "token"}, + "cert": {ClientCredBundle: "bundle.pem"}, + } { + t.Run(name, func(t *testing.T) { + _, err := DialOptions(cfg) if err == nil { t.Fatalf("DialOptions() error = nil, want error") } }) } } + +func TestDialOptionsRequiresModeCredential(t *testing.T) { + for name, cfg := range map[string]ClientConfig{ + "cert mode without bundle": {CAFile: "ca.pem"}, + "token mode without token": {CAFile: "ca.pem", UseTokenAuth: true}, + "cert path does not satisfy token mode": {CAFile: "ca.pem", UseTokenAuth: true, ClientCredBundle: "bundle.pem"}, + } { + t.Run(name, func(t *testing.T) { + _, err := DialOptions(cfg) + if err == nil { + t.Fatalf("DialOptions() error = nil, want error") + } + }) + } +} + +// TestDialOptionsMTLSHandshake dials a server that requires and verifies +// client certificates — the configuration ateapi will move to — and checks +// that DialOptions with a client credential bundle completes the handshake, +// while a certificate-less client is rejected. +func TestDialOptionsMTLSHandshake(t *testing.T) { + ca := newTestCA(t) + dir := t.TempDir() + caFile := filepath.Join(dir, "ca.pem") + writeFile(t, caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.certDER})) + + clientBundle := filepath.Join(dir, "client-bundle.pem") + writeFile(t, clientBundle, ca.issueClientBundle(t, "spiffe://cluster.local/ns/ate-system/sa/ate-controller")) + + serverCert := ca.issueServerCert(t) + caPool := x509.NewCertPool() + caPool.AddCert(ca.cert) + srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: caPool, + MinVersion: tls.VersionTLS13, + }))) + healthpb.RegisterHealthServer(srv, health.NewServer()) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + go srv.Serve(lis) + defer srv.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + t.Run("with client cert", func(t *testing.T) { + // TokenFile is ignored in cert mode (the default). + opts, err := DialOptions(ClientConfig{ + CAFile: caFile, + ClientCredBundle: clientBundle, + TokenFile: filepath.Join(dir, "does-not-exist-token"), + }) + if err != nil { + t.Fatalf("DialOptions() error = %v", err) + } + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code != codes.OK { + t.Fatalf("health check code = %v, want %v", code, codes.OK) + } + }) + + t.Run("without client cert is rejected", func(t *testing.T) { + opts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + RootCAs: caPool, + MinVersion: tls.VersionTLS13, + }))} + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code == codes.OK { + t.Fatalf("health check code = %v, want handshake failure", code) + } + }) +} + +// TestDialOptionsTokenSendsBearer dials a server that accepts certless +// clients — ateapi's current configuration — and checks that DialOptions +// with a token file attaches the token as an `authorization: Bearer` header +// on every RPC. +func TestDialOptionsTokenSendsBearer(t *testing.T) { + ca := newTestCA(t) + dir := t.TempDir() + caFile := filepath.Join(dir, "ca.pem") + writeFile(t, caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.certDER})) + tokenFile := filepath.Join(dir, "token") + writeFile(t, tokenFile, []byte("test-token\n")) + + gotAuth := make(chan string, 1) + srv := grpc.NewServer( + grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{ca.issueServerCert(t)}, + ClientAuth: tls.VerifyClientCertIfGiven, + MinVersion: tls.VersionTLS13, + })), + grpc.UnaryInterceptor(func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, _ := metadata.FromIncomingContext(ctx) + gotAuth <- strings.Join(md.Get("authorization"), ",") + return handler(ctx, req) + }), + ) + healthpb.RegisterHealthServer(srv, health.NewServer()) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + go srv.Serve(lis) + defer srv.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // ClientCredBundle stays set (as the base manifests leave it) but is + // ignored in token mode — it doesn't even need to exist on disk. + opts, err := DialOptions(ClientConfig{ + CAFile: caFile, + UseTokenAuth: true, + TokenFile: tokenFile, + ClientCredBundle: filepath.Join(dir, "does-not-exist.pem"), + }) + if err != nil { + t.Fatalf("DialOptions() error = %v", err) + } + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code != codes.OK { + t.Fatalf("health check code = %v, want %v", code, codes.OK) + } + if got, want := <-gotAuth, "Bearer test-token"; got != want { + t.Errorf("authorization header = %q, want %q", got, want) + } +} + +func healthCheckCode(ctx context.Context, t *testing.T, target string, opts []grpc.DialOption) codes.Code { + t.Helper() + conn, err := grpc.NewClient(target, opts...) + if err != nil { + t.Fatalf("grpc.NewClient() error = %v", err) + } + defer conn.Close() + _, err = healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}) + return status.Code(err) +} + +type testCA struct { + cert *x509.Certificate + certDER []byte + key *ecdsa.PrivateKey +} + +func newTestCA(t *testing.T) *testCA { + t.Helper() + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("create CA certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse CA certificate: %v", err) + } + return &testCA{cert: cert, certDER: der, key: key} +} + +// issueClientBundle returns a PEM credential bundle (leaf certificate + PKCS8 +// private key) for a client certificate carrying the given SPIFFE URI SAN. +func (ca *testCA) issueClientBundle(t *testing.T, spiffeID string) []byte { + t.Helper() + uri, err := url.Parse(spiffeID) + if err != nil { + t.Fatalf("parse SPIFFE ID: %v", err) + } + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "test-client"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + URIs: []*url.URL{uri}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("create client certificate: %v", err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal PKCS8 key: %v", err) + } + return append( + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})..., + ) +} + +func (ca *testCA) issueServerCert(t *testing.T) tls.Certificate { + t.Helper() + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(3), + Subject: pkix.Name{CommonName: "test-server"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("create server certificate: %v", err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +func generateKey(t *testing.T) *ecdsa.PrivateKey { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + return key +} + +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/internal/ateapiauth/server.go b/internal/ateapiauth/server.go index b8e5167bf..b2891c9da 100644 --- a/internal/ateapiauth/server.go +++ b/internal/ateapiauth/server.go @@ -12,75 +12,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ateapiauth adds optional Kubernetes ServiceAccount JWT -// authentication on top of the ateapi gRPC server, and a matching client -// dial helper. It does not replace the existing TLS / mTLS path — the -// server's transport credentials still apply unchanged. Set Mode=ModeJWT -// on the server to require an `authorization: Bearer ` header -// on every RPC; Mode=ModeMTLS (the default) leaves identity to the -// transport-layer mTLS credentials. +// Package ateapiauth authenticates clients of the ateapi gRPC server, and +// provides a matching client dial helper. The server interceptor takes +// identity from the transport-layer mTLS credentials when the client +// presented a certificate, and otherwise requires an authorization +// header `Bearer `. Requests with no credentials are rejected. package ateapiauth import ( "context" "fmt" + "log/slog" "strings" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/status" ) -// Mode selects whether the JWT interceptor enforces a Bearer token. -type Mode string - -const ( - ModeMTLS Mode = "mtls" - ModeJWT Mode = "jwt" -) - -// ParseMode parses a flag value into a Mode, defaulting to ModeMTLS on empty. -// ModeMTLS means identity is established by the transport-layer mTLS -// credentials; the interceptor performs no app-level checks. ModeJWT -// additionally requires a Kubernetes SA Bearer token on every RPC. -func ParseMode(s string) (Mode, error) { - switch Mode(s) { - case "", ModeMTLS: - return ModeMTLS, nil - case ModeJWT: - return ModeJWT, nil - default: - return "", fmt.Errorf("unknown auth mode %q (want mtls|jwt)", s) - } -} - func ValidateServerConfig(cfg ServerConfig) error { - switch cfg.Mode { - case "", ModeMTLS: - return nil - case ModeJWT: - if cfg.VerifyBearerToken == nil { - return fmt.Errorf("jwt mode requires bearer token verifier") - } - return nil - default: - return fmt.Errorf("unknown auth mode %q", cfg.Mode) + if cfg.VerifyBearerToken == nil { + return fmt.Errorf("a bearer token verifier is required") } + return nil } // ServerConfig configures the server-side auth interceptor. type ServerConfig struct { - Mode Mode - - // VerifyBearerToken verifies a Bearer token presented by a client. Required - // for ModeJWT and ignored for ModeMTLS. - VerifyBearerToken func(context.Context, string) error + // VerifyBearerToken verifies a Bearer token presented by a client and + // returns the authenticated principal's ID (e.g. the JWT subject). It + // authenticates clients that did not present a certificate identity + // (e.g. kubectl-ate, which dials without a client certificate). + VerifyBearerToken func(context.Context, string) (string, error) } // UnaryServerInterceptor returns a gRPC unary interceptor enforcing cfg. func UnaryServerInterceptor(cfg ServerConfig) grpc.UnaryServerInterceptor { - auth := serverAuthenticatorFor(cfg) + auth := newChainedAuthenticator(cfg) return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { newCtx, err := auth.authenticate(ctx) if err != nil { @@ -92,7 +64,7 @@ func UnaryServerInterceptor(cfg ServerConfig) grpc.UnaryServerInterceptor { // StreamServerInterceptor returns a gRPC stream interceptor enforcing cfg. func StreamServerInterceptor(cfg ServerConfig) grpc.StreamServerInterceptor { - auth := serverAuthenticatorFor(cfg) + auth := newChainedAuthenticator(cfg) return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { newCtx, err := auth.authenticate(ss.Context()) if err != nil { @@ -109,33 +81,69 @@ type wrappedStream struct { func (w *wrappedStream) Context() context.Context { return w.ctx } -type serverAuthenticator interface { - authenticate(context.Context) (context.Context, error) +func newChainedAuthenticator(cfg ServerConfig) chainedServerAuthenticator { + return chainedServerAuthenticator{ + jwt: jwtServerAuthenticator{ + verifyBearerToken: cfg.VerifyBearerToken, + }, + } } -func serverAuthenticatorFor(cfg ServerConfig) serverAuthenticator { - switch cfg.Mode { - case "", ModeMTLS: - return mtlsServerAuthenticator{} - case ModeJWT: - return jwtServerAuthenticator{ - verifyBearerToken: cfg.VerifyBearerToken, - } +// chainedServerAuthenticator first checks mTLS peer, then checks +// a bearer token in the header. +type chainedServerAuthenticator struct { + // jwt authenticates clients that did not present a certificate identity + // (e.g. kubectl-ate, which dials without a client certificate). + jwt jwtServerAuthenticator +} + +func (a chainedServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { + if id, ok := mtlsPeerIdentity(ctx); ok { + return principal.InjectContext(ctx, principal.PrincipalInfo{ + ID: id, + Kind: principal.KindMTLS, + }), nil + } + return a.jwt.authenticate(ctx) +} + +// mtlsPeerIdentity extracts the client identity (the first URI SAN, a SPIFFE +// ID) from the transport-authenticated peer certificate. +func mtlsPeerIdentity(ctx context.Context) (string, bool) { + p, ok := peer.FromContext(ctx) + if !ok || p.AuthInfo == nil { + slog.DebugContext(ctx, "No mTLS peer identity: no peer or auth info in context.") + return "", false } - return invalidServerAuthenticator{mode: cfg.Mode} -} + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok { + slog.DebugContext(ctx, "No mTLS peer identity: no TLS info in context.") + return "", false + } -type mtlsServerAuthenticator struct{} + if len(tlsInfo.State.PeerCertificates) == 0 { + slog.DebugContext(ctx, "No mTLS peer identity: no peer certificates in TLS info.") + return "", false + } -func (mtlsServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { - // TODO: Extract the transport-authenticated client identity and attach it - // to ctx once ateapi has an authorization layer. - return ctx, nil + clientCert := tlsInfo.State.PeerCertificates[0] + if len(clientCert.URIs) == 0 { + slog.DebugContext(ctx, "No mTLS peer identity: no URIs in peer certificate.") + return "", false + } + + id := clientCert.URIs[0].String() + if id == "" { + slog.DebugContext(ctx, "No mTLS peer identity: client cert URI is empty string") + return "", false + } + slog.InfoContext(ctx, "Authentication successful", slog.String("id", id)) + return id, true } type jwtServerAuthenticator struct { - verifyBearerToken func(context.Context, string) error + verifyBearerToken func(context.Context, string) (string, error) } func (a jwtServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { @@ -143,20 +151,14 @@ func (a jwtServerAuthenticator) authenticate(ctx context.Context) (context.Conte if !ok { return nil, status.Error(codes.Unauthenticated, "missing bearer token") } - if err := a.verifyBearerToken(ctx, bearer); err != nil { + id, err := a.verifyBearerToken(ctx, bearer) + if err != nil { return nil, status.Errorf(codes.Unauthenticated, "invalid bearer token: %v", err) } - // TODO: Attach the verified JWT identity to ctx once ateapi has an - // authorization layer that consumes it. - return ctx, nil -} - -type invalidServerAuthenticator struct { - mode Mode -} - -func (a invalidServerAuthenticator) authenticate(context.Context) (context.Context, error) { - return nil, status.Errorf(codes.Internal, "invalid auth mode %q", a.mode) + return principal.InjectContext(ctx, principal.PrincipalInfo{ + ID: id, + Kind: principal.KindJWT, + }), nil } func bearerToken(ctx context.Context) (string, bool) { diff --git a/internal/ateapiauth/server_test.go b/internal/ateapiauth/server_test.go index 5a0c0631e..e9538b5d8 100644 --- a/internal/ateapiauth/server_test.go +++ b/internal/ateapiauth/server_test.go @@ -16,49 +16,29 @@ package ateapiauth import ( "context" + "crypto/tls" + "crypto/x509" "fmt" + "net/url" "testing" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/status" ) -func TestParseMode(t *testing.T) { - cases := []struct { - in string - want Mode - wantErr bool - }{ - {"", ModeMTLS, false}, - {"mtls", ModeMTLS, false}, - {"jwt", ModeJWT, false}, - {"none", "", true}, - {"bogus", "", true}, - } - for _, tc := range cases { - got, err := ParseMode(tc.in) - if (err != nil) != tc.wantErr { - t.Errorf("ParseMode(%q) err=%v wantErr=%v", tc.in, err, tc.wantErr) - } - if !tc.wantErr && got != tc.want { - t.Errorf("ParseMode(%q)=%v want %v", tc.in, got, tc.want) - } - } -} - func TestValidateServerConfig(t *testing.T) { tests := []struct { name string cfg ServerConfig wantErr bool }{ - {name: "mtls zero config", cfg: ServerConfig{Mode: ModeMTLS}}, - {name: "empty mode zero config", cfg: ServerConfig{}}, - {name: "jwt valid", cfg: ServerConfig{Mode: ModeJWT, VerifyBearerToken: func(context.Context, string) error { return nil }}}, - {name: "jwt missing verifier", cfg: ServerConfig{Mode: ModeJWT}, wantErr: true}, - {name: "unknown mode", cfg: ServerConfig{Mode: Mode("bogus")}, wantErr: true}, + {name: "valid", cfg: ServerConfig{VerifyBearerToken: func(context.Context, string) (string, error) { return "", nil }}}, + {name: "missing verifier", cfg: ServerConfig{}, wantErr: true}, } for _, tt := range tests { @@ -71,17 +51,117 @@ func TestValidateServerConfig(t *testing.T) { } } -func TestMTLSServerAuthenticatorAllowsAnonymous(t *testing.T) { - _, err := (mtlsServerAuthenticator{}).authenticate(context.Background()) - if err != nil { - t.Fatalf("ModeMTLS should not error: %v", err) +func TestChainedServerAuthenticatorPrincipal(t *testing.T) { + const subject = "system:serviceaccount:ate-system:ate-client" + spiffeID := &url.URL{Scheme: "spiffe", Host: "ate.dev", Path: "/ns/default/sa/router"} + spiffePeer := func(ctx context.Context) context.Context { + return peer.NewContext(ctx, &peer.Peer{ + AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{spiffeID}}}, + }}, + }) + } + withBearer := func(ctx context.Context, token string) context.Context { + return metadata.NewIncomingContext(ctx, metadata.Pairs("authorization", "Bearer "+token)) + } + verifyGoodToken := func(_ context.Context, bearer string) (string, error) { + if bearer != "good-token" { + return "", fmt.Errorf("bad token") + } + return subject, nil + } + + tests := []struct { + name string + ctx context.Context + // verify is the bearer token verifier; nil means the test fails if + // it is called (the certificate identity must take precedence). + verify func(context.Context, string) (string, error) + want principal.PrincipalInfo + wantCode codes.Code + }{ + { + name: "no peer and no token", + ctx: context.Background(), + wantCode: codes.Unauthenticated, + }, + { + name: "peer without certificates and no token", + ctx: peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: credentials.TLSInfo{}, + }), + wantCode: codes.Unauthenticated, + }, + { + name: "no peer with valid bearer", + ctx: withBearer(context.Background(), "good-token"), + verify: verifyGoodToken, + want: principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT}, + wantCode: codes.OK, + }, + { + name: "no peer with invalid bearer", + ctx: withBearer(context.Background(), "bad-token"), + verify: verifyGoodToken, + wantCode: codes.Unauthenticated, + }, + { + name: "certificate without URI SAN with valid bearer", + ctx: withBearer(peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{{}}, + }}, + }), "good-token"), + verify: verifyGoodToken, + want: principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT}, + wantCode: codes.OK, + }, + { + name: "certificate with SPIFFE URI SAN", + ctx: spiffePeer(context.Background()), + want: principal.PrincipalInfo{ID: spiffeID.String(), Kind: principal.KindMTLS}, + wantCode: codes.OK, + }, + { + name: "certificate takes precedence over bearer", + ctx: withBearer(spiffePeer(context.Background()), "good-token"), + want: principal.PrincipalInfo{ID: spiffeID.String(), Kind: principal.KindMTLS}, + wantCode: codes.OK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + verify := tt.verify + if verify == nil { + verify = func(context.Context, string) (string, error) { + t.Fatal("bearer token verifier called; certificate identity must take precedence") + return "", nil + } + } + auth := newChainedAuthenticator(ServerConfig{VerifyBearerToken: verify}) + newCtx, err := auth.authenticate(tt.ctx) + if code := status.Code(err); code != tt.wantCode { + t.Fatalf("authenticate: code=%v (err=%v), want %v", code, err, tt.wantCode) + } + if tt.wantCode != codes.OK { + return + } + got, ok := principal.FromContext(newCtx) + if !ok { + t.Fatal("no principal in context") + } + if got != tt.want { + t.Errorf("principal=%+v want %+v", got, tt.want) + } + }) } } func TestJWTServerAuthenticatorRequiresBearer(t *testing.T) { auth := jwtServerAuthenticator{ - verifyBearerToken: func(context.Context, string) error { - return fmt.Errorf("bad token") + verifyBearerToken: func(context.Context, string) (string, error) { + return "", fmt.Errorf("bad token") }, } @@ -99,6 +179,29 @@ func TestJWTServerAuthenticatorRequiresBearer(t *testing.T) { } } +func TestJWTServerAuthenticatorInjectsPrincipal(t *testing.T) { + const subject = "system:serviceaccount:default:router" + auth := jwtServerAuthenticator{ + verifyBearerToken: func(context.Context, string) (string, error) { + return subject, nil + }, + } + + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer good-token")) + newCtx, err := auth.authenticate(ctx) + if err != nil { + t.Fatalf("authenticate: %v", err) + } + got, ok := principal.FromContext(newCtx) + if !ok { + t.Fatal("no principal in context") + } + want := principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT} + if got != want { + t.Errorf("principal=%+v want %+v", got, want) + } +} + func TestBearerToken(t *testing.T) { cases := []struct { name string diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 771c0e9bd..ed72a7599 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -17,11 +17,11 @@ package ateclient import ( "context" "crypto/tls" + "crypto/x509" "fmt" "io" "net/http" "os" - "strings" "sync" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -45,6 +45,16 @@ import ( metricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned" ) +const ( + apiServerName = "api.ate-system.svc" + + // serviceDNSSignerName and liveBundleSelector mirror the + // clusterTrustBundle projected-volume sources that in-cluster clients + // mount to verify ateapi's serving cert. + serviceDNSSignerName = "servicedns.podcert.ate.dev/identity" + liveBundleSelector = "podcert.ate.dev/canarying=live" +) + // Client wraps the gRPC ControlClient and DebugClient and ensures the port-forward connection is closed when done. type Client struct { ateapipb.ControlClient @@ -78,7 +88,7 @@ func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, var cli *Client if endpoint != "" { - cli, err = dialDirect(kubeconfigPath, k8sContext, endpoint, traceEnabled) + cli, err = dialDirect(ctx, kubeconfigPath, k8sContext, endpoint, traceEnabled) } else { cli, err = dialPortForward(ctx, kubeconfigPath, k8sContext, traceEnabled) } @@ -92,13 +102,27 @@ func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, return cli, nil } -func dialDirect(kubeconfigPath, k8sContext, endpoint string, traceEnabled bool) (*Client, error) { - // Always assume TLS to match production behavior - creds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) +func dialDirect(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, traceEnabled bool) (*Client, error) { + clientset, err := NewK8sClientset(kubeconfigPath, k8sContext) + if err != nil { + return nil, fmt.Errorf("failed to create k8s client: %w", err) + } + + // Verify the server before attaching the bearer token below: the token + // must never be sent over an unauthenticated channel. + tlsCfg, err := serverTLSConfig(ctx, clientset) + if err != nil { + return nil, err + } var opts []grpc.DialOption - opts = append(opts, grpc.WithTransportCredentials(creds)) + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))) opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) + tokenOpt, err := bearerTokenDialOption(ctx, clientset) + if err != nil { + return nil, err + } + opts = append(opts, tokenOpt) if traceEnabled { opts = append(opts, grpc.WithUnaryInterceptor(newTraceInterceptor())) @@ -207,19 +231,21 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext string, tra localPort := forwardedPorts[0].Local localEndpoint := fmt.Sprintf("127.0.0.1:%d", localPort) - // The ate-api-server uses TLS with pod certificates, so we need InsecureSkipVerify - // to talk to it over localhost. - transportCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) - - var opts []grpc.DialOption - opts = append(opts, grpc.WithTransportCredentials(transportCreds)) - opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) - jwtOpts, err := jwtDialOptions(ctx, clientset) + tlsCfg, err := serverTLSConfig(ctx, clientset) if err != nil { close(stopCh) return nil, err } - opts = append(opts, jwtOpts...) + + var opts []grpc.DialOption + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))) + opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) + tokenOpt, err := bearerTokenDialOption(ctx, clientset) + if err != nil { + close(stopCh) + return nil, err + } + opts = append(opts, tokenOpt) if traceEnabled { opts = append(opts, grpc.WithUnaryInterceptor(newTraceInterceptor())) @@ -242,19 +268,43 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext string, tra }, nil } -func jwtDialOptions(ctx context.Context, clientset *kubernetes.Clientset) ([]grpc.DialOption, error) { - jwtMode, err := isJWTMode(ctx, clientset) +func serverTLSConfig(ctx context.Context, clientset kubernetes.Interface) (*tls.Config, error) { + ctbs, err := clientset.CertificatesV1beta1().ClusterTrustBundles().List(ctx, metav1.ListOptions{ + LabelSelector: liveBundleSelector, + }) if err != nil { - return nil, err - } - if !jwtMode { - return nil, nil + return nil, fmt.Errorf("failed to list ClusterTrustBundles: %w", err) } + pool := x509.NewCertPool() + found := false + for _, ctb := range ctbs.Items { + if ctb.Spec.SignerName != serviceDNSSignerName { + continue + } + if !pool.AppendCertsFromPEM([]byte(ctb.Spec.TrustBundle)) { + return nil, fmt.Errorf("ClusterTrustBundle %q contains no valid certificates", ctb.ObjectMeta.Name) + } + found = true + } + if !found { + return nil, fmt.Errorf("no live ClusterTrustBundle found for signer %q", serviceDNSSignerName) + } + + return &tls.Config{ + MinVersion: tls.VersionTLS13, + RootCAs: pool, + ServerName: apiServerName, + }, nil +} + +// bearerTokenDialOption attaches a ServiceAccount token for the ate-client SA +// as per-RPC credentials. +func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset) (grpc.DialOption, error) { expirationSeconds := int64(3600) tokenRequest := &authv1.TokenRequest{ Spec: authv1.TokenRequestSpec{ - Audiences: []string{"api.ate-system.svc"}, + Audiences: []string{apiServerName}, ExpirationSeconds: &expirationSeconds, }, } @@ -265,38 +315,7 @@ func jwtDialOptions(ctx context.Context, clientset *kubernetes.Clientset) ([]grp if token.Status.Token == "" { return nil, fmt.Errorf("failed to request ateapi bearer token: token response was empty") } - return []grpc.DialOption{grpc.WithPerRPCCredentials(bearerTokenCreds(token.Status.Token))}, nil -} - -func isJWTMode(ctx context.Context, clientset *kubernetes.Clientset) (bool, error) { - // TODO: Replace deployment introspection with an explicit client-readable - // config file once ateapi auth mode is part of install/runtime config. - deployment, err := clientset.AppsV1().Deployments("ate-system").Get(ctx, "ate-api-server", metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("failed to get ate-api-server deployment: %w", err) - } - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name != "ate-api-server" { - continue - } - return isJWTAuthModeArg(container.Args), nil - } - return false, fmt.Errorf("failed to find ate-api-server container in deployment") -} - -func isJWTAuthModeArg(args []string) bool { - for i, arg := range args { - if arg == "--auth-mode=jwt" { - return true - } - if strings.HasPrefix(arg, "--auth-mode=") { - return strings.TrimPrefix(arg, "--auth-mode=") == "jwt" - } - if arg == "--auth-mode" && i+1 < len(args) { - return args[i+1] == "jwt" - } - } - return false + return grpc.WithPerRPCCredentials(bearerTokenCreds(token.Status.Token)), nil } type bearerTokenCreds string diff --git a/internal/ateclient/builder_test.go b/internal/ateclient/builder_test.go index 62c44feb9..3c9f43393 100644 --- a/internal/ateclient/builder_test.go +++ b/internal/ateclient/builder_test.go @@ -14,26 +14,139 @@ package ateclient -import "testing" +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "testing" + "time" -func TestIsJWTAuthModeArg(t *testing.T) { - tests := []struct { - name string - args []string - want bool - }{ - {name: "equals jwt", args: []string{"--auth-mode=jwt"}, want: true}, - {name: "split jwt", args: []string{"--auth-mode", "jwt"}, want: true}, - {name: "equals mtls", args: []string{"--auth-mode=mtls"}, want: false}, - {name: "split mtls", args: []string{"--auth-mode", "mtls"}, want: false}, - {name: "missing value", args: []string{"--auth-mode"}, want: false}, - {name: "unrelated", args: []string{"--foo=bar"}, want: false}, + certsv1beta1 "k8s.io/api/certificates/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" +) + +func TestBearerTokenCreds(t *testing.T) { + md, err := bearerTokenCreds("some-token").GetRequestMetadata(context.Background()) + if err != nil { + t.Fatalf("GetRequestMetadata: %v", err) + } + if got, want := md["authorization"], "Bearer some-token"; got != want { + t.Errorf("authorization=%q want %q", got, want) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isJWTAuthModeArg(tt.args); got != tt.want { - t.Fatalf("isJWTAuthModeArg(%v) = %v, want %v", tt.args, got, tt.want) + if _, err := bearerTokenCreds("").GetRequestMetadata(context.Background()); err == nil { + t.Error("GetRequestMetadata with empty token: want error, got nil") + } + + if !bearerTokenCreds("some-token").RequireTransportSecurity() { + t.Error("RequireTransportSecurity() = false, want true") + } +} + +// testCAPEM generates a self-signed CA and returns its PEM encoding. +func testCAPEM(t *testing.T, cn string) []byte { + t.Helper() + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generating key for %s: %v", cn, err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: cn}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, priv) + if err != nil { + t.Fatalf("creating CA cert for %s: %v", cn, err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func trustBundle(name, signer string, live bool, pemData []byte) *certsv1beta1.ClusterTrustBundle { + ctb := &certsv1beta1.ClusterTrustBundle{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: certsv1beta1.ClusterTrustBundleSpec{ + SignerName: signer, + TrustBundle: string(pemData), + }, + } + if live { + ctb.ObjectMeta.Labels = map[string]string{"podcert.ate.dev/canarying": "live"} + } + return ctb +} + +func TestServerTLSConfig(t *testing.T) { + servicednsCA1 := testCAPEM(t, "servicedns-ca-1") + servicednsCA2 := testCAPEM(t, "servicedns-ca-2") + podidentityCA := testCAPEM(t, "podidentity-ca") + canaryCA := testCAPEM(t, "servicedns-ca-canary") + + clientset := fake.NewSimpleClientset( + trustBundle("servicedns.podcert.ate.dev:identity:primary-bundle", serviceDNSSignerName, true, append(servicednsCA1, servicednsCA2...)), + trustBundle("podidentity.podcert.ate.dev:identity:primary-bundle", "podidentity.podcert.ate.dev/identity", true, podidentityCA), + trustBundle("servicedns.podcert.ate.dev:identity:canary-bundle", serviceDNSSignerName, false, canaryCA), + ) + + cfg, err := serverTLSConfig(context.Background(), clientset) + if err != nil { + t.Fatalf("serverTLSConfig: %v", err) + } + + if got, want := cfg.ServerName, "api.ate-system.svc"; got != want { + t.Errorf("ServerName=%q want %q", got, want) + } + if cfg.MinVersion < tls.VersionTLS13 { + t.Errorf("MinVersion=%x want at least %x", cfg.MinVersion, tls.VersionTLS13) + } + if cfg.InsecureSkipVerify { + t.Error("InsecureSkipVerify=true, want false") + } + + // The pool must contain exactly the live servicedns CAs: not the + // podidentity bundle and not the non-live canary bundle. + wantPool := x509.NewCertPool() + wantPool.AppendCertsFromPEM(servicednsCA1) + wantPool.AppendCertsFromPEM(servicednsCA2) + if !cfg.RootCAs.Equal(wantPool) { + t.Error("RootCAs does not match the live servicedns trust bundle") + } +} + +func TestServerTLSConfigErrors(t *testing.T) { + for _, tc := range []struct { + name string + objects []runtime.Object + }{ + {name: "no bundles"}, + { + name: "only other signers", + objects: []runtime.Object{ + trustBundle("podidentity.podcert.ate.dev:identity:primary-bundle", "podidentity.podcert.ate.dev/identity", true, testCAPEM(t, "podidentity-ca")), + }, + }, + { + name: "bundle with no valid certificates", + objects: []runtime.Object{ + trustBundle("servicedns.podcert.ate.dev:identity:primary-bundle", serviceDNSSignerName, true, []byte("not a pem")), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + clientset := fake.NewSimpleClientset(tc.objects...) + if _, err := serverTLSConfig(context.Background(), clientset); err == nil { + t.Error("serverTLSConfig: want error, got nil") } }) } diff --git a/internal/ateinterceptors/ateinterceptors.go b/internal/ateinterceptors/ateinterceptors.go index 81fb5a195..495a226ec 100644 --- a/internal/ateinterceptors/ateinterceptors.go +++ b/internal/ateinterceptors/ateinterceptors.go @@ -21,6 +21,7 @@ import ( "strconv" "time" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -46,12 +47,15 @@ func ServerUnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServer strconv.FormatInt(elapsed.Microseconds(), 10), )) + pInfo, _ := principal.FromContext(ctx) + slog.InfoContext(ctx, "Handle RPC", slog.String("method", info.FullMethod), slog.Any("req", sanitizeForLog(req)), slog.Any("resp", sanitizeForLog(resp)), slog.Any("err", err), slog.String("elapsed-time", elapsed.String()), + slog.Any("principal", pInfo), ) if err != nil { diff --git a/cmd/ateapi/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go similarity index 85% rename from cmd/ateapi/internal/credbundle/credbundle.go rename to internal/credbundle/credbundle.go index 58d86df76..659a53899 100644 --- a/cmd/ateapi/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -38,6 +38,17 @@ func Loader(path string) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { } } +// ClientLoader is the client-side counterpart to Loader. It returns a function +// suitable for use as GetClientCertificate in a tls.Config, re-reading the +// bundle on each handshake so that in-place pod-certificate rotations are +// picked up. +func ClientLoader(path string) func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + // TODO: Introduce caching. + return func(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return Parse(path) + } +} + // Parse reads a private key and certificate chain from a credential bundle file as written by the // Kubernetes Pod Certificates mechanism. func Parse(bundlePath string) (*tls.Certificate, error) { diff --git a/cmd/ateapi/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go similarity index 100% rename from cmd/ateapi/internal/credbundle/credbundle_test.go rename to internal/credbundle/credbundle_test.go diff --git a/internal/principal/principal.go b/internal/principal/principal.go new file mode 100644 index 000000000..c6169aaed --- /dev/null +++ b/internal/principal/principal.go @@ -0,0 +1,45 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package principal + +import "context" + +// Kind values for PrincipalInfo, named after the authentication method that +// established the identity. +const ( + KindMTLS = "mtls" + KindJWT = "jwt" +) + +// PrincipalInfo contains information about an authenticated principal. +type PrincipalInfo struct { + ID string + Kind string +} + +type contextKey struct{} + +var principalKey = contextKey{} + +// FromContext returns the PrincipalInfo from the context, if any. +func FromContext(ctx context.Context) (PrincipalInfo, bool) { + p, ok := ctx.Value(principalKey).(PrincipalInfo) + return p, ok +} + +// InjectContext returns a new context with the given PrincipalInfo. +func InjectContext(ctx context.Context, p PrincipalInfo) context.Context { + return context.WithValue(ctx, principalKey, p) +} diff --git a/internal/substratex509/substratex509.go b/internal/substratex509/substratex509.go new file mode 100644 index 000000000..58374e4af --- /dev/null +++ b/internal/substratex509/substratex509.go @@ -0,0 +1,133 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package substratex509 contains routines for creating and parsing x509 +// certificates that embed Substrate-specific X.509 extensions communicating +// the identity of a given workload. It is modeled on the upstream Kubernetes +// component-helpers/kubernetesx509 package, but encodes extension values as +// JSON instead of ASN.1. +package substratex509 + +import ( + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "fmt" + "strings" +) + +var ( + // GoogleSubstratePEN is the ASN.1 Private Enterprise Number arc used to + // name X.509 extensions that communicate Substrate-specific concepts. + GoogleSubstratePEN = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 12} + + // oidPodIdentity identifies the Kubernetes PodIdentity X.509 extension specifically in substrate. + oidPodIdentity = makeSubstrateOID(1) +) + +func makeSubstrateOID(subIDs ...int) asn1.ObjectIdentifier { + base := asn1.ObjectIdentifier{} + base = append(base, GoogleSubstratePEN...) + base = append(base, subIDs...) + return base +} + +// PodIdentity is the Kubernetes Pod Identity of a pod, as embedded in the +// oidPodIdentity extension of its certificate. +type PodIdentity struct { + Namespace string + ServiceAccountName string + ServiceAccountUID string + PodName string + PodUID string + NodeName string + NodeUID string +} + +func AddPodIdentityToCertificate(pod *PodIdentity, template *x509.Certificate) error { + if err := validatePodIdentity(pod); err != nil { + return fmt.Errorf("while validating PodIdentity input: %w", err) + } + podIdentityBytes, err := json.Marshal(pod) + if err != nil { + return fmt.Errorf("while json-marshaling PodIdentity extension: %w", err) + } + + template.ExtraExtensions = append(template.ExtraExtensions, pkix.Extension{ + Id: oidPodIdentity, + Value: podIdentityBytes, + }) + + return nil +} + +func PodIdentityFromCertificate(cert *x509.Certificate) (*PodIdentity, error) { + podIdentityCount := 0 + + var podIdentityValue []byte + for _, ext := range cert.Extensions { + if ext.Id.Equal(oidPodIdentity) { + podIdentityCount++ + podIdentityValue = ext.Value + } + } + + if podIdentityCount == 0 { + return nil, nil + } + if podIdentityCount > 1 { + return nil, fmt.Errorf("certificate contains multiple PodIdentity extensions") + } + + pod := &PodIdentity{} + if err := json.Unmarshal(podIdentityValue, pod); err != nil { + return nil, fmt.Errorf("while json-unmarshaling PodIdentity extension: %w", err) + } + + if err := validatePodIdentity(pod); err != nil { + return nil, fmt.Errorf("while validating PodIdentity extension: %w", err) + } + + return pod, nil +} + +func validatePodIdentity(pod *PodIdentity) error { + var empty []string + if pod.Namespace == "" { + empty = append(empty, "Namespace") + } + if pod.ServiceAccountName == "" { + empty = append(empty, "ServiceAccountName") + } + if pod.ServiceAccountUID == "" { + empty = append(empty, "ServiceAccountUID") + } + if pod.PodName == "" { + empty = append(empty, "PodName") + } + if pod.PodUID == "" { + empty = append(empty, "PodUID") + } + if pod.NodeName == "" { + empty = append(empty, "NodeName") + } + if pod.NodeUID == "" { + empty = append(empty, "NodeUID") + } + if len(empty) > 0 { + return fmt.Errorf("empty fields: %s", strings.Join(empty, ", ")) + } + return nil +} diff --git a/internal/substratex509/substratex509_test.go b/internal/substratex509/substratex509_test.go new file mode 100644 index 000000000..42aae802e --- /dev/null +++ b/internal/substratex509/substratex509_test.go @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package substratex509 + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "math/big" + "reflect" + "strings" + "testing" + "time" +) + +// mintCert self-signs a certificate from the template and parses it back, so +// that ExtraExtensions round-trip into Extensions. +func mintCert(t *testing.T, template *x509.Certificate) *x509.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating key: %v", err) + } + template.SerialNumber = big.NewInt(1) + template.NotBefore = time.Now().Add(-time.Hour) + template.NotAfter = time.Now().Add(time.Hour) + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing certificate: %v", err) + } + return cert +} + +// mintCertWithExtension mints a certificate carrying a single PodIdentity +// extension with the given raw value, bypassing the validation in +// AddPodIdentityToCertificate. +func mintCertWithExtension(t *testing.T, value []byte) *x509.Certificate { + t.Helper() + return mintCert(t, &x509.Certificate{ + ExtraExtensions: []pkix.Extension{ + {Id: testPodIdentityOID, Value: value}, + }, + }) +} + +// testPodIdentityOID spells out the intended PodIdentity extension OID +// to prevent accidental modification to the value. +var testPodIdentityOID = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 12, 1} + +func TestPodIdentityFromCertificate(t *testing.T) { + // fullPodIdentity has every field populated, as required by + // AddPodIdentityToCertificate. Cases that mutate it copy it first. + fullPodIdentity := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + for _, tc := range []struct { + name string + cert func(t *testing.T) *x509.Certificate + want *PodIdentity + wantErr string // substring of the expected error; "" means no error + }{ + { + name: "RoundTrip", + cert: func(t *testing.T) *x509.Certificate { + template := &x509.Certificate{} + if err := AddPodIdentityToCertificate(&fullPodIdentity, template); err != nil { + t.Fatalf("AddPodIdentityToCertificate: %v", err) + } + return mintCert(t, template) + }, + want: &fullPodIdentity, + }, + { + name: "Absent", + cert: func(t *testing.T) *x509.Certificate { + return mintCert(t, &x509.Certificate{}) + }, + want: nil, + }, + { + name: "Duplicate", + cert: func(t *testing.T) *x509.Certificate { + // Go's x509 parser rejects certificates with duplicate + // extensions, so a duplicate can only reach + // PodIdentityFromCertificate via a Certificate constructed + // by other means. Build one directly. + ext := pkix.Extension{Id: testPodIdentityOID, Value: []byte(`{"PodUID":"pod-uid"}`)} + return &x509.Certificate{Extensions: []pkix.Extension{ext, ext}} + }, + wantErr: "multiple PodIdentity extensions", + }, + { + name: "EmptyField", + cert: func(t *testing.T) *x509.Certificate { + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + NodeName: "node-1", + NodeUID: "node-uid", + } + value, err := json.Marshal(pod) + if err != nil { + t.Fatalf("marshaling PodIdentity: %v", err) + } + return mintCertWithExtension(t, value) + }, + wantErr: "PodUID", + }, + { + name: "Malformed", + cert: func(t *testing.T) *x509.Certificate { + return mintCertWithExtension(t, []byte("not json")) + }, + wantErr: "json-unmarshaling", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := PodIdentityFromCertificate(tc.cert(t)) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("PodIdentityFromCertificate succeeded, want error containing %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("PodIdentityFromCertificate error = %q, want it to contain %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("PodIdentityFromCertificate: %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("PodIdentityFromCertificate = %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestAddPodIdentityToCertificateEmptyField(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*PodIdentity) + }{ + {"Namespace", func(p *PodIdentity) { p.Namespace = "" }}, + {"ServiceAccountName", func(p *PodIdentity) { p.ServiceAccountName = "" }}, + {"ServiceAccountUID", func(p *PodIdentity) { p.ServiceAccountUID = "" }}, + {"PodName", func(p *PodIdentity) { p.PodName = "" }}, + {"PodUID", func(p *PodIdentity) { p.PodUID = "" }}, + {"NodeName", func(p *PodIdentity) { p.NodeName = "" }}, + {"NodeUID", func(p *PodIdentity) { p.NodeUID = "" }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + tc.mutate(&pod) + err := AddPodIdentityToCertificate(&pod, &x509.Certificate{}) + if err == nil { + t.Fatalf("AddPodIdentityToCertificate succeeded, want error for empty %s", tc.name) + } + if !strings.Contains(err.Error(), tc.name) { + t.Errorf("error %q does not name the empty field %s", err, tc.name) + } + }) + } +} + +// The extension value is plain JSON so that non-Go verifiers can parse it +// without an ASN.1 library. +func TestExtensionValueIsJSON(t *testing.T) { + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + template := &x509.Certificate{} + if err := AddPodIdentityToCertificate(&pod, template); err != nil { + t.Fatalf("AddPodIdentityToCertificate: %v", err) + } + cert := mintCert(t, template) + + for _, ext := range cert.Extensions { + if ext.Id.Equal(testPodIdentityOID) { + if !json.Valid(ext.Value) { + t.Errorf("extension value is not valid JSON: %q", ext.Value) + } + return + } + } + t.Fatal("PodIdentity extension not found in certificate") +} diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index f86294953..5322a661a 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -90,7 +90,8 @@ spec: - --client-jwt-audience=api.ate-system.svc - --session-id-jwt-pool=/run/session-id-jwt-pool/pool.json - --session-id-ca-pool=/run/session-id-ca-pool/pool.json - - --workerpool-ca-certs=/run/workerpool-ca-certs/trust-bundle.pem + - --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem env: - name: POD_NAME valueFrom: @@ -117,8 +118,14 @@ spec: name: ate-api-server-envvars optional: true volumeMounts: + # servicedns: the apiserver's own gRPC serving cert (DNS SAN api.ate-system.svc). - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" + # podidentity: the apiserver's client identity (SPIFFE) when it dials + # valkey and atelet, plus the trust bundle for verifying atelet's + # serving certificate. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" - name: "session-id-jwt-pool" mountPath: "/run/session-id-jwt-pool" # Note: See README.md for how to generate this secret. @@ -128,9 +135,6 @@ spec: - name: "session-id-ca-pool" mountPath: "/run/session-id-ca-pool" readOnly: true - - name: "workerpool-ca-certs" - mountPath: "/run/workerpool-ca-certs" - readOnly: true ports: - containerPort: 443 - name: prometheus @@ -149,6 +153,19 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem - name: "session-id-jwt-pool" projected: sources: @@ -173,15 +190,6 @@ spec: items: - key: "pool" path: "pool.json" - - name: "workerpool-ca-certs" - projected: - sources: - - clusterTrustBundle: - signerName: podidentity.podcert.ate.dev/identity - labelSelector: - matchLabels: - podcert.ate.dev/canarying: live - path: trust-bundle.pem --- # 6. Expose the Session Assigner apiVersion: v1 diff --git a/manifests/ate-install/ate-controller.yaml b/manifests/ate-install/ate-controller.yaml index 65ee46075..a6955bb81 100644 --- a/manifests/ate-install/ate-controller.yaml +++ b/manifests/ate-install/ate-controller.yaml @@ -90,6 +90,7 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atecontroller args: - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem ports: - name: metrics containerPort: 8080 @@ -97,3 +98,27 @@ spec: - name: healthz containerPort: 8081 protocol: TCP + volumeMounts: + # Controller's own client identity presented to ateapi. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" + # Trust bundle used to verify ateapi's servicedns serving cert. + - name: "servicedns-ca" + mountPath: "/run/servicedns-ca" + volumes: + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: "servicedns-ca" + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 3ce7f82f1..b8e057a86 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -69,6 +69,8 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atelet args: - --gcp-auth-for-image-pulls=true + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem # atelet does no mounts, netlink, device, or namespace operations (those # live in the ateom worker pod) — it only reads/writes the # /var/lib/ateom-gvisor hostPath as root, so it needs no Linux @@ -118,8 +120,27 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ateom-gvisor + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev volumes: - name: run-ateom hostPath: path: /var/lib/ateom-gvisor type: DirectoryOrCreate + # Identity for mutual TLS with the ate-apiserver. atelet is not behind a + # Service, so it uses a podidentity (SPIFFE) cert rather than a servicedns + # serving cert (which requires DNS SANs it would not have); the + # clusterTrustBundle verifies the apiserver's client certificate. + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 967f22c29..ad6f9dfd6 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -131,12 +131,15 @@ spec: - "--port-xds=18000" - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - - "--ateapi-address=api.ate-system.svc:443" - "--status-port=4040" - "--port-https=8443" - "--envoy-cert-path=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--otlp-collector-address=opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317" + # Client auth to ateapi (mtls): verify the serving cert against the + # servicedns trust bundle and present the podidentity client cert. + - "--ateapi-address=api.ate-system.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" env: - name: POD_NAME valueFrom: @@ -163,6 +166,13 @@ spec: containerPort: 4040 - name: metrics containerPort: 9090 + volumeMounts: + # Router's own client identity presented to ateapi. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" + # Trust bundle used to verify ateapi's servicedns serving cert. + - name: "servicedns-ca" + mountPath: "/run/servicedns-ca" - name: envoy image: envoyproxy/envoy:v1.30-latest command: @@ -194,6 +204,22 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: "servicedns-ca" + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem --- apiVersion: v1 kind: Service diff --git a/manifests/ate-install/kind-jwt/kustomization.yaml b/manifests/ate-install/kind-token-client/kustomization.yaml similarity index 56% rename from manifests/ate-install/kind-jwt/kustomization.yaml rename to manifests/ate-install/kind-token-client/kustomization.yaml index 8769b7a10..d95c6be06 100644 --- a/manifests/ate-install/kind-jwt/kustomization.yaml +++ b/manifests/ate-install/kind-token-client/kustomization.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Overlay: in-cluster clients authenticate to ateapi with a projected +# ServiceAccount token instead of a client certificate. The server always +# accepts both. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -19,17 +22,11 @@ resources: - ../kind patches: - - path: ../jwt/patches.yaml - - target: - group: apps - version: v1 - kind: Deployment - name: ate-api-server - namespace: ate-system - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: --auth-mode=jwt + - path: ../token-client/patches.yaml + # --ateapi-use-token-auth switches the client to the projected + # ServiceAccount token below. The base manifests' --ateapi-client-cert is + # removed by index; the preceding `test` op pins the expected value so the + # build fails loudly if the base args are ever reordered. - target: group: apps version: v1 @@ -37,9 +34,14 @@ patches: name: ate-controller namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/1 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/1 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token @@ -50,9 +52,14 @@ patches: name: atenet-router namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/13 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 8463bfd79..6b28ff2fd 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -33,6 +33,8 @@ patches: args: - --gcp-auth-for-image-pulls=false - --localhost-registry-replacement=kind-registry:5000 + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: http://opentelemetry-collector.otel-system.svc:4317 diff --git a/manifests/ate-install/jwt/kustomization.yaml b/manifests/ate-install/token-client/kustomization.yaml similarity index 60% rename from manifests/ate-install/jwt/kustomization.yaml rename to manifests/ate-install/token-client/kustomization.yaml index a6331c498..82e39fc7f 100644 --- a/manifests/ate-install/jwt/kustomization.yaml +++ b/manifests/ate-install/token-client/kustomization.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Overlay: in-cluster clients authenticate to ateapi with a projected +# ServiceAccount token instead of a client certificate. The server always +# accepts both. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -26,16 +29,10 @@ resources: patches: - path: patches.yaml - - target: - group: apps - version: v1 - kind: Deployment - name: ate-api-server - namespace: ate-system - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: --auth-mode=jwt + # --ateapi-use-token-auth switches the client to the projected + # ServiceAccount token below. The base manifests' --ateapi-client-cert is + # removed by index; the preceding `test` op pins the expected value so the + # build fails loudly if the base args are ever reordered. - target: group: apps version: v1 @@ -43,9 +40,14 @@ patches: name: ate-controller namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/1 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/1 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token @@ -56,9 +58,14 @@ patches: name: atenet-router namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/13 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token diff --git a/manifests/ate-install/jwt/patches.yaml b/manifests/ate-install/token-client/patches.yaml similarity index 100% rename from manifests/ate-install/jwt/patches.yaml rename to manifests/ate-install/token-client/patches.yaml diff --git a/manifests/ate-install/valkey.yaml b/manifests/ate-install/valkey.yaml index 23325d0fe..fb4b43a21 100644 --- a/manifests/ate-install/valkey.yaml +++ b/manifests/ate-install/valkey.yaml @@ -28,6 +28,8 @@ data: # Load certificates from projected volume tls-cert-file /run/servicedns.podcert.ate.dev/credential-bundle.pem tls-key-file /run/servicedns.podcert.ate.dev/credential-bundle.pem + tls-client-cert-file /run/podidentity.podcert.ate.dev/credential-bundle.pem + tls-client-key-file /run/podidentity.podcert.ate.dev/credential-bundle.pem tls-ca-cert-file /etc/valkey-ca/ca.crt tls-auth-clients yes @@ -117,6 +119,8 @@ spec: mountPath: /etc/valkey - name: servicedns mountPath: /run/servicedns.podcert.ate.dev + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev - name: valkey-ca-certs mountPath: /etc/valkey-ca readOnly: true @@ -133,6 +137,13 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem - name: valkey-ca-certs projected: sources: @@ -166,8 +177,11 @@ spec: - name: init image: valkey/valkey:9.1@sha256:4963247afc4cd33c7d3b2d2816b9f7f8eeebab148d29056c2ca4d7cbc966f2d9 volumeMounts: - - name: servicedns - mountPath: /run/servicedns.podcert.ate.dev + # The init job is a client of the cluster, not a Service-backed server, + # so it uses a podidentity client cert rather than a servicedns + # serving cert (which requires DNS SANs it would not have). + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev - name: valkey-ca-certs mountPath: /etc/valkey-ca readOnly: true @@ -191,19 +205,19 @@ spec: done echo "Checking if Valkey cluster is already initialized..." - until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem --key /run/servicedns.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc ping >/dev/null 2>&1; do + until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc ping >/dev/null 2>&1; do echo "Waiting for valkey-cluster-0 to respond to ping..." sleep 2 done - INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem --key /run/servicedns.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc cluster info 2>/dev/null | grep cluster_state || true) + INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc cluster info 2>/dev/null | grep cluster_state || true) if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then echo "Initializing Valkey cluster..." valkey-cli --tls \ --cacert /etc/valkey-ca/ca.crt \ - --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem \ - --key /run/servicedns.podcert.ate.dev/credential-bundle.pem \ + --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --key /run/podidentity.podcert.ate.dev/credential-bundle.pem \ --cluster create ${VALKEY_NODES} \ --cluster-replicas 1 \ --cluster-yes @@ -212,11 +226,11 @@ spec: echo "Cluster already initialized." fi volumes: - - name: servicedns + - name: podidentity projected: sources: - podCertificate: - signerName: servicedns.podcert.ate.dev/identity + signerName: podidentity.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem - name: valkey-ca-certs diff --git a/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/source.go b/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/source.go new file mode 100644 index 000000000..df34ed777 --- /dev/null +++ b/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/source.go @@ -0,0 +1,7 @@ +package x509svid + +// Source represents a source of X509-SVIDs. +type Source interface { + // GetX509SVID returns an X509-SVID from the source. + GetX509SVID() (*SVID, error) +} diff --git a/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/svid.go b/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/svid.go new file mode 100644 index 000000000..c2e234d77 --- /dev/null +++ b/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/svid.go @@ -0,0 +1,249 @@ +package x509svid + +import ( + "bytes" + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/rsa" + "crypto/x509" + "errors" + "fmt" + "os" + + "github.com/spiffe/go-spiffe/v2/internal/pemutil" + "github.com/spiffe/go-spiffe/v2/internal/x509util" + "github.com/spiffe/go-spiffe/v2/spiffeid" +) + +// SVID represents a SPIFFE X509-SVID. +type SVID struct { + // ID is the SPIFFE ID of the X509-SVID. + ID spiffeid.ID + + // Certificates are the X.509 certificates of the X509-SVID. The leaf + // certificate is the X509-SVID certificate. Any remaining certificates ( + // if any) chain the X509-SVID certificate back to a X.509 root for the + // trust domain. + Certificates []*x509.Certificate + + // PrivateKey is the private key for the X509-SVID. + PrivateKey crypto.Signer + + // Hint is an operator-specified string used to provide guidance on how this + // identity should be used by a workload when more than one SVID is returned. + Hint string +} + +// Load loads the X509-SVID from PEM encoded files on disk. certFile and +// keyFile may be the same file. +func Load(certFile, keyFile string) (*SVID, error) { + certBytes, err := os.ReadFile(certFile) + if err != nil { + return nil, wrapX509svidErr(fmt.Errorf("cannot read certificate file: %w", err)) + } + + keyBytes, err := os.ReadFile(keyFile) + if err != nil { + return nil, wrapX509svidErr(fmt.Errorf("cannot read key file: %w", err)) + } + + return Parse(certBytes, keyBytes) +} + +// Parse parses the X509-SVID from PEM blocks containing certificate and key +// bytes. The certificate must be one or more PEM blocks with ASN.1 DER. The +// key must be a PEM block with PKCS#8 ASN.1 DER. +func Parse(certBytes, keyBytes []byte) (*SVID, error) { + certs, err := pemutil.ParseCertificates(certBytes) + if err != nil { + return nil, wrapX509svidErr(fmt.Errorf("cannot parse PEM encoded certificate: %v", err)) + } + + privateKey, err := pemutil.ParsePrivateKey(keyBytes) + if err != nil { + return nil, wrapX509svidErr(fmt.Errorf("cannot parse PEM encoded private key: %v", err)) + } + + return newSVID(certs, privateKey) +} + +// ParseRaw parses the X509-SVID from certificate and key bytes. The +// certificate must be ASN.1 DER (concatenated with no intermediate +// padding if there are more than one certificate). The key must be a PKCS#8 +// ASN.1 DER. +func ParseRaw(certBytes, keyBytes []byte) (*SVID, error) { + certificates, err := x509.ParseCertificates(certBytes) + if err != nil { + return nil, wrapX509svidErr(fmt.Errorf("cannot parse DER encoded certificate: %v", err)) + } + + privateKey, err := x509.ParsePKCS8PrivateKey(keyBytes) + if err != nil { + return nil, wrapX509svidErr(fmt.Errorf("cannot parse DER encoded private key: %v", err)) + } + + return newSVID(certificates, privateKey) +} + +// Marshal marshals the X509-SVID and returns PEM encoded blocks for the SVID +// and private key. +func (s *SVID) Marshal() ([]byte, []byte, error) { + if len(s.Certificates) == 0 { + return nil, nil, wrapX509svidErr(errors.New("no certificates to marshal")) + } + certBytes := pemutil.EncodeCertificates(s.Certificates) + keyBytes, err := pemutil.EncodePKCS8PrivateKey(s.PrivateKey) + if err != nil { + return nil, nil, wrapX509svidErr(fmt.Errorf("cannot encode private key: %v", err)) + } + + return certBytes, keyBytes, nil +} + +// MarshalRaw marshals the X509-SVID and returns ASN.1 DER for the certificates +// (concatenated with no intermediate padding) and PKCS8 ASN1.DER for the +// private key. +func (s *SVID) MarshalRaw() ([]byte, []byte, error) { + key, err := x509.MarshalPKCS8PrivateKey(s.PrivateKey) + if err != nil { + return nil, nil, wrapX509svidErr(fmt.Errorf("cannot marshal private key: %v", err)) + } + + if len(s.Certificates) == 0 { + return nil, nil, wrapX509svidErr(errors.New("no certificates to marshal")) + } + + certs := x509util.ConcatRawCertsFromCerts(s.Certificates) + return certs, key, nil +} + +// GetX509SVID returns the X509-SVID. It implements the Source interface. +func (s *SVID) GetX509SVID() (*SVID, error) { + return s, nil +} + +func newSVID(certificates []*x509.Certificate, privateKey crypto.PrivateKey) (*SVID, error) { + spiffeID, err := validateCertificates(certificates) + if err != nil { + return nil, wrapX509svidErr(fmt.Errorf("certificate validation failed: %v", err)) + } + + signer, err := validatePrivateKey(privateKey, certificates[0]) + if err != nil { + return nil, wrapX509svidErr(fmt.Errorf("private key validation failed: %v", err)) + } + + return &SVID{ + Certificates: certificates, + PrivateKey: signer, + ID: *spiffeID, + }, nil +} + +// validate the slice of certificates constitutes a valid SVID chain according +// to the spiffe standard and returns the spiffe id of the leaf certificate +func validateCertificates(certificates []*x509.Certificate) (*spiffeid.ID, error) { + if len(certificates) == 0 { + return nil, errors.New("no certificates found") + } + + leafID, err := validateLeafCertificate(certificates[0]) + if err != nil { + return nil, err + } + + err = validateSigningCertificates(certificates[1:]) + if err != nil { + return nil, err + } + + return leafID, nil +} + +func validateLeafCertificate(leaf *x509.Certificate) (*spiffeid.ID, error) { + leafID, err := IDFromCert(leaf) + if err != nil { + return nil, fmt.Errorf("cannot get leaf certificate SPIFFE ID: %v", err) + } + if leaf.IsCA { + return nil, errors.New("leaf certificate must not have CA flag set to true") + } + + err = validateKeyUsage(leaf) + if err != nil { + return nil, err + } + + return &leafID, err +} + +func validateSigningCertificates(signingCerts []*x509.Certificate) error { + for _, cert := range signingCerts { + if !cert.IsCA { + return errors.New("signing certificate must have CA flag set to true") + } + if cert.KeyUsage&x509.KeyUsageCertSign == 0 { + return errors.New("signing certificate must have 'keyCertSign' set as key usage") + } + } + + return nil +} + +func validateKeyUsage(leaf *x509.Certificate) error { + switch { + case leaf.KeyUsage&x509.KeyUsageDigitalSignature == 0: + return errors.New("leaf certificate must have 'digitalSignature' set as key usage") + case leaf.KeyUsage&x509.KeyUsageCertSign > 0: + return errors.New("leaf certificate must not have 'keyCertSign' set as key usage") + case leaf.KeyUsage&x509.KeyUsageCRLSign > 0: + return errors.New("leaf certificate must not have 'cRLSign' set as key usage") + } + return nil +} + +func validatePrivateKey(privateKey crypto.PrivateKey, leaf *x509.Certificate) (crypto.Signer, error) { + if privateKey == nil { + return nil, errors.New("no private key found") + } + + matched, err := keyMatches(privateKey, leaf.PublicKey) + if err != nil { + return nil, err + } + if !matched { + return nil, errors.New("leaf certificate does not match private key") + } + + signer, ok := privateKey.(crypto.Signer) + if !ok { + return nil, fmt.Errorf("expected crypto.Signer; got %T", privateKey) + } + + return signer, nil +} + +func keyMatches(privateKey crypto.PrivateKey, publicKey crypto.PublicKey) (bool, error) { + switch privateKey := privateKey.(type) { + case *rsa.PrivateKey: + rsaPublicKey, ok := publicKey.(*rsa.PublicKey) + return ok && rsaPublicKeyEqual(&privateKey.PublicKey, rsaPublicKey), nil + case *ecdsa.PrivateKey: + ecdsaPublicKey, ok := publicKey.(*ecdsa.PublicKey) + return ok && ecdsaPublicKeyEqual(&privateKey.PublicKey, ecdsaPublicKey), nil + case ed25519.PrivateKey: + ed25519PublicKey, ok := publicKey.(ed25519.PublicKey) + return ok && bytes.Equal(privateKey.Public().(ed25519.PublicKey), ed25519PublicKey), nil + default: + return false, fmt.Errorf("unsupported private key type %T", privateKey) + } +} + +func rsaPublicKeyEqual(a, b *rsa.PublicKey) bool { + return a.E == b.E && a.N.Cmp(b.N) == 0 +} + +func ecdsaPublicKeyEqual(a, b *ecdsa.PublicKey) bool { + return a.Curve == b.Curve && a.X.Cmp(b.X) == 0 && a.Y.Cmp(b.Y) == 0 +} diff --git a/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/verify.go b/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/verify.go new file mode 100644 index 000000000..178450166 --- /dev/null +++ b/vendor/github.com/spiffe/go-spiffe/v2/svid/x509svid/verify.go @@ -0,0 +1,116 @@ +package x509svid + +import ( + "crypto/x509" + "errors" + "fmt" + "time" + + "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" + "github.com/spiffe/go-spiffe/v2/internal/x509util" + "github.com/spiffe/go-spiffe/v2/spiffeid" +) + +// VerifyOption is an option used when verifying X509-SVIDs. +type VerifyOption interface { + apply(config *verifyConfig) +} + +// WithTime sets the time used when verifying validity periods on the X509-SVID. +// If not used, the current time will be used. +func WithTime(now time.Time) VerifyOption { + return verifyOption(func(config *verifyConfig) { + config.now = now + }) +} + +// Verify verifies an X509-SVID chain using the X.509 bundle source. It +// returns the SPIFFE ID of the X509-SVID and one or more chains back to a root +// in the bundle. +func Verify(certs []*x509.Certificate, bundleSource x509bundle.Source, opts ...VerifyOption) (spiffeid.ID, [][]*x509.Certificate, error) { + config := &verifyConfig{} + for _, opt := range opts { + opt.apply(config) + } + + switch { + case len(certs) == 0: + return spiffeid.ID{}, nil, wrapX509svidErr(errors.New("empty certificates chain")) + case bundleSource == nil: + return spiffeid.ID{}, nil, wrapX509svidErr(errors.New("bundleSource is required")) + } + + leaf := certs[0] + id, err := IDFromCert(leaf) + if err != nil { + return spiffeid.ID{}, nil, wrapX509svidErr(fmt.Errorf("could not get leaf SPIFFE ID: %w", err)) + } + + switch { + case leaf.IsCA: + return id, nil, wrapX509svidErr(errors.New("leaf certificate with CA flag set to true")) + case leaf.KeyUsage&x509.KeyUsageCertSign > 0: + return id, nil, wrapX509svidErr(errors.New("leaf certificate with KeyCertSign key usage")) + case leaf.KeyUsage&x509.KeyUsageCRLSign > 0: + return id, nil, wrapX509svidErr(errors.New("leaf certificate with KeyCrlSign key usage")) + } + + bundle, err := bundleSource.GetX509BundleForTrustDomain(id.TrustDomain()) + if err != nil { + return id, nil, wrapX509svidErr(fmt.Errorf("could not get X509 bundle: %w", err)) + } + + verifiedChains, err := leaf.Verify(x509.VerifyOptions{ + Roots: x509util.NewCertPool(bundle.X509Authorities()), + Intermediates: x509util.NewCertPool(certs[1:]), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + CurrentTime: config.now, + }) + if err != nil { + return id, nil, wrapX509svidErr(fmt.Errorf("could not verify leaf certificate: %w", err)) + } + + return id, verifiedChains, nil +} + +// ParseAndVerify parses and verifies an X509-SVID chain using the X.509 +// bundle source. It returns the SPIFFE ID of the X509-SVID and one or more +// chains back to a root in the bundle. +func ParseAndVerify(rawCerts [][]byte, bundleSource x509bundle.Source, opts ...VerifyOption) (spiffeid.ID, [][]*x509.Certificate, error) { + var certs []*x509.Certificate + for _, rawCert := range rawCerts { + cert, err := x509.ParseCertificate(rawCert) + if err != nil { + return spiffeid.ID{}, nil, wrapX509svidErr(fmt.Errorf("unable to parse certificate: %w", err)) + } + certs = append(certs, cert) + } + return Verify(certs, bundleSource, opts...) +} + +// IDFromCert extracts the SPIFFE ID from the URI SAN of the provided +// certificate. It will return an an error if the certificate does not have +// exactly one URI SAN with a well-formed SPIFFE ID. +func IDFromCert(cert *x509.Certificate) (spiffeid.ID, error) { + switch { + case len(cert.URIs) == 0: + return spiffeid.ID{}, errors.New("certificate contains no URI SAN") + case len(cert.URIs) > 1: + return spiffeid.ID{}, errors.New("certificate contains more than one URI SAN") + } + return spiffeid.FromURI(cert.URIs[0]) +} + +type verifyConfig struct { + now time.Time +} + +type verifyOption func(config *verifyConfig) + +func (fn verifyOption) apply(config *verifyConfig) { + fn(config) +} + +func wrapX509svidErr(err error) error { + return fmt.Errorf("x509svid: %w", err) +} diff --git a/vendor/google.golang.org/grpc/health/client.go b/vendor/google.golang.org/grpc/health/client.go new file mode 100644 index 000000000..740745c45 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/client.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "fmt" + "io" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/status" +) + +var ( + backoffStrategy = backoff.DefaultExponential + backoffFunc = func(ctx context.Context, retries int) bool { + d := backoffStrategy.Backoff(retries) + timer := time.NewTimer(d) + select { + case <-timer.C: + return true + case <-ctx.Done(): + timer.Stop() + return false + } + } +) + +func init() { + internal.HealthCheckFunc = clientHealthCheck +} + +const healthCheckMethod = "/grpc.health.v1.Health/Watch" + +// This function implements the protocol defined at: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md +func clientHealthCheck(ctx context.Context, newStream func(string) (any, error), setConnectivityState func(connectivity.State, error), service string) error { + tryCnt := 0 + +retryConnection: + for { + // Backs off if the connection has failed in some way without receiving a message in the previous retry. + if tryCnt > 0 && !backoffFunc(ctx, tryCnt-1) { + return nil + } + tryCnt++ + + if ctx.Err() != nil { + return nil + } + setConnectivityState(connectivity.Connecting, nil) + rawS, err := newStream(healthCheckMethod) + if err != nil { + continue retryConnection + } + + s, ok := rawS.(grpc.ClientStream) + // Ideally, this should never happen. But if it happens, the server is marked as healthy for LBing purposes. + if !ok { + setConnectivityState(connectivity.Ready, nil) + return fmt.Errorf("newStream returned %v (type %T); want grpc.ClientStream", rawS, rawS) + } + + if err = s.SendMsg(&healthpb.HealthCheckRequest{Service: service}); err != nil && err != io.EOF { + // Stream should have been closed, so we can safely continue to create a new stream. + continue retryConnection + } + s.CloseSend() + + resp := new(healthpb.HealthCheckResponse) + for { + err = s.RecvMsg(resp) + + // Reports healthy for the LBing purposes if health check is not implemented in the server. + if status.Code(err) == codes.Unimplemented { + setConnectivityState(connectivity.Ready, nil) + return err + } + + // Reports unhealthy if server's Watch method gives an error other than UNIMPLEMENTED. + if err != nil { + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but received health check RPC error: %v", err)) + continue retryConnection + } + + // As a message has been received, removes the need for backoff for the next retry by resetting the try count. + tryCnt = 0 + if resp.Status == healthpb.HealthCheckResponse_SERVING { + setConnectivityState(connectivity.Ready, nil) + } else { + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but health check failed. status=%s", resp.Status)) + } + } + } +} diff --git a/vendor/google.golang.org/grpc/health/logging.go b/vendor/google.golang.org/grpc/health/logging.go new file mode 100644 index 000000000..83c6acf55 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/logging.go @@ -0,0 +1,23 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import "google.golang.org/grpc/grpclog" + +var logger = grpclog.Component("health_service") diff --git a/vendor/google.golang.org/grpc/health/producer.go b/vendor/google.golang.org/grpc/health/producer.go new file mode 100644 index 000000000..f938e5790 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/producer.go @@ -0,0 +1,106 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/status" +) + +func init() { + producerBuilderSingleton = &producerBuilder{} + internal.RegisterClientHealthCheckListener = registerClientSideHealthCheckListener +} + +type producerBuilder struct{} + +var producerBuilderSingleton *producerBuilder + +// Build constructs and returns a producer and its cleanup function. +func (*producerBuilder) Build(cci any) (balancer.Producer, func()) { + p := &healthServiceProducer{ + cc: cci.(grpc.ClientConnInterface), + cancel: func() {}, + } + return p, func() { + p.mu.Lock() + defer p.mu.Unlock() + p.cancel() + } +} + +type healthServiceProducer struct { + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + cc grpc.ClientConnInterface + + mu sync.Mutex + cancel func() +} + +// registerClientSideHealthCheckListener accepts a listener to provide server +// health state via the health service. +func registerClientSideHealthCheckListener(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) func() { + pr, closeFn := sc.GetOrBuildProducer(producerBuilderSingleton) + p := pr.(*healthServiceProducer) + p.mu.Lock() + defer p.mu.Unlock() + p.cancel() + if listener == nil { + return closeFn + } + + ctx, cancel := context.WithCancel(ctx) + p.cancel = cancel + + go p.startHealthCheck(ctx, sc, serviceName, listener) + return closeFn +} + +func (p *healthServiceProducer) startHealthCheck(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) { + newStream := func(method string) (any, error) { + return p.cc.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, method) + } + + setConnectivityState := func(state connectivity.State, err error) { + listener(balancer.SubConnState{ + ConnectivityState: state, + ConnectionError: err, + }) + } + + // Call the function through the internal variable as tests use it for + // mocking. + err := internal.HealthCheckFunc(ctx, newStream, setConnectivityState, serviceName) + if err == nil { + return + } + if status.Code(err) == codes.Unimplemented { + logger.Errorf("Subchannel health check is unimplemented at server side, thus health check is disabled for SubConn %p", sc) + } else { + logger.Errorf("Health checking failed for SubConn %p: %v", sc, err) + } +} diff --git a/vendor/google.golang.org/grpc/health/server.go b/vendor/google.golang.org/grpc/health/server.go new file mode 100644 index 000000000..d8eebe99d --- /dev/null +++ b/vendor/google.golang.org/grpc/health/server.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package health provides a service that exposes server's health and it must be +// imported to enable support for client-side health checks. +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc/codes" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +const ( + // maxAllowedServices defines the maximum number of resources a List + // operation can return. An error is returned if the number of services + // exceeds this limit. + maxAllowedServices = 100 +) + +// Server implements `service Health`. +type Server struct { + healthgrpc.UnimplementedHealthServer + mu sync.RWMutex + // If shutdown is true, it's expected all serving status is NOT_SERVING, and + // will stay in NOT_SERVING. + shutdown bool + // statusMap stores the serving status of the services this Server monitors. + statusMap map[string]healthpb.HealthCheckResponse_ServingStatus + updates map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus +} + +// NewServer returns a new Server. +func NewServer() *Server { + return &Server{ + statusMap: map[string]healthpb.HealthCheckResponse_ServingStatus{"": healthpb.HealthCheckResponse_SERVING}, + updates: make(map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus), + } +} + +// Check implements `service Health`. +func (s *Server) Check(_ context.Context, in *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if servingStatus, ok := s.statusMap[in.Service]; ok { + return &healthpb.HealthCheckResponse{ + Status: servingStatus, + }, nil + } + return nil, status.Error(codes.NotFound, "unknown service") +} + +// List implements `service Health`. +func (s *Server) List(_ context.Context, _ *healthpb.HealthListRequest) (*healthpb.HealthListResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if len(s.statusMap) > maxAllowedServices { + return nil, status.Errorf(codes.ResourceExhausted, "server health list exceeds maximum capacity: %d", maxAllowedServices) + } + + statusMap := make(map[string]*healthpb.HealthCheckResponse, len(s.statusMap)) + for k, v := range s.statusMap { + statusMap[k] = &healthpb.HealthCheckResponse{Status: v} + } + + return &healthpb.HealthListResponse{Statuses: statusMap}, nil +} + +// Watch implements `service Health`. +func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + service := in.Service + // update channel is used for getting service status updates. + update := make(chan healthpb.HealthCheckResponse_ServingStatus, 1) + s.mu.Lock() + // Puts the initial status to the channel. + if servingStatus, ok := s.statusMap[service]; ok { + update <- servingStatus + } else { + update <- healthpb.HealthCheckResponse_SERVICE_UNKNOWN + } + + // Registers the update channel to the correct place in the updates map. + if _, ok := s.updates[service]; !ok { + s.updates[service] = make(map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus) + } + s.updates[service][stream] = update + defer func() { + s.mu.Lock() + delete(s.updates[service], stream) + s.mu.Unlock() + }() + s.mu.Unlock() + + var lastSentStatus healthpb.HealthCheckResponse_ServingStatus = -1 + for { + select { + // Status updated. Sends the up-to-date status to the client. + case servingStatus := <-update: + if lastSentStatus == servingStatus { + continue + } + lastSentStatus = servingStatus + err := stream.Send(&healthpb.HealthCheckResponse{Status: servingStatus}) + if err != nil { + return status.Error(codes.Canceled, "Stream has ended.") + } + // Context done. Removes the update channel from the updates map. + case <-stream.Context().Done(): + return status.Error(codes.Canceled, "Stream has ended.") + } + } +} + +// SetServingStatus is called when need to reset the serving status of a service +// or insert a new service entry into the statusMap. +func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.mu.Lock() + defer s.mu.Unlock() + if s.shutdown { + logger.Infof("health: status changing for %s to %v is ignored because health service is shutdown", service, servingStatus) + return + } + + s.setServingStatusLocked(service, servingStatus) +} + +func (s *Server) setServingStatusLocked(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.statusMap[service] = servingStatus + for _, update := range s.updates[service] { + // Clears previous updates, that are not sent to the client, from the channel. + // This can happen if the client is not reading and the server gets flow control limited. + select { + case <-update: + default: + } + // Puts the most recent update to the channel. + update <- servingStatus + } +} + +// Shutdown sets all serving status to NOT_SERVING, and configures the server to +// ignore all future status changes. +// +// This changes serving status for all services. To set status for a particular +// services, call SetServingStatus(). +func (s *Server) Shutdown() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = true + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_NOT_SERVING) + } +} + +// Resume sets all serving status to SERVING, and configures the server to +// accept all future status changes. +// +// This changes serving status for all services. To set status for a particular +// services, call SetServingStatus(). +func (s *Server) Resume() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = false + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_SERVING) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d92c6e288..1eaaa34ac 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -712,6 +712,7 @@ github.com/spiffe/go-spiffe/v2/internal/jwtutil github.com/spiffe/go-spiffe/v2/internal/pemutil github.com/spiffe/go-spiffe/v2/internal/x509util github.com/spiffe/go-spiffe/v2/spiffeid +github.com/spiffe/go-spiffe/v2/svid/x509svid # github.com/tklauser/go-sysconf v0.4.0 ## explicit; go 1.25.0 github.com/tklauser/go-sysconf @@ -1024,6 +1025,7 @@ google.golang.org/grpc/experimental/opentelemetry google.golang.org/grpc/experimental/stats google.golang.org/grpc/grpclog google.golang.org/grpc/grpclog/internal +google.golang.org/grpc/health google.golang.org/grpc/health/grpc_health_v1 google.golang.org/grpc/internal google.golang.org/grpc/internal/admin