mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-10-02 03:24:42 +08:00
egress: add an Envoy egress gateway.
The gateway terminates actor's CONNECT request. It requires downstream mTLS, so only a worker's atunnel can reach it. The gateway consists of an Envoy and an `atenet router --standalone` ext_proc sidecar.
This commit is contained in:
@@ -59,6 +59,7 @@ func NewRouterCmd() *cobra.Command {
|
||||
cmd.Flags().StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80).")
|
||||
cmd.Flags().StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.")
|
||||
cmd.Flags().StringVar(&cfg.UpstreamSpiffePrefix, "upstream-spiffe-prefix", "spiffe://cluster.local/", "SPIFFE URI SAN prefix (trust domain) the actor's atunnel server cert must match. Empty falls back to default SAN check against the dialed pod IP (which SPIFFE-only certs never match).")
|
||||
cmd.Flags().StringVar(&cfg.ActorIdentityCAFile, "actor-identity-ca-file", "", "PEM trust bundle for the actor-identity CA, used to verify the actor client certificates presented on egress CONNECTs. Required by the egress gateway's ext_proc sidecar; empty (the default) leaves egress authentication unconfigured and every egress CONNECT is denied.")
|
||||
// Envoy learns the collector over xDS rather than from its own environment,
|
||||
// so the router has to carry the address for it. Defaulting to
|
||||
// OTEL_EXPORTER_OTLP_ENDPOINT — the same variable the router's own exporter
|
||||
|
||||
@@ -67,8 +67,17 @@ type routerConfig struct {
|
||||
// UpstreamSpiffePrefix validates the actor's atunnel server cert by its
|
||||
// SPIFFE URI SAN prefix (trust domain) instead of the dialed pod IP.
|
||||
UpstreamSpiffePrefix string
|
||||
LogLevel string
|
||||
MetricsAddr string
|
||||
|
||||
// ActorIdentityCAFile is the PEM trust bundle for the actor-identity CA,
|
||||
// used by the egress gateway's ext_proc sidecar to verify the actor client
|
||||
// certificates atunnel presents on egress CONNECTs. Only that deployment
|
||||
// sets it; empty leaves egress authentication unconfigured, which makes
|
||||
// every egress CONNECT fail closed and is correct for an ingress-only
|
||||
// router (it never sees the egress listener).
|
||||
ActorIdentityCAFile string
|
||||
|
||||
LogLevel string
|
||||
MetricsAddr string
|
||||
// OtlpCollectorAddress is the OTLP gRPC collector that Envoy reports
|
||||
// tracing spans to, as host:port or an http:// URL. It defaults to
|
||||
// OTEL_EXPORTER_OTLP_ENDPOINT — Envoy gets its whole configuration over
|
||||
|
||||
@@ -16,6 +16,7 @@ package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -47,17 +48,23 @@ type ExtProcServer struct {
|
||||
routeDuration metric.Float64Histogram
|
||||
parking *parkingLot
|
||||
routeViaAuthority bool
|
||||
// actorIdentityRoots are the actor-identity CA certificates the egress
|
||||
// handler verifies actor client certificates against. Only the ext_proc
|
||||
// sidecar of the egress gateway sets it; nil makes every egress CONNECT
|
||||
// fail closed and is the correct state for an ingress-only router.
|
||||
actorIdentityRoots *x509.CertPool
|
||||
}
|
||||
|
||||
func NewExtProcServer(port int, apiClient ateapipb.ControlClient, routeDuration metric.Float64Histogram, parkCfg ParkedRequestConfig, parkMetrics *parkingMetrics, routeViaAuthority bool) *ExtProcServer {
|
||||
func NewExtProcServer(port int, apiClient ateapipb.ControlClient, routeDuration metric.Float64Histogram, parkCfg ParkedRequestConfig, parkMetrics *parkingMetrics, routeViaAuthority bool, actorIdentityRoots *x509.CertPool) *ExtProcServer {
|
||||
return &ExtProcServer{
|
||||
port: port,
|
||||
apiClient: apiClient,
|
||||
recorder: NewQueryRecorder(100),
|
||||
resumer: NewActorResumer(apiClient, withParking(parkCfg)),
|
||||
routeDuration: routeDuration,
|
||||
parking: newParkingLot(parkCfg, parkMetrics),
|
||||
routeViaAuthority: routeViaAuthority,
|
||||
port: port,
|
||||
apiClient: apiClient,
|
||||
recorder: NewQueryRecorder(100),
|
||||
resumer: NewActorResumer(apiClient, withParking(parkCfg)),
|
||||
routeDuration: routeDuration,
|
||||
parking: newParkingLot(parkCfg, parkMetrics),
|
||||
routeViaAuthority: routeViaAuthority,
|
||||
actorIdentityRoots: actorIdentityRoots,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,7 +95,24 @@ func (s *ExtProcServer) Process(stream extprocv3.ExternalProcessor_ProcessServer
|
||||
switch reqType := req.Request.(type) {
|
||||
case *extprocv3.ProcessingRequest_RequestHeaders:
|
||||
start := time.Now()
|
||||
hResponse, rqm, target, tmplNs, tmplName, resumeOutcome, err := s.handleRequestHeaders(stream.Context(), reqType.RequestHeaders)
|
||||
// One ext_proc server handles both directions: actor egress
|
||||
// CONNECT requests and ingress requests. Which one is decided by
|
||||
// the accepting listener, not by anything in the request itself
|
||||
// (see isEgressRequest).
|
||||
//
|
||||
// SEE(lior): main grew a ResumeOutcome return on
|
||||
// handleRequestHeaders while this branch was out. Rather than
|
||||
// splitting the dispatch into two separately-typed call sites, the
|
||||
// egress handler was widened to the same signature and returns
|
||||
// ResumeOutcomeNone — egress requires an already-RUNNING actor and
|
||||
// never resumes one, so "none" is accurate, and it keeps the
|
||||
// route-duration metric's resume label a closed set (no new empty
|
||||
// value) across both directions.
|
||||
handle := s.handleRequestHeaders
|
||||
if isEgressRequest(req) {
|
||||
handle = s.handleEgressRequestHeaders
|
||||
}
|
||||
hResponse, rqm, target, tmplNs, tmplName, resumeOutcome, err := handle(stream.Context(), reqType.RequestHeaders)
|
||||
elapsed := time.Since(start)
|
||||
outcomeStr := classifyOutcome(err)
|
||||
resumeStr := string(resumeOutcome)
|
||||
|
||||
@@ -0,0 +1,451 @@
|
||||
// Copyright 2026 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
|
||||
envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/agent-substrate/substrate/internal/resources"
|
||||
"github.com/agent-substrate/substrate/internal/substratex509"
|
||||
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
|
||||
)
|
||||
|
||||
const (
|
||||
// EgressListenerName is the Envoy listener that terminates actor egress
|
||||
// CONNECTs. It must stay in sync with the listener name in
|
||||
// manifests/ate-install/ateway-egress.yaml.
|
||||
EgressListenerName = "egress"
|
||||
// ListenerNameAttribute is the CEL attribute carrying the name of the
|
||||
// listener that accepted the request. The egress Envoy asks for it via
|
||||
// request_attributes on its ext_proc filter.
|
||||
ListenerNameAttribute = "xds.listener_name"
|
||||
|
||||
// forwardedClientCertHeader is the header Envoy fills in with details of
|
||||
// the mTLS peer, including the PEM chain it validated. The egress listener
|
||||
// sets forward_client_cert_details: SANITIZE_SET, so whatever a client
|
||||
// sends under this name is discarded and replaced by Envoy's own value.
|
||||
//
|
||||
// This is the only channel that can carry a whole certificate to ext_proc:
|
||||
// the CEL request attributes Envoy exposes (subject, SANs, SHA-256 digest)
|
||||
// cannot express the custom ActorIdentity X.509 extension this gateway
|
||||
// authorizes on.
|
||||
forwardedClientCertHeader = "x-forwarded-client-cert"
|
||||
// xfccChainKey is the x-forwarded-client-cert key holding the URL-encoded
|
||||
// PEM of the full presented chain, leaf first.
|
||||
xfccChainKey = "chain"
|
||||
)
|
||||
|
||||
// isEgressRequest reports whether an ext_proc RequestHeaders callback arrived on
|
||||
// the egress gateway's listener rather than on an ingress listener. This lets
|
||||
// one ext_proc server handle both directions off the same stream.
|
||||
//
|
||||
// Dispatch is by listener, not by :method, because the two handlers apply
|
||||
// opposite trust models: on egress the actor identity comes from a client
|
||||
// certificate Envoy validated, while on ingress every request header is
|
||||
// unauthenticated client input. Keying on :method would let any external client
|
||||
// sending CONNECT select the egress handler and use its denial messages as an
|
||||
// actor-existence and status oracle. Envoy asserts the listener name; the
|
||||
// request cannot influence it.
|
||||
//
|
||||
// An unrecognized or absent attribute means ingress, the fail-safe direction: an
|
||||
// egress request misrouted to the ingress handler fails to parse as an actor DNS
|
||||
// name and 404s, whereas the reverse leaks control-plane state.
|
||||
func isEgressRequest(req *extprocv3.ProcessingRequest) bool {
|
||||
return listenerName(req) == EgressListenerName
|
||||
}
|
||||
|
||||
// listenerName returns the xds.listener_name attribute Envoy attached to the
|
||||
// request, or "" when the listener did not request the attribute. The
|
||||
// attributes map is keyed by the ext_proc filter's name within the HCM chain,
|
||||
// which we do not want to hardcode here, so scan every entry.
|
||||
func listenerName(req *extprocv3.ProcessingRequest) string {
|
||||
for _, attrs := range req.GetAttributes() {
|
||||
if v, ok := attrs.GetFields()[ListenerNameAttribute]; ok {
|
||||
return v.GetStringValue()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// handleEgressRequestHeaders authenticates the actor behind an egress CONNECT
|
||||
// before the gateway tunnels it out, using the actor certificate atunnel
|
||||
// presented in the mTLS handshake. Nothing the actor can write — no CONNECT
|
||||
// header, no request metadata — contributes to the identity; the only inputs
|
||||
// are the certificate the actor-identity CA signed and the control plane's own
|
||||
// view of that actor.
|
||||
//
|
||||
// Authorization by destination and credential/token injection are deliberately
|
||||
// a TODO once we have SessionIdentity RPC service figured out.
|
||||
//
|
||||
// The signature mirrors handleRequestHeaders so Process can dispatch to either
|
||||
// with a single branch. The (target, tmplNs, tmplName) results are unused for
|
||||
// egress and returned empty.
|
||||
//
|
||||
// SEE(lior): the trailing ResumeOutcome exists only to match
|
||||
// handleRequestHeaders after main added it. Egress never resumes an actor — it
|
||||
// requires one already RUNNING — so every path returns ResumeOutcomeNone.
|
||||
func (s *ExtProcServer) handleEgressRequestHeaders(
|
||||
ctx context.Context,
|
||||
reqHeaders *extprocv3.HttpHeaders,
|
||||
) (*extprocv3.HeadersResponse, *requestMetadata, string, string, string, ResumeOutcome, error) {
|
||||
metadata := newRequestMetadata(reqHeaders.Headers.GetHeaders())
|
||||
|
||||
// Dispatch is by listener, so reaching here means the egress listener
|
||||
// accepted the request. That listener only routes CONNECT (its sole route
|
||||
// is a connect_matcher), so anything else is a config drift rather than a
|
||||
// client the gateway should tunnel for.
|
||||
if !strings.EqualFold(metadata.method, "CONNECT") {
|
||||
return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_MethodNotAllowed,
|
||||
"egress denied: expected CONNECT, got %q", metadata.method)
|
||||
}
|
||||
|
||||
// No roots means the gateway cannot authenticate anyone. Fail closed, and
|
||||
// as 503 rather than 403: this is our misconfiguration, not the actor's.
|
||||
if s.actorIdentityRoots == nil {
|
||||
return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_ServiceUnavailable,
|
||||
"egress unavailable: no actor-identity CA configured")
|
||||
}
|
||||
|
||||
identity, err := s.authenticateActorCertificate(metadata)
|
||||
if err != nil {
|
||||
// The message stays generic on purpose: an actor that fails
|
||||
// authentication has not proven it is anyone, so it gets no detail
|
||||
// about why. The specific reason is logged below instead.
|
||||
slog.WarnContext(ctx, "egress denied: actor certificate rejected", slog.Any("err", err))
|
||||
return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden,
|
||||
"egress denied: invalid actor certificate")
|
||||
}
|
||||
|
||||
atespace := identity.Atespace
|
||||
actorName := identity.ActorName
|
||||
actorUID := identity.ActorUid
|
||||
// For a CONNECT the :authority is the actor's original destination (IP:port).
|
||||
destination := metadata.host
|
||||
|
||||
// The CA only ever mints these from control-plane state, so a name that is
|
||||
// not a legal resource name means the CA or its inputs are compromised.
|
||||
if !resources.IsValidResourceName(atespace) || !resources.IsValidResourceName(actorName) {
|
||||
return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden,
|
||||
"egress denied: invalid actor identity %q/%q", atespace, actorName)
|
||||
}
|
||||
|
||||
// Confirm the certified actor still exists. The name is only a lookup key
|
||||
// here; the UID below is what actually authorizes.
|
||||
actor, err := s.apiClient.GetActor(ctx, &ateapipb.GetActorRequest{
|
||||
Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: actorName},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, metadata, "", "", "", ResumeOutcomeNone, mapEgressIdentityError(atespace, actorName, err)
|
||||
}
|
||||
|
||||
// Authorize on the UID, not the name. Names are reused: delete an actor and
|
||||
// recreate it under the same atespace/name and it is a different actor with
|
||||
// a different UID. A certificate outliving its actor must not carry over to
|
||||
// the successor, so the UID the CA certified has to match the UID the
|
||||
// control plane holds right now.
|
||||
if uid := actor.GetMetadata().GetUid(); uid != actorUID {
|
||||
slog.WarnContext(ctx, "egress denied: actor UID mismatch",
|
||||
slog.String("atespace", atespace),
|
||||
slog.String("actor", actorName),
|
||||
slog.String("certificateActorUid", actorUID),
|
||||
slog.String("currentActorUid", uid))
|
||||
return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden,
|
||||
"egress denied: actor %q/%q is not the actor this certificate was issued to", atespace, actorName)
|
||||
}
|
||||
|
||||
// The actor performing egress must actually be running.
|
||||
if actor.GetStatus() != ateapipb.Actor_STATUS_RUNNING {
|
||||
return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden,
|
||||
"egress denied: actor %q/%q is %s, not running", atespace, actorName, actor.GetStatus())
|
||||
}
|
||||
|
||||
slog.InfoContext(ctx, "egress identity authenticated",
|
||||
slog.String("atespace", atespace),
|
||||
slog.String("actor", actorName),
|
||||
slog.String("actorUid", actorUID),
|
||||
slog.String("destination", destination),
|
||||
slog.String("status", actor.GetStatus().String()))
|
||||
|
||||
// Identity is authenticated; let the CONNECT proceed unchanged. Milestone 2
|
||||
// would additionally authorize `destination` and inject upstream credentials
|
||||
// here by returning a HeaderMutation.
|
||||
return &extprocv3.HeadersResponse{
|
||||
Response: &extprocv3.CommonResponse{},
|
||||
}, metadata, "", "", "", ResumeOutcomeNone, nil
|
||||
}
|
||||
|
||||
// authenticateActorCertificate turns the mTLS peer certificate Envoy recorded
|
||||
// on the request into a verified ActorIdentity, or an error describing why it
|
||||
// cannot be trusted.
|
||||
func (s *ExtProcServer) authenticateActorCertificate(metadata *requestMetadata) (*substratex509.ActorIdentity, error) {
|
||||
header := metadata.headers[forwardedClientCertHeader]
|
||||
if header == "" {
|
||||
return nil, fmt.Errorf("request carries no %s header", forwardedClientCertHeader)
|
||||
}
|
||||
chain, err := parseXFCCChain(header)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.verifyActorCertificate(chain)
|
||||
}
|
||||
|
||||
// verifyActorCertificate checks that chain[0] is a live, non-CA, client-auth
|
||||
// actor certificate issued by the actor-identity CA, and returns the single
|
||||
// ActorIdentity it carries.
|
||||
//
|
||||
// ###########################################################################
|
||||
// SEE(lior): READ THIS IF YOU ARE WONDERING WHY WE VERIFY THE CHAIN TWICE.
|
||||
//
|
||||
// The egress listener already does full mTLS: require_client_certificate with
|
||||
// the actor-identity CA as its trusted_ca, so Envoy refuses the handshake for
|
||||
// anything this function would also reject on chain, expiry, or signature. The
|
||||
// re-verification below is therefore redundant *today*, and it is here on
|
||||
// purpose:
|
||||
//
|
||||
// - Envoy validates the chain but cannot look at the ActorIdentity
|
||||
// extension. This function has to parse the certificate regardless, and
|
||||
// parsing an unverified certificate and then trusting its contents is the
|
||||
// failure mode that keeps producing CVEs. Verifying what we parse keeps the
|
||||
// trust decision in one place instead of split across a YAML file and a Go
|
||||
// file.
|
||||
// - It makes the handler safe under Envoy config drift. Someone relaxing
|
||||
// require_client_certificate, widening trusted_ca, or putting another proxy
|
||||
// in front should not silently turn this into an unauthenticated endpoint.
|
||||
// - It costs one signature verification per CONNECT, not per request: the
|
||||
// tunnel is established once and then carries raw TCP.
|
||||
//
|
||||
// If you decide the Envoy-side check is authoritative and this is dead weight,
|
||||
// this function is the thing to delete — but keep the ActorIdentity extraction
|
||||
// and the IsCA/EKU/purpose checks below it, because Envoy does none of those.
|
||||
// ###########################################################################
|
||||
func (s *ExtProcServer) verifyActorCertificate(chain []*x509.Certificate) (*substratex509.ActorIdentity, error) {
|
||||
leaf := chain[0]
|
||||
intermediates := x509.NewCertPool()
|
||||
for _, cert := range chain[1:] {
|
||||
intermediates.AddCert(cert)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if now.Before(leaf.NotBefore) || !now.Before(leaf.NotAfter) {
|
||||
return nil, fmt.Errorf("actor certificate is outside its validity period (%s..%s)",
|
||||
leaf.NotBefore.Format(time.RFC3339), leaf.NotAfter.Format(time.RFC3339))
|
||||
}
|
||||
// An actor certificate is an end-entity credential. Refusing IsCA here stops
|
||||
// a leaked or mis-issued CA certificate from being replayed as a leaf: chain
|
||||
// verification alone would happily accept one.
|
||||
if leaf.IsCA {
|
||||
return nil, fmt.Errorf("actor certificate is a CA certificate")
|
||||
}
|
||||
// Require ClientAuth explicitly rather than relying on VerifyOptions.KeyUsages:
|
||||
// an empty ExtKeyUsage means "any usage" to crypto/x509 and would pass. This
|
||||
// mirrors the check atunnel makes on the certificate when it mints it.
|
||||
if !slices.Contains(leaf.ExtKeyUsage, x509.ExtKeyUsageClientAuth) {
|
||||
return nil, fmt.Errorf("actor certificate cannot authenticate a TLS client")
|
||||
}
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{
|
||||
Roots: s.actorIdentityRoots,
|
||||
Intermediates: intermediates,
|
||||
CurrentTime: now,
|
||||
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("actor certificate is not signed by the actor-identity CA: %w", err)
|
||||
}
|
||||
|
||||
// ActorIdentityFromCertificate returns (nil, nil) when the extension is
|
||||
// absent, an error when there is more than one or when its contents are
|
||||
// malformed, empty, or carry a purpose other than atunnel.
|
||||
identity, err := substratex509.ActorIdentityFromCertificate(leaf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("actor certificate has no single valid ActorIdentity extension: %w", err)
|
||||
}
|
||||
if identity == nil {
|
||||
return nil, fmt.Errorf("actor certificate has no ActorIdentity extension")
|
||||
}
|
||||
// Restate what ActorIdentityFromCertificate enforces. The gateway is the
|
||||
// component that gets hurt if that helper ever loosens, and "reject anything
|
||||
// not scoped to atunnel" is the property this endpoint depends on: a
|
||||
// certificate minted for some future purpose must not open a tunnel.
|
||||
if identity.Atespace == "" || identity.ActorName == "" || identity.ActorUid == "" {
|
||||
return nil, fmt.Errorf("actor certificate identity is incomplete")
|
||||
}
|
||||
if identity.Purpose != substratex509.ActorIdentityPurposeAtunnel {
|
||||
return nil, fmt.Errorf("actor certificate purpose %q is not %q",
|
||||
identity.Purpose, substratex509.ActorIdentityPurposeAtunnel)
|
||||
}
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
// parseXFCCChain extracts the presented certificate chain, leaf first, from an
|
||||
// x-forwarded-client-cert header value.
|
||||
func parseXFCCChain(header string) ([]*x509.Certificate, error) {
|
||||
// One element per proxy hop. SANITIZE_SET makes Envoy the only writer, so
|
||||
// anything but exactly one element means either an unexpected proxy in front
|
||||
// of the gateway or a listener that lost SANITIZE_SET — in both cases we no
|
||||
// longer know which element describes our actual peer, so refuse to guess.
|
||||
elements := splitXFCCUnquoted(header, ',')
|
||||
if len(elements) != 1 {
|
||||
return nil, fmt.Errorf("expected exactly one %s element, got %d", forwardedClientCertHeader, len(elements))
|
||||
}
|
||||
encoded, ok := xfccValue(elements[0], xfccChainKey)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s carries no %q value", forwardedClientCertHeader, xfccChainKey)
|
||||
}
|
||||
// Envoy percent-encodes the PEM. PathUnescape, not QueryUnescape: base64
|
||||
// bodies contain '+', and query unescaping would decode it to a space and
|
||||
// silently corrupt the DER.
|
||||
chainPEM, err := url.PathUnescape(encoded)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decoding the client certificate chain: %w", err)
|
||||
}
|
||||
|
||||
var chain []*x509.Certificate
|
||||
rest := []byte(chainPEM)
|
||||
for {
|
||||
var block *pem.Block
|
||||
block, rest = pem.Decode(rest)
|
||||
if block == nil {
|
||||
break
|
||||
}
|
||||
if block.Type != "CERTIFICATE" {
|
||||
continue
|
||||
}
|
||||
cert, err := x509.ParseCertificate(block.Bytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing the client certificate chain: %w", err)
|
||||
}
|
||||
chain = append(chain, cert)
|
||||
}
|
||||
if len(chain) == 0 {
|
||||
return nil, fmt.Errorf("%s carries no certificate", forwardedClientCertHeader)
|
||||
}
|
||||
return chain, nil
|
||||
}
|
||||
|
||||
// xfccValue returns the value of key in one x-forwarded-client-cert element.
|
||||
// Keys are matched case-insensitively; Envoy emits "Chain", but the header is
|
||||
// consumed by enough different proxies that assuming its casing is not worth
|
||||
// the failure mode.
|
||||
func xfccValue(element, key string) (string, bool) {
|
||||
for _, pair := range splitXFCCUnquoted(element, ';') {
|
||||
k, v, found := strings.Cut(pair, "=")
|
||||
if !found || !strings.EqualFold(strings.TrimSpace(k), key) {
|
||||
continue
|
||||
}
|
||||
return unquoteXFCC(strings.TrimSpace(v)), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// splitXFCCUnquoted splits on sep, ignoring separators inside a quoted value.
|
||||
// x-forwarded-client-cert quotes any value containing its own delimiters, which
|
||||
// the PEM ones always do.
|
||||
func splitXFCCUnquoted(s string, sep rune) []string {
|
||||
var parts []string
|
||||
var current strings.Builder
|
||||
quoted := false
|
||||
escaped := false
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case escaped:
|
||||
current.WriteRune(r)
|
||||
escaped = false
|
||||
case quoted && r == '\\':
|
||||
current.WriteRune(r)
|
||||
escaped = true
|
||||
case r == '"':
|
||||
quoted = !quoted
|
||||
current.WriteRune(r)
|
||||
case r == sep && !quoted:
|
||||
parts = append(parts, current.String())
|
||||
current.Reset()
|
||||
default:
|
||||
current.WriteRune(r)
|
||||
}
|
||||
}
|
||||
parts = append(parts, current.String())
|
||||
|
||||
trimmed := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
trimmed = append(trimmed, part)
|
||||
}
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
// unquoteXFCC strips the surrounding quotes from an x-forwarded-client-cert
|
||||
// value and undoes the backslash escaping inside them.
|
||||
func unquoteXFCC(value string) string {
|
||||
if len(value) < 2 || !strings.HasPrefix(value, `"`) || !strings.HasSuffix(value, `"`) {
|
||||
return value
|
||||
}
|
||||
inner := value[1 : len(value)-1]
|
||||
var out strings.Builder
|
||||
escaped := false
|
||||
for _, r := range inner {
|
||||
if escaped {
|
||||
out.WriteRune(r)
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if r == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
out.WriteRune(r)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// mapEgressIdentityError converts a GetActor failure into a client-facing
|
||||
// ext_proc denial. An unknown actor is treated as a forbidden (the actor was
|
||||
// deleted out from under a still-valid certificate); transient control-plane
|
||||
// failures fail closed with 503.
|
||||
func mapEgressIdentityError(atespace, actorName string, err error) error {
|
||||
switch status.Code(err) {
|
||||
case codes.NotFound:
|
||||
return newReqError(envoy_type.StatusCode_Forbidden,
|
||||
"egress denied: unknown actor %q/%q", atespace, actorName)
|
||||
case codes.Unavailable, codes.DeadlineExceeded:
|
||||
return newReqError(envoy_type.StatusCode_ServiceUnavailable,
|
||||
"egress identity check unavailable for %q/%q: %v", atespace, actorName, err)
|
||||
default:
|
||||
return newReqError(envoy_type.StatusCode_Forbidden,
|
||||
"egress denied for %q/%q: %v", atespace, actorName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// loadActorIdentityRoots reads the actor-identity CA trust bundle the egress
|
||||
// gateway verifies actor client certificates against.
|
||||
func loadActorIdentityRoots(pemBytes []byte) (*x509.CertPool, error) {
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(pemBytes) {
|
||||
return nil, fmt.Errorf("actor-identity CA bundle contains no certificates")
|
||||
}
|
||||
return roots, nil
|
||||
}
|
||||
@@ -0,0 +1,698 @@
|
||||
// Copyright 2026 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/asn1"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
||||
extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
|
||||
envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/types/known/structpb"
|
||||
|
||||
"github.com/agent-substrate/substrate/internal/substratex509"
|
||||
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
|
||||
)
|
||||
|
||||
// connectRequest builds a RequestHeaders ProcessingRequest for an egress
|
||||
// CONNECT, optionally attributed to a listener. filterKey is the ext_proc
|
||||
// filter name Envoy keys the attributes map by.
|
||||
func connectRequest(filterKey, listener string) *extprocv3.ProcessingRequest {
|
||||
req := &extprocv3.ProcessingRequest{
|
||||
Request: &extprocv3.ProcessingRequest_RequestHeaders{
|
||||
RequestHeaders: &extprocv3.HttpHeaders{
|
||||
Headers: &corev3.HeaderMap{
|
||||
Headers: []*corev3.HeaderValue{
|
||||
{Key: ":method", RawValue: []byte("CONNECT")},
|
||||
{Key: ":authority", RawValue: []byte("10.0.0.9:443")},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if listener != "" {
|
||||
req.Attributes = map[string]*structpb.Struct{
|
||||
filterKey: {
|
||||
Fields: map[string]*structpb.Value{
|
||||
ListenerNameAttribute: structpb.NewStringValue(listener),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func TestIsEgressRequest(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filterKey string
|
||||
listener string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "egress listener",
|
||||
filterKey: "envoy.filters.http.ext_proc",
|
||||
listener: EgressListenerName,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "egress listener under a renamed filter",
|
||||
filterKey: "some.custom.ext_proc.name",
|
||||
listener: EgressListenerName,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// The pre-listener-dispatch hole: an external client sending
|
||||
// CONNECT to the ingress gateway must not reach the egress handler,
|
||||
// whose denials would otherwise report whether an arbitrary actor
|
||||
// exists and is running.
|
||||
name: "CONNECT on the ingress HTTP listener",
|
||||
filterKey: "envoy.filters.http.ext_proc",
|
||||
listener: IngressHTTPListener,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "CONNECT on the ingress HTTPS listener",
|
||||
filterKey: "envoy.filters.http.ext_proc",
|
||||
listener: IngressHTTPSListener,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// A listener that never requested the attribute falls back to
|
||||
// ingress, the fail-safe direction.
|
||||
name: "no attributes at all",
|
||||
listener: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "unrecognised listener",
|
||||
filterKey: "envoy.filters.http.ext_proc",
|
||||
listener: "some-other-listener",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isEgressRequest(connectRequest(tc.filterKey, tc.listener)); got != tc.want {
|
||||
t.Errorf("isEgressRequest() = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A request the client dresses up to look like egress must not be enough: only
|
||||
// the Envoy-asserted listener name selects the egress handler.
|
||||
func TestIsEgressRequestIgnoresClientSuppliedAttributeHeader(t *testing.T) {
|
||||
req := connectRequest("envoy.filters.http.ext_proc", IngressHTTPListener)
|
||||
rh := req.GetRequestHeaders().GetHeaders()
|
||||
rh.Headers = append(rh.Headers,
|
||||
&corev3.HeaderValue{Key: ListenerNameAttribute, RawValue: []byte(EgressListenerName)},
|
||||
&corev3.HeaderValue{Key: "x-envoy-listener-name", RawValue: []byte(EgressListenerName)},
|
||||
)
|
||||
|
||||
if isEgressRequest(req) {
|
||||
t.Error("isEgressRequest() = true for a client-forged listener header, want false")
|
||||
}
|
||||
}
|
||||
|
||||
// --- actor certificate authentication -------------------------------------
|
||||
|
||||
const (
|
||||
testEgressAtespace = "default"
|
||||
testEgressActor = "my-actor"
|
||||
testEgressActorUID = "1b4e28ba-2fa1-11d2-883f-0016d3cca427"
|
||||
)
|
||||
|
||||
// testCA is a throwaway CA standing in for the actor-identity CA.
|
||||
type testCA struct {
|
||||
cert *x509.Certificate
|
||||
key *ecdsa.PrivateKey
|
||||
}
|
||||
|
||||
func newTestCA(t *testing.T, commonName string) *testCA {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("generating CA key: %v", err)
|
||||
}
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: commonName},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageCertSign,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatalf("creating CA certificate: %v", err)
|
||||
}
|
||||
cert, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing CA certificate: %v", err)
|
||||
}
|
||||
return &testCA{cert: cert, key: key}
|
||||
}
|
||||
|
||||
func (ca *testCA) roots() *x509.CertPool {
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(ca.cert)
|
||||
return pool
|
||||
}
|
||||
|
||||
// oidActorIdentity mirrors the unexported OID substratex509 encodes the
|
||||
// ActorIdentity extension under: the Substrate PEN arc, sub-identifier 2.
|
||||
var oidActorIdentity = append(append(asn1.ObjectIdentifier{}, substratex509.GoogleSubstratePEN...), 2)
|
||||
|
||||
// addActorIdentityUnchecked encodes identity into template the way
|
||||
// substratex509.AddActorIdentityToCertificate does, minus its validation, so
|
||||
// tests can mint the malformed identities a real CA would refuse to produce and
|
||||
// confirm the gateway rejects them anyway.
|
||||
func addActorIdentityUnchecked(identity *substratex509.ActorIdentity, template *x509.Certificate) error {
|
||||
value, err := json.Marshal(identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
template.ExtraExtensions = append(template.ExtraExtensions, pkix.Extension{Id: oidActorIdentity, Value: value})
|
||||
return nil
|
||||
}
|
||||
|
||||
// actorCertOptions mutates the leaf template so each test can break exactly one
|
||||
// property of an otherwise-valid actor certificate.
|
||||
type actorCertOptions struct {
|
||||
identity *substratex509.ActorIdentity
|
||||
extraIdentity *substratex509.ActorIdentity
|
||||
mutate func(*x509.Certificate)
|
||||
}
|
||||
|
||||
// issueActorCert mints a leaf off ca, mirroring what ateapi's actoridentity
|
||||
// service produces.
|
||||
func (ca *testCA) issueActorCert(t *testing.T, opts actorCertOptions) *x509.Certificate {
|
||||
t.Helper()
|
||||
cert, err := x509.ParseCertificate(ca.issueActorCertDER(t, opts))
|
||||
if err != nil {
|
||||
t.Fatalf("parsing leaf certificate: %v", err)
|
||||
}
|
||||
return cert
|
||||
}
|
||||
|
||||
// issueActorCertDER is issueActorCert without the parse, for the certificates
|
||||
// crypto/x509 itself refuses to read back.
|
||||
func (ca *testCA) issueActorCertDER(t *testing.T, opts actorCertOptions) []byte {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("generating leaf key: %v", err)
|
||||
}
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(2),
|
||||
Subject: pkix.Name{CommonName: testEgressActor},
|
||||
NotBefore: time.Now().Add(-5 * time.Minute),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: false,
|
||||
}
|
||||
identity := opts.identity
|
||||
if identity == nil {
|
||||
identity = &substratex509.ActorIdentity{
|
||||
Atespace: testEgressAtespace,
|
||||
ActorName: testEgressActor,
|
||||
ActorUid: testEgressActorUID,
|
||||
Purpose: substratex509.ActorIdentityPurposeAtunnel,
|
||||
}
|
||||
}
|
||||
// AddActorIdentityToCertificate validates its input, so identities a real CA
|
||||
// would refuse to mint are encoded directly.
|
||||
if err := addActorIdentityUnchecked(identity, template); err != nil {
|
||||
t.Fatalf("adding ActorIdentity extension: %v", err)
|
||||
}
|
||||
if opts.extraIdentity != nil {
|
||||
if err := addActorIdentityUnchecked(opts.extraIdentity, template); err != nil {
|
||||
t.Fatalf("adding second ActorIdentity extension: %v", err)
|
||||
}
|
||||
}
|
||||
if opts.mutate != nil {
|
||||
opts.mutate(template)
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key)
|
||||
if err != nil {
|
||||
t.Fatalf("signing leaf certificate: %v", err)
|
||||
}
|
||||
return der
|
||||
}
|
||||
|
||||
// xfccHeader renders chain the way Envoy's SANITIZE_SET +
|
||||
// set_current_client_cert_details{chain: true} does.
|
||||
func xfccHeader(chain ...*x509.Certificate) string {
|
||||
der := make([][]byte, 0, len(chain))
|
||||
for _, cert := range chain {
|
||||
der = append(der, cert.Raw)
|
||||
}
|
||||
return xfccHeaderDER(der...)
|
||||
}
|
||||
|
||||
func xfccHeaderDER(chain ...[]byte) string {
|
||||
var buf strings.Builder
|
||||
for _, der := range chain {
|
||||
_ = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
}
|
||||
return fmt.Sprintf(`By=spiffe://cluster.local/ns/ate-system/sa/ateway-egress;Hash=abc123;Chain="%s"`,
|
||||
url.PathEscape(buf.String()))
|
||||
}
|
||||
|
||||
// egressServer builds an ExtProcServer whose GetActor returns actor/err.
|
||||
func egressServer(roots *x509.CertPool, actor *ateapipb.Actor, err error) *ExtProcServer {
|
||||
client := &egressMockClient{actor: actor, err: err}
|
||||
return NewExtProcServer(50051, client, nil, ParkedRequestConfig{}, nil, false, roots)
|
||||
}
|
||||
|
||||
type egressMockClient struct {
|
||||
ateapipb.ControlClient
|
||||
actor *ateapipb.Actor
|
||||
err error
|
||||
}
|
||||
|
||||
func (m *egressMockClient) GetActor(context.Context, *ateapipb.GetActorRequest, ...grpc.CallOption) (*ateapipb.Actor, error) {
|
||||
if m.err != nil {
|
||||
return nil, m.err
|
||||
}
|
||||
return m.actor, nil
|
||||
}
|
||||
|
||||
func runningActor() *ateapipb.Actor {
|
||||
return &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{
|
||||
Atespace: testEgressAtespace,
|
||||
Name: testEgressActor,
|
||||
Uid: testEgressActorUID,
|
||||
},
|
||||
Status: ateapipb.Actor_STATUS_RUNNING,
|
||||
}
|
||||
}
|
||||
|
||||
// egressHeaders builds the CONNECT the egress listener hands to ext_proc.
|
||||
func egressHeaders(xfcc string) *extprocv3.HttpHeaders {
|
||||
headers := []*corev3.HeaderValue{
|
||||
{Key: ":method", RawValue: []byte("CONNECT")},
|
||||
{Key: ":authority", RawValue: []byte("93.184.216.34:80")},
|
||||
}
|
||||
if xfcc != "" {
|
||||
headers = append(headers, &corev3.HeaderValue{Key: forwardedClientCertHeader, RawValue: []byte(xfcc)})
|
||||
}
|
||||
return &extprocv3.HttpHeaders{Headers: &corev3.HeaderMap{Headers: headers}}
|
||||
}
|
||||
|
||||
func wantStatus(t *testing.T, err error, want envoy_type.StatusCode) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("expected a denial with status %d, got none", want)
|
||||
}
|
||||
var re *reqError
|
||||
if !errors.As(err, &re) {
|
||||
t.Fatalf("error %v is not a *reqError", err)
|
||||
}
|
||||
if re.statusCode != int(want) {
|
||||
t.Errorf("status = %d, want %d (%v)", re.statusCode, want, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleEgressRequestHeadersAllowsVerifiedActor(t *testing.T) {
|
||||
ca := newTestCA(t, "actor-identity-ca")
|
||||
leaf := ca.issueActorCert(t, actorCertOptions{})
|
||||
s := egressServer(ca.roots(), runningActor(), nil)
|
||||
|
||||
resp, _, _, _, _, resume, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf)))
|
||||
if err != nil {
|
||||
t.Fatalf("handleEgressRequestHeaders() error = %v, want nil", err)
|
||||
}
|
||||
if resp == nil {
|
||||
t.Fatal("handleEgressRequestHeaders() returned no response")
|
||||
}
|
||||
if resume != ResumeOutcomeNone {
|
||||
t.Errorf("resume outcome = %q, want %q", resume, ResumeOutcomeNone)
|
||||
}
|
||||
}
|
||||
|
||||
// Every way an actor certificate can fail to prove an identity has to end in a
|
||||
// denial, never in a tunnel.
|
||||
func TestHandleEgressRequestHeadersRejectsBadCertificates(t *testing.T) {
|
||||
ca := newTestCA(t, "actor-identity-ca")
|
||||
otherCA := newTestCA(t, "some-other-ca")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
xfcc func(t *testing.T) string
|
||||
want envoy_type.StatusCode
|
||||
}{
|
||||
{
|
||||
name: "no client certificate at all",
|
||||
xfcc: func(*testing.T) string { return "" },
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "signed by an unknown CA",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(otherCA.issueActorCert(t, actorCertOptions{}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "expired",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) {
|
||||
c.NotBefore = time.Now().Add(-2 * time.Hour)
|
||||
c.NotAfter = time.Now().Add(-time.Hour)
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "not yet valid",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) {
|
||||
c.NotBefore = time.Now().Add(time.Hour)
|
||||
c.NotAfter = time.Now().Add(2 * time.Hour)
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "no ClientAuth EKU",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) {
|
||||
c.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
// An empty EKU means "any usage" to crypto/x509 and would pass
|
||||
// VerifyOptions.KeyUsages; the explicit ClientAuth check is what
|
||||
// catches it.
|
||||
name: "empty EKU",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) {
|
||||
c.ExtKeyUsage = nil
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "is a CA certificate",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) {
|
||||
c.IsCA = true
|
||||
c.KeyUsage |= x509.KeyUsageCertSign
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "no ActorIdentity extension",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) {
|
||||
c.ExtraExtensions = nil
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
// Stays in DER: crypto/x509 refuses to parse a certificate with a
|
||||
// duplicate extension OID at all, so the helper cannot hand back an
|
||||
// *x509.Certificate here. That refusal is the first of the two
|
||||
// layers guarding "exactly one ActorIdentity" — this case proves the
|
||||
// handler denies rather than panics when the parse fails, and
|
||||
// substratex509 rejects a second copy if a parser ever allowed one.
|
||||
name: "two ActorIdentity extensions",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeaderDER(ca.issueActorCertDER(t, actorCertOptions{
|
||||
extraIdentity: &substratex509.ActorIdentity{
|
||||
Atespace: testEgressAtespace,
|
||||
ActorName: "a-different-actor",
|
||||
ActorUid: "8f14e45f-ceea-467a-9575-25a0d5d5e4b0",
|
||||
Purpose: substratex509.ActorIdentityPurposeAtunnel,
|
||||
},
|
||||
}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "generic purpose",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{
|
||||
Atespace: testEgressAtespace,
|
||||
ActorName: testEgressActor,
|
||||
ActorUid: testEgressActorUID,
|
||||
Purpose: "generic",
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "missing purpose",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{
|
||||
Atespace: testEgressAtespace,
|
||||
ActorName: testEgressActor,
|
||||
ActorUid: testEgressActorUID,
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "empty atespace",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{
|
||||
ActorName: testEgressActor,
|
||||
ActorUid: testEgressActorUID,
|
||||
Purpose: substratex509.ActorIdentityPurposeAtunnel,
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "empty actor name",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{
|
||||
Atespace: testEgressAtespace,
|
||||
ActorUid: testEgressActorUID,
|
||||
Purpose: substratex509.ActorIdentityPurposeAtunnel,
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "empty actor UID",
|
||||
xfcc: func(t *testing.T) string {
|
||||
return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{
|
||||
Atespace: testEgressAtespace,
|
||||
ActorName: testEgressActor,
|
||||
Purpose: substratex509.ActorIdentityPurposeAtunnel,
|
||||
}}))
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
// SANITIZE_SET makes Envoy the only writer of the header, so two
|
||||
// elements means we can no longer tell which one is our peer.
|
||||
name: "two XFCC elements",
|
||||
xfcc: func(t *testing.T) string {
|
||||
leaf := ca.issueActorCert(t, actorCertOptions{})
|
||||
return xfccHeader(leaf) + "," + xfccHeader(leaf)
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "XFCC without a Chain value",
|
||||
xfcc: func(*testing.T) string {
|
||||
return `By=spiffe://cluster.local/ns/ate-system/sa/ateway-egress;Hash=abc123`
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "XFCC Chain that is not a certificate",
|
||||
xfcc: func(*testing.T) string {
|
||||
return `Chain="` + url.PathEscape("-----BEGIN CERTIFICATE-----\nbm90LWEtY2VydA==\n-----END CERTIFICATE-----\n") + `"`
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := egressServer(ca.roots(), runningActor(), nil)
|
||||
_, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(tc.xfcc(t)))
|
||||
wantStatus(t, err, tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The certificate authenticates; these cover what the control plane says about
|
||||
// the actor it names.
|
||||
func TestHandleEgressRequestHeadersAuthorization(t *testing.T) {
|
||||
ca := newTestCA(t, "actor-identity-ca")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
actor *ateapipb.Actor
|
||||
err error
|
||||
want envoy_type.StatusCode
|
||||
}{
|
||||
{
|
||||
// The actor was deleted and recreated under the same name: the
|
||||
// certificate is still cryptographically valid but names a UID that
|
||||
// no longer exists, and must not carry over to the successor.
|
||||
name: "actor UID does not match the certificate",
|
||||
actor: &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{
|
||||
Atespace: testEgressAtespace,
|
||||
Name: testEgressActor,
|
||||
Uid: "d41d8cd9-8f00-4204-a980-0998ecf8427e",
|
||||
},
|
||||
Status: ateapipb.Actor_STATUS_RUNNING,
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "actor has no UID",
|
||||
actor: &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{Atespace: testEgressAtespace, Name: testEgressActor},
|
||||
Status: ateapipb.Actor_STATUS_RUNNING,
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "actor is not running",
|
||||
actor: &ateapipb.Actor{
|
||||
Metadata: &ateapipb.ResourceMetadata{
|
||||
Atespace: testEgressAtespace,
|
||||
Name: testEgressActor,
|
||||
Uid: testEgressActorUID,
|
||||
},
|
||||
Status: ateapipb.Actor_STATUS_SUSPENDED,
|
||||
},
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "actor no longer exists",
|
||||
err: status.Error(codes.NotFound, "no such actor"),
|
||||
want: envoy_type.StatusCode_Forbidden,
|
||||
},
|
||||
{
|
||||
name: "control plane unreachable",
|
||||
err: status.Error(codes.Unavailable, "ateapi is down"),
|
||||
want: envoy_type.StatusCode_ServiceUnavailable,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := egressServer(ca.roots(), tc.actor, tc.err)
|
||||
leaf := ca.issueActorCert(t, actorCertOptions{})
|
||||
_, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf)))
|
||||
wantStatus(t, err, tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An ingress-only router has no actor-identity CA. If an egress CONNECT somehow
|
||||
// reaches it, it must fail closed rather than tunnel unauthenticated traffic.
|
||||
func TestHandleEgressRequestHeadersWithoutConfiguredCA(t *testing.T) {
|
||||
ca := newTestCA(t, "actor-identity-ca")
|
||||
leaf := ca.issueActorCert(t, actorCertOptions{})
|
||||
s := egressServer(nil, runningActor(), nil)
|
||||
|
||||
_, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf)))
|
||||
wantStatus(t, err, envoy_type.StatusCode_ServiceUnavailable)
|
||||
}
|
||||
|
||||
func TestHandleEgressRequestHeadersRejectsNonConnect(t *testing.T) {
|
||||
ca := newTestCA(t, "actor-identity-ca")
|
||||
leaf := ca.issueActorCert(t, actorCertOptions{})
|
||||
s := egressServer(ca.roots(), runningActor(), nil)
|
||||
|
||||
headers := egressHeaders(xfccHeader(leaf))
|
||||
for _, h := range headers.Headers.Headers {
|
||||
if h.Key == ":method" {
|
||||
h.RawValue = []byte("GET")
|
||||
}
|
||||
}
|
||||
_, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), headers)
|
||||
wantStatus(t, err, envoy_type.StatusCode_MethodNotAllowed)
|
||||
}
|
||||
|
||||
// PEM bodies routinely contain '+'. Decoding the header as a query string would
|
||||
// turn those into spaces and corrupt the DER, so pin the round trip.
|
||||
func TestParseXFCCChainPreservesPlusInPEM(t *testing.T) {
|
||||
ca := newTestCA(t, "actor-identity-ca")
|
||||
// Serials differ per certificate, so mint until one encodes with a '+'.
|
||||
var leaf *x509.Certificate
|
||||
for i := 0; i < 50; i++ {
|
||||
candidate := ca.issueActorCert(t, actorCertOptions{})
|
||||
if strings.Contains(string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: candidate.Raw})), "+") {
|
||||
leaf = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if leaf == nil {
|
||||
t.Skip("no certificate with a '+' in its PEM body after 50 attempts")
|
||||
}
|
||||
|
||||
chain, err := parseXFCCChain(xfccHeader(leaf))
|
||||
if err != nil {
|
||||
t.Fatalf("parseXFCCChain() error = %v", err)
|
||||
}
|
||||
if len(chain) != 1 || !chain[0].Equal(leaf) {
|
||||
t.Fatalf("parseXFCCChain() did not round-trip the certificate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXFCCChainIncludesIntermediates(t *testing.T) {
|
||||
ca := newTestCA(t, "actor-identity-ca")
|
||||
leaf := ca.issueActorCert(t, actorCertOptions{})
|
||||
|
||||
chain, err := parseXFCCChain(xfccHeader(leaf, ca.cert))
|
||||
if err != nil {
|
||||
t.Fatalf("parseXFCCChain() error = %v", err)
|
||||
}
|
||||
if len(chain) != 2 {
|
||||
t.Fatalf("parseXFCCChain() returned %d certificates, want 2", len(chain))
|
||||
}
|
||||
if !chain[0].Equal(leaf) {
|
||||
t.Error("parseXFCCChain() did not return the leaf first")
|
||||
}
|
||||
}
|
||||
@@ -28,12 +28,14 @@ type requestMetadata struct {
|
||||
headers map[string]string
|
||||
path string
|
||||
host string
|
||||
method string
|
||||
}
|
||||
|
||||
func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata {
|
||||
headersMap := make(map[string]string)
|
||||
var path string
|
||||
var host string
|
||||
var method string
|
||||
|
||||
for _, h := range headers {
|
||||
k := strings.ToLower(h.Key)
|
||||
@@ -49,12 +51,16 @@ func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata {
|
||||
if k == authorityHeader || k == "host" {
|
||||
host = val
|
||||
}
|
||||
if k == ":method" {
|
||||
method = val
|
||||
}
|
||||
}
|
||||
|
||||
return &requestMetadata{
|
||||
headers: headersMap,
|
||||
path: path,
|
||||
host: host,
|
||||
method: method,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) {
|
||||
resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) {
|
||||
return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}}, nil
|
||||
},
|
||||
}, nil, ParkedRequestConfig{}, nil, false)
|
||||
}, nil, ParkedRequestConfig{}, nil, false, nil)
|
||||
|
||||
reqHeaders := &extprocv3.HttpHeaders{
|
||||
Headers: &corev3.HeaderMap{
|
||||
@@ -196,7 +196,7 @@ func TestExtProcHeadersEvaluation(t *testing.T) {
|
||||
// Parking disabled: these cases assert fail-fast mapping of resume
|
||||
// errors (e.g. FailedPrecondition -> immediate 503). Parking behavior
|
||||
// is covered separately in TestExtProc_ParkingLotFull and resumer_test.go.
|
||||
s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{}, nil, false)
|
||||
s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{}, nil, false, nil)
|
||||
|
||||
reqHeaders := &extprocv3.HttpHeaders{
|
||||
Headers: &corev3.HeaderMap{
|
||||
@@ -277,7 +277,7 @@ func TestExtProc_ParkingLotFull(t *testing.T) {
|
||||
|
||||
// A 1-slot lot with the slot already occupied deterministically simulates a
|
||||
// full lot without needing a concurrent in-flight request.
|
||||
s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{Budget: time.Second, Max: 1}, nil, false)
|
||||
s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{Budget: time.Second, Max: 1}, nil, false, nil)
|
||||
release, ok := s.parking.enter(context.Background())
|
||||
if !ok {
|
||||
t.Fatal("priming enter should be admitted")
|
||||
@@ -396,7 +396,7 @@ func TestRecordRouteDuration_Attributes(t *testing.T) {
|
||||
t.Fatalf("failed to create histogram: %v", err)
|
||||
}
|
||||
|
||||
s := NewExtProcServer(50051, nil, h, ParkedRequestConfig{}, nil, false)
|
||||
s := NewExtProcServer(50051, nil, h, ParkedRequestConfig{}, nil, false, nil)
|
||||
s.recordRouteDuration(context.Background(), 10*time.Millisecond, "team-a-ns", "tmpl-a", classifyOutcome(nil), string(ResumeOutcomeTriggered))
|
||||
|
||||
var rm metricdata.ResourceMetrics
|
||||
|
||||
@@ -16,6 +16,7 @@ package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -215,7 +216,21 @@ func (s *RouterServer) Run(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create parking metrics: %w", err)
|
||||
}
|
||||
s.extprocSrv = NewExtProcServer(s.cfg.ExtprocPort, s.apiClient, routeDuration, parkCfg, parkMetrics, s.cfg.atenetRouter().routeViaAuthority())
|
||||
// Load the actor-identity CA up front so a missing or unusable bundle
|
||||
// fails startup, rather than turning into a 503 on the first actor
|
||||
// egress attempt. An unset flag is the ingress-only case and is fine.
|
||||
var actorIdentityRoots *x509.CertPool
|
||||
if s.cfg.ActorIdentityCAFile != "" {
|
||||
pemBytes, err := os.ReadFile(s.cfg.ActorIdentityCAFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading --actor-identity-ca-file: %w", err)
|
||||
}
|
||||
actorIdentityRoots, err = loadActorIdentityRoots(pemBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading --actor-identity-ca-file %q: %w", s.cfg.ActorIdentityCAFile, err)
|
||||
}
|
||||
}
|
||||
s.extprocSrv = NewExtProcServer(s.cfg.ExtprocPort, s.apiClient, routeDuration, parkCfg, parkMetrics, s.cfg.atenetRouter().routeViaAuthority(), actorIdentityRoots)
|
||||
}
|
||||
s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg)
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ func TestStatuszEndpoint(t *testing.T) {
|
||||
t.Fatalf("Failed generating router server: %v", err)
|
||||
}
|
||||
|
||||
srv.extprocSrv = NewExtProcServer(cfg.ExtprocPort, &mockClient{}, nil, defaultParkedRequestConfig(), nil, false)
|
||||
srv.extprocSrv = NewExtProcServer(cfg.ExtprocPort, &mockClient{}, nil, defaultParkedRequestConfig(), nil, false, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
+56
-4
@@ -79,6 +79,7 @@ function usage() {
|
||||
echo ""
|
||||
echo " --create-jwt-authority-pool-secret Create JWT authority pool secret"
|
||||
echo " --create-actor-id-ca-pool-secret Create actor ID CA pool secret"
|
||||
echo " --create-actor-id-ca-certs-secret Create actor ID CA certs secret"
|
||||
echo " --create-podcertificate-controller-cas Create podcertificate controller CAs"
|
||||
echo " --create-valkey-ca-certs-secret Create Valkey CA certs secret"
|
||||
echo " --create-api-server-env-vars Create ate-api-server env vars"
|
||||
@@ -222,10 +223,13 @@ apply_otel_config() {
|
||||
}
|
||||
|
||||
# Extract a CA pool secret's RootCertificateDER and emit it as a PEM certificate.
|
||||
# The namespace defaults to the podcertificate controller's, where the signer
|
||||
# CAs live; the actor-identity CA pool is in ate-system, so it passes its own.
|
||||
ca_pool_root_pem() {
|
||||
local secret="$1"
|
||||
local namespace="${2:-podcertificate-controller-system}"
|
||||
local pool_json=""
|
||||
pool_json=$(run_kubectl get secret -n podcertificate-controller-system "${secret}" -o jsonpath='{.data.pool}' | base64 --decode)
|
||||
pool_json=$(run_kubectl get secret -n "${namespace}" "${secret}" -o jsonpath='{.data.pool}' | base64 --decode)
|
||||
local der_base64=""
|
||||
der_base64=$(echo "${pool_json}" | grep -o '"RootCertificateDER":"[^"]*' | sed 's/"RootCertificateDER":"//')
|
||||
echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem
|
||||
@@ -275,6 +279,43 @@ create_actor_id_ca_pool_secret() {
|
||||
--secret-namespace=ate-system
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SEE(lior): actor-identity CA trust bundle for the egress PEP.
|
||||
#
|
||||
# The egress gateway has to verify actor client certificates, which means it
|
||||
# needs the actor-identity CA *root* — and only the root. The authoritative
|
||||
# copy today is the actor-id-ca-pool Secret, but its pool.json also carries the
|
||||
# CA signing key, so mounting that Secret into the gateway would put the key
|
||||
# that mints every actor identity inside a pod that only ever needs to verify
|
||||
# them. This derives a cert-only Secret instead, following exactly the pattern
|
||||
# create_valkey_ca_certs_secret already uses for the signer roots.
|
||||
#
|
||||
# TODO(liorlieberman): revisit. The other CAs reach their consumers as
|
||||
# ClusterTrustBundles published by the podcertificate controller, and the
|
||||
# actor-identity CA arguably should too — then the gateway would project a
|
||||
# trust bundle like it already does for servicedns/podidentity and this
|
||||
# install-time Secret would go away. Doing that needs a signer/controller path
|
||||
# that does not exist yet, so this is the interim shape.
|
||||
# ---------------------------------------------------------------------------
|
||||
create_actor_id_ca_certs_secret() {
|
||||
log_step "create_actor_id_ca_certs_secret"
|
||||
# Extract into its own variable first: errexit cannot see a substitution fail
|
||||
# inside the create-secret argument list, which would silently produce an
|
||||
# empty trust bundle and an egress gateway that rejects every actor.
|
||||
local actorid_root=""
|
||||
actorid_root=$(ca_pool_root_pem actor-id-ca-pool ate-system)
|
||||
if [[ -z "${actorid_root}" ]]; then
|
||||
echo "error: failed to extract the actor-identity CA root for actor-id-ca-certs" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
run_kubectl create secret generic actor-id-ca-certs \
|
||||
--from-literal=ca.crt="${actorid_root}" \
|
||||
-n ate-system \
|
||||
--dry-run=client -o yaml \
|
||||
| run_kubectl apply -f -
|
||||
}
|
||||
|
||||
create_podcertificate_controller_cas() {
|
||||
log_step "create_podcertificate_controller_cas"
|
||||
run_kubectl create namespace podcertificate-controller-system || true
|
||||
@@ -405,6 +446,7 @@ deploy_ate_system() {
|
||||
run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout=120s
|
||||
run_kubectl rollout status deployment/ate-controller -n ate-system --timeout=120s
|
||||
run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s
|
||||
run_kubectl rollout status deployment/ateway-egress -n ate-system --timeout=120s
|
||||
run_kubectl rollout status statefulset/valkey-cluster -n ate-system --timeout=120s
|
||||
run_kubectl rollout status daemonset/atelet -n ate-system --timeout=120s
|
||||
}
|
||||
@@ -416,6 +458,9 @@ ensure_apiserver_prerequisites() {
|
||||
|| create_jwt_authority_pool_secret
|
||||
run_kubectl get secret -n ate-system actor-id-ca-pool >/dev/null 2>&1 \
|
||||
|| create_actor_id_ca_pool_secret
|
||||
# SEE(lior): derived from actor-id-ca-pool above, so it must come after it.
|
||||
run_kubectl get secret -n ate-system actor-id-ca-certs >/dev/null 2>&1 \
|
||||
|| create_actor_id_ca_certs_secret
|
||||
run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool >/dev/null 2>&1 \
|
||||
|| create_podcertificate_controller_cas
|
||||
run_kubectl get secret -n ate-system valkey-ca-certs >/dev/null 2>&1 \
|
||||
@@ -476,11 +521,15 @@ deploy_atenet() {
|
||||
router_manifest="$(render_atenet_router_manifest)"
|
||||
echo "${router_manifest}" | run_kubectl apply -f -
|
||||
|
||||
run_ko apply -f manifests/ate-install/ateway-egress.yaml
|
||||
run_ko apply -f manifests/ate-install/atenet-dns.yaml
|
||||
run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s
|
||||
# The Deployment in atenet-dns.yaml is named "dns"; every other resource in
|
||||
# that file is "atenet-dns". Waiting on the filename rather than the actual
|
||||
# Deployment made this step fail with NotFound on every successful deploy.
|
||||
run_kubectl rollout status deployment/ateway-egress -n ate-system --timeout=120s
|
||||
# SEE(lior): this branch also added a `deployment/atenet-dns` wait here, which
|
||||
# main has since fixed to `deployment/dns` (the Deployment in atenet-dns.yaml
|
||||
# is named "dns"; every other resource in that file is "atenet-dns", so the
|
||||
# old name was NotFound on every successful deploy). Dropped this branch's
|
||||
# duplicate in favour of main's corrected line.
|
||||
run_kubectl rollout status deployment/dns -n ate-system --timeout=120s
|
||||
}
|
||||
|
||||
@@ -599,6 +648,8 @@ delete_atenet() {
|
||||
run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-router.yaml
|
||||
run_kubectl delete --ignore-not-found \
|
||||
-f manifests/ate-install/components/agentgateway/configmap.yaml
|
||||
run_kubectl delete --ignore-not-found -f manifests/ate-install/ateway-egress.yaml
|
||||
run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-dns.yaml
|
||||
}
|
||||
|
||||
deploy_benchmarks() {
|
||||
@@ -734,6 +785,7 @@ while [[ "$#" -gt 0 ]]; do
|
||||
|
||||
--create-jwt-authority-pool-secret) create_jwt_authority_pool_secret ;;
|
||||
--create-actor-id-ca-pool-secret) create_actor_id_ca_pool_secret ;;
|
||||
--create-actor-id-ca-certs-secret) create_actor_id_ca_certs_secret ;;
|
||||
--create-podcertificate-controller-cas) create_podcertificate_controller_cas ;;
|
||||
--create-valkey-ca-certs-secret) create_valkey_ca_certs_secret ;;
|
||||
--create-api-server-env-vars) create_api_server_env_vars ;;
|
||||
|
||||
@@ -104,6 +104,15 @@ spec:
|
||||
- --actor-id-ca-pool=/run/actor-id-ca-pool/pool.json
|
||||
- --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem
|
||||
- --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem
|
||||
# ONLY UNTIL THE EGRESS API IS DECIDED, FOR POC: turn on pluggable actor
|
||||
# egress cluster-wide. ateapi stamps this address onto every atelet
|
||||
# Run/Restore, ateom hands it to atunnel, and actor TCP egress is
|
||||
# transparently redirected (nftables) into atunnel, which wraps it in
|
||||
# mTLS + HTTP CONNECT to the egress gateway.
|
||||
#
|
||||
# SEE(lior): this flag lived on atelet on the pre-rebase branch; PR 708
|
||||
# moved egress-gateway configuration to ateapi, so it is set here now.
|
||||
- --egress-gateway-address=ateway-egress.ate-system.svc:443
|
||||
# Graceful shutdown knobs. The sum must fit within terminationGracePeriodSeconds.
|
||||
- --drain-delay=13s
|
||||
- --drain-timeout=15s
|
||||
|
||||
@@ -0,0 +1,393 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# contract expected by atunnel's egress client:
|
||||
# * downstream mTLS on :443 (present servicedns identity, require + verify an
|
||||
# actor-identity client cert),
|
||||
# * terminate the actor's HTTP CONNECT and tunnel raw TCP to the CONNECT
|
||||
# authority (the actor's original destination, always sent as IP:port).
|
||||
#
|
||||
# SEE(lior): the client cert is the actor's own identity — minted per actor by
|
||||
# ateapi's actoridentity service off the actor-identity CA and carrying the
|
||||
# ActorIdentity X.509 extension — not the pod's podidentity cert. The pod cert
|
||||
# says "some substrate pod"; only the actor cert says *which actor*, which is
|
||||
# the thing the gateway has to authorize. The CONNECT carries no identity
|
||||
# headers at all; everything the PEP decides on comes out of the certificate.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: ateway-egress
|
||||
namespace: ate-system
|
||||
---
|
||||
# RBAC for the co-located ext_proc sidecar (atenet router), which watches
|
||||
# ActorTemplates as part of its normal operation.
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: ateway-egress
|
||||
rules:
|
||||
- apiGroups:
|
||||
- "ate.dev"
|
||||
resources:
|
||||
- actortemplates
|
||||
verbs:
|
||||
- get
|
||||
- watch
|
||||
- list
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: ateway-egress
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: ateway-egress
|
||||
namespace: ate-system
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: ateway-egress
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: ateway-egress
|
||||
namespace: ate-system
|
||||
data:
|
||||
envoy.yaml: |
|
||||
admin:
|
||||
address:
|
||||
socket_address: { address: 0.0.0.0, port_value: 15000 }
|
||||
static_resources:
|
||||
listeners:
|
||||
- name: egress
|
||||
address:
|
||||
socket_address: { address: 0.0.0.0, port_value: 443 }
|
||||
filter_chains:
|
||||
- transport_socket:
|
||||
name: envoy.transport_sockets.tls
|
||||
typed_config:
|
||||
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
|
||||
require_client_certificate: true
|
||||
common_tls_context:
|
||||
tls_certificates:
|
||||
# Gateway server identity (servicedns signer). credential-bundle.pem
|
||||
# holds the leaf cert and key concatenated. watched_directory picks
|
||||
# up kubelet's projected certificate rotation without a restart.
|
||||
- certificate_chain: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem }
|
||||
private_key: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem }
|
||||
watched_directory: { path: /run/servicedns.podcert.ate.dev }
|
||||
validation_context:
|
||||
# Actors authenticate with an actor-identity client cert.
|
||||
# Envoy enforces chain, signature, and validity period here, so
|
||||
# an unsigned or expired actor cert never reaches ext_proc; the
|
||||
# ActorIdentity extension is checked there because Envoy cannot
|
||||
# read custom X.509 extensions.
|
||||
trusted_ca: { filename: /run/actor-id-ca-certs/ca.crt }
|
||||
filters:
|
||||
- name: envoy.filters.network.http_connection_manager
|
||||
typed_config:
|
||||
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
|
||||
stat_prefix: egress_connect
|
||||
codec_type: HTTP1
|
||||
upgrade_configs:
|
||||
- upgrade_type: CONNECT
|
||||
# SEE(lior): this pair is how the actor's certificate reaches the
|
||||
# ext_proc handler. ext_proc can request Envoy attributes, but none
|
||||
# of them carry a custom X.509 extension, so the only way to check
|
||||
# ActorIdentity in Go is to have Envoy hand over the raw chain.
|
||||
# SANITIZE_SET drops whatever x-forwarded-client-cert the client
|
||||
# sent and writes Envoy's own view of the verified peer, and
|
||||
# chain: true puts the full URL-encoded PEM chain in it. Removing
|
||||
# either of these breaks egress closed: the handler sees no XFCC
|
||||
# and denies every CONNECT.
|
||||
forward_client_cert_details: SANITIZE_SET
|
||||
set_current_client_cert_details:
|
||||
chain: true
|
||||
# Emit the access log as soon as the CONNECT tunnel is established
|
||||
# (atunnel keeps the tunnel open, so the default log-on-close would
|
||||
# not fire during the demo).
|
||||
access_log_options:
|
||||
flush_log_on_tunnel_successfully_established: true
|
||||
access_log:
|
||||
- name: envoy.access_loggers.stdout
|
||||
typed_config:
|
||||
"@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
|
||||
log_format:
|
||||
text_format_source:
|
||||
# SEE(lior): the peer fields come from the verified client
|
||||
# certificate, not from request headers. The actor sends no
|
||||
# identity headers any more, and logging attacker-controlled
|
||||
# ones would have made this log lie about who egressed.
|
||||
# peer= is the leaf subject (CN carries the actor); the
|
||||
# atespace/UID live in the ActorIdentity extension, which
|
||||
# Envoy cannot format — the ext_proc handler logs those.
|
||||
inline_string: "[egress] authority=%REQ(:AUTHORITY)% peer=%DOWNSTREAM_PEER_SUBJECT% peer_san=%DOWNSTREAM_PEER_URI_SAN% peer_serial=%DOWNSTREAM_PEER_SERIAL% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% up_bytes=%BYTES_RECEIVED% down_bytes=%BYTES_SENT%\n"
|
||||
route_config:
|
||||
name: connect_route
|
||||
virtual_hosts:
|
||||
- name: connect
|
||||
domains: ["*"]
|
||||
routes:
|
||||
- match: { connect_matcher: {} }
|
||||
route:
|
||||
cluster: egress_forward_proxy
|
||||
upgrade_configs:
|
||||
- upgrade_type: CONNECT
|
||||
connect_config: {}
|
||||
http_filters:
|
||||
# Actor-identity authorization: on every egress CONNECT, ext_proc
|
||||
# reads the actor certificate out of x-forwarded-client-cert,
|
||||
# re-verifies it against the actor-identity CA, pulls the
|
||||
# ActorIdentity extension, and asks the ate API whether that UID is
|
||||
# a real, running actor. Denials come back as an immediate 403.
|
||||
# Fails closed if the router is down.
|
||||
- name: envoy.filters.http.ext_proc
|
||||
typed_config:
|
||||
"@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor
|
||||
grpc_service:
|
||||
envoy_grpc:
|
||||
cluster_name: ext_proc_server
|
||||
timeout: 2s
|
||||
failure_mode_allow: false
|
||||
# How the ext_proc server tells egress from ingress. It applies
|
||||
# opposite trust models to the two directions, and dispatches on
|
||||
# this Envoy-asserted listener name so that no client can select
|
||||
# the egress path by crafting a request. The value must match
|
||||
# EgressListenerName in cmd/atenet/internal/router/extproc_egress.go;
|
||||
# renaming the listener above without updating it fails closed
|
||||
# (egress requests take the ingress path and 404).
|
||||
request_attributes:
|
||||
- xds.listener_name
|
||||
processing_mode:
|
||||
request_header_mode: SEND
|
||||
response_header_mode: SKIP
|
||||
request_body_mode: NONE
|
||||
response_body_mode: NONE
|
||||
request_trailer_mode: SKIP
|
||||
response_trailer_mode: SKIP
|
||||
- name: envoy.filters.http.dynamic_forward_proxy
|
||||
typed_config:
|
||||
"@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig
|
||||
dns_cache_config:
|
||||
name: egress_dns_cache
|
||||
dns_lookup_family: V4_ONLY
|
||||
- name: envoy.filters.http.router
|
||||
typed_config:
|
||||
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
|
||||
clusters:
|
||||
# ext_proc gRPC server = the atenet router, co-located in this pod as a
|
||||
# sidecar and called over localhost (same topology the ingress gateway uses
|
||||
# for its Envoy + ext_proc).
|
||||
- name: ext_proc_server
|
||||
type: STATIC
|
||||
lb_policy: ROUND_ROBIN
|
||||
connect_timeout: 1s
|
||||
typed_extension_protocol_options:
|
||||
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
|
||||
"@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
|
||||
explicit_http_config:
|
||||
http2_protocol_options: {}
|
||||
load_assignment:
|
||||
cluster_name: ext_proc_server
|
||||
endpoints:
|
||||
- lb_endpoints:
|
||||
- endpoint:
|
||||
address:
|
||||
socket_address:
|
||||
address: 127.0.0.1
|
||||
port_value: 50051
|
||||
# Dials the terminated-CONNECT target (IP:port from the authority). atunnel
|
||||
# always sends an IP:port, so DNS resolution is effectively a passthrough.
|
||||
- name: egress_forward_proxy
|
||||
lb_policy: CLUSTER_PROVIDED
|
||||
connect_timeout: 5s
|
||||
cluster_type:
|
||||
name: envoy.clusters.dynamic_forward_proxy
|
||||
typed_config:
|
||||
"@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig
|
||||
dns_cache_config:
|
||||
name: egress_dns_cache
|
||||
dns_lookup_family: V4_ONLY
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ateway-egress
|
||||
namespace: ate-system
|
||||
labels:
|
||||
app: ateway-egress
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: ateway-egress
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ateway-egress
|
||||
spec:
|
||||
serviceAccountName: ateway-egress
|
||||
securityContext:
|
||||
# Allow the non-root envoy user to bind :443.
|
||||
sysctls:
|
||||
- name: net.ipv4.ip_unprivileged_port_start
|
||||
value: "0"
|
||||
containers:
|
||||
- name: envoy
|
||||
image: envoyproxy/envoy:v1.34-latest
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
args:
|
||||
- -c
|
||||
- /etc/envoy/envoy.yaml
|
||||
- --service-node
|
||||
- ateway-egress
|
||||
- --service-cluster
|
||||
- ateway-egress
|
||||
ports:
|
||||
- name: https
|
||||
containerPort: 443
|
||||
- name: admin
|
||||
containerPort: 15000
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: admin
|
||||
periodSeconds: 10
|
||||
startupProbe:
|
||||
failureThreshold: 60
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: admin
|
||||
periodSeconds: 1
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/envoy
|
||||
readOnly: true
|
||||
- name: servicedns
|
||||
mountPath: /run/servicedns.podcert.ate.dev
|
||||
readOnly: true
|
||||
- name: podidentity
|
||||
mountPath: /run/podidentity.podcert.ate.dev
|
||||
readOnly: true
|
||||
- name: actor-id-ca-certs
|
||||
mountPath: /run/actor-id-ca-certs
|
||||
readOnly: true
|
||||
# Co-located ext_proc server (the atenet router, ext_proc-only). The egress
|
||||
# Envoy calls it over localhost to authenticate actor identity against the
|
||||
# ate API on every CONNECT. This mirrors the ingress gateway topology
|
||||
# (Envoy + ext_proc in one pod); a shared/standalone ext_proc is a future step.
|
||||
- name: ext-proc
|
||||
image: ko://github.com/agent-substrate/substrate/cmd/atenet
|
||||
args:
|
||||
- router
|
||||
- --standalone
|
||||
- --namespace=ate-system
|
||||
- --port-extproc=50051
|
||||
- --extproc-address=127.0.0.1
|
||||
- --ateapi-address=api.ate-system.svc:443
|
||||
- --ateapi-auth=mtls
|
||||
# Same bundle Envoy validates the handshake against. The handler
|
||||
# re-verifies the chain in Go and reads the ActorIdentity extension out
|
||||
# of it; without this flag the router has no actor-identity roots and
|
||||
# denies every egress CONNECT with a 503.
|
||||
- --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt
|
||||
- --otlp-collector-address=
|
||||
env:
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
ports:
|
||||
- name: extproc
|
||||
containerPort: 50051
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: extproc
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: actor-id-ca-certs
|
||||
mountPath: /run/actor-id-ca-certs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: ateway-egress
|
||||
- name: servicedns
|
||||
projected:
|
||||
sources:
|
||||
- podCertificate:
|
||||
signerName: servicedns.podcert.ate.dev/identity
|
||||
keyType: ECDSAP256
|
||||
credentialBundlePath: credential-bundle.pem
|
||||
- clusterTrustBundle:
|
||||
signerName: servicedns.podcert.ate.dev/identity
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
podcert.ate.dev/canarying: live
|
||||
path: trust-bundle.pem
|
||||
- name: podidentity
|
||||
projected:
|
||||
sources:
|
||||
- podCertificate:
|
||||
signerName: podidentity.podcert.ate.dev/identity
|
||||
keyType: ECDSAP256
|
||||
credentialBundlePath: credential-bundle.pem
|
||||
- clusterTrustBundle:
|
||||
signerName: podidentity.podcert.ate.dev/identity
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
podcert.ate.dev/canarying: live
|
||||
path: trust-bundle.pem
|
||||
# SEE(lior): the actor-identity CA root, as a cert-only Secret that
|
||||
# install-ate.sh derives from the actor-id-ca-pool Secret. It is a plain
|
||||
# Secret rather than a projected clusterTrustBundle — unlike the two above
|
||||
# — because the actor-identity CA has no signer/controller publishing a
|
||||
# trust bundle for it yet; its only in-cluster home is actor-id-ca-pool,
|
||||
# whose pool.json also holds the CA *signing key*. Mounting that here would
|
||||
# put the key that mints every actor identity into the pod that merely
|
||||
# verifies them, so install-ate.sh extracts the root and nothing else.
|
||||
#
|
||||
# TODO(liorlieberman): revisit once the actor-identity CA is published as a
|
||||
# ClusterTrustBundle; this volume then becomes a third projected source and
|
||||
# the install-time Secret goes away. See create_actor_id_ca_certs_secret in
|
||||
# hack/install-ate.sh.
|
||||
- name: actor-id-ca-certs
|
||||
secret:
|
||||
secretName: actor-id-ca-certs
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ateway-egress
|
||||
namespace: ate-system
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: ateway-egress
|
||||
ports:
|
||||
- name: https
|
||||
port: 443
|
||||
targetPort: https
|
||||
protocol: TCP
|
||||
@@ -23,6 +23,7 @@ resources:
|
||||
- ../ate-controller.yaml
|
||||
- ../atelet.yaml
|
||||
- ../atenet-dns.yaml
|
||||
- ../ateway-egress.yaml
|
||||
- ../atenet-router.yaml
|
||||
- ../valkey.yaml
|
||||
- ../pod-certificate-controller.yaml
|
||||
|
||||
@@ -26,6 +26,7 @@ resources:
|
||||
- ../ate-controller.yaml
|
||||
- ./atelet
|
||||
- ../atenet-dns.yaml
|
||||
- ../ateway-egress.yaml
|
||||
- ../atenet-router.yaml
|
||||
- ../valkey.yaml
|
||||
- ../pod-certificate-controller.yaml
|
||||
|
||||
Reference in New Issue
Block a user