mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-30 07:33:59 +08:00
* feat(kubernetes): support HA gateway rebalancing Signed-off-by: Drew Newberry <anewberry@nvidia.com> * perf(server): cache peer connections, tokens, and owner lookups Every forwarded relay rebuilt its setup from scratch: an owner lookup, a blocking read of the peer token, a TLS connect to the owning replica, and a TokenReview plus Pod GET on the receiving side. Sandbox service routing does this per HTTP request, so the apiserver calls scaled with traffic. Cache all of it on ServerState: - peer channels pooled per endpoint, so relays multiplex over one connection instead of redialing - peer tokens keyed by SHA-256, expiring at min(ttl, token exp) so a hit cannot accept an expired token - owner records for 3s against a 45s ownership TTL, still freshness checked before use Entries are evicted when a relay fails. Also raise HTTP/2 max_concurrent_streams to 1024, since pooling funnels every relay between two replicas onto one connection and hyper's default of 200 sits below the 256 pending-relay budget. Signed-off-by: divesh <dgude@nvidia.com> * perf(server): pool upstream connections for sandbox services Each HTTP request to a sandbox service opened its own supervisor relay, paying a new TCP connection and HTTP/1 handshake every time. Worse, it counted against the 32 in-flight relay cap, so a service handling more than 32 concurrent requests failed outright. Pool idle upstreams per endpoint and port, up to 8 each for 15s. Reuse is safe because the pool only returns a connection hyper reports as ready, and HTTP/1 cannot start a request until the previous body has drained. Upgrades are never pooled since they take the connection over, and a failed send evicts that endpoint. Pruning is bounded per key, with the full sweep limited to once per 30s. Signed-off-by: divesh <dgude@nvidia.com> * fix(server): address HA gateway review findings (#3449) - Let a gateway own supervisor sessions without a peer endpoint. Requiring one whenever the store is PostgreSQL broke every single-instance PostgreSQL deployment, because no sandbox supervisor could connect. A cross-replica request to an owner that advertises no endpoint now fails immediately naming the cause, instead of retrying until the wait timeout. - Close a supervisor session on heartbeat only when another replica owns it, or after renewals fail for the ownership TTL. A database error no longer drops every session heartbeating during an outage. - Clamp owner record ages at zero so a skewed or corrupt stored timestamp cannot produce a negative age. - Bound the cross-object advisory lock with a lock timeout, so a stuck holder fails instead of blocking every mutation in the fleet. - Refuse to start when a peer endpoint is configured on a multi-replica backend but peer authentication is unavailable, and warn when a multi-replica backend has no peer endpoint at all. - Reject a plaintext peer endpoint when the gateway serves TLS. - Skip the sandbox watch poller on single-replica backends, where the local update bus already sees every write. - Rate-limit the peer owner cache sweep so an insert no longer scans the whole map under the lock. - Retry GET and HEAD on a pooled upstream the sandbox closed, instead of returning 502, and drop an emptied endpoint from the pool right away. - Document the gateway peer environment variables and the post-rollout ownership skew operators should expect. Signed-off-by: divesh <dgude@nvidia.com> * fix(server): harden HA supervisor ownership Signed-off-by: Drew Newberry <anewberry@nvidia.com> --------- Signed-off-by: Drew Newberry <anewberry@nvidia.com> Signed-off-by: divesh <dgude@nvidia.com> Co-authored-by: Drew Newberry <anewberry@nvidia.com> Co-authored-by: divesh <dgude@nvidia.com> Co-authored-by: Divesh Chowdary <47188680+FrostGod@users.noreply.github.com>
168 lines
4.5 KiB
YAML
168 lines
4.5 KiB
YAML
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" }}
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: {{ include "openshell.fullname" . }}-node-reader-{{ .Release.Namespace }}
|
|
labels:
|
|
{{- include "openshell.labels" . | nindent 4 }}
|
|
rules:
|
|
# Validate projected ServiceAccount tokens during sandbox bootstrap and
|
|
# internal gateway peer authentication.
|
|
- apiGroups:
|
|
- authentication.k8s.io
|
|
resources:
|
|
- tokenreviews
|
|
verbs:
|
|
- create
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- nodes
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
# Read namespace annotations for OpenShift SCC UID/GID range resolution.
|
|
# Managed/operator modes additionally need list+watch for cluster-wide
|
|
# namespace discovery. Managed mode needs create+delete for namespace
|
|
# lifecycle.
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- namespaces
|
|
verbs:
|
|
- get
|
|
{{- if ne $workspaceMode "shared" }}
|
|
- list
|
|
- watch
|
|
{{- end }}
|
|
{{- if eq $workspaceMode "managed" }}
|
|
- create
|
|
- delete
|
|
{{- end }}
|
|
{{- if ne $workspaceMode "shared" }}
|
|
# Cluster-wide sandbox CRD access for managed/operator workspace modes.
|
|
- apiGroups:
|
|
- agents.x-k8s.io
|
|
resources:
|
|
- sandboxes
|
|
- sandboxes/status
|
|
verbs:
|
|
- create
|
|
- delete
|
|
- get
|
|
- list
|
|
- patch
|
|
- update
|
|
- watch
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- events
|
|
verbs:
|
|
- get
|
|
- list
|
|
- watch
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- pods
|
|
verbs:
|
|
- create
|
|
- delete
|
|
- get
|
|
- list
|
|
- patch
|
|
- watch
|
|
{{- end }}
|
|
{{- $copiedSecretNames := list }}
|
|
{{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }}
|
|
{{- $copiedSecretNames = append $copiedSecretNames .Values.server.tls.clientTlsSecretName }}
|
|
{{- end }}
|
|
{{- if eq $workspaceMode "managed" }}
|
|
{{- range .Values.server.sandboxImagePullSecrets }}
|
|
{{- if .name }}
|
|
{{- $copiedSecretNames = append $copiedSecretNames .name }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if ne $workspaceMode "shared" }}
|
|
- apiGroups: [""]
|
|
resources: ["services"]
|
|
verbs: ["create", "get"]
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
# Bootstrap Secrets are generation-scoped. Recovery lists them by label and
|
|
# deletes stale generations with UID preconditions before retrying creation.
|
|
verbs: ["create", "list", "delete"]
|
|
- apiGroups: ["networking.k8s.io"]
|
|
resources: ["networkpolicies"]
|
|
verbs: ["create", "get"]
|
|
{{- end }}
|
|
{{- $copiedSecretNames = uniq $copiedSecretNames }}
|
|
{{- if $copiedSecretNames }}
|
|
# Copy only explicitly configured TLS and image-pull Secrets into workspace
|
|
# namespaces. Server-side apply authorizes these requests as patch operations.
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- secrets
|
|
resourceNames:
|
|
{{- range $copiedSecretNames }}
|
|
- {{ . | quote }}
|
|
{{- end }}
|
|
verbs:
|
|
- get
|
|
- patch
|
|
# Server-side apply uses PATCH for an existing Secret, but the API server
|
|
# additionally authorizes CREATE when the named Secret does not exist yet.
|
|
# Kubernetes RBAC cannot restrict CREATE by resourceNames because create
|
|
# authorization happens before the object name is available to RBAC. Keep
|
|
# reads and mutations name-restricted above; this broader grant is required
|
|
# only to create the initial copy in a gateway-owned managed namespace.
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- secrets
|
|
verbs:
|
|
- create
|
|
{{- end }}
|
|
{{- if and (ne $workspaceMode "shared") .Values.server.credentialDrivers.kubernetesSecrets.enabled }}
|
|
# The kubernetes-secrets credential driver uses dynamic hashed names in
|
|
# workspace namespaces, so Kubernetes RBAC cannot restrict resourceNames.
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- secrets
|
|
verbs:
|
|
- get
|
|
- create
|
|
- patch
|
|
- delete
|
|
{{- end }}
|
|
{{- if eq $workspaceMode "managed" }}
|
|
# ServiceAccount creation in managed namespaces.
|
|
- apiGroups:
|
|
- ""
|
|
resources:
|
|
- serviceaccounts
|
|
verbs:
|
|
- create
|
|
- get
|
|
{{- if .Values.networkPolicy.enabled }}
|
|
# Apply gateway-only SSH ingress isolation in managed namespaces.
|
|
- apiGroups:
|
|
- networking.k8s.io
|
|
resources:
|
|
- networkpolicies
|
|
verbs:
|
|
- get
|
|
- create
|
|
- patch
|
|
- update
|
|
{{- end }}
|
|
{{- end }}
|