diff --git a/charts/miles-run/templates/_helpers.tpl b/charts/miles-run/templates/_helpers.tpl index ac37693f5e..db361029c0 100644 --- a/charts/miles-run/templates/_helpers.tpl +++ b/charts/miles-run/templates/_helpers.tpl @@ -6,6 +6,15 @@ {{- include "miles-common.selectorLabels" . }} {{- end }} +{{- define "miles-run.accountUsedByAWorker" -}} +{{- $name := .name -}} +{{- range $worker := .context.Values.run.staticWorkers -}} +{{- if eq (default "" $worker.serviceAccountName) $name -}} +used +{{- end -}} +{{- end -}} +{{- end }} + {{- define "miles-run.podDefaults" -}} {{- include "miles-run.podDefaultsFor" (dict "context" . "gated" false) }} {{- end }} diff --git a/charts/miles-run/templates/orchestrator-rbac.yaml b/charts/miles-run/templates/orchestrator-rbac.yaml deleted file mode 100644 index 9c49fb1eac..0000000000 --- a/charts/miles-run/templates/orchestrator-rbac.yaml +++ /dev/null @@ -1,43 +0,0 @@ -{{- if .Values.run.orchestrator.command }} -{{- $name := .Values.run.objectNames.orchestrator }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ $name | quote }} - namespace: {{ .Release.Namespace | quote }} - labels: - {{- include "miles-run.labels" (dict "context" . "component" "orchestrator") | nindent 4 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ $name | quote }} - namespace: {{ .Release.Namespace | quote }} - labels: - {{- include "miles-run.labels" (dict "context" . "component" "orchestrator") | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch", "delete"] - {{- if include "miles-run.autoUninstallEnabled" . }} - - apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["create"] - {{- end }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ $name | quote }} - namespace: {{ .Release.Namespace | quote }} - labels: - {{- include "miles-run.labels" (dict "context" . "component" "orchestrator") | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $name | quote }} -subjects: - - kind: ServiceAccount - name: {{ $name | quote }} - namespace: {{ .Release.Namespace | quote }} -{{- end }} diff --git a/charts/miles-run/templates/orchestrator.yaml b/charts/miles-run/templates/orchestrator.yaml index 2a2efa0378..4d7297d8a4 100644 --- a/charts/miles-run/templates/orchestrator.yaml +++ b/charts/miles-run/templates/orchestrator.yaml @@ -46,7 +46,7 @@ spec: {{- end }} spec: {{- include "miles-run.podDefaults" . | nindent 6 }} - serviceAccountName: {{ $name | quote }} + serviceAccountName: {{ .Values.run.objectNames.platformReadDelete | quote }} restartPolicy: Always containers: - name: orchestrator diff --git a/charts/miles-run/templates/platform-rbac.yaml b/charts/miles-run/templates/platform-rbac.yaml new file mode 100644 index 0000000000..5e88a990a9 --- /dev/null +++ b/charts/miles-run/templates/platform-rbac.yaml @@ -0,0 +1,56 @@ +{{- $context := . }} +{{- $read := .Values.run.objectNames.platformRead }} +{{- $readDelete := .Values.run.objectNames.platformReadDelete }} +{{- $levels := list + (dict "access" "read" "name" $read + "used" (include "miles-run.accountUsedByAWorker" (dict "context" . "name" $read))) + (dict "access" "read-delete" "name" $readDelete + "used" (or .Values.run.orchestrator.command + (include "miles-run.accountUsedByAWorker" (dict "context" . "name" $readDelete)))) }} +{{- range $level := $levels }} +{{- if $level.used }} +{{- $name := $level.name }} +{{- $labels := dict "context" $context "component" (printf "platform-%s" $level.access) }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name | quote }} + namespace: {{ $context.Release.Namespace | quote }} + labels: + {{- include "miles-run.labels" $labels | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name | quote }} + namespace: {{ $context.Release.Namespace | quote }} + labels: + {{- include "miles-run.labels" $labels | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: {{ if eq $level.access "read" }}["get", "list", "watch"]{{ else }}["get", "list", "watch", "delete"]{{ end }} + {{- if and (ne $level.access "read") (include "miles-run.autoUninstallEnabled" $context) }} + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name | quote }} + namespace: {{ $context.Release.Namespace | quote }} + labels: + {{- include "miles-run.labels" $labels | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name | quote }} +subjects: + - kind: ServiceAccount + name: {{ $name | quote }} + namespace: {{ $context.Release.Namespace | quote }} +{{- end }} +{{- end }} diff --git a/charts/miles-run/values.schema.json b/charts/miles-run/values.schema.json index 20d2b2c664..6bf618df0b 100644 --- a/charts/miles-run/values.schema.json +++ b/charts/miles-run/values.schema.json @@ -329,6 +329,18 @@ "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", "type": "string" }, + "platformRead": { + "maxLength": 63, + "minLength": 1, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "type": "string" + }, + "platformReadDelete": { + "maxLength": 63, + "minLength": 1, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "type": "string" + }, "mooncakeMaster": { "maxLength": 63, "minLength": 1, @@ -356,6 +368,8 @@ }, "required": [ "orchestrator", + "platformRead", + "platformReadDelete", "mooncakeMaster", "colocatePairing", "uninstall", diff --git a/charts/miles-run/values.yaml b/charts/miles-run/values.yaml index 2312c43bd9..f6d934dbbd 100644 --- a/charts/miles-run/values.yaml +++ b/charts/miles-run/values.yaml @@ -29,6 +29,8 @@ run: stateFile: /cluster-storage/miles_data/miles-runs/unset/state/orchestrator.state objectNames: orchestrator: miles-run-orchestrator + platformRead: miles-run-platform-read + platformReadDelete: miles-run-platform-read-delete mooncakeMaster: miles-run-mooncake-master colocatePairing: miles-run-colocate-pairing uninstall: miles-run-uninstall diff --git a/miles/ray/specs/inference.py b/miles/ray/specs/inference.py index 6a2ff4f11e..19e9fef7d0 100644 --- a/miles/ray/specs/inference.py +++ b/miles/ray/specs/inference.py @@ -16,7 +16,7 @@ from miles.utils.workers.launch_gate import GATE_PORT_NAME from miles.utils.workers.naming import compute_worker_name from miles.utils.workers.registration.hub import RegistrationHub from miles.utils.workers.registration.reporter import RegistrationReporter -from miles.utils.workers.types import DeployComponent +from miles.utils.workers.types import DeployComponent, PlatformAccess from miles.utils.workers.worker_handle import BaseWorkerHandle from miles.utils.workers.worker_provider.base import BaseWorkerProvider from miles.utils.workers.worker_provider.static import StaticWorkerProvider, parse_host_and_port @@ -44,7 +44,7 @@ INFERENCE_REGISTRATION_REPORTER_WORKER_CLASS = "miles.utils.workers.registration def spec_inference_controller(args) -> ServeWorkerSpec: return ServeWorkerSpec( name=INFERENCE_CONTROLLER_POOL_ID, - needs_platform_read_permission=True, + platform_access=PlatformAccess.READ, port_infos=[], env_var=lambda _ctx: {}, scheduling=SchedulingSpec( @@ -71,6 +71,7 @@ def specs_inference_registration_reporter(args) -> list[ServeWorkerSpec]: ServeWorkerSpec( name=INFERENCE_REGISTRATION_REPORTER_POOL_ID, deploy_component=DeployComponent.INFERENCE, + platform_access=PlatformAccess.READ, port_infos=[], env_var=lambda _ctx: {}, scheduling=SchedulingSpec( diff --git a/miles/ray/specs/train.py b/miles/ray/specs/train.py index 342f439ad9..5a7638f644 100644 --- a/miles/ray/specs/train.py +++ b/miles/ray/specs/train.py @@ -19,7 +19,7 @@ from miles.utils.workers.naming import ( compute_worker_name, format_name_index, ) -from miles.utils.workers.types import DeployComponent, DeploymentIdentity +from miles.utils.workers.types import DeployComponent, DeploymentIdentity, PlatformAccess from miles.utils.workers.worker_handle import BaseWorkerHandle from miles.utils.workers.worker_provider.base import BaseWorkerProvider from miles.utils.workers.worker_provider.static import StaticWorkerProvider, parse_host_and_port @@ -127,7 +127,7 @@ def _compute_spec_trainer_controller( return ServeWorkerSpec( name=compute_trainer_controller_pool_id(trainer_id), deploy_component=DeployComponent.TRAINER, - needs_platform_read_permission=True, + platform_access=PlatformAccess.READ_DELETE, port_infos=[], env_var=lambda _ctx: {}, scheduling=SchedulingSpec( diff --git a/miles/utils/external_utils/command_utils/helm_backend/launcher/values/builder.py b/miles/utils/external_utils/command_utils/helm_backend/launcher/values/builder.py index 5ad15ad400..e1df984f08 100644 --- a/miles/utils/external_utils/command_utils/helm_backend/launcher/values/builder.py +++ b/miles/utils/external_utils/command_utils/helm_backend/launcher/values/builder.py @@ -22,6 +22,7 @@ from miles.utils.external_utils.command_utils.helm_backend.launcher.values.misc from miles.utils.external_utils.command_utils.helm_backend.launcher.values.pool_entry import build_entry from miles.utils.external_utils.command_utils.helm_backend.naming import RunNames from miles.utils.workers.naming import compute_cell_id +from miles.utils.workers.types import PlatformAccess from miles.utils.workers.worker_provider.kubernetes.helm.naming import static_cell_addrs from miles.utils.workers.worker_spec import RPC_PORT_NAME, BaseWorkerSpec, NamedHostAndPorts, ServeWorkerSpec @@ -93,6 +94,8 @@ def _pairing_config(specs: list[BaseWorkerSpec], plan: LaunchPlan) -> PairingCon def _object_names(release: str) -> ObjectNames: return ObjectNames( orchestrator=naming.component_name(release, naming.ORCHESTRATOR_COMPONENT), + platform_read=naming.platform_account_name(release=release, access=PlatformAccess.READ), + platform_read_delete=naming.platform_account_name(release=release, access=PlatformAccess.READ_DELETE), mooncake_master=MooncakeInfo.master_object_name(release), colocate_pairing=naming.component_name(release, _COLOCATE_PAIRING_COMPONENT), uninstall=RunNames.uninstall_job(release=release), diff --git a/miles/utils/external_utils/command_utils/helm_backend/launcher/values/helm_values_types.py b/miles/utils/external_utils/command_utils/helm_backend/launcher/values/helm_values_types.py index 6b6d0ada74..e9abcd042d 100644 --- a/miles/utils/external_utils/command_utils/helm_backend/launcher/values/helm_values_types.py +++ b/miles/utils/external_utils/command_utils/helm_backend/launcher/values/helm_values_types.py @@ -92,6 +92,8 @@ class PoolEntry(ValuesModel): class ObjectNames(ValuesModel): orchestrator: _ObjectName + platform_read: _ObjectName + platform_read_delete: _ObjectName mooncake_master: _ObjectName colocate_pairing: _ObjectName uninstall: _ObjectName diff --git a/miles/utils/external_utils/command_utils/helm_backend/launcher/values/pool_entry.py b/miles/utils/external_utils/command_utils/helm_backend/launcher/values/pool_entry.py index 39310a6621..bb1198a1d3 100644 --- a/miles/utils/external_utils/command_utils/helm_backend/launcher/values/pool_entry.py +++ b/miles/utils/external_utils/command_utils/helm_backend/launcher/values/pool_entry.py @@ -24,6 +24,7 @@ from miles.utils.external_utils.command_utils.helm_backend.launcher.values.place ) from miles.utils.workers.argv_utils import python_argv_prefix from miles.utils.workers.naming import compute_port_name +from miles.utils.workers.types import PlatformAccess from miles.utils.workers.worker_provider.kubernetes.helm import env from miles.utils.workers.worker_spec import ( BaseWorkerSpec, @@ -70,11 +71,7 @@ def build_entry( ports=[PortEntry(name=compute_port_name(port.name), port=port.static_port) for port in spec.port_infos], env=_command_env_of_spec(spec, context, addresses=addresses, is_sub_node=is_sub_node) or None, meta=_meta_of_spec(spec) or None, - service_account_name=( - naming.component_name(plan.release, naming.ORCHESTRATOR_COMPONENT) - if spec.needs_platform_read_permission - else None - ), + service_account_name=_service_account_name(spec, plan=plan), replicas=spec.scheduling.num_cells, size=pods_per_cell if pods_per_cell > 1 else None, resources={"limits": {"nvidia.com/gpu": gpus_per_pod}} if gpus_per_pod else None, @@ -82,6 +79,12 @@ def build_entry( ) +def _service_account_name(spec: BaseWorkerSpec, *, plan: LaunchPlan) -> str | None: + if (access := spec.platform_access) is PlatformAccess.NONE: + return None + return naming.platform_account_name(release=plan.release, access=access) + + def _command_env_of_spec( spec: BaseWorkerSpec, context: LaunchCommandContext, diff --git a/miles/utils/external_utils/command_utils/helm_backend/naming.py b/miles/utils/external_utils/command_utils/helm_backend/naming.py index c82b6009ea..6a359b3f4b 100644 --- a/miles/utils/external_utils/command_utils/helm_backend/naming.py +++ b/miles/utils/external_utils/command_utils/helm_backend/naming.py @@ -8,7 +8,7 @@ from pathlib import Path from pydantic import model_validator from miles.utils.pydantic_utils import FrozenStrictBaseModel -from miles.utils.workers.types import DeployComponent +from miles.utils.workers.types import DeployComponent, PlatformAccess from miles.utils.workers.worker_provider.kubernetes.helm.naming import CHART_NAME, component_name ORCHESTRATOR_COMPONENT = "orchestrator" @@ -31,6 +31,11 @@ _RECORDED_STATE_FILE_KEY = "state_file" _SUPERSEDED_MARKER_SUFFIX = ".superseded" +def platform_account_name(*, release: str, access: PlatformAccess) -> str: + assert access is not PlatformAccess.NONE, "a worker that never reaches the platform runs on no account of its own" + return component_name(release, f"platform-{access.value}") + + class ReleaseName(FrozenStrictBaseModel): run_id: str deploy_component: DeployComponent diff --git a/miles/utils/workers/types.py b/miles/utils/workers/types.py index 1002bb23e9..d0369d287b 100644 --- a/miles/utils/workers/types.py +++ b/miles/utils/workers/types.py @@ -36,6 +36,12 @@ class DeployComponent(Enum): return self in (DeployComponent.TRAINER, DeployComponent.INFERENCE) +class PlatformAccess(Enum): + NONE = "none" + READ = "read" + READ_DELETE = "read-delete" + + class HotRestartComponent(Enum): ORCHESTRATION = "orchestration" ROLLOUT_EXECUTOR = "rollout_executor" diff --git a/miles/utils/workers/worker_spec.py b/miles/utils/workers/worker_spec.py index 1d27fb25f0..caf131d39e 100644 --- a/miles/utils/workers/worker_spec.py +++ b/miles/utils/workers/worker_spec.py @@ -6,7 +6,7 @@ from pydantic import ConfigDict, model_validator from miles.utils.math_utils import exact_div from miles.utils.pydantic_utils import FrozenStrictBaseModel from miles.utils.workers.backend_capability.base import BackendCapability -from miles.utils.workers.types import DeployComponent +from miles.utils.workers.types import DeployComponent, PlatformAccess RPC_PORT_NAME = "rpc" MASTER_PORT_NAME = "master" @@ -102,7 +102,7 @@ class BaseWorkerSpec(FrozenStrictBaseModel): scheduling: SchedulingSpec meta: SpecMetaFn | None = None deploy_component: DeployComponent = DeployComponent.PRIMARY - needs_platform_read_permission: bool = False + platform_access: PlatformAccess = PlatformAccess.NONE @model_validator(mode="after") def _reject_selector_component(self) -> "BaseWorkerSpec": diff --git a/tests/fast/charts/miles_run/test_auto_uninstall.py b/tests/fast/charts/miles_run/test_auto_uninstall.py index a6a067ea64..5d060b41b4 100644 --- a/tests/fast/charts/miles_run/test_auto_uninstall.py +++ b/tests/fast/charts/miles_run/test_auto_uninstall.py @@ -7,6 +7,7 @@ from tests.fast.charts.utils import ( NAMESPACE, RUN_CHART_DIR, RUN_ORCHESTRATOR_NAME, + RUN_PLATFORM_READ_DELETE_NAME, RUN_RELEASE_NAME, RUN_UNINSTALL_JOB_NAME, RUN_UNINSTALL_MANIFEST_NAME, @@ -127,13 +128,13 @@ class TestOrchestratorTrigger: def test_lets_the_orchestrator_create_that_one_job(self): """The wrapper creates it as the run's own account, which is otherwise allowed pods and nothing else.""" - role = named_object(_enabled_objects(), "Role", RUN_ORCHESTRATOR_NAME) + role = named_object(_enabled_objects(), "Role", RUN_PLATFORM_READ_DELETE_NAME) assert {"apiGroups": ["batch"], "resources": ["jobs"], "verbs": ["create"]} in role["rules"] def test_grants_no_job_rights_to_a_run_that_never_creates_one(self): """A run that cannot uninstall itself has no business creating workloads of any kind.""" - role = named_object(render_run(*DISABLE_AUTO_UNINSTALL), "Role", RUN_ORCHESTRATOR_NAME) + role = named_object(render_run(*DISABLE_AUTO_UNINSTALL), "Role", RUN_PLATFORM_READ_DELETE_NAME) assert [rule["apiGroups"] for rule in role["rules"]] == [[""]] diff --git a/tests/fast/charts/miles_run/test_colocate.py b/tests/fast/charts/miles_run/test_colocate.py index a83c313e91..3b707677a9 100644 --- a/tests/fast/charts/miles_run/test_colocate.py +++ b/tests/fast/charts/miles_run/test_colocate.py @@ -50,7 +50,7 @@ POOLS = ( ) PAIRING = "myrun-miles-run-colocate-pairing" -ORCHESTRATOR_ROLE = "myrun-miles-run-orchestrator" +ORCHESTRATOR_ROLE = "myrun-miles-run-platform-read-delete" def layout( diff --git a/tests/fast/charts/miles_run/test_orchestrator.py b/tests/fast/charts/miles_run/test_orchestrator.py index e005bd490f..d51fd8f774 100644 --- a/tests/fast/charts/miles_run/test_orchestrator.py +++ b/tests/fast/charts/miles_run/test_orchestrator.py @@ -13,6 +13,7 @@ from tests.fast.charts.utils import ( ) ORCHESTRATOR = "myrun-miles-run-orchestrator" +ORCHESTRATOR_ACCOUNT = "myrun-miles-run-platform-read-delete" WRAPPER_MODULE = "miles.utils.external_utils.command_utils.helm_backend.orchestrator.wrapper" @@ -98,17 +99,17 @@ class TestOrchestratorIdentity: def test_grants_the_orchestrator_the_pod_rights_observation_and_healing_need(self): """Healing a cell means deleting its pods, and observing one means watching them.""" objects = render_run() - role = named_object(objects, "Role", ORCHESTRATOR) - binding = named_object(objects, "RoleBinding", ORCHESTRATOR) + role = named_object(objects, "Role", ORCHESTRATOR_ACCOUNT) + binding = named_object(objects, "RoleBinding", ORCHESTRATOR_ACCOUNT) assert role["rules"] == [ {"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list", "watch", "delete"]}, {"apiGroups": ["batch"], "resources": ["jobs"], "verbs": ["create"]}, ] - assert binding["roleRef"]["name"] == ORCHESTRATOR - assert binding["subjects"] == [dict(kind="ServiceAccount", name=ORCHESTRATOR, namespace=NAMESPACE)] + assert binding["roleRef"]["name"] == ORCHESTRATOR_ACCOUNT + assert binding["subjects"] == [dict(kind="ServiceAccount", name=ORCHESTRATOR_ACCOUNT, namespace=NAMESPACE)] def test_runs_the_orchestrator_under_the_account_that_binding_names(self): """A role bound to an account nobody runs as grants nothing at all.""" - assert pod_spec_of(render_run(), "StatefulSet", ORCHESTRATOR)["serviceAccountName"] == ORCHESTRATOR - assert named_object(render_run(), "ServiceAccount", ORCHESTRATOR)["metadata"]["namespace"] == NAMESPACE + assert pod_spec_of(render_run(), "StatefulSet", ORCHESTRATOR)["serviceAccountName"] == ORCHESTRATOR_ACCOUNT + assert named_object(render_run(), "ServiceAccount", ORCHESTRATOR_ACCOUNT)["metadata"]["namespace"] == NAMESPACE diff --git a/tests/fast/charts/miles_run/test_platform_rbac.py b/tests/fast/charts/miles_run/test_platform_rbac.py new file mode 100644 index 0000000000..d50da2b2ea --- /dev/null +++ b/tests/fast/charts/miles_run/test_platform_rbac.py @@ -0,0 +1,96 @@ +import json +from typing import Any + +from tests.fast.charts.utils import ( + RUN_ORCHESTRATOR_NAME, + RUN_PLATFORM_READ_DELETE_NAME, + RUN_PLATFORM_READ_NAME, + RUN_RELEASE_NAME, + named_object, + objects_of_kind, + pod_spec_of, + render_run, + requires_helm, + with_object_names, +) + +_READER = { + "name": "inference-registration-reporter", + "command": ["x"], + "serviceAccountName": RUN_PLATFORM_READ_NAME, +} +_DELETER = { + "name": "trainer-controller-actor", + "command": ["x"], + "serviceAccountName": RUN_PLATFORM_READ_DELETE_NAME, +} +_PLAIN = {"name": "inference-engine", "command": ["x"]} + + +def _render_without_orchestrator(*workers: dict[str, Any]) -> list[dict[str, Any]]: + return render_run( + "--set-json", + "run.orchestrator.command=[]", + "--set-json", + f"run.staticWorkers={json.dumps(with_object_names(list(workers)))}", + ) + + +@requires_helm +class TestTheAccountsAReleaseGrantsItsPlatformClients: + def test_grants_the_read_account_to_a_release_whose_client_only_reads_pods(self): + """A split run's inference release carries the registration reporter, which lists pods to report them.""" + objects = _render_without_orchestrator(_READER) + + assert named_object(objects, "ServiceAccount", RUN_PLATFORM_READ_NAME) + assert named_object(objects, "RoleBinding", RUN_PLATFORM_READ_NAME) + assert named_object(objects, "Role", RUN_PLATFORM_READ_NAME)["rules"] == [ + {"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list", "watch"]} + ] + assert ( + pod_spec_of(objects, "StatefulSet", f"{RUN_RELEASE_NAME}-miles-run-inference-registration-reporter")[ + "serviceAccountName" + ] + == RUN_PLATFORM_READ_NAME + ) + + def test_grants_the_read_delete_account_to_a_release_whose_client_also_deletes_pods(self): + """A split run's trainer release suspends cells by deleting pods, which the read-only account refuses.""" + objects = _render_without_orchestrator(_DELETER) + + assert named_object(objects, "ServiceAccount", RUN_PLATFORM_READ_DELETE_NAME) + assert named_object(objects, "RoleBinding", RUN_PLATFORM_READ_DELETE_NAME) + assert named_object(objects, "Role", RUN_PLATFORM_READ_DELETE_NAME)["rules"] == [ + {"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list", "watch", "delete"]}, + {"apiGroups": ["batch"], "resources": ["jobs"], "verbs": ["create"]}, + ] + + def test_grants_a_reader_none_of_the_rights_the_read_delete_account_carries(self): + """A reporter may read pod state without inheriting deletion or job-creation rights.""" + objects = _render_without_orchestrator(_READER) + + assert not any( + obj["metadata"]["name"] == RUN_PLATFORM_READ_DELETE_NAME for obj in objects_of_kind(objects, "Role") + ) + + def test_grants_nothing_to_a_release_no_platform_client_lives_in(self): + """A release of plain engines binds no account, so rights over pods would be handed to nobody.""" + objects = _render_without_orchestrator(_PLAIN) + + assert objects_of_kind(objects, "ServiceAccount") == [] + assert objects_of_kind(objects, "Role") == [] + assert objects_of_kind(objects, "RoleBinding") == [] + + def test_still_grants_the_release_that_runs_the_orchestration_script(self): + """The orchestrator deletes pods to heal cells, and it is the reason this rbac existed at all.""" + objects = render_run() + + assert named_object(objects, "ServiceAccount", RUN_PLATFORM_READ_DELETE_NAME) + assert named_object(objects, "Role", RUN_PLATFORM_READ_DELETE_NAME)["rules"] == [ + {"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list", "watch", "delete"]}, + {"apiGroups": ["batch"], "resources": ["jobs"], "verbs": ["create"]}, + ] + assert ( + pod_spec_of(objects, "StatefulSet", RUN_ORCHESTRATOR_NAME)["serviceAccountName"] + == RUN_PLATFORM_READ_DELETE_NAME + ) diff --git a/tests/fast/charts/utils.py b/tests/fast/charts/utils.py index d3b40c75c8..6409b5f361 100644 --- a/tests/fast/charts/utils.py +++ b/tests/fast/charts/utils.py @@ -25,6 +25,8 @@ RELEASE_NAME = "miles-workbench-myuser" UNINSTALLER_SERVICE_ACCOUNT = "miles-uninstaller" RUN_RELEASE_NAME = "myrun" RUN_ORCHESTRATOR_NAME = f"{RUN_RELEASE_NAME}-miles-run-orchestrator" +RUN_PLATFORM_READ_NAME = f"{RUN_RELEASE_NAME}-miles-run-platform-read" +RUN_PLATFORM_READ_DELETE_NAME = f"{RUN_RELEASE_NAME}-miles-run-platform-read-delete" RUN_UNINSTALL_JOB_NAME = f"{RUN_RELEASE_NAME}-miles-run-uninstall" RUN_UNINSTALL_MANIFEST_NAME = f"{RUN_RELEASE_NAME}-miles-run-uninstall-manifest" RUN_ID = "260101-000000-000" @@ -100,6 +102,10 @@ def run_helm_template_run(*args: str) -> subprocess.CompletedProcess: "--set", f"run.objectNames.orchestrator={RUN_ORCHESTRATOR_NAME}", "--set", + f"run.objectNames.platformRead={RUN_PLATFORM_READ_NAME}", + "--set", + f"run.objectNames.platformReadDelete={RUN_PLATFORM_READ_DELETE_NAME}", + "--set", f"run.objectNames.mooncakeMaster={RUN_RELEASE_NAME}-miles-run-mooncake-master", "--set", f"run.objectNames.colocatePairing={RUN_RELEASE_NAME}-miles-run-colocate-pairing", diff --git a/tests/fast/ray/specs/test_inference.py b/tests/fast/ray/specs/test_inference.py index e4415bfbc5..76f3644037 100644 --- a/tests/fast/ray/specs/test_inference.py +++ b/tests/fast/ray/specs/test_inference.py @@ -29,6 +29,7 @@ from miles.ray.specs.inference import ( spec_inference_controller, spec_session_server, specs_inference_engine, + specs_inference_registration_reporter, specs_router, ) from miles.rollout.session.config import SessionServerConfig @@ -38,6 +39,7 @@ from miles.utils.external_utils.command_utils.helm_backend.launcher.values.misc from miles.utils.function_registry import load_function from miles.utils.workers.argv_utils import parse_config_argv from miles.utils.workers.registration.hub import RegistrationHub +from miles.utils.workers.types import PlatformAccess from miles.utils.workers.worker_provider.static import StaticWorkerProvider from miles.utils.workers.worker_spec import ( RPC_PORT_NAME, @@ -1199,6 +1201,18 @@ class TestSpecInferenceController: assert load_function(spec.worker_class) is InferenceController + def test_it_declares_only_the_platform_reads_its_provider_performs(self, tmp_path): + """Watching engine pods needs reads, while deleting them remains outside this worker's capability.""" + spec = spec_inference_controller(self._args(tmp_path)) + + assert spec.platform_access is PlatformAccess.READ + + def test_the_registration_reporter_declares_only_platform_reads(self, tmp_path): + """The reporter watches engine pods, so it needs reads and none of the orchestrator's other rights.""" + (spec,) = specs_inference_registration_reporter(self._args(tmp_path, deploy_component="inference")) + + assert spec.platform_access is PlatformAccess.READ + def test_the_worker_name_is_stable(self): """The driver looks the controller up by name, so this name is part of the release's contract.""" assert inference_controller_worker_name() == "inference-controller-00000-00000" diff --git a/tests/fast/ray/specs/test_train.py b/tests/fast/ray/specs/test_train.py index 0fcb2c4bad..91cf925850 100644 --- a/tests/fast/ray/specs/test_train.py +++ b/tests/fast/ray/specs/test_train.py @@ -33,6 +33,7 @@ from miles.ray.train_actor import TrainRayActor from miles.utils.external_utils.command_utils.helm_backend.launcher.values.builder import build_values from miles.utils.external_utils.command_utils.helm_backend.launcher.values.misc import SECTION_OF_CATEGORY, LaunchPlan from miles.utils.workers.rpc.common.metadata import _find_rpc_config +from miles.utils.workers.types import PlatformAccess from miles.utils.workers.worker_spec import WorkerCtorContext @@ -626,6 +627,12 @@ class TestSpecTrainerController: assert spec.worker_class == TRAINER_CONTROLLER_WORKER_CLASS + def test_it_keeps_the_platform_delete_capability_used_to_suspend_cells(self): + """Fault injection deletes trainer pods, so moving readers off the orchestrator account must not break it.""" + spec = specs_trainer_controller(_make_args())[0] + + assert spec.platform_access is PlatformAccess.READ_DELETE + def test_the_worker_and_cell_names_are_stable(self): """The driver looks the controller up by name, so these names are part of the release's contract.""" assert trainer_controller_worker_name("actor") == "trainer-controller-actor-00000-00000" diff --git a/tests/fast/tools/kubernetes/test_generate_chart_schema.py b/tests/fast/tools/kubernetes/test_generate_chart_schema.py index 7d80553852..9f4a21f78c 100644 --- a/tests/fast/tools/kubernetes/test_generate_chart_schema.py +++ b/tests/fast/tools/kubernetes/test_generate_chart_schema.py @@ -52,6 +52,8 @@ def _minimal_run_values() -> dict[str, Any]: "stateFile": "/cluster-storage/miles_data/miles-runs/260101-000000-000/state/orchestrator.state", "objectNames": { "orchestrator": "r-miles-run-orchestrator", + "platformRead": "r-miles-run-platform-read", + "platformReadDelete": "r-miles-run-platform-read-delete", "mooncakeMaster": "r-miles-run-mooncake-master", "colocatePairing": "r-miles-run-colocate-pairing", "uninstall": "r-miles-run-uninstall", diff --git a/tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_builder.py b/tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_builder.py index 1ac2c8c081..9ddfad03e5 100644 --- a/tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_builder.py +++ b/tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_builder.py @@ -222,6 +222,8 @@ class TestObjectNames: """A schema-required field the launcher only sometimes writes would refuse half the runs.""" assert build_values([], LAYOUT).as_values()["run"]["objectNames"] == { "orchestrator": "r-miles-run-orchestrator", + "platformRead": "r-miles-run-platform-read", + "platformReadDelete": "r-miles-run-platform-read-delete", "mooncakeMaster": "r-miles-run-mooncake-master", "colocatePairing": "r-miles-run-colocate-pairing", "uninstall": "r-miles-run-uninstall", diff --git a/tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_pool_entry.py b/tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_pool_entry.py index a56c07780f..bed51749a8 100644 --- a/tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_pool_entry.py +++ b/tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_pool_entry.py @@ -13,6 +13,7 @@ from miles.utils.external_utils.command_utils.helm_backend.launcher.values impor from miles.utils.external_utils.command_utils.helm_backend.launcher.values.builder import build_values from miles.utils.external_utils.command_utils.helm_backend.launcher.values.helm_values_types import PortEntry from miles.utils.external_utils.command_utils.helm_backend.launcher.values.misc import LaunchPlan +from miles.utils.workers.types import PlatformAccess from miles.utils.workers.worker_spec import BaseWorkerSpec, SchedulingSpec STAMP = "2026-08-12T09:00:00+00:00" @@ -180,13 +181,21 @@ class TestTheRestartStamp: class TestTheAccountAPoolRunsUnder: - def test_a_pool_that_observes_the_platform_gets_the_account_that_may_read_it(self): + def test_a_pool_that_only_reads_the_platform_gets_the_account_that_may_read_it(self): """Only these workers reconcile against pods, and the namespace default cannot list one.""" - spec = session_server(num_cells=1).model_copy(update={"needs_platform_read_permission": True}) + spec = session_server(num_cells=1).model_copy(update={"platform_access": PlatformAccess.READ}) entry = build_values([spec], LAYOUT).as_values()["run"]["staticWorkers"][0] - assert entry["serviceAccountName"] == "r-miles-run-orchestrator" + assert entry["serviceAccountName"] == "r-miles-run-platform-read" + + def test_a_pool_that_also_deletes_platform_pods_gets_the_account_that_may_delete_them(self): + """A trainer controller suspends cells by deleting pods and must retain that existing capability.""" + spec = session_server(num_cells=1).model_copy(update={"platform_access": PlatformAccess.READ_DELETE}) + + entry = build_values([spec], LAYOUT).as_values()["run"]["staticWorkers"][0] + + assert entry["serviceAccountName"] == "r-miles-run-platform-read-delete" def test_every_other_pool_stays_on_the_namespace_default(self): """An engine talks to no api server, and an account it never needs is one it could misuse.""" @@ -196,7 +205,7 @@ class TestTheAccountAPoolRunsUnder: def test_refuses_a_pool_whose_template_renders_no_account_at_all(self): """The engine template ignores the key, so the pod would run on the default and 403 far from here.""" - spec = engine(num_cells=1, gpus_per_engine=8).model_copy(update={"needs_platform_read_permission": True}) + spec = engine(num_cells=1, gpus_per_engine=8).model_copy(update={"platform_access": PlatformAccess.READ}) with pytest.raises(AssertionError, match="renders a service account"): build_values([spec], LAYOUT).as_values() diff --git a/tests/fast/utils/external_utils/command_utils/helm_backend/test_naming.py b/tests/fast/utils/external_utils/command_utils/helm_backend/test_naming.py index 671539de41..f4da5063be 100644 --- a/tests/fast/utils/external_utils/command_utils/helm_backend/test_naming.py +++ b/tests/fast/utils/external_utils/command_utils/helm_backend/test_naming.py @@ -1,3 +1,12 @@ +import json +import re +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from miles.utils.external_utils.command_utils.helm_backend import naming from miles.utils.external_utils.command_utils.helm_backend.naming import ( RunFiles, _orchestrator_state_path, @@ -14,6 +23,12 @@ def _write(path, status: OrchestratorStatus, *, exit_code: int | None = None) -> OrchestratorState(status=status, exit_code=exit_code).write(path) +def _record(run_directory, launch_token: str, state_file) -> None: + path = Path(run_directory) / "launches" / f"launch-{launch_token}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"state_file": str(state_file)})) + + def _state_file(tmp_path): return _orchestrator_state_path(tmp_path, "260101-000000-000001") @@ -48,20 +63,70 @@ class TestRunDir: class TestLatestExitFile: - def test_names_no_file_before_a_launch_has_written_one(self, tmp_path): + def test_names_no_file_before_a_launch_has_recorded_one(self, tmp_path): """A run directory a launch has only just created holds no verdict to collect.""" assert RunFiles.latest_state_file(run_directory=tmp_path) is None + def test_names_the_newest_launch_s_state_file_before_it_is_written(self, tmp_path): + """A generation whose pods never came up must not hand the previous generation's verdict over.""" + _record(tmp_path, "260101-000100-000002", _orchestrator_state_path(tmp_path, "260101-000100-000002")) + pending = _orchestrator_state_path(tmp_path, "260101-000200-000001") + _record(tmp_path, "260101-000200-000001", pending) + + assert RunFiles.latest_state_file(run_directory=tmp_path) == pending + def test_picks_the_newest_launch_rather_than_the_newest_write(self, tmp_path): """An earlier launch torn down after a later one started writes last, and its verdict is not the run's.""" later = _orchestrator_state_path(tmp_path, "260101-000200-000001") earlier = _orchestrator_state_path(tmp_path, "260101-000100-000002") _write(later, OrchestratorStatus.EXITED, exit_code=0) _write(earlier, OrchestratorStatus.EXITED, exit_code=1) + _record(tmp_path, "260101-000100-000002", earlier) + _record(tmp_path, "260101-000200-000001", later) assert RunFiles.latest_state_file(run_directory=tmp_path) == later +def _mint_token(monkeypatch: pytest.MonkeyPatch, *, when: datetime, tail: int) -> str: + monkeypatch.setattr(naming, "datetime", SimpleNamespace(now=lambda: when)) + monkeypatch.setattr(naming.random, "Random", lambda: SimpleNamespace(randint=lambda low, high: tail)) + return naming._new_launch_token() + + +class TestNewLaunchToken: + def test_two_launches_of_one_second_order_by_the_time_they_were_minted(self, monkeypatch): + """Everything that picks the newest launch sorts these names, and the random tail ordered them by chance.""" + earlier = _mint_token(monkeypatch, when=datetime(2026, 1, 1, 0, 0, 0, 1), tail=999999) + later = _mint_token(monkeypatch, when=datetime(2026, 1, 1, 0, 0, 0, 2), tail=0) + + assert earlier < later + + def test_a_token_carries_the_microsecond_it_was_minted_at(self, monkeypatch): + """Two launches of one run are apart by microseconds, which is the resolution the name has to keep.""" + token = _mint_token(monkeypatch, when=datetime(2026, 1, 1, 0, 0, 0, 123), tail=7) + + assert token == "260101-000000-000123-000007" + + def test_two_tokens_minted_in_one_microsecond_still_differ(self, monkeypatch): + """The random tail no longer decides the order, but it is still what keeps two names apart.""" + when = datetime(2026, 1, 1, 0, 0, 0, 5) + + assert _mint_token(monkeypatch, when=when, tail=1) != _mint_token(monkeypatch, when=when, tail=2) + + def test_every_part_of_a_real_token_is_fixed_width(self): + """A part that can be shorter sorts before a longer one whatever time it names.""" + assert re.fullmatch(r"\d{6}-\d{6}-\d{6}-\d{6}", naming._new_launch_token()) + + def test_a_real_token_never_names_a_time_before_the_one_minted_earlier(self): + """The clock part is what the newest-launch lookup reads, and it has to be non-decreasing.""" + stamp_length = len("260101-000000-000000") + + first = naming._new_launch_token() + second = naming._new_launch_token() + + assert first[:stamp_length] <= second[:stamp_length] + + class TestSupersededMarker: def test_the_marker_sits_beside_the_state_file_it_supersedes(self): """Both the launcher that writes it and the orchestrator that reads it know only that path.""" diff --git a/tests/snapshots/charts/miles-run/typical-values.yaml b/tests/snapshots/charts/miles-run/typical-values.yaml index 14ba6ef3df..b0e91e7776 100644 --- a/tests/snapshots/charts/miles-run/typical-values.yaml +++ b/tests/snapshots/charts/miles-run/typical-values.yaml @@ -196,6 +196,8 @@ run: colocatePairing: myrun-miles-run-colocate-pairing mooncakeMaster: myrun-miles-run-mooncake-master orchestrator: myrun-miles-run-orchestrator + platformRead: myrun-miles-run-platform-read + platformReadDelete: myrun-miles-run-platform-read-delete uninstall: myrun-miles-run-uninstall uninstallManifest: myrun-miles-run-uninstall-manifest orchestrator: @@ -247,7 +249,7 @@ run: - name: rpc port: 8000 replicas: 1 - serviceAccountName: myrun-miles-run-orchestrator + serviceAccountName: myrun-miles-run-platform-read - command: - - -m @@ -279,7 +281,7 @@ run: - -m - miles.rollout.session.server - --config-json - - '{"host":"0.0.0.0","port":8000,"instance_id":"0123456789abcdef-0","backend_url":"http://myrun-miles-run-inference-router-0-0.myrun-miles-run-inference-router-0:8000","timeout":null,"hf_checkpoint":"/typical-model","chat_template_path":null,"tito_model":"default","apply_chat_template_kwargs":{},"use_rollout_routing_replay":false,"use_rollout_indexer_replay":false,"use_sampling_support_replay":false,"sglang_speculative_algorithm":null,"num_layers":36,"moe_router_topk":2,"save_debug_trajectory_data":null,"lora_rank":0,"lora_adapter_path":null,"lora_train_only":false,"use_session_server":true,"session_message_matcher":"strict","pause_generation_mode":"retract","session_sample_picker_path":"miles.rollout.session.v2.picker_hub.drop_retries","session_sample_postprocessor_path":"miles.rollout.session.v2.postprocessor_hub.default_postprocess"}' + - '{"host":"0.0.0.0","port":8000,"instance_id":"0123456789abcdef-0","backend_url":"http://myrun-miles-run-inference-router-0-0.myrun-miles-run-inference-router-0:8000","timeout":null,"hf_checkpoint":"/typical-model","chat_template_path":null,"tito_model":"default","apply_chat_template_kwargs":{},"use_rollout_routing_replay":false,"use_rollout_indexer_replay":false,"use_sampling_support_replay":false,"sglang_speculative_algorithm":null,"num_layers":36,"moe_router_topk":2,"save_debug_trajectory_data":null,"lora_rank":0,"lora_adapter_path":null,"lora_train_only":false,"use_session_server":true,"session_message_matcher":"strict","pause_generation_mode":"retract","session_sample_picker_path":"miles.rollout.session.v2.picker_hub.drop_same_prompt_retries","session_sample_postprocessor_path":"miles.rollout.session.v2.postprocessor_hub.default_postprocess"}' name: session-server objectName: myrun-miles-run-session-server poolId: session-server @@ -307,7 +309,7 @@ run: - name: rpc port: 8000 replicas: 1 - serviceAccountName: myrun-miles-run-orchestrator + serviceAccountName: myrun-miles-run-platform-read-delete trainerEngines: - command: - bash diff --git a/tests/snapshots/charts/miles-run/typical.yaml b/tests/snapshots/charts/miles-run/typical.yaml index 4846951679..2c56ccbc01 100644 --- a/tests/snapshots/charts/miles-run/typical.yaml +++ b/tests/snapshots/charts/miles-run/typical.yaml @@ -13,17 +13,31 @@ metadata: app.kubernetes.io/version: "0.1.0" app.kubernetes.io/managed-by: "Helm" --- -# Source: miles-run/templates/orchestrator-rbac.yaml +# Source: miles-run/templates/platform-rbac.yaml apiVersion: v1 kind: ServiceAccount metadata: - name: "myrun-miles-run-orchestrator" + name: "myrun-miles-run-platform-read" namespace: "myns" labels: helm.sh/chart: "miles-run-0.1.0" app.kubernetes.io/name: "miles-run" app.kubernetes.io/instance: "myrun" - app.kubernetes.io/component: "orchestrator" + app.kubernetes.io/component: "platform-read" + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: "Helm" +--- +# Source: miles-run/templates/platform-rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "myrun-miles-run-platform-read-delete" + namespace: "myns" + labels: + helm.sh/chart: "miles-run-0.1.0" + app.kubernetes.io/name: "miles-run" + app.kubernetes.io/instance: "myrun" + app.kubernetes.io/component: "platform-read-delete" app.kubernetes.io/version: "0.1.0" app.kubernetes.io/managed-by: "Helm" --- @@ -97,17 +111,35 @@ rules: resources: ["pods"] verbs: ["get", "list", "watch", "patch", "update"] --- -# Source: miles-run/templates/orchestrator-rbac.yaml +# Source: miles-run/templates/platform-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: "myrun-miles-run-orchestrator" + name: "myrun-miles-run-platform-read" namespace: "myns" labels: helm.sh/chart: "miles-run-0.1.0" app.kubernetes.io/name: "miles-run" app.kubernetes.io/instance: "myrun" - app.kubernetes.io/component: "orchestrator" + app.kubernetes.io/component: "platform-read" + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: "Helm" +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +# Source: miles-run/templates/platform-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "myrun-miles-run-platform-read-delete" + namespace: "myns" + labels: + helm.sh/chart: "miles-run-0.1.0" + app.kubernetes.io/name: "miles-run" + app.kubernetes.io/instance: "myrun" + app.kubernetes.io/component: "platform-read-delete" app.kubernetes.io/version: "0.1.0" app.kubernetes.io/managed-by: "Helm" rules: @@ -140,26 +172,48 @@ subjects: name: "myrun-miles-run-colocate-pairing" namespace: "myns" --- -# Source: miles-run/templates/orchestrator-rbac.yaml +# Source: miles-run/templates/platform-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: "myrun-miles-run-orchestrator" + name: "myrun-miles-run-platform-read" namespace: "myns" labels: helm.sh/chart: "miles-run-0.1.0" app.kubernetes.io/name: "miles-run" app.kubernetes.io/instance: "myrun" - app.kubernetes.io/component: "orchestrator" + app.kubernetes.io/component: "platform-read" app.kubernetes.io/version: "0.1.0" app.kubernetes.io/managed-by: "Helm" roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: "myrun-miles-run-orchestrator" + name: "myrun-miles-run-platform-read" subjects: - kind: ServiceAccount - name: "myrun-miles-run-orchestrator" + name: "myrun-miles-run-platform-read" + namespace: "myns" +--- +# Source: miles-run/templates/platform-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "myrun-miles-run-platform-read-delete" + namespace: "myns" + labels: + helm.sh/chart: "miles-run-0.1.0" + app.kubernetes.io/name: "miles-run" + app.kubernetes.io/instance: "myrun" + app.kubernetes.io/component: "platform-read-delete" + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: "Helm" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "myrun-miles-run-platform-read-delete" +subjects: + - kind: ServiceAccount + name: "myrun-miles-run-platform-read-delete" namespace: "myns" --- @@ -350,7 +404,7 @@ spec: - name: pairing image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" @@ -446,13 +500,13 @@ spec: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - serviceAccountName: "myrun-miles-run-orchestrator" + serviceAccountName: "myrun-miles-run-platform-read-delete" restartPolicy: Always containers: - name: orchestrator image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" @@ -569,12 +623,12 @@ spec: - name: worker image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" - name: "cluster-storage" - mountPath: "" + mountPath: "/root/miles" subPath: "myuser/miles" - name: "cluster-storage" mountPath: "/root/Megatron-LM" @@ -677,13 +731,13 @@ spec: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - serviceAccountName: "myrun-miles-run-orchestrator" + serviceAccountName: "myrun-miles-run-platform-read" restartPolicy: Always containers: - name: worker image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" @@ -796,7 +850,7 @@ spec: - name: worker image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" @@ -915,7 +969,7 @@ spec: - name: worker image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" @@ -935,7 +989,7 @@ spec: - "-m" - "miles.rollout.session.server" - "--config-json" - - "{\"host\":\"0.0.0.0\",\"port\":8000,\"instance_id\":\"0123456789abcdef-0\",\"backend_url\":\"http://myrun-miles-run-inference-router-0-0.myrun-miles-run-inference-router-0:8000\",\"timeout\":null,\"hf_checkpoint\":\"/tests/fast/charts/miles_run/typical-model\",\"chat_template_path\":null,\"tito_model\":\"default\",\"apply_chat_template_kwargs\":{},\"use_rollout_routing_replay\":false,\"use_rollout_indexer_replay\":false,\"use_sampling_support_replay\":false,\"sglang_speculative_algorithm\":null,\"num_layers\":36,\"moe_router_topk\":2,\"save_debug_trajectory_data\":null,\"lora_rank\":0,\"lora_adapter_path\":null,\"lora_train_only\":false,\"use_session_server\":true,\"session_message_matcher\":\"strict\",\"pause_generation_mode\":\"retract\",\"session_sample_picker_path\":\"miles.rollout.session.v2.picker_hub.drop_retries\",\"session_sample_postprocessor_path\":\"miles.rollout.session.v2.postprocessor_hub.default_postprocess\"}" + - "{\"host\":\"0.0.0.0\",\"port\":8000,\"instance_id\":\"0123456789abcdef-0\",\"backend_url\":\"http://myrun-miles-run-inference-router-0-0.myrun-miles-run-inference-router-0:8000\",\"timeout\":null,\"hf_checkpoint\":\"/typical-model\",\"chat_template_path\":null,\"tito_model\":\"default\",\"apply_chat_template_kwargs\":{},\"use_rollout_routing_replay\":false,\"use_rollout_indexer_replay\":false,\"use_sampling_support_replay\":false,\"sglang_speculative_algorithm\":null,\"num_layers\":36,\"moe_router_topk\":2,\"save_debug_trajectory_data\":null,\"lora_rank\":0,\"lora_adapter_path\":null,\"lora_train_only\":false,\"use_session_server\":true,\"session_message_matcher\":\"strict\",\"pause_generation_mode\":\"retract\",\"session_sample_picker_path\":\"miles.rollout.session.v2.picker_hub.drop_same_prompt_retries\",\"session_sample_postprocessor_path\":\"miles.rollout.session.v2.postprocessor_hub.default_postprocess\"}" env: - name: MILES_CELL_INDEX valueFrom: @@ -1016,13 +1070,13 @@ spec: - effect: NoSchedule key: nvidia.com/gpu operator: Exists - serviceAccountName: "myrun-miles-run-orchestrator" + serviceAccountName: "myrun-miles-run-platform-read-delete" restartPolicy: Always containers: - name: worker image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" @@ -1138,7 +1192,7 @@ spec: - name: "engine" image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" @@ -1347,7 +1401,7 @@ spec: - name: "engine" image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" @@ -1547,7 +1601,7 @@ spec: - name: "trainer" image: "myregistry.example/myteam/miles:v0.9.3-cu128" imagePullPolicy: "IfNotPresent" - workingDir: "" + workingDir: "/root/miles" volumeMounts: - name: "cluster-storage" mountPath: "/cluster-storage" diff --git a/tests/snapshots/helm_backend/kubernetes_launch.txt b/tests/snapshots/helm_backend/kubernetes_launch.txt index 3a416659fe..3c26653ca8 100644 --- a/tests/snapshots/helm_backend/kubernetes_launch.txt +++ b/tests/snapshots/helm_backend/kubernetes_launch.txt @@ -15,100 +15,103 @@ run: env: CUDA_DEVICE_MAX_CONNECTIONS: '1' PYTHONUNBUFFERED: '1' - stateFile: /cluster-storage/miles_data/miles-runs/260101-000000-000/state/orchestrator-260101-000000-000001.state - id: 260101-000000-000 + id: '260101-000000-000' inferenceEngines: - command: - - python - - -m - - sglang.launch_server - - --node-rank - - $(LWS_WORKER_INDEX) - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):9000 + - 'python' + - '-m' + - 'sglang.launch_server' + - '--node-rank' + - '$(LWS_WORKER_INDEX)' + - '--dist-init-addr' + - '$(LWS_LEADER_ADDRESS):9000' env: NVSHMEM_DISABLE_NCCL: '1' meta: - gpu_ids: 0,1,2,3,4,5,6,7 - name: inference-engine-0-0 - objectName: miles-run-260101-000000-000-all-inference-engine-0-0 + gpu_ids: '0,1,2,3,4,5,6,7' + name: 'inference-engine-0-0' + objectName: 'miles-run-26-98cf6da492ec-inference-engine-0-0' + poolId: 'inference-engine-0-0' ports: - - name: primary + - name: 'primary' port: 8000 - - name: dist-init + - name: 'dist-init' port: 9000 replicas: 2 resources: limits: nvidia.com/gpu: 8 size: 2 - poolId: inference-engine-0-0 + launchRecord: '/cluster-storage/miles_data/miles-runs/260101-000000-000/launches/launch-260101-000000-000001.json' objectNames: - colocatePairing: miles-run-260101-000000-000-all-colocate-pairing - mooncakeMaster: miles-run-260101-000000-000-all-mooncake-master - orchestrator: miles-run-260101-000000-000-all-orchestrator - uninstall: miles-run-260101-000000-000-all-uninstall - uninstallManifest: miles-run-260101-000000-000-all-uninstall-manifest + colocatePairing: 'miles-run-260101-98cf6da492ec-colocate-pairing' + mooncakeMaster: 'miles-run-260101-98cf6da492ec-mooncake-master' + orchestrator: 'miles-run-260101-000000-000-all-orchestrator' + platformRead: 'miles-run-260101-000000-000-all-platform-read' + platformReadDelete: 'miles-run-26-98cf6da492ec-platform-read-delete' + uninstall: 'miles-run-260101-000000-000-all-uninstall' + uninstallManifest: 'miles-run-2601-98cf6da492ec-uninstall-manifest' orchestrator: command: - - python - - /repo/train.py - - --rollout-num-gpus + - 'python' + - '/repo/train.py' + - '--rollout-num-gpus' - '8' - - --cluster-backend - - kubernetes - - --run-uuid - - f52ecf8e9d7d4889 + - '--cluster-backend' + - 'kubernetes' + - '--run-uuid' + - 'f52ecf8e9d7d4889' + stateFile: '/cluster-storage/miles_data/miles-runs/260101-000000-000/state/orchestrator-260101-000000-000001.state' staticWorkers: - command: - - python - - -m - - sglang_router.launch_router - - --host - - 0.0.0.0 - - --port + - 'python' + - '-m' + - 'sglang_router.launch_router' + - '--host' + - '0.0.0.0' + - '--port' - '30000' - name: inference-router-0 - objectName: miles-run-260101-000000-000-all-inference-router-0 + name: 'inference-router-0' + objectName: 'miles-run-2601-98cf6da492ec-inference-router-0' + poolId: 'inference-router-0' ports: - - name: primary + - name: 'primary' port: 30000 replicas: 1 - poolId: inference-router-0 trainerEngines: - command: - - - - -m - - miles.utils.workers.process_supervisor - - --num-subprocesses + - '' + - '-m' + - 'miles.utils.workers.process_supervisor' + - '--num-subprocesses' - '8' - - -- - - - - -m - - miles.utils.workers.serving.serve - - --specs - - miles.ray.specs.entrypoint.compute_specs_from_argv - - --pool-id - - trainer-engine-actor - - -- - - --rollout-num-gpus + - '--' + - '' + - '-m' + - 'miles.utils.workers.serving.serve' + - '--specs' + - 'miles.ray.specs.entrypoint.compute_specs_from_argv' + - '--pool-id' + - 'trainer-engine-actor' + - '--' + - '--rollout-num-gpus' - '8' - - --cluster-backend - - kubernetes - - --run-uuid - - f52ecf8e9d7d4889 + - '--cluster-backend' + - 'kubernetes' + - '--run-uuid' + - 'f52ecf8e9d7d4889' meta: - gpu_ids: 0,1,2,3,4,5,6,7 - name: trainer-engine-actor - objectName: miles-run-260101-000000-000-all-trainer-engine-actor + gpu_ids: '0,1,2,3,4,5,6,7' + name: 'trainer-engine-actor' + objectName: 'miles-run-26-98cf6da492ec-trainer-engine-actor' + poolId: 'trainer-engine-actor' ports: - - name: master + - name: 'master' port: 9000 - - name: rpc + - name: 'rpc' port: 8000 replicas: 2 resources: limits: nvidia.com/gpu: 8 - poolId: trainer-engine-actor