trending/miles
1
0
Fork 0
mirror of https://github.com/radixark/miles.git synced 2026-10-01 23:06:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Give every worker that watches pods the platform read access it needs (#3056)

Browse Source
This commit is contained in:
fzyzcjy
2026-09-26 20:36:27 +08:00
committed by GitHub
parent 4ce14ed227
commit 3d3fb97bac
28 changed files with 480 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
charts/miles-run/templates/_helpers.tpl
+9
View File
@@ -6,6 +6,15 @@
{{- include "miles-common.selectorLabels" . }}
{{- end }}
{{- define "miles-run.accountUsedByAWorker" -}}
{{- $name := .name -}}
{{- range $worker := .context.Values.run.staticWorkers -}}
{{- if eq (default "" $worker.serviceAccountName) $name -}}
used
{{- end -}}
{{- end -}}
{{- end }}
{{- define "miles-run.podDefaults" -}}
{{- include "miles-run.podDefaultsFor" (dict "context" . "gated" false) }}
{{- end }}
charts/miles-run/templates/orchestrator-rbac.yaml
-43
View File
@@ -1,43 +0,0 @@
{{- if .Values.run.orchestrator.command }}
{{- $name := .Values.run.objectNames.orchestrator }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ $name | quote }}
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "miles-run.labels" (dict "context" . "component" "orchestrator") | nindent 4 }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ $name | quote }}
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "miles-run.labels" (dict "context" . "component" "orchestrator") | nindent 4 }}
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "delete"]
{{- if include "miles-run.autoUninstallEnabled" . }}
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["create"]
{{- end }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ $name | quote }}
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "miles-run.labels" (dict "context" . "component" "orchestrator") | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ $name | quote }}
subjects:
- kind: ServiceAccount
name: {{ $name | quote }}
namespace: {{ .Release.Namespace | quote }}
{{- end }}
charts/miles-run/templates/orchestrator.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ spec:
{{- end }}
spec:
{{- include "miles-run.podDefaults" . | nindent 6 }}
serviceAccountName: {{ $name | quote }}
serviceAccountName: {{ .Values.run.objectNames.platformReadDelete | quote }}
restartPolicy: Always
containers:
- name: orchestrator
charts/miles-run/templates/platform-rbac.yaml
+56
View File
@@ -0,0 +1,56 @@
{{- $context := . }}
{{- $read := .Values.run.objectNames.platformRead }}
{{- $readDelete := .Values.run.objectNames.platformReadDelete }}
{{- $levels := list
(dict "access" "read" "name" $read
"used" (include "miles-run.accountUsedByAWorker" (dict "context" . "name" $read)))
(dict "access" "read-delete" "name" $readDelete
"used" (or .Values.run.orchestrator.command
(include "miles-run.accountUsedByAWorker" (dict "context" . "name" $readDelete)))) }}
{{- range $level := $levels }}
{{- if $level.used }}
{{- $name := $level.name }}
{{- $labels := dict "context" $context "component" (printf "platform-%s" $level.access) }}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ $name | quote }}
namespace: {{ $context.Release.Namespace | quote }}
labels:
{{- include "miles-run.labels" $labels | nindent 4 }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ $name | quote }}
namespace: {{ $context.Release.Namespace | quote }}
labels:
{{- include "miles-run.labels" $labels | nindent 4 }}
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: {{ if eq $level.access "read" }}["get", "list", "watch"]{{ else }}["get", "list", "watch", "delete"]{{ end }}
{{- if and (ne $level.access "read") (include "miles-run.autoUninstallEnabled" $context) }}
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["create"]
{{- end }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ $name | quote }}
namespace: {{ $context.Release.Namespace | quote }}
labels:
{{- include "miles-run.labels" $labels | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ $name | quote }}
subjects:
- kind: ServiceAccount
name: {{ $name | quote }}
namespace: {{ $context.Release.Namespace | quote }}
{{- end }}
{{- end }}
charts/miles-run/values.schema.json
+14
View File
@@ -329,6 +329,18 @@
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
"type": "string"
},
"platformRead": {
"maxLength": 63,
"minLength": 1,
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
"type": "string"
},
"platformReadDelete": {
"maxLength": 63,
"minLength": 1,
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
"type": "string"
},
"mooncakeMaster": {
"maxLength": 63,
"minLength": 1,
@@ -356,6 +368,8 @@
},
"required": [
"orchestrator",
"platformRead",
"platformReadDelete",
"mooncakeMaster",
"colocatePairing",
"uninstall",
charts/miles-run/values.yaml
+2
View File
@@ -29,6 +29,8 @@ run:
stateFile: /cluster-storage/miles_data/miles-runs/unset/state/orchestrator.state
objectNames:
orchestrator: miles-run-orchestrator
platformRead: miles-run-platform-read
platformReadDelete: miles-run-platform-read-delete
mooncakeMaster: miles-run-mooncake-master
colocatePairing: miles-run-colocate-pairing
uninstall: miles-run-uninstall
miles/ray/specs/inference.py
+3 -2
View File
@@ -16,7 +16,7 @@ from miles.utils.workers.launch_gate import GATE_PORT_NAME
from miles.utils.workers.naming import compute_worker_name
from miles.utils.workers.registration.hub import RegistrationHub
from miles.utils.workers.registration.reporter import RegistrationReporter
from miles.utils.workers.types import DeployComponent
from miles.utils.workers.types import DeployComponent, PlatformAccess
from miles.utils.workers.worker_handle import BaseWorkerHandle
from miles.utils.workers.worker_provider.base import BaseWorkerProvider
from miles.utils.workers.worker_provider.static import StaticWorkerProvider, parse_host_and_port
@@ -44,7 +44,7 @@ INFERENCE_REGISTRATION_REPORTER_WORKER_CLASS = "miles.utils.workers.registration
def spec_inference_controller(args) -> ServeWorkerSpec:
return ServeWorkerSpec(
name=INFERENCE_CONTROLLER_POOL_ID,
needs_platform_read_permission=True,
platform_access=PlatformAccess.READ,
port_infos=[],
env_var=lambda _ctx: {},
scheduling=SchedulingSpec(
@@ -71,6 +71,7 @@ def specs_inference_registration_reporter(args) -> list[ServeWorkerSpec]:
ServeWorkerSpec(
name=INFERENCE_REGISTRATION_REPORTER_POOL_ID,
deploy_component=DeployComponent.INFERENCE,
platform_access=PlatformAccess.READ,
port_infos=[],
env_var=lambda _ctx: {},
scheduling=SchedulingSpec(
miles/ray/specs/train.py
+2 -2
View File
@@ -19,7 +19,7 @@ from miles.utils.workers.naming import (
compute_worker_name,
format_name_index,
)
from miles.utils.workers.types import DeployComponent, DeploymentIdentity
from miles.utils.workers.types import DeployComponent, DeploymentIdentity, PlatformAccess
from miles.utils.workers.worker_handle import BaseWorkerHandle
from miles.utils.workers.worker_provider.base import BaseWorkerProvider
from miles.utils.workers.worker_provider.static import StaticWorkerProvider, parse_host_and_port
@@ -127,7 +127,7 @@ def _compute_spec_trainer_controller(
return ServeWorkerSpec(
name=compute_trainer_controller_pool_id(trainer_id),
deploy_component=DeployComponent.TRAINER,
needs_platform_read_permission=True,
platform_access=PlatformAccess.READ_DELETE,
port_infos=[],
env_var=lambda _ctx: {},
scheduling=SchedulingSpec(
miles/utils/external_utils/command_utils/helm_backend/launcher/values/builder.py
+3
View File
@@ -22,6 +22,7 @@ from miles.utils.external_utils.command_utils.helm_backend.launcher.values.misc
from miles.utils.external_utils.command_utils.helm_backend.launcher.values.pool_entry import build_entry
from miles.utils.external_utils.command_utils.helm_backend.naming import RunNames
from miles.utils.workers.naming import compute_cell_id
from miles.utils.workers.types import PlatformAccess
from miles.utils.workers.worker_provider.kubernetes.helm.naming import static_cell_addrs
from miles.utils.workers.worker_spec import RPC_PORT_NAME, BaseWorkerSpec, NamedHostAndPorts, ServeWorkerSpec
@@ -93,6 +94,8 @@ def _pairing_config(specs: list[BaseWorkerSpec], plan: LaunchPlan) -> PairingCon
def _object_names(release: str) -> ObjectNames:
return ObjectNames(
orchestrator=naming.component_name(release, naming.ORCHESTRATOR_COMPONENT),
platform_read=naming.platform_account_name(release=release, access=PlatformAccess.READ),
platform_read_delete=naming.platform_account_name(release=release, access=PlatformAccess.READ_DELETE),
mooncake_master=MooncakeInfo.master_object_name(release),
colocate_pairing=naming.component_name(release, _COLOCATE_PAIRING_COMPONENT),
uninstall=RunNames.uninstall_job(release=release),
miles/utils/external_utils/command_utils/helm_backend/launcher/values/helm_values_types.py
+2
View File
@@ -92,6 +92,8 @@ class PoolEntry(ValuesModel):
class ObjectNames(ValuesModel):
orchestrator: _ObjectName
platform_read: _ObjectName
platform_read_delete: _ObjectName
mooncake_master: _ObjectName
colocate_pairing: _ObjectName
uninstall: _ObjectName
miles/utils/external_utils/command_utils/helm_backend/launcher/values/pool_entry.py
+8 -5
View File
@@ -24,6 +24,7 @@ from miles.utils.external_utils.command_utils.helm_backend.launcher.values.place
)
from miles.utils.workers.argv_utils import python_argv_prefix
from miles.utils.workers.naming import compute_port_name
from miles.utils.workers.types import PlatformAccess
from miles.utils.workers.worker_provider.kubernetes.helm import env
from miles.utils.workers.worker_spec import (
BaseWorkerSpec,
@@ -70,11 +71,7 @@ def build_entry(
ports=[PortEntry(name=compute_port_name(port.name), port=port.static_port) for port in spec.port_infos],
env=_command_env_of_spec(spec, context, addresses=addresses, is_sub_node=is_sub_node) or None,
meta=_meta_of_spec(spec) or None,
service_account_name=(
naming.component_name(plan.release, naming.ORCHESTRATOR_COMPONENT)
if spec.needs_platform_read_permission
else None
),
service_account_name=_service_account_name(spec, plan=plan),
replicas=spec.scheduling.num_cells,
size=pods_per_cell if pods_per_cell > 1 else None,
resources={"limits": {"nvidia.com/gpu": gpus_per_pod}} if gpus_per_pod else None,
@@ -82,6 +79,12 @@ def build_entry(
)
def _service_account_name(spec: BaseWorkerSpec, *, plan: LaunchPlan) -> str | None:
if (access := spec.platform_access) is PlatformAccess.NONE:
return None
return naming.platform_account_name(release=plan.release, access=access)
def _command_env_of_spec(
spec: BaseWorkerSpec,
context: LaunchCommandContext,
miles/utils/external_utils/command_utils/helm_backend/naming.py
+6 -1
View File
@@ -8,7 +8,7 @@ from pathlib import Path
from pydantic import model_validator
from miles.utils.pydantic_utils import FrozenStrictBaseModel
from miles.utils.workers.types import DeployComponent
from miles.utils.workers.types import DeployComponent, PlatformAccess
from miles.utils.workers.worker_provider.kubernetes.helm.naming import CHART_NAME, component_name
ORCHESTRATOR_COMPONENT = "orchestrator"
@@ -31,6 +31,11 @@ _RECORDED_STATE_FILE_KEY = "state_file"
_SUPERSEDED_MARKER_SUFFIX = ".superseded"
def platform_account_name(*, release: str, access: PlatformAccess) -> str:
assert access is not PlatformAccess.NONE, "a worker that never reaches the platform runs on no account of its own"
return component_name(release, f"platform-{access.value}")
class ReleaseName(FrozenStrictBaseModel):
run_id: str
deploy_component: DeployComponent
miles/utils/workers/types.py
+6
View File
@@ -36,6 +36,12 @@ class DeployComponent(Enum):
return self in (DeployComponent.TRAINER, DeployComponent.INFERENCE)
class PlatformAccess(Enum):
NONE = "none"
READ = "read"
READ_DELETE = "read-delete"
class HotRestartComponent(Enum):
ORCHESTRATION = "orchestration"
ROLLOUT_EXECUTOR = "rollout_executor"
miles/utils/workers/worker_spec.py
+2 -2
View File
@@ -6,7 +6,7 @@ from pydantic import ConfigDict, model_validator
from miles.utils.math_utils import exact_div
from miles.utils.pydantic_utils import FrozenStrictBaseModel
from miles.utils.workers.backend_capability.base import BackendCapability
from miles.utils.workers.types import DeployComponent
from miles.utils.workers.types import DeployComponent, PlatformAccess
RPC_PORT_NAME = "rpc"
MASTER_PORT_NAME = "master"
@@ -102,7 +102,7 @@ class BaseWorkerSpec(FrozenStrictBaseModel):
scheduling: SchedulingSpec
meta: SpecMetaFn | None = None
deploy_component: DeployComponent = DeployComponent.PRIMARY
needs_platform_read_permission: bool = False
platform_access: PlatformAccess = PlatformAccess.NONE
@model_validator(mode="after")
def _reject_selector_component(self) -> "BaseWorkerSpec":
tests/fast/charts/miles_run/test_auto_uninstall.py
+3 -2
View File
@@ -7,6 +7,7 @@ from tests.fast.charts.utils import (
NAMESPACE,
RUN_CHART_DIR,
RUN_ORCHESTRATOR_NAME,
RUN_PLATFORM_READ_DELETE_NAME,
RUN_RELEASE_NAME,
RUN_UNINSTALL_JOB_NAME,
RUN_UNINSTALL_MANIFEST_NAME,
@@ -127,13 +128,13 @@ class TestOrchestratorTrigger:
def test_lets_the_orchestrator_create_that_one_job(self):
"""The wrapper creates it as the run's own account, which is otherwise allowed pods and nothing else."""
role = named_object(_enabled_objects(), "Role", RUN_ORCHESTRATOR_NAME)
role = named_object(_enabled_objects(), "Role", RUN_PLATFORM_READ_DELETE_NAME)
assert {"apiGroups": ["batch"], "resources": ["jobs"], "verbs": ["create"]} in role["rules"]
def test_grants_no_job_rights_to_a_run_that_never_creates_one(self):
"""A run that cannot uninstall itself has no business creating workloads of any kind."""
role = named_object(render_run(*DISABLE_AUTO_UNINSTALL), "Role", RUN_ORCHESTRATOR_NAME)
role = named_object(render_run(*DISABLE_AUTO_UNINSTALL), "Role", RUN_PLATFORM_READ_DELETE_NAME)
assert [rule["apiGroups"] for rule in role["rules"]] == [[""]]
tests/fast/charts/miles_run/test_colocate.py
+1 -1
View File
@@ -50,7 +50,7 @@ POOLS = (
)
PAIRING = "myrun-miles-run-colocate-pairing"
ORCHESTRATOR_ROLE = "myrun-miles-run-orchestrator"
ORCHESTRATOR_ROLE = "myrun-miles-run-platform-read-delete"
def layout(
tests/fast/charts/miles_run/test_orchestrator.py
+7 -6
View File
@@ -13,6 +13,7 @@ from tests.fast.charts.utils import (
)
ORCHESTRATOR = "myrun-miles-run-orchestrator"
ORCHESTRATOR_ACCOUNT = "myrun-miles-run-platform-read-delete"
WRAPPER_MODULE = "miles.utils.external_utils.command_utils.helm_backend.orchestrator.wrapper"
@@ -98,17 +99,17 @@ class TestOrchestratorIdentity:
def test_grants_the_orchestrator_the_pod_rights_observation_and_healing_need(self):
"""Healing a cell means deleting its pods, and observing one means watching them."""
objects = render_run()
role = named_object(objects, "Role", ORCHESTRATOR)
binding = named_object(objects, "RoleBinding", ORCHESTRATOR)
role = named_object(objects, "Role", ORCHESTRATOR_ACCOUNT)
binding = named_object(objects, "RoleBinding", ORCHESTRATOR_ACCOUNT)
assert role["rules"] == [
{"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list", "watch", "delete"]},
{"apiGroups": ["batch"], "resources": ["jobs"], "verbs": ["create"]},
]
assert binding["roleRef"]["name"] == ORCHESTRATOR
assert binding["subjects"] == [dict(kind="ServiceAccount", name=ORCHESTRATOR, namespace=NAMESPACE)]
assert binding["roleRef"]["name"] == ORCHESTRATOR_ACCOUNT
assert binding["subjects"] == [dict(kind="ServiceAccount", name=ORCHESTRATOR_ACCOUNT, namespace=NAMESPACE)]
def test_runs_the_orchestrator_under_the_account_that_binding_names(self):
"""A role bound to an account nobody runs as grants nothing at all."""
assert pod_spec_of(render_run(), "StatefulSet", ORCHESTRATOR)["serviceAccountName"] == ORCHESTRATOR
assert named_object(render_run(), "ServiceAccount", ORCHESTRATOR)["metadata"]["namespace"] == NAMESPACE
assert pod_spec_of(render_run(), "StatefulSet", ORCHESTRATOR)["serviceAccountName"] == ORCHESTRATOR_ACCOUNT
assert named_object(render_run(), "ServiceAccount", ORCHESTRATOR_ACCOUNT)["metadata"]["namespace"] == NAMESPACE
tests/fast/charts/miles_run/test_platform_rbac.py
+96
View File
@@ -0,0 +1,96 @@
import json
from typing import Any
from tests.fast.charts.utils import (
RUN_ORCHESTRATOR_NAME,
RUN_PLATFORM_READ_DELETE_NAME,
RUN_PLATFORM_READ_NAME,
RUN_RELEASE_NAME,
named_object,
objects_of_kind,
pod_spec_of,
render_run,
requires_helm,
with_object_names,
)
_READER = {
"name": "inference-registration-reporter",
"command": ["x"],
"serviceAccountName": RUN_PLATFORM_READ_NAME,
}
_DELETER = {
"name": "trainer-controller-actor",
"command": ["x"],
"serviceAccountName": RUN_PLATFORM_READ_DELETE_NAME,
}
_PLAIN = {"name": "inference-engine", "command": ["x"]}
def _render_without_orchestrator(*workers: dict[str, Any]) -> list[dict[str, Any]]:
return render_run(
"--set-json",
"run.orchestrator.command=[]",
"--set-json",
f"run.staticWorkers={json.dumps(with_object_names(list(workers)))}",
)
@requires_helm
class TestTheAccountsAReleaseGrantsItsPlatformClients:
def test_grants_the_read_account_to_a_release_whose_client_only_reads_pods(self):
"""A split run's inference release carries the registration reporter, which lists pods to report them."""
objects = _render_without_orchestrator(_READER)
assert named_object(objects, "ServiceAccount", RUN_PLATFORM_READ_NAME)
assert named_object(objects, "RoleBinding", RUN_PLATFORM_READ_NAME)
assert named_object(objects, "Role", RUN_PLATFORM_READ_NAME)["rules"] == [
{"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list", "watch"]}
]
assert (
pod_spec_of(objects, "StatefulSet", f"{RUN_RELEASE_NAME}-miles-run-inference-registration-reporter")[
"serviceAccountName"
]
== RUN_PLATFORM_READ_NAME
)
def test_grants_the_read_delete_account_to_a_release_whose_client_also_deletes_pods(self):
"""A split run's trainer release suspends cells by deleting pods, which the read-only account refuses."""
objects = _render_without_orchestrator(_DELETER)
assert named_object(objects, "ServiceAccount", RUN_PLATFORM_READ_DELETE_NAME)
assert named_object(objects, "RoleBinding", RUN_PLATFORM_READ_DELETE_NAME)
assert named_object(objects, "Role", RUN_PLATFORM_READ_DELETE_NAME)["rules"] == [
{"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list", "watch", "delete"]},
{"apiGroups": ["batch"], "resources": ["jobs"], "verbs": ["create"]},
]
def test_grants_a_reader_none_of_the_rights_the_read_delete_account_carries(self):
"""A reporter may read pod state without inheriting deletion or job-creation rights."""
objects = _render_without_orchestrator(_READER)
assert not any(
obj["metadata"]["name"] == RUN_PLATFORM_READ_DELETE_NAME for obj in objects_of_kind(objects, "Role")
)
def test_grants_nothing_to_a_release_no_platform_client_lives_in(self):
"""A release of plain engines binds no account, so rights over pods would be handed to nobody."""
objects = _render_without_orchestrator(_PLAIN)
assert objects_of_kind(objects, "ServiceAccount") == []
assert objects_of_kind(objects, "Role") == []
assert objects_of_kind(objects, "RoleBinding") == []
def test_still_grants_the_release_that_runs_the_orchestration_script(self):
"""The orchestrator deletes pods to heal cells, and it is the reason this rbac existed at all."""
objects = render_run()
assert named_object(objects, "ServiceAccount", RUN_PLATFORM_READ_DELETE_NAME)
assert named_object(objects, "Role", RUN_PLATFORM_READ_DELETE_NAME)["rules"] == [
{"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list", "watch", "delete"]},
{"apiGroups": ["batch"], "resources": ["jobs"], "verbs": ["create"]},
]
assert (
pod_spec_of(objects, "StatefulSet", RUN_ORCHESTRATOR_NAME)["serviceAccountName"]
== RUN_PLATFORM_READ_DELETE_NAME
)
tests/fast/charts/utils.py
+6
View File
@@ -25,6 +25,8 @@ RELEASE_NAME = "miles-workbench-myuser"
UNINSTALLER_SERVICE_ACCOUNT = "miles-uninstaller"
RUN_RELEASE_NAME = "myrun"
RUN_ORCHESTRATOR_NAME = f"{RUN_RELEASE_NAME}-miles-run-orchestrator"
RUN_PLATFORM_READ_NAME = f"{RUN_RELEASE_NAME}-miles-run-platform-read"
RUN_PLATFORM_READ_DELETE_NAME = f"{RUN_RELEASE_NAME}-miles-run-platform-read-delete"
RUN_UNINSTALL_JOB_NAME = f"{RUN_RELEASE_NAME}-miles-run-uninstall"
RUN_UNINSTALL_MANIFEST_NAME = f"{RUN_RELEASE_NAME}-miles-run-uninstall-manifest"
RUN_ID = "260101-000000-000"
@@ -100,6 +102,10 @@ def run_helm_template_run(*args: str) -> subprocess.CompletedProcess:
"--set",
f"run.objectNames.orchestrator={RUN_ORCHESTRATOR_NAME}",
"--set",
f"run.objectNames.platformRead={RUN_PLATFORM_READ_NAME}",
"--set",
f"run.objectNames.platformReadDelete={RUN_PLATFORM_READ_DELETE_NAME}",
"--set",
f"run.objectNames.mooncakeMaster={RUN_RELEASE_NAME}-miles-run-mooncake-master",
"--set",
f"run.objectNames.colocatePairing={RUN_RELEASE_NAME}-miles-run-colocate-pairing",
tests/fast/ray/specs/test_inference.py
+14
View File
@@ -29,6 +29,7 @@ from miles.ray.specs.inference import (
spec_inference_controller,
spec_session_server,
specs_inference_engine,
specs_inference_registration_reporter,
specs_router,
)
from miles.rollout.session.config import SessionServerConfig
@@ -38,6 +39,7 @@ from miles.utils.external_utils.command_utils.helm_backend.launcher.values.misc
from miles.utils.function_registry import load_function
from miles.utils.workers.argv_utils import parse_config_argv
from miles.utils.workers.registration.hub import RegistrationHub
from miles.utils.workers.types import PlatformAccess
from miles.utils.workers.worker_provider.static import StaticWorkerProvider
from miles.utils.workers.worker_spec import (
RPC_PORT_NAME,
@@ -1199,6 +1201,18 @@ class TestSpecInferenceController:
assert load_function(spec.worker_class) is InferenceController
def test_it_declares_only_the_platform_reads_its_provider_performs(self, tmp_path):
"""Watching engine pods needs reads, while deleting them remains outside this worker's capability."""
spec = spec_inference_controller(self._args(tmp_path))
assert spec.platform_access is PlatformAccess.READ
def test_the_registration_reporter_declares_only_platform_reads(self, tmp_path):
"""The reporter watches engine pods, so it needs reads and none of the orchestrator's other rights."""
(spec,) = specs_inference_registration_reporter(self._args(tmp_path, deploy_component="inference"))
assert spec.platform_access is PlatformAccess.READ
def test_the_worker_name_is_stable(self):
"""The driver looks the controller up by name, so this name is part of the release's contract."""
assert inference_controller_worker_name() == "inference-controller-00000-00000"
tests/fast/ray/specs/test_train.py
+7
View File
@@ -33,6 +33,7 @@ from miles.ray.train_actor import TrainRayActor
from miles.utils.external_utils.command_utils.helm_backend.launcher.values.builder import build_values
from miles.utils.external_utils.command_utils.helm_backend.launcher.values.misc import SECTION_OF_CATEGORY, LaunchPlan
from miles.utils.workers.rpc.common.metadata import _find_rpc_config
from miles.utils.workers.types import PlatformAccess
from miles.utils.workers.worker_spec import WorkerCtorContext
@@ -626,6 +627,12 @@ class TestSpecTrainerController:
assert spec.worker_class == TRAINER_CONTROLLER_WORKER_CLASS
def test_it_keeps_the_platform_delete_capability_used_to_suspend_cells(self):
"""Fault injection deletes trainer pods, so moving readers off the orchestrator account must not break it."""
spec = specs_trainer_controller(_make_args())[0]
assert spec.platform_access is PlatformAccess.READ_DELETE
def test_the_worker_and_cell_names_are_stable(self):
"""The driver looks the controller up by name, so these names are part of the release's contract."""
assert trainer_controller_worker_name("actor") == "trainer-controller-actor-00000-00000"
tests/fast/tools/kubernetes/test_generate_chart_schema.py
+2
View File
@@ -52,6 +52,8 @@ def _minimal_run_values() -> dict[str, Any]:
"stateFile": "/cluster-storage/miles_data/miles-runs/260101-000000-000/state/orchestrator.state",
"objectNames": {
"orchestrator": "r-miles-run-orchestrator",
"platformRead": "r-miles-run-platform-read",
"platformReadDelete": "r-miles-run-platform-read-delete",
"mooncakeMaster": "r-miles-run-mooncake-master",
"colocatePairing": "r-miles-run-colocate-pairing",
"uninstall": "r-miles-run-uninstall",
tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_builder.py
+2
View File
@@ -222,6 +222,8 @@ class TestObjectNames:
"""A schema-required field the launcher only sometimes writes would refuse half the runs."""
assert build_values([], LAYOUT).as_values()["run"]["objectNames"] == {
"orchestrator": "r-miles-run-orchestrator",
"platformRead": "r-miles-run-platform-read",
"platformReadDelete": "r-miles-run-platform-read-delete",
"mooncakeMaster": "r-miles-run-mooncake-master",
"colocatePairing": "r-miles-run-colocate-pairing",
"uninstall": "r-miles-run-uninstall",
tests/fast/utils/external_utils/command_utils/helm_backend/launcher/values/test_pool_entry.py
+13 -4
View File
@@ -13,6 +13,7 @@ from miles.utils.external_utils.command_utils.helm_backend.launcher.values impor
from miles.utils.external_utils.command_utils.helm_backend.launcher.values.builder import build_values
from miles.utils.external_utils.command_utils.helm_backend.launcher.values.helm_values_types import PortEntry
from miles.utils.external_utils.command_utils.helm_backend.launcher.values.misc import LaunchPlan
from miles.utils.workers.types import PlatformAccess
from miles.utils.workers.worker_spec import BaseWorkerSpec, SchedulingSpec
STAMP = "2026-08-12T09:00:00+00:00"
@@ -180,13 +181,21 @@ class TestTheRestartStamp:
class TestTheAccountAPoolRunsUnder:
def test_a_pool_that_observes_the_platform_gets_the_account_that_may_read_it(self):
def test_a_pool_that_only_reads_the_platform_gets_the_account_that_may_read_it(self):
"""Only these workers reconcile against pods, and the namespace default cannot list one."""
spec = session_server(num_cells=1).model_copy(update={"needs_platform_read_permission": True})
spec = session_server(num_cells=1).model_copy(update={"platform_access": PlatformAccess.READ})
entry = build_values([spec], LAYOUT).as_values()["run"]["staticWorkers"][0]
assert entry["serviceAccountName"] == "r-miles-run-orchestrator"
assert entry["serviceAccountName"] == "r-miles-run-platform-read"
def test_a_pool_that_also_deletes_platform_pods_gets_the_account_that_may_delete_them(self):
"""A trainer controller suspends cells by deleting pods and must retain that existing capability."""
spec = session_server(num_cells=1).model_copy(update={"platform_access": PlatformAccess.READ_DELETE})
entry = build_values([spec], LAYOUT).as_values()["run"]["staticWorkers"][0]
assert entry["serviceAccountName"] == "r-miles-run-platform-read-delete"
def test_every_other_pool_stays_on_the_namespace_default(self):
"""An engine talks to no api server, and an account it never needs is one it could misuse."""
@@ -196,7 +205,7 @@ class TestTheAccountAPoolRunsUnder:
def test_refuses_a_pool_whose_template_renders_no_account_at_all(self):
"""The engine template ignores the key, so the pod would run on the default and 403 far from here."""
spec = engine(num_cells=1, gpus_per_engine=8).model_copy(update={"needs_platform_read_permission": True})
spec = engine(num_cells=1, gpus_per_engine=8).model_copy(update={"platform_access": PlatformAccess.READ})
with pytest.raises(AssertionError, match="renders a service account"):
build_values([spec], LAYOUT).as_values()
tests/fast/utils/external_utils/command_utils/helm_backend/test_naming.py
+66 -1
View File
@@ -1,3 +1,12 @@
import json
import re
from datetime import datetime
from pathlib import Path
from types import SimpleNamespace
import pytest
from miles.utils.external_utils.command_utils.helm_backend import naming
from miles.utils.external_utils.command_utils.helm_backend.naming import (
RunFiles,
_orchestrator_state_path,
@@ -14,6 +23,12 @@ def _write(path, status: OrchestratorStatus, *, exit_code: int | None = None) ->
OrchestratorState(status=status, exit_code=exit_code).write(path)
def _record(run_directory, launch_token: str, state_file) -> None:
path = Path(run_directory) / "launches" / f"launch-{launch_token}.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"state_file": str(state_file)}))
def _state_file(tmp_path):
return _orchestrator_state_path(tmp_path, "260101-000000-000001")
@@ -48,20 +63,70 @@ class TestRunDir:
class TestLatestExitFile:
def test_names_no_file_before_a_launch_has_written_one(self, tmp_path):
def test_names_no_file_before_a_launch_has_recorded_one(self, tmp_path):
"""A run directory a launch has only just created holds no verdict to collect."""
assert RunFiles.latest_state_file(run_directory=tmp_path) is None
def test_names_the_newest_launch_s_state_file_before_it_is_written(self, tmp_path):
"""A generation whose pods never came up must not hand the previous generation's verdict over."""
_record(tmp_path, "260101-000100-000002", _orchestrator_state_path(tmp_path, "260101-000100-000002"))
pending = _orchestrator_state_path(tmp_path, "260101-000200-000001")
_record(tmp_path, "260101-000200-000001", pending)
assert RunFiles.latest_state_file(run_directory=tmp_path) == pending
def test_picks_the_newest_launch_rather_than_the_newest_write(self, tmp_path):
"""An earlier launch torn down after a later one started writes last, and its verdict is not the run's."""
later = _orchestrator_state_path(tmp_path, "260101-000200-000001")
earlier = _orchestrator_state_path(tmp_path, "260101-000100-000002")
_write(later, OrchestratorStatus.EXITED, exit_code=0)
_write(earlier, OrchestratorStatus.EXITED, exit_code=1)
_record(tmp_path, "260101-000100-000002", earlier)
_record(tmp_path, "260101-000200-000001", later)
assert RunFiles.latest_state_file(run_directory=tmp_path) == later
def _mint_token(monkeypatch: pytest.MonkeyPatch, *, when: datetime, tail: int) -> str:
monkeypatch.setattr(naming, "datetime", SimpleNamespace(now=lambda: when))
monkeypatch.setattr(naming.random, "Random", lambda: SimpleNamespace(randint=lambda low, high: tail))
return naming._new_launch_token()
class TestNewLaunchToken:
def test_two_launches_of_one_second_order_by_the_time_they_were_minted(self, monkeypatch):
"""Everything that picks the newest launch sorts these names, and the random tail ordered them by chance."""
earlier = _mint_token(monkeypatch, when=datetime(2026, 1, 1, 0, 0, 0, 1), tail=999999)
later = _mint_token(monkeypatch, when=datetime(2026, 1, 1, 0, 0, 0, 2), tail=0)
assert earlier < later
def test_a_token_carries_the_microsecond_it_was_minted_at(self, monkeypatch):
"""Two launches of one run are apart by microseconds, which is the resolution the name has to keep."""
token = _mint_token(monkeypatch, when=datetime(2026, 1, 1, 0, 0, 0, 123), tail=7)
assert token == "260101-000000-000123-000007"
def test_two_tokens_minted_in_one_microsecond_still_differ(self, monkeypatch):
"""The random tail no longer decides the order, but it is still what keeps two names apart."""
when = datetime(2026, 1, 1, 0, 0, 0, 5)
assert _mint_token(monkeypatch, when=when, tail=1) != _mint_token(monkeypatch, when=when, tail=2)
def test_every_part_of_a_real_token_is_fixed_width(self):
"""A part that can be shorter sorts before a longer one whatever time it names."""
assert re.fullmatch(r"\d{6}-\d{6}-\d{6}-\d{6}", naming._new_launch_token())
def test_a_real_token_never_names_a_time_before_the_one_minted_earlier(self):
"""The clock part is what the newest-launch lookup reads, and it has to be non-decreasing."""
stamp_length = len("260101-000000-000000")
first = naming._new_launch_token()
second = naming._new_launch_token()
assert first[:stamp_length] <= second[:stamp_length]
class TestSupersededMarker:
def test_the_marker_sits_beside_the_state_file_it_supersedes(self):
"""Both the launcher that writes it and the orchestrator that reads it know only that path."""
tests/snapshots/charts/miles-run/typical-values.yaml
+5 -3
View File
@@ -196,6 +196,8 @@ run:
colocatePairing: myrun-miles-run-colocate-pairing
mooncakeMaster: myrun-miles-run-mooncake-master
orchestrator: myrun-miles-run-orchestrator
platformRead: myrun-miles-run-platform-read
platformReadDelete: myrun-miles-run-platform-read-delete
uninstall: myrun-miles-run-uninstall
uninstallManifest: myrun-miles-run-uninstall-manifest
orchestrator:
@@ -247,7 +249,7 @@ run:
- name: rpc
port: 8000
replicas: 1
serviceAccountName: myrun-miles-run-orchestrator
serviceAccountName: myrun-miles-run-platform-read
- command:
- <PYTHON>
- -m
@@ -279,7 +281,7 @@ run:
- -m
- miles.rollout.session.server
- --config-json
- '{"host":"0.0.0.0","port":8000,"instance_id":"0123456789abcdef-0","backend_url":"http://myrun-miles-run-inference-router-0-0.myrun-miles-run-inference-router-0:8000","timeout":null,"hf_checkpoint":"<FIXTURE_DIR>/typical-model","chat_template_path":null,"tito_model":"default","apply_chat_template_kwargs":{},"use_rollout_routing_replay":false,"use_rollout_indexer_replay":false,"use_sampling_support_replay":false,"sglang_speculative_algorithm":null,"num_layers":36,"moe_router_topk":2,"save_debug_trajectory_data":null,"lora_rank":0,"lora_adapter_path":null,"lora_train_only":false,"use_session_server":true,"session_message_matcher":"strict","pause_generation_mode":"retract","session_sample_picker_path":"miles.rollout.session.v2.picker_hub.drop_retries","session_sample_postprocessor_path":"miles.rollout.session.v2.postprocessor_hub.default_postprocess"}'
- '{"host":"0.0.0.0","port":8000,"instance_id":"0123456789abcdef-0","backend_url":"http://myrun-miles-run-inference-router-0-0.myrun-miles-run-inference-router-0:8000","timeout":null,"hf_checkpoint":"<FIXTURE_DIR>/typical-model","chat_template_path":null,"tito_model":"default","apply_chat_template_kwargs":{},"use_rollout_routing_replay":false,"use_rollout_indexer_replay":false,"use_sampling_support_replay":false,"sglang_speculative_algorithm":null,"num_layers":36,"moe_router_topk":2,"save_debug_trajectory_data":null,"lora_rank":0,"lora_adapter_path":null,"lora_train_only":false,"use_session_server":true,"session_message_matcher":"strict","pause_generation_mode":"retract","session_sample_picker_path":"miles.rollout.session.v2.picker_hub.drop_same_prompt_retries","session_sample_postprocessor_path":"miles.rollout.session.v2.postprocessor_hub.default_postprocess"}'
name: session-server
objectName: myrun-miles-run-session-server
poolId: session-server
@@ -307,7 +309,7 @@ run:
- name: rpc
port: 8000
replicas: 1
serviceAccountName: myrun-miles-run-orchestrator
serviceAccountName: myrun-miles-run-platform-read-delete
trainerEngines:
- command:
- bash
tests/snapshots/charts/miles-run/typical.yaml
+80 -26
View File
@@ -13,17 +13,31 @@ metadata:
app.kubernetes.io/version: "0.1.0"
app.kubernetes.io/managed-by: "Helm"
---
# Source: miles-run/templates/orchestrator-rbac.yaml
# Source: miles-run/templates/platform-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: "myrun-miles-run-orchestrator"
name: "myrun-miles-run-platform-read"
namespace: "myns"
labels:
helm.sh/chart: "miles-run-0.1.0"
app.kubernetes.io/name: "miles-run"
app.kubernetes.io/instance: "myrun"
app.kubernetes.io/component: "orchestrator"
app.kubernetes.io/component: "platform-read"
app.kubernetes.io/version: "0.1.0"
app.kubernetes.io/managed-by: "Helm"
---
# Source: miles-run/templates/platform-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: "myrun-miles-run-platform-read-delete"
namespace: "myns"
labels:
helm.sh/chart: "miles-run-0.1.0"
app.kubernetes.io/name: "miles-run"
app.kubernetes.io/instance: "myrun"
app.kubernetes.io/component: "platform-read-delete"
app.kubernetes.io/version: "0.1.0"
app.kubernetes.io/managed-by: "Helm"
---
@@ -97,17 +111,35 @@ rules:
resources: ["pods"]
verbs: ["get", "list", "watch", "patch", "update"]
---
# Source: miles-run/templates/orchestrator-rbac.yaml
# Source: miles-run/templates/platform-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: "myrun-miles-run-orchestrator"
name: "myrun-miles-run-platform-read"
namespace: "myns"
labels:
helm.sh/chart: "miles-run-0.1.0"
app.kubernetes.io/name: "miles-run"
app.kubernetes.io/instance: "myrun"
app.kubernetes.io/component: "orchestrator"
app.kubernetes.io/component: "platform-read"
app.kubernetes.io/version: "0.1.0"
app.kubernetes.io/managed-by: "Helm"
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch"]
---
# Source: miles-run/templates/platform-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: "myrun-miles-run-platform-read-delete"
namespace: "myns"
labels:
helm.sh/chart: "miles-run-0.1.0"
app.kubernetes.io/name: "miles-run"
app.kubernetes.io/instance: "myrun"
app.kubernetes.io/component: "platform-read-delete"
app.kubernetes.io/version: "0.1.0"
app.kubernetes.io/managed-by: "Helm"
rules:
@@ -140,26 +172,48 @@ subjects:
name: "myrun-miles-run-colocate-pairing"
namespace: "myns"
---
# Source: miles-run/templates/orchestrator-rbac.yaml
# Source: miles-run/templates/platform-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: "myrun-miles-run-orchestrator"
name: "myrun-miles-run-platform-read"
namespace: "myns"
labels:
helm.sh/chart: "miles-run-0.1.0"
app.kubernetes.io/name: "miles-run"
app.kubernetes.io/instance: "myrun"
app.kubernetes.io/component: "orchestrator"
app.kubernetes.io/component: "platform-read"
app.kubernetes.io/version: "0.1.0"
app.kubernetes.io/managed-by: "Helm"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: "myrun-miles-run-orchestrator"
name: "myrun-miles-run-platform-read"
subjects:
- kind: ServiceAccount
name: "myrun-miles-run-orchestrator"
name: "myrun-miles-run-platform-read"
namespace: "myns"
---
# Source: miles-run/templates/platform-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: "myrun-miles-run-platform-read-delete"
namespace: "myns"
labels:
helm.sh/chart: "miles-run-0.1.0"
app.kubernetes.io/name: "miles-run"
app.kubernetes.io/instance: "myrun"
app.kubernetes.io/component: "platform-read-delete"
app.kubernetes.io/version: "0.1.0"
app.kubernetes.io/managed-by: "Helm"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: "myrun-miles-run-platform-read-delete"
subjects:
- kind: ServiceAccount
name: "myrun-miles-run-platform-read-delete"
namespace: "myns"
---
@@ -350,7 +404,7 @@ spec:
- name: pairing
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
@@ -446,13 +500,13 @@ spec:
- effect: NoSchedule
key: nvidia.com/gpu
operator: Exists
serviceAccountName: "myrun-miles-run-orchestrator"
serviceAccountName: "myrun-miles-run-platform-read-delete"
restartPolicy: Always
containers:
- name: orchestrator
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
@@ -569,12 +623,12 @@ spec:
- name: worker
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
- name: "cluster-storage"
mountPath: "<REPO_ROOT>"
mountPath: "/root/miles"
subPath: "myuser/miles"
- name: "cluster-storage"
mountPath: "/root/Megatron-LM"
@@ -677,13 +731,13 @@ spec:
- effect: NoSchedule
key: nvidia.com/gpu
operator: Exists
serviceAccountName: "myrun-miles-run-orchestrator"
serviceAccountName: "myrun-miles-run-platform-read"
restartPolicy: Always
containers:
- name: worker
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
@@ -796,7 +850,7 @@ spec:
- name: worker
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
@@ -915,7 +969,7 @@ spec:
- name: worker
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
@@ -935,7 +989,7 @@ spec:
- "-m"
- "miles.rollout.session.server"
- "--config-json"
- "{\"host\":\"0.0.0.0\",\"port\":8000,\"instance_id\":\"0123456789abcdef-0\",\"backend_url\":\"http://myrun-miles-run-inference-router-0-0.myrun-miles-run-inference-router-0:8000\",\"timeout\":null,\"hf_checkpoint\":\"<REPO_ROOT>/tests/fast/charts/miles_run/typical-model\",\"chat_template_path\":null,\"tito_model\":\"default\",\"apply_chat_template_kwargs\":{},\"use_rollout_routing_replay\":false,\"use_rollout_indexer_replay\":false,\"use_sampling_support_replay\":false,\"sglang_speculative_algorithm\":null,\"num_layers\":36,\"moe_router_topk\":2,\"save_debug_trajectory_data\":null,\"lora_rank\":0,\"lora_adapter_path\":null,\"lora_train_only\":false,\"use_session_server\":true,\"session_message_matcher\":\"strict\",\"pause_generation_mode\":\"retract\",\"session_sample_picker_path\":\"miles.rollout.session.v2.picker_hub.drop_retries\",\"session_sample_postprocessor_path\":\"miles.rollout.session.v2.postprocessor_hub.default_postprocess\"}"
- "{\"host\":\"0.0.0.0\",\"port\":8000,\"instance_id\":\"0123456789abcdef-0\",\"backend_url\":\"http://myrun-miles-run-inference-router-0-0.myrun-miles-run-inference-router-0:8000\",\"timeout\":null,\"hf_checkpoint\":\"<FIXTURE_DIR>/typical-model\",\"chat_template_path\":null,\"tito_model\":\"default\",\"apply_chat_template_kwargs\":{},\"use_rollout_routing_replay\":false,\"use_rollout_indexer_replay\":false,\"use_sampling_support_replay\":false,\"sglang_speculative_algorithm\":null,\"num_layers\":36,\"moe_router_topk\":2,\"save_debug_trajectory_data\":null,\"lora_rank\":0,\"lora_adapter_path\":null,\"lora_train_only\":false,\"use_session_server\":true,\"session_message_matcher\":\"strict\",\"pause_generation_mode\":\"retract\",\"session_sample_picker_path\":\"miles.rollout.session.v2.picker_hub.drop_same_prompt_retries\",\"session_sample_postprocessor_path\":\"miles.rollout.session.v2.postprocessor_hub.default_postprocess\"}"
env:
- name: MILES_CELL_INDEX
valueFrom:
@@ -1016,13 +1070,13 @@ spec:
- effect: NoSchedule
key: nvidia.com/gpu
operator: Exists
serviceAccountName: "myrun-miles-run-orchestrator"
serviceAccountName: "myrun-miles-run-platform-read-delete"
restartPolicy: Always
containers:
- name: worker
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
@@ -1138,7 +1192,7 @@ spec:
- name: "engine"
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
@@ -1347,7 +1401,7 @@ spec:
- name: "engine"
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
@@ -1547,7 +1601,7 @@ spec:
- name: "trainer"
image: "myregistry.example/myteam/miles:v0.9.3-cu128"
imagePullPolicy: "IfNotPresent"
workingDir: "<REPO_ROOT>"
workingDir: "/root/miles"
volumeMounts:
- name: "cluster-storage"
mountPath: "/cluster-storage"
tests/snapshots/helm_backend/kubernetes_launch.txt
+64 -61
View File
@@ -15,100 +15,103 @@ run:
env:
CUDA_DEVICE_MAX_CONNECTIONS: '1'
PYTHONUNBUFFERED: '1'
stateFile: <SANDBOX>/cluster-storage/miles_data/miles-runs/260101-000000-000/state/orchestrator-260101-000000-000001.state
id: 260101-000000-000
id: '260101-000000-000'
inferenceEngines:
- command:
- python
- -m
- sglang.launch_server
- --node-rank
- $(LWS_WORKER_INDEX)
- --dist-init-addr
- $(LWS_LEADER_ADDRESS):9000
- 'python'
- '-m'
- 'sglang.launch_server'
- '--node-rank'
- '$(LWS_WORKER_INDEX)'
- '--dist-init-addr'
- '$(LWS_LEADER_ADDRESS):9000'
env:
NVSHMEM_DISABLE_NCCL: '1'
meta:
gpu_ids: 0,1,2,3,4,5,6,7
name: inference-engine-0-0
objectName: miles-run-260101-000000-000-all-inference-engine-0-0
gpu_ids: '0,1,2,3,4,5,6,7'
name: 'inference-engine-0-0'
objectName: 'miles-run-26-98cf6da492ec-inference-engine-0-0'
poolId: 'inference-engine-0-0'
ports:
- name: primary
- name: 'primary'
port: 8000
- name: dist-init
- name: 'dist-init'
port: 9000
replicas: 2
resources:
limits:
nvidia.com/gpu: 8
size: 2
poolId: inference-engine-0-0
launchRecord: '<SANDBOX>/cluster-storage/miles_data/miles-runs/260101-000000-000/launches/launch-260101-000000-000001.json'
objectNames:
colocatePairing: miles-run-260101-000000-000-all-colocate-pairing
mooncakeMaster: miles-run-260101-000000-000-all-mooncake-master
orchestrator: miles-run-260101-000000-000-all-orchestrator
uninstall: miles-run-260101-000000-000-all-uninstall
uninstallManifest: miles-run-260101-000000-000-all-uninstall-manifest
colocatePairing: 'miles-run-260101-98cf6da492ec-colocate-pairing'
mooncakeMaster: 'miles-run-260101-98cf6da492ec-mooncake-master'
orchestrator: 'miles-run-260101-000000-000-all-orchestrator'
platformRead: 'miles-run-260101-000000-000-all-platform-read'
platformReadDelete: 'miles-run-26-98cf6da492ec-platform-read-delete'
uninstall: 'miles-run-260101-000000-000-all-uninstall'
uninstallManifest: 'miles-run-2601-98cf6da492ec-uninstall-manifest'
orchestrator:
command:
- python
- /repo/train.py
- --rollout-num-gpus
- 'python'
- '/repo/train.py'
- '--rollout-num-gpus'
- '8'
- --cluster-backend
- kubernetes
- --run-uuid
- f52ecf8e9d7d4889
- '--cluster-backend'
- 'kubernetes'
- '--run-uuid'
- 'f52ecf8e9d7d4889'
stateFile: '<SANDBOX>/cluster-storage/miles_data/miles-runs/260101-000000-000/state/orchestrator-260101-000000-000001.state'
staticWorkers:
- command:
- python
- -m
- sglang_router.launch_router
- --host
- 0.0.0.0
- --port
- 'python'
- '-m'
- 'sglang_router.launch_router'
- '--host'
- '0.0.0.0'
- '--port'
- '30000'
name: inference-router-0
objectName: miles-run-260101-000000-000-all-inference-router-0
name: 'inference-router-0'
objectName: 'miles-run-2601-98cf6da492ec-inference-router-0'
poolId: 'inference-router-0'
ports:
- name: primary
- name: 'primary'
port: 30000
replicas: 1
poolId: inference-router-0
trainerEngines:
- command:
- <PYTHON>
- -m
- miles.utils.workers.process_supervisor
- --num-subprocesses
- '<PYTHON>'
- '-m'
- 'miles.utils.workers.process_supervisor'
- '--num-subprocesses'
- '8'
- --
- <PYTHON>
- -m
- miles.utils.workers.serving.serve
- --specs
- miles.ray.specs.entrypoint.compute_specs_from_argv
- --pool-id
- trainer-engine-actor
- --
- --rollout-num-gpus
- '--'
- '<PYTHON>'
- '-m'
- 'miles.utils.workers.serving.serve'
- '--specs'
- 'miles.ray.specs.entrypoint.compute_specs_from_argv'
- '--pool-id'
- 'trainer-engine-actor'
- '--'
- '--rollout-num-gpus'
- '8'
- --cluster-backend
- kubernetes
- --run-uuid
- f52ecf8e9d7d4889
- '--cluster-backend'
- 'kubernetes'
- '--run-uuid'
- 'f52ecf8e9d7d4889'
meta:
gpu_ids: 0,1,2,3,4,5,6,7
name: trainer-engine-actor
objectName: miles-run-260101-000000-000-all-trainer-engine-actor
gpu_ids: '0,1,2,3,4,5,6,7'
name: 'trainer-engine-actor'
objectName: 'miles-run-26-98cf6da492ec-trainer-engine-actor'
poolId: 'trainer-engine-actor'
ports:
- name: master
- name: 'master'
port: 9000
- name: rpc
- name: 'rpc'
port: 8000
replicas: 2
resources:
limits:
nvidia.com/gpu: 8
poolId: trainer-engine-actor