mirror of
https://github.com/lightseekorg/tokenspeed.git
synced 2026-11-21 01:53:23 +08:00
fix(memory): scope expandable segments around memory saver operations (#3640)
Enabling expandable CUDA segments together with memory saver can fail initialization or bypass the allocator that tracks managed storage. This change temporarily disables expandable segments during managed weight/KV and persistent-buffer allocations, CUDA graph capture, and pause/resume. It restores the effective allocator configuration and both environment aliases afterward, including on exceptions and when the last configuration string omits the active expandable setting. The allocator setting is process-wide. The adapter serializes its own scopes, and callers must keep unrelated CUDA allocation threads idle during startup or drained maintenance. Validation: - `pre-commit run --all-files` passed. - H200 with PyTorch 2.9.0 and `torch_memory_saver` 0.0.10: 19 passed, 1 skipped. The skipped case uses `PYTORCH_ALLOC_CONF`, which does not enable expandable segments in the tested Torch build. - GPU tests cover expandable allocations outside managed scopes, weight/KV/buffer isolation, graph replay, three pause/resume cycles, exception and sticky-setting restoration, and normal process exit. --------- Signed-off-by: lyzuu <349859374+lyzuu@users.noreply.github.com> Co-authored-by: lyzuu <349859374+lyzuu@users.noreply.github.com> Co-authored-by: longcat-ia-team <312376714+longcat-ia-team@users.noreply.github.com>
This commit is contained in:
co-authored by
lyzuu
longcat-ia-team
parent
bc61c9c3fe
commit
943bef04fb
@@ -98,6 +98,11 @@ The upstream implementation at
|
||||
contains both behaviors and can be built against the deployment's CUDA toolkit.
|
||||
Record the exact source and wheel hashes when using a local build.
|
||||
|
||||
With the native CUDA allocator, TokenSpeed temporarily disables expandable
|
||||
segments in memory-saver allocation, graph capture, pause, and resume scopes.
|
||||
Other allocations can still use expandable segments. PyTorch's setting is
|
||||
process-wide, so other CUDA allocation threads must be idle during these scopes.
|
||||
|
||||
When CUDA graphs operate on remapped storage, set `NCCL_GRAPH_REGISTER=0` before
|
||||
launch so NCCL does not retain graph registrations across remapping. This does
|
||||
not suspend NCCL or release its communication buffers.
|
||||
|
||||
@@ -20,13 +20,56 @@
|
||||
|
||||
"""Map runtime allocation owners to shared pools and memory-saver lifecycle tags."""
|
||||
|
||||
import os
|
||||
import re
|
||||
from abc import ABC
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from threading import RLock
|
||||
|
||||
import torch
|
||||
|
||||
_allocator_lock = RLock()
|
||||
_allocator_env = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _non_expandable_allocation():
|
||||
if (
|
||||
torch.version.cuda is None
|
||||
or torch.cuda.memory.get_allocator_backend() != "native"
|
||||
):
|
||||
yield
|
||||
return
|
||||
# PyTorch's setting is process-wide. Serialize saver scopes at startup and
|
||||
# during drained maintenance, when other CUDA allocators are idle.
|
||||
with _allocator_lock:
|
||||
settings = torch.cuda.memory._snapshot()["allocator_settings"]
|
||||
config = settings["PYTORCH_CUDA_ALLOC_CONF"]
|
||||
expandable = settings["expandable_segments"]
|
||||
previous = {name: os.environ.get(name) for name in _allocator_env}
|
||||
try:
|
||||
if expandable:
|
||||
torch.cuda.memory._set_allocator_settings(
|
||||
f"{config},expandable_segments:False"
|
||||
)
|
||||
for name, value in previous.items():
|
||||
if value is not None:
|
||||
os.environ[name] = re.sub(
|
||||
r"(expandable_segments\s*:\s*)True\b", r"\g<1>False", value
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
for name, value in previous.items():
|
||||
if value is not None:
|
||||
os.environ[name] = value
|
||||
if expandable:
|
||||
# The last config string may omit the still-active setting.
|
||||
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
|
||||
torch.cuda.memory._set_allocator_settings(config)
|
||||
|
||||
|
||||
_NAMED_REGION_POOLS = {
|
||||
"executor_inputs": "persistent_buffers",
|
||||
"runtime_states": "persistent_buffers",
|
||||
@@ -126,7 +169,9 @@ class _TorchMemorySaverAdapterReal(TorchMemorySaverAdapter):
|
||||
return
|
||||
token = _active_buffer_pool.set(pool)
|
||||
try:
|
||||
with pool.region(tag="weights", enable_cpu_backup=True):
|
||||
with _non_expandable_allocation(), pool.region(
|
||||
tag="weights", enable_cpu_backup=True
|
||||
):
|
||||
yield
|
||||
finally:
|
||||
_active_buffer_pool.reset(token)
|
||||
@@ -144,7 +189,7 @@ class _TorchMemorySaverAdapterReal(TorchMemorySaverAdapter):
|
||||
pending: list[Callable[[], None]] = []
|
||||
token = _communication_initializers.set(pending)
|
||||
try:
|
||||
with _primary_memory_saver.region(
|
||||
with _non_expandable_allocation(), _primary_memory_saver.region(
|
||||
tag=tag or "default", enable_cpu_backup=enable_cpu_backup
|
||||
):
|
||||
yield
|
||||
@@ -157,6 +202,7 @@ class _TorchMemorySaverAdapterReal(TorchMemorySaverAdapter):
|
||||
|
||||
return active_region()
|
||||
|
||||
@contextmanager
|
||||
def cuda_graph(self, graph, *, pool, stream, tag: str):
|
||||
"""Back up graph RNG state before capturing discardable graph buffers."""
|
||||
# PyTorch creates seed/offset storage when the generator is first
|
||||
@@ -167,19 +213,22 @@ class _TorchMemorySaverAdapterReal(TorchMemorySaverAdapter):
|
||||
)
|
||||
# Requires the library's preload hook mode, which configure_subprocess
|
||||
# sets up for the scheduler processes.
|
||||
return _primary_memory_saver.cuda_graph(
|
||||
with _non_expandable_allocation(), _primary_memory_saver.cuda_graph(
|
||||
cuda_graph=graph,
|
||||
pool=pool,
|
||||
stream=stream,
|
||||
tag=tag,
|
||||
enable_cpu_backup=False,
|
||||
)
|
||||
):
|
||||
yield
|
||||
|
||||
def pause(self, tag: str | None = None):
|
||||
return _primary_memory_saver.pause(tag=tag)
|
||||
with _non_expandable_allocation():
|
||||
return _primary_memory_saver.pause(tag=tag)
|
||||
|
||||
def resume(self, tag: str | None = None):
|
||||
return _primary_memory_saver.resume(tag=tag)
|
||||
with _non_expandable_allocation():
|
||||
return _primary_memory_saver.resume(tag=tag)
|
||||
|
||||
|
||||
class _TorchMemorySaverAdapterNoop(TorchMemorySaverAdapter):
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Verify scoped allocator compatibility with the real memory saver in a child."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
from test.ci_system.ci_register import register_cuda_ci
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
|
||||
from tokenspeed.runtime.utils.torch_memory_saver_adapter import (
|
||||
TorchMemorySaverAdapter,
|
||||
initialize_communication,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=30, suite="runtime-1gpu")
|
||||
|
||||
|
||||
_CONFIG = (
|
||||
"max_split_size_mb:128,roundup_power2_divisions:[256:1,512:2,>:4],"
|
||||
"expandable_segments:True"
|
||||
)
|
||||
_ENV_NAMES = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")
|
||||
|
||||
|
||||
def worker(configuration):
|
||||
torch.cuda.set_device(0)
|
||||
if configuration in ("runtime_override", "sticky"):
|
||||
torch.cuda.memory._set_allocator_settings(_CONFIG)
|
||||
if configuration == "sticky":
|
||||
torch.cuda.memory._set_allocator_settings("max_split_size_mb:128")
|
||||
original_settings = torch.cuda.memory._snapshot()["allocator_settings"]
|
||||
original_env = {name: os.environ.get(name) for name in _ENV_NAMES}
|
||||
if (
|
||||
configuration == "PYTORCH_ALLOC_CONF"
|
||||
and not original_settings["expandable_segments"]
|
||||
):
|
||||
return False
|
||||
assert original_settings["expandable_segments"]
|
||||
saver = TorchMemorySaverAdapter.create(enable=True)
|
||||
|
||||
def assert_restored():
|
||||
assert torch.cuda.memory._snapshot()["allocator_settings"] == original_settings
|
||||
assert {name: os.environ.get(name) for name in _ENV_NAMES} == original_env
|
||||
|
||||
before = torch.full((24 << 20,), 17, dtype=torch.uint8, device="cuda")
|
||||
# Even a failed load must restore settings and leave the next scope usable.
|
||||
with pytest.raises(RuntimeError, match="load failed"):
|
||||
with saver.region(tag="weights", enable_cpu_backup=True):
|
||||
weights = torch.full((8 << 20,), 5, dtype=torch.float32, device="cuda")
|
||||
raise RuntimeError("load failed")
|
||||
assert_restored()
|
||||
communication = []
|
||||
with saver.region(tag="kv_cache", enable_cpu_backup=False):
|
||||
kv = torch.full((32 << 20,), 9, dtype=torch.uint8, device="cuda")
|
||||
initialize_communication(
|
||||
lambda: communication.append(
|
||||
torch.full((24 << 20,), 19, dtype=torch.uint8, device="cuda")
|
||||
)
|
||||
)
|
||||
assert_restored()
|
||||
after = communication[0]
|
||||
with saver.named_region("attention_state"):
|
||||
with saver.named_region("workspace"):
|
||||
constant = torch.full((1 << 20,), 11, dtype=torch.uint8, device="cuda")
|
||||
assert_restored()
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
warmup = weights[:1024] * 2
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
del warmup
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with saver.cuda_graph(
|
||||
graph, pool=torch.cuda.graph_pool_handle(), stream=stream, tag="cuda_graph"
|
||||
):
|
||||
output = weights[:1024] * 2
|
||||
assert_restored()
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
assert torch.all(output == 10).item()
|
||||
|
||||
segments = torch.cuda.memory_snapshot()
|
||||
for tensor, expandable in (
|
||||
(before, True),
|
||||
(after, True),
|
||||
(weights, False),
|
||||
(kv, False),
|
||||
(output, False),
|
||||
(constant, False),
|
||||
):
|
||||
segment = next(
|
||||
s
|
||||
for s in segments
|
||||
if s["address"] <= tensor.data_ptr() < s["address"] + s["total_size"]
|
||||
)
|
||||
assert segment["is_expandable"] is expandable
|
||||
tensors = (weights, kv, output, before, after)
|
||||
pointers = [t.data_ptr() for t in tensors]
|
||||
for cycle in range(3):
|
||||
torch.cuda.synchronize()
|
||||
free_before, _ = torch.cuda.mem_get_info()
|
||||
for tag in ("cuda_graph", "kv_cache", "weights"):
|
||||
saver.pause(tag=tag)
|
||||
assert_restored()
|
||||
free_paused, _ = torch.cuda.mem_get_info()
|
||||
assert free_paused - free_before >= 32 << 20
|
||||
assert torch.all(before == 17).item()
|
||||
assert torch.all(after == 19).item()
|
||||
for tag in ("weights", "kv_cache", "cuda_graph"):
|
||||
saver.resume(tag=tag)
|
||||
assert_restored()
|
||||
assert pointers == [t.data_ptr() for t in tensors]
|
||||
assert torch.all(weights == 5).item()
|
||||
assert torch.all(constant == 11).item()
|
||||
kv.fill_(cycle + 1)
|
||||
assert torch.all(kv == cycle + 1).item()
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
assert torch.all(output == 10).item()
|
||||
print(
|
||||
f"cycle={cycle}: released={free_paused - free_before} bytes; data, pointers, graph and configuration restored",
|
||||
flush=True,
|
||||
)
|
||||
print("Scoped expandable allocator compatibility passed", flush=True)
|
||||
return True
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.version.hip is not None,
|
||||
reason="requires NVIDIA CUDA memory saver",
|
||||
)
|
||||
@pytest.mark.parametrize("configuration", [*_ENV_NAMES, "runtime_override", "sticky"])
|
||||
def test_expandable_allocations_around_managed_storage(configuration):
|
||||
pytest.importorskip("torch_memory_saver")
|
||||
if Version(version("torch-memory-saver")) < Version("0.0.10"):
|
||||
pytest.skip("requires memory saver with managed-pointer teardown support")
|
||||
with TorchMemorySaverAdapter.create(enable=True).configure_subprocess():
|
||||
env = dict(os.environ)
|
||||
for name in _ENV_NAMES:
|
||||
env.pop(name, None)
|
||||
if configuration in _ENV_NAMES:
|
||||
env[configuration] = _CONFIG
|
||||
else:
|
||||
env["PYTORCH_ALLOC_CONF"] = "expandable_segments:False"
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"test.runtime.distributed.test_memory_saver_expandable",
|
||||
configuration,
|
||||
],
|
||||
cwd=Path(__file__).resolve().parents[3],
|
||||
env=env,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=180,
|
||||
)
|
||||
if result.returncode == 77 and configuration == "PYTORCH_ALLOC_CONF":
|
||||
pytest.skip("this PyTorch build only reads PYTORCH_CUDA_ALLOC_CONF")
|
||||
assert result.returncode == 0, result.stdout
|
||||
assert "Scoped expandable allocator compatibility passed" in result.stdout
|
||||
print(result.stdout, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(0 if worker(sys.argv[1]) else 77)
|
||||
Reference in New Issue
Block a user