fix(memory): scope expandable segments around memory saver operations (#3640)

Enabling expandable CUDA segments together with memory saver can fail
initialization or bypass the allocator that tracks managed storage. This
change temporarily disables expandable segments during managed weight/KV
and persistent-buffer allocations, CUDA graph capture, and pause/resume.
It restores the effective allocator configuration and both environment
aliases afterward, including on exceptions and when the last
configuration string omits the active expandable setting.

The allocator setting is process-wide. The adapter serializes its own
scopes, and callers must keep unrelated CUDA allocation threads idle
during startup or drained maintenance.

Validation:
- `pre-commit run --all-files` passed.
- H200 with PyTorch 2.9.0 and `torch_memory_saver` 0.0.10: 19 passed, 1
skipped. The skipped case uses `PYTORCH_ALLOC_CONF`, which does not
enable expandable segments in the tested Torch build.
- GPU tests cover expandable allocations outside managed scopes,
weight/KV/buffer isolation, graph replay, three pause/resume cycles,
exception and sticky-setting restoration, and normal process exit.

---------

Signed-off-by: lyzuu <349859374+lyzuu@users.noreply.github.com>
Co-authored-by: lyzuu <349859374+lyzuu@users.noreply.github.com>
Co-authored-by: longcat-ia-team <312376714+longcat-ia-team@users.noreply.github.com>
This commit is contained in:
lyzuu
2026-11-19 14:55:22 -08:00
committed by GitHub
co-authored by lyzuu longcat-ia-team
parent bc61c9c3fe
commit 943bef04fb
3 changed files with 251 additions and 6 deletions
+5
View File
@@ -98,6 +98,11 @@ The upstream implementation at
contains both behaviors and can be built against the deployment's CUDA toolkit.
Record the exact source and wheel hashes when using a local build.
With the native CUDA allocator, TokenSpeed temporarily disables expandable
segments in memory-saver allocation, graph capture, pause, and resume scopes.
Other allocations can still use expandable segments. PyTorch's setting is
process-wide, so other CUDA allocation threads must be idle during these scopes.
When CUDA graphs operate on remapped storage, set `NCCL_GRAPH_REGISTER=0` before
launch so NCCL does not retain graph registrations across remapping. This does
not suspend NCCL or release its communication buffers.
@@ -20,13 +20,56 @@
"""Map runtime allocation owners to shared pools and memory-saver lifecycle tags."""
import os
import re
from abc import ABC
from collections.abc import Callable
from contextlib import contextmanager
from contextvars import ContextVar
from threading import RLock
import torch
_allocator_lock = RLock()
_allocator_env = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")
@contextmanager
def _non_expandable_allocation():
if (
torch.version.cuda is None
or torch.cuda.memory.get_allocator_backend() != "native"
):
yield
return
# PyTorch's setting is process-wide. Serialize saver scopes at startup and
# during drained maintenance, when other CUDA allocators are idle.
with _allocator_lock:
settings = torch.cuda.memory._snapshot()["allocator_settings"]
config = settings["PYTORCH_CUDA_ALLOC_CONF"]
expandable = settings["expandable_segments"]
previous = {name: os.environ.get(name) for name in _allocator_env}
try:
if expandable:
torch.cuda.memory._set_allocator_settings(
f"{config},expandable_segments:False"
)
for name, value in previous.items():
if value is not None:
os.environ[name] = re.sub(
r"(expandable_segments\s*:\s*)True\b", r"\g<1>False", value
)
yield
finally:
for name, value in previous.items():
if value is not None:
os.environ[name] = value
if expandable:
# The last config string may omit the still-active setting.
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
torch.cuda.memory._set_allocator_settings(config)
_NAMED_REGION_POOLS = {
"executor_inputs": "persistent_buffers",
"runtime_states": "persistent_buffers",
@@ -126,7 +169,9 @@ class _TorchMemorySaverAdapterReal(TorchMemorySaverAdapter):
return
token = _active_buffer_pool.set(pool)
try:
with pool.region(tag="weights", enable_cpu_backup=True):
with _non_expandable_allocation(), pool.region(
tag="weights", enable_cpu_backup=True
):
yield
finally:
_active_buffer_pool.reset(token)
@@ -144,7 +189,7 @@ class _TorchMemorySaverAdapterReal(TorchMemorySaverAdapter):
pending: list[Callable[[], None]] = []
token = _communication_initializers.set(pending)
try:
with _primary_memory_saver.region(
with _non_expandable_allocation(), _primary_memory_saver.region(
tag=tag or "default", enable_cpu_backup=enable_cpu_backup
):
yield
@@ -157,6 +202,7 @@ class _TorchMemorySaverAdapterReal(TorchMemorySaverAdapter):
return active_region()
@contextmanager
def cuda_graph(self, graph, *, pool, stream, tag: str):
"""Back up graph RNG state before capturing discardable graph buffers."""
# PyTorch creates seed/offset storage when the generator is first
@@ -167,19 +213,22 @@ class _TorchMemorySaverAdapterReal(TorchMemorySaverAdapter):
)
# Requires the library's preload hook mode, which configure_subprocess
# sets up for the scheduler processes.
return _primary_memory_saver.cuda_graph(
with _non_expandable_allocation(), _primary_memory_saver.cuda_graph(
cuda_graph=graph,
pool=pool,
stream=stream,
tag=tag,
enable_cpu_backup=False,
)
):
yield
def pause(self, tag: str | None = None):
return _primary_memory_saver.pause(tag=tag)
with _non_expandable_allocation():
return _primary_memory_saver.pause(tag=tag)
def resume(self, tag: str | None = None):
return _primary_memory_saver.resume(tag=tag)
with _non_expandable_allocation():
return _primary_memory_saver.resume(tag=tag)
class _TorchMemorySaverAdapterNoop(TorchMemorySaverAdapter):
@@ -0,0 +1,191 @@
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Verify scoped allocator compatibility with the real memory saver in a child."""
import os
import subprocess
import sys
from importlib.metadata import version
from pathlib import Path
from test.ci_system.ci_register import register_cuda_ci
import pytest
import torch
from packaging.version import Version
from tokenspeed.runtime.utils.torch_memory_saver_adapter import (
TorchMemorySaverAdapter,
initialize_communication,
)
register_cuda_ci(est_time=30, suite="runtime-1gpu")
_CONFIG = (
"max_split_size_mb:128,roundup_power2_divisions:[256:1,512:2,>:4],"
"expandable_segments:True"
)
_ENV_NAMES = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")
def worker(configuration):
torch.cuda.set_device(0)
if configuration in ("runtime_override", "sticky"):
torch.cuda.memory._set_allocator_settings(_CONFIG)
if configuration == "sticky":
torch.cuda.memory._set_allocator_settings("max_split_size_mb:128")
original_settings = torch.cuda.memory._snapshot()["allocator_settings"]
original_env = {name: os.environ.get(name) for name in _ENV_NAMES}
if (
configuration == "PYTORCH_ALLOC_CONF"
and not original_settings["expandable_segments"]
):
return False
assert original_settings["expandable_segments"]
saver = TorchMemorySaverAdapter.create(enable=True)
def assert_restored():
assert torch.cuda.memory._snapshot()["allocator_settings"] == original_settings
assert {name: os.environ.get(name) for name in _ENV_NAMES} == original_env
before = torch.full((24 << 20,), 17, dtype=torch.uint8, device="cuda")
# Even a failed load must restore settings and leave the next scope usable.
with pytest.raises(RuntimeError, match="load failed"):
with saver.region(tag="weights", enable_cpu_backup=True):
weights = torch.full((8 << 20,), 5, dtype=torch.float32, device="cuda")
raise RuntimeError("load failed")
assert_restored()
communication = []
with saver.region(tag="kv_cache", enable_cpu_backup=False):
kv = torch.full((32 << 20,), 9, dtype=torch.uint8, device="cuda")
initialize_communication(
lambda: communication.append(
torch.full((24 << 20,), 19, dtype=torch.uint8, device="cuda")
)
)
assert_restored()
after = communication[0]
with saver.named_region("attention_state"):
with saver.named_region("workspace"):
constant = torch.full((1 << 20,), 11, dtype=torch.uint8, device="cuda")
assert_restored()
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
warmup = weights[:1024] * 2
torch.cuda.current_stream().wait_stream(stream)
del warmup
graph = torch.cuda.CUDAGraph()
with saver.cuda_graph(
graph, pool=torch.cuda.graph_pool_handle(), stream=stream, tag="cuda_graph"
):
output = weights[:1024] * 2
assert_restored()
graph.replay()
torch.cuda.synchronize()
assert torch.all(output == 10).item()
segments = torch.cuda.memory_snapshot()
for tensor, expandable in (
(before, True),
(after, True),
(weights, False),
(kv, False),
(output, False),
(constant, False),
):
segment = next(
s
for s in segments
if s["address"] <= tensor.data_ptr() < s["address"] + s["total_size"]
)
assert segment["is_expandable"] is expandable
tensors = (weights, kv, output, before, after)
pointers = [t.data_ptr() for t in tensors]
for cycle in range(3):
torch.cuda.synchronize()
free_before, _ = torch.cuda.mem_get_info()
for tag in ("cuda_graph", "kv_cache", "weights"):
saver.pause(tag=tag)
assert_restored()
free_paused, _ = torch.cuda.mem_get_info()
assert free_paused - free_before >= 32 << 20
assert torch.all(before == 17).item()
assert torch.all(after == 19).item()
for tag in ("weights", "kv_cache", "cuda_graph"):
saver.resume(tag=tag)
assert_restored()
assert pointers == [t.data_ptr() for t in tensors]
assert torch.all(weights == 5).item()
assert torch.all(constant == 11).item()
kv.fill_(cycle + 1)
assert torch.all(kv == cycle + 1).item()
graph.replay()
torch.cuda.synchronize()
assert torch.all(output == 10).item()
print(
f"cycle={cycle}: released={free_paused - free_before} bytes; data, pointers, graph and configuration restored",
flush=True,
)
print("Scoped expandable allocator compatibility passed", flush=True)
return True
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.version.hip is not None,
reason="requires NVIDIA CUDA memory saver",
)
@pytest.mark.parametrize("configuration", [*_ENV_NAMES, "runtime_override", "sticky"])
def test_expandable_allocations_around_managed_storage(configuration):
pytest.importorskip("torch_memory_saver")
if Version(version("torch-memory-saver")) < Version("0.0.10"):
pytest.skip("requires memory saver with managed-pointer teardown support")
with TorchMemorySaverAdapter.create(enable=True).configure_subprocess():
env = dict(os.environ)
for name in _ENV_NAMES:
env.pop(name, None)
if configuration in _ENV_NAMES:
env[configuration] = _CONFIG
else:
env["PYTORCH_ALLOC_CONF"] = "expandable_segments:False"
result = subprocess.run(
[
sys.executable,
"-m",
"test.runtime.distributed.test_memory_saver_expandable",
configuration,
],
cwd=Path(__file__).resolve().parents[3],
env=env,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=180,
)
if result.returncode == 77 and configuration == "PYTORCH_ALLOC_CONF":
pytest.skip("this PyTorch build only reads PYTORCH_CUDA_ALLOC_CONF")
assert result.returncode == 0, result.stdout
assert "Scoped expandable allocator compatibility passed" in result.stdout
print(result.stdout, end="")
if __name__ == "__main__":
sys.exit(0 if worker(sys.argv[1]) else 77)