"""Python face of the disk row store extension (``kernel/csrc/row_store``) and its CUDA-graph sync. ``RowStore`` reads fixed-width rows straight out of checkpoint shard files (io_uring or a pread pool, O_DIRECT) into pinned staging by row id; ``PleStore`` adds PLE's n-gram hashing. Graph sync: a captured graph cannot wait for host I/O, so a captured ``lookup`` WAITs on a host-visible flag (pinned on CUDA, signal memory on HIP) through a stream memop (``wait_reset``). The host ``signal``s once the fill landed; where the driver rejects memops in capture, the caller falls back to launch-gating (fill before replay). ``probe_wait_sync`` decides which, once, at startup. """ from __future__ import annotations import torch from freetoken.kernel import _row_store from freetoken.kernel.pinned import alloc_pinned_tensor RowStore = _row_store.RowStore PleStore = _row_store.PleStore def alloc_sync_flag() -> torch.Tensor: if torch.version.hip is not None: return _row_store.alloc_signal_tensor() return alloc_pinned_tensor(1, dtype=torch.int64).zero_() def probe_wait_sync(mode: str, device: torch.device) -> bool: """``mode``: ``auto`` (probe), ``wait`` (require memops) or ``gate`` (never use them).""" if mode not in ("auto", "wait", "gate"): raise ValueError(f"unknown row-store sync mode {mode!r}; expected auto, wait or gate") if mode == "gate": return False if torch.version.hip is not None: with torch.cuda.device(device): ok = _row_store.capture_memops_supported() else: scratch = alloc_pinned_tensor(1, dtype=torch.int64) scratch.zero_() stream = torch.cuda.current_stream(device) ok = ( _row_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0 and _row_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0 ) if ok: stream.synchronize() ok = int(scratch[0]) == 7 if mode == "wait" and not ok: raise RuntimeError("wait-sync requested but stream memops are unavailable") return ok def wait_reset(stream: torch.cuda.Stream, flag: torch.Tensor) -> None: """Enqueue WAIT(flag >= 1) then flag := 0 on ``stream`` (capturable).""" _row_store.memop_wait_reset(stream.cuda_stream, flag.data_ptr()) def signal(flag: torch.Tensor) -> None: """Host side: release a pending WAIT (a release-store of 1).""" _row_store.signal_flag(flag.data_ptr()) __all__ = ["RowStore", "PleStore", "alloc_sync_flag", "probe_wait_sync", "wait_reset", "signal"]