Files
FreeToken/python/freetoken/layers/base.py
Xiaoze Fan fd863cce3e feat: add --embed-weights host for a pinned host embedding table (#806)
* feat: add --embed-weights host for a pinned host embedding table

* refactor: check the tie from the model config and share the op walk
2026-10-23 14:41:37 -07:00

120 lines
3.8 KiB
Python

from __future__ import annotations
from abc import abstractmethod
from typing import Any, Dict, Generic, Iterator, List, TypeAlias, TypeVar
import torch
_STATE_DICT: TypeAlias = Dict[str, torch.Tensor]
def _concat_prefix(prefix: str, name: str) -> str:
return f"{prefix}.{name}" if prefix else name
class BaseOP:
@abstractmethod
def forward(self, *args: Any, **kwargs: Any) -> Any: ...
def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT:
result = result if result is not None else {}
for name, param in self.__dict__.items():
if name.startswith("_"):
continue
if isinstance(param, torch.Tensor):
result[_concat_prefix(prefix, name)] = param
elif isinstance(param, BaseOP):
param.state_dict(prefix=_concat_prefix(prefix, name), result=result)
return result
def load_state_dict(
self,
state_dict: _STATE_DICT,
*,
prefix: str = "",
_internal: bool = False,
) -> None:
for name, param in self.__dict__.items():
if name.startswith("_"):
continue
if isinstance(param, torch.Tensor):
item = state_dict.pop(_concat_prefix(prefix, name))
assert isinstance(item, torch.Tensor)
assert param.shape == item.shape and param.dtype == item.dtype
setattr(self, name, item)
elif isinstance(param, BaseOP):
param.load_state_dict(
state_dict, prefix=_concat_prefix(prefix, name), _internal=True
)
if not _internal and state_dict:
raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
class StateLessOP(BaseOP):
def __init__(self):
super().__init__()
def load_state_dict(
self,
state_dict: _STATE_DICT,
*,
prefix: str = "",
_internal: bool = False,
) -> None:
if not _internal and state_dict:
raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT:
return result if result is not None else {}
T = TypeVar("T", bound=BaseOP)
class OPList(BaseOP, Generic[T]):
def __init__(self, ops: List[T]):
super().__init__()
self.op_list = ops
def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT:
result = result if result is not None else {}
for i, op in enumerate(self.op_list):
op.state_dict(prefix=_concat_prefix(prefix, str(i)), result=result)
return result
def load_state_dict(
self,
state_dict: _STATE_DICT,
*,
prefix: str = "",
_internal: bool = False,
) -> None:
for i, op in enumerate(self.op_list):
op.load_state_dict(state_dict, prefix=_concat_prefix(prefix, str(i)), _internal=True)
if not _internal and state_dict:
raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
def iter_ops(root: BaseOP) -> Iterator[BaseOP]:
"""Each op under ``root`` once, ``root`` first; an op's attributes are read after it is yielded."""
seen: set[int] = set()
def walk(op: BaseOP) -> Iterator[BaseOP]:
if id(op) in seen:
return
seen.add(id(op))
yield op
for value in vars(op).values():
if isinstance(value, BaseOP):
yield from walk(value)
elif isinstance(value, (list, tuple)):
for item in value:
if isinstance(item, BaseOP):
yield from walk(item)
return walk(root)