mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat: add --embed-weights host for a pinned host embedding table * refactor: check the tie from the model config and share the op walk
120 lines
3.8 KiB
Python
120 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
from abc import abstractmethod
|
|
from typing import Any, Dict, Generic, Iterator, List, TypeAlias, TypeVar
|
|
|
|
import torch
|
|
|
|
_STATE_DICT: TypeAlias = Dict[str, torch.Tensor]
|
|
|
|
|
|
def _concat_prefix(prefix: str, name: str) -> str:
|
|
return f"{prefix}.{name}" if prefix else name
|
|
|
|
|
|
class BaseOP:
|
|
@abstractmethod
|
|
def forward(self, *args: Any, **kwargs: Any) -> Any: ...
|
|
|
|
def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT:
|
|
result = result if result is not None else {}
|
|
|
|
for name, param in self.__dict__.items():
|
|
if name.startswith("_"):
|
|
continue
|
|
if isinstance(param, torch.Tensor):
|
|
result[_concat_prefix(prefix, name)] = param
|
|
elif isinstance(param, BaseOP):
|
|
param.state_dict(prefix=_concat_prefix(prefix, name), result=result)
|
|
|
|
return result
|
|
|
|
def load_state_dict(
|
|
self,
|
|
state_dict: _STATE_DICT,
|
|
*,
|
|
prefix: str = "",
|
|
_internal: bool = False,
|
|
) -> None:
|
|
for name, param in self.__dict__.items():
|
|
if name.startswith("_"):
|
|
continue
|
|
if isinstance(param, torch.Tensor):
|
|
item = state_dict.pop(_concat_prefix(prefix, name))
|
|
assert isinstance(item, torch.Tensor)
|
|
assert param.shape == item.shape and param.dtype == item.dtype
|
|
setattr(self, name, item)
|
|
elif isinstance(param, BaseOP):
|
|
param.load_state_dict(
|
|
state_dict, prefix=_concat_prefix(prefix, name), _internal=True
|
|
)
|
|
|
|
if not _internal and state_dict:
|
|
raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
|
|
|
|
|
|
class StateLessOP(BaseOP):
|
|
def __init__(self):
|
|
super().__init__()
|
|
|
|
def load_state_dict(
|
|
self,
|
|
state_dict: _STATE_DICT,
|
|
*,
|
|
prefix: str = "",
|
|
_internal: bool = False,
|
|
) -> None:
|
|
if not _internal and state_dict:
|
|
raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
|
|
|
|
def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT:
|
|
return result if result is not None else {}
|
|
|
|
|
|
T = TypeVar("T", bound=BaseOP)
|
|
|
|
|
|
class OPList(BaseOP, Generic[T]):
|
|
def __init__(self, ops: List[T]):
|
|
super().__init__()
|
|
self.op_list = ops
|
|
|
|
def state_dict(self, *, prefix: str = "", result: _STATE_DICT | None = None) -> _STATE_DICT:
|
|
result = result if result is not None else {}
|
|
for i, op in enumerate(self.op_list):
|
|
op.state_dict(prefix=_concat_prefix(prefix, str(i)), result=result)
|
|
return result
|
|
|
|
def load_state_dict(
|
|
self,
|
|
state_dict: _STATE_DICT,
|
|
*,
|
|
prefix: str = "",
|
|
_internal: bool = False,
|
|
) -> None:
|
|
for i, op in enumerate(self.op_list):
|
|
op.load_state_dict(state_dict, prefix=_concat_prefix(prefix, str(i)), _internal=True)
|
|
|
|
if not _internal and state_dict:
|
|
raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
|
|
|
|
|
|
def iter_ops(root: BaseOP) -> Iterator[BaseOP]:
|
|
"""Each op under ``root`` once, ``root`` first; an op's attributes are read after it is yielded."""
|
|
seen: set[int] = set()
|
|
|
|
def walk(op: BaseOP) -> Iterator[BaseOP]:
|
|
if id(op) in seen:
|
|
return
|
|
seen.add(id(op))
|
|
yield op
|
|
for value in vars(op).values():
|
|
if isinstance(value, BaseOP):
|
|
yield from walk(value)
|
|
elif isinstance(value, (list, tuple)):
|
|
for item in value:
|
|
if isinstance(item, BaseOP):
|
|
yield from walk(item)
|
|
|
|
return walk(root)
|