mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(server): add /v1/cache/flush to drop the prefix cache when idle * refactor(scheduler): run every cache flush at the idle safe point * test: cover cache flush on the overlap loop and on a live server
58 lines
1.1 KiB
Python
58 lines
1.1 KiB
Python
from .backend import (
|
|
AbortBackendMsg,
|
|
BaseBackendMsg,
|
|
BatchBackendMsg,
|
|
CacheRebuildBackendMsg,
|
|
ExitMsg,
|
|
FlushCacheBackendMsg,
|
|
MMItem,
|
|
UserMsg,
|
|
)
|
|
from .frontend import (
|
|
BaseFrontendMsg,
|
|
BatchFrontendMsg,
|
|
CacheRebuildReply,
|
|
FlushCacheReply,
|
|
UserReply,
|
|
)
|
|
from .tokenizer import (
|
|
AbortMsg,
|
|
BaseTokenizerMsg,
|
|
BatchTokenizerMsg,
|
|
CacheRebuildMsg,
|
|
CacheRebuildResultMsg,
|
|
DetokenizeMsg,
|
|
ErrorReplyMsg,
|
|
FlushCacheMsg,
|
|
FlushCacheResultMsg,
|
|
PromptAdmittedMsg,
|
|
TokenizeMsg,
|
|
)
|
|
|
|
__all__ = [
|
|
"AbortMsg",
|
|
"AbortBackendMsg",
|
|
"BaseBackendMsg",
|
|
"BatchBackendMsg",
|
|
"CacheRebuildBackendMsg",
|
|
"ExitMsg",
|
|
"FlushCacheBackendMsg",
|
|
"MMItem",
|
|
"UserMsg",
|
|
"BaseTokenizerMsg",
|
|
"BatchTokenizerMsg",
|
|
"CacheRebuildMsg",
|
|
"CacheRebuildResultMsg",
|
|
"DetokenizeMsg",
|
|
"ErrorReplyMsg",
|
|
"FlushCacheMsg",
|
|
"FlushCacheResultMsg",
|
|
"PromptAdmittedMsg",
|
|
"TokenizeMsg",
|
|
"BaseFrontendMsg",
|
|
"BatchFrontendMsg",
|
|
"CacheRebuildReply",
|
|
"FlushCacheReply",
|
|
"UserReply",
|
|
]
|