mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
77 lines
3.3 KiB
Python
77 lines
3.3 KiB
Python
"""Normalize a serve's raw stdout/stderr byte stream into display lines.
|
|
|
|
Two jobs, kept pure (no clock, no IO — so they unit-test trivially and the caller owns any
|
|
throttling): strip ANSI control sequences, and turn a terminal-style stream (``\\n`` newlines +
|
|
``\\r`` in-place redraws, as tqdm emits) into ("line" | "partial", text) events. A ``\\r`` marks
|
|
the current line as displayed-then-overwritten → emitted as a ``partial`` (the caller may throttle
|
|
these); a ``\\n`` finalizes a ``line``; ``\\r\\n`` is treated as a single newline.
|
|
|
|
Lives in the torch-free ``daemon`` package and imports only ``re`` — never anything from
|
|
``freetoken.server`` / ``freetoken.utils`` (both pull torch/transformers at import).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
# CSI/escape sequences: ESC [ ... final-byte, plus a few standalone two-char escapes. Enough to
|
|
# clean tqdm bars and coloured log lines without a full terminal emulator.
|
|
_ANSI_RE = re.compile(r"\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
|
|
|
|
|
|
def strip_ansi(text: str) -> str:
|
|
return _ANSI_RE.sub("", text)
|
|
|
|
|
|
class LineAssembler:
|
|
"""Feed raw decoded chunks; get back finalized display lines. Stateful across chunks (a line
|
|
may span reads) but deterministic and clock-free. ``feed`` returns a list of (kind, text);
|
|
``flush`` emits whatever trails after EOF."""
|
|
|
|
__slots__ = ("_cur", "_cr")
|
|
|
|
def __init__(self) -> None:
|
|
self._cur: list[str] = []
|
|
self._cr = False # a bare '\r' was seen; the next char decides newline-vs-redraw
|
|
|
|
def feed(self, text: str) -> list[tuple[str, str]]:
|
|
out: list[tuple[str, str]] = []
|
|
for ch in text:
|
|
if ch == "\r":
|
|
# Coalesce a RUN of '\r'. A lone '\r' is a tqdm redraw (cursor→col 0); a run is
|
|
# still just "return to col 0", so it collapses to one. This is what makes Windows
|
|
# '\r\r\n' behave like '\r\n': the child's text-mode stdout adds a CR to output that
|
|
# already ended in CRLF, and without coalescing the first '\r' was misread as a
|
|
# redraw (emitting the real line as a throttled 'partial' plus a spurious empty
|
|
# 'line' — the phantom blank rows in the Logs view).
|
|
self._cr = True
|
|
continue
|
|
if self._cr:
|
|
self._cr = False
|
|
if ch == "\n":
|
|
# '\r'+…+'\n' — a single newline (covers '\r\n' and Windows '\r\r\n').
|
|
out.append(("line", "".join(self._cur)))
|
|
self._cur = []
|
|
continue
|
|
# bare '\r' run then a real char: the line was drawn, then overwritten (redraw).
|
|
out.append(("partial", "".join(self._cur)))
|
|
self._cur = [ch]
|
|
continue
|
|
if ch == "\n":
|
|
out.append(("line", "".join(self._cur)))
|
|
self._cur = []
|
|
else:
|
|
self._cur.append(ch)
|
|
return out
|
|
|
|
def flush(self) -> list[tuple[str, str]]:
|
|
out: list[tuple[str, str]] = []
|
|
if self._cr:
|
|
self._cr = False
|
|
out.append(("partial", "".join(self._cur)))
|
|
self._cur = []
|
|
elif self._cur:
|
|
out.append(("line", "".join(self._cur)))
|
|
self._cur = []
|
|
return out
|