# SPDX-License-Identifier: Apache-2.0 # Renderer-probe approach adapted from vLLM PR #58772. """Preserve or fold late system instructions without hoisting the prompt head.""" from __future__ import annotations from copy import deepcopy from typing import Any, Callable # Allow small role-framing differences, not a repeated environment header. _INLINE_ROLE_SLACK_TOKENS = 8 class InlineSystemError(ValueError): """An input instruction cannot be placed without corrupting tool ordering.""" def has_inline_system(messages: list[dict]) -> bool: started = False for message in messages: if message.get("role") == "system": if started: return True else: started = True return False def _add_text(message: dict, text: str, *, prepend: bool = False) -> dict: result = dict(message) content = result.get("content") def join(a, b): return "\n\n".join(part for part in (a, b) if part) if content is None or isinstance(content, str): result["content"] = join(text, content) if prepend else join(content, text) elif isinstance(content, list): parts = deepcopy(content) index = 0 if prepend else -1 if parts and parts[index].get("type") == "text": previous = parts[index].get("text") or "" parts[index]["text"] = join(text, previous) if prepend else join(previous, text) else: parts.insert(0 if prepend else len(parts), {"type": "text", "text": text}) result["content"] = parts else: raise InlineSystemError("inline system folding requires text or content blocks") return result def _after_tool_results(messages: list[dict]) -> list[dict]: out, held = [], [] pending_ids: set[str] = set() for message in messages: role = message.get("role") if role == "system" and pending_ids: held.append(message) continue if role == "tool": out.append(message) pending_ids.discard(message.get("tool_call_id", "")) if not pending_ids: out.extend(held) held.clear() continue if pending_ids and held: raise InlineSystemError("inline system instruction interrupts incomplete tool results") out.append(message) pending_ids = {call.get("id", "") for call in message.get("tool_calls", [])} if role == "assistant" else set() if held: raise InlineSystemError("inline system instruction is waiting for missing tool results") return out def normalize_inline_system(messages: list[dict], mode: str) -> list[dict]: if mode not in ("preserve", "fold"): raise InlineSystemError(f"invalid inline system mode: {mode}") if not has_inline_system(messages): return messages ordered = _after_tool_results(deepcopy(messages)) # Without a stable leading system, some renderers attach tools to the first # late system message. Folding avoids moving tools when that message changes. if mode == "preserve" and ordered[0].get("role") == "system": return ordered out, pending = [], [] started = False for message in ordered: role = message.get("role") if role == "system" and started: text = message.get("content") or "" if not isinstance(text, str): raise InlineSystemError("inline system instructions must be text") if text: if out and out[-1].get("role") in ("user", "tool") and not pending: out[-1] = _add_text(out[-1], text) else: pending.append(text) continue if pending: text = "\n\n".join(pending) pending.clear() if role == "user": message = _add_text(message, text, prepend=True) else: out.append({"role": "user", "content": text}) out.append(message) started = started or role != "system" if pending: out.append({"role": "user", "content": "\n\n".join(pending)}) return out def _insertion(base: list[int], new: list[int]) -> tuple[int, list[int]] | None: start = 0 while start < min(len(base), len(new)) and base[start] == new[start]: start += 1 suffix = 0 while suffix < min(len(base), len(new)) - start and base[-suffix-1] == new[-suffix-1]: suffix += 1 if start + suffix != len(base): return None return start, new[start:len(new)-suffix] def probe_inline_system(render: Callable[[list[dict]], str], tokenizer: Any) -> tuple[str, str]: """Conservatively require a distinct, prefix-stable system segment.""" note, reasoning = "Echo inline instruction", "Foxtrot previous reasoning" system = {"role": "system", "content": "Alpha stable instructions"} user = {"role": "user", "content": "Bravo first question"} cases = [ ("middle", [system, user, {"role": "assistant", "content": "Charlie first answer"}, {"role": "user", "content": "Delta second question"}], 3), ("trailing", [system, user], 2), ("tool result", [system, user, {"role": "assistant", "content": "", "reasoning_content": reasoning, "thinking": reasoning, "reasoning": reasoning, "tool_calls": [{"id": "probe_call", "type": "function", "function": {"name": "read", "arguments": {}}}]}, {"role": "tool", "tool_call_id": "probe_call", "content": "Golf tool output"}], 4), ] max_system_tokens = None try: special_ids = set(tokenizer.all_special_ids) | set(tokenizer.get_added_vocab().values()) if not special_ids: return "fold", "renderer has no identifiable special tokens" for name, messages, index in cases: base_text = render(deepcopy(messages)) base = tokenizer.encode(base_text, add_special_tokens=False) with_note = messages[:index] + [{"role": "system", "content": note}] + messages[index:] new_text = render(deepcopy(with_note)) new = tokenizer.encode(new_text, add_special_tokens=False) found = _insertion(base, new) if found is None: return "fold", f"{name}: other prompt tokens change" position, segment = found if index == len(messages) and position >= len(base): return "fold", f"{name}: instruction follows the generation opening" if tokenizer.decode(segment).count(note) != 1 or not special_ids.intersection(segment): return "fold", f"{name}: no distinct role-marked instruction" if (reasoning in base_text) != (reasoning in new_text): return "fold", f"{name}: reasoning retention changes" if name == "middle": as_user = messages[:index] + [{"role": "user", "content": note}] + messages[index:] user_tokens = tokenizer.encode(render(deepcopy(as_user)), add_special_tokens=False) user_segment = _insertion(base, user_tokens) if user_segment is None: return "fold", "middle: cannot isolate a user instruction" if user_segment[1] == segment: return "fold", "system instructions render as ordinary user turns" max_system_tokens = len(user_segment[1]) + _INLINE_ROLE_SLACK_TOKENS if max_system_tokens is not None and len(segment) > max_system_tokens: return "fold", f"{name}: system insertion overhead ({len(segment)} > {max_system_tokens} tokens)" except Exception as exc: return "fold", f"probe could not render: {type(exc).__name__}" return "preserve", "renderer preserves distinct system turns and existing prompt tokens"