mirror of
https://github.com/earendil-works/pi.git
synced 2026-10-02 00:35:27 +08:00
feat(coding-agent): highlight heredocs and inline scripts in bash calls
Heredoc bodies and interpreter inline scripts (python3 - <<'EOF', cat > file.rs <<'EOF', node -e '...') are split from the shell text and highlighted in their own language. The language comes from the interpreter behind wrappers such as uv run or timeout, the cat/tee target extension, or the heredoc delimiter. Long bodies collapse to 10 lines until expanded. PowerShell calls are unchanged. Streaming highlight caching is shared with the write renderer: growing text only highlights new lines plus a bounded prefix, is fully highlighted once when the arguments complete, and is reused for output updates until the theme changes.
This commit is contained in:
@@ -219,6 +219,8 @@ export interface ShellToolConfig {
|
||||
label: string;
|
||||
shellName: string;
|
||||
prompt: string;
|
||||
/** Highlight heredoc bodies and inline scripts, which requires POSIX shell syntax. */
|
||||
embeddedCode: boolean;
|
||||
promptSnippet: string;
|
||||
promptGuidelines?: readonly string[];
|
||||
tempFilePrefix: string;
|
||||
@@ -376,7 +378,7 @@ export function createShellToolDefinition(
|
||||
clearUpdateTimer();
|
||||
}
|
||||
},
|
||||
...createShellRenderers(config.prompt),
|
||||
...createShellRenderers(config.prompt, { embeddedCode: config.embeddedCode }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -385,6 +387,7 @@ const bashToolConfig: ShellToolConfig = {
|
||||
label: "bash",
|
||||
shellName: "bash",
|
||||
prompt: "$",
|
||||
embeddedCode: true,
|
||||
promptSnippet: bashToolSystemPromptContribution.snippet,
|
||||
promptGuidelines: bashToolSystemPromptContribution.guidelines,
|
||||
tempFilePrefix: "pi-bash",
|
||||
|
||||
@@ -41,6 +41,7 @@ const powershellToolConfig: ShellToolConfig = {
|
||||
label: "powershell",
|
||||
shellName: "PowerShell",
|
||||
prompt: "PS>",
|
||||
embeddedCode: false,
|
||||
promptSnippet: powershellToolSystemPromptContribution.snippet,
|
||||
promptGuidelines: powershellToolSystemPromptContribution.guidelines,
|
||||
tempFilePrefix: "pi-powershell",
|
||||
|
||||
@@ -36,6 +36,66 @@ export function normalizeDisplayText(text: string): string {
|
||||
return text.replace(/\r/g, "");
|
||||
}
|
||||
|
||||
/** Leading lines of a streaming highlight that are re-highlighted with full context on each update. */
|
||||
const STREAMING_FULL_HIGHLIGHT_PREFIX_LINES = 50;
|
||||
|
||||
/** Highlighted display lines of streamed source text, for {@link updateStreamingHighlight}. */
|
||||
export interface StreamingHighlight {
|
||||
source: string;
|
||||
language: string | undefined;
|
||||
theme: Theme;
|
||||
/** Display lines of `source` after normalizeDisplayText and replaceTabs. */
|
||||
lines: string[];
|
||||
/** Styled version of each entry in `lines`. */
|
||||
highlighted: string[];
|
||||
/** False when some lines were highlighted one at a time, without multi-line context. */
|
||||
exact: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Highlight source text that arrives in growing chunks, such as streamed tool arguments.
|
||||
*
|
||||
* Renderers run on every argument delta, so highlighting the full text each time costs O(n^2) while
|
||||
* a long body streams in. When the new source extends the previous one, only the changed last line
|
||||
* and new lines are highlighted, one line at a time. The first lines, which collapsed previews show,
|
||||
* are re-highlighted with full context. Once `complete` is set, the text is highlighted in full once
|
||||
* and then reused until the source, language, or theme changes.
|
||||
*/
|
||||
export function updateStreamingHighlight(
|
||||
previous: StreamingHighlight | undefined,
|
||||
source: string,
|
||||
options: {
|
||||
language: string | undefined;
|
||||
theme: Theme;
|
||||
complete: boolean;
|
||||
highlight: (code: string) => string[];
|
||||
},
|
||||
): StreamingHighlight {
|
||||
const { language, theme, complete, highlight } = options;
|
||||
const reusable = previous !== undefined && previous.language === language && previous.theme === theme;
|
||||
if (reusable && previous.source === source && (previous.exact || !complete)) return previous;
|
||||
if (reusable && !complete && source.startsWith(previous.source)) {
|
||||
const highlightLine = (line: string) => highlight(line)[0] ?? "";
|
||||
const { lines, highlighted } = previous;
|
||||
const added = replaceTabs(normalizeDisplayText(source.slice(previous.source.length))).split("\n");
|
||||
const last = lines.length - 1;
|
||||
lines[last] += added[0];
|
||||
highlighted[last] = highlightLine(lines[last]);
|
||||
for (let i = 1; i < added.length; i++) {
|
||||
lines.push(added[i]);
|
||||
highlighted.push(highlightLine(added[i]));
|
||||
}
|
||||
const prefixCount = Math.min(STREAMING_FULL_HIGHLIGHT_PREFIX_LINES, lines.length);
|
||||
const prefix = highlight(lines.slice(0, prefixCount).join("\n"));
|
||||
for (let i = 0; i < prefixCount; i++) highlighted[i] = prefix[i] ?? highlighted[i];
|
||||
previous.source = source;
|
||||
previous.exact = prefixCount === lines.length;
|
||||
return previous;
|
||||
}
|
||||
const lines = replaceTabs(normalizeDisplayText(source)).split("\n");
|
||||
return { source, language, theme, lines, highlighted: highlight(lines.join("\n")), exact: true };
|
||||
}
|
||||
|
||||
export function getTextOutput(
|
||||
result: { content: Array<{ type: string; text?: string; data?: string; mimeType?: string }> } | undefined,
|
||||
showImages: boolean,
|
||||
|
||||
@@ -9,13 +9,28 @@
|
||||
import { Container, Text, truncateToWidth } from "@earendil-works/pi-tui";
|
||||
import { keyHint } from "../../../modes/interactive/components/keybinding-hints.ts";
|
||||
import { truncateToVisualLines } from "../../../modes/interactive/components/visual-truncate.ts";
|
||||
import { theme } from "../../../modes/interactive/theme/theme.ts";
|
||||
import {
|
||||
getLanguageFromPath,
|
||||
getThemeInstance,
|
||||
highlightCode,
|
||||
type Theme,
|
||||
theme,
|
||||
} from "../../../modes/interactive/theme/theme.ts";
|
||||
import { splitShellCommand } from "../../../utils/shell-embedded-code.ts";
|
||||
import type { ToolDefinition, ToolRenderResultOptions } from "../../extensions/types.ts";
|
||||
import type { BashToolDetails } from "../bash.ts";
|
||||
import { getTextOutput, invalidArgText, str } from "../render-utils.ts";
|
||||
import {
|
||||
getTextOutput,
|
||||
invalidArgText,
|
||||
type StreamingHighlight,
|
||||
str,
|
||||
updateStreamingHighlight,
|
||||
} from "../render-utils.ts";
|
||||
import { DEFAULT_MAX_BYTES, formatSize } from "../truncate.ts";
|
||||
|
||||
const BASH_PREVIEW_LINES = 5;
|
||||
/** Lines of each heredoc body or inline script shown before the call is expanded. */
|
||||
const EMBEDDED_CODE_PREVIEW_LINES = 10;
|
||||
export const BASH_UPDATE_THROTTLE_MS = 100;
|
||||
type BashResultRenderState = {
|
||||
cachedWidth: number | undefined;
|
||||
@@ -40,12 +55,102 @@ function formatDuration(ms: number): string {
|
||||
|
||||
return `${Math.floor(minutes / 60)}h ${minutes % 60}m ${remainder}s`;
|
||||
}
|
||||
function formatShellCall(args: { command?: string; timeout?: number } | undefined, prompt: string): string {
|
||||
function styleShellLines(text: string): string[] {
|
||||
return text.split("\n").map((line) => (line ? theme.fg("toolTitle", theme.bold(line)) : ""));
|
||||
}
|
||||
/** Style normalized embedded code. Data with no known language keeps the plain output color. */
|
||||
function styleEmbeddedLines(code: string, language: string | undefined): string[] {
|
||||
if (!language) return code.split("\n").map((line) => theme.fg("toolOutput", line));
|
||||
return highlightCode(code, language);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lines to show for one embedded segment. The first line continues the preceding shell line and the
|
||||
* last line continues into the following shell text, so collapsed previews keep both and elide the
|
||||
* middle.
|
||||
*/
|
||||
function formatEmbeddedCode(lines: string[], expanded: boolean): string[] {
|
||||
const hidden = lines.length - 1 - EMBEDDED_CODE_PREVIEW_LINES;
|
||||
if (expanded || hidden <= 0) return lines;
|
||||
const hint = `${theme.fg("muted", `... (${hidden} more lines,`)} ${keyHint("app.tools.expand", "to expand")}${theme.fg("muted", ")")}`;
|
||||
return [...lines.slice(0, EMBEDDED_CODE_PREVIEW_LINES), hint, lines[lines.length - 1]];
|
||||
}
|
||||
|
||||
class ShellCallRenderComponent extends Text {
|
||||
theme: Theme | undefined;
|
||||
command: string | undefined;
|
||||
expanded = false;
|
||||
timeout: number | undefined;
|
||||
/** Formatted call text for the fields above. */
|
||||
rendered: string | undefined;
|
||||
/** Highlights of the embedded segments of `command`, in order. */
|
||||
embedded: StreamingHighlight[] = [];
|
||||
|
||||
constructor() {
|
||||
super("", 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
function formatShellCall(
|
||||
component: ShellCallRenderComponent,
|
||||
args: { command?: string; timeout?: number } | undefined,
|
||||
prompt: string,
|
||||
options: { expanded: boolean; embeddedCode: boolean; argsComplete: boolean },
|
||||
): string {
|
||||
const command = str(args?.command);
|
||||
const timeout = args?.timeout as number | undefined;
|
||||
const timeoutSuffix = timeout ? theme.fg("muted", ` (timeout ${timeout}s)`) : "";
|
||||
const commandDisplay = command === null ? invalidArgText(theme) : command ? command : theme.fg("toolOutput", "...");
|
||||
return theme.fg("toolTitle", theme.bold(`${prompt} ${commandDisplay}`)) + timeoutSuffix;
|
||||
if (!command || !options.embeddedCode) {
|
||||
component.rendered = undefined;
|
||||
const commandDisplay =
|
||||
command === null ? invalidArgText(theme) : command ? command : theme.fg("toolOutput", "...");
|
||||
return theme.fg("toolTitle", theme.bold(`${prompt} ${commandDisplay}`)) + timeoutSuffix;
|
||||
}
|
||||
|
||||
const currentTheme = getThemeInstance();
|
||||
// renderCall also runs for every throttled output update with unchanged arguments. Reuse the text
|
||||
// unless the theme changed or the final arguments still need a full highlight.
|
||||
if (
|
||||
component.rendered !== undefined &&
|
||||
component.theme === currentTheme &&
|
||||
component.command === command &&
|
||||
component.expanded === options.expanded &&
|
||||
component.timeout === timeout &&
|
||||
(!options.argsComplete || component.embedded.every((entry) => entry.exact))
|
||||
) {
|
||||
return component.rendered;
|
||||
}
|
||||
|
||||
const lines = [""];
|
||||
const append = (styled: string[]) => {
|
||||
lines[lines.length - 1] += styled[0] ?? "";
|
||||
for (let i = 1; i < styled.length; i++) lines.push(styled[i]);
|
||||
};
|
||||
append(styleShellLines(`${prompt} `));
|
||||
const embedded: StreamingHighlight[] = [];
|
||||
for (const segment of splitShellCommand(command, { languageFromPath: getLanguageFromPath })) {
|
||||
if (!segment.embedded) {
|
||||
append(styleShellLines(segment.text));
|
||||
continue;
|
||||
}
|
||||
const language = segment.language;
|
||||
const highlight = updateStreamingHighlight(component.embedded[embedded.length], segment.text, {
|
||||
language,
|
||||
theme: currentTheme,
|
||||
complete: options.argsComplete,
|
||||
highlight: (code) => styleEmbeddedLines(code, language),
|
||||
});
|
||||
embedded.push(highlight);
|
||||
append(formatEmbeddedCode(highlight.highlighted, options.expanded));
|
||||
}
|
||||
|
||||
component.theme = currentTheme;
|
||||
component.command = command;
|
||||
component.expanded = options.expanded;
|
||||
component.timeout = timeout;
|
||||
component.embedded = embedded;
|
||||
component.rendered = lines.join("\n") + timeoutSuffix;
|
||||
return component.rendered;
|
||||
}
|
||||
function rebuildBashResultRenderComponent(
|
||||
component: BashResultRenderComponent,
|
||||
@@ -129,8 +234,14 @@ function rebuildBashResultRenderComponent(
|
||||
}
|
||||
}
|
||||
|
||||
/** Shell renderers are shared by bash and powershell, which differ only in the prompt they display. */
|
||||
export function createShellRenderers(prompt: string): Pick<ToolDefinition<any, any>, "renderCall" | "renderResult"> {
|
||||
/**
|
||||
* Shell renderers are shared by bash and powershell. `embeddedCode` highlights heredoc bodies and
|
||||
* inline scripts, which is only implemented for POSIX shell syntax.
|
||||
*/
|
||||
export function createShellRenderers(
|
||||
prompt: string,
|
||||
options: { embeddedCode: boolean },
|
||||
): Pick<ToolDefinition<any, any>, "renderCall" | "renderResult"> {
|
||||
return {
|
||||
renderCall(args, _theme, context) {
|
||||
const state = context.state;
|
||||
@@ -138,9 +249,17 @@ export function createShellRenderers(prompt: string): Pick<ToolDefinition<any, a
|
||||
state.startedAt = Date.now();
|
||||
state.endedAt = undefined;
|
||||
}
|
||||
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
text.setText(formatShellCall(args as { command?: string; timeout?: number } | undefined, prompt));
|
||||
return text;
|
||||
const component =
|
||||
(context.lastComponent as ShellCallRenderComponent | undefined) ?? new ShellCallRenderComponent();
|
||||
const previous = component.rendered;
|
||||
const text = formatShellCall(component, args as { command?: string; timeout?: number } | undefined, prompt, {
|
||||
expanded: context.expanded,
|
||||
embeddedCode: options.embeddedCode,
|
||||
argsComplete: context.argsComplete,
|
||||
});
|
||||
// setText drops the wrapped-line cache, so skip it when output updates leave the call unchanged.
|
||||
if (text !== previous) component.setText(text);
|
||||
return component;
|
||||
},
|
||||
renderResult(result, options, _theme, context) {
|
||||
const state = context.state;
|
||||
|
||||
@@ -32,8 +32,8 @@ export {
|
||||
export function createAllToolRenderers(): Record<ToolName, ToolRenderers> {
|
||||
return {
|
||||
read: readRenderers,
|
||||
bash: createShellRenderers("$"),
|
||||
powershell: createShellRenderers("PS>"),
|
||||
bash: createShellRenderers("$", { embeddedCode: true }),
|
||||
powershell: createShellRenderers("PS>", { embeddedCode: false }),
|
||||
edit: editRenderers,
|
||||
write: writeRenderers,
|
||||
grep: grepRenderers,
|
||||
|
||||
@@ -8,84 +8,29 @@
|
||||
|
||||
import { Container, Text } from "@earendil-works/pi-tui";
|
||||
import { keyHint } from "../../../modes/interactive/components/keybinding-hints.ts";
|
||||
import { getLanguageFromPath, highlightCode, type Theme } from "../../../modes/interactive/theme/theme.ts";
|
||||
import {
|
||||
getLanguageFromPath,
|
||||
getThemeInstance,
|
||||
highlightCode,
|
||||
type Theme,
|
||||
} from "../../../modes/interactive/theme/theme.ts";
|
||||
import type { ToolDefinition, ToolRenderResultOptions } from "../../extensions/types.ts";
|
||||
import { normalizeDisplayText, renderToolPath, replaceTabs, str } from "../render-utils.ts";
|
||||
import {
|
||||
normalizeDisplayText,
|
||||
renderToolPath,
|
||||
replaceTabs,
|
||||
type StreamingHighlight,
|
||||
str,
|
||||
updateStreamingHighlight,
|
||||
} from "../render-utils.ts";
|
||||
|
||||
type WriteHighlightCache = {
|
||||
rawPath: string | null;
|
||||
lang: string;
|
||||
rawContent: string;
|
||||
normalizedLines: string[];
|
||||
highlightedLines: string[];
|
||||
};
|
||||
class WriteCallRenderComponent extends Text {
|
||||
cache?: WriteHighlightCache;
|
||||
cache?: StreamingHighlight;
|
||||
|
||||
constructor() {
|
||||
super("", 0, 0);
|
||||
}
|
||||
}
|
||||
const WRITE_PARTIAL_FULL_HIGHLIGHT_LINES = 50;
|
||||
function highlightSingleLine(line: string, lang: string): string {
|
||||
const highlighted = highlightCode(line, lang);
|
||||
return highlighted[0] ?? "";
|
||||
}
|
||||
function refreshWriteHighlightPrefix(cache: WriteHighlightCache): void {
|
||||
const prefixCount = Math.min(WRITE_PARTIAL_FULL_HIGHLIGHT_LINES, cache.normalizedLines.length);
|
||||
if (prefixCount === 0) return;
|
||||
const prefixSource = cache.normalizedLines.slice(0, prefixCount).join("\n");
|
||||
const prefixHighlighted = highlightCode(prefixSource, cache.lang);
|
||||
for (let i = 0; i < prefixCount; i++) {
|
||||
cache.highlightedLines[i] =
|
||||
prefixHighlighted[i] ?? highlightSingleLine(cache.normalizedLines[i] ?? "", cache.lang);
|
||||
}
|
||||
}
|
||||
function rebuildWriteHighlightCacheFull(rawPath: string | null, fileContent: string): WriteHighlightCache | undefined {
|
||||
const lang = rawPath ? getLanguageFromPath(rawPath) : undefined;
|
||||
if (!lang) return undefined;
|
||||
const displayContent = normalizeDisplayText(fileContent);
|
||||
const normalized = replaceTabs(displayContent);
|
||||
return {
|
||||
rawPath,
|
||||
lang,
|
||||
rawContent: fileContent,
|
||||
normalizedLines: normalized.split("\n"),
|
||||
highlightedLines: highlightCode(normalized, lang),
|
||||
};
|
||||
}
|
||||
function updateWriteHighlightCacheIncremental(
|
||||
cache: WriteHighlightCache | undefined,
|
||||
rawPath: string | null,
|
||||
fileContent: string,
|
||||
): WriteHighlightCache | undefined {
|
||||
const lang = rawPath ? getLanguageFromPath(rawPath) : undefined;
|
||||
if (!lang) return undefined;
|
||||
if (!cache) return rebuildWriteHighlightCacheFull(rawPath, fileContent);
|
||||
if (cache.lang !== lang || cache.rawPath !== rawPath) return rebuildWriteHighlightCacheFull(rawPath, fileContent);
|
||||
if (!fileContent.startsWith(cache.rawContent)) return rebuildWriteHighlightCacheFull(rawPath, fileContent);
|
||||
if (fileContent.length === cache.rawContent.length) return cache;
|
||||
|
||||
const deltaRaw = fileContent.slice(cache.rawContent.length);
|
||||
const deltaDisplay = normalizeDisplayText(deltaRaw);
|
||||
const deltaNormalized = replaceTabs(deltaDisplay);
|
||||
cache.rawContent = fileContent;
|
||||
if (cache.normalizedLines.length === 0) {
|
||||
cache.normalizedLines.push("");
|
||||
cache.highlightedLines.push("");
|
||||
}
|
||||
|
||||
const segments = deltaNormalized.split("\n");
|
||||
const lastIndex = cache.normalizedLines.length - 1;
|
||||
cache.normalizedLines[lastIndex] += segments[0];
|
||||
cache.highlightedLines[lastIndex] = highlightSingleLine(cache.normalizedLines[lastIndex], cache.lang);
|
||||
for (let i = 1; i < segments.length; i++) {
|
||||
cache.normalizedLines.push(segments[i]);
|
||||
cache.highlightedLines.push(highlightSingleLine(segments[i], cache.lang));
|
||||
}
|
||||
refreshWriteHighlightPrefix(cache);
|
||||
return cache;
|
||||
}
|
||||
function trimTrailingEmptyLines(lines: string[]): string[] {
|
||||
let end = lines.length;
|
||||
while (end > 0 && lines[end - 1] === "") {
|
||||
@@ -97,7 +42,7 @@ function formatWriteCall(
|
||||
args: { path?: string; file_path?: string; content?: string } | undefined,
|
||||
options: ToolRenderResultOptions,
|
||||
theme: Theme,
|
||||
cache: WriteHighlightCache | undefined,
|
||||
cache: StreamingHighlight | undefined,
|
||||
cwd: string,
|
||||
): string {
|
||||
const rawPath = str(args?.file_path ?? args?.path);
|
||||
@@ -110,7 +55,7 @@ function formatWriteCall(
|
||||
} else if (fileContent) {
|
||||
const lang = rawPath ? getLanguageFromPath(rawPath) : undefined;
|
||||
const renderedLines = lang
|
||||
? (cache?.highlightedLines ?? highlightCode(replaceTabs(normalizeDisplayText(fileContent)), lang))
|
||||
? (cache?.highlighted ?? highlightCode(replaceTabs(normalizeDisplayText(fileContent)), lang))
|
||||
: normalizeDisplayText(fileContent).split("\n");
|
||||
const lines = trimTrailingEmptyLines(renderedLines);
|
||||
const totalLines = lines.length;
|
||||
@@ -149,13 +94,16 @@ export const writeRenderers: Pick<ToolDefinition<any, any>, "renderCall" | "rend
|
||||
const fileContent = str(renderArgs?.content);
|
||||
const component =
|
||||
(context.lastComponent as WriteCallRenderComponent | undefined) ?? new WriteCallRenderComponent();
|
||||
if (fileContent !== null) {
|
||||
component.cache = context.argsComplete
|
||||
? rebuildWriteHighlightCacheFull(rawPath, fileContent)
|
||||
: updateWriteHighlightCacheIncremental(component.cache, rawPath, fileContent);
|
||||
} else {
|
||||
component.cache = undefined;
|
||||
}
|
||||
const lang = rawPath ? getLanguageFromPath(rawPath) : undefined;
|
||||
component.cache =
|
||||
fileContent !== null && lang
|
||||
? updateStreamingHighlight(component.cache, fileContent, {
|
||||
language: lang,
|
||||
theme: getThemeInstance(),
|
||||
complete: context.argsComplete,
|
||||
highlight: (code) => highlightCode(code, lang),
|
||||
})
|
||||
: undefined;
|
||||
component.setText(
|
||||
formatWriteCall(
|
||||
renderArgs,
|
||||
|
||||
@@ -754,6 +754,16 @@ export const theme: Theme = new Proxy({} as Theme, {
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* The active theme instance. Switching or reloading a theme installs a new instance, so renderers can
|
||||
* compare identities to invalidate cached styled output.
|
||||
*/
|
||||
export function getThemeInstance(): Theme {
|
||||
const t = (globalThis as Record<symbol, Theme>)[THEME_KEY];
|
||||
if (!t) throw new Error("Theme not initialized. Call initTheme() first.");
|
||||
return t;
|
||||
}
|
||||
|
||||
function setGlobalTheme(t: Theme): void {
|
||||
(globalThis as Record<symbol, Theme>)[THEME_KEY] = t;
|
||||
(globalThis as Record<symbol, Theme>)[THEME_KEY_OLD] = t;
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
/**
|
||||
* Split a POSIX shell command into shell text and embedded code for display.
|
||||
*
|
||||
* Models often run scripts through the shell tool instead of writing files first:
|
||||
*
|
||||
* cd repo && python3 - <<'EOF'
|
||||
* print("hi")
|
||||
* EOF
|
||||
* node -e 'console.log(1)'
|
||||
*
|
||||
* Highlighting the whole command as shell renders the heredoc body and inline scripts as shell,
|
||||
* where a lone quote in the embedded code colors the rest of the command. This module finds
|
||||
* heredoc bodies and interpreter inline scripts (`-c`, `-e`, ...) and infers their language from
|
||||
* the command that consumes them, so each part can be highlighted on its own.
|
||||
*
|
||||
* The scanner is approximate and only used for display. It tracks quotes, command boundaries,
|
||||
* redirects, and `$(...)` nesting well enough for the commands models write, and never throws.
|
||||
*/
|
||||
|
||||
export interface ShellSegment {
|
||||
/** Exact source text. Concatenating all segment texts yields the original command. */
|
||||
text: string;
|
||||
/** False for shell syntax, true for a heredoc body or inline script. */
|
||||
embedded: boolean;
|
||||
/** Highlight language for embedded code. Undefined for plain data such as a commit message. */
|
||||
language?: string;
|
||||
}
|
||||
|
||||
export interface SplitShellCommandOptions {
|
||||
/** Resolve a highlight language from a file path, used for `cat > file <<'EOF'`. */
|
||||
languageFromPath?: (path: string) => string | undefined;
|
||||
}
|
||||
|
||||
interface ShellCommand {
|
||||
words: string[];
|
||||
/** Word being built, or undefined between words. */
|
||||
word: string | undefined;
|
||||
/** Set while the next word is a redirect target. */
|
||||
pendingRedirect: { fd: string; dup: boolean } | undefined;
|
||||
/** Target of the last stdout redirect (`> file`, `>> file`, `&> file`). */
|
||||
stdoutTarget: string | undefined;
|
||||
}
|
||||
|
||||
interface ShellFrame {
|
||||
kind: "shell";
|
||||
command: ShellCommand;
|
||||
/** Open parentheses inside this `$(...)` frame. */
|
||||
parenDepth: number;
|
||||
/** True for `$(...)`, `<(...)`, and `>(...)`, which end at the matching `)`. */
|
||||
substitution: boolean;
|
||||
}
|
||||
|
||||
interface DoubleQuoteFrame {
|
||||
kind: "double";
|
||||
}
|
||||
|
||||
type Frame = ShellFrame | DoubleQuoteFrame;
|
||||
|
||||
interface PendingHeredoc {
|
||||
delimiter: string;
|
||||
stripTabs: boolean;
|
||||
command: ShellCommand;
|
||||
}
|
||||
|
||||
interface InterpreterSpec {
|
||||
language: string;
|
||||
/** Flag that introduces an inline script, e.g. `-c` or `-e`. */
|
||||
inlineFlag?: RegExp;
|
||||
/** The first positional argument is the script, as in `awk '{ print $1 }'`. */
|
||||
inlineFirstPositional?: boolean;
|
||||
/** Options whose value is the next word, e.g. `-r json` for ruby. */
|
||||
valueOptions?: ReadonlySet<string>;
|
||||
/** Stdin is code even when positional arguments are present, e.g. `psql mydb <<'SQL'`. */
|
||||
stdinAlwaysCode?: boolean;
|
||||
}
|
||||
|
||||
const SHELL_INLINE_FLAG = /^-[A-Za-z]*c$/;
|
||||
const NODE_INLINE_FLAG = /^(?:-[A-Za-z]*[ep]|--eval|--print)$/;
|
||||
|
||||
const PYTHON_SPEC: InterpreterSpec = {
|
||||
language: "python",
|
||||
inlineFlag: /^-[A-Za-z]*c$/,
|
||||
valueOptions: new Set(["-W", "-X", "-m"]),
|
||||
};
|
||||
const NODE_SPEC: InterpreterSpec = {
|
||||
language: "javascript",
|
||||
inlineFlag: NODE_INLINE_FLAG,
|
||||
valueOptions: new Set(["-r", "--require", "--import", "--loader", "--input-type", "--conditions", "-C"]),
|
||||
};
|
||||
const TYPESCRIPT_SPEC: InterpreterSpec = {
|
||||
language: "typescript",
|
||||
inlineFlag: NODE_INLINE_FLAG,
|
||||
valueOptions: NODE_SPEC.valueOptions,
|
||||
};
|
||||
const SHELL_SPEC: InterpreterSpec = {
|
||||
language: "bash",
|
||||
inlineFlag: SHELL_INLINE_FLAG,
|
||||
valueOptions: new Set(["-o", "+o", "-O", "+O"]),
|
||||
};
|
||||
const AWK_SPEC: InterpreterSpec = {
|
||||
language: "awk",
|
||||
inlineFirstPositional: true,
|
||||
valueOptions: new Set(["-F", "-v", "-f"]),
|
||||
};
|
||||
|
||||
const INTERPRETERS: Record<string, InterpreterSpec> = {
|
||||
python: PYTHON_SPEC,
|
||||
pypy: PYTHON_SPEC,
|
||||
node: NODE_SPEC,
|
||||
bun: NODE_SPEC,
|
||||
tsx: TYPESCRIPT_SPEC,
|
||||
"ts-node": TYPESCRIPT_SPEC,
|
||||
ruby: { language: "ruby", inlineFlag: /^-[A-Za-z]*e$/, valueOptions: new Set(["-r", "-I"]) },
|
||||
perl: { language: "perl", inlineFlag: /^-[A-Za-z0-9]*[eE]$/, valueOptions: new Set(["-I", "-M", "-m"]) },
|
||||
php: { language: "php", inlineFlag: /^-r$/ },
|
||||
lua: { language: "lua", inlineFlag: /^-e$/ },
|
||||
swift: { language: "swift", inlineFlag: /^-e$/ },
|
||||
osascript: { language: "applescript", inlineFlag: /^-e$/, valueOptions: new Set(["-l"]) },
|
||||
sh: SHELL_SPEC,
|
||||
bash: SHELL_SPEC,
|
||||
zsh: SHELL_SPEC,
|
||||
dash: SHELL_SPEC,
|
||||
ksh: SHELL_SPEC,
|
||||
awk: AWK_SPEC,
|
||||
gawk: AWK_SPEC,
|
||||
mawk: AWK_SPEC,
|
||||
psql: {
|
||||
language: "sql",
|
||||
inlineFlag: /^(?:-c|--command)$/,
|
||||
valueOptions: new Set(["-d", "-h", "-p", "-U", "-f"]),
|
||||
stdinAlwaysCode: true,
|
||||
},
|
||||
mysql: {
|
||||
language: "sql",
|
||||
inlineFlag: /^(?:-e|--execute)$/,
|
||||
valueOptions: new Set(["-u", "-h", "-P", "-D"]),
|
||||
stdinAlwaysCode: true,
|
||||
},
|
||||
duckdb: { language: "sql", inlineFlag: /^-c$/, stdinAlwaysCode: true },
|
||||
sqlite3: { language: "sql", stdinAlwaysCode: true },
|
||||
};
|
||||
|
||||
/** Heredoc delimiter names that state the body language, e.g. `node tool.js <<'JS'`. */
|
||||
const DELIMITER_LANGUAGES: Record<string, string> = {
|
||||
PY: "python",
|
||||
PYTHON: "python",
|
||||
JS: "javascript",
|
||||
JAVASCRIPT: "javascript",
|
||||
NODE: "javascript",
|
||||
MJS: "javascript",
|
||||
TS: "typescript",
|
||||
TYPESCRIPT: "typescript",
|
||||
RB: "ruby",
|
||||
RUBY: "ruby",
|
||||
PL: "perl",
|
||||
PERL: "perl",
|
||||
PHP: "php",
|
||||
LUA: "lua",
|
||||
SWIFT: "swift",
|
||||
SQL: "sql",
|
||||
SH: "bash",
|
||||
BASH: "bash",
|
||||
ZSH: "bash",
|
||||
SHELL: "bash",
|
||||
JSON: "json",
|
||||
YAML: "yaml",
|
||||
YML: "yaml",
|
||||
TOML: "ini",
|
||||
HTML: "xml",
|
||||
XML: "xml",
|
||||
CSS: "css",
|
||||
RS: "rust",
|
||||
RUST: "rust",
|
||||
GO: "go",
|
||||
C: "c",
|
||||
CPP: "cpp",
|
||||
MD: "markdown",
|
||||
MARKDOWN: "markdown",
|
||||
DIFF: "diff",
|
||||
PATCH: "diff",
|
||||
};
|
||||
|
||||
/** Words that precede the real command without changing what consumes stdin. */
|
||||
const TRANSPARENT_WORDS = new Set([
|
||||
"sudo",
|
||||
"time",
|
||||
"nohup",
|
||||
"exec",
|
||||
"command",
|
||||
"builtin",
|
||||
"do",
|
||||
"then",
|
||||
"else",
|
||||
"elif",
|
||||
"if",
|
||||
"while",
|
||||
"until",
|
||||
"!",
|
||||
"{",
|
||||
]);
|
||||
|
||||
const UV_VALUE_OPTIONS = new Set([
|
||||
"--with",
|
||||
"--with-editable",
|
||||
"--with-requirements",
|
||||
"--python",
|
||||
"-p",
|
||||
"--project",
|
||||
"--directory",
|
||||
"--extra",
|
||||
"--group",
|
||||
"--env-file",
|
||||
"--package",
|
||||
"--index",
|
||||
"--from",
|
||||
]);
|
||||
|
||||
const ENV_VALUE_OPTIONS = new Set(["-u", "-C", "-S"]);
|
||||
const TIMEOUT_VALUE_OPTIONS = new Set(["-s", "-k", "-n", "--signal", "--kill-after"]);
|
||||
|
||||
const ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=/;
|
||||
|
||||
function newCommand(): ShellCommand {
|
||||
return { words: [], word: undefined, pendingRedirect: undefined, stdoutTarget: undefined };
|
||||
}
|
||||
|
||||
function skipOptions(words: string[], index: number, valueOptions?: ReadonlySet<string>): number {
|
||||
let i = index;
|
||||
while (i < words.length && words[i].startsWith("-") && words[i] !== "-") {
|
||||
if (words[i] === "--") return i + 1;
|
||||
if (valueOptions?.has(words[i])) i++;
|
||||
i++;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
/** Strip assignments and wrappers such as `sudo` or `uv run --with x`, returning the program index. */
|
||||
function findProgramIndex(words: string[]): number {
|
||||
let i = 0;
|
||||
while (i < words.length) {
|
||||
const word = words[i];
|
||||
if (ASSIGNMENT.test(word) || TRANSPARENT_WORDS.has(word)) {
|
||||
i++;
|
||||
} else if (word === "env") {
|
||||
i = skipOptions(words, i + 1, ENV_VALUE_OPTIONS);
|
||||
while (i < words.length && ASSIGNMENT.test(words[i])) i++;
|
||||
} else if (word === "timeout" || word === "nice") {
|
||||
i = skipOptions(words, i + 1, TIMEOUT_VALUE_OPTIONS);
|
||||
if (word === "timeout") i++;
|
||||
} else if (word === "uv" && words[i + 1] === "run") {
|
||||
i = skipOptions(words, i + 2, UV_VALUE_OPTIONS);
|
||||
} else if (word === "uv" && words[i + 1] === "tool" && words[i + 2] === "run") {
|
||||
i = skipOptions(words, i + 3, UV_VALUE_OPTIONS);
|
||||
} else if (word === "npx" || word === "bunx" || (word === "pnpm" && words[i + 1] === "exec")) {
|
||||
i = skipOptions(words, word === "pnpm" ? i + 2 : i + 1);
|
||||
} else {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
function programName(word: string): string {
|
||||
const base = word.slice(word.lastIndexOf("/") + 1).replace(/\.exe$/i, "");
|
||||
if (/^python\d*(?:\.\d+)?$/.test(base) || /^pypy\d*$/.test(base)) return "python";
|
||||
return base;
|
||||
}
|
||||
|
||||
/** `python -m pkg` hands all later arguments to the module, so `-c` is no longer Python's flag. */
|
||||
function runsPythonModule(spec: InterpreterSpec, args: string[]): boolean {
|
||||
return spec === PYTHON_SPEC && args.some((arg) => arg.startsWith("-m"));
|
||||
}
|
||||
|
||||
function substituteVariables(value: string, variables: ReadonlyMap<string, string>): string {
|
||||
return value.replace(
|
||||
/\$(?:\{([A-Za-z_][A-Za-z0-9_]*)\}|([A-Za-z_][A-Za-z0-9_]*))/g,
|
||||
(match, braced: string | undefined, bare: string | undefined) => variables.get(braced ?? bare ?? "") ?? match,
|
||||
);
|
||||
}
|
||||
|
||||
/** Language of a heredoc body, from the command that reads it or the delimiter name. */
|
||||
function heredocLanguage(
|
||||
heredoc: PendingHeredoc,
|
||||
variables: ReadonlyMap<string, string>,
|
||||
options: SplitShellCommandOptions,
|
||||
): string | undefined {
|
||||
const words = heredoc.command.words;
|
||||
const programIndex = findProgramIndex(words);
|
||||
const program = programIndex < words.length ? programName(words[programIndex]) : undefined;
|
||||
const pathLanguage = (path: string | undefined) =>
|
||||
path ? options.languageFromPath?.(substituteVariables(path, variables)) : undefined;
|
||||
|
||||
let language: string | undefined;
|
||||
if (program === "cat") {
|
||||
language = pathLanguage(heredoc.command.stdoutTarget);
|
||||
} else if (program === "tee") {
|
||||
const argsStart = skipOptions(words, programIndex + 1);
|
||||
language = pathLanguage(words[argsStart]) ?? pathLanguage(heredoc.command.stdoutTarget);
|
||||
} else if (program !== undefined) {
|
||||
const spec = INTERPRETERS[program];
|
||||
if (spec) {
|
||||
const positionalIndex = skipOptions(words, programIndex + 1, spec.valueOptions);
|
||||
const positional = words[positionalIndex];
|
||||
const readsScriptFromStdin = positional === undefined || positional === "-";
|
||||
const hasInlineScript = spec.inlineFlag
|
||||
? words.slice(programIndex + 1).some((word) => spec.inlineFlag?.test(word))
|
||||
: false;
|
||||
const runsModule = runsPythonModule(spec, words.slice(programIndex + 1));
|
||||
if (spec.stdinAlwaysCode || (readsScriptFromStdin && !hasInlineScript && !runsModule)) {
|
||||
language = spec.language;
|
||||
}
|
||||
}
|
||||
}
|
||||
return language ?? DELIMITER_LANGUAGES[heredoc.delimiter.toUpperCase()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Language of a quoted word that starts at the current position, if it is an interpreter's inline
|
||||
* script such as the argument of `python3 -c` or `node -e`. Returns undefined for regular words.
|
||||
*/
|
||||
function inlineScriptLanguage(command: ShellCommand): string | undefined {
|
||||
const words = command.words;
|
||||
const programIndex = findProgramIndex(words);
|
||||
if (programIndex >= words.length) return undefined;
|
||||
const spec = INTERPRETERS[programName(words[programIndex])];
|
||||
if (!spec) return undefined;
|
||||
const args = words.slice(programIndex + 1);
|
||||
// Options after a positional argument belong to a script, as in `python3 tool.py -c x`.
|
||||
if (skipOptions(args, 0, spec.valueOptions) !== args.length || runsPythonModule(spec, args)) return undefined;
|
||||
const previous = args[args.length - 1];
|
||||
if (spec.inlineFlag && previous !== undefined && spec.inlineFlag.test(previous)) return spec.language;
|
||||
if (spec.inlineFirstPositional) return spec.language;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Split a shell command into shell text, heredoc bodies, and inline scripts. */
|
||||
export function splitShellCommand(command: string, options: SplitShellCommandOptions = {}): ShellSegment[] {
|
||||
const segments: ShellSegment[] = [];
|
||||
const variables = new Map<string, string>();
|
||||
const frames: Frame[] = [{ kind: "shell", command: newCommand(), parenDepth: 0, substitution: false }];
|
||||
let pendingHeredocs: PendingHeredoc[] = [];
|
||||
let segmentStart = 0;
|
||||
let i = 0;
|
||||
|
||||
const pushSegment = (end: number, embedded: boolean, language?: string) => {
|
||||
if (end <= segmentStart) return;
|
||||
const text = command.slice(segmentStart, end);
|
||||
const previous = segments[segments.length - 1];
|
||||
if (!embedded && previous && !previous.embedded) {
|
||||
previous.text += text;
|
||||
} else {
|
||||
segments.push(embedded ? { text, embedded, language } : { text, embedded });
|
||||
}
|
||||
segmentStart = end;
|
||||
};
|
||||
|
||||
const shellFrame = (): ShellFrame => {
|
||||
for (let index = frames.length - 1; index >= 0; index--) {
|
||||
const frame = frames[index];
|
||||
if (frame.kind === "shell") return frame;
|
||||
}
|
||||
return frames[0] as ShellFrame;
|
||||
};
|
||||
|
||||
const appendToWord = (text: string) => {
|
||||
const current = shellFrame().command;
|
||||
current.word = (current.word ?? "") + text;
|
||||
};
|
||||
|
||||
const finishWord = (current: ShellCommand) => {
|
||||
if (current.word === undefined) return;
|
||||
const word = current.word;
|
||||
current.word = undefined;
|
||||
if (current.pendingRedirect) {
|
||||
const { fd, dup } = current.pendingRedirect;
|
||||
current.pendingRedirect = undefined;
|
||||
if (!dup && (fd === "1" || fd === "&")) current.stdoutTarget = word;
|
||||
return;
|
||||
}
|
||||
current.words.push(word);
|
||||
};
|
||||
|
||||
const endCommand = (frame: ShellFrame) => {
|
||||
finishWord(frame.command);
|
||||
const words = frame.command.words;
|
||||
const assignments = words[0] === "export" ? words.slice(1) : words;
|
||||
if (assignments.length > 0 && assignments.every((word) => ASSIGNMENT.test(word))) {
|
||||
for (const word of assignments) {
|
||||
const eq = word.indexOf("=");
|
||||
variables.set(word.slice(0, eq), substituteVariables(word.slice(eq + 1), variables));
|
||||
}
|
||||
}
|
||||
frame.command = newCommand();
|
||||
};
|
||||
|
||||
/** Consume heredoc bodies that start at `bodyStart`, returning the index after the last delimiter line. */
|
||||
const consumeHeredocs = (bodyStart: number): number => {
|
||||
let position = bodyStart;
|
||||
for (const heredoc of pendingHeredocs) {
|
||||
const language = heredocLanguage(heredoc, variables, options);
|
||||
pushSegment(position, false);
|
||||
let lineStart = position;
|
||||
let delimiterStart = -1;
|
||||
let delimiterEnd = command.length;
|
||||
while (lineStart < command.length) {
|
||||
const newline = command.indexOf("\n", lineStart);
|
||||
const lineEnd = newline === -1 ? command.length : newline;
|
||||
let line = command.slice(lineStart, lineEnd);
|
||||
if (heredoc.stripTabs) line = line.replace(/^\t+/, "");
|
||||
// A delimiter on the last line of a streaming command may still be growing, but an
|
||||
// exact match is also how a complete command ends, so both cases are treated as closed.
|
||||
if (line === heredoc.delimiter) {
|
||||
delimiterStart = lineStart;
|
||||
delimiterEnd = newline === -1 ? command.length : newline + 1;
|
||||
break;
|
||||
}
|
||||
if (newline === -1) break;
|
||||
lineStart = newline + 1;
|
||||
}
|
||||
if (delimiterStart === -1) {
|
||||
pushSegment(command.length, true, language);
|
||||
pendingHeredocs = [];
|
||||
return command.length;
|
||||
}
|
||||
pushSegment(delimiterStart, true, language);
|
||||
position = delimiterEnd;
|
||||
}
|
||||
pendingHeredocs = [];
|
||||
return position;
|
||||
};
|
||||
|
||||
/** Emit an inline script inside quotes that open at `quoteIndex` and close at `closeIndex`. */
|
||||
const emitInlineScript = (quoteIndex: number, closeIndex: number, language: string) => {
|
||||
pushSegment(quoteIndex + 1, false);
|
||||
pushSegment(closeIndex, true, language);
|
||||
};
|
||||
|
||||
const findDoubleQuoteEnd = (start: number): number => {
|
||||
for (let index = start; index < command.length; index++) {
|
||||
const char = command[index];
|
||||
if (char === "\\") index++;
|
||||
else if (char === '"') return index;
|
||||
}
|
||||
return command.length;
|
||||
};
|
||||
|
||||
while (i < command.length) {
|
||||
const frame = frames[frames.length - 1];
|
||||
const char = command[i];
|
||||
const next = command[i + 1];
|
||||
|
||||
if (frame.kind === "double") {
|
||||
if (char === "\\") {
|
||||
appendToWord(next ?? "");
|
||||
i += 2;
|
||||
} else if (char === '"') {
|
||||
frames.pop();
|
||||
i++;
|
||||
} else if (char === "$" && next === "(" && command[i + 2] !== "(") {
|
||||
appendToWord("$()");
|
||||
frames.push({ kind: "shell", command: newCommand(), parenDepth: 0, substitution: true });
|
||||
i += 2;
|
||||
} else {
|
||||
appendToWord(char);
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const current = frame.command;
|
||||
switch (char) {
|
||||
case "\\": {
|
||||
if (next !== "\n") appendToWord(next ?? "");
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
case "'": {
|
||||
const close = command.indexOf("'", i + 1);
|
||||
const end = close === -1 ? command.length : close;
|
||||
const language = current.word === undefined ? inlineScriptLanguage(current) : undefined;
|
||||
if (language) emitInlineScript(i, end, language);
|
||||
appendToWord(command.slice(i + 1, end));
|
||||
i = end + 1;
|
||||
break;
|
||||
}
|
||||
case '"': {
|
||||
const language = current.word === undefined ? inlineScriptLanguage(current) : undefined;
|
||||
if (language) {
|
||||
const end = findDoubleQuoteEnd(i + 1);
|
||||
emitInlineScript(i, end, language);
|
||||
appendToWord(command.slice(i + 1, end));
|
||||
i = end + 1;
|
||||
} else {
|
||||
appendToWord("");
|
||||
frames.push({ kind: "double" });
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "$": {
|
||||
if (next === "(" && command[i + 2] === "(") {
|
||||
// Arithmetic expansion: `<<` inside is a shift, not a heredoc.
|
||||
const close = command.indexOf("))", i + 3);
|
||||
const end = close === -1 ? command.length : close + 2;
|
||||
appendToWord(command.slice(i, end));
|
||||
i = end;
|
||||
} else if (next === "(") {
|
||||
appendToWord("$()");
|
||||
frames.push({ kind: "shell", command: newCommand(), parenDepth: 0, substitution: true });
|
||||
i += 2;
|
||||
} else {
|
||||
appendToWord(char);
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "#": {
|
||||
if (current.word !== undefined) {
|
||||
appendToWord(char);
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
const newline = command.indexOf("\n", i);
|
||||
i = newline === -1 ? command.length : newline;
|
||||
break;
|
||||
}
|
||||
case " ":
|
||||
case "\t": {
|
||||
finishWord(current);
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
case "\n": {
|
||||
endCommand(frame);
|
||||
i = pendingHeredocs.length > 0 ? consumeHeredocs(i + 1) : i + 1;
|
||||
break;
|
||||
}
|
||||
case ";":
|
||||
case "|": {
|
||||
endCommand(frame);
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
case "&": {
|
||||
if (next === ">") {
|
||||
finishWord(current);
|
||||
current.pendingRedirect = { fd: "&", dup: false };
|
||||
i += command[i + 2] === ">" ? 3 : 2;
|
||||
} else {
|
||||
endCommand(frame);
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "(": {
|
||||
endCommand(frame);
|
||||
frame.parenDepth++;
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
case ")": {
|
||||
endCommand(frame);
|
||||
if (frame.substitution && frame.parenDepth === 0) frames.pop();
|
||||
else frame.parenDepth = Math.max(0, frame.parenDepth - 1);
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
case "<": {
|
||||
if (next === "(") {
|
||||
appendToWord("<()");
|
||||
frames.push({ kind: "shell", command: newCommand(), parenDepth: 0, substitution: true });
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
finishWord(current);
|
||||
if (next !== "<") {
|
||||
current.pendingRedirect = { fd: "0", dup: false };
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
if (command[i + 2] === "<") {
|
||||
// Here-string: the next word is data, not an argument.
|
||||
current.pendingRedirect = { fd: "0", dup: false };
|
||||
i += 3;
|
||||
break;
|
||||
}
|
||||
const match = /^<<(-?)[ \t]*(?:'([^'\n]*)'|"([^"\n]*)"|\\?([^\s;&|<>()'"]+))/.exec(command.slice(i));
|
||||
if (!match) {
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
const delimiter = match[2] ?? match[3] ?? match[4] ?? "";
|
||||
// While streaming, the operator line may end before the delimiter is complete.
|
||||
const complete = i + match[0].length < command.length;
|
||||
if (delimiter && complete) {
|
||||
pendingHeredocs.push({ delimiter, stripTabs: match[1] === "-", command: current });
|
||||
}
|
||||
i += match[0].length;
|
||||
break;
|
||||
}
|
||||
case ">": {
|
||||
let fd = "1";
|
||||
if (current.word !== undefined && /^\d+$/.test(current.word)) {
|
||||
fd = current.word;
|
||||
current.word = undefined;
|
||||
} else {
|
||||
finishWord(current);
|
||||
}
|
||||
if (next === "(") {
|
||||
appendToWord(">()");
|
||||
frames.push({ kind: "shell", command: newCommand(), parenDepth: 0, substitution: true });
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
if (command[i] === ">" || command[i] === "|") i++;
|
||||
const dup = command[i] === "&";
|
||||
if (dup) i++;
|
||||
current.pendingRedirect = { fd, dup };
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
appendToWord(char);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pushSegment(command.length, false);
|
||||
return segments;
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { type ShellSegment, splitShellCommand } from "../src/utils/shell-embedded-code.ts";
|
||||
|
||||
const EXTENSIONS: Record<string, string> = { rs: "rust", ts: "typescript", py: "python", json: "json" };
|
||||
|
||||
function split(command: string): ShellSegment[] {
|
||||
const segments = splitShellCommand(command, {
|
||||
languageFromPath: (path) => EXTENSIONS[path.slice(path.lastIndexOf(".") + 1)],
|
||||
});
|
||||
expect(segments.map((segment) => segment.text).join("")).toBe(command);
|
||||
return segments;
|
||||
}
|
||||
|
||||
function embedded(command: string): Array<{ text: string; language: string | undefined }> {
|
||||
return split(command)
|
||||
.filter((segment) => segment.embedded)
|
||||
.map((segment) => ({ text: segment.text, language: segment.language }));
|
||||
}
|
||||
|
||||
describe("splitShellCommand", () => {
|
||||
test("leaves plain commands as a single shell segment", () => {
|
||||
expect(split("cd repo && cargo test 2>&1 | grep -E 'a|b'")).toEqual([
|
||||
{ text: "cd repo && cargo test 2>&1 | grep -E 'a|b'", embedded: false },
|
||||
]);
|
||||
});
|
||||
|
||||
test("splits a python heredoc and keeps following commands as shell", () => {
|
||||
const command = "cd repo && python3 - <<'EOF'\ns = open('a').read()\nprint(\"it's\")\nEOF\ncargo test";
|
||||
expect(split(command)).toEqual([
|
||||
{ text: "cd repo && python3 - <<'EOF'\n", embedded: false },
|
||||
{ text: "s = open('a').read()\nprint(\"it's\")\n", embedded: true, language: "python" },
|
||||
{ text: "EOF\ncargo test", embedded: false },
|
||||
]);
|
||||
});
|
||||
|
||||
test("infers interpreters behind wrappers and redirects", () => {
|
||||
expect(embedded("uv run --no-project --with httpx --with bs4 python - <<'PY'\nx = 1\nPY")).toEqual([
|
||||
{ text: "x = 1\n", language: "python" },
|
||||
]);
|
||||
expect(embedded("./python.exe - <<'PY' > /tmp/log 2>&1\nx = 1\nPY")).toEqual([
|
||||
{ text: "x = 1\n", language: "python" },
|
||||
]);
|
||||
expect(embedded("node --input-type=module <<'JS'\nconsole.log(1)\nJS")).toEqual([
|
||||
{ text: "console.log(1)\n", language: "javascript" },
|
||||
]);
|
||||
expect(embedded("for d in a b; do timeout 20 bash <<'EOF'\necho $d\nEOF\ndone")).toEqual([
|
||||
{ text: "echo $d\n", language: "bash" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("uses the file extension for cat and tee targets", () => {
|
||||
expect(embedded("cat >> deser-json/tests/test_de.rs <<'EOF'\n#[test]\nfn x() {}\nEOF")).toEqual([
|
||||
{ text: "#[test]\nfn x() {}\n", language: "rust" },
|
||||
]);
|
||||
expect(embedded("cat <<'EOF' > out.json\n{}\nEOF")).toEqual([{ text: "{}\n", language: "json" }]);
|
||||
expect(embedded("tee -a notes.py <<'EOF'\nx = 1\nEOF")).toEqual([{ text: "x = 1\n", language: "python" }]);
|
||||
expect(embedded("F=test/zz-tmp-$$.test.ts && cat > $F <<'EOF'\nconst x = 1;\nEOF")).toEqual([
|
||||
{ text: "const x = 1;\n", language: "typescript" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("falls back to the delimiter name when stdin is data for a script", () => {
|
||||
expect(embedded("node tools/workspace.js exec --email a@b.c <<'JS' >out.json &\nreturn 1;\nJS")).toEqual([
|
||||
{ text: "return 1;\n", language: "javascript" },
|
||||
]);
|
||||
expect(embedded("python3 script.py <<'EOF'\ndata\nEOF")).toEqual([{ text: "data\n", language: undefined }]);
|
||||
expect(embedded("git commit -F - <<'EOF'\nfix: thing\nEOF")).toEqual([
|
||||
{ text: "fix: thing\n", language: undefined },
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles <<- and consecutive heredocs", () => {
|
||||
expect(embedded("python3 - <<-PY\n\tx = 1\n\tPY\nnode <<'JS'\ny()\nJS\n")).toEqual([
|
||||
{ text: "\tx = 1\n", language: "python" },
|
||||
{ text: "y()\n", language: "javascript" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("treats an unterminated heredoc as running to the end while streaming", () => {
|
||||
expect(split("python3 - <<'EOF'\nprint(1)\nEO")).toEqual([
|
||||
{ text: "python3 - <<'EOF'\n", embedded: false },
|
||||
{ text: "print(1)\nEO", embedded: true, language: "python" },
|
||||
]);
|
||||
expect(split("python3 - <<'EO")).toEqual([{ text: "python3 - <<'EO", embedded: false }]);
|
||||
});
|
||||
|
||||
test("ignores << inside quotes, arithmetic, and here-strings", () => {
|
||||
expect(embedded("rg -n 'a <<EOF b' && echo $((1 << 3)) && cat <<< \"x\"\nfoo")).toEqual([]);
|
||||
});
|
||||
|
||||
test("extracts inline interpreter scripts", () => {
|
||||
expect(split('cd x && python3 -c "\nimport sys\nprint(sys.argv)\n" 2>&1 | tail')).toEqual([
|
||||
{ text: 'cd x && python3 -c "', embedded: false },
|
||||
{ text: "\nimport sys\nprint(sys.argv)\n", embedded: true, language: "python" },
|
||||
{ text: '" 2>&1 | tail', embedded: false },
|
||||
]);
|
||||
expect(embedded("node -e 'console.log(1)'")).toEqual([{ text: "console.log(1)", language: "javascript" }]);
|
||||
expect(embedded("ruby -rjson -e 'puts 1'")).toEqual([{ text: "puts 1", language: "ruby" }]);
|
||||
expect(embedded("perl -0pi -e 's/a/b/' file")).toEqual([{ text: "s/a/b/", language: "perl" }]);
|
||||
expect(embedded("zsh -i -c 'echo hi'")).toEqual([{ text: "echo hi", language: "bash" }]);
|
||||
expect(embedded("awk -F: '{ print $1 }' /etc/passwd")).toEqual([{ text: "{ print $1 }", language: "awk" }]);
|
||||
expect(embedded("osascript -e 'tell app \"Finder\" to quit'")).toEqual([
|
||||
{ text: 'tell app "Finder" to quit', language: "applescript" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("does not treat flags of other tools as inline scripts", () => {
|
||||
expect(embedded("grep -e 'foo' x && rg -n 'bar' && git commit -m \"feat: x\n\nbody\"")).toEqual([]);
|
||||
expect(embedded("sed -e 's/a/b/' x && python3 script.py -c 'x'")).toEqual([]);
|
||||
expect(embedded("python3 -m pytest -c 'conf.ini' && python -mpytest -c 'conf.ini'")).toEqual([]);
|
||||
expect(embedded("python3 -m tool <<'EOF'\ndata\nEOF")).toEqual([{ text: "data\n", language: undefined }]);
|
||||
});
|
||||
|
||||
test("finds heredocs inside command substitution", () => {
|
||||
expect(embedded("git commit -m \"$(cat <<'EOF'\nfix: x\nEOF\n)\"")).toEqual([
|
||||
{ text: "fix: x\n", language: undefined },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
import type { TUI } from "@earendil-works/pi-tui";
|
||||
import { beforeAll, describe, expect, test, vi } from "vitest";
|
||||
|
||||
const highlightedLines = vi.hoisted(() => ({ count: 0 }));
|
||||
|
||||
vi.mock("../src/utils/syntax-highlight.ts", async (importOriginal) => {
|
||||
const original = await importOriginal<typeof import("../src/utils/syntax-highlight.ts")>();
|
||||
return {
|
||||
...original,
|
||||
highlight: (code: string, options: Parameters<typeof original.highlight>[1]) => {
|
||||
highlightedLines.count += code.split("\n").length;
|
||||
return original.highlight(code, options);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import type { ToolDefinition } from "../src/core/extensions/types.ts";
|
||||
import { createBashToolDefinition } from "../src/core/tools/bash.ts";
|
||||
import { createWriteToolDefinition } from "../src/core/tools/write.ts";
|
||||
import { ToolExecutionComponent } from "../src/modes/interactive/components/tool-execution.ts";
|
||||
import { initTheme } from "../src/modes/interactive/theme/theme.ts";
|
||||
|
||||
function createToolComponent(
|
||||
name: string,
|
||||
args: Record<string, unknown>,
|
||||
definition: ToolDefinition<any, any>,
|
||||
): ToolExecutionComponent {
|
||||
return new ToolExecutionComponent(
|
||||
name,
|
||||
`tool-${name}-cache`,
|
||||
args,
|
||||
{},
|
||||
definition,
|
||||
{ requestRender: () => {} } as unknown as TUI,
|
||||
process.cwd(),
|
||||
);
|
||||
}
|
||||
|
||||
function createComponent(command: string): ToolExecutionComponent {
|
||||
return createToolComponent(
|
||||
"bash",
|
||||
{ command },
|
||||
createBashToolDefinition(process.cwd(), { exposeSessionEnvironment: false }),
|
||||
);
|
||||
}
|
||||
|
||||
describe("streaming call highlighting", () => {
|
||||
beforeAll(() => {
|
||||
initTheme("dark");
|
||||
});
|
||||
|
||||
test("highlights streamed heredocs incrementally and reuses the result for output updates", () => {
|
||||
const header = "cat > src/lib.rs <<'EOF'\n";
|
||||
const bodyLines = Array.from({ length: 400 }, (_, i) => `fn f${i}() -> u32 { ${i} }`);
|
||||
const command = `${header}${bodyLines.join("\n")}\nEOF`;
|
||||
|
||||
const component = createComponent(header);
|
||||
component.setExpanded(true);
|
||||
highlightedLines.count = 0;
|
||||
for (let i = 1; i <= bodyLines.length; i++) {
|
||||
component.updateArgs({ command: `${header}${bodyLines.slice(0, i).join("\n")}\n` });
|
||||
}
|
||||
component.updateArgs({ command });
|
||||
// Each update highlights its new line plus a bounded prefix; a full rehighlight per update
|
||||
// would cost 400 * 401 / 2 = 80,200 lines.
|
||||
expect(highlightedLines.count).toBeLessThan(bodyLines.length * 60);
|
||||
|
||||
component.setArgsComplete();
|
||||
component.markExecutionStarted();
|
||||
const final = component.render(200);
|
||||
const fresh = createComponent(command);
|
||||
fresh.setExpanded(true);
|
||||
fresh.setArgsComplete();
|
||||
fresh.markExecutionStarted();
|
||||
expect(final).toEqual(fresh.render(200));
|
||||
|
||||
highlightedLines.count = 0;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
component.updateResult({ content: [{ type: "text", text: `line ${i}` }], isError: false }, true);
|
||||
}
|
||||
component.updateResult({ content: [{ type: "text", text: "done" }], isError: false }, false);
|
||||
expect(highlightedLines.count).toBe(0);
|
||||
});
|
||||
|
||||
test("highlights streamed write content incrementally and fully once complete", () => {
|
||||
const definition = createWriteToolDefinition(process.cwd());
|
||||
const contentLines = Array.from({ length: 400 }, (_, i) => `const s${i} = \`multi`);
|
||||
const content = `${contentLines.join("\n")}\n`;
|
||||
|
||||
const component = createToolComponent("write", { path: "src/a.ts", content: "" }, definition);
|
||||
component.setExpanded(true);
|
||||
highlightedLines.count = 0;
|
||||
for (let i = 1; i <= contentLines.length; i++) {
|
||||
component.updateArgs({ path: "src/a.ts", content: `${contentLines.slice(0, i).join("\n")}\n` });
|
||||
}
|
||||
expect(highlightedLines.count).toBeLessThan(contentLines.length * 60);
|
||||
|
||||
// The unterminated template literal spans lines, so only a full highlight matches a fresh render.
|
||||
component.setArgsComplete();
|
||||
const fresh = createToolComponent("write", { path: "src/a.ts", content }, definition);
|
||||
fresh.setExpanded(true);
|
||||
fresh.setArgsComplete();
|
||||
expect(component.render(200)).toEqual(fresh.render(200));
|
||||
|
||||
highlightedLines.count = 0;
|
||||
component.updateResult({ content: [{ type: "text", text: "ok" }], isError: false }, false);
|
||||
expect(highlightedLines.count).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -279,6 +279,36 @@ describe("ToolExecutionComponent parity", () => {
|
||||
expect(completed).toContain(`Took ${formatted}`);
|
||||
});
|
||||
|
||||
test("bash renderer highlights heredoc bodies and collapses long ones", () => {
|
||||
const body = Array.from({ length: 15 }, (_, i) => `import mod${i + 1}`).join("\n");
|
||||
const command = `cd repo && python3 - <<'EOF'\n${body}\nEOF\ncargo test`;
|
||||
const component = new ToolExecutionComponent(
|
||||
"bash",
|
||||
"tool-bash-heredoc",
|
||||
{ command },
|
||||
{},
|
||||
createBashToolDefinition(process.cwd(), { exposeSessionEnvironment: false }),
|
||||
createFakeTui(),
|
||||
process.cwd(),
|
||||
);
|
||||
|
||||
const collapsed = component.render(120);
|
||||
const plain = collapsed.map((line) => stripAnsi(line).trimEnd());
|
||||
expect(plain).toContain(" $ cd repo && python3 - <<'EOF'");
|
||||
expect(plain).toContain(" import mod10");
|
||||
expect(plain).not.toContain(" import mod11");
|
||||
expect(plain.some((line) => line.startsWith(" ... (5 more lines,"))).toBe(true);
|
||||
expect(plain).toContain(" EOF");
|
||||
expect(plain).toContain(" cargo test");
|
||||
const bodyLine = collapsed[plain.indexOf(" import mod1")];
|
||||
expect(bodyLine).toContain(theme.fg("syntaxKeyword", "import"));
|
||||
|
||||
component.setExpanded(true);
|
||||
const expanded = component.render(120).map((line) => stripAnsi(line).trimEnd());
|
||||
expect(expanded).toContain(" import mod15");
|
||||
expect(expanded.some((line) => line.includes("more lines"))).toBe(false);
|
||||
});
|
||||
|
||||
test("does not duplicate built-in headers when passed the active built-in definition", () => {
|
||||
const component = new ToolExecutionComponent(
|
||||
"read",
|
||||
|
||||
Reference in New Issue
Block a user