mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-02 07:34:45 +08:00
feat(cli): add interactive agent REPL with Chat gRPC service
Add `nav agent` command with a client-side REPL that communicates with LLMs through a new Chat gRPC streaming service on the gateway server and executes tools by shelling out to `nav` CLI subcommands. Server-side: - chat.proto with ChatStream server-streaming RPC, ChatMessage, Tool, ToolCall, ContentDelta, and ChatError messages - ChatService resolves inference routes by routing_hint and delegates to protocol-specific translators (OpenAI and Anthropic) - OpenAI and Anthropic SSE streaming with tool call accumulation - GrpcRouter extended from 2 to 3 services in the multiplexer Client-side: - 9 tools (cluster_status, sandbox CRUD/logs, provider list/get, inference_route_list) executed via subprocess shell-out - Binary resolution via current_exe() with NAV_AGENT_CLI env override - rustyline-based interactive REPL with multi-turn tool-calling loop - --routing-hint and --system-prompt CLI arguments Closes #50
This commit is contained in:
Drew Newberry
Generated
+97
@@ -681,6 +681,15 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
|
||||
|
||||
[[package]]
|
||||
name = "clipboard-win"
|
||||
version = "5.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4"
|
||||
dependencies = [
|
||||
"error-code",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cmake"
|
||||
version = "0.1.57"
|
||||
@@ -1197,6 +1206,12 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "endian-type"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
|
||||
|
||||
[[package]]
|
||||
name = "enum_dispatch"
|
||||
version = "0.3.13"
|
||||
@@ -1245,6 +1260,12 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "error-code"
|
||||
version = "3.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59"
|
||||
|
||||
[[package]]
|
||||
name = "etcetera"
|
||||
version = "0.8.0"
|
||||
@@ -1273,6 +1294,17 @@ version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "fd-lock"
|
||||
version = "4.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"rustix 1.1.3",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ff"
|
||||
version = "0.13.1"
|
||||
@@ -2611,6 +2643,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"rustls",
|
||||
"rustls-pemfile",
|
||||
"rustyline",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
@@ -2733,6 +2766,7 @@ dependencies = [
|
||||
"prost-types",
|
||||
"rand 0.9.2",
|
||||
"rcgen",
|
||||
"reqwest",
|
||||
"russh",
|
||||
"rustls",
|
||||
"rustls-pemfile",
|
||||
@@ -2753,6 +2787,15 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nibble_vec"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.29.0"
|
||||
@@ -3446,6 +3489,16 @@ version = "5.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "radix_trie"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
|
||||
dependencies = [
|
||||
"endian-type",
|
||||
"nibble_vec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
@@ -3600,6 +3653,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
@@ -3619,12 +3673,14 @@ dependencies = [
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
"tower 0.5.3",
|
||||
"tower-http 0.6.8",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"wasm-streams",
|
||||
"web-sys",
|
||||
"webpki-roots 1.0.6",
|
||||
]
|
||||
@@ -3891,6 +3947,28 @@ version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||
|
||||
[[package]]
|
||||
name = "rustyline"
|
||||
version = "15.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"clipboard-win",
|
||||
"fd-lock",
|
||||
"home",
|
||||
"libc",
|
||||
"log",
|
||||
"memchr",
|
||||
"nix",
|
||||
"radix_trie",
|
||||
"unicode-segmentation",
|
||||
"unicode-width 0.2.2",
|
||||
"utf8parse",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.22"
|
||||
@@ -5145,6 +5223,12 @@ version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.14"
|
||||
@@ -5327,6 +5411,19 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-streams"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.85"
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ serde_json = "1"
|
||||
serde_yaml = "0.9"
|
||||
|
||||
# HTTP client
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
|
||||
|
||||
# Utilities
|
||||
futures = "0.3"
|
||||
|
||||
@@ -45,6 +45,12 @@ rustls-pemfile = { workspace = true }
|
||||
tokio-rustls = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
|
||||
# Streams
|
||||
tokio-stream = { workspace = true }
|
||||
|
||||
# REPL
|
||||
rustyline = "15"
|
||||
|
||||
# Error handling
|
||||
miette = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,387 @@
|
||||
//! Interactive CLI agent for Navigator cluster operations.
|
||||
//!
|
||||
//! Runs a REPL loop that communicates with an LLM through the gateway's Chat
|
||||
//! gRPC service and executes tools by shelling out to `nav` CLI commands.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use navigator_core::proto::{
|
||||
ChatMessage, ChatStreamRequest, Tool, ToolCall, chat_stream_event::Event,
|
||||
};
|
||||
use owo_colors::OwoColorize;
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
use crate::tls::{TlsOptions, grpc_chat_client};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tool definitions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct ToolDef {
|
||||
name: &'static str,
|
||||
description: &'static str,
|
||||
parameters_schema: &'static str,
|
||||
/// Build CLI arguments from parsed JSON parameters.
|
||||
build_args: fn(&serde_json::Map<String, serde_json::Value>) -> Vec<String>,
|
||||
}
|
||||
|
||||
fn tool_definitions() -> Vec<ToolDef> {
|
||||
vec![
|
||||
ToolDef {
|
||||
name: "cluster_status",
|
||||
description: "Check the cluster health, connectivity, and version.",
|
||||
parameters_schema: r#"{"type":"object","properties":{}}"#,
|
||||
build_args: |_| vec!["cluster".into(), "status".into()],
|
||||
},
|
||||
ToolDef {
|
||||
name: "sandbox_list",
|
||||
description: "List all sandboxes in the cluster with their name, namespace, creation time, and phase.",
|
||||
parameters_schema: r#"{"type":"object","properties":{}}"#,
|
||||
build_args: |_| vec!["sandbox".into(), "list".into()],
|
||||
},
|
||||
ToolDef {
|
||||
name: "sandbox_get",
|
||||
description: "Get detailed information about a specific sandbox by name.",
|
||||
parameters_schema: r#"{"type":"object","properties":{"name":{"type":"string","description":"Sandbox name"}},"required":["name"]}"#,
|
||||
build_args: |args| {
|
||||
let mut v = vec!["sandbox".into(), "get".into()];
|
||||
if let Some(name) = args.get("name").and_then(|n| n.as_str()) {
|
||||
v.push(name.to_string());
|
||||
}
|
||||
v
|
||||
},
|
||||
},
|
||||
ToolDef {
|
||||
name: "sandbox_create",
|
||||
description: "Create a new sandbox. Optionally specify a container image and/or a command to run.",
|
||||
parameters_schema: r#"{"type":"object","properties":{"name":{"type":"string","description":"Optional sandbox name"},"image":{"type":"string","description":"Container image (e.g. ubuntu:24.04, python:3.12-slim)"},"command":{"type":"string","description":"Command to run in the sandbox"}}}"#,
|
||||
build_args: |args| {
|
||||
let mut v = vec!["sandbox".into(), "create".into()];
|
||||
if let Some(name) = args.get("name").and_then(|n| n.as_str()) {
|
||||
v.push("--name".into());
|
||||
v.push(name.to_string());
|
||||
}
|
||||
if let Some(image) = args.get("image").and_then(|n| n.as_str()) {
|
||||
v.push("--image".into());
|
||||
v.push(image.to_string());
|
||||
}
|
||||
if let Some(cmd) = args.get("command").and_then(|n| n.as_str()) {
|
||||
v.push("--keep".into());
|
||||
v.push("--".into());
|
||||
// Split command on whitespace for the trailing args.
|
||||
for part in cmd.split_whitespace() {
|
||||
v.push(part.to_string());
|
||||
}
|
||||
}
|
||||
v
|
||||
},
|
||||
},
|
||||
ToolDef {
|
||||
name: "sandbox_delete",
|
||||
description: "Delete a sandbox by name.",
|
||||
parameters_schema: r#"{"type":"object","properties":{"name":{"type":"string","description":"Sandbox name"}},"required":["name"]}"#,
|
||||
build_args: |args| {
|
||||
let mut v = vec!["sandbox".into(), "delete".into()];
|
||||
if let Some(name) = args.get("name").and_then(|n| n.as_str()) {
|
||||
v.push(name.to_string());
|
||||
}
|
||||
v
|
||||
},
|
||||
},
|
||||
ToolDef {
|
||||
name: "sandbox_logs",
|
||||
description: "View recent logs for a sandbox. Returns the latest log lines from the gateway and sandbox.",
|
||||
parameters_schema: r#"{"type":"object","properties":{"name":{"type":"string","description":"Sandbox name"},"lines":{"type":"integer","description":"Number of log lines to return (default: 50)"}},"required":["name"]}"#,
|
||||
build_args: |args| {
|
||||
let mut v = vec!["sandbox".into(), "logs".into()];
|
||||
if let Some(name) = args.get("name").and_then(|n| n.as_str()) {
|
||||
v.push(name.to_string());
|
||||
}
|
||||
if let Some(lines) = args.get("lines").and_then(serde_json::Value::as_u64) {
|
||||
v.push("-n".into());
|
||||
v.push(lines.to_string());
|
||||
}
|
||||
v
|
||||
},
|
||||
},
|
||||
ToolDef {
|
||||
name: "provider_list",
|
||||
description: "List all configured providers in the cluster.",
|
||||
parameters_schema: r#"{"type":"object","properties":{}}"#,
|
||||
build_args: |_| vec!["provider".into(), "list".into()],
|
||||
},
|
||||
ToolDef {
|
||||
name: "provider_get",
|
||||
description: "Get details of a specific provider by name.",
|
||||
parameters_schema: r#"{"type":"object","properties":{"name":{"type":"string","description":"Provider name"}},"required":["name"]}"#,
|
||||
build_args: |args| {
|
||||
let mut v = vec!["provider".into(), "get".into()];
|
||||
if let Some(name) = args.get("name").and_then(|n| n.as_str()) {
|
||||
v.push(name.to_string());
|
||||
}
|
||||
v
|
||||
},
|
||||
},
|
||||
ToolDef {
|
||||
name: "inference_route_list",
|
||||
description: "List all inference routes configured in the cluster.",
|
||||
parameters_schema: r#"{"type":"object","properties":{}}"#,
|
||||
build_args: |_| vec!["inference".into(), "list".into()],
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// Convert tool definitions to proto Tool messages.
|
||||
fn to_proto_tools(defs: &[ToolDef]) -> Vec<Tool> {
|
||||
defs.iter()
|
||||
.map(|d| Tool {
|
||||
name: d.name.to_string(),
|
||||
description: d.description.to_string(),
|
||||
parameters_schema: d.parameters_schema.to_string(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Binary resolution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Resolve the navigator CLI binary path.
|
||||
///
|
||||
/// Priority:
|
||||
/// 1. `NAV_AGENT_CLI` environment variable (for dev — set to `nav`)
|
||||
/// 2. `std::env::current_exe()` (the running binary itself)
|
||||
fn resolve_cli_binary() -> Result<PathBuf> {
|
||||
if let Ok(override_path) = std::env::var("NAV_AGENT_CLI") {
|
||||
return Ok(PathBuf::from(override_path));
|
||||
}
|
||||
std::env::current_exe()
|
||||
.into_diagnostic()
|
||||
.map_err(|e| miette::miette!("failed to resolve navigator binary path: {e}"))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tool execution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Execute a tool call by shelling out to the navigator CLI.
|
||||
async fn execute_tool(
|
||||
binary: &Path,
|
||||
cluster: &str,
|
||||
tool_defs: &[ToolDef],
|
||||
tool_call: &ToolCall,
|
||||
) -> String {
|
||||
let Some(tool_def) = tool_defs.iter().find(|d| d.name == tool_call.name) else {
|
||||
return format!("Unknown tool: {}", tool_call.name);
|
||||
};
|
||||
|
||||
let args_map: serde_json::Map<String, serde_json::Value> =
|
||||
match serde_json::from_str(&tool_call.arguments) {
|
||||
Ok(v) => v,
|
||||
Err(e) => return format!("Failed to parse tool arguments: {e}"),
|
||||
};
|
||||
|
||||
let cli_args = (tool_def.build_args)(&args_map);
|
||||
|
||||
eprintln!(
|
||||
" {} {} {}",
|
||||
"tool:".dimmed(),
|
||||
tool_call.name.cyan(),
|
||||
cli_args.join(" ").dimmed(),
|
||||
);
|
||||
|
||||
let result = tokio::process::Command::new(binary)
|
||||
.arg("--cluster")
|
||||
.arg(cluster)
|
||||
.args(&cli_args)
|
||||
.output()
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(output) => {
|
||||
let mut text = String::from_utf8_lossy(&output.stdout).to_string();
|
||||
if !output.stderr.is_empty() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
// Skip empty stderr or lines that are just whitespace.
|
||||
let stderr_trimmed = stderr.trim();
|
||||
if !stderr_trimmed.is_empty() {
|
||||
if !text.is_empty() {
|
||||
text.push('\n');
|
||||
}
|
||||
text.push_str(stderr_trimmed);
|
||||
}
|
||||
}
|
||||
if !output.status.success() {
|
||||
let _ = write!(
|
||||
text,
|
||||
"\nCommand exited with code: {}",
|
||||
output.status.code().unwrap_or(-1)
|
||||
);
|
||||
}
|
||||
if text.is_empty() {
|
||||
"Command completed successfully (no output).".to_string()
|
||||
} else {
|
||||
text
|
||||
}
|
||||
}
|
||||
Err(e) => format!("Failed to execute tool: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Default system prompt
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const DEFAULT_SYSTEM_PROMPT: &str = "\
|
||||
You are a Navigator cluster operations assistant. You help users manage and \
|
||||
operate their Navigator cluster through available tools.
|
||||
|
||||
You have access to tools that interact with the Navigator cluster. Use them \
|
||||
to answer questions about the cluster state, manage sandboxes, view logs, \
|
||||
and inspect configuration.
|
||||
|
||||
When the user asks about the cluster, sandboxes, providers, or inference \
|
||||
routes, use the appropriate tools to get current information rather than \
|
||||
guessing. Be concise and direct in your responses.
|
||||
|
||||
If a tool call fails, explain the error to the user and suggest how to fix it.";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Agent loop
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Run the interactive agent REPL.
|
||||
pub async fn run_agent(
|
||||
server: &str,
|
||||
cluster_name: &str,
|
||||
routing_hint: &str,
|
||||
system_prompt: Option<&str>,
|
||||
tls: &TlsOptions,
|
||||
) -> Result<()> {
|
||||
let mut client = grpc_chat_client(server, tls).await?;
|
||||
let binary = resolve_cli_binary()?;
|
||||
let tool_defs = tool_definitions();
|
||||
let proto_tools = to_proto_tools(&tool_defs);
|
||||
|
||||
let system_text = system_prompt.unwrap_or(DEFAULT_SYSTEM_PROMPT);
|
||||
|
||||
let mut messages: Vec<ChatMessage> = vec![ChatMessage {
|
||||
role: "system".to_string(),
|
||||
content: system_text.to_string(),
|
||||
tool_calls: vec![],
|
||||
tool_call_id: String::new(),
|
||||
}];
|
||||
|
||||
eprintln!(
|
||||
"{}\n",
|
||||
"Navigator Agent (type 'exit' or Ctrl-D to quit)"
|
||||
.cyan()
|
||||
.bold()
|
||||
);
|
||||
|
||||
let mut rl = rustyline::DefaultEditor::new().into_diagnostic()?;
|
||||
|
||||
loop {
|
||||
let input = match rl.readline("> ") {
|
||||
Ok(line) => line,
|
||||
Err(
|
||||
rustyline::error::ReadlineError::Interrupted | rustyline::error::ReadlineError::Eof,
|
||||
) => {
|
||||
eprintln!("\n{}", "Goodbye.".dimmed());
|
||||
break;
|
||||
}
|
||||
Err(e) => return Err(miette::miette!("readline error: {e}")),
|
||||
};
|
||||
|
||||
let trimmed = input.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if trimmed == "exit" || trimmed == "quit" {
|
||||
eprintln!("{}", "Goodbye.".dimmed());
|
||||
break;
|
||||
}
|
||||
|
||||
let _ = rl.add_history_entry(trimmed);
|
||||
|
||||
messages.push(ChatMessage {
|
||||
role: "user".to_string(),
|
||||
content: trimmed.to_string(),
|
||||
tool_calls: vec![],
|
||||
tool_call_id: String::new(),
|
||||
});
|
||||
|
||||
// Tool-calling loop: keep calling the LLM until it responds without tool calls.
|
||||
loop {
|
||||
let request = ChatStreamRequest {
|
||||
messages: messages.clone(),
|
||||
tools: proto_tools.clone(),
|
||||
routing_hint: routing_hint.to_string(),
|
||||
};
|
||||
|
||||
let response = client
|
||||
.chat_stream(request)
|
||||
.await
|
||||
.into_diagnostic()
|
||||
.map_err(|e| miette::miette!("chat stream failed: {e}"))?;
|
||||
|
||||
let mut stream = response.into_inner();
|
||||
let mut final_message: Option<ChatMessage> = None;
|
||||
let mut had_content = false;
|
||||
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.into_diagnostic()?;
|
||||
match event.event {
|
||||
Some(Event::ContentDelta(delta)) => {
|
||||
print!("{}", delta.text);
|
||||
std::io::stdout().flush().ok();
|
||||
had_content = true;
|
||||
}
|
||||
Some(Event::Message(msg)) => {
|
||||
final_message = Some(msg);
|
||||
}
|
||||
Some(Event::Error(e)) => {
|
||||
eprintln!("\n{} {}", "Error:".red().bold(), e.message);
|
||||
break;
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
|
||||
if had_content {
|
||||
println!();
|
||||
}
|
||||
|
||||
let Some(msg) = final_message else {
|
||||
break;
|
||||
};
|
||||
|
||||
let has_tool_calls = !msg.tool_calls.is_empty();
|
||||
messages.push(msg.clone());
|
||||
|
||||
if !has_tool_calls {
|
||||
break;
|
||||
}
|
||||
|
||||
// Execute all tool calls and feed results back.
|
||||
for tc in &msg.tool_calls {
|
||||
let result = execute_tool(&binary, cluster_name, &tool_defs, tc).await;
|
||||
messages.push(ChatMessage {
|
||||
role: "tool".to_string(),
|
||||
content: result,
|
||||
tool_calls: vec![],
|
||||
tool_call_id: tc.id.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
// Continue the loop to send tool results back to the LLM.
|
||||
}
|
||||
|
||||
println!();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
//!
|
||||
//! This crate provides the CLI implementation for Navigator.
|
||||
|
||||
pub mod agent;
|
||||
pub mod bootstrap;
|
||||
pub mod completers;
|
||||
pub mod run;
|
||||
|
||||
@@ -131,6 +131,24 @@ enum Commands {
|
||||
command: ProviderCommands,
|
||||
},
|
||||
|
||||
/// Interactive agent for cluster operations.
|
||||
///
|
||||
/// Starts an interactive REPL that uses an LLM (via the gateway Chat service)
|
||||
/// to help operate the cluster. The agent can list sandboxes, view logs,
|
||||
/// check cluster status, and more by calling Navigator CLI commands as tools.
|
||||
///
|
||||
/// Requires an inference route with the matching routing hint (default: "agent").
|
||||
/// Create one with: nav inference create --routing-hint agent --base-url <url> --model-id <model> --api-key <key>
|
||||
Agent {
|
||||
/// Routing hint to select the inference route (default: "agent").
|
||||
#[arg(long, default_value = "agent")]
|
||||
routing_hint: String,
|
||||
|
||||
/// Path to a custom system prompt file.
|
||||
#[arg(long)]
|
||||
system_prompt: Option<PathBuf>,
|
||||
},
|
||||
|
||||
/// Generate shell completions.
|
||||
#[command(after_long_help = COMPLETIONS_HELP)]
|
||||
Completions {
|
||||
@@ -1239,6 +1257,28 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(Commands::Agent {
|
||||
routing_hint,
|
||||
system_prompt,
|
||||
}) => {
|
||||
let ctx = resolve_cluster(&cli.cluster)?;
|
||||
let tls = tls.with_cluster_name(&ctx.name);
|
||||
let prompt = system_prompt
|
||||
.map(|p| {
|
||||
std::fs::read_to_string(&p).map_err(|e| {
|
||||
miette::miette!("failed to read system prompt from {}: {e}", p.display())
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
navigator_cli::agent::run_agent(
|
||||
&ctx.endpoint,
|
||||
&ctx.name,
|
||||
&routing_hint,
|
||||
prompt.as_deref(),
|
||||
&tls,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Some(Commands::Completions { shell }) => {
|
||||
let exe = std::env::current_exe()
|
||||
.map_err(|e| miette::miette!("failed to find current executable: {e}"))?;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use miette::{IntoDiagnostic, Result, WrapErr};
|
||||
use navigator_core::proto::chat_client::ChatClient;
|
||||
use navigator_core::proto::inference_client::InferenceClient;
|
||||
use navigator_core::proto::navigator_client::NavigatorClient;
|
||||
use rustls::{
|
||||
@@ -212,6 +213,19 @@ pub async fn grpc_client(server: &str, tls: &TlsOptions) -> Result<NavigatorClie
|
||||
Ok(NavigatorClient::new(channel))
|
||||
}
|
||||
|
||||
pub async fn grpc_chat_client(server: &str, tls: &TlsOptions) -> Result<ChatClient<Channel>> {
|
||||
let mut endpoint = Endpoint::from_shared(server.to_string())
|
||||
.into_diagnostic()?
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.http2_keep_alive_interval(Duration::from_secs(10))
|
||||
.keep_alive_while_idle(true);
|
||||
let materials = require_tls_materials(server, tls)?;
|
||||
let tls_config = build_tonic_tls_config(&materials);
|
||||
endpoint = endpoint.tls_config(tls_config).into_diagnostic()?;
|
||||
let channel = endpoint.connect().await.into_diagnostic()?;
|
||||
Ok(ChatClient::new(channel))
|
||||
}
|
||||
|
||||
pub async fn grpc_inference_client(
|
||||
server: &str,
|
||||
tls: &TlsOptions,
|
||||
|
||||
@@ -14,6 +14,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
"../../proto/datamodel.proto",
|
||||
"../../proto/sandbox.proto",
|
||||
"../../proto/inference.proto",
|
||||
"../../proto/chat.proto",
|
||||
"../../proto/test.proto",
|
||||
];
|
||||
|
||||
|
||||
@@ -63,6 +63,20 @@ pub mod inference {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::all,
|
||||
clippy::pedantic,
|
||||
clippy::nursery,
|
||||
unused_qualifications,
|
||||
rust_2018_idioms
|
||||
)]
|
||||
pub mod chat {
|
||||
pub mod v1 {
|
||||
include!(concat!(env!("OUT_DIR"), "/navigator.chat.v1.rs"));
|
||||
}
|
||||
}
|
||||
|
||||
pub use chat::v1::*;
|
||||
pub use datamodel::v1::*;
|
||||
pub use inference::v1::*;
|
||||
pub use navigator::*;
|
||||
|
||||
@@ -61,6 +61,7 @@ kube = { workspace = true }
|
||||
kube-runtime = { workspace = true }
|
||||
k8s-openapi = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
reqwest = { workspace = true, features = ["stream"] }
|
||||
hmac = "0.12"
|
||||
sha2 = "0.10"
|
||||
hex = "0.4"
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
//! Anthropic messages protocol translation and SSE streaming.
|
||||
|
||||
use futures::StreamExt;
|
||||
use navigator_core::proto::{
|
||||
ChatMessage, ChatStreamEvent, ChatStreamRequest, ContentDelta, SandboxResolvedRoute, Tool,
|
||||
ToolCall, chat_stream_event::Event,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::sync::mpsc;
|
||||
use tonic::Status;
|
||||
use tracing::debug;
|
||||
|
||||
/// Translate proto messages to Anthropic messages API format.
|
||||
///
|
||||
/// Anthropic separates the system prompt from the messages array.
|
||||
/// Returns `(system_text, messages)`.
|
||||
fn translate_messages(messages: &[ChatMessage]) -> (Option<String>, Vec<Value>) {
|
||||
let mut system = None;
|
||||
let mut out = Vec::new();
|
||||
|
||||
for m in messages {
|
||||
match m.role.as_str() {
|
||||
"system" => {
|
||||
system = Some(m.content.clone());
|
||||
}
|
||||
"assistant" if !m.tool_calls.is_empty() => {
|
||||
// Assistant message with tool_use content blocks.
|
||||
let mut content_blocks: Vec<Value> = Vec::new();
|
||||
|
||||
if !m.content.is_empty() {
|
||||
content_blocks.push(json!({
|
||||
"type": "text",
|
||||
"text": m.content,
|
||||
}));
|
||||
}
|
||||
|
||||
for tc in &m.tool_calls {
|
||||
let input: Value =
|
||||
serde_json::from_str(&tc.arguments).unwrap_or_else(|_| json!({}));
|
||||
content_blocks.push(json!({
|
||||
"type": "tool_use",
|
||||
"id": tc.id,
|
||||
"name": tc.name,
|
||||
"input": input,
|
||||
}));
|
||||
}
|
||||
|
||||
out.push(json!({
|
||||
"role": "assistant",
|
||||
"content": content_blocks,
|
||||
}));
|
||||
}
|
||||
"tool" => {
|
||||
// Tool result block.
|
||||
out.push(json!({
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": m.tool_call_id,
|
||||
"content": m.content,
|
||||
}],
|
||||
}));
|
||||
}
|
||||
role => {
|
||||
out.push(json!({
|
||||
"role": role,
|
||||
"content": m.content,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(system, out)
|
||||
}
|
||||
|
||||
/// Translate proto Tool definitions to Anthropic tool format.
|
||||
fn translate_tools(tools: &[Tool]) -> Vec<Value> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let input_schema: Value =
|
||||
serde_json::from_str(&t.parameters_schema).unwrap_or_else(|_| {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
})
|
||||
});
|
||||
json!({
|
||||
"name": t.name,
|
||||
"description": t.description,
|
||||
"input_schema": input_schema,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Build the Anthropic request body.
|
||||
fn build_request_body(request: &ChatStreamRequest, model: &str) -> Value {
|
||||
let (system, messages) = translate_messages(&request.messages);
|
||||
|
||||
let mut body = json!({
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": 8192,
|
||||
"stream": true,
|
||||
});
|
||||
|
||||
if let Some(sys) = system {
|
||||
body["system"] = json!(sys);
|
||||
}
|
||||
|
||||
if !request.tools.is_empty() {
|
||||
body["tools"] = json!(translate_tools(&request.tools));
|
||||
}
|
||||
|
||||
body
|
||||
}
|
||||
|
||||
/// Stream a chat completion from an Anthropic backend.
|
||||
pub async fn stream_chat(
|
||||
client: &reqwest::Client,
|
||||
route: &SandboxResolvedRoute,
|
||||
request: &ChatStreamRequest,
|
||||
tx: &mpsc::Sender<Result<ChatStreamEvent, Status>>,
|
||||
) -> Result<(), Status> {
|
||||
let base = route.base_url.trim_end_matches('/');
|
||||
let url = format!("{base}/v1/messages");
|
||||
let body = build_request_body(request, &route.model_id);
|
||||
|
||||
debug!(url = %url, "sending Anthropic chat request");
|
||||
|
||||
let response = client
|
||||
.post(&url)
|
||||
.header("x-api-key", &route.api_key)
|
||||
.header("anthropic-version", "2023-06-01")
|
||||
.header("content-type", "application/json")
|
||||
.header("accept", "text/event-stream")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Status::unavailable(format!("failed to connect to LLM backend: {e}")))?;
|
||||
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let error_body = response
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "failed to read error body".to_string());
|
||||
return Err(Status::internal(format!(
|
||||
"LLM backend returned {status}: {error_body}"
|
||||
)));
|
||||
}
|
||||
|
||||
// Parse SSE stream.
|
||||
let mut content = String::new();
|
||||
let mut tool_calls: Vec<ToolCallAccumulator> = Vec::new();
|
||||
let mut current_tool_index: Option<usize> = None;
|
||||
let mut stream = response.bytes_stream();
|
||||
let mut buffer = String::new();
|
||||
let mut current_event_type = String::new();
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|e| Status::internal(format!("stream read error: {e}")))?;
|
||||
let text = String::from_utf8_lossy(&chunk);
|
||||
buffer.push_str(&text);
|
||||
|
||||
while let Some(line_end) = buffer.find('\n') {
|
||||
let line = buffer[..line_end].trim_end_matches('\r').to_string();
|
||||
buffer = buffer[line_end + 1..].to_string();
|
||||
|
||||
if line.is_empty() {
|
||||
// End of event — reset event type.
|
||||
current_event_type.clear();
|
||||
continue;
|
||||
}
|
||||
|
||||
if line.starts_with(':') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(event_type) = line.strip_prefix("event: ") {
|
||||
current_event_type = event_type.trim().to_string();
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(data) = line.strip_prefix("data: ") {
|
||||
let parsed: Value = match serde_json::from_str(data) {
|
||||
Ok(v) => v,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
match current_event_type.as_str() {
|
||||
"content_block_start" => {
|
||||
// May start a tool_use block.
|
||||
if let Some(cb) = parsed.get("content_block").filter(|cb| {
|
||||
cb.get("type").and_then(|t| t.as_str()) == Some("tool_use")
|
||||
}) {
|
||||
let id = cb
|
||||
.get("id")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let name = cb
|
||||
.get("name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
tool_calls.push(ToolCallAccumulator {
|
||||
id,
|
||||
name,
|
||||
arguments: String::new(),
|
||||
});
|
||||
current_tool_index = Some(tool_calls.len() - 1);
|
||||
}
|
||||
}
|
||||
"content_block_delta" => {
|
||||
if let Some(delta) = parsed.get("delta") {
|
||||
let delta_type =
|
||||
delta.get("type").and_then(|t| t.as_str()).unwrap_or("");
|
||||
|
||||
match delta_type {
|
||||
"text_delta" => {
|
||||
if let Some(text) = delta
|
||||
.get("text")
|
||||
.and_then(|t| t.as_str())
|
||||
.filter(|t| !t.is_empty())
|
||||
{
|
||||
content.push_str(text);
|
||||
let _ = tx
|
||||
.send(Ok(ChatStreamEvent {
|
||||
event: Some(Event::ContentDelta(ContentDelta {
|
||||
text: text.to_string(),
|
||||
})),
|
||||
}))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
"input_json_delta" => {
|
||||
if let (Some(idx), Some(partial)) = (
|
||||
current_tool_index,
|
||||
delta.get("partial_json").and_then(|p| p.as_str()),
|
||||
) && let Some(acc) = tool_calls.get_mut(idx)
|
||||
{
|
||||
acc.arguments.push_str(partial);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
"content_block_stop" => {
|
||||
current_tool_index = None;
|
||||
}
|
||||
// message_stop and other events — handled after the loop.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Send final complete message.
|
||||
let final_tool_calls: Vec<ToolCall> = tool_calls
|
||||
.into_iter()
|
||||
.map(|acc| ToolCall {
|
||||
id: acc.id,
|
||||
name: acc.name,
|
||||
arguments: acc.arguments,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let final_message = ChatMessage {
|
||||
role: "assistant".to_string(),
|
||||
content,
|
||||
tool_calls: final_tool_calls,
|
||||
tool_call_id: String::new(),
|
||||
};
|
||||
|
||||
let _ = tx
|
||||
.send(Ok(ChatStreamEvent {
|
||||
event: Some(Event::Message(final_message)),
|
||||
}))
|
||||
.await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Accumulates streamed tool call fragments.
|
||||
struct ToolCallAccumulator {
|
||||
id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! Chat gRPC service — streaming LLM completions with tool calling.
|
||||
//!
|
||||
//! Resolves an inference route by `routing_hint` (default `"agent"`),
|
||||
//! translates the request to the backend's native protocol, and streams
|
||||
//! token deltas back to the caller.
|
||||
|
||||
mod anthropic;
|
||||
mod openai;
|
||||
|
||||
use navigator_core::proto::{
|
||||
ChatStreamEvent, ChatStreamRequest, SandboxResolvedRoute, chat_server::Chat,
|
||||
chat_stream_event::Event,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use tracing::{error, info};
|
||||
|
||||
use crate::{ServerState, inference::list_sandbox_routes};
|
||||
|
||||
const DEFAULT_ROUTING_HINT: &str = "agent";
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ChatService {
|
||||
state: Arc<ServerState>,
|
||||
http_client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl ChatService {
|
||||
pub fn new(state: Arc<ServerState>) -> Self {
|
||||
let http_client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(300))
|
||||
.build()
|
||||
.expect("failed to build reqwest client");
|
||||
Self { state, http_client }
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl Chat for ChatService {
|
||||
type ChatStreamStream = ReceiverStream<Result<ChatStreamEvent, Status>>;
|
||||
|
||||
async fn chat_stream(
|
||||
&self,
|
||||
request: Request<ChatStreamRequest>,
|
||||
) -> Result<Response<Self::ChatStreamStream>, Status> {
|
||||
let req = request.into_inner();
|
||||
|
||||
let routing_hint = if req.routing_hint.is_empty() {
|
||||
DEFAULT_ROUTING_HINT.to_string()
|
||||
} else {
|
||||
req.routing_hint.clone()
|
||||
};
|
||||
|
||||
// Resolve the inference route.
|
||||
let routes = list_sandbox_routes(
|
||||
self.state.store.as_ref(),
|
||||
std::slice::from_ref(&routing_hint),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if routes.is_empty() {
|
||||
return Err(Status::failed_precondition(format!(
|
||||
"no enabled inference route with routing_hint '{routing_hint}'. \
|
||||
Create one with: nav inference create --routing-hint {routing_hint} ..."
|
||||
)));
|
||||
}
|
||||
|
||||
// Pick the first route that has a supported chat protocol.
|
||||
let (route, protocol) = pick_chat_route(&routes).ok_or_else(|| {
|
||||
Status::failed_precondition(
|
||||
"no inference route supports openai_chat_completions or anthropic_messages",
|
||||
)
|
||||
})?;
|
||||
|
||||
info!(
|
||||
routing_hint = %routing_hint,
|
||||
protocol = %protocol,
|
||||
model = %route.model_id,
|
||||
"starting chat stream"
|
||||
);
|
||||
|
||||
let (tx, rx) = mpsc::channel(256);
|
||||
let client = self.http_client.clone();
|
||||
let route = route.clone();
|
||||
let protocol = protocol.to_string();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let result = match protocol.as_str() {
|
||||
"openai_chat_completions" => openai::stream_chat(&client, &route, &req, &tx).await,
|
||||
"anthropic_messages" => anthropic::stream_chat(&client, &route, &req, &tx).await,
|
||||
_ => Err(Status::internal(format!(
|
||||
"unsupported protocol: {protocol}"
|
||||
))),
|
||||
};
|
||||
|
||||
if let Err(e) = result {
|
||||
error!(error = %e, "chat stream failed");
|
||||
let _ = tx
|
||||
.send(Ok(ChatStreamEvent {
|
||||
event: Some(Event::Error(navigator_core::proto::ChatError {
|
||||
message: e.message().to_string(),
|
||||
status_code: e.code() as u32,
|
||||
})),
|
||||
}))
|
||||
.await;
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::new(ReceiverStream::new(rx)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Pick the first route that supports a chat protocol.
|
||||
///
|
||||
/// Returns the route and the protocol string.
|
||||
fn pick_chat_route(routes: &[SandboxResolvedRoute]) -> Option<(&SandboxResolvedRoute, &str)> {
|
||||
// Prefer openai_chat_completions, then anthropic_messages.
|
||||
for route in routes {
|
||||
for proto in &route.protocols {
|
||||
if proto == "openai_chat_completions" || proto == "anthropic_messages" {
|
||||
return Some((route, proto.as_str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
//! `OpenAI` chat completions protocol translation and SSE streaming.
|
||||
|
||||
use futures::StreamExt;
|
||||
use navigator_core::proto::{
|
||||
ChatMessage, ChatStreamEvent, ChatStreamRequest, ContentDelta, SandboxResolvedRoute, Tool,
|
||||
ToolCall, chat_stream_event::Event,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::sync::mpsc;
|
||||
use tonic::Status;
|
||||
use tracing::debug;
|
||||
|
||||
/// Translate proto messages to `OpenAI` JSON message format.
|
||||
fn translate_messages(messages: &[ChatMessage]) -> Vec<Value> {
|
||||
messages
|
||||
.iter()
|
||||
.map(|m| {
|
||||
let mut msg = json!({
|
||||
"role": m.role,
|
||||
});
|
||||
|
||||
match m.role.as_str() {
|
||||
"assistant" if !m.tool_calls.is_empty() => {
|
||||
// Assistant message with tool calls — may have empty content.
|
||||
if m.content.is_empty() {
|
||||
msg["content"] = Value::Null;
|
||||
} else {
|
||||
msg["content"] = json!(m.content);
|
||||
}
|
||||
msg["tool_calls"] = json!(
|
||||
m.tool_calls
|
||||
.iter()
|
||||
.map(|tc| {
|
||||
json!({
|
||||
"id": tc.id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tc.name,
|
||||
"arguments": tc.arguments,
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
"tool" => {
|
||||
msg["content"] = json!(m.content);
|
||||
msg["tool_call_id"] = json!(m.tool_call_id);
|
||||
}
|
||||
_ => {
|
||||
msg["content"] = json!(m.content);
|
||||
}
|
||||
}
|
||||
|
||||
msg
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Translate proto Tool definitions to `OpenAI` tool format.
|
||||
fn translate_tools(tools: &[Tool]) -> Vec<Value> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let parameters: Value =
|
||||
serde_json::from_str(&t.parameters_schema).unwrap_or_else(|_| json!({}));
|
||||
json!({
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": t.name,
|
||||
"description": t.description,
|
||||
"parameters": parameters,
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Build the `OpenAI` request body.
|
||||
fn build_request_body(request: &ChatStreamRequest, model: &str) -> Value {
|
||||
let mut body = json!({
|
||||
"model": model,
|
||||
"messages": translate_messages(&request.messages),
|
||||
"stream": true,
|
||||
});
|
||||
|
||||
if !request.tools.is_empty() {
|
||||
body["tools"] = json!(translate_tools(&request.tools));
|
||||
}
|
||||
|
||||
body
|
||||
}
|
||||
|
||||
/// Stream a chat completion from an OpenAI-compatible backend.
|
||||
pub async fn stream_chat(
|
||||
client: &reqwest::Client,
|
||||
route: &SandboxResolvedRoute,
|
||||
request: &ChatStreamRequest,
|
||||
tx: &mpsc::Sender<Result<ChatStreamEvent, Status>>,
|
||||
) -> Result<(), Status> {
|
||||
let base = route.base_url.trim_end_matches('/');
|
||||
let url = format!("{base}/v1/chat/completions");
|
||||
let body = build_request_body(request, &route.model_id);
|
||||
|
||||
debug!(url = %url, "sending OpenAI chat request");
|
||||
|
||||
let response = client
|
||||
.post(&url)
|
||||
.bearer_auth(&route.api_key)
|
||||
.header("content-type", "application/json")
|
||||
.header("accept", "text/event-stream")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Status::unavailable(format!("failed to connect to LLM backend: {e}")))?;
|
||||
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let error_body = response
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "failed to read error body".to_string());
|
||||
return Err(Status::internal(format!(
|
||||
"LLM backend returned {status}: {error_body}"
|
||||
)));
|
||||
}
|
||||
|
||||
// Parse SSE stream and accumulate the full response.
|
||||
let mut content = String::new();
|
||||
let mut tool_calls: Vec<ToolCallAccumulator> = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
|
||||
let mut buffer = String::new();
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|e| Status::internal(format!("stream read error: {e}")))?;
|
||||
let text = String::from_utf8_lossy(&chunk);
|
||||
buffer.push_str(&text);
|
||||
|
||||
// Process complete SSE lines from the buffer.
|
||||
while let Some(line_end) = buffer.find('\n') {
|
||||
let line = buffer[..line_end].trim_end_matches('\r').to_string();
|
||||
buffer = buffer[line_end + 1..].to_string();
|
||||
|
||||
if line.is_empty() || line.starts_with(':') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(data) = line.strip_prefix("data: ") {
|
||||
if data.trim() == "[DONE]" {
|
||||
break;
|
||||
}
|
||||
|
||||
let parsed: Value = match serde_json::from_str(data) {
|
||||
Ok(v) => v,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
if let Some(choices) = parsed.get("choices").and_then(|c| c.as_array()) {
|
||||
for choice in choices {
|
||||
let Some(delta) = choice.get("delta") else {
|
||||
continue;
|
||||
};
|
||||
|
||||
// Content delta.
|
||||
if let Some(text) = delta
|
||||
.get("content")
|
||||
.and_then(|c| c.as_str())
|
||||
.filter(|t| !t.is_empty())
|
||||
{
|
||||
content.push_str(text);
|
||||
let _ = tx
|
||||
.send(Ok(ChatStreamEvent {
|
||||
event: Some(Event::ContentDelta(ContentDelta {
|
||||
text: text.to_string(),
|
||||
})),
|
||||
}))
|
||||
.await;
|
||||
}
|
||||
|
||||
// Tool call deltas (streamed incrementally).
|
||||
if let Some(tcs) = delta.get("tool_calls").and_then(|tc| tc.as_array()) {
|
||||
for tc_delta in tcs {
|
||||
let index =
|
||||
tc_delta.get("index").and_then(Value::as_u64).unwrap_or(0);
|
||||
let index = usize::try_from(index).unwrap_or(0);
|
||||
|
||||
// Grow the accumulator vec as needed.
|
||||
while tool_calls.len() <= index {
|
||||
tool_calls.push(ToolCallAccumulator::default());
|
||||
}
|
||||
|
||||
let acc = &mut tool_calls[index];
|
||||
|
||||
if let Some(id) = tc_delta.get("id").and_then(|v| v.as_str()) {
|
||||
acc.id = id.to_string();
|
||||
}
|
||||
|
||||
if let Some(func) = tc_delta.get("function") {
|
||||
if let Some(name) = func.get("name").and_then(|n| n.as_str()) {
|
||||
acc.name = name.to_string();
|
||||
}
|
||||
if let Some(args) =
|
||||
func.get("arguments").and_then(|a| a.as_str())
|
||||
{
|
||||
acc.arguments.push_str(args);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Send the final complete message.
|
||||
let final_tool_calls: Vec<ToolCall> = tool_calls
|
||||
.into_iter()
|
||||
.map(|acc| ToolCall {
|
||||
id: acc.id,
|
||||
name: acc.name,
|
||||
arguments: acc.arguments,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let final_message = ChatMessage {
|
||||
role: "assistant".to_string(),
|
||||
content,
|
||||
tool_calls: final_tool_calls,
|
||||
tool_call_id: String::new(),
|
||||
};
|
||||
|
||||
let _ = tx
|
||||
.send(Ok(ChatStreamEvent {
|
||||
event: Some(Event::Message(final_message)),
|
||||
}))
|
||||
.await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Accumulates streamed tool call fragments.
|
||||
#[derive(Default)]
|
||||
struct ToolCallAccumulator {
|
||||
id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
}
|
||||
@@ -286,7 +286,7 @@ async fn resolve_sandbox_inference_bundle(
|
||||
///
|
||||
/// Routes are matched by `routing_hint` against the `allowed_routes` list
|
||||
/// from the sandbox's inference policy. Only enabled routes are returned.
|
||||
async fn list_sandbox_routes(
|
||||
pub(crate) async fn list_sandbox_routes(
|
||||
store: &Store,
|
||||
allowed_routes: &[String],
|
||||
) -> Result<Vec<SandboxResolvedRoute>, Status> {
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! - Protocol multiplexing (gRPC + HTTP on same port)
|
||||
//! - mTLS support
|
||||
|
||||
mod chat;
|
||||
mod grpc;
|
||||
mod http;
|
||||
mod inference;
|
||||
|
||||
@@ -12,7 +12,9 @@ use hyper_util::{
|
||||
rt::{TokioExecutor, TokioIo},
|
||||
server::conn::auto::Builder,
|
||||
};
|
||||
use navigator_core::proto::{inference_server::InferenceServer, navigator_server::NavigatorServer};
|
||||
use navigator_core::proto::{
|
||||
chat_server::ChatServer, inference_server::InferenceServer, navigator_server::NavigatorServer,
|
||||
};
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
@@ -20,7 +22,9 @@ use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use crate::{NavigatorService, ServerState, http_router, inference::InferenceService};
|
||||
use crate::{
|
||||
NavigatorService, ServerState, chat::ChatService, http_router, inference::InferenceService,
|
||||
};
|
||||
|
||||
/// Multiplexed gRPC/HTTP service.
|
||||
#[derive(Clone)]
|
||||
@@ -43,7 +47,8 @@ impl MultiplexService {
|
||||
{
|
||||
let navigator = NavigatorServer::new(NavigatorService::new(self.state.clone()));
|
||||
let inference = InferenceServer::new(InferenceService::new(self.state.clone()));
|
||||
let grpc_service = GrpcRouter::new(navigator, inference);
|
||||
let chat = ChatServer::new(ChatService::new(self.state.clone()));
|
||||
let grpc_service = GrpcRouter::new(navigator, inference, chat);
|
||||
let http_service = http_router(self.state.clone());
|
||||
|
||||
let service = MultiplexedService::new(grpc_service, http_service);
|
||||
@@ -56,26 +61,29 @@ impl MultiplexService {
|
||||
}
|
||||
}
|
||||
|
||||
/// Combined gRPC service that routes between Navigator and Inference services
|
||||
/// based on the request path prefix.
|
||||
/// Combined gRPC service that routes between Navigator, Inference, and Chat
|
||||
/// services based on the request path prefix.
|
||||
#[derive(Clone)]
|
||||
pub struct GrpcRouter<N, I> {
|
||||
pub struct GrpcRouter<N, I, C> {
|
||||
navigator: N,
|
||||
inference: I,
|
||||
chat: C,
|
||||
}
|
||||
|
||||
impl<N, I> GrpcRouter<N, I> {
|
||||
fn new(navigator: N, inference: I) -> Self {
|
||||
impl<N, I, C> GrpcRouter<N, I, C> {
|
||||
fn new(navigator: N, inference: I, chat: C) -> Self {
|
||||
Self {
|
||||
navigator,
|
||||
inference,
|
||||
chat,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const INFERENCE_PATH_PREFIX: &str = "/navigator.inference.v1.Inference/";
|
||||
const CHAT_PATH_PREFIX: &str = "/navigator.chat.v1.Chat/";
|
||||
|
||||
impl<N, I, B> tower::Service<Request<B>> for GrpcRouter<N, I>
|
||||
impl<N, I, C, B> tower::Service<Request<B>> for GrpcRouter<N, I, C>
|
||||
where
|
||||
N: tower::Service<Request<B>> + Clone + Send + 'static,
|
||||
N::Response: Send,
|
||||
@@ -86,6 +94,11 @@ where
|
||||
+ Send
|
||||
+ 'static,
|
||||
I::Future: Send,
|
||||
C: tower::Service<Request<B>, Response = N::Response, Error = N::Error>
|
||||
+ Clone
|
||||
+ Send
|
||||
+ 'static,
|
||||
C::Future: Send,
|
||||
B: Send + 'static,
|
||||
{
|
||||
type Response = N::Response;
|
||||
@@ -97,9 +110,12 @@ where
|
||||
}
|
||||
|
||||
fn call(&mut self, req: Request<B>) -> Self::Future {
|
||||
let is_inference = req.uri().path().starts_with(INFERENCE_PATH_PREFIX);
|
||||
let path = req.uri().path();
|
||||
|
||||
if is_inference {
|
||||
if path.starts_with(CHAT_PATH_PREFIX) {
|
||||
let mut svc = self.chat.clone();
|
||||
Box::pin(async move { svc.ready().await?.call(req).await })
|
||||
} else if path.starts_with(INFERENCE_PATH_PREFIX) {
|
||||
let mut svc = self.inference.clone();
|
||||
Box::pin(async move { svc.ready().await?.call(req).await })
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package navigator.chat.v1;
|
||||
|
||||
option java_multiple_files = true;
|
||||
option java_package = "com.anthropic.navigator.chat.v1";
|
||||
|
||||
// Chat service provides streaming LLM completions with tool calling support.
|
||||
//
|
||||
// The server resolves an inference route by routing_hint (default "agent"),
|
||||
// translates the request to the backend's native protocol (OpenAI or Anthropic),
|
||||
// and streams token deltas back to the client.
|
||||
service Chat {
|
||||
// Stream a single chat completion. Returns token chunks followed by
|
||||
// a final message containing any tool_calls.
|
||||
rpc ChatStream(ChatStreamRequest) returns (stream ChatStreamEvent);
|
||||
}
|
||||
|
||||
// --- Request ---
|
||||
|
||||
message ChatStreamRequest {
|
||||
// Conversation history (system, user, assistant, tool messages).
|
||||
repeated ChatMessage messages = 1;
|
||||
// Tool definitions available to the model.
|
||||
repeated Tool tools = 2;
|
||||
// Routing hint to select the inference route (default: "agent").
|
||||
string routing_hint = 3;
|
||||
}
|
||||
|
||||
message ChatMessage {
|
||||
// One of: "system", "user", "assistant", "tool".
|
||||
string role = 1;
|
||||
// Text content (for system/user/assistant text, or tool result).
|
||||
string content = 2;
|
||||
// Tool calls returned by the assistant (only present when role=assistant).
|
||||
repeated ToolCall tool_calls = 3;
|
||||
// Tool call ID this message is responding to (only present when role=tool).
|
||||
string tool_call_id = 4;
|
||||
}
|
||||
|
||||
message Tool {
|
||||
// Function name.
|
||||
string name = 1;
|
||||
// Human-readable description of what the tool does.
|
||||
string description = 2;
|
||||
// JSON Schema string describing the parameters.
|
||||
string parameters_schema = 3;
|
||||
}
|
||||
|
||||
message ToolCall {
|
||||
// Unique ID for this tool invocation.
|
||||
string id = 1;
|
||||
// Function name to call.
|
||||
string name = 2;
|
||||
// JSON string of arguments.
|
||||
string arguments = 3;
|
||||
}
|
||||
|
||||
// --- Response (streaming) ---
|
||||
|
||||
message ChatStreamEvent {
|
||||
oneof event {
|
||||
// Incremental text chunk (streamed as tokens arrive).
|
||||
ContentDelta content_delta = 1;
|
||||
// Final complete message (always the last event).
|
||||
ChatMessage message = 2;
|
||||
// Error from the LLM backend.
|
||||
ChatError error = 3;
|
||||
}
|
||||
}
|
||||
|
||||
message ContentDelta {
|
||||
string text = 1;
|
||||
}
|
||||
|
||||
message ChatError {
|
||||
string message = 1;
|
||||
uint32 status_code = 2;
|
||||
}
|
||||
Reference in New Issue
Block a user