mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-19 05:50:41 +08:00
Extend python/openshell/ with the verbs needed to express the multi-agent-notepad flow entirely in Python: - ProviderClient (create/get/list/delete); credentials_from_env resolves env var names to mirror the CLI's --credential flag. - policy_from_yaml() loads the CLI's YAML format into a SandboxPolicy proto. Lazy-imports PyYAML, added to runtime deps. - SandboxClient.upload() gzip-tars the source and pipes the tarball over ExecSandbox(stdin=...) to 'tar xzf -'. No paramiko dependency. - SandboxClient.create() gains friendly kwargs (name, image, providers, policy, environment, gpu, labels) alongside the existing spec= path. architecture/python-sdk.md documents the design principle (mirror CLI verbs, not raw protos), the public surface, and what's out of scope (image build, gateway bootstrap, draft policy, port-forward, RelayStream). examples/multi-agent-notepad/notebook.ipynb is a Python equivalent of demo.sh: env validation, policy render, provider creation, parallel worker sandboxes via ThreadPoolExecutor, synthesis sandbox, cleanup. 13 new unit tests; all 23 sandbox tests pass; ruff/ty/markdownlint clean.
13 KiB
13 KiB
In [ ]:
import json
import os
import shutil
import tempfile
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
import grpc
from openshell import SandboxClient, SandboxError, policy_from_yaml
# Resolve the demo directory whether the notebook runs from its own folder or repo root.
_candidates = [Path.cwd(), Path.cwd() / "examples" / "multi-agent-notepad"]
SCRIPT_DIR = next((c for c in _candidates if (c / "runner.sh").exists()), None)
if SCRIPT_DIR is None:
raise SystemExit("Could not locate runner.sh; run from examples/multi-agent-notepad/ or the repo root.")
POLICY_TEMPLATE = SCRIPT_DIR / "policy.template.yaml"
PROMPTS_DIR = SCRIPT_DIR / "prompts"
RUNNER_SH = SCRIPT_DIR / "runner.sh"
DEMO_TOPIC = os.environ.get("DEMO_TOPIC", "How should teams evaluate sandboxed coding agents?")
DEMO_AGENT_COUNT = int(os.environ.get("DEMO_AGENT_COUNT", "3"))
DEMO_BRANCH = os.environ.get("DEMO_BRANCH", "main")
DEMO_RUN_ID = os.environ.get("DEMO_RUN_ID", time.strftime("%Y%m%d-%H%M%S"))
if not 1 <= DEMO_AGENT_COUNT <= 8:
raise SystemExit("DEMO_AGENT_COUNT must be between 1 and 8")
CODEX_PROVIDER = f"codex-oauth-{DEMO_RUN_ID}"
GITHUB_PROVIDER = f"github-memory-{DEMO_RUN_ID}"
WORKER_SLICES = [
"Adoption criteria: what makes a sandboxed coding-agent workflow trustworthy enough to try?",
"Operational risks: what can go wrong when many autonomous agents run at once?",
"Security controls: which controls make the biggest difference for credential and network safety?",
"Developer experience: where does the workflow need to feel simple for users to adopt it?",
"Measurement: what signals show whether the agents produced useful work?",
"Scaling: what changes when the team runs dozens of agents instead of five?",
"Governance: what review and approval steps should remain human-controlled?",
"Repository hygiene: what makes the shared markdown notepad easy to review and clean up?",
]
print(f"Run id: {DEMO_RUN_ID}")
print(f"Topic: {DEMO_TOPIC}")
print(f"Agent count: {DEMO_AGENT_COUNT}")In [ ]:
github_owner = os.environ["DEMO_GITHUB_OWNER"]
github_repo = os.environ["DEMO_GITHUB_REPO"]
_ = os.environ["DEMO_GITHUB_TOKEN"] # checked here so we fail fast
codex_auth_path = Path.home() / ".codex" / "auth.json"
if not codex_auth_path.exists():
raise SystemExit("missing local Codex sign-in; run: codex login")
tokens = json.loads(codex_auth_path.read_text())["tokens"]
for key in ("access_token", "refresh_token", "account_id"):
if not tokens.get(key):
raise SystemExit(f"local Codex sign-in is missing {key}; run: codex login")
os.environ["CODEX_AUTH_ACCESS_TOKEN"] = tokens["access_token"]
os.environ["CODEX_AUTH_REFRESH_TOKEN"] = tokens["refresh_token"]
os.environ["CODEX_AUTH_ACCOUNT_ID"] = tokens["account_id"]
client = SandboxClient.from_active_cluster()
health = client.health()
print(f"Gateway reachable. Server version: {health.version!r}")In [ ]:
rendered = (
POLICY_TEMPLATE.read_text()
.replace("__OWNER__", github_owner)
.replace("__REPO__", github_repo)
.replace("__RUN_ID__", DEMO_RUN_ID)
)
policy = policy_from_yaml(rendered)
print(f"Loaded policy v{policy.version} with rule sets: {sorted(policy.network_policies.keys())}")In [ ]:
def safe_provider_delete(name: str) -> None:
try:
client.providers.delete(name)
except grpc.RpcError as exc:
if exc.code() != grpc.StatusCode.NOT_FOUND:
raise
for stale in (CODEX_PROVIDER, GITHUB_PROVIDER):
safe_provider_delete(stale)
client.providers.create(
name=CODEX_PROVIDER,
provider_type="generic",
credentials_from_env=[
"CODEX_AUTH_ACCESS_TOKEN",
"CODEX_AUTH_REFRESH_TOKEN",
"CODEX_AUTH_ACCOUNT_ID",
],
)
client.providers.create(
name=GITHUB_PROVIDER,
provider_type="generic",
credentials_from_env=["DEMO_GITHUB_TOKEN"],
)
print(f"Created providers: {CODEX_PROVIDER}, {GITHUB_PROVIDER}")In [ ]:
work_dir = Path(tempfile.mkdtemp(prefix="openshell-notebook-"))
payload_dir = work_dir / "payload"
(payload_dir / "prompts").mkdir(parents=True)
shutil.copy(RUNNER_SH, payload_dir / "demo-runner.sh")
(payload_dir / "demo-runner.sh").chmod(0o755)
for prompt in PROMPTS_DIR.glob("*.md"):
shutil.copy(prompt, payload_dir / "prompts" / prompt.name)
print(f"Payload staged at {payload_dir}")
print(" files:", sorted(p.relative_to(payload_dir).as_posix() for p in payload_dir.rglob("*") if p.is_file()))In [ ]:
def run_worker(index: int) -> str:
name = f"codex-gh-{DEMO_RUN_ID}-a{index}"
slice_text = WORKER_SLICES[(index - 1) % len(WORKER_SLICES)]
sandbox = client.create(
name=name,
image="base",
providers=[CODEX_PROVIDER, GITHUB_PROVIDER],
policy=policy,
)
client.wait_ready(name)
client.upload(sandbox.id, payload_dir, "/sandbox")
result = client.exec(
sandbox.id,
[
"bash",
"/sandbox/payload/demo-runner.sh",
"worker",
github_owner,
github_repo,
DEMO_BRANCH,
DEMO_RUN_ID,
str(index),
str(DEMO_AGENT_COUNT),
DEMO_TOPIC,
slice_text,
],
)
if result.exit_code != 0:
raise RuntimeError(
f"worker {index} failed (exit {result.exit_code}); stderr tail:\n{result.stderr[-1000:]}"
)
return name
worker_names: list[str] = []
with ThreadPoolExecutor(max_workers=DEMO_AGENT_COUNT) as pool:
futures = {pool.submit(run_worker, i): i for i in range(1, DEMO_AGENT_COUNT + 1)}
for fut in as_completed(futures):
idx = futures[fut]
try:
worker_names.append(fut.result())
print(f" worker {idx} complete")
except Exception as exc:
print(f" worker {idx} FAILED: {exc}")
if len(worker_names) != DEMO_AGENT_COUNT:
raise SystemExit(f"only {len(worker_names)}/{DEMO_AGENT_COUNT} workers completed")In [ ]:
synth_name = f"codex-gh-{DEMO_RUN_ID}-summary"
synth = client.create(
name=synth_name,
image="base",
providers=[CODEX_PROVIDER, GITHUB_PROVIDER],
policy=policy,
)
client.wait_ready(synth_name)
client.upload(synth.id, payload_dir, "/sandbox")
result = client.exec(
synth.id,
[
"bash",
"/sandbox/payload/demo-runner.sh",
"synthesis",
github_owner,
github_repo,
DEMO_BRANCH,
DEMO_RUN_ID,
"0",
str(DEMO_AGENT_COUNT),
DEMO_TOPIC,
],
)
if result.exit_code != 0:
raise SandboxError(f"synthesis failed (exit {result.exit_code}); stderr:\n{result.stderr[-1500:]}")
print("synthesis complete")In [ ]:
def safe_sandbox_delete(name: str) -> None:
try:
client.delete(name)
except grpc.RpcError as exc:
if exc.code() != grpc.StatusCode.NOT_FOUND:
print(f" delete {name} failed: {exc.code()}")
for i in range(1, DEMO_AGENT_COUNT + 1):
safe_sandbox_delete(f"codex-gh-{DEMO_RUN_ID}-a{i}")
safe_sandbox_delete(synth_name)
for prov in (CODEX_PROVIDER, GITHUB_PROVIDER):
safe_provider_delete(prov)
shutil.rmtree(work_dir, ignore_errors=True)
client.close()
base_url = f"https://github.com/{github_owner}/{github_repo}/tree/{DEMO_BRANCH}/runs/{DEMO_RUN_ID}"
print(f"\nDone. Shared agent notepad: {base_url}")
print(f" summary: {base_url}/summary.md")
for i in range(1, DEMO_AGENT_COUNT + 1):
print(f" agent {i}: {base_url}/notes/agent-{i}.md")