From eecad6868a991292d5d2af620ade4bfc18e048de Mon Sep 17 00:00:00 2001 From: Rehaan Ahmad Date: Sun, 7 Jun 2026 18:11:21 -0700 Subject: [PATCH] changes --- SKILL.md | 23 ++++++++++++++++++----- src/commands/exp.rs | 40 +++++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/SKILL.md b/SKILL.md index 9925bd7b..e18f3a43 100644 --- a/SKILL.md +++ b/SKILL.md @@ -129,8 +129,12 @@ run command: ``` 6. **Keep the budget saturated.** With a cap of N concurrent runs, never leave a slot idle while there's useful parallel work. Block on the next finish with - `orx exp wait --project ` (returns on the first change to any run), - then immediately launch the next queued child to refill the freed slot. + `orx exp wait --project ` — it returns the moment any run + *completes* (reaches a terminal state), which is exactly the "a slot freed" + signal. Then immediately launch the next queued child to refill the freed + slot. **Don't hand-roll a poll loop over `orx runs`** — `exp wait` already + does the polling, knows the real terminal states, and wakes only on + completions. 7. **Analyze, then iterate.** When a run finishes, **actually read its results** before deciding — `orx artifact EVAL.md`, `orx chart wandb …`, `orx query …`. Don't infer from status alone. Promising children become parents @@ -209,15 +213,24 @@ to act as soon as a run finishes. Two modes, picked by argument: ```sh orx exp wait # level trigger: poll this experiment's latest run # until it reaches a terminal state (done/failed/cancelled) -orx exp wait --project # edge trigger: return on the FIRST change to ANY run - # in the project (new run, or a status transition) +orx exp wait --project # edge trigger: return when the FIRST run in the + # project COMPLETES (transitions into done/failed/cancelled) orx exp wait --interval 10 --timeout 3600 # tune polling ``` - Pass **exactly one** of `` or `--project` (not both, not neither). +- `--project` is the **budget-loop** primitive: it wakes only on a **completion** + (a run reaching `done`/`failed`/`cancelled`) — i.e. a freed slot. Run *starts*, + new queued runs, and `queued→running` transitions are intentionally ignored, so + it won't wake you on non-events. **Prefer this over a custom poll loop over + `orx runs`.** +- Because it only fires on a *new* completion, call `--project` **while runs are + in flight** (right after launching). If every run is already terminal when you + call it, there's nothing left to complete and it blocks until `--timeout`. - `--interval` is seconds between polls (default `5`); `--timeout` gives up after N seconds (default `1800`) and exits **non-zero** so callers can branch on it. -- Progress lines go to **stderr**; the final state line(s) go to **stdout**. +- Progress lines go to **stderr**; the final completion line(s) go to **stdout**, + each as ` -> ` (or ` (new)`). ## Experiment description / notes — `orx exp desc` diff --git a/src/commands/exp.rs b/src/commands/exp.rs index c9bea362..dd356307 100644 --- a/src/commands/exp.rs +++ b/src/commands/exp.rs @@ -46,7 +46,7 @@ fn is_terminal(status: &str) -> bool { /// /// Two modes, picked by argument: /// - `` — level trigger: poll the experiment's latest run until it reaches a terminal state (done/failed/cancelled). -/// - `--project` — edge trigger: snapshot every run in the project and return on the first change of any kind (new run or status change). +/// - `--project` — edge trigger: snapshot every run in the project and return when the first run *completes* — i.e. transitions into a terminal state (done/failed/cancelled). This is the "a slot just freed" signal a budget-saturation loop wants; run starts and queued→running transitions are intentionally ignored. /// /// Polls every `--interval` seconds (default 5), gives up after `--timeout` /// seconds (default 1800) with a non-zero exit so callers can branch on it. @@ -103,8 +103,16 @@ async fn wait_experiment( } } -/// Edge trigger: return as soon as any run in the project changes vs. the -/// snapshot taken on entry (new run id, or a status transition). +/// Edge trigger: return as soon as any run in the project *completes* — i.e. +/// transitions into a terminal state (done/failed/cancelled) vs. the snapshot +/// taken on entry. Run starts, new queued runs, and queued→running transitions +/// are ignored: the useful project-wide signal is "a slot just freed", so a +/// budget-saturation loop can analyze the finished run and launch the next one. +/// +/// Note: runs already terminal at entry don't count (they're in the snapshot as +/// terminal). If *every* run is already terminal when this is called, there's +/// nothing left to complete and it blocks until `--timeout`. Call it right after +/// launching, while runs are in flight. async fn wait_project( creds: &crate::config::Credentials, project_id: &str, @@ -117,27 +125,33 @@ async fn wait_project( .into_iter() .map(|r| (r.id, r.status)) .collect(); + let in_flight = snapshot.values().filter(|s| !is_terminal(s)).count(); eprintln!( - "Watching {} run(s) in project — returning on the first change…", - snapshot.len() + "Watching {} run(s) in project ({} in flight) — returning on the first completion…", + snapshot.len(), + in_flight ); loop { sleep_until_or_timeout(interval, deadline).await?; let current = list_runs(creds, project_id).await?.runs; - let mut changes: Vec = Vec::new(); + let mut completed: Vec = Vec::new(); for r in ¤t { + if !is_terminal(&r.status) { + continue; + } + // Fire only on a *new* terminal: a run that was non-terminal before, + // or a brand-new run that's already terminal. Skip runs that were + // already terminal in the entry snapshot. match snapshot.get(&r.id) { - None => changes.push(format!("{} {} (new)", r.id, r.status)), - Some(prev) if prev != &r.status => { - changes.push(format!("{} {} -> {}", r.id, prev, r.status)) - } - Some(_) => {} + Some(prev) if is_terminal(prev) => {} + Some(prev) => completed.push(format!("{} {} -> {}", r.id, prev, r.status)), + None => completed.push(format!("{} {} (new)", r.id, r.status)), } } - if !changes.is_empty() { - for c in &changes { + if !completed.is_empty() { + for c in &completed { println!("{}", c); } return Ok(());