mirror of
https://github.com/nightly-labs/openbot.git
synced 2026-11-22 09:42:08 +08:00
* Start a workflow loop at its leftmost entry on the canvas A workflow loop that one more connection also enters (for example the task to a reviewer inside a search loop, #12199) got its loop edge from a depth-first walk in the order in which the edges were added. When the extra edge came first, the walk entered the loop at the reviewer and marked search > reviewer as the loop edge, not enough > fork. Then "Run from this step" on the reviewer counted the search as a later step and ran it again, and a full run started the reviewer before the search. workflowLoopEdges now finds each group of steps that reach each other. When edges from outside enter a group at more than one step, the loop starts at the leftmost entry on the canvas; a tie, a single entry, or no positions keep the step that the old walk reached first. The group without its loop edges gets the same treatment for inner loops. On 30,000 random graphs without positions the result is the same as the old walk, and no random graph keeps a loop after its loop edges are removed. The host run, the desktop and web canvas, and mobile all call this shared function, so they keep the same loop edges. No wire protocol, IPC or schema change: loop edges are computed from the graph, never saved or sent. A joined server must be on the new version for its runs to get the fix. This ports the rule of #12240 to current main, without the doc comments that AGENTS.md now forbids. Fixes #12199 * Start a single-entry workflow loop at its entry and break an x tie by y A loop with one entry from outside now starts at that entry, not at the first loop node that the walk reaches. When two entries have the same x, the topmost entry is the start. The architecture document gives the rule.
271 lines
12 KiB
TypeScript
271 lines
12 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
import { type WorkflowGraph, type WorkflowStep, workflowLoopEdges } from "@openbot/contracts/ipc";
|
|
import { Effect } from "effect";
|
|
import { runCauseEffect } from "../src/backend/effect-boundary";
|
|
import { OpenBotDatabase } from "../src/backend/openbot-database";
|
|
import { WorkflowGitHubStore } from "../src/backend/workflows/workflow-github-store";
|
|
import { WorkflowRunStore } from "../src/backend/workflows/workflow-run-store";
|
|
import { WorkflowStore } from "../src/backend/workflows/workflow-store";
|
|
import { workflowScheduleDueAt } from "../src/backend/workflows/workflow-triggers";
|
|
import { createWorkflows, type WorkflowsHandle } from "../src/backend/workflows/workflows";
|
|
|
|
const OUT = resolve(import.meta.dirname, "../.openbot-build/workflow-loop-e2e");
|
|
const WAIT_MS = 5_000;
|
|
|
|
interface Report {
|
|
checks: string[];
|
|
outputs: string[];
|
|
passed: boolean;
|
|
failure?: string;
|
|
}
|
|
|
|
const report: Report = { checks: [], outputs: [], passed: false };
|
|
const check = (name: string, ok: boolean, detail?: unknown) => {
|
|
assert.ok(ok, `${name}${detail === undefined ? "" : `: ${JSON.stringify(detail)}`}`);
|
|
report.checks.push(name);
|
|
};
|
|
|
|
const GRAPH: WorkflowGraph = {
|
|
nodes: [
|
|
{ id: "prompt", kind: "prompt", at: { x: 0, y: 0 }, fields: { text: "Draft" } },
|
|
{ id: "review", kind: "approval", at: { x: 300, y: 0 }, fields: { question: "Review?" } },
|
|
{ id: "check", kind: "approval", at: { x: 600, y: 0 }, fields: { question: "Again?" } },
|
|
{ id: "output", kind: "output", at: { x: 900, y: 0 }, fields: { target: "notify" } },
|
|
],
|
|
edges: [
|
|
{ id: "prompt>review", from: "prompt", to: "review", type: "text" },
|
|
{ id: "review>check", from: "review", to: "check", type: "text" },
|
|
{ id: "check>output", from: "check", to: "output", type: "text" },
|
|
{ id: "check>review", from: "check", to: "review", type: "text" },
|
|
],
|
|
strokes: [],
|
|
};
|
|
|
|
const RESEARCH_CHAIN: WorkflowGraph["edges"] = [
|
|
{ id: "task>fork", from: "task", to: "fork", type: "text" },
|
|
{ id: "fork>search", from: "fork", to: "search", type: "text" },
|
|
{ id: "search>reviewer", from: "search", to: "reviewer", type: "text" },
|
|
{ id: "reviewer>enough", from: "reviewer", to: "enough", type: "text" },
|
|
{ id: "enough>fork", from: "enough", to: "fork", type: "text" },
|
|
];
|
|
const TASK_TO_REVIEWER: WorkflowGraph["edges"][number] = {
|
|
id: "task>reviewer",
|
|
from: "task",
|
|
to: "reviewer",
|
|
type: "text",
|
|
};
|
|
const RESEARCH: WorkflowGraph = {
|
|
nodes: [
|
|
{ id: "task", kind: "prompt", at: { x: 0, y: 0 }, fields: { text: "Find sources" } },
|
|
{ id: "fork", kind: "approval", at: { x: 300, y: 0 }, fields: { question: "Search?" } },
|
|
{ id: "search", kind: "approval", at: { x: 600, y: 0 }, fields: { question: "Sources?" } },
|
|
{ id: "reviewer", kind: "approval", at: { x: 900, y: 0 }, fields: { question: "Good?" } },
|
|
{ id: "enough", kind: "approval", at: { x: 1200, y: 0 }, fields: { question: "Enough?" } },
|
|
],
|
|
edges: [TASK_TO_REVIEWER, ...RESEARCH_CHAIN],
|
|
strokes: [],
|
|
};
|
|
|
|
const directory = await mkdtemp(join(tmpdir(), "openbot-workflow-loop-"));
|
|
const database = new OpenBotDatabase(directory);
|
|
let service: WorkflowsHandle | null = null;
|
|
|
|
async function open() {
|
|
await Effect.runPromise(database.initialize());
|
|
const workflows = await Effect.runPromise(
|
|
createWorkflows({
|
|
store: new WorkflowStore({ database, nextDueAt: workflowScheduleDueAt(() => "UTC") }),
|
|
runs: new WorkflowRunStore({ database }),
|
|
github: new WorkflowGitHubStore({ database }),
|
|
githubApp: () => null,
|
|
delivery: () => null,
|
|
turnAnswer: () => null,
|
|
sendHandoff: () => Effect.fail({ cause: new Error("No agent step runs in this test.") }),
|
|
stopDelivery: () => Effect.void,
|
|
generateText: () => Effect.fail({ message: "No model step runs in this test." }),
|
|
notify: (notice) => {
|
|
if (notice.kind === "output") report.outputs.push(notice.text);
|
|
},
|
|
connectionOn: () => false,
|
|
post: () => Effect.die(new Error("No post runs in this test.")),
|
|
changed: () => {},
|
|
rearm: () => {},
|
|
timezone: () => "UTC",
|
|
}),
|
|
);
|
|
service = workflows;
|
|
return workflows;
|
|
}
|
|
|
|
async function close() {
|
|
if (service) await Effect.runPromise(service.close()).catch(() => {});
|
|
service = null;
|
|
database.close();
|
|
}
|
|
|
|
async function waiting(workflows: WorkflowsHandle, runId: string): Promise<WorkflowStep> {
|
|
const deadline = Date.now() + WAIT_MS;
|
|
for (;;) {
|
|
const step = (await runCauseEffect(workflows.run(runId))).steps.find((each) => each.status === "approval");
|
|
if (step) return step;
|
|
if (Date.now() > deadline) throw new Error("No step waits for a decision.");
|
|
await new Promise((done) => setImmediate(done));
|
|
}
|
|
}
|
|
const inputOf = (step: WorkflowStep) => new WorkflowRunStore({ database }).step(step.id)?.input;
|
|
const runStatus = async (workflows: WorkflowsHandle, runId: string) =>
|
|
(await runCauseEffect(workflows.run(runId))).run.status;
|
|
|
|
try {
|
|
const workflows = await open();
|
|
check(
|
|
"the graph has one loop: the connection back",
|
|
[...workflowLoopEdges(GRAPH.nodes, GRAPH.edges)].join() === "check>review",
|
|
);
|
|
|
|
const created = await runCauseEffect(workflows.create({ title: "Review loop" }));
|
|
const workflowId = created.summary.id;
|
|
const saved = await runCauseEffect(workflows.saveGraph({ workflowId, baseRevision: created.revision, graph: GRAPH }));
|
|
check("a graph with a loop saves", saved.ok, saved);
|
|
const detail = await runCauseEffect(workflows.get(workflowId));
|
|
check(
|
|
"the saved graph keeps the loop",
|
|
detail.graph.edges.some((edge) => edge.id === "check>review"),
|
|
);
|
|
|
|
const run = (await runCauseEffect(workflows.startRun({ workflowId, requestId: "loop-1" }))).run;
|
|
let step = await waiting(workflows, run.id);
|
|
check(
|
|
"pass 1 waits at the first approval",
|
|
step.nodeId === "review" && step.attempt === 1 && inputOf(step) === "Draft",
|
|
step,
|
|
);
|
|
check("the run waits for a decision", (await runStatus(workflows, run.id)) === "approval");
|
|
|
|
await runCauseEffect(workflows.decide({ stepId: step.id, approved: true }));
|
|
step = await waiting(workflows, run.id);
|
|
check("pass 1 goes on to the second approval", step.nodeId === "check" && step.attempt === 1, step);
|
|
|
|
await runCauseEffect(workflows.decide({ stepId: step.id, approved: true }));
|
|
step = await waiting(workflows, run.id);
|
|
check(
|
|
"the loop starts pass 2 at the first approval, with the loop's answer as one more input",
|
|
step.nodeId === "review" && step.attempt === 2 && inputOf(step) === "Input 1:\n\nDraft\n\nInput 2:\n\nDraft",
|
|
step,
|
|
);
|
|
check("the output runs after pass 1", report.outputs.length === 1, report.outputs);
|
|
|
|
await runCauseEffect(workflows.decide({ stepId: step.id, approved: true }));
|
|
step = await waiting(workflows, run.id);
|
|
check("pass 2 goes on to the second approval", step.nodeId === "check" && step.attempt === 2, step);
|
|
check("the output does not run before the pass ends", report.outputs.length === 1, report.outputs);
|
|
|
|
await runCauseEffect(workflows.decide({ stepId: step.id, approved: true }));
|
|
step = await waiting(workflows, run.id);
|
|
check("pass 3 starts", step.nodeId === "review" && step.attempt === 3, step);
|
|
check("the output runs after pass 2", report.outputs.length === 2, report.outputs);
|
|
|
|
await runCauseEffect(workflows.decide({ stepId: step.id, approved: false }));
|
|
check("a rejection ends the loop and stops the run", (await runStatus(workflows, run.id)) === "stopped");
|
|
const steps = (await runCauseEffect(workflows.run(run.id))).steps;
|
|
check(
|
|
"each pass is the next attempt of its node",
|
|
steps.filter((each) => each.nodeId === "review").length === 3 &&
|
|
steps.filter((each) => each.nodeId === "check").length === 2 &&
|
|
steps.filter((each) => each.nodeId === "prompt").length === 1,
|
|
steps.map((each) => `${each.nodeId}:${each.attempt}:${each.status}`),
|
|
);
|
|
|
|
const from = (await runCauseEffect(workflows.startRun({ workflowId, requestId: "loop-2", fromNodeId: "check" }))).run;
|
|
step = await waiting(workflows, from.id);
|
|
const reused = (await runCauseEffect(workflows.run(from.id))).steps.find((each) => each.nodeId === "review");
|
|
check(
|
|
"a run from a node in the loop starts at that node with the earlier answer before it",
|
|
step.nodeId === "check" && step.attempt === 1 && reused?.status === "done" && reused.attempt === 1,
|
|
{ step, reused },
|
|
);
|
|
const checked = step;
|
|
await runCauseEffect(workflows.decide({ stepId: checked.id, approved: true }));
|
|
step = await waiting(workflows, from.id);
|
|
const answer = (await runCauseEffect(workflows.run(from.id))).steps.find((each) => each.id === checked.id)?.output;
|
|
check(
|
|
"its loop goes back to the node before it, with only the loop's answer",
|
|
step.nodeId === "review" && step.attempt === 2 && answer !== undefined && inputOf(step) === answer?.trim(),
|
|
{ step, answer, input: inputOf(step) },
|
|
);
|
|
await runCauseEffect(workflows.stopRun(from.id));
|
|
check("a stop ends the loop", (await runStatus(workflows, from.id)) === "stopped");
|
|
|
|
check(
|
|
"a loop that one more connection also enters goes back to its leftmost entry, in any edge order",
|
|
[[...RESEARCH_CHAIN, TASK_TO_REVIEWER], RESEARCH.edges].every(
|
|
(edges) => [...workflowLoopEdges(RESEARCH.nodes, edges)].join() === "enough>fork",
|
|
),
|
|
);
|
|
const column = RESEARCH.nodes.map((node, index) => ({ ...node, at: { x: 0, y: index * 200 } }));
|
|
check(
|
|
"in one column, the loop goes back to its top entry, in any edge order",
|
|
[[...RESEARCH_CHAIN, TASK_TO_REVIEWER], RESEARCH.edges].every(
|
|
(edges) => [...workflowLoopEdges(column, edges)].join() === "enough>fork",
|
|
),
|
|
);
|
|
const research = await runCauseEffect(workflows.create({ title: "Research loop" }));
|
|
const researchId = research.summary.id;
|
|
const researchSaved = await runCauseEffect(
|
|
workflows.saveGraph({ workflowId: researchId, baseRevision: research.revision, graph: RESEARCH }),
|
|
);
|
|
check("the research loop saves", researchSaved.ok, researchSaved);
|
|
const full = (await runCauseEffect(workflows.startRun({ workflowId: researchId, requestId: "research-1" }))).run;
|
|
const order: string[] = [];
|
|
for (let index = 0; index < 4; index += 1) {
|
|
step = await waiting(workflows, full.id);
|
|
order.push(step.nodeId);
|
|
await runCauseEffect(workflows.decide({ stepId: step.id, approved: true }));
|
|
}
|
|
step = await waiting(workflows, full.id);
|
|
check(
|
|
"a full run goes through the fork and the search before the reviewer, then loops back to the fork",
|
|
order.join() === "fork,search,reviewer,enough" && step.nodeId === "fork" && step.attempt === 2,
|
|
{ order, step },
|
|
);
|
|
await runCauseEffect(workflows.stopRun(full.id));
|
|
const searched = (await runCauseEffect(workflows.run(full.id))).steps.find((each) => each.nodeId === "search");
|
|
|
|
const resumed = (
|
|
await runCauseEffect(
|
|
workflows.startRun({ workflowId: researchId, requestId: "research-2", fromNodeId: "reviewer" }),
|
|
)
|
|
).run;
|
|
step = await waiting(workflows, resumed.id);
|
|
const resumedSteps = (await runCauseEffect(workflows.run(resumed.id))).steps;
|
|
const reusedSearch = resumedSteps.find((each) => each.nodeId === "search");
|
|
check(
|
|
"a run from the reviewer inside the loop starts at the reviewer and reuses the last search answer",
|
|
step.nodeId === "reviewer" &&
|
|
reusedSearch?.status === "done" &&
|
|
reusedSearch.output === searched?.output &&
|
|
resumedSteps
|
|
.filter((each) => each.nodeId === "search" || each.nodeId === "fork")
|
|
.every((each) => each.status === "done") &&
|
|
(inputOf(step) ?? "").includes(searched?.output?.trim() ?? "missing"),
|
|
{ step, input: inputOf(step), steps: resumedSteps.map((each) => `${each.nodeId}:${each.attempt}:${each.status}`) },
|
|
);
|
|
await runCauseEffect(workflows.stopRun(resumed.id));
|
|
|
|
report.passed = true;
|
|
} catch (error) {
|
|
report.failure = error instanceof Error ? (error.stack ?? error.message) : String(error);
|
|
} finally {
|
|
await close();
|
|
await rm(directory, { recursive: true, force: true });
|
|
mkdirSync(OUT, { recursive: true });
|
|
writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
|
process.stdout.write(`${report.passed ? "passed" : "FAILED"}: ${report.checks.length} checks; ${OUT}/report.json\n`);
|
|
if (report.failure) process.stderr.write(`${report.failure}\n`);
|
|
process.exitCode = report.passed ? 0 : 1;
|
|
}
|