import assert from "node:assert/strict"; import { mkdirSync, writeFileSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import type { WorkflowGitHubHistory, WorkflowGraph, WorkflowNode } from "@openbot/contracts/ipc"; import { sourceText } from "@openbot/i18n/source"; import { Effect } from "effect"; import { OpenBotDatabase } from "../src/backend/openbot-database"; import { EXTERNAL_EVENT_END } from "../src/backend/webhook-trigger"; import type { WorkflowGitHubApi, WorkflowGitHubChange } from "../src/backend/workflows/workflow-github-api"; import { WorkflowGitHubStore } from "../src/backend/workflows/workflow-github-store"; import { WorkflowRunStore } from "../src/backend/workflows/workflow-run-store"; import { WorkflowStore } from "../src/backend/workflows/workflow-store"; import { workflowScheduleDueAt } from "../src/backend/workflows/workflow-triggers"; import { createWorkflows, type WorkflowRunEnd, type WorkflowsDependencies, type WorkflowsHandle, } from "../src/backend/workflows/workflows"; import { receiveGitHubDelivery } from "../src/main/github-relay"; const OUT = resolve(import.meta.dirname, "../.openbot-build/github-workflow-e2e"); const BOT = "openbotgit"; interface Report { checks: string[]; watched?: string[]; history?: WorkflowGitHubHistory; changes?: WorkflowGitHubChange[]; passed: boolean; failure?: string; } const report: Report = { checks: [], passed: false }; const check = (name: string, ok: boolean, detail?: unknown) => { assert.ok(ok, `${name}${detail === undefined ? "" : `: ${JSON.stringify(detail)}`}`); report.checks.push(name); }; const graphOf = (repository: string, step: "approval" | "notify", filters = "", event = "comment"): WorkflowGraph => ({ nodes: [ { id: "trigger", kind: "trigger", at: { x: 0, y: 0 }, fields: { trigger: "github", repository, event, filters }, }, step === "approval" ? { id: "step", kind: "approval", at: { x: 300, y: 0 }, fields: { question: "Go on?" } } : { id: "step", kind: "output", at: { x: 300, y: 0 }, fields: { target: "notify" } }, ], edges: [{ id: "edge", from: "trigger", to: "step", type: "text" }], strokes: [], }); const lineOf = (repository: string, steps: WorkflowNode[], filters = ""): WorkflowGraph => { const nodes: WorkflowNode[] = [ { id: "trigger", kind: "trigger", at: { x: 0, y: 0 }, fields: { trigger: "github", repository, event: "comment", filters }, }, ...steps, ]; return { nodes, edges: nodes.slice(1).map((node, index) => ({ id: `edge-${index}`, from: nodes[index]?.id ?? "", to: node.id, type: "text" as const, })), strokes: [], }; }; const SAY = "Thanks, I look at it."; const saying = (graph: WorkflowGraph): WorkflowGraph => ({ ...graph, nodes: [...graph.nodes, { id: "say", kind: "prompt", at: { x: 300, y: 200 }, fields: { text: SAY } }], edges: [...graph.edges, { id: "edge-say", from: "say", to: "out", type: "text" }], }); const githubOutput = (fields: Record): WorkflowNode => ({ id: "out", kind: "output", at: { x: 600, y: 0 }, fields: { target: "github", repository: "trigger", ...fields }, }); let commentId = 100; const comment = (repository: string, number: number, login: string, type: "User" | "Bot" = "User", text?: string) => { commentId += 1; return { id: commentId, body: JSON.stringify({ action: "created", repository: { full_name: repository }, sender: { login, type }, issue: { number, title: `Issue ${number}`, body: "Steps to see it.", labels: [], user: { login: "ana", type: "User" }, }, comment: { id: commentId, body: text ?? `Comment ${commentId}`, user: { login, type }, author_association: "MEMBER", }, }), }; }; const directory = await mkdtemp(join(tmpdir(), "openbot-github-workflow-")); const database = new OpenBotDatabase(directory); let service: WorkflowsHandle | null = null; try { await Effect.runPromise(database.initialize()); const watched: string[][] = []; const changes: WorkflowGitHubChange[] = []; report.changes = changes; const changeWaiters: Array<() => void> = []; const changeMade = () => new Promise((resolve) => changeWaiters.push(resolve)); let createdId = 9_000; const hung = new Set(); const githubApi: WorkflowGitHubApi = { attachFiles: (change) => Effect.succeed({ ...change, files: null }), change: (change) => Effect.suspend(() => { changes.push(change); for (const resolve of changeWaiters.splice(0)) resolve(); if (change.repository.startsWith("acme/slow") && !hung.has(change.repository)) { hung.add(change.repository); return Effect.never; } createdId += 1; return Effect.succeed({ created: change.action === "comment" ? { kind: "comment" as const, id: createdId } : null, url: null, }); }), options: () => Effect.succeed({ labels: [ { name: "Bug", color: "d73a4a" }, { name: "docs", color: "0075ca" }, ], people: ["ana", "ben"], }), recentEvents: (repository) => Effect.succeed([ { event: "issue_comment", body: JSON.parse(comment(repository, 11, "dependabot[bot]", "Bot").body) }, { event: "issue_comment", body: JSON.parse(comment(repository, 12, "ana").body) }, ]), }; const runEnds = new Map void>(); const ended = new Map(); const runEnd = (runId: string) => new Promise((resolve) => { const done = ended.get(runId); if (done) resolve(done); else runEnds.set(runId, resolve); }); const dependencies: WorkflowsDependencies = { store: new WorkflowStore({ database, nextDueAt: workflowScheduleDueAt(() => "UTC") }), runs: new WorkflowRunStore({ database }), github: new WorkflowGitHubStore({ database }), githubApp: () => BOT, watchGitHub: (repositories) => watched.push(repositories), githubApi, delivery: () => null, turnAnswer: () => null, sendHandoff: () => Effect.fail({ cause: new Error("No agent step runs in this test.") }), stopDelivery: () => Effect.void, generateText: () => Effect.succeed('{"labels": ["bug", "wontfix"]}'), notify: () => {}, connectionOn: () => false, post: () => Effect.die(new Error("No output posts in this test.")), changed: () => {}, rearm: () => {}, timezone: () => "UTC", runEnded: (end) => { ended.set(end.runId, end); runEnds.get(end.runId)?.(end); }, }; let workflows = await Effect.runPromise(createWorkflows(dependencies)); service = workflows; const run = (effect: Effect.Effect) => Effect.runPromise(effect); let deliveryNumber = 0; const send = (body: string, deliveryId = `delivery-${++deliveryNumber}`, event = "issue_comment") => run( receiveGitHubDelivery(workflows, { event, deliveryId, body: new TextEncoder().encode(body), }), ).then((answer) => ({ ...answer, deliveryId })); const history = (workflowId: string) => run(workflows.githubHistory(workflowId)); const delivery = async (workflowId: string, deliveryId: string) => (await history(workflowId)).deliveries.find((candidate) => candidate.id === deliveryId); const queue = await run( workflows.create({ title: "Answer comments", graph: graphOf("acme/app", "approval", "notBot") }), ); const chain = await run(workflows.create({ title: "Chain", graph: graphOf("acme/chain", "notify") })); report.watched = watched.at(-1) ?? []; check( "the saved triggers set the watched repositories", [...report.watched].sort().join() === "acme/app,acme/chain", watched, ); const first = await send(comment("acme/app", 1, "ana").body); check("a comment is saved", first.status === 202, first); const firstRow = await delivery(queue.summary.id, first.deliveryId); check("a comment starts a run", firstRow?.outcome === "started" && firstRow.runId !== null, firstRow); const second = await send(comment("acme/app", 1, "ben").body); check( "a second comment on the issue waits", (await delivery(queue.summary.id, second.deliveryId))?.outcome === "waiting", ); const other = await send(comment("acme/app", 2, "ana").body); const otherRow = await delivery(queue.summary.id, other.deliveryId); check("a comment on another issue runs at the same time", otherRow?.outcome === "started", otherRow); const again = await send(comment("acme/app", 2, "ana").body, other.deliveryId); const otherRows = (await history(queue.summary.id)).deliveries.filter((row) => row.id === other.deliveryId); check("the same delivery again records nothing", again.status === 202 && otherRows.length === 1, otherRows); const firstRun = await run(workflows.run(firstRow?.runId ?? "")); const approval = firstRun.steps.find((step) => step.status === "approval"); check("the first run waits for its approval", approval !== undefined, firstRun.steps); await run(workflows.decide({ stepId: approval?.id ?? "", approved: true })); await run(workflows.sweep()); const started = await delivery(queue.summary.id, second.deliveryId); check("the waiting comment starts when the run of its issue ends", started?.outcome === "started", started); const runs = (await history(queue.summary.id)).runs; check( "the history names the item of each run", runs.some((entry) => entry.runId === started?.runId && entry.item.number === 1), runs, ); const dependabot = await send(comment("acme/app", 3, "dependabot[bot]", "Bot").body); const dependabotRow = await delivery(queue.summary.id, dependabot.deliveryId); check('"not a bot" refuses another bot', dependabotRow?.reason === "filter", dependabotRow); const ownBot = await send(comment("acme/app", 4, `${BOT}[bot]`, "Bot").body); const ownBotRow = await delivery(queue.summary.id, ownBot.deliveryId); check('"not a bot" refuses OpenBot\'s own bot', ownBotRow?.reason === "filter", ownBotRow); const chainRows = []; for (let index = 0; index < 4; index += 1) { const sent = await send(comment("acme/chain", 9, `${BOT}[bot]`, "Bot").body); chainRows.push(await delivery(chain.summary.id, sent.deliveryId)); } check( "OpenBot's bot starts runs until the chain is too long", chainRows.slice(0, 3).every((row) => row?.outcome === "started") && chainRows[3]?.reason === "chain", chainRows, ); const labeled = await run( workflows.create({ title: "Labeled", graph: graphOf("acme/labels", "notify", "notBot", "issue.labeled") }), ); const botIssue = { action: "labeled", repository: { full_name: "acme/labels" }, sender: { login: "ana", type: "User" }, label: { name: "Bug" }, issue: { number: 3, title: "Bump a package", labels: [], user: { login: "dependabot[bot]", type: "Bot" } }, }; const labelSent = await send(JSON.stringify(botIssue), undefined, "issues"); const labelRow = await delivery(labeled.summary.id, labelSent.deliveryId); check("\"not a bot\" refuses a person's label on a bot's issue", labelRow?.reason === "filter", labelRow); const personIssue = { ...botIssue, sender: { login: "github-actions[bot]", type: "Bot" }, issue: { ...botIssue.issue, number: 4, user: { login: "ana", type: "User" } }, }; const botLabelSent = await send(JSON.stringify(personIssue), undefined, "issues"); const botLabelRow = await delivery(labeled.summary.id, botLabelSent.deliveryId); check("\"not a bot\" refuses a bot's label on a person's issue", botLabelRow?.reason === "filter", botLabelRow); const personLabelSent = await send( JSON.stringify({ ...personIssue, sender: { login: "ben", type: "User" } }), undefined, "issues", ); const personLabelRow = await delivery(labeled.summary.id, personLabelSent.deliveryId); check( "\"not a bot\" lets a person's label on a person's issue start a run", personLabelRow?.outcome === "started", personLabelRow, ); const unwatched = await send(comment("acme/other", 1, "ana").body); check("an event of a repository that no trigger watches gets 404", unwatched.status === 404, unwatched); const invalid = await send("{"); check("a body that is not JSON gets 400", invalid.status === 400, invalid); const now = Date.now() + 1_000; await run(workflows.githubOffline(new Date(now), new Date(now + 30_000))); await run(workflows.githubOffline(new Date(now + 60_000), new Date(now + 90_000))); const gaps = (await history(chain.summary.id)).deliveries.filter((row) => row.reason === "offline"); check( "two close times offline are one gap", gaps.length === 1 && gaps[0]?.receivedAt === new Date(now + 90_000).toISOString(), gaps, ); const outputStep = async (runId: string) => (await run(workflows.run(runId))).steps.find((step) => step.nodeId === "out"); const startedRun = async (workflowId: string, body: string) => { const sent = await send(body); const row = await delivery(workflowId, sent.deliveryId); assert.ok(row?.runId, `no run: ${JSON.stringify(row)}`); return row.runId; }; const reply = await run( workflows.create({ title: "Reply", graph: saying(lineOf("acme/out", [githubOutput({ action: "comment" })])) }), ); const replyRun = await startedRun(reply.summary.id, comment("acme/out", 5, "ana").body); const replyEnd = await runEnd(replyRun); const replyStep = await outputStep(replyRun); const posted = changes.filter((change) => change.repository === "acme/out"); check( "a comment output posts one comment on the run's issue, with the text and not the event", replyEnd.status === "done" && posted.length === 1 && posted[0]?.number === 5 && posted[0]?.body === SAY && replyStep?.output === "acme/out#5", { replyEnd, posted, replyStep }, ); const ownComment = comment("acme/out", 5, `${BOT}[bot]`, "Bot"); const ownBody = JSON.parse(ownComment.body); ownBody.comment.id = createdId; const own = await send(JSON.stringify(ownBody)); const ownRow = await delivery(reply.summary.id, own.deliveryId); check("the event of the output's own comment is skipped", ownRow?.reason === "self", ownRow); const withRepositories = (graph: WorkflowGraph, repositories: string): WorkflowGraph => ({ ...graph, nodes: graph.nodes.map((node) => node.id === "trigger" ? { ...node, fields: { ...node.fields, repositories } } : node, ), }); const many = await run( workflows.create({ title: "Many", graph: withRepositories( saying(lineOf("acme/many-a", [githubOutput({ action: "comment" })])), "acme/many-a\nacme/many-b", ), }), ); const manyRun = await startedRun(many.summary.id, comment("acme/many-b", 6, "ana").body); const manyEnd = await runEnd(manyRun); const manyPosted = changes.filter((change) => change.repository.startsWith("acme/many-")); check( "a trigger with two repositories starts on the second one, and its output comments there", manyEnd.status === "done" && manyPosted.length === 1 && manyPosted[0]?.repository === "acme/many-b" && manyPosted[0]?.number === 6, { manyEnd, manyPosted }, ); await run( workflows.create({ title: "Changed by an older client", graph: withRepositories(graphOf("acme/now", "notify"), "acme/before\nacme/before-b"), }), ); const stale = await send(comment("acme/before-b", 1, "ana").body); check("a list whose first line is not `repository` is not watched", stale.status === 404, stale); const back = await run( workflows.create({ title: "Back", graph: withRepositories(graphOf("acme/back-a", "notify"), "acme/back-a\nacme/back-c"), }), ); const resave = async (repository: string) => { const current = await run(workflows.get(back.summary.id)); const graph = { ...current.graph, nodes: current.graph.nodes.map((node) => node.id === "trigger" ? { ...node, fields: { ...node.fields, repository } } : node, ), }; await run(workflows.saveGraph({ workflowId: back.summary.id, baseRevision: current.revision, graph })); }; await resave("acme/back-b"); await resave("acme/back-a"); const backC = await send(comment("acme/back-c", 1, "ana").body); check("a list that an older client left behind does not come back", backC.status === 404, backC); const dropGraph = withRepositories(graphOf("acme/drop-a", "approval"), "acme/drop-a\nacme/drop-b"); const drop = await run(workflows.create({ title: "Drop", graph: dropGraph })); const dropRun = await startedRun(drop.summary.id, comment("acme/drop-b", 1, "ana").body); const dropWaiting = await send(comment("acme/drop-b", 1, "ben").body); const dropDetail = await run(workflows.get(drop.summary.id)); await run( workflows.saveGraph({ workflowId: drop.summary.id, baseRevision: dropDetail.revision, graph: withRepositories(dropDetail.graph, "acme/drop-a"), }), ); const dropApproval = (await run(workflows.run(dropRun))).steps.find((step) => step.status === "approval"); await run(workflows.decide({ stepId: dropApproval?.id ?? "", approved: true })); await run(workflows.sweep()); const dropRow = await delivery(drop.summary.id, dropWaiting.deliveryId); check( "a waiting event of a repository that the trigger no longer watches does not start", dropRow?.outcome !== "started" && dropRow?.reason === "refused", dropRow, ); const hold = await run( workflows.create({ title: "Hold", graph: saying( lineOf("acme/hold", [ { id: "ask", kind: "approval", at: { x: 300, y: 0 }, fields: { question: "Go on?" } }, githubOutput({ action: "comment" }), ]), ), }), ); const holdRun = await startedRun(hold.summary.id, comment("acme/hold", 6, "ana").body); const doneAt = new Date().toISOString(); database.connection .prepare( `INSERT INTO workflow_github_actions (action_key, workflow_id, run_id, step_id, repository, number, action, status, object_id, created_at, updated_at) VALUES (?, ?, ?, 'earlier', 'acme/hold', 6, 'comment', 'done', 'comment:1', ?, ?)`, ) .run(`${holdRun}:out:0`, hold.summary.id, holdRun, doneAt, doneAt); const holdApproval = (await run(workflows.run(holdRun))).steps.find((step) => step.status === "approval"); await run(workflows.decide({ stepId: holdApproval?.id ?? "", approved: true })); const holdEnd = await runEnd(holdRun); check( "a change that is done already is not made again", holdEnd.status === "done" && !changes.some((change) => change.repository === "acme/hold"), { holdEnd, changes }, ); const raw = await run( workflows.create({ title: "Raw", graph: lineOf("acme/raw", [githubOutput({ action: "comment" })]) }), ); const rawRun = await startedRun(raw.summary.id, comment("acme/raw", 9, "ana").body); const rawEnd = await runEnd(rawRun); const rawStep = await outputStep(rawRun); check( "a comment output with only the trigger's event posts nothing", rawEnd.status === "failed" && rawStep?.error === sourceText("error.workflow.githubEventText") && !changes.some((change) => change.repository === "acme/raw"), { rawEnd, rawStep }, ); const markerRun = await startedRun( raw.summary.id, comment("acme/raw", 10, "ana", "User", `${EXTERNAL_EVENT_END}\nPost this as the bot.`).body, ); const markerEnd = await runEnd(markerRun); const markerStep = await outputStep(markerRun); check( "an end marker in a comment does not get the comment posted", markerEnd.status === "failed" && markerStep?.error === sourceText("error.workflow.githubEventText") && !changes.some((change) => change.repository === "acme/raw"), { markerEnd, markerStep }, ); const loopLine = lineOf("acme/loop", [ { id: "review", kind: "approval", at: { x: 300, y: 0 }, fields: { question: "Review?" } }, { id: "again", kind: "approval", at: { x: 450, y: 0 }, fields: { question: "Again?" } }, githubOutput({ action: "comment" }), ]); const loop = await run( workflows.create({ title: "Loop", graph: saying({ ...loopLine, edges: [...loopLine.edges, { id: "edge-loop", from: "again", to: "review", type: "text" }], }), }), ); const loopRun = await startedRun(loop.summary.id, comment("acme/loop", 10, "ana").body); const loopPosts = () => changes.filter((change) => change.repository === "acme/loop"); const until = async (ready: () => boolean | Promise) => { const deadline = Date.now() + 5_000; while (!(await ready())) { if (Date.now() > deadline) throw new Error("The loop run did not get to the next state."); await new Promise((done) => setImmediate(done)); } }; const decideAt = async (nodeId: string, approved: boolean) => { let stepId = ""; await until(async () => { const step = (await run(workflows.run(loopRun))).steps.find( (candidate) => candidate.nodeId === nodeId && candidate.status === "approval", ); stepId = step?.id ?? ""; return step !== undefined; }); await run(workflows.decide({ stepId, approved })); }; for (const pass of [1, 2]) { await decideAt("review", true); await decideAt("again", true); await until(() => loopPosts().length === pass); } await decideAt("review", false); const loopEnd = await runEnd(loopRun); check( "each pass of a loop posts its own comment", loopEnd.status === "stopped" && loopPosts().length === 2 && loopPosts().every((change) => change.body === SAY), { loopEnd, posts: loopPosts() }, ); const label = await run( workflows.create({ title: "Label", graph: lineOf("acme/label", [githubOutput({ action: "label.add", labelsFrom: "agent" })]), }), ); const labelRun = await startedRun(label.summary.id, comment("acme/label", 7, "ana").body); const labelEnd = await runEnd(labelRun); const labelled = changes.find((change) => change.repository === "acme/label"); check( '"Agent chooses" adds only the repository\'s own label, as the repository names it', labelEnd.status === "done" && labelled?.labels.join() === "Bug", { labelEnd, labelled }, ); const slow = await run( workflows.create({ title: "Slow", graph: saying(lineOf("acme/slow", [githubOutput({ action: "comment" })])) }), ); const slowLabel = await run( workflows.create({ title: "Slow label", graph: lineOf("acme/slowlabel", [githubOutput({ action: "label.add", labels: "docs" })]), }), ); let slowCalled = changeMade(); const slowRun = await startedRun(slow.summary.id, comment("acme/slow", 8, "ana").body); await slowCalled; slowCalled = changeMade(); const slowLabelRun = await startedRun(slowLabel.summary.id, comment("acme/slowlabel", 8, "ana").body); await slowCalled; await Effect.runPromise(workflows.close()); service = null; workflows = await Effect.runPromise(createWorkflows(dependencies)); service = workflows; await run(workflows.sweep()); const slowEnd = await runEnd(slowRun); const slowStep = await outputStep(slowRun); check( "a restart does not post a comment that was on its way again", slowEnd.status === "failed" && changes.filter((change) => change.repository === "acme/slow").length === 1 && slowStep?.error === sourceText("error.workflow.githubUncertain"), { slowEnd, slowStep }, ); const slowLabelEnd = await runEnd(slowLabelRun); check( "a restart sends a label that was on its way again", slowLabelEnd.status === "done" && changes.filter((change) => change.repository === "acme/slowlabel").length === 2, { slowLabelEnd, changes }, ); const last = await run( workflows.create({ title: "Last", graph: saying(lineOf("acme/last", [githubOutput({ action: "comment" })], "notBot")), }), ); const lastDetail = await run( workflows.githubLastEvent({ workflowId: last.summary.id, nodeId: "trigger", requestId: "last-1" }), ); const lastEnd = await runEnd(lastDetail.run.id); const lastItem = dependencies.github.runItem(lastDetail.run.id); check( '"Run with last event" starts a run with the newest item that the filters let through', lastEnd.status === "done" && lastItem?.number === 12, { lastEnd, lastItem }, ); report.history = await history(queue.summary.id); report.passed = true; } catch (error) { report.failure = error instanceof Error ? (error.stack ?? error.message) : String(error); process.exitCode = 1; } finally { if (service) await Effect.runPromise(service.close()); database.close(); await rm(directory, { recursive: true, force: true }); mkdirSync(OUT, { recursive: true }); writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`); process.stdout.write( `${report.passed ? "passed" : "failed"}: ${report.checks.length} checks; ${join(OUT, "report.json")}\n`, ); if (report.failure) process.stderr.write(`${report.failure}\n`); }