feat(phase-11/17): agent framework tradeoffs — LangGraph vs CrewAI vs AutoGen vs Agno

New lesson. Decision matrix across state model, branching style, observability, and cost; working pick_framework.py that encodes the tradeoffs as a decision tree and passes seven canonical test cases.
2026-10-02 01:54:39 +08:00 · 2026-04-23 01:02:23 +01:00
parent 507e3be18c
commit fc30e13369
4 changed files with 412 additions and 0 deletions
@@ -0,0 +1,42 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 380" font-family="ui-monospace, SFMono-Regular, Menlo, monospace" font-size="12">
+  <style>
+    .axis { stroke: #111827; stroke-width: 1.5; fill: none; }
+    .grid { stroke: #d1d5db; stroke-width: 1; fill: none; }
+    .label { fill: #111827; }
+    .title { fill: #111827; font-weight: 600; font-size: 13; }
+    .bubble { fill: #ffffff; stroke: #111827; stroke-width: 1.5; }
+  </style>
+
+  <text x="380" y="24" text-anchor="middle" class="title">Framework fit by problem shape</text>
+
+  <!-- axes -->
+  <line class="axis" x1="90" y1="320" x2="720" y2="320"/>
+  <line class="axis" x1="90" y1="60" x2="90" y2="320"/>
+
+  <text x="405" y="355" text-anchor="middle" class="label">Branching style: explicit ---- LLM-selected</text>
+  <text x="40" y="190" text-anchor="middle" class="label" transform="rotate(-90 40 190)">State: ephemeral ---- durable</text>
+
+  <!-- gridlines -->
+  <line class="grid" x1="405" y1="60" x2="405" y2="320"/>
+  <line class="grid" x1="90" y1="190" x2="720" y2="190"/>
+
+  <!-- LangGraph (durable, explicit) -->
+  <circle cx="200" cy="120" r="46" class="bubble"/>
+  <text x="200" y="118" text-anchor="middle" class="title">LangGraph</text>
+  <text x="200" y="136" text-anchor="middle" class="label">StateGraph</text>
+
+  <!-- CrewAI (ephemeral, mixed but closer to LLM-routed) -->
+  <circle cx="540" cy="240" r="46" class="bubble"/>
+  <text x="540" y="238" text-anchor="middle" class="title">CrewAI</text>
+  <text x="540" y="256" text-anchor="middle" class="label">Roles + tasks</text>
+
+  <!-- AutoGen (ephemeral, LLM-selected speakers) -->
+  <circle cx="620" cy="150" r="46" class="bubble"/>
+  <text x="620" y="148" text-anchor="middle" class="title">AutoGen</text>
+  <text x="620" y="166" text-anchor="middle" class="label">Chat / GroupChat</text>
+
+  <!-- Agno (session-durable, explicit-ish tools) -->
+  <circle cx="290" cy="250" r="46" class="bubble"/>
+  <text x="290" y="248" text-anchor="middle" class="title">Agno</text>
+  <text x="290" y="266" text-anchor="middle" class="label">Agent + storage</text>
+</svg>
@@ -0,0 +1,186 @@
+"""Decision-tree recommender for agent frameworks.
+
+Takes a problem descriptor and recommends LangGraph, CrewAI, AutoGen, Agno, or
+"no framework" with a one-sentence justification. The tree encodes the tradeoffs
+described in docs/en.md.
+
+Run:
+    python main.py           # runs the bundled test suite
+    python main.py --ask     # interactive prompt mode
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import dataclass
+from typing import Callable
+
+
+@dataclass(frozen=True)
+class Problem:
+    """Shape descriptor for an agentic task."""
+
+    has_typed_state: bool = False
+    has_roles: bool = False
+    has_dialogue: bool = False
+    has_parallel_fanout: bool = False
+    needs_resume: bool = False
+    needs_human_interrupt: bool = False
+    total_llm_calls: int = 1
+    needs_session_memory: bool = False
+
+
+@dataclass(frozen=True)
+class Recommendation:
+    framework: str
+    reason: str
+
+
+def recommend(p: Problem) -> Recommendation:
+    # Smallest-first: if it's 2 or fewer calls, skip the framework entirely.
+    if p.total_llm_calls <= 2 and not any(
+        (p.has_roles, p.has_dialogue, p.needs_resume, p.has_parallel_fanout, p.needs_human_interrupt)
+    ):
+        return Recommendation(
+            "plain python",
+            "Two or fewer LLM calls with no state, roles, dialogue, fanout, "
+            "or resume needs; a framework is pure overhead.",
+        )
+
+    # Durable state or human interrupts or time-travel -> LangGraph.
+    if p.needs_resume or p.needs_human_interrupt or p.has_parallel_fanout:
+        return Recommendation(
+            "langgraph",
+            "Typed state, checkpointer, interrupts, and Send fanout are only "
+            "first-class in LangGraph.",
+        )
+
+    # Dialogue-shaped problem -> AutoGen.
+    if p.has_dialogue and not p.has_typed_state:
+        return Recommendation(
+            "autogen",
+            "Proposer-critic or teacher-student dialogue is AutoGen's native "
+            "shape; GroupChat selects speakers without hand-wiring.",
+        )
+
+    # Role-driven pipeline -> CrewAI.
+    if p.has_roles and not p.has_typed_state:
+        return Recommendation(
+            "crewai",
+            "Specialist roles with a short sequential or hierarchical plan "
+            "are cheapest to express in CrewAI.",
+        )
+
+    # Single agent + sessions -> Agno.
+    if p.needs_session_memory and not p.has_roles and not p.has_dialogue:
+        return Recommendation(
+            "agno",
+            "Single agent with tools and persistent session memory; Agno's "
+            "storage drivers are built in.",
+        )
+
+    # Typed state but no other signals still points at LangGraph.
+    if p.has_typed_state:
+        return Recommendation(
+            "langgraph",
+            "Typed state is LangGraph's core abstraction; map your TypedDict "
+            "onto a StateGraph.",
+        )
+
+    # Fallback.
+    return Recommendation(
+        "langgraph",
+        "Default for multi-step agents with any uncertainty about future state "
+        "or branching needs.",
+    )
+
+
+# Tests -----------------------------------------------------------------------
+
+
+def _check(label: str, actual: Recommendation, expected_framework: str) -> bool:
+    ok = actual.framework == expected_framework
+    tag = "OK " if ok else "FAIL"
+    print(f"[{tag}] {label:<60}  -> {actual.framework:<14} // {actual.reason}")
+    return ok
+
+
+def run_tests() -> int:
+    cases: list[tuple[str, Problem, str]] = [
+        (
+            "two-call summarizer, no state",
+            Problem(total_llm_calls=2),
+            "plain python",
+        ),
+        (
+            "long-running workflow with human approval",
+            Problem(has_typed_state=True, needs_human_interrupt=True, total_llm_calls=8),
+            "langgraph",
+        ),
+        (
+            "research with parallel fanout to three retrievers",
+            Problem(has_typed_state=True, has_parallel_fanout=True, total_llm_calls=5),
+            "langgraph",
+        ),
+        (
+            "proposer-critic coding loop",
+            Problem(has_dialogue=True, total_llm_calls=10),
+            "autogen",
+        ),
+        (
+            "marketing pipeline with researcher/writer/editor roles",
+            Problem(has_roles=True, total_llm_calls=4),
+            "crewai",
+        ),
+        (
+            "chat assistant with persistent user memory",
+            Problem(needs_session_memory=True, total_llm_calls=6),
+            "agno",
+        ),
+        (
+            "workflow that must resume after crash",
+            Problem(has_typed_state=True, needs_resume=True, total_llm_calls=12),
+            "langgraph",
+        ),
+    ]
+
+    failures = 0
+    for label, problem, expected in cases:
+        if not _check(label, recommend(problem), expected):
+            failures += 1
+    print()
+    print(f"{len(cases) - failures}/{len(cases)} cases passed.")
+    return 0 if failures == 0 else 1
+
+
+def run_interactive() -> int:
+    def yes(prompt: str) -> bool:
+        return input(f"{prompt} [y/N] ").strip().lower().startswith("y")
+
+    p = Problem(
+        has_typed_state=yes("Typed state / explicit state schema?"),
+        has_roles=yes("Specialist roles with distinct goals?"),
+        has_dialogue=yes("Multi-agent dialogue (speaker-ordering emergent)?"),
+        has_parallel_fanout=yes("Parallel fanout across N sub-workers?"),
+        needs_resume=yes("Must resume after process restart?"),
+        needs_human_interrupt=yes("Needs human approval mid-run?"),
+        total_llm_calls=int(input("Approx LLM calls per run? ").strip() or "1"),
+        needs_session_memory=yes("Needs durable per-user session memory?"),
+    )
+    r = recommend(p)
+    print()
+    print(json.dumps({"framework": r.framework, "reason": r.reason}, indent=2))
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--ask", action="store_true", help="interactive mode")
+    args = parser.parse_args()
+    return run_interactive() if args.ask else run_tests()
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,134 @@
+# Agent Framework Tradeoffs — LangGraph vs CrewAI vs AutoGen vs Agno
+
+> Every framework sells the same demo (research agent builds a report) and hides the same bug (state schema fights with the orchestration layer). Pick the framework whose abstractions match the shape of your problem; everything else is glue you write twice.
+
+**Type:** Learn
+**Languages:** Python
+**Prerequisites:** Phase 11 · 09 (Function Calling), Phase 11 · 16 (LangGraph)
+**Time:** ~45 minutes
+
+## The Problem
+
+You have a task that needs more than one LLM call. Maybe it is a research workflow (plan, search, summarize, cite). Maybe it is a code-review pipeline (parse diff, critique, patch, validate). Maybe it is a multi-turn assistant that books flights, writes emails, and files expense reports. You pick a framework.
+
+Three days later, you discover the framework's abstractions leak. CrewAI gives you roles but fights you when the "researcher" needs to hand a structured plan to the "writer." AutoGen gives you chat between agents but has no first-class state so your checkpoint is a pickle of a conversation log. LangGraph gives you a state graph but forces you to name every transition before you know what the agent will do. Agno gives you a single-agent primitive that screams when you try to fan out to three concurrent workers.
+
+The fix is not "pick the best framework." It is to match the framework's core abstraction to the shape of your problem. This lesson draws that map.
+
+## The Concept
+
+![Agent framework matrix: core abstraction vs problem shape](../assets/framework-matrix.svg)
+
+Four frameworks dominate the 2026 landscape. Their core abstractions are not the same.
+
+| Framework | Core abstraction | Best fit | Worst fit |
+|-----------|------------------|----------|-----------|
+| **LangGraph** | `StateGraph` — typed state, nodes, conditional edges, checkpointer. | Workflows with explicit state and human-in-the-loop interrupts; production agents needing time-travel debugging. | Loose, role-driven brainstorming where the topology is unknown. |
+| **CrewAI** | `Crew` — roles (goal, backstory), tasks, process (sequential or hierarchical). | Role-playing or persona-driven workflows with a short linear/hierarchical plan. | Anything stateful beyond the crew's turn history; complex branching. |
+| **AutoGen** | `ConversableAgent` pair — two or more agents that speak in turns until an exit condition. | Multi-agent *dialogue* (teacher-student, proposer-critic, actor-reviewer) where the thinking emerges from the chat. | Deterministic workflows with a known DAG; anything needing durable state across restarts. |
+| **Agno** | `Agent` — a single LLM + tools + memory, composable into teams. | Fast-to-build single agents and lightweight teams; strong multi-modality and built-in storage drivers. | Deep, explicitly-branched graphs with custom reducers. |
+
+### What "abstraction" actually means
+
+A framework's core abstraction is the thing you draw on the whiteboard when you pitch the architecture.
+
+- **LangGraph** → you draw a graph. Nodes are steps, edges are transitions, and the state object at every point is typed. The mental model is a state machine.
+- **CrewAI** → you draw an org chart. Each role has a job description and a manager routes tasks. The mental model is a small team of specialists.
+- **AutoGen** → you draw a Slack DM. Two agents message each other; a third joins if you need a moderator. The mental model is chat.
+- **Agno** → you draw a single box with tools hanging off it. Put boxes next to each other for a team. The mental model is "agent with batteries included."
+
+### The state question
+
+State is where most framework choices break down in production.
+
+- **LangGraph.** Typed state (`TypedDict` or Pydantic model), per-field reducers, first-class checkpointer (SQLite/Postgres/Redis). Resume, interrupt, and time-travel are free. *(See Phase 11 · 16.)*
+- **CrewAI.** State flows as strings between tasks via the `context` field, or structured through `output_pydantic`. No durable per-crew store out of the box; you bolt on your own if the crew must survive a restart.
+- **AutoGen.** State is the chat history and any user-defined `context`. Conversation transcripts persist; arbitrary workflow state does not unless you write adapters.
+- **Agno.** Built-in storage drivers (SQLite, Postgres, Mongo, Redis, DynamoDB) attached to an `Agent` via `storage=` — conversation sessions and user memories persist automatically. Not a full graph checkpointer; a session store.
+
+### The branching question
+
+Every non-trivial agent branches. Who decides the branch matters.
+
+- **LangGraph** — you decide, via conditional edges. Routing is a Python function with named branches. Branches are first-class in the compiled graph; the checkpointer records which branch was taken.
+- **CrewAI** — the manager decides in hierarchical mode; in sequential mode you decide at build time. Routing is implicit in the task list; there is no first-class "if" outside the manager's prompt.
+- **AutoGen** — the agents decide via chat. Branching is emergent from who speaks next. `GroupChatManager` selects the next speaker; you can hand-write a `speaker_selection_method` but the default is LLM-driven.
+- **Agno** — the agent decides by which tool to call next. Teams have a coordinator/router/collaborator mode; branching beyond that is the developer's responsibility.
+
+### The observability question
+
+- **LangGraph** — OpenTelemetry via LangSmith or any OTel exporter. Every node transition is a trace span; checkpoints double as replayable traces. LangSmith is the first-party option; Langfuse/Phoenix also have adapters.
+- **CrewAI** — first-class OpenTelemetry since late-2025; integrations with Langfuse, Phoenix, Opik, AgentOps.
+- **AutoGen** — OpenTelemetry integration via `autogen-core`; AgentOps and Opik have connectors. Tracing granularity is per-agent-message, not per-node.
+- **Agno** — built-in `monitoring=True` flag plus OpenTelemetry exporters; tight integration with Langfuse for session traces.
+
+### Cost and latency
+
+All four frameworks add per-call overhead (framework logic, validation, serialization). Rough order of increasing overhead: Agno ≈ LangGraph < CrewAI ≈ AutoGen. The difference is dominated by how much extra LLM routing the framework does. CrewAI's hierarchical manager spends tokens deciding who goes next; AutoGen's `GroupChatManager` likewise. LangGraph only spends tokens where you write `llm.invoke`. Agno's single-agent path is thin.
+
+When cost per run matters, prefer explicit routing (LangGraph edges, AutoGen `speaker_selection_method`) over LLM-selected routing.
+
+### Interoperability
+
+- **LangGraph** ↔ **LangChain** tools, retrievers, LLMs. First-class MCP adapter (tools imported as MCP servers).
+- **CrewAI** ↔ tools inherit from `BaseTool`; LangChain tools, LlamaIndex tools, and MCP tools all adapt in. Crew-to-crew delegation via `allow_delegation=True`.
+- **AutoGen** → `FunctionTool` wraps any Python callable; MCP adapter available. Tight coupling to AG2 ecosystem for agent-to-agent patterns.
+- **Agno** → `@tool` decorator or BaseTool subclass; MCP adapter; tools can be shared across agents and teams.
+
+## The Skill
+
+> You can explain, in one sentence, why a given framework is right for a given agent problem.
+
+Pre-build checklist:
+
+1. **Draw the shape.** Is this a graph (typed state, named transitions)? A role play (specialists hand off work)? A chat (agents talk until done)? A single agent with tools?
+2. **Decide who branches.** Developer-decided branching → LangGraph. Manager-agent-decided → CrewAI hierarchical. Chat-emergent → AutoGen. Tool-call-decided → Agno.
+3. **Check the state budget.** Do you need resume-from-checkpoint? Time-travel? Human interrupts mid-run? If yes, LangGraph is the default; Agno sessions cover conversation-scoped state.
+4. **Check the cost budget.** LLM-selected routing costs extra tokens per turn. If the agent runs thousands of times a day, prefer explicit routing.
+5. **Budget the framework overhead.** Every framework is another dependency. If the task is two LLM calls and a tool, write 30 lines of plain Python; no framework is cheaper than no framework.
+
+Refuse to reach for a framework before you can draw the graph, the org chart, the chat, or the agent box. Refuse to pick one that forces you to fight its state model for the thing you actually need.
+
+## The Decision Matrix
+
+| Problem shape | Preferred framework | Why |
+|---------------|---------------------|-----|
+| Workflow DAG with typed state, human approvals, long-running | LangGraph | First-class state, checkpointer, interrupts, time-travel. |
+| Research / writing pipeline with distinct roles | CrewAI (sequential) or LangGraph subgraphs | Role-per-task is cheap to express in CrewAI; scale up with LangGraph when branching gets complex. |
+| Proposer-critic or teacher-student dialogue | AutoGen | Two-agent chat is its native shape. |
+| Single agent with tools, sessions, memory | Agno | Thinnest setup, built-in storage and memory. |
+| Thousands of parallel fanouts with reducers | LangGraph + `Send` | The only one with a first-class parallel dispatch primitive. |
+| Quick prototype, no framework commitment | Plain Python + provider SDK | No framework is the fastest framework. |
+
+## Exercises
+
+1. **Easy.** Take the same task — "research Anthropic's headquarters, write a 200-word brief, cite sources" — and implement it in LangGraph (four nodes: plan, search, write, cite) and in CrewAI (three roles: researcher, writer, editor). Report token cost per run and lines of code.
+2. **Medium.** Build the same task in AutoGen (researcher ↔ writer chat, editor joins via `GroupChat`) and Agno (a single agent with `search_tools` and `write_tools`, plus a session store). Rank the four implementations on (a) cost per run, (b) ability to resume after a crash, (c) ability to inject a human approval before the write step.
+3. **Hard.** Build a decision-tree script `pick_framework.py` that takes a short problem description (JSON: `{has_typed_state, has_roles, has_dialogue, has_parallel_fanout, needs_resume}`) and returns a recommendation with one-sentence justification. Verify it on six cases you design yourself.
+
+## Key Terms
+
+| Term | What people say | What it actually means |
+|------|-----------------|-----------------------|
+| Orchestration | "How the agents coordinate" | The layer that decides which node/role/agent runs next. |
+| Durable state | "Resume after a restart" | State that survives process death, attached to a checkpoint or session store. |
+| LLM-selected routing | "Let the model decide" | A planner LLM picks the next step each turn; flexible but pays tokens on every decision. |
+| Explicit routing | "Developer decides" | A Python function or static edge picks the next step; cheap and auditable. |
+| Crew | "A CrewAI team" | Roles + tasks + process (sequential or hierarchical) bound into a single runnable. |
+| GroupChat | "AutoGen's multi-agent chat" | A managed conversation between N agents with a speaker selector. |
+| Team (Agno) | "Multi-agent Agno" | Route / coordinate / collaborate mode over a set of agents. |
+| StateGraph | "LangGraph's graph" | Typed-state, node, conditional-edge, checkpointer primitive. |
+
+## Further Reading
+
+- [LangGraph documentation](https://langchain-ai.github.io/langgraph/) — StateGraph, checkpointers, interrupts, time-travel.
+- [CrewAI documentation](https://docs.crewai.com/) — Crews, Flows, Agents, Tasks, Processes.
+- [AutoGen documentation](https://microsoft.github.io/autogen/) — ConversableAgent, GroupChat, teams, tools.
+- [Agno documentation](https://docs.agno.com/) — Agent, Team, Workflow, storage, memory.
+- [Anthropic — Building effective agents (Dec 2024)](https://www.anthropic.com/research/building-effective-agents) — pattern library (prompt chaining, routing, parallelization, orchestrator-workers, evaluator-optimizer) framework-agnostic.
+- [Yao et al., "ReAct: Synergizing Reasoning and Acting" (ICLR 2023)](https://arxiv.org/abs/2210.03629) — the primitive every framework dresses up.
+- [Wu et al., "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation" (2023)](https://arxiv.org/abs/2308.08155) — AutoGen's design paper.
+- [Park et al., "Generative Agents: Interactive Simulacra of Human Behavior" (UIST 2023)](https://arxiv.org/abs/2304.03442) — role-play foundation that CrewAI-style persona stacks build on.
+- Phase 11 · 16 (LangGraph) — the framework this lesson benchmarks against.
+- Phase 11 · 19 (Reflexion) — a pattern that maps cleanly to LangGraph but awkwardly to CrewAI.
+- Phase 11 · 22 (Production observability) — how to instrument whichever framework you pick.
@@ -0,0 +1,50 @@
+{
+  "lesson": "phase-11/17-agent-framework-tradeoffs",
+  "questions": [
+    {
+      "id": 1,
+      "prompt": "Which framework is the right first pick for a workflow that must resume after a crash, accept a human approval mid-run, and fan out to three retrievers in parallel?",
+      "choices": ["CrewAI", "AutoGen", "LangGraph", "Agno"],
+      "answer": 2
+    },
+    {
+      "id": 2,
+      "prompt": "Why does LLM-selected routing cost more tokens per turn than explicit routing?",
+      "choices": [
+        "It pre-fetches the next node in parallel to hedge latency.",
+        "A planner LLM call picks the next step each turn, adding prompt and completion tokens for every decision.",
+        "It duplicates the tool list for every agent in the crew.",
+        "It sends the whole conversation history to a verifier model."
+      ],
+      "answer": 1
+    },
+    {
+      "id": 3,
+      "prompt": "Proposer-critic dialogue in code review naturally maps to which framework's core abstraction?",
+      "choices": [
+        "CrewAI's sequential Crew",
+        "LangGraph's StateGraph",
+        "AutoGen's GroupChat / ConversableAgent pair",
+        "Agno's single Agent with tools"
+      ],
+      "answer": 2
+    },
+    {
+      "id": 4,
+      "prompt": "Which framework has built-in storage drivers (SQLite, Postgres, Redis, Mongo, DynamoDB) attached directly to the Agent primitive for session and memory persistence?",
+      "choices": ["LangGraph", "CrewAI", "AutoGen", "Agno"],
+      "answer": 3
+    },
+    {
+      "id": 5,
+      "prompt": "You have a two-call summarizer: fetch text, summarize. Which option is the right framework choice?",
+      "choices": [
+        "LangGraph StateGraph — always use a framework for reliability.",
+        "CrewAI with researcher + summarizer roles — roles make it clearer.",
+        "Plain Python with the provider SDK — no framework is the fastest framework for tiny pipelines.",
+        "AutoGen GroupChat — two agents can argue about the best summary."
+      ],
+      "answer": 2
+    }
+  ]
+}