mirror of
https://github.com/rohitg00/ai-engineering-from-scratch.git
synced 2026-10-02 01:54:39 +08:00
Add lesson: Context Engineering (the new discipline for 2026)
This commit is contained in:
Rohit Ghumare
@@ -0,0 +1,465 @@
|
||||
import json
|
||||
import numpy as np
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
def count_tokens(text):
|
||||
if not text:
|
||||
return 0
|
||||
return int(len(text.split()) * 1.3)
|
||||
|
||||
|
||||
def count_tokens_json(obj):
|
||||
return count_tokens(json.dumps(obj))
|
||||
|
||||
|
||||
class ContextBudget:
|
||||
def __init__(self, max_tokens=128000, generation_reserve=4000):
|
||||
self.max_tokens = max_tokens
|
||||
self.generation_reserve = generation_reserve
|
||||
self.available = max_tokens - generation_reserve
|
||||
self.allocations = OrderedDict()
|
||||
|
||||
def allocate(self, component, content, max_tokens=None):
|
||||
tokens = count_tokens(content)
|
||||
if max_tokens and tokens > max_tokens:
|
||||
words = content.split()
|
||||
target_words = int(max_tokens / 1.3)
|
||||
content = " ".join(words[:target_words])
|
||||
tokens = count_tokens(content)
|
||||
|
||||
used = sum(self.allocations.values())
|
||||
if used + tokens > self.available:
|
||||
allowed = self.available - used
|
||||
if allowed <= 0:
|
||||
return None, 0
|
||||
words = content.split()
|
||||
target_words = int(allowed / 1.3)
|
||||
content = " ".join(words[:target_words])
|
||||
tokens = count_tokens(content)
|
||||
|
||||
self.allocations[component] = tokens
|
||||
return content, tokens
|
||||
|
||||
def remaining(self):
|
||||
used = sum(self.allocations.values())
|
||||
return self.available - used
|
||||
|
||||
def utilization(self):
|
||||
used = sum(self.allocations.values())
|
||||
return used / self.max_tokens
|
||||
|
||||
def report(self):
|
||||
total_used = sum(self.allocations.values())
|
||||
lines = []
|
||||
lines.append(f"\n Context Budget Report ({self.max_tokens:,} token window)")
|
||||
lines.append(" " + "-" * 55)
|
||||
for component, tokens in self.allocations.items():
|
||||
pct = tokens / self.max_tokens * 100
|
||||
bar = "#" * int(pct * 2) if pct >= 0.5 else ""
|
||||
lines.append(f" {component:<25} {tokens:>6} tokens ({pct:>5.1f}%) {bar}")
|
||||
lines.append(" " + "-" * 55)
|
||||
lines.append(f" {'Used':<25} {total_used:>6} tokens ({total_used/self.max_tokens*100:.1f}%)")
|
||||
lines.append(f" {'Generation reserve':<25} {self.generation_reserve:>6} tokens")
|
||||
lines.append(f" {'Remaining':<25} {self.remaining():>6} tokens")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def reorder_lost_in_middle(items, scores):
|
||||
paired = sorted(zip(scores, items), reverse=True)
|
||||
sorted_items = [item for _, item in paired]
|
||||
|
||||
if len(sorted_items) <= 2:
|
||||
return sorted_items
|
||||
|
||||
first_half = sorted_items[::2]
|
||||
second_half = sorted_items[1::2]
|
||||
second_half.reverse()
|
||||
|
||||
return first_half + second_half
|
||||
|
||||
|
||||
def score_relevance(query, documents):
|
||||
query_words = set(query.lower().split())
|
||||
scores = []
|
||||
for doc in documents:
|
||||
doc_words = set(doc.lower().split())
|
||||
if not query_words:
|
||||
scores.append(0.0)
|
||||
continue
|
||||
overlap = len(query_words & doc_words) / len(query_words)
|
||||
scores.append(round(overlap, 3))
|
||||
return scores
|
||||
|
||||
|
||||
class ConversationManager:
|
||||
def __init__(self, max_history_tokens=5000):
|
||||
self.turns = []
|
||||
self.summaries = []
|
||||
self.max_history_tokens = max_history_tokens
|
||||
|
||||
def add_turn(self, role, content):
|
||||
self.turns.append({"role": role, "content": content})
|
||||
self._compress_if_needed()
|
||||
|
||||
def _compress_if_needed(self):
|
||||
total = sum(count_tokens(t["content"]) for t in self.turns)
|
||||
if total <= self.max_history_tokens:
|
||||
return
|
||||
|
||||
while total > self.max_history_tokens and len(self.turns) > 4:
|
||||
old_turns = self.turns[:2]
|
||||
summary = self._summarize_turns(old_turns)
|
||||
self.summaries.append(summary)
|
||||
self.turns = self.turns[2:]
|
||||
total = sum(count_tokens(t["content"]) for t in self.turns)
|
||||
|
||||
def _summarize_turns(self, turns):
|
||||
parts = []
|
||||
for t in turns:
|
||||
content = t["content"]
|
||||
if len(content) > 100:
|
||||
content = content[:100] + "..."
|
||||
parts.append(f"{t['role']}: {content}")
|
||||
return "Previous: " + " | ".join(parts)
|
||||
|
||||
def get_context(self):
|
||||
parts = []
|
||||
if self.summaries:
|
||||
parts.append("[Conversation Summary]")
|
||||
for s in self.summaries:
|
||||
parts.append(s)
|
||||
if self.turns:
|
||||
parts.append("[Recent Conversation]")
|
||||
for t in self.turns:
|
||||
parts.append(f"{t['role']}: {t['content']}")
|
||||
return "\n".join(parts)
|
||||
|
||||
def token_count(self):
|
||||
return count_tokens(self.get_context())
|
||||
|
||||
def stats(self):
|
||||
return {
|
||||
"live_turns": len(self.turns),
|
||||
"summaries": len(self.summaries),
|
||||
"tokens": self.token_count(),
|
||||
}
|
||||
|
||||
|
||||
TOOL_REGISTRY = {
|
||||
"read_file": {
|
||||
"description": "Read contents of a file from disk",
|
||||
"tokens": 120,
|
||||
"categories": ["code", "files"],
|
||||
},
|
||||
"write_file": {
|
||||
"description": "Write content to a file on disk",
|
||||
"tokens": 150,
|
||||
"categories": ["code", "files"],
|
||||
},
|
||||
"search_code": {
|
||||
"description": "Search for patterns across the codebase",
|
||||
"tokens": 130,
|
||||
"categories": ["code"],
|
||||
},
|
||||
"run_command": {
|
||||
"description": "Execute a shell command and return output",
|
||||
"tokens": 140,
|
||||
"categories": ["code", "system"],
|
||||
},
|
||||
"create_calendar_event": {
|
||||
"description": "Create a new event on the calendar",
|
||||
"tokens": 180,
|
||||
"categories": ["calendar"],
|
||||
},
|
||||
"list_emails": {
|
||||
"description": "List recent emails from inbox",
|
||||
"tokens": 160,
|
||||
"categories": ["email"],
|
||||
},
|
||||
"send_email": {
|
||||
"description": "Compose and send an email message",
|
||||
"tokens": 200,
|
||||
"categories": ["email"],
|
||||
},
|
||||
"web_search": {
|
||||
"description": "Search the web for information",
|
||||
"tokens": 140,
|
||||
"categories": ["research"],
|
||||
},
|
||||
"query_database": {
|
||||
"description": "Run a SQL query against the database",
|
||||
"tokens": 170,
|
||||
"categories": ["code", "data"],
|
||||
},
|
||||
"generate_chart": {
|
||||
"description": "Generate a visualization from data",
|
||||
"tokens": 190,
|
||||
"categories": ["data", "visualization"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def classify_intent(query):
|
||||
query_lower = query.lower()
|
||||
|
||||
intent_keywords = {
|
||||
"code": ["code", "function", "bug", "error", "file", "implement", "refactor", "debug", "test", "fix", "class", "module"],
|
||||
"calendar": ["meeting", "schedule", "calendar", "appointment", "event", "tuesday", "tomorrow"],
|
||||
"email": ["email", "mail", "send", "inbox", "message", "reply"],
|
||||
"research": ["search", "find", "what is", "how does", "explain", "look up", "documentation"],
|
||||
"data": ["data", "query", "database", "chart", "graph", "analytics", "sql", "stats"],
|
||||
}
|
||||
|
||||
scores = {}
|
||||
for intent, keywords in intent_keywords.items():
|
||||
score = sum(1 for kw in keywords if kw in query_lower)
|
||||
if score > 0:
|
||||
scores[intent] = score
|
||||
|
||||
if not scores:
|
||||
return ["code"]
|
||||
|
||||
max_score = max(scores.values())
|
||||
return [intent for intent, score in scores.items() if score >= max_score * 0.5]
|
||||
|
||||
|
||||
def select_tools(query, token_budget=2000):
|
||||
intents = classify_intent(query)
|
||||
relevant = {}
|
||||
total_tokens = 0
|
||||
|
||||
for name, tool in TOOL_REGISTRY.items():
|
||||
if any(cat in intents for cat in tool["categories"]):
|
||||
if total_tokens + tool["tokens"] <= token_budget:
|
||||
relevant[name] = tool
|
||||
total_tokens += tool["tokens"]
|
||||
|
||||
return relevant, total_tokens
|
||||
|
||||
|
||||
class ContextEngine:
|
||||
def __init__(self, max_tokens=128000, generation_reserve=4000):
|
||||
self.max_tokens = max_tokens
|
||||
self.generation_reserve = generation_reserve
|
||||
self.conversation = ConversationManager(max_history_tokens=5000)
|
||||
self.system_prompt = (
|
||||
"You are a helpful AI assistant. You have access to tools for "
|
||||
"code editing, file management, web search, and data analysis. "
|
||||
"Use the appropriate tools for each task. Be concise and accurate."
|
||||
)
|
||||
self.knowledge_base = [
|
||||
"Python 3.12 introduced type parameter syntax for generic classes using bracket notation.",
|
||||
"The project uses PostgreSQL 16 with pgvector for embedding storage.",
|
||||
"Authentication is handled by Supabase Auth with JWT tokens.",
|
||||
"The frontend is built with Next.js 15 using the App Router.",
|
||||
"API rate limits are set to 100 requests per minute per user.",
|
||||
"The deployment pipeline uses GitHub Actions with Docker multi-stage builds.",
|
||||
"Test coverage must be above 80% for all new modules.",
|
||||
"The codebase follows the repository pattern for data access.",
|
||||
"Error logging uses structured JSON format with correlation IDs.",
|
||||
"The vector search index uses HNSW with 128 dimensions and cosine distance.",
|
||||
]
|
||||
|
||||
def assemble(self, query):
|
||||
budget = ContextBudget(self.max_tokens, self.generation_reserve)
|
||||
|
||||
budget.allocate("system_prompt", self.system_prompt, max_tokens=1000)
|
||||
|
||||
tools, tool_tokens = select_tools(query, token_budget=2000)
|
||||
tool_text = json.dumps(list(tools.keys()))
|
||||
budget.allocate("tools", tool_text, max_tokens=2000)
|
||||
|
||||
relevance = score_relevance(query, self.knowledge_base)
|
||||
threshold = 0.05
|
||||
relevant_docs = [
|
||||
doc for doc, score in zip(self.knowledge_base, relevance)
|
||||
if score >= threshold
|
||||
]
|
||||
|
||||
if relevant_docs:
|
||||
doc_scores = [s for s in relevance if s >= threshold]
|
||||
reordered = reorder_lost_in_middle(relevant_docs, doc_scores)
|
||||
doc_text = "\n".join(reordered)
|
||||
budget.allocate("retrieved_context", doc_text, max_tokens=3000)
|
||||
|
||||
history_text = self.conversation.get_context()
|
||||
if history_text.strip():
|
||||
budget.allocate("conversation_history", history_text, max_tokens=5000)
|
||||
|
||||
budget.allocate("user_query", query, max_tokens=500)
|
||||
|
||||
return budget
|
||||
|
||||
def chat(self, query):
|
||||
self.conversation.add_turn("user", query)
|
||||
budget = self.assemble(query)
|
||||
response = f"[Simulated response to: {query[:50]}...]"
|
||||
self.conversation.add_turn("assistant", response)
|
||||
return budget
|
||||
|
||||
|
||||
def run_budget_demo():
|
||||
print("=" * 60)
|
||||
print(" STEP 1: Context Budget Manager")
|
||||
print("=" * 60)
|
||||
|
||||
budget = ContextBudget(max_tokens=128000, generation_reserve=4000)
|
||||
budget.allocate("system_prompt", "You are a helpful assistant. " * 20, max_tokens=500)
|
||||
budget.allocate("tools", json.dumps(list(TOOL_REGISTRY.keys())), max_tokens=2000)
|
||||
budget.allocate("retrieved_docs", "The project uses PostgreSQL. " * 50, max_tokens=3000)
|
||||
budget.allocate("history", "user: How do I fix this?\nassistant: Check the logs." * 10, max_tokens=5000)
|
||||
budget.allocate("query", "Fix the authentication bug in the JWT validation module", max_tokens=500)
|
||||
print(budget.report())
|
||||
|
||||
|
||||
def run_reorder_demo():
|
||||
print(f"\n{'=' * 60}")
|
||||
print(" STEP 2: Lost-in-the-Middle Reordering")
|
||||
print("=" * 60)
|
||||
|
||||
docs = [
|
||||
"Doc A: PostgreSQL connection pooling (most relevant)",
|
||||
"Doc B: Redis caching layer (somewhat relevant)",
|
||||
"Doc C: CSS styling guide (not relevant)",
|
||||
"Doc D: Database migration scripts (relevant)",
|
||||
"Doc E: CI/CD pipeline config (slightly relevant)",
|
||||
"Doc F: API authentication flow (relevant)",
|
||||
"Doc G: Frontend routing (not relevant)",
|
||||
]
|
||||
scores = [0.95, 0.60, 0.05, 0.80, 0.30, 0.75, 0.10]
|
||||
|
||||
reordered = reorder_lost_in_middle(docs, scores)
|
||||
|
||||
print(f"\n Original order (by insertion):")
|
||||
for doc, score in zip(docs, scores):
|
||||
print(f" {score:.2f} {doc}")
|
||||
|
||||
print(f"\n Reordered (high relevance at start + end, low in middle):")
|
||||
for i, doc in enumerate(reordered):
|
||||
position = "START" if i < 2 else "END" if i >= len(reordered) - 2 else "middle"
|
||||
print(f" [{position:>6}] {doc}")
|
||||
|
||||
|
||||
def run_conversation_demo():
|
||||
print(f"\n{'=' * 60}")
|
||||
print(" STEP 3: Conversation History Compression")
|
||||
print("=" * 60)
|
||||
|
||||
conv = ConversationManager(max_history_tokens=200)
|
||||
|
||||
exchanges = [
|
||||
("How do I set up the database?", "Run docker-compose up to start PostgreSQL. Then run the migrations with npm run migrate."),
|
||||
("What about the environment variables?", "Copy .env.example to .env and fill in DATABASE_URL and JWT_SECRET."),
|
||||
("The migrations are failing with a connection error.", "Check that PostgreSQL is running on port 5432 and the DATABASE_URL matches."),
|
||||
("Fixed it. Now how do I seed test data?", "Run npm run seed which loads fixtures from the test/fixtures directory."),
|
||||
("Can I run the tests now?", "Yes, run npm test. Make sure the test database is separate from development."),
|
||||
]
|
||||
|
||||
for i, (user_msg, assistant_msg) in enumerate(exchanges):
|
||||
conv.add_turn("user", user_msg)
|
||||
conv.add_turn("assistant", assistant_msg)
|
||||
stats = conv.stats()
|
||||
print(f"\n After turn {i + 1}:")
|
||||
print(f" Live turns: {stats['live_turns']}, Summaries: {stats['summaries']}, Tokens: {stats['tokens']}")
|
||||
|
||||
print(f"\n Final context:")
|
||||
for line in conv.get_context().split("\n"):
|
||||
print(f" {line}")
|
||||
|
||||
|
||||
def run_tool_selection_demo():
|
||||
print(f"\n{'=' * 60}")
|
||||
print(" STEP 4: Dynamic Tool Selection")
|
||||
print("=" * 60)
|
||||
|
||||
test_queries = [
|
||||
"Fix the bug in auth.py where JWT tokens expire too early",
|
||||
"Schedule a meeting with the design team for next Tuesday at 2pm",
|
||||
"Show me the database query performance stats and generate a chart",
|
||||
"Search for best practices on error handling in Python",
|
||||
"Send an email to the team about the deployment schedule",
|
||||
"Read the config file and check for database connection settings",
|
||||
]
|
||||
|
||||
print(f"\n All tools: {list(TOOL_REGISTRY.keys())} ({sum(t['tokens'] for t in TOOL_REGISTRY.values())} total tokens)")
|
||||
|
||||
for q in test_queries:
|
||||
tools, tokens = select_tools(q)
|
||||
intents = classify_intent(q)
|
||||
all_tokens = sum(t["tokens"] for t in TOOL_REGISTRY.values())
|
||||
savings = all_tokens - tokens
|
||||
print(f"\n Query: {q[:60]}...")
|
||||
print(f" Intents: {intents}")
|
||||
print(f" Selected: {list(tools.keys())}")
|
||||
print(f" Tokens: {tokens} (saved {savings} by pruning)")
|
||||
|
||||
|
||||
def run_full_pipeline_demo():
|
||||
print(f"\n{'=' * 60}")
|
||||
print(" STEP 5: Full Context Assembly Pipeline")
|
||||
print("=" * 60)
|
||||
|
||||
engine = ContextEngine(max_tokens=128000, generation_reserve=4000)
|
||||
|
||||
queries = [
|
||||
"Fix the bug in the authentication module where JWT tokens expire too early",
|
||||
"What is the best approach for implementing vector search with PostgreSQL?",
|
||||
"Schedule a team standup meeting for tomorrow morning",
|
||||
]
|
||||
|
||||
for q in queries:
|
||||
print(f"\n Query: {q}")
|
||||
budget = engine.chat(q)
|
||||
print(budget.report())
|
||||
|
||||
print(f"\n --- After building up conversation history ---")
|
||||
for i in range(6):
|
||||
engine.conversation.add_turn("user", f"Follow-up question {i+1} about the database migration and authentication setup")
|
||||
engine.conversation.add_turn("assistant", f"Detailed response {i+1} covering the technical architecture and implementation steps")
|
||||
|
||||
budget = engine.chat("Now implement all the changes we discussed in the previous turns")
|
||||
print(budget.report())
|
||||
conv_stats = engine.conversation.stats()
|
||||
print(f"\n Conversation state: {conv_stats['live_turns']} live turns, {conv_stats['summaries']} summaries, {conv_stats['tokens']} tokens")
|
||||
|
||||
|
||||
def run_relevance_demo():
|
||||
print(f"\n{'=' * 60}")
|
||||
print(" STEP 6: Relevance Scoring + Filtering")
|
||||
print("=" * 60)
|
||||
|
||||
knowledge = [
|
||||
"Python 3.12 introduced type parameter syntax for generic classes.",
|
||||
"The project uses PostgreSQL 16 with pgvector for embedding storage.",
|
||||
"Authentication is handled by Supabase Auth with JWT tokens.",
|
||||
"The frontend is built with Next.js 15 using the App Router.",
|
||||
"API rate limits are set to 100 requests per minute per user.",
|
||||
"The deployment pipeline uses GitHub Actions with Docker builds.",
|
||||
"Test coverage must be above 80% for all new modules.",
|
||||
"Error logging uses structured JSON format with correlation IDs.",
|
||||
]
|
||||
|
||||
query = "How do I fix the JWT authentication token expiry bug?"
|
||||
scores = score_relevance(query, knowledge)
|
||||
|
||||
print(f"\n Query: {query}")
|
||||
print(f"\n Relevance scores:")
|
||||
for doc, score in sorted(zip(knowledge, scores), key=lambda x: -x[1]):
|
||||
marker = "*" if score >= 0.05 else " "
|
||||
print(f" {marker} {score:.3f} {doc[:70]}...")
|
||||
|
||||
threshold = 0.05
|
||||
included = sum(1 for s in scores if s >= threshold)
|
||||
excluded = len(scores) - included
|
||||
print(f"\n Threshold {threshold}: {included} included, {excluded} excluded")
|
||||
print(f" Token savings: ~{excluded * 20} tokens from excluded docs")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_budget_demo()
|
||||
run_reorder_demo()
|
||||
run_conversation_demo()
|
||||
run_tool_selection_demo()
|
||||
run_full_pipeline_demo()
|
||||
run_relevance_demo()
|
||||
@@ -0,0 +1,580 @@
|
||||
# Context Engineering: Windows, Budgets, Memory, and Retrieval
|
||||
|
||||
> Prompt engineering is dead. Context engineering killed it. A prompt is a string you type. Context is everything that goes into the model's window: system instructions, retrieved documents, tool definitions, conversation history, few-shot examples, and yes, the prompt. The best AI engineers in 2026 are context engineers. They decide what goes in, what stays out, and in what order.
|
||||
|
||||
**Type:** Build
|
||||
**Languages:** Python
|
||||
**Prerequisites:** Phase 10 (LLMs from Scratch), Phase 11 Lesson 01-02
|
||||
**Time:** ~90 minutes
|
||||
|
||||
## The Problem
|
||||
|
||||
Claude has a 200K token context window. GPT-4o has 128K. Gemini 1.5 has 1M. These numbers sound enormous until you fill them.
|
||||
|
||||
Here is a real breakdown for a coding assistant. System prompt: 500 tokens. Tool definitions for 50 tools: 8,000 tokens. Retrieved documentation: 4,000 tokens. Conversation history (10 turns): 6,000 tokens. Current user query: 200 tokens. Generation budget (max output): 4,000 tokens. Total: 22,700 tokens. That is only 18% of a 128K window.
|
||||
|
||||
But attention does not scale linearly with context length. A model with 128K tokens of context pays quadratic attention cost (O(n^2) in vanilla transformers, though most production models use efficient attention variants). More importantly, retrieval accuracy degrades. The "Needle in a Haystack" test shows that models struggle to find information placed in the middle of long contexts. Claude 3.5 Sonnet retrieves needles at the start and end with near-perfect accuracy, but accuracy drops 10-20% for needles placed between positions 40% and 70% of the context.
|
||||
|
||||
The practical lesson: having 200K tokens available does not mean using 200K tokens is effective. A carefully curated 10K token context often outperforms a dumped 100K token context. Context engineering is the discipline of maximizing signal-to-noise ratio within the context window.
|
||||
|
||||
Every token you put in the window displaces a token that could carry more relevant information. Every irrelevant tool definition, every stale conversation turn, every chunk of retrieved text that does not answer the question -- each one makes the model slightly worse at the task.
|
||||
|
||||
## The Concept
|
||||
|
||||
### The Context Window is a Scarce Resource
|
||||
|
||||
Think of the context window as RAM, not disk. It is fast and directly accessible, but limited. You cannot fit everything. You must choose.
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Window["Context Window (128K tokens)"]
|
||||
direction TB
|
||||
S["System Prompt\n~500 tokens"] --> T["Tool Definitions\n~2K-8K tokens"]
|
||||
T --> R["Retrieved Context\n~2K-10K tokens"]
|
||||
R --> H["Conversation History\n~2K-20K tokens"]
|
||||
H --> F["Few-shot Examples\n~1K-3K tokens"]
|
||||
F --> Q["User Query\n~100-500 tokens"]
|
||||
Q --> G["Generation Budget\n~2K-8K tokens"]
|
||||
end
|
||||
|
||||
style S fill:#1a1a2e,stroke:#e94560,color:#fff
|
||||
style T fill:#1a1a2e,stroke:#0f3460,color:#fff
|
||||
style R fill:#1a1a2e,stroke:#ffa500,color:#fff
|
||||
style H fill:#1a1a2e,stroke:#51cf66,color:#fff
|
||||
style F fill:#1a1a2e,stroke:#9b59b6,color:#fff
|
||||
style Q fill:#1a1a2e,stroke:#e94560,color:#fff
|
||||
style G fill:#1a1a2e,stroke:#0f3460,color:#fff
|
||||
```
|
||||
|
||||
Each component competes for space. Adding more tool definitions means less room for conversation history. Adding more retrieved context means less room for few-shot examples. Context engineering is the art of allocating this budget to maximize task performance.
|
||||
|
||||
### Lost-in-the-Middle
|
||||
|
||||
The most important empirical finding in context engineering. Models attend better to information at the beginning and end of the context. Information in the middle gets lower attention scores and is more likely to be ignored.
|
||||
|
||||
Liu et al. (2023) tested this systematically. They placed a relevant document among 20 irrelevant documents at various positions and measured answer accuracy. When the relevant document was first or last, accuracy was 85-90%. When it was in the middle (position 10 of 20), accuracy dropped to 60-70%.
|
||||
|
||||
This has direct engineering implications:
|
||||
|
||||
- Put the most important information first (system prompt, critical instructions)
|
||||
- Put the current query and most relevant context last (recency bias helps)
|
||||
- Treat the middle of the context as the lowest-priority zone
|
||||
- If you must include information in the middle, duplicate the key point at the end
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph Attention["Attention Distribution Across Context"]
|
||||
direction LR
|
||||
P1["Position 0-20%\nHIGH attention\n(system prompt)"]
|
||||
P2["Position 20-40%\nMODERATE"]
|
||||
P3["Position 40-70%\nLOW attention\n(lost in middle)"]
|
||||
P4["Position 70-90%\nMODERATE"]
|
||||
P5["Position 90-100%\nHIGH attention\n(current query)"]
|
||||
end
|
||||
|
||||
style P1 fill:#51cf66,color:#000
|
||||
style P2 fill:#ffa500,color:#000
|
||||
style P3 fill:#ff6b6b,color:#fff
|
||||
style P4 fill:#ffa500,color:#000
|
||||
style P5 fill:#51cf66,color:#000
|
||||
```
|
||||
|
||||
### Context Components
|
||||
|
||||
**System prompt**: sets the persona, constraints, and behavioral rules. This goes first and stays constant across turns. Claude Code uses roughly 6,000 tokens for its system prompt including tool definitions and behavioral instructions. Keep it tight. Every word in the system prompt is repeated on every API call.
|
||||
|
||||
**Tool definitions**: each tool adds 50-200 tokens (name, description, parameter schema). 50 tools at 150 tokens each is 7,500 tokens before any conversation happens. Dynamic tool selection -- only including tools relevant to the current query -- can reduce this by 60-80%.
|
||||
|
||||
**Retrieved context**: documents from a vector database, search results, file contents. The quality of retrieval directly determines the quality of the response. Bad retrieval is worse than no retrieval -- it fills the window with noise and actively misleads the model.
|
||||
|
||||
**Conversation history**: every previous user message and assistant response. Grows linearly with conversation length. A 50-turn conversation at 200 tokens per turn is 10,000 tokens of history. Most of it is irrelevant to the current query.
|
||||
|
||||
**Few-shot examples**: input/output pairs that demonstrate the desired behavior. Two to three well-chosen examples often improve output quality more than thousands of tokens of instructions. But they cost space.
|
||||
|
||||
**Generation budget**: the tokens reserved for the model's response. If you fill the window to capacity, the model has no room to answer. Reserve at least 2,000-4,000 tokens for generation.
|
||||
|
||||
### Context Compression Strategies
|
||||
|
||||
**History summarization**: instead of keeping all previous turns verbatim, periodically summarize the conversation. "We discussed X, decided Y, and the user wants Z" in 100 tokens replaces 10 turns that took 2,000 tokens. Run summarization when history exceeds a threshold (e.g., 5,000 tokens).
|
||||
|
||||
**Relevance filtering**: score each retrieved document against the current query and drop documents below a threshold. If you retrieved 10 chunks but only 3 are relevant, discard the other 7. Better to have 3 highly relevant chunks than 10 mediocre ones.
|
||||
|
||||
**Tool pruning**: classify the user's query intent and only include tools relevant to that intent. A code question does not need calendar tools. A scheduling question does not need file system tools. This can reduce tool definitions from 8,000 tokens to 1,000.
|
||||
|
||||
**Recursive summarization**: for very long documents, summarize in stages. First summarize each section, then summarize the summaries. A 50-page document becomes a 500-token digest that captures the key points.
|
||||
|
||||
### Memory Systems
|
||||
|
||||
Context engineering spans three time horizons.
|
||||
|
||||
**Short-term memory**: the current conversation. Stored in the context window directly. Grows with each turn. Managed by summarization and truncation.
|
||||
|
||||
**Long-term memory**: facts and preferences that persist across conversations. "The user prefers TypeScript." "The project uses PostgreSQL." Stored in a database, retrieved on session start. Claude Code stores this in CLAUDE.md files. ChatGPT stores it in its memory feature.
|
||||
|
||||
**Episodic memory**: specific past interactions that might be relevant. "Last Tuesday, we debugged a similar issue in the auth module." Stored as embeddings, retrieved when the current conversation matches a past episode.
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Memory["Memory Architecture"]
|
||||
direction TB
|
||||
STM["Short-term Memory\n(current conversation)\nDirect in context window"]
|
||||
LTM["Long-term Memory\n(facts, preferences)\nDB -> retrieved on session start"]
|
||||
EM["Episodic Memory\n(past interactions)\nEmbeddings -> retrieved on similarity"]
|
||||
end
|
||||
|
||||
Q["Current Query"] --> STM
|
||||
Q --> LTM
|
||||
Q --> EM
|
||||
|
||||
STM --> CW["Context Window"]
|
||||
LTM --> CW
|
||||
EM --> CW
|
||||
|
||||
style STM fill:#1a1a2e,stroke:#51cf66,color:#fff
|
||||
style LTM fill:#1a1a2e,stroke:#0f3460,color:#fff
|
||||
style EM fill:#1a1a2e,stroke:#e94560,color:#fff
|
||||
style CW fill:#1a1a2e,stroke:#ffa500,color:#fff
|
||||
```
|
||||
|
||||
### Dynamic Context Assembly
|
||||
|
||||
The key insight: different queries need different context. A static system prompt + static tools + static history is wasteful. The best systems dynamically assemble context per query.
|
||||
|
||||
1. Classify the query intent
|
||||
2. Select relevant tools (not all tools)
|
||||
3. Retrieve relevant documents (not a fixed set)
|
||||
4. Include relevant history turns (not all history)
|
||||
5. Add few-shot examples that match the task type
|
||||
6. Order everything by importance: critical first, important last, optional in the middle
|
||||
|
||||
This is what separates a good AI application from a great one. The model is the same. The context is the differentiator.
|
||||
|
||||
## Build It
|
||||
|
||||
### Step 1: Token Counter
|
||||
|
||||
You cannot budget what you cannot measure. Build a simple token counter (approximation using whitespace splitting, since the exact count depends on the tokenizer).
|
||||
|
||||
```python
|
||||
import json
|
||||
import numpy as np
|
||||
from collections import OrderedDict
|
||||
|
||||
def count_tokens(text):
|
||||
if not text:
|
||||
return 0
|
||||
return int(len(text.split()) * 1.3)
|
||||
|
||||
def count_tokens_json(obj):
|
||||
return count_tokens(json.dumps(obj))
|
||||
```
|
||||
|
||||
### Step 2: Context Budget Manager
|
||||
|
||||
The core abstraction. A budget manager tracks how many tokens each component uses and enforces limits.
|
||||
|
||||
```python
|
||||
class ContextBudget:
|
||||
def __init__(self, max_tokens=128000, generation_reserve=4000):
|
||||
self.max_tokens = max_tokens
|
||||
self.generation_reserve = generation_reserve
|
||||
self.available = max_tokens - generation_reserve
|
||||
self.allocations = OrderedDict()
|
||||
|
||||
def allocate(self, component, content, max_tokens=None):
|
||||
tokens = count_tokens(content)
|
||||
if max_tokens and tokens > max_tokens:
|
||||
words = content.split()
|
||||
target_words = int(max_tokens / 1.3)
|
||||
content = " ".join(words[:target_words])
|
||||
tokens = count_tokens(content)
|
||||
|
||||
used = sum(self.allocations.values())
|
||||
if used + tokens > self.available:
|
||||
allowed = self.available - used
|
||||
if allowed <= 0:
|
||||
return None, 0
|
||||
words = content.split()
|
||||
target_words = int(allowed / 1.3)
|
||||
content = " ".join(words[:target_words])
|
||||
tokens = count_tokens(content)
|
||||
|
||||
self.allocations[component] = tokens
|
||||
return content, tokens
|
||||
|
||||
def remaining(self):
|
||||
used = sum(self.allocations.values())
|
||||
return self.available - used
|
||||
|
||||
def utilization(self):
|
||||
used = sum(self.allocations.values())
|
||||
return used / self.max_tokens
|
||||
|
||||
def report(self):
|
||||
total_used = sum(self.allocations.values())
|
||||
lines = []
|
||||
lines.append(f"Context Budget Report ({self.max_tokens:,} token window)")
|
||||
lines.append("-" * 50)
|
||||
for component, tokens in self.allocations.items():
|
||||
pct = tokens / self.max_tokens * 100
|
||||
bar = "#" * int(pct / 2)
|
||||
lines.append(f" {component:<25} {tokens:>6} tokens ({pct:>5.1f}%) {bar}")
|
||||
lines.append("-" * 50)
|
||||
lines.append(f" {'Used':<25} {total_used:>6} tokens ({total_used/self.max_tokens*100:.1f}%)")
|
||||
lines.append(f" {'Generation reserve':<25} {self.generation_reserve:>6} tokens")
|
||||
lines.append(f" {'Remaining':<25} {self.remaining():>6} tokens")
|
||||
return "\n".join(lines)
|
||||
```
|
||||
|
||||
### Step 3: Lost-in-the-Middle Reordering
|
||||
|
||||
Implement the reordering strategy: most important items go first and last, least important go in the middle.
|
||||
|
||||
```python
|
||||
def reorder_lost_in_middle(items, scores):
|
||||
paired = sorted(zip(scores, items), reverse=True)
|
||||
sorted_items = [item for _, item in paired]
|
||||
|
||||
if len(sorted_items) <= 2:
|
||||
return sorted_items
|
||||
|
||||
first_half = sorted_items[::2]
|
||||
second_half = sorted_items[1::2]
|
||||
second_half.reverse()
|
||||
|
||||
return first_half + second_half
|
||||
|
||||
def score_relevance(query, documents):
|
||||
query_words = set(query.lower().split())
|
||||
scores = []
|
||||
for doc in documents:
|
||||
doc_words = set(doc.lower().split())
|
||||
if not query_words:
|
||||
scores.append(0.0)
|
||||
continue
|
||||
overlap = len(query_words & doc_words) / len(query_words)
|
||||
scores.append(round(overlap, 3))
|
||||
return scores
|
||||
```
|
||||
|
||||
### Step 4: Conversation History Compressor
|
||||
|
||||
Summarize old conversation turns to reclaim token budget.
|
||||
|
||||
```python
|
||||
class ConversationManager:
|
||||
def __init__(self, max_history_tokens=5000):
|
||||
self.turns = []
|
||||
self.summaries = []
|
||||
self.max_history_tokens = max_history_tokens
|
||||
|
||||
def add_turn(self, role, content):
|
||||
self.turns.append({"role": role, "content": content})
|
||||
self._compress_if_needed()
|
||||
|
||||
def _compress_if_needed(self):
|
||||
total = sum(count_tokens(t["content"]) for t in self.turns)
|
||||
if total <= self.max_history_tokens:
|
||||
return
|
||||
|
||||
while total > self.max_history_tokens and len(self.turns) > 4:
|
||||
old_turns = self.turns[:2]
|
||||
summary = self._summarize_turns(old_turns)
|
||||
self.summaries.append(summary)
|
||||
self.turns = self.turns[2:]
|
||||
total = sum(count_tokens(t["content"]) for t in self.turns)
|
||||
|
||||
def _summarize_turns(self, turns):
|
||||
parts = []
|
||||
for t in turns:
|
||||
content = t["content"]
|
||||
if len(content) > 100:
|
||||
content = content[:100] + "..."
|
||||
parts.append(f"{t['role']}: {content}")
|
||||
return "Previous: " + " | ".join(parts)
|
||||
|
||||
def get_context(self):
|
||||
parts = []
|
||||
if self.summaries:
|
||||
parts.append("[Conversation Summary]")
|
||||
for s in self.summaries:
|
||||
parts.append(s)
|
||||
parts.append("[Recent Conversation]")
|
||||
for t in self.turns:
|
||||
parts.append(f"{t['role']}: {t['content']}")
|
||||
return "\n".join(parts)
|
||||
|
||||
def token_count(self):
|
||||
return count_tokens(self.get_context())
|
||||
```
|
||||
|
||||
### Step 5: Dynamic Tool Selector
|
||||
|
||||
Only include tools relevant to the current query. Classify intent, then filter.
|
||||
|
||||
```python
|
||||
TOOL_REGISTRY = {
|
||||
"read_file": {
|
||||
"description": "Read contents of a file",
|
||||
"tokens": 120,
|
||||
"categories": ["code", "files"],
|
||||
},
|
||||
"write_file": {
|
||||
"description": "Write content to a file",
|
||||
"tokens": 150,
|
||||
"categories": ["code", "files"],
|
||||
},
|
||||
"search_code": {
|
||||
"description": "Search for patterns in codebase",
|
||||
"tokens": 130,
|
||||
"categories": ["code"],
|
||||
},
|
||||
"run_command": {
|
||||
"description": "Execute a shell command",
|
||||
"tokens": 140,
|
||||
"categories": ["code", "system"],
|
||||
},
|
||||
"create_calendar_event": {
|
||||
"description": "Create a new calendar event",
|
||||
"tokens": 180,
|
||||
"categories": ["calendar"],
|
||||
},
|
||||
"list_emails": {
|
||||
"description": "List recent emails",
|
||||
"tokens": 160,
|
||||
"categories": ["email"],
|
||||
},
|
||||
"send_email": {
|
||||
"description": "Send an email message",
|
||||
"tokens": 200,
|
||||
"categories": ["email"],
|
||||
},
|
||||
"web_search": {
|
||||
"description": "Search the web for information",
|
||||
"tokens": 140,
|
||||
"categories": ["research"],
|
||||
},
|
||||
"query_database": {
|
||||
"description": "Run a SQL query on the database",
|
||||
"tokens": 170,
|
||||
"categories": ["code", "data"],
|
||||
},
|
||||
"generate_chart": {
|
||||
"description": "Generate a chart from data",
|
||||
"tokens": 190,
|
||||
"categories": ["data", "visualization"],
|
||||
},
|
||||
}
|
||||
|
||||
def classify_intent(query):
|
||||
query_lower = query.lower()
|
||||
|
||||
intent_keywords = {
|
||||
"code": ["code", "function", "bug", "error", "file", "implement", "refactor", "debug", "test"],
|
||||
"calendar": ["meeting", "schedule", "calendar", "appointment", "event"],
|
||||
"email": ["email", "mail", "send", "inbox", "message"],
|
||||
"research": ["search", "find", "what is", "how does", "explain", "look up"],
|
||||
"data": ["data", "query", "database", "chart", "graph", "analytics", "sql"],
|
||||
}
|
||||
|
||||
scores = {}
|
||||
for intent, keywords in intent_keywords.items():
|
||||
score = sum(1 for kw in keywords if kw in query_lower)
|
||||
if score > 0:
|
||||
scores[intent] = score
|
||||
|
||||
if not scores:
|
||||
return ["code"]
|
||||
|
||||
max_score = max(scores.values())
|
||||
return [intent for intent, score in scores.items() if score >= max_score * 0.5]
|
||||
|
||||
def select_tools(query, token_budget=2000):
|
||||
intents = classify_intent(query)
|
||||
relevant = {}
|
||||
total_tokens = 0
|
||||
|
||||
for name, tool in TOOL_REGISTRY.items():
|
||||
if any(cat in intents for cat in tool["categories"]):
|
||||
if total_tokens + tool["tokens"] <= token_budget:
|
||||
relevant[name] = tool
|
||||
total_tokens += tool["tokens"]
|
||||
|
||||
return relevant, total_tokens
|
||||
```
|
||||
|
||||
### Step 6: Full Context Assembly Pipeline
|
||||
|
||||
Wire everything together. Given a query, dynamically assemble the optimal context.
|
||||
|
||||
```python
|
||||
class ContextEngine:
|
||||
def __init__(self, max_tokens=128000, generation_reserve=4000):
|
||||
self.budget = ContextBudget(max_tokens, generation_reserve)
|
||||
self.conversation = ConversationManager(max_history_tokens=5000)
|
||||
self.system_prompt = (
|
||||
"You are a helpful AI assistant. You have access to tools for "
|
||||
"code editing, file management, web search, and data analysis. "
|
||||
"Use the appropriate tools for each task. Be concise and accurate."
|
||||
)
|
||||
self.knowledge_base = [
|
||||
"Python 3.12 introduced type parameter syntax for generic classes using bracket notation.",
|
||||
"The project uses PostgreSQL 16 with pgvector for embedding storage.",
|
||||
"Authentication is handled by Supabase Auth with JWT tokens.",
|
||||
"The frontend is built with Next.js 15 using the App Router.",
|
||||
"API rate limits are set to 100 requests per minute per user.",
|
||||
"The deployment pipeline uses GitHub Actions with Docker multi-stage builds.",
|
||||
"Test coverage must be above 80% for all new modules.",
|
||||
"The codebase follows the repository pattern for data access.",
|
||||
]
|
||||
|
||||
def assemble(self, query):
|
||||
self.budget = ContextBudget(self.budget.max_tokens, self.budget.generation_reserve)
|
||||
|
||||
system_content, _ = self.budget.allocate("system_prompt", self.system_prompt, max_tokens=1000)
|
||||
|
||||
tools, tool_tokens = select_tools(query, token_budget=2000)
|
||||
tool_text = json.dumps(list(tools.keys()))
|
||||
tool_content, _ = self.budget.allocate("tools", tool_text, max_tokens=2000)
|
||||
|
||||
relevance = score_relevance(query, self.knowledge_base)
|
||||
threshold = 0.1
|
||||
relevant_docs = [
|
||||
doc for doc, score in zip(self.knowledge_base, relevance)
|
||||
if score >= threshold
|
||||
]
|
||||
|
||||
if relevant_docs:
|
||||
doc_scores = [s for s in relevance if s >= threshold]
|
||||
reordered = reorder_lost_in_middle(relevant_docs, doc_scores)
|
||||
doc_text = "\n".join(reordered)
|
||||
doc_content, _ = self.budget.allocate("retrieved_context", doc_text, max_tokens=3000)
|
||||
|
||||
history_text = self.conversation.get_context()
|
||||
if history_text.strip():
|
||||
history_content, _ = self.budget.allocate("conversation_history", history_text, max_tokens=5000)
|
||||
|
||||
query_content, _ = self.budget.allocate("user_query", query, max_tokens=500)
|
||||
|
||||
return self.budget
|
||||
|
||||
def chat(self, query):
|
||||
self.conversation.add_turn("user", query)
|
||||
budget = self.assemble(query)
|
||||
response = f"[Response to: {query[:50]}...]"
|
||||
self.conversation.add_turn("assistant", response)
|
||||
return budget
|
||||
|
||||
|
||||
def run_demo():
|
||||
print("=" * 60)
|
||||
print(" Context Engineering Pipeline Demo")
|
||||
print("=" * 60)
|
||||
|
||||
engine = ContextEngine(max_tokens=128000, generation_reserve=4000)
|
||||
|
||||
print("\n--- Query 1: Code task ---")
|
||||
budget = engine.chat("Fix the bug in the authentication module where JWT tokens expire too early")
|
||||
print(budget.report())
|
||||
|
||||
print("\n--- Query 2: Research task ---")
|
||||
budget = engine.chat("What is the best approach for implementing vector search in PostgreSQL?")
|
||||
print(budget.report())
|
||||
|
||||
print("\n--- Query 3: After conversation history builds up ---")
|
||||
for i in range(8):
|
||||
engine.conversation.add_turn("user", f"Follow-up question number {i+1} about the implementation details of the system")
|
||||
engine.conversation.add_turn("assistant", f"Here is the response to follow-up {i+1} with technical details about the architecture")
|
||||
|
||||
budget = engine.chat("Now implement the changes we discussed")
|
||||
print(budget.report())
|
||||
|
||||
print("\n--- Tool Selection Examples ---")
|
||||
test_queries = [
|
||||
"Fix the bug in auth.py",
|
||||
"Schedule a meeting with the team for Tuesday",
|
||||
"Show me the database query performance stats",
|
||||
"Search for best practices on error handling",
|
||||
]
|
||||
|
||||
for q in test_queries:
|
||||
tools, tokens = select_tools(q)
|
||||
intents = classify_intent(q)
|
||||
print(f"\n Query: {q}")
|
||||
print(f" Intents: {intents}")
|
||||
print(f" Tools: {list(tools.keys())} ({tokens} tokens)")
|
||||
|
||||
print("\n--- Lost-in-the-Middle Reordering ---")
|
||||
docs = ["Doc A (most relevant)", "Doc B (somewhat relevant)", "Doc C (least relevant)",
|
||||
"Doc D (relevant)", "Doc E (moderately relevant)"]
|
||||
scores = [0.95, 0.60, 0.20, 0.80, 0.50]
|
||||
reordered = reorder_lost_in_middle(docs, scores)
|
||||
print(f" Original order: {docs}")
|
||||
print(f" Scores: {scores}")
|
||||
print(f" Reordered: {reordered}")
|
||||
print(f" (Most relevant at start and end, least relevant in middle)")
|
||||
```
|
||||
|
||||
## Use It
|
||||
|
||||
### Claude Code's Context Strategy
|
||||
|
||||
Claude Code manages context with a layered approach. The system prompt includes behavioral rules and tool definitions (~6K tokens). When you open a file, its contents are injected as context. When you search, results are added. Old conversation turns are summarized. CLAUDE.md provides long-term memory that persists across sessions.
|
||||
|
||||
The key engineering decision: Claude Code does not dump your entire codebase into the context. It retrieves relevant files on demand. This is context engineering in practice.
|
||||
|
||||
### Cursor's Dynamic Context Loading
|
||||
|
||||
Cursor indexes your entire codebase into embeddings. When you type a query, it retrieves the most relevant files and code blocks using vector similarity. Only those pieces go into the context window. A 500K-line codebase is compressed into the 5-10 most relevant code blocks.
|
||||
|
||||
This is the pattern: embed everything, retrieve on demand, include only what matters.
|
||||
|
||||
### ChatGPT Memory
|
||||
|
||||
ChatGPT stores user preferences and facts as long-term memory. On each conversation start, relevant memories are retrieved and included in the system prompt. "The user prefers Python" costs 5 tokens but saves hundreds of tokens of repeated instructions across conversations.
|
||||
|
||||
### RAG as Context Engineering
|
||||
|
||||
Retrieval-Augmented Generation is context engineering formalized. Instead of stuffing knowledge into the model's weights (training) or the system prompt (static context), you retrieve relevant documents at query time and inject them into the context window. The entire RAG pipeline -- chunking, embedding, retrieval, reranking -- exists to solve one problem: putting the right information in the context window.
|
||||
|
||||
## Ship It
|
||||
|
||||
This lesson produces `outputs/prompt-context-optimizer.md` -- a reusable prompt that audits a context assembly strategy and recommends optimizations. Feed it your system prompt, tool count, average history length, and retrieval strategy, and it identifies token waste and suggests improvements.
|
||||
|
||||
It also produces `outputs/skill-context-engineering.md` -- a decision framework for designing context assembly pipelines based on task type, context window size, and latency budget.
|
||||
|
||||
## Exercises
|
||||
|
||||
1. Add a "token waste detector" to the ContextBudget class. It should flag components using more than 30% of the budget and suggest compression strategies specific to each component type (summarize history, prune tools, re-rank documents).
|
||||
|
||||
2. Implement semantic deduplication for retrieved context. If two retrieved documents are more than 80% similar (by word overlap or cosine similarity of their embeddings), keep only the higher-scored one. Measure how much token budget this recovers.
|
||||
|
||||
3. Build a "context replay" tool. Given a conversation transcript, replay it through the ContextEngine and visualize how the budget allocation changes turn by turn. Plot token usage per component over time. Identify the turn where context starts getting compressed.
|
||||
|
||||
4. Implement a priority-based tool selector. Instead of binary include/exclude, assign each tool a relevance score to the current query. Include tools in descending relevance order until the tool budget is exhausted. Compare task performance with 5, 10, 20, and 50 tools included.
|
||||
|
||||
5. Build a multi-strategy context compressor. Implement three compression strategies (truncation, summarization, extraction of key sentences) and benchmark them on a set of 20 documents. Measure the tradeoff between compression ratio and information retention (does the compressed version still contain the answer to the query?).
|
||||
|
||||
## Key Terms
|
||||
|
||||
| Term | What people say | What it actually means |
|
||||
|------|----------------|----------------------|
|
||||
| Context window | "How much the model can read" | The maximum number of tokens (input + output) the model processes in a single forward pass -- 128K for GPT-4o, 200K for Claude 3.5 |
|
||||
| Context engineering | "Advanced prompt engineering" | The discipline of deciding what goes into the context window, in what order, and at what priority -- encompasses retrieval, compression, tool selection, and memory management |
|
||||
| Lost-in-the-middle | "Models forget stuff in the middle" | Empirical finding that LLMs attend better to the beginning and end of context, with 10-20% accuracy drop for information placed in the middle |
|
||||
| Token budget | "How many tokens you have left" | An explicit allocation of context window capacity across components (system prompt, tools, history, retrieval, generation) with per-component limits |
|
||||
| Dynamic context | "Loading stuff on the fly" | Assembling the context window differently for each query based on intent classification, relevant tool selection, and retrieval results |
|
||||
| History summarization | "Compressing the conversation" | Replacing verbatim old conversation turns with a concise summary, reducing token cost while preserving key information |
|
||||
| Tool pruning | "Only including relevant tools" | Classifying query intent and only including tool definitions that match, reducing tool token cost by 60-80% |
|
||||
| Long-term memory | "Remembering across sessions" | Facts and preferences stored in a database and retrieved at session start -- CLAUDE.md, ChatGPT Memory, and similar systems |
|
||||
| Episodic memory | "Remembering specific past events" | Past interactions stored as embeddings and retrieved when the current query is similar to a past conversation |
|
||||
| Generation budget | "Room for the answer" | Tokens reserved for the model's output -- if the context fills the window completely, the model has no room to respond |
|
||||
|
||||
## Further Reading
|
||||
|
||||
- [Liu et al., 2023 -- "Lost in the Middle: How Language Models Use Long Contexts"](https://arxiv.org/abs/2307.03172) -- the definitive study on position-dependent attention, showing that models struggle with information in the middle of long contexts
|
||||
- [Anthropic's Contextual Retrieval blog post](https://www.anthropic.com/news/contextual-retrieval) -- how Anthropic approaches context-aware chunk retrieval, reducing retrieval failure by 49%
|
||||
- [Simon Willison's "Context Engineering"](https://simonwillison.net/2025/Jun/27/context-engineering/) -- the blog post that named the discipline and distinguished it from prompt engineering
|
||||
- [LangChain documentation on RAG](https://python.langchain.com/docs/tutorials/rag/) -- practical implementation of retrieval-augmented generation as a context engineering pattern
|
||||
- [Greg Kamradt's Needle in a Haystack test](https://github.com/gkamradt/LLMTest_NeedleInAHaystack) -- the benchmark that revealed position-dependent retrieval failures across all major models
|
||||
- [Andrej Karpathy on context length](https://karpathy.ai/) -- practical observations on how context window usage affects model performance in real applications
|
||||
@@ -0,0 +1,80 @@
|
||||
---
|
||||
name: prompt-context-optimizer
|
||||
description: Audit a context assembly strategy and recommend optimizations to reduce token waste and improve response quality
|
||||
phase: 11
|
||||
lesson: 05
|
||||
---
|
||||
|
||||
You are a context engineering consultant. I will describe how an LLM application assembles its context window. You will audit the strategy and recommend specific optimizations.
|
||||
|
||||
## Audit Protocol
|
||||
|
||||
### 1. Token Budget Analysis
|
||||
|
||||
Calculate the current token allocation:
|
||||
|
||||
- System prompt: how many tokens? Is there redundancy?
|
||||
- Tool definitions: how many tools, total tokens? Are all tools relevant to every query?
|
||||
- Retrieved context: how many chunks, total tokens? What is the retrieval quality?
|
||||
- Conversation history: how many turns kept verbatim? Is summarization used?
|
||||
- Few-shot examples: how many, total tokens? Are they static or dynamic?
|
||||
- Generation reserve: how many tokens? Is it sufficient for the expected output?
|
||||
- Total used vs available: what is the utilization percentage?
|
||||
|
||||
### 2. Waste Detection
|
||||
|
||||
Flag specific sources of token waste:
|
||||
|
||||
**Over-allocation**: components using more than 30% of the budget. A system prompt consuming 10,000 tokens is almost certainly too verbose.
|
||||
|
||||
**Static context**: tool definitions or few-shot examples that never change per query. If 80% of tools are irrelevant to most queries, you are wasting tool tokens 80% of the time.
|
||||
|
||||
**Stale history**: conversation turns from 20 messages ago that are irrelevant to the current query. Verbatim history is the biggest token waste in long conversations.
|
||||
|
||||
**Low-relevance retrieval**: retrieved chunks with low similarity scores that dilute the signal. Better to include 3 highly relevant chunks than 10 mediocre ones.
|
||||
|
||||
**Duplicate information**: the same fact appearing in the system prompt, retrieved context, and conversation history.
|
||||
|
||||
### 3. Ordering Analysis
|
||||
|
||||
Check for lost-in-the-middle problems:
|
||||
|
||||
- Is the most important information at the start and end of the context?
|
||||
- Are retrieved documents ordered by relevance, or by insertion order?
|
||||
- Is the user query near the end of the context (where attention is highest)?
|
||||
|
||||
### 4. Recommendations
|
||||
|
||||
For each waste source, provide a specific fix:
|
||||
|
||||
- **System prompt**: reduce to essential instructions, move examples to dynamic few-shot
|
||||
- **Tools**: implement intent-based tool selection, only include relevant tools per query
|
||||
- **Retrieval**: add reranking, raise similarity threshold, deduplicate chunks
|
||||
- **History**: summarize turns older than N, keep only the last K verbatim
|
||||
- **Ordering**: reorder by lost-in-the-middle pattern (important first and last)
|
||||
- **Generation**: ensure at least 2K tokens reserved, increase for long-form outputs
|
||||
|
||||
### 5. Impact Estimate
|
||||
|
||||
For each recommendation, estimate:
|
||||
|
||||
- Tokens saved per query
|
||||
- Expected quality impact (positive, neutral, or negative)
|
||||
- Implementation effort (minutes to hours)
|
||||
|
||||
## Input Format
|
||||
|
||||
Provide:
|
||||
- Context window size (e.g., 128K tokens)
|
||||
- Current token breakdown by component
|
||||
- Number of tools defined
|
||||
- Retrieval strategy (vector search, keyword, hybrid)
|
||||
- History management (keep all, truncate, summarize)
|
||||
- Any observed quality issues
|
||||
|
||||
## Output Format
|
||||
|
||||
1. **Budget Summary**: current allocation table with waste flags
|
||||
2. **Top 3 Waste Sources**: specific problems with estimated token cost
|
||||
3. **Recommendations**: ordered by impact/effort ratio
|
||||
4. **Projected Savings**: estimated tokens recovered and quality improvement
|
||||
@@ -0,0 +1,77 @@
|
||||
---
|
||||
name: skill-context-engineering
|
||||
description: Decision framework for designing context assembly pipelines based on task type, window size, and latency budget
|
||||
version: 1.0.0
|
||||
phase: 11
|
||||
lesson: 05
|
||||
tags: [context-engineering, context-window, rag, memory, tool-selection, lost-in-the-middle]
|
||||
---
|
||||
|
||||
# Context Engineering
|
||||
|
||||
When building an LLM application, apply this framework to design the context assembly pipeline.
|
||||
|
||||
## Core principles
|
||||
|
||||
1. **Context is scarce.** A 128K window sounds large but fills fast. Budget every component explicitly.
|
||||
2. **Attention is uneven.** Models attend more to the start and end. Put critical information there. The middle is the dead zone.
|
||||
3. **Dynamic beats static.** Different queries need different context. Assemble per query, not once at startup.
|
||||
4. **Less is more.** A curated 10K context outperforms a dumped 100K context. Signal-to-noise ratio matters more than total information.
|
||||
5. **Measure everything.** You cannot optimize what you do not measure. Count tokens per component on every request.
|
||||
|
||||
## Context budget guidelines
|
||||
|
||||
| Component | Typical Range | Priority | Compression Strategy |
|
||||
|-----------|-------------|----------|---------------------|
|
||||
| System prompt | 200-1,000 tokens | Fixed, high | Write tight, remove redundancy |
|
||||
| Tool definitions | 500-3,000 tokens | Dynamic, medium | Prune by query intent |
|
||||
| Retrieved context | 1,000-5,000 tokens | Dynamic, high | Rerank + threshold + deduplicate |
|
||||
| Conversation history | 500-5,000 tokens | Dynamic, medium | Summarize old turns |
|
||||
| Few-shot examples | 500-2,000 tokens | Dynamic, high | Select by task similarity |
|
||||
| User query | 50-500 tokens | Fixed, highest | N/A |
|
||||
| Generation reserve | 2,000-8,000 tokens | Fixed | Adjust by expected output length |
|
||||
|
||||
## When to use each memory type
|
||||
|
||||
**Short-term (conversation history):** The current session. Managed by summarization. Compress turns older than 5-10 exchanges. Keep the last 3-4 turns verbatim.
|
||||
|
||||
**Long-term (facts database):** Preferences and project facts that persist across sessions. Retrieve on session start. Examples: "user prefers Python", "project uses PostgreSQL", "team follows trunk-based development". Store in CLAUDE.md, a database, or a structured memory system.
|
||||
|
||||
**Episodic (past interactions):** Specific past conversations relevant to the current task. Store as embeddings, retrieve by similarity. "Last week we debugged a similar auth issue" is episodic memory.
|
||||
|
||||
## Tool selection strategy
|
||||
|
||||
Do not include all tools in every request. This wastes tokens and confuses the model.
|
||||
|
||||
1. Classify the query intent (code, email, calendar, research, data)
|
||||
2. Map intents to tool categories
|
||||
3. Include only matching tools
|
||||
4. If intent is ambiguous, include tools from the top 2 categories
|
||||
5. Always include a "general" tool (like web search) as fallback
|
||||
|
||||
Expected savings: 60-80% of tool definition tokens on queries with clear intent.
|
||||
|
||||
## Retrieval best practices
|
||||
|
||||
- **Rerank after retrieval.** Vector similarity is a rough filter. A reranker (cross-encoder or LLM-based) improves precision significantly.
|
||||
- **Set a relevance threshold.** Do not include chunks below 0.3 cosine similarity. They add noise.
|
||||
- **Deduplicate.** If two chunks share 80%+ content, keep only the higher-scored one.
|
||||
- **Apply lost-in-the-middle ordering.** Place the most relevant chunks first and last.
|
||||
- **Limit total retrieval tokens.** 3-5 highly relevant chunks beat 15 mediocre ones.
|
||||
|
||||
## History management
|
||||
|
||||
- Keep the last 3-4 turns verbatim (the model needs recent context)
|
||||
- Summarize older turns into a digest ("We discussed X, decided Y, and blocked on Z")
|
||||
- Drop system-generated turns that add no information (tool invocations with no user-facing content)
|
||||
- Trigger compression when history exceeds 30% of the available budget
|
||||
|
||||
## Red flags
|
||||
|
||||
- System prompt exceeds 2,000 tokens: probably includes information that should be dynamic
|
||||
- All tools included on every request: implement intent-based selection
|
||||
- No relevance filtering on retrieval: you are dumping noise into the window
|
||||
- History grows unbounded: summarization is not implemented
|
||||
- No generation reserve: the model truncates its responses
|
||||
- Same information in 3 places (system prompt, retrieved doc, history): deduplicate
|
||||
- Context utilization over 60%: you are leaving too little room for the model to "think"
|
||||
Reference in New Issue
Block a user