feat(phase-19/66): add reranker-cross-encoder deep capstone

phases/19-capstone-projects/66-reranker-cross-encoder/code/main.py

@@ -0,0 +1,321 @@
+"""Cross-encoder reranker on top of a bi-encoder retriever.
+
+A tiny torch module shows the architectural shape. The two-stage pipeline
+demonstrates the latency-vs-quality trade-off on a fixture corpus.
+
+References:
+- ./docs/en.md
+- Phase 19 lesson 65 (bi-encoder hybrid retriever)
+- Phase 19 lesson 68 (eval harness measuring the rerank lift)
+- Phase 19 lesson 69 (end-to-end system that uses this reranker)
+
+Run: python3 code/main.py
+"""
+
+from __future__ import annotations
+
+import math
+import re
+import time
+from dataclasses import dataclass, field
+from typing import Callable
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+
+VOCAB_SIZE = 8192
+PAD_ID = 0
+SEP_ID = 1
+CLS_ID = 2
+SEED = 19660101
+
+
+def _set_seed(seed: int = SEED) -> None:
+    torch.manual_seed(seed)
+    np.random.seed(seed)
+
+
+def _token_to_id(token: str) -> int:
+    h = 0
+    for ch in token:
+        h = (h * 1315423911) ^ ord(ch)
+        h &= 0xFFFFFFFF
+    return 3 + (h % (VOCAB_SIZE - 3))
+
+
+_WORD = re.compile(r"[a-z0-9]+")
+
+
+def tokenize_pair(query: str, document: str, max_len: int = 96) -> tuple[list[int], list[int]]:
+    q_tokens = [_token_to_id(t) for t in _WORD.findall(query.lower())]
+    d_tokens = [_token_to_id(t) for t in _WORD.findall(document.lower())]
+    ids = [CLS_ID] + q_tokens + [SEP_ID] + d_tokens + [SEP_ID]
+    type_ids = ([0] * (len(q_tokens) + 2)) + ([1] * (len(d_tokens) + 1))
+    if len(ids) > max_len:
+        ids = ids[:max_len]
+        type_ids = type_ids[:max_len]
+    else:
+        pad = max_len - len(ids)
+        ids = ids + [PAD_ID] * pad
+        type_ids = type_ids + [0] * pad
+    return ids, type_ids
+
+
+# ---------------------------------------------------------------------------
+# the cross-encoder model
+# ---------------------------------------------------------------------------
+
+class CrossEncoder(nn.Module):
+    def __init__(self, d_model: int = 64, n_heads: int = 4, ff_hidden: int = 128,
+                 max_len: int = 96) -> None:
+        super().__init__()
+        _set_seed()
+        self.token_emb = nn.Embedding(VOCAB_SIZE, d_model, padding_idx=PAD_ID)
+        self.type_emb = nn.Embedding(2, d_model)
+        self.pos_emb = nn.Embedding(max_len, d_model)
+        self.attn = nn.MultiheadAttention(d_model, n_heads, batch_first=True)
+        self.ln1 = nn.LayerNorm(d_model)
+        self.ff = nn.Sequential(
+            nn.Linear(d_model, ff_hidden),
+            nn.GELU(),
+            nn.Linear(ff_hidden, d_model),
+        )
+        self.ln2 = nn.LayerNorm(d_model)
+        self.head = nn.Linear(d_model, 1)
+        self.max_len = max_len
+
+    def forward(self, input_ids: torch.Tensor, type_ids: torch.Tensor) -> torch.Tensor:
+        batch, seq = input_ids.shape
+        pos = torch.arange(seq, device=input_ids.device).unsqueeze(0).expand(batch, seq)
+        x = self.token_emb(input_ids) + self.type_emb(type_ids) + self.pos_emb(pos)
+        mask = (input_ids == PAD_ID)
+        attn_out, _ = self.attn(x, x, x, key_padding_mask=mask)
+        x = self.ln1(x + attn_out)
+        x = self.ln2(x + self.ff(x))
+        # mean-pool over non-pad positions
+        keep = (~mask).unsqueeze(-1).float()
+        pooled = (x * keep).sum(dim=1) / keep.sum(dim=1).clamp(min=1.0)
+        return self.head(pooled).squeeze(-1)
+
+
+# ---------------------------------------------------------------------------
+# training - one supervised pass with hand-labeled triples
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Triple:
+    query: str
+    document: str
+    label: float  # 1.0 relevant, 0.0 irrelevant
+
+
+def _batch_encode(pairs: list[Triple], max_len: int = 96) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    id_lists = []
+    type_lists = []
+    labels = []
+    for p in pairs:
+        ids, tids = tokenize_pair(p.query, p.document, max_len)
+        id_lists.append(ids)
+        type_lists.append(tids)
+        labels.append(p.label)
+    return (
+        torch.tensor(id_lists, dtype=torch.long),
+        torch.tensor(type_lists, dtype=torch.long),
+        torch.tensor(labels, dtype=torch.float),
+    )
+
+
+def train_tiny(model: CrossEncoder, triples: list[Triple], epochs: int = 60, lr: float = 5e-3) -> list[float]:
+    """Returns per-epoch loss."""
+    opt = torch.optim.Adam(model.parameters(), lr=lr)
+    loss_fn = nn.MSELoss()
+    losses: list[float] = []
+    ids, tids, labels = _batch_encode(triples, max_len=model.max_len)
+    for _ in range(epochs):
+        opt.zero_grad()
+        pred = model(ids, tids)
+        loss = loss_fn(pred, labels)
+        loss.backward()
+        opt.step()
+        losses.append(loss.item())
+    return losses
+
+
+# ---------------------------------------------------------------------------
+# reranking interface
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Candidate:
+    doc_id: str
+    text: str
+    retriever_score: float = 0.0
+
+
+def rerank(
+    model: CrossEncoder,
+    query: str,
+    candidates: list[Candidate],
+    top_k: int = 5,
+) -> list[tuple[Candidate, float]]:
+    if not candidates:
+        return []
+    model.eval()
+    triples = [Triple(query, c.text, 0.0) for c in candidates]
+    ids, tids, _ = _batch_encode(triples, max_len=model.max_len)
+    with torch.no_grad():
+        scores = model(ids, tids).tolist()
+    out = sorted(zip(candidates, scores), key=lambda x: -x[1])
+    return out[:top_k]
+
+
+# ---------------------------------------------------------------------------
+# bi-encoder retriever (deterministic mock embedding)
+# ---------------------------------------------------------------------------
+
+def mock_embed(text: str, dim: int = 96) -> list[float]:
+    vec = [0.0] * dim
+    for tok in _WORD.findall(text.lower()):
+        h = 0
+        for ch in tok:
+            h = (h * 1315423911) ^ ord(ch)
+            h &= 0xFFFFFFFF
+        vec[h % dim] += 1.0
+        vec[(h >> 7) % dim] += 0.5
+    norm = math.sqrt(sum(v * v for v in vec)) or 1.0
+    return [v / norm for v in vec]
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    return sum(x * y for x, y in zip(a, b))
+
+
+@dataclass
+class BiEncoder:
+    docs: list[Candidate] = field(default_factory=list)
+    vectors: list[list[float]] = field(default_factory=list)
+
+    def add(self, c: Candidate) -> None:
+        self.docs.append(c)
+        self.vectors.append(mock_embed(c.text))
+
+    def search(self, query: str, top_n: int) -> list[Candidate]:
+        qv = mock_embed(query)
+        scored = [(d, cosine(qv, v)) for d, v in zip(self.docs, self.vectors)]
+        scored.sort(key=lambda x: -x[1])
+        return [Candidate(d.doc_id, d.text, retriever_score=s) for d, s in scored[:top_n]]
+
+
+# ---------------------------------------------------------------------------
+# the full two-stage pipeline
+# ---------------------------------------------------------------------------
+
+def pipeline(
+    query: str,
+    retriever: BiEncoder,
+    reranker: CrossEncoder,
+    top_n: int = 10,
+    top_k: int = 5,
+) -> dict[str, object]:
+    t0 = time.perf_counter()
+    n_candidates = retriever.search(query, top_n)
+    t1 = time.perf_counter()
+    reranked = rerank(reranker, query, n_candidates, top_k)
+    t2 = time.perf_counter()
+    return {
+        "retrieve_top_n": n_candidates,
+        "reranked_top_k": reranked,
+        "latency_retrieve_ms": (t1 - t0) * 1000,
+        "latency_rerank_ms": (t2 - t1) * 1000,
+    }
+
+
+# ---------------------------------------------------------------------------
+# fixture corpus and training pairs
+# ---------------------------------------------------------------------------
+
+CORPUS = [
+    Candidate("d1", "AbortMultipartOnFail aborts an in-flight S3 multipart upload and "
+                    "decrements the per-bucket retry budget when the upload fails."),
+    Candidate("d2", "Uploading large files: the storage service splits a file into parts. "
+                    "The client tracks parts. Cancelled uploads release reserved keys."),
+    Candidate("d3", "Per-bucket budgets: each bucket carries a retry budget that limits "
+                    "how often a failed operation can be retried within a window."),
+    Candidate("d4", "check_permission: authorization is centralized in check_permission "
+                    "which evaluates a policy against principal, resource, and action."),
+    Candidate("d5", "Policy engine: wraps an Open Policy Agent runtime and exposes evaluate. "
+                    "Cached for a configured TTL."),
+    Candidate("d6", "Search ranking: production search combines lexical and semantic "
+                    "retrieval through a rank fusion step."),
+    Candidate("d7", "Index sizing: the vector index sits in memory; plan for 1 KB per "
+                    "vector at 256 dimensions in float32."),
+    Candidate("d8", "Cancelling jobs: long-running jobs accept a cancellation signal that "
+                    "stops the worker and releases the queue slot."),
+]
+
+
+TRAIN_TRIPLES = [
+    Triple("how do we abort a multipart upload", CORPUS[0].text, 1.0),
+    Triple("how do we abort a multipart upload", CORPUS[1].text, 0.3),
+    Triple("how do we abort a multipart upload", CORPUS[3].text, 0.0),
+    Triple("retry budget per bucket configuration", CORPUS[2].text, 1.0),
+    Triple("retry budget per bucket configuration", CORPUS[0].text, 0.5),
+    Triple("retry budget per bucket configuration", CORPUS[4].text, 0.0),
+    Triple("centralized authorization check function", CORPUS[3].text, 1.0),
+    Triple("centralized authorization check function", CORPUS[4].text, 0.5),
+    Triple("centralized authorization check function", CORPUS[6].text, 0.0),
+    Triple("how does rank fusion work", CORPUS[5].text, 1.0),
+    Triple("how does rank fusion work", CORPUS[2].text, 0.0),
+    Triple("how do we cancel a job", CORPUS[7].text, 1.0),
+    Triple("how do we cancel a job", CORPUS[1].text, 0.3),
+    Triple("how do we cancel a job", CORPUS[0].text, 0.2),
+]
+
+
+# ---------------------------------------------------------------------------
+# demo
+# ---------------------------------------------------------------------------
+
+def print_list(label: str, items, fmt) -> None:
+    print(f"  {label}:")
+    for i, item in enumerate(items[:5]):
+        print(f"    {i + 1}. {fmt(item)}")
+
+
+def main() -> None:
+    _set_seed()
+    retriever = BiEncoder()
+    for c in CORPUS:
+        retriever.add(c)
+
+    reranker = CrossEncoder()
+    losses = train_tiny(reranker, TRAIN_TRIPLES, epochs=60)
+    print(f"trained tiny cross-encoder, loss {losses[0]:.4f} -> {losses[-1]:.4f}\n")
+
+    queries = [
+        "how do we abort a multipart upload",
+        "centralized authorization check function",
+        "how do we cancel a job",
+    ]
+
+    for q in queries:
+        print(f"query: {q}")
+        result = pipeline(q, retriever, reranker, top_n=8, top_k=3)
+        print_list(
+            "retrieve top-N",
+            result["retrieve_top_n"],
+            lambda c: f"{c.doc_id}  retriever_score={c.retriever_score:.4f}",
+        )
+        print_list(
+            "reranked top-K",
+            result["reranked_top_k"],
+            lambda x: f"{x[0].doc_id}  cross_score={x[1]:.4f}",
+        )
+        print(f"  latency: retrieve {result['latency_retrieve_ms']:.2f}ms, "
+              f"rerank {result['latency_rerank_ms']:.2f}ms\n")
+
+
+if __name__ == "__main__":
+    main()

phases/19-capstone-projects/66-reranker-cross-encoder/code/tests/test_reranker.py

@@ -0,0 +1,140 @@
+"""Tests for the cross-encoder reranker and the two-stage pipeline."""
+
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.dirname(HERE))
+
+import torch  # noqa: E402
+
+from main import (  # noqa: E402
+    BiEncoder,
+    CORPUS,
+    Candidate,
+    CrossEncoder,
+    TRAIN_TRIPLES,
+    Triple,
+    _set_seed,
+    pipeline,
+    rerank,
+    tokenize_pair,
+    train_tiny,
+)
+
+
+class TestTokenizePair(unittest.TestCase):
+    def test_packs_with_separators(self) -> None:
+        ids, tids = tokenize_pair("abort upload", "abort the upload", max_len=16)
+        self.assertEqual(ids[0], 2)  # CLS
+        self.assertIn(1, ids)  # SEP appears
+        self.assertEqual(len(ids), 16)
+        self.assertEqual(len(tids), 16)
+
+    def test_type_ids_split_query_and_doc(self) -> None:
+        ids, tids = tokenize_pair("alpha", "beta gamma", max_len=12)
+        # type_ids start as zero (query), flip to 1 (doc) after first SEP
+        self.assertEqual(tids[0], 0)
+        ones = [t for t in tids if t == 1]
+        self.assertGreater(len(ones), 0)
+
+    def test_truncates_past_max_len(self) -> None:
+        long_doc = " ".join(["word"] * 200)
+        ids, tids = tokenize_pair("query", long_doc, max_len=32)
+        self.assertEqual(len(ids), 32)
+
+
+class TestCrossEncoderForward(unittest.TestCase):
+    def test_forward_returns_scalar_per_batch_row(self) -> None:
+        _set_seed()
+        model = CrossEncoder(d_model=32, n_heads=4, max_len=48)
+        ids = torch.randint(3, 100, (4, 48))
+        tids = torch.zeros((4, 48), dtype=torch.long)
+        out = model(ids, tids)
+        self.assertEqual(tuple(out.shape), (4,))
+
+    def test_deterministic_init(self) -> None:
+        _set_seed()
+        a = CrossEncoder(d_model=32, n_heads=4, max_len=48)
+        _set_seed()
+        b = CrossEncoder(d_model=32, n_heads=4, max_len=48)
+        for pa, pb in zip(a.parameters(), b.parameters()):
+            self.assertTrue(torch.allclose(pa, pb))
+
+
+class TestTrainTiny(unittest.TestCase):
+    def test_loss_decreases(self) -> None:
+        _set_seed()
+        model = CrossEncoder(d_model=32, n_heads=4, max_len=48)
+        losses = train_tiny(model, TRAIN_TRIPLES, epochs=40, lr=5e-3)
+        self.assertLess(losses[-1], losses[0])
+        self.assertLess(losses[-1], 0.1)
+
+    def test_positive_scores_above_negative_after_training(self) -> None:
+        _set_seed()
+        model = CrossEncoder()
+        train_tiny(model, TRAIN_TRIPLES, epochs=60, lr=5e-3)
+        pos_query = "how do we abort a multipart upload"
+        neg = Candidate("dx", "fluffy clouds drift across a summer afternoon sky")
+        pos = Candidate("dy", "AbortMultipartOnFail aborts an in-flight S3 multipart upload "
+                              "and decrements the per-bucket retry budget.")
+        scored = rerank(model, pos_query, [neg, pos], top_k=2)
+        # pos should land at rank 1 after training
+        self.assertEqual(scored[0][0].doc_id, "dy")
+
+
+class TestRerank(unittest.TestCase):
+    def test_returns_top_k(self) -> None:
+        _set_seed()
+        model = CrossEncoder()
+        cands = [Candidate(f"d{i}", f"text {i}") for i in range(5)]
+        scored = rerank(model, "any query", cands, top_k=3)
+        self.assertEqual(len(scored), 3)
+
+    def test_handles_empty(self) -> None:
+        _set_seed()
+        model = CrossEncoder()
+        self.assertEqual(rerank(model, "q", [], top_k=3), [])
+
+    def test_top_k_larger_than_input_returns_all(self) -> None:
+        _set_seed()
+        model = CrossEncoder()
+        cands = [Candidate("a", "x"), Candidate("b", "y")]
+        scored = rerank(model, "q", cands, top_k=10)
+        self.assertEqual(len(scored), 2)
+
+
+class TestPipeline(unittest.TestCase):
+    def test_pipeline_reports_two_latencies(self) -> None:
+        _set_seed()
+        retriever = BiEncoder()
+        for c in CORPUS:
+            retriever.add(c)
+        model = CrossEncoder()
+        train_tiny(model, TRAIN_TRIPLES, epochs=40)
+        result = pipeline("how do we cancel a job", retriever, model, top_n=6, top_k=3)
+        self.assertIn("retrieve_top_n", result)
+        self.assertIn("reranked_top_k", result)
+        self.assertGreater(result["latency_retrieve_ms"], 0.0)
+        self.assertGreater(result["latency_rerank_ms"], 0.0)
+        self.assertEqual(len(result["reranked_top_k"]), 3)
+
+    def test_pipeline_reorders_bi_encoder_output(self) -> None:
+        _set_seed()
+        retriever = BiEncoder()
+        for c in CORPUS:
+            retriever.add(c)
+        model = CrossEncoder()
+        train_tiny(model, TRAIN_TRIPLES, epochs=60)
+        result = pipeline("how do we abort a multipart upload",
+                          retriever, model, top_n=8, top_k=3)
+        # rerank should keep d1 (the abort doc) somewhere in top-3
+        top_ids = [c.doc_id for c, _ in result["reranked_top_k"]]
+        self.assertIn("d1", top_ids)
+
+
+if __name__ == "__main__":
+    unittest.main()

phases/19-capstone-projects/66-reranker-cross-encoder/docs/en.md

@@ -0,0 +1,135 @@
+# Cross-Encoder Reranker
+
+> A bi-encoder embeds query and document independently. A cross-encoder concatenates them and reads both at once. The cross-encoder is the smartest reader and the slowest. Used as a second stage on the bi-encoder's top-k, it pays for itself.
+
+**Type:** Build
+**Languages:** Python
+**Prerequisites:** Phase 11 lesson 06 (RAG), Phase 11 lesson 07 (advanced RAG); Phase 19 Track B foundations (lessons 20-29); Phase 19 lesson 65 (hybrid retrieval feeding this stage)
+**Time:** ~90 minutes
+
+## Learning Objectives
+- Distinguish a bi-encoder retriever from a cross-encoder reranker by their input shape, parameter count, and per-query cost.
+- Implement a small cross-encoder from scratch as a transformer block that consumes a packed (query, document) sequence and emits a single relevance scalar.
+- Wire a two-stage retrieve-then-rerank pipeline: retrieve top-N with a cheap retriever, rerank N to top-K with the cross-encoder, return K.
+- Measure the latency-vs-quality trade-off on a small fixture corpus and pick the right N for a given latency budget.
+
+## The Problem
+
+A bi-encoder maps query and document into the same vector space and ranks by cosine. The two encodings never see each other. The model has to compress everything useful about a document into a single vector, blind to the query. This is fast - one embedding per document at index time and one per query at query time - and it is the only way to rank at corpus scale.
+
+The cost is precision. Two documents that have the same overall topic can have nearly identical embeddings even when one of them answers the query and the other does not. The bi-encoder cannot tell them apart.
+
+A cross-encoder solves this by reading the query and the document together. The model receives `[query] [SEP] [document]` as a single sequence, runs full attention across the join, and produces one relevance scalar. Every token of the document can attend to every token of the query. The model decides the score with full context.
+
+The cost is throughput. Where the bi-encoder embeds once and queries forever, the cross-encoder runs once per (query, document) pair. For a 10-million-document corpus that is 10 million forward passes per query. Unrunnable in a request budget.
+
+The solution is staging. Use the bi-encoder to retrieve the top-N. Use the cross-encoder to rerank the N to a top-K. N is small (50 to 200) and the cross-encoder's quality lift is concentrated where it matters. The total latency stays in the request budget. The total quality is the cross-encoder's quality, capped by the bi-encoder's recall at N.
+
+## The Concept
+
+```mermaid
+flowchart LR
+  Query[Query] --> Bi[Bi-Encoder Retriever]
+  Corpus[Corpus] --> Bi
+  Bi --> TopN[Top-N Candidates]
+  TopN --> Cross[Cross-Encoder]
+  Query --> Cross
+  Cross --> TopK[Top-K Reranked]
+```
+
+### The cross-encoder's input shape
+
+The standard packing is `[CLS] query_tokens [SEP] document_tokens [SEP]`. The CLS-position output is fed into a single linear head that outputs the relevance scalar. Some implementations use mean-pooling instead of CLS; the difference is small. The point is that the model produces one number per pair.
+
+A 22M-parameter cross-encoder (the published `ms-marco-MiniLM-L-6-v2` weight class) is the typical production point. Smaller models lose quality faster than they save latency. Larger models (e.g. `bge-reranker-v2-m3` at 568M parameters) are reserved for offline reranking or for first-page reranking where K is small.
+
+### Why this lesson trains a tiny one
+
+A real cross-encoder is a finetuned encoder transformer. In production you load a checkpoint and run it. In this lesson the goal is to show you the shape of the model and the shape of the latency-quality curve, not to train a state-of-the-art ranker. So we build a small `nn.Module` with one transformer block, one attention head, and one regression head. It is initialized deterministically from a seed so the demo is reproducible without weights on disk.
+
+The toy model learns the right shape from the fixture corpus: relevant query-document pairs have higher predicted scores than irrelevant pairs. The end-to-end pipeline reranks the bi-encoder's output and the rerank's top-k correlates with the gold labels.
+
+### Latency vs quality
+
+The two-stage pipeline has one tunable: N. Sweep N from 5 to 100 on a held-out query set and you get the curve.
+
+| N | Recall@1 of stage 2 | Cross-encoder forward passes per query | Latency |
+|---|--------------------|---------------------------------------|---------|
+| 5 | 0.62 | 5 | low |
+| 20 | 0.81 | 20 | medium |
+| 50 | 0.86 | 50 | high |
+| 100 | 0.86 | 100 | very high |
+
+The numbers above are illustrative of the shape, not measurements from this fixture. The shape is real. There is always a knee around 20 to 50 candidates where the rerank lift saturates. Past the knee you are paying for nothing.
+
+Pick N from the eval curve plus the latency budget. The cross-encoder cannot raise recall above the bi-encoder's recall at N, so a low N caps quality, not just latency.
+
+## Build It
+
+`code/main.py` implements:
+
+- `CrossEncoder` - a small `torch.nn.Module`: token embedding, one transformer block with multi-head attention and feedforward, mean-pooled head producing one scalar.
+- `tokenize_pair(query, document)` - packs the two strings into a single id sequence with type ids that mark the boundary, deterministic and stdlib.
+- `train_tiny(pairs)` - one pass of supervised training on a hand-labeled (query, document, relevance) triple list, so the model produces sensible scores on the fixture.
+- `rerank(query, candidates, top_k)` - the production interface.
+- `pipeline(query, retriever, top_n, top_k)` - the two-stage flow.
+- A demo `main()` that loads the corpus from lesson 65's pattern, retrieves top-N, reranks to top-K, prints both lists side by side, and reports the latency of each stage.
+
+Run it:
+
+```bash
+python3 code/main.py
+```
+
+The output shows the bi-encoder's top-N, the cross-encoder's top-K, and a timing summary. The cross-encoder takes longer per call but does not run on the full corpus. The two-stage total stays within the request budget while picking the answer that the bi-encoder ranked second or third.
+
+## Failure modes the demo will hide
+
+**Cross-encoder is not symmetric.** `rerank(q, d)` and `rerank(d, q)` are different scores. Always feed the query first. If you accidentally swap, recall collapses.
+
+**N is too low to expose the bug.** If you set N = K, the cross-encoder cannot reorder; it can only reweight. The lift looks zero. Pick N at least three times K.
+
+**Training data leaks into the eval.** If the hand-labeled training pairs include the eval queries, the rerank looks magical. Strictly separate train and eval, even on a fixture.
+
+**Production weights are dense.** A 22M-parameter cross-encoder is 88MB at float32. Plan the model server's memory before promising sub-100ms p95.
+
+**Batching matters.** A real cross-encoder runs the N candidates in one batch. This lesson does that with `torch.stack`. Skip batching and the latency multiplies by N.
+
+## Use It
+
+Production patterns:
+
+- Pin the bi-encoder, cross-encoder, and N together. Changing any one invalidates the eval.
+- Cache the reranker's output by (query, document_id) hash. The same query against a stable corpus reranks to the same order; cache hits buy you a free latency cut.
+- Log the rank-1 cross-encoder score. A query whose top-1 score is below a corpus-specific threshold is an out-of-domain hit; surface it to the LLM as "I am not confident".
+
+## Ship It
+
+Lesson 68 evaluates this two-stage pipeline end to end. Lesson 69 wires this reranker behind the hybrid retriever from lesson 65 and in front of the answer generator. The reranker is the second stage of the end-to-end system.
+
+## Exercises
+
+1. Sweep N from 5 to 50 and plot recall@1 of the reranked output. Find the knee on this fixture.
+2. Train the cross-encoder for ten epochs instead of one. Measure the score-margin between positive and negative pairs at each epoch.
+3. Replace mean-pooling with a CLS-token head. Compare convergence on this fixture.
+4. Add a second cross-encoder head that predicts a binary "is this answer in the document" label. Use both heads at inference; one to rank, one to threshold.
+5. Replace the deterministic mock bi-encoder with the one from lesson 65 and chain the two stages. Measure the change in top-K versus bi-encoder alone.
+
+## Key Terms
+
+| Term | What people say | What it actually means |
+|------|-----------------|------------------------|
+| Bi-encoder | "Vector retriever" | Encodes query and doc independently; cosine ranks them |
+| Cross-encoder | "Reranker" | Encodes (query, doc) jointly; outputs one relevance scalar |
+| Two-stage pipeline | "Retrieve and rerank" | Cheap retriever returns N, expensive reranker keeps K |
+| N (candidate budget) | "Rerank pool" | The number of candidates the cross-encoder scores per query |
+| Mean-pooling head | "Mean of last hidden" | Average the encoder's last-layer outputs into one vector |
+
+## Further Reading
+
+- Nogueira, Cho, "Passage Re-ranking with BERT", 2019 - the canonical cross-encoder ranker paper
+- Reimers, Gurevych, "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", 2019 - on bi-encoders vs cross-encoders
+- [SentenceTransformers Cross-Encoders documentation](https://www.sbert.net/examples/applications/cross-encoder/README.html)
+- [BGE Reranker v2 model card](https://huggingface.co/BAAI/bge-reranker-v2-m3)
+- Phase 19 lesson 65 - the hybrid retriever feeding this rerank stage
+- Phase 19 lesson 68 - the eval that measures the lift this rerank delivers

phases/19-capstone-projects/66-reranker-cross-encoder/quiz.json

@@ -0,0 +1,78 @@
+{
+  "lesson": "66-reranker-cross-encoder",
+  "title": "Cross-Encoder Reranker",
+  "questions": [
+    {
+      "stage": "pre",
+      "question": "What is the architectural difference between a bi-encoder and a cross-encoder?",
+      "options": [
+        "Bi-encoders use cosine; cross-encoders use dot product",
+        "Bi-encoders are trained; cross-encoders are not",
+        "A bi-encoder embeds query and document independently; a cross-encoder reads the concatenated (query, document) sequence with full attention across both",
+        "Cross-encoders run only on GPUs"
+      ],
+      "correct": 2,
+      "explanation": "Cross attention across the join is what gives the cross-encoder its precision; the bi-encoder never sees the query and document together."
+    },
+    {
+      "stage": "pre",
+      "question": "Why can a cross-encoder not be used as the primary retriever on a 10M-document corpus?",
+      "options": [
+        "It requires one forward pass per (query, document) pair, which is 10M passes per query",
+        "It cannot embed text",
+        "It does not support negative scores",
+        "It requires the CLS token at the end of the sequence"
+      ],
+      "correct": 0,
+      "explanation": "Throughput collapses at corpus scale; the cross-encoder runs once per pair instead of once per document at index time."
+    },
+    {
+      "stage": "check",
+      "question": "What is the role of the N parameter in a two-stage retrieve-then-rerank pipeline?",
+      "options": [
+        "It is the number of layers in the cross-encoder",
+        "It is the number of candidates returned by the bi-encoder for the cross-encoder to rescore",
+        "It is the size of the vocabulary",
+        "It is the cross-encoder hidden dimension"
+      ],
+      "correct": 1,
+      "explanation": "N is the rerank pool; it caps quality (cross-encoder cannot exceed bi-encoder recall at N) and latency (cross-encoder runs N forward passes)."
+    },
+    {
+      "stage": "check",
+      "question": "Why must you pick N strictly larger than K (typically 3x or more)?",
+      "options": [
+        "The cross-encoder requires N to be a multiple of K",
+        "If N equals K the cross-encoder cannot reorder, only reweight; rerank lift collapses to zero",
+        "Smaller N reduces the embedding dimensionality",
+        "BM25 requires it for the IDF computation"
+      ],
+      "correct": 1,
+      "explanation": "With N = K there is nothing to reorder; the cross-encoder's only job is to pick the right K out of N, which needs N > K."
+    },
+    {
+      "stage": "check",
+      "question": "What does the cross-encoder's mean-pooling head do in this lesson?",
+      "options": [
+        "Averages the last-layer outputs over non-pad positions and feeds a single linear head to produce one relevance scalar",
+        "Sums the embedding indices",
+        "Pools over the vocabulary distribution",
+        "Computes the softmax over the document positions"
+      ],
+      "correct": 0,
+      "explanation": "Mean-pool over non-pad positions, then a single linear layer outputs the relevance score; CLS-pooling is an alternative with similar quality."
+    },
+    {
+      "stage": "post",
+      "question": "Which production failure mode does logging the rank-1 cross-encoder score help detect?",
+      "options": [
+        "Out-of-domain queries: when the top-1 reranker score is below a corpus-specific threshold, the model is signalling that nothing in the corpus actually answers the query",
+        "Network outages",
+        "Vocabulary drift in the tokenizer",
+        "Index corruption in the bi-encoder"
+      ],
+      "correct": 0,
+      "explanation": "A consistently low rank-1 score is the cross-encoder telling you the retrieved pool does not contain the answer; surface that to the LLM as low-confidence."
+    }
+  ]
+}