TOC extraction on a Judge — the TOC stage with no generative model (HAL-1370)

cmd/tocdump/main.go

@@ -17,6 +17,7 @@ import (
 	"bufio"
 	"context"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"os"
@@ -35,20 +36,23 @@ import (
 )
 
 type dump struct {
-	Doc      string         `json:"doc"`
-	Pages    int            `json:"pages"`
-	Seconds  float64        `json:"seconds"`
-	Requests int            `json:"requests"`
-	InTokens int            `json:"in_tokens"`
-	CostUSD  float64        `json:"cost_usd"`
-	Nodes    []tree.TOCNode `json:"nodes"`
-	Err      string         `json:"err,omitempty"`
+	Doc        string         `json:"doc"`
+	Pages      int            `json:"pages"`
+	Seconds    float64        `json:"seconds"`
+	Requests   int            `json:"requests"`
+	InTokens   int            `json:"in_tokens"`
+	CostUSD    float64        `json:"cost_usd"`
+	Generative int            `json:"generative_calls"`
+	Degraded   []string       `json:"degraded,omitempty"`
+	Nodes      []tree.TOCNode `json:"nodes"`
+	Err        string         `json:"err,omitempty"`
 }
 
 func main() {
 	docs := flag.String("docs", "", "directory of PDFs")
 	out := flag.String("out", "", "directory for per-document tree JSON")
 	noJudge := flag.Bool("no-judge", false, "run detection/verification on the chat model instead")
+	judgeOnly := flag.Bool("judge-only", false, "no chat model at all: any generative call fails loudly, so the tree is provably Judge-built")
 	minimal := flag.Bool("minimal", false, "TOCBuilder.MinimalContext: prefilter + truncation + two-stage scan")
 	// 300s, not the pipeline's 90s default: GLM's extraction call on the
 	// z.ai gateway routinely exceeds 90s on a 100+ page filing, and a
@@ -91,22 +95,27 @@ func main() {
 		}
 		d.Pages = len(pages)
 
-		b := &ingest.TOCBuilder{LLM: client, Judge: judge, LLMCallTimeout: *callTimeout, MinimalContext: *minimal}
+		llm := client
+		if *judgeOnly {
+			llm = refusingClient{}
+		}
+		b := &ingest.TOCBuilder{LLM: llm, Judge: judge, LLMCallTimeout: *callTimeout, MinimalContext: *minimal}
 		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
 		start := time.Now()
 		nodes, usage, err := b.Build(ctx, pages)
 		cancel()
 		d.Seconds = time.Since(start).Seconds()
 		d.Requests, d.InTokens, d.CostUSD = usage.LLMCalls, usage.InputTokens, usage.CostUSD
+		d.Generative, d.Degraded = usage.GenerativeCalls, usage.Degraded
 		if err != nil {
 			d.Err = err.Error()
 		}
 		d.Nodes = nodes
 		write(*out, d)
 
 		leaves := countLeaves(nodes)
-		fmt.Printf("  %-28s %4d pages  %6.1fs  %3d req  %3d leaves  $%.4f %s\n",
-			name, d.Pages, d.Seconds, d.Requests, leaves, d.CostUSD, d.Err)
+		fmt.Printf("  %-28s %4d pages  %6.1fs  %3d req  %d gen  %3d leaves  $%.4f %s %s\n",
+			name, d.Pages, d.Seconds, d.Requests, d.Generative, leaves, d.CostUSD, d.Err, strings.Join(d.Degraded, "; "))
 	}
 }
 
@@ -216,3 +225,13 @@ func dotEnv(key string) string {
 	}
 	return ""
 }
+
+// refusingClient is the chat model for -judge-only: every call fails,
+// so a tree can only have come from the Judge.
+type refusingClient struct{}
+
+func (refusingClient) Complete(context.Context, llmgate.Request) (*llmgate.Response, error) {
+	return nil, errors.New("tocdump -judge-only: a generative call was attempted")
+}
+
+func (refusingClient) CountTokens(_ context.Context, s string) (int, error) { return len(s) / 4, nil }

cmd/tocdump/titles.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Compare the leaf titles of two tocdump runs.
+
+    python cmd/tocdump/titles.py trees-after trees-jev
+
+Reports, per document and overall, what share of the reference run's leaf
+titles the candidate run also has (recall), and the reverse (precision),
+after normalising case, punctuation and whitespace. Titles are compared
+as sets; a title is matched if the normalised forms are equal or one is
+a prefix of the other (contents entries are often wrapped or truncated).
+"""
+import json, re, sys
+from pathlib import Path
+
+def norm(t):
+    t = t.lower()
+    t = re.sub(r"[^a-z0-9 ]+", " ", t)
+    return " ".join(t.split())
+
+def leaves(ns):
+    for n in ns:
+        if n.get("nodes"):
+            yield from leaves(n["nodes"])
+        else:
+            yield n
+
+def titles(path):
+    d = json.loads(Path(path).read_text())
+    return [norm(n["title"]) for n in leaves(d.get("nodes") or [])], d
+
+def matched(a, bs):
+    for b in bs:
+        if a == b or (len(a) >= 12 and (a.startswith(b) or b.startswith(a))):
+            return True
+    return False
+
+def main(ref_dir, cand_dir):
+    ref_dir, cand_dir = Path(ref_dir), Path(cand_dir)
+    rows = []
+    R = C = RM = CM = 0
+    for rp in sorted(ref_dir.glob("*.json")):
+        cp = cand_dir / rp.name
+        if not cp.exists():
+            continue
+        rt, rd = titles(rp); ct, cd = titles(cp)
+        if not rt or not ct:
+            rows.append((rp.stem, len(rt), len(ct), None, None, cd.get("seconds", 0), cd.get("generative_calls", "?")))
+            continue
+        rm = sum(1 for t in rt if matched(t, ct)); cm = sum(1 for t in ct if matched(t, rt))
+        R += len(rt); C += len(ct); RM += rm; CM += cm
+        rows.append((rp.stem, len(rt), len(ct), rm / len(rt), cm / len(ct), cd.get("seconds", 0), cd.get("generative_calls", "?")))
+    print(f"{'document':26} {'ref':>4} {'cand':>4} {'recall':>7} {'prec':>6} {'cand s':>7} gen")
+    for name, r, c, rec, prec, sec, gen in rows:
+        rec_s = f"{rec:.3f}" if rec is not None else "  n/a"
+        prec_s = f"{prec:.3f}" if prec is not None else "  n/a"
+        print(f"{name:26} {r:4d} {c:4d} {rec_s:>7} {prec_s:>6} {sec:7.1f} {gen}")
+    if R and C:
+        print(f"\noverall: recall {RM/R:.3f} ({RM}/{R})  precision {CM/C:.3f} ({CM}/{C})")
+
+if __name__ == "__main__":
+    main(*sys.argv[1:3])

docs/evaluations/2026-09-18-toc-extraction-on-a-judge.md

@@ -0,0 +1,111 @@
+# TOC extraction on a Judge — the TOC stage with no generative model
+
+**Date:** 2026-09-18
+**Harness:** [`cmd/tocdump -judge-only`](../../cmd/tocdump/main.go) (any chat-completion call fails loudly), [`cmd/tocdump/titles.py`](../../cmd/tocdump/titles.py) (leaf-title recall against the GLM trees), [`cmd/tocdump/coverage.py`](../../cmd/tocdump/coverage.py) (evidence-page gate)
+**Corpus:** FinanceBench, 21 10-K filings; gold evidence pages from `PatronusAI/financebench`
+**Issue:** HAL-1370
+**Question:** with detection and page resolution on Jev, extraction was the only generative call left in the TOC stage and all of its wall clock — 103–840 s per filing. Can code list the contents entries and Jev confirm them, with the same tree at the end?
+
+## Result: yes. The whole TOC stage runs with no chat model, in one to two minutes of API time per filing, for under a tenth of a cent, and lands every gold evidence page.
+
+| | GLM extraction + Jev resolver ([previous evaluation](2026-09-18-jev-page-resolver.md)) | **Jev only** |
+|---|---|---|
+| documents with a tree | 19 / 21 | **20 / 21** |
+| generative calls per document | 1 | **0** |
+| gold evidence pages inside a leaf | 44 / 44 | **45 / 45** |
+| median leaf span | 36 p | 37 p |
+| TOC stage per document | 103–840 s (extraction) + resolver | 12–143 s, mean 44 s, all of it Jev latency |
+| cost per document | $0.017–0.095 | **$0.0003–0.0015** |
+| requests per document | 3 | 3 |
+
+NIKE_2019_10K, which GLM never extracted (past 900 s twice), has a
+21-leaf tree. GENERALMILLS_2020_10K is still the one-page parse
+(HAL-1365); it is the only filing without a tree.
+
+## Are the trees the same trees?
+
+Leaf titles of the Jev-built tree against the GLM-built tree, per
+document, after normalising case and punctuation and allowing a prefix
+match for wrapped titles:
+
+| document | GLM leaves | Jev leaves | recall | precision |
+|---|---|---|---|---|
+| ADOBE, AMAZON, AMD, BESTBUY, COSTCO, NETFLIX, ORACLE, PEPSICO, WALMART | 20–24 | same | 1.000 | 1.000 |
+| BOEING | 23 | 24 | 1.000 | 1.000 |
+| LOCKHEEDMARTIN | 23 | 24 | 1.000 | 0.958 |
+| JOHNSON_JOHNSON | 36 | 36 | 0.972 | 0.972 |
+| KRAFTHEINZ | 63 | 65 | 0.968 | 0.954 |
+| AMCOR | 29 | 30 | 0.966 | 0.933 |
+| VERIZON | 24 | 23 | 0.958 | 1.000 |
+| MGMRESORTS | 23 | 24 | 0.957 | 0.917 |
+| 3M | 56 | 55 | 0.893 | 0.909 |
+| PFIZER | 40 | 37 | 0.875 | 0.946 |
+| INTEL | 26 | 25 | 0.846 | 0.880 |
+| **overall** | 543 | 543 | **0.961** | **0.965** |
+
+The GLM tree is the reference only because it existed first; where the
+two differ it is not always GLM that is right. The misses on PFIZER are
+unnumbered sub-headings GLM read out of the body (`Available
+Information`, `Forward-Looking Information…`) that the contents page
+does not list; the extra on LOCKHEEDMARTIN is a real entry. INTEL's
+contents page lists `Fundamentals of Our Business` and similar headings
+under a different layout that the parser splits into fewer entries; it
+is the one document under 0.9 and worth a look on its own.
+
+## What the parser had to learn
+
+The first corpus pass scored 0.866 recall with WALMART at 0.045. Every
+miss was a shape of contents text, not a Judge verdict:
+
+| shape | filing | handling |
+|---|---|---|
+| no space between the item number and its title: `Item 1Business 7 Item 1ARisk Factors 14` | WALMART | re-insert the space; a number directly after a label is never a page |
+| part label inline before its first item: `PART II Item 5.`, `PART I. Item 1.` | VERIZON, ORACLE, WALMART | the label becomes a container, the run restarts |
+| a title wrapped so its page lands mid-title: `… Issuer Purchases of 20 Equity Securities Item 6.` | VERIZON | the words before the next label are the previous entry's tail |
+| an item with no page at all: `Item 1. Business` alone on its line | VERIZON | an item-labelled run is an entry; the resolver finds its page |
+| parenthesised sub-entries: `… SCHEDULES 111 15(a)(1) Financial Statements 111` | PFIZER | a digit opens an entry |
+| the same line twice, once truncated in bold, from the parser's empty-leaf fold | 3M | exact duplicates dropped; a page-less prefix of a fuller entry dropped |
+| a column label fused to the first entry: `Page Part I Item 1…` | WALMART | leading furniture skipped |
+
+Second pass: recall 0.961, precision 0.965, WALMART 1.000.
+
+## Method
+
+- **Candidates from code.** `parseContentsEntries` tokenises each line
+  of the detected contents pages; an entry closes at a 1–3 digit token
+  followed by the end of the line or something that opens an entry (a
+  label such as Item or Note, a number, a capitalised word). Structure
+  comes from the numbering: Items under the enclosing Part, dotted
+  numbers by depth.
+- **One Judge request.** A `Noul` per entry — *is this a section a
+  reader could turn to, rather than the table's own title, a running
+  header, a column label, or a fragment?* — with the contents text
+  shared once (~3k characters) and the entry as per-question state.
+- **Pages from the resolver** (HAL-1367), which searches the body for
+  each title. The printed page numbers are not trusted for placement;
+  after resolution they calibrate an offset — the median of physical
+  minus printed over the resolved leaves — that places whatever the
+  resolver could not.
+- **Fallbacks.** Fewer than three confirmed entries declines to the
+  generative extractor unchanged. A failed Judge request also falls
+  back, recorded on `Usage.Degraded`. `Usage.GenerativeCalls` makes a
+  Judge-only build provable; `tocdump -judge-only` makes any generative
+  call an error.
+
+## What this means for the pipeline
+
+Detection, extraction, resolution: three requests, no generative call,
+roughly a cent per hundred filings. The wall clock is now the Judge
+API's latency in the hour of the run (the same documents took 5–12 s an
+hour earlier and 40–140 s in this pass), not the work. The remaining
+generative steps in ingest are the per-section summaries, HyDE
+questions and multi-axis summaries, none of which the page-based
+retrieval strategy needs.
+
+## Reproduce
+
+```bash
+go run ./cmd/tocdump -docs ~/.cache/vlbench/financebench -out /tmp/trees-jev -judge-only -minimal
+python cmd/tocdump/titles.py ~/.cache/vlbench/trees-after /tmp/trees-jev
+python cmd/tocdump/coverage.py ~/.cache/vlbench/trees-resolved /tmp/trees-jev
+```

pkg/ingest/toc_builder.go

@@ -137,6 +137,11 @@ type Usage struct {
 	CostUSD      float64
 	LLMCalls     int
 
+	// GenerativeCalls counts the chat-completion calls among LLMCalls.
+	// Zero on a Judge-only build; the number to watch when the goal is
+	// to have no generative call in the TOC stage at all.
+	GenerativeCalls int
+
 	// Degraded lists the Judge-path steps that could not complete and
 	// what Build did instead. Empty means every step ran as configured.
 	// A document built with a non-empty Degraded is not wrong, but it is
@@ -162,6 +167,7 @@ func (u *Usage) add(r *llmgate.Response) {
 	u.TotalTokens += r.Usage.TotalTokens
 	u.CostUSD += r.Usage.CostUSD
 	u.LLMCalls++
+	u.GenerativeCalls++
 }
 
 // Build runs the three-phase pipeline on pages and returns a
@@ -199,15 +205,34 @@ func (b *TOCBuilder) Build(ctx context.Context, pages []PageText) ([]tree.TOCNod
 	}
 
 	// Phase 2: extract.
+	//
+	// With a Judge and a contents page, code lists the entries and the
+	// Judge confirms them — one request, seconds. The generative
+	// extractor runs only when the contents did not parse into enough
+	// entries to trust, or when the Judge failed (recorded as a
+	// degradation, since it is minutes and a body window instead).
 	var nodes []tree.TOCNode
+	var printed map[string]int
 	var err error
-	if len(tocPages) > 0 {
-		nodes, err = b.extractFromTOCPages(ctx, pages, tocPages, &usage)
-	} else {
-		nodes, err = b.generateNoTOC(ctx, pages, &usage)
+	extracted := false
+	if len(tocPages) > 0 && b.Judge != nil {
+		var handled bool
+		nodes, printed, handled, err = b.extractFromTOCPagesJudge(ctx, pages, tocPages, &usage)
+		if err != nil {
+			log.Printf("toc: judge extraction failed, falling back to the generative extractor: %v", err)
+			usage.degrade("extraction", "generative extractor after a failed Judge request: "+err.Error())
+		}
+		extracted = handled && err == nil
 	}
-	if err != nil {
-		return nil, usage, err
+	if !extracted {
+		if len(tocPages) > 0 {
+			nodes, err = b.extractFromTOCPages(ctx, pages, tocPages, &usage)
+		} else {
+			nodes, err = b.generateNoTOC(ctx, pages, &usage)
+		}
+		if err != nil {
+			return nil, usage, err
+		}
 	}
 	if len(nodes) == 0 {
 		return nil, usage, nil
@@ -237,6 +262,12 @@ func (b *TOCBuilder) Build(ctx context.Context, pages []PageText) ([]tree.TOCNod
 		b.verifyTitlesConcurrent(ctx, nodes, pages, concurrency, &usage)
 	}
 
+	// Leaves the resolver could not place get their printed page plus
+	// the offset the resolved leaves agree on.
+	if n := calibrateFromPrinted(nodes, printed, lastPage(pages)); n > 0 {
+		log.Printf("toc: %d leaves placed from printed page numbers", n)
+	}
+
 	// Derive end pages from sibling order. Done last so verified
 	// start pages drive the derivation.
 	deriveEndPages(nodes, lastPage(pages))