A failed Judge request keeps extraction's pages; wire the Judge into production ingest (HAL-1369)

cmd/engine/main.go

@@ -19,6 +19,8 @@ import (
 	"time"
 
 	"github.com/hallelx2/llmgate"
+	"github.com/hallelx2/llmgate/judge/typesafe"
+	"github.com/hallelx2/llmgate/middleware/retry"
 	"github.com/hallelx2/llmgate/pricing"
 	"github.com/hallelx2/llmgate/provider/anthropic"
 	"github.com/hallelx2/llmgate/provider/gemini"
@@ -203,10 +205,22 @@ func run() error {
 		)
 	}
 
+	judge, err := buildJudge(cfg.LLM.Judge)
+	if err != nil {
+		logger.Error("judge: config invalid", "err", err)
+		os.Exit(1)
+	}
+	if judge != nil {
+		logger.Info("judge: typesafe enabled — contents-page detection and page resolution run on the Judge")
+	} else {
+		logger.Warn("judge: none configured — TOC judgements run on the generative driver, one call per page (set TYPESAFE_API_KEY)")
+	}
 	pipeline := ingest.NewPipeline(ingest.Pipeline{
 		DB:                     pool,
 		Storage:                store,
 		LLM:                    llmClient,
+		Judge:                  judge,
+		JudgeThreshold:         cfg.LLM.Judge.Threshold,
 		Parsers:                ingest.RegistryFromIngestParams(tableOptsFromConfig(cfg.Ingest.Tables), cfg.Ingest.MaxSections, time.Duration(cfg.Ingest.ParseTimeoutSeconds)*time.Second),
 		Logger:                 logger,
 		Mode:                   cfg.Ingest.Mode,
@@ -396,6 +410,26 @@ func modelFor(c config.LLMConfig) string {
 	return ""
 }
 
+// buildJudge returns the configured Judge, or nil when llm.judge has
+// no API key. Retries share the same schedule as every other provider
+// call; a Judge request that fails past them is handled by the TOC
+// builder, which keeps extraction's pages rather than degrading
+// silently (HAL-1369).
+func buildJudge(c config.JudgeBlock) (llmgate.Judge, error) {
+	if c.TypeSafe.APIKey == "" {
+		return nil, nil
+	}
+	j, err := typesafe.New(typesafe.Config{
+		APIKey:  c.TypeSafe.APIKey,
+		BaseURL: c.TypeSafe.BaseURL,
+		Model:   c.TypeSafe.Model,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return retry.NewJudge(retry.Config{MaxRetries: 3})(j), nil
+}
+
 func buildLLM(c config.LLMConfig) (llmgate.Client, error) {
 	switch c.Driver {
 	case "anthropic":

cmd/server/main.go

@@ -27,6 +27,8 @@ import (
 	"time"
 
 	"github.com/hallelx2/llmgate"
+	"github.com/hallelx2/llmgate/judge/typesafe"
+	"github.com/hallelx2/llmgate/middleware/retry"
 	"github.com/hallelx2/llmgate/pricing"
 	"github.com/hallelx2/llmgate/provider/anthropic"
 	"github.com/hallelx2/llmgate/provider/gemini"
@@ -201,10 +203,22 @@ func run() error {
 	}
 
 	// ── Ingest pipeline ───────────────────────────────────────────
+	judge, err := buildJudge(cfg.Engine.LLM.Judge)
+	if err != nil {
+		logger.Error("judge: config invalid", "err", err)
+		os.Exit(1)
+	}
+	if judge != nil {
+		logger.Info("judge: typesafe enabled — contents-page detection and page resolution run on the Judge")
+	} else {
+		logger.Warn("judge: none configured — TOC judgements run on the generative driver, one call per page (set TYPESAFE_API_KEY)")
+	}
 	pipeline := ingest.NewPipeline(ingest.Pipeline{
 		DB:                     pool,
 		Storage:                store,
 		LLM:                    llmClient,
+		Judge:                  judge,
+		JudgeThreshold:         cfg.Engine.LLM.Judge.Threshold,
 		Parsers:                ingest.RegistryFromIngestParams(tableOptsFromConfig(cfg.Engine.Ingest.Tables), cfg.Engine.Ingest.MaxSections, time.Duration(cfg.Engine.Ingest.ParseTimeoutSeconds)*time.Second),
 		Logger:                 logger,
 		Mode:                   cfg.Engine.Ingest.Mode,
@@ -396,6 +410,26 @@ func modelFor(c enginecfg.LLMConfig) string {
 	return ""
 }
 
+// buildJudge returns the configured Judge, or nil when llm.judge has
+// no API key. Retries share the same schedule as every other provider
+// call; a Judge request that fails past them is handled by the TOC
+// builder, which keeps extraction's pages rather than degrading
+// silently (HAL-1369).
+func buildJudge(c enginecfg.JudgeBlock) (llmgate.Judge, error) {
+	if c.TypeSafe.APIKey == "" {
+		return nil, nil
+	}
+	j, err := typesafe.New(typesafe.Config{
+		APIKey:  c.TypeSafe.APIKey,
+		BaseURL: c.TypeSafe.BaseURL,
+		Model:   c.TypeSafe.Model,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return retry.NewJudge(retry.Config{MaxRetries: 3})(j), nil
+}
+
 func buildLLM(c enginecfg.LLMConfig) (llmgate.Client, error) {
 	switch c.Driver {
 	case "anthropic":

config.example.yaml

@@ -107,6 +107,20 @@ llm:
     model: "gemini-2.0-flash"
     reasoning_model: "gemini-2.5-pro"
 
+  # Judge: a System One model (TypeSafe Jev) that answers the ingest
+  # pipeline's judgements — contents-page detection and page resolution —
+  # in one batched request per document instead of a generative call per
+  # page. Enabled exactly when an api_key is present; also read from
+  # VLE_TYPESAFE_API_KEY or TYPESAFE_API_KEY. Without it, long filings
+  # lose every leaf's page (HAL-1367) and the TOC stage runs minutes
+  # slower on the driver above.
+  judge:
+    typesafe:
+      api_key: ""
+      # base_url: ""     # override the endpoint; empty = api.typesafe.ai
+      # model: ""        # override the model alias
+    # threshold: 0.5     # Noul probability that counts as "yes"
+
 retrieval:
   # strategy: single-pass | chunked-tree | agentic | treewalk
   #

config.server.example.yaml

@@ -90,6 +90,13 @@ engine:
     #   api_key: ""
     #   model: "gemini-2.0-flash"
     #   reasoning_model: ""
+    # Judge (TypeSafe Jev): batched contents-page detection and page
+    # resolution during ingest. Enabled when api_key is set — also read
+    # from VLE_TYPESAFE_API_KEY / TYPESAFE_API_KEY. See HAL-1367.
+    judge:
+      typesafe:
+        api_key: ""
+      # threshold: 0.5
 
   retrieval:
     strategy: "chunked-tree"   # "single-pass" or "chunked-tree"

pkg/config/config.go

@@ -376,6 +376,32 @@ type LLMConfig struct {
 	Anthropic AnthropicBlock `yaml:"anthropic"`
 	OpenAI    OpenAIBlock    `yaml:"openai"`
 	Gemini    GeminiBlock    `yaml:"gemini"`
+
+	// Judge configures the System One model that answers the pipeline's
+	// judgements — contents-page detection and page resolution — in one
+	// batched request each, instead of a generative call per page. Left
+	// empty, those steps run on the generative driver above, one call at
+	// a time, and a 10-K's leaves lose their pages (HAL-1367).
+	Judge JudgeBlock `yaml:"judge"`
+}
+
+// JudgeBlock configures the Judge. Only TypeSafe is supported today; the
+// Judge is enabled exactly when an API key is present.
+type JudgeBlock struct {
+	TypeSafe TypeSafeBlock `yaml:"typesafe"`
+
+	// Threshold is the Noul probability at or above which a Judge answer
+	// counts as yes. Zero selects the builder's default (0.5).
+	Threshold float64 `yaml:"threshold"`
+}
+
+// TypeSafeBlock configures the TypeSafe System One provider.
+type TypeSafeBlock struct {
+	APIKey string `yaml:"api_key"`
+	// BaseURL overrides the endpoint. Empty = api.typesafe.ai.
+	BaseURL string `yaml:"base_url"`
+	// Model overrides the model alias. Empty = the provider's default.
+	Model string `yaml:"model"`
 }
 
 // AnthropicBlock configures the Anthropic provider.
@@ -903,6 +929,13 @@ func applyEnvOverrides(c *Config) {
 	if v := os.Getenv("VLE_GEMINI_API_KEY"); v != "" {
 		c.LLM.Gemini.APIKey = v
 	}
+	// The Judge key accepts the deploy layer's VLS_ prefix and the bare
+	// TYPESAFE_API_KEY — what the llmgate live tests and the bench
+	// commands already read — so one export enables it everywhere.
+	// VLE_-prefixed wins if several are set.
+	if v := firstEnv("VLE_TYPESAFE_API_KEY", "VLS_TYPESAFE_API_KEY", "TYPESAFE_API_KEY"); v != "" {
+		c.LLM.Judge.TypeSafe.APIKey = v
+	}
 	// Accept both VLE_-prefixed and bare QSTASH_* env vars. The bare
 	// names match what the Upstash console documents and what the
 	// dashboard already uses, so ops can set them once for both

pkg/config/judge_config_test.go

@@ -0,0 +1,29 @@
+package config
+
+import "testing"
+
+func TestJudgeKeyFromEnv(t *testing.T) {
+	t.Setenv("VLE_TYPESAFE_API_KEY", "")
+	t.Setenv("TYPESAFE_API_KEY", "bare")
+	c := Default()
+	applyEnvOverrides(&c)
+	if c.LLM.Judge.TypeSafe.APIKey != "bare" {
+		t.Errorf("bare TYPESAFE_API_KEY not picked up: %q", c.LLM.Judge.TypeSafe.APIKey)
+	}
+	t.Setenv("VLE_TYPESAFE_API_KEY", "prefixed")
+	c = Default()
+	applyEnvOverrides(&c)
+	if c.LLM.Judge.TypeSafe.APIKey != "prefixed" {
+		t.Errorf("VLE_ prefix should win: %q", c.LLM.Judge.TypeSafe.APIKey)
+	}
+}
+
+func TestJudgeIsOffByDefault(t *testing.T) {
+	t.Setenv("VLE_TYPESAFE_API_KEY", "")
+	t.Setenv("TYPESAFE_API_KEY", "")
+	c := Default()
+	applyEnvOverrides(&c)
+	if c.LLM.Judge.TypeSafe.APIKey != "" {
+		t.Errorf("no key configured, got %q", c.LLM.Judge.TypeSafe.APIKey)
+	}
+}

pkg/ingest/ingest.go

@@ -170,6 +170,18 @@ type Pipeline struct {
 	// per-stage semaphore). Default applied by NewPipeline: 12.
 	GlobalLLMConcurrency int
 
+	// Judge, when non-nil, answers the TOC stage's judgements —
+	// contents-page detection and page resolution — in one batched
+	// request each. Without it those steps run on LLM, one generative
+	// call per page, and long filings lose every leaf's page (HAL-1367).
+	// Wired from config llm.judge by cmd/server and cmd/engine; nil in
+	// Pipeline literals that do not set it, which keeps the old path.
+	Judge llmgate.Judge
+
+	// JudgeThreshold is the Noul probability at or above which a Judge
+	// answer counts as yes. Zero selects TOCBuilder's default.
+	JudgeThreshold float64
+
 	// TOCEnabled toggles the LLM-built table-of-contents stage. The
 	// stage runs after summarize+HyDE on PDF inputs and persists the
 	// resulting tree on documents.toc_tree (JSONB). Failures are
@@ -460,6 +472,12 @@ func (p *Pipeline) runTOCBuilder(ctx context.Context, docID tree.DocumentID, par
 		Concurrency:    p.TOCConcurrency,
 		TOCCheckPages:  p.TOCCheckPages,
 		LLMCallTimeout: p.LLMCallTimeout,
+		Judge:          p.Judge,
+		JudgeThreshold: p.JudgeThreshold,
+		// The minimum-context path: prefilter, truncation, two-stage
+		// scan. Measured on FinanceBench as the fastest detection that
+		// lost nothing (HAL-1366).
+		MinimalContext: p.Judge != nil,
 	}
 	nodes, usage, err := builder.Build(ctx, pages)
 	if err != nil {
@@ -470,7 +488,13 @@ func (p *Pipeline) runTOCBuilder(ctx context.Context, docID tree.DocumentID, par
 		"llm_calls", usage.LLMCalls,
 		"input_tokens", usage.InputTokens,
 		"output_tokens", usage.OutputTokens,
+		"judge", p.Judge != nil,
 	)
+	// A degraded build is not a failed one, but it is not what was
+	// configured either, and it must not look like success in the log.
+	for _, d := range usage.Degraded {
+		log.Warn("ingest: toc-builder degraded", "step", d)
+	}
 	if len(nodes) == 0 {
 		return nil
 	}

pkg/ingest/toc_builder.go

@@ -136,6 +136,19 @@ type Usage struct {
 	TotalTokens  int
 	CostUSD      float64
 	LLMCalls     int
+
+	// Degraded lists the Judge-path steps that could not complete and
+	// what Build did instead. Empty means every step ran as configured.
+	// A document built with a non-empty Degraded is not wrong, but it is
+	// not what was asked for, and the caller must be able to see that:
+	// VERIZON_2022_10K once ingested with no page on any leaf after a
+	// single failed Judge request, and reported success (HAL-1369).
+	Degraded []string
+}
+
+// degrade records a Judge-path step that fell back.
+func (u *Usage) degrade(step, what string) {
+	u.Degraded = append(u.Degraded, step+": "+what)
 }
 
 // add folds the per-response usage from one LLM call into the
@@ -210,10 +223,16 @@ func (b *TOCBuilder) Build(ctx context.Context, pages []PageText) ([]tree.TOCNod
 	// verification with search, and it is what makes the extraction body
 	// window irrelevant to page accuracy (HAL-1367). Plain verification
 	// remains the fallback when resolution cannot run.
-	if resolved, handled := b.resolvePagesJudge(ctx, nodes, pages, tocPages, &usage); handled {
-		applyResolvedPages(nodes, resolved)
-	} else if verdicts, handled := b.verifyTitlesJudge(ctx, nodes, pages, &usage); handled {
-		applyJudgeVerdicts(nodes, verdicts)
+	//
+	// A failed Judge request is not "no Judge". The generative verifier
+	// asks about extraction's printed page numbers, which are wrong for
+	// every leaf past the cover of a long filing, so falling to it after
+	// a transport failure zeroes the tree and looks like success
+	// (HAL-1369). With a Judge configured, resolution is retried once
+	// with a fresh budget; if it still fails, extraction's pages are
+	// kept as they are and the degradation is recorded on Usage.
+	if b.Judge != nil {
+		b.resolvePagesOrKeep(ctx, nodes, pages, tocPages, &usage)
 	} else {
 		b.verifyTitlesConcurrent(ctx, nodes, pages, concurrency, &usage)
 	}
@@ -229,6 +248,36 @@ func (b *TOCBuilder) Build(ctx context.Context, pages []PageText) ([]tree.TOCNod
 	return nodes, usage, nil
 }
 
+// resolverAttempts is how many times Build asks the Judge to resolve
+// pages before keeping extraction's. Two: the first failure is almost
+// always transport, and a resolver batch is two cheap requests.
+const resolverAttempts = 2
+
+// resolvePagesOrKeep runs Judge page resolution with one retry. On
+// exhaustion it leaves the tree exactly as extraction produced it and
+// records the fact; it never routes a Judge-path document through the
+// generative verifier.
+func (b *TOCBuilder) resolvePagesOrKeep(ctx context.Context, nodes []tree.TOCNode, pages []PageText, exclude []int, usage *Usage) {
+	var lastErr error
+	for attempt := 1; attempt <= resolverAttempts; attempt++ {
+		resolved, handled, err := b.resolvePagesJudgeErr(ctx, nodes, pages, exclude, usage)
+		if err == nil {
+			if handled {
+				applyResolvedPages(nodes, resolved)
+			}
+			// handled=false with no error means there was nothing to
+			// resolve (no leaves with titles); extraction's pages stand.
+			return
+		}
+		lastErr = err
+		log.Printf("toc: judge page resolution attempt %d/%d failed: %v", attempt, resolverAttempts, err)
+		if ctx.Err() != nil {
+			break
+		}
+	}
+	usage.degrade("page resolution", fmt.Sprintf("kept extraction's pages after %d failed Judge attempts: %v", resolverAttempts, lastErr))
+}
+
 // detectTOCPages scans the first tocCheck pages with the
 // TreeWalk-style single-page detector. Returns the 1-indexed page
 // numbers (in order) the LLM judged as table-of-contents pages.