diff --git a/docs/plans/2026-02-23-llm-validation-impl.md b/docs/plans/2026-02-23-llm-validation-impl.md new file mode 100644 index 0000000..417514c --- /dev/null +++ b/docs/plans/2026-02-23-llm-validation-impl.md @@ -0,0 +1,614 @@ +# LLM Validation of Solve Results Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add a "Validate" button that sends all solve results to Claude Sonnet, scores them on contextual fit, and reorders the result list by score. + +**Architecture:** New backend endpoint reads buffered solve results, sends them to Anthropic API with surrounding text context, returns scored results. Frontend shows a validate panel with editable context fields, replaces results with LLM-scored list. + +**Tech Stack:** Python/FastAPI, Anthropic SDK (claude-sonnet-4-6, tool_use), vanilla JS frontend. + +--- + +### Task 1: Create the LLM validation module + +**Files:** +- Create: `unredact/pipeline/llm_validate.py` +- Test: `tests/test_llm_validate.py` + +**Step 1: Write the failing test** + +Create `tests/test_llm_validate.py`: + +```python +"""Tests for LLM solve result validation.""" + +import pytest + +from unredact.pipeline.llm_validate import build_validation_prompt, SCORE_TOOL + + +class TestBuildValidationPrompt: + def test_basic_prompt_structure(self): + candidates = ["Smith", "house", "running"] + prompt = build_validation_prompt( + left_context="Dear Mr.", + right_context=", we are writing", + candidates=candidates, + ) + assert "Dear Mr." in prompt + assert ", we are writing" in prompt + assert "1. Smith" in prompt + assert "2. house" in prompt + assert "3. running" in prompt + assert "[REDACTED]" in prompt + + def test_empty_context(self): + prompt = build_validation_prompt( + left_context="", + right_context="", + candidates=["word"], + ) + assert "1. word" in prompt + + def test_score_tool_schema(self): + assert SCORE_TOOL["name"] == "score_candidates" + schema = SCORE_TOOL["input_schema"] + assert "scores" in schema["properties"] + items = schema["properties"]["scores"]["items"] + assert "index" in items["properties"] + assert "score" in items["properties"] +``` + +**Step 2: Run test to verify it fails** + +Run: `pytest tests/test_llm_validate.py -v` +Expected: FAIL — module doesn't exist + +**Step 3: Implement the module** + +Create `unredact/pipeline/llm_validate.py`: + +```python +"""LLM-based validation of solve candidates. + +Sends candidate words to Claude Sonnet with surrounding text context. +Returns a contextual fit score (0-100) for each candidate. +""" + +from __future__ import annotations + +import logging +import os + +import anthropic + +log = logging.getLogger(__name__) + +_VALIDATION_MODEL = "claude-sonnet-4-6" + +SCORE_TOOL = { + "name": "score_candidates", + "description": "Score each candidate word on how well it fits the redacted gap contextually.", + "input_schema": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "description": "1-based index of the candidate in the list.", + }, + "score": { + "type": "integer", + "description": "Contextual fit score from 0-100.", + "minimum": 0, + "maximum": 100, + }, + }, + "required": ["index", "score"], + }, + }, + }, + "required": ["scores"], + }, +} + + +def build_validation_prompt( + left_context: str, + right_context: str, + candidates: list[str], +) -> str: + """Build the prompt for LLM validation of solve candidates.""" + numbered = "\n".join(f"{i+1}. {c}" for i, c in enumerate(candidates)) + return ( + "You are analyzing a redacted document. A section of text has been " + "blacked out. The text surrounding the redaction reads:\n\n" + f'Left context: "{left_context}"\n' + "[REDACTED]\n" + f'Right context: "{right_context}"\n\n' + "Below is a list of candidate words/phrases that fit the redacted " + "space by pixel width. Score each from 0-100 on how well it fits " + "contextually:\n\n" + "- 90-100: Near-certain fit (grammatically correct, semantically " + "meaningful, contextually expected)\n" + "- 60-89: Plausible (makes sense but not the most likely)\n" + "- 30-59: Unlikely (grammatically possible but doesn't make much sense)\n" + "- 0-29: Very poor fit (nonsensical, wrong part of speech, doesn't " + "work in context)\n\n" + 'Example: If left context is "Dear Mr." and right is ", we are ' + 'writing to inform you":\n' + '- "Smith" -> 95 (common surname, perfect fit)\n' + '- "house" -> 5 (not a surname, makes no sense after "Mr.")\n\n' + f"Candidates:\n{numbered}" + ) + + +# Reuse the client from llm_detect +from unredact.pipeline.llm_detect import _get_client + + +async def validate_candidates( + left_context: str, + right_context: str, + candidates: list[str], +) -> list[int]: + """Score candidates using LLM. Returns list of scores in same order as candidates. + + Raises on API error — caller should handle. + """ + if not candidates: + return [] + + prompt = build_validation_prompt(left_context, right_context, candidates) + client = _get_client() + + response = await client.messages.create( + model=_VALIDATION_MODEL, + max_tokens=16384, + tools=[SCORE_TOOL], + tool_choice={"type": "tool", "name": "score_candidates"}, + messages=[{"role": "user", "content": prompt}], + ) + + # Parse tool response + scores = [0] * len(candidates) + for block in response.content: + if block.type == "tool_use" and block.name == "score_candidates": + for item in block.input.get("scores", []): + idx = item.get("index", 0) - 1 # 1-based to 0-based + score = item.get("score", 0) + if 0 <= idx < len(candidates): + scores[idx] = score + break + + return scores +``` + +**Step 4: Run test to verify it passes** + +Run: `pytest tests/test_llm_validate.py -v` +Expected: PASS (tests only check prompt building and schema, not API calls) + +**Step 5: Commit** + +```bash +git add unredact/pipeline/llm_validate.py tests/test_llm_validate.py +git commit -m "feat: add LLM validation module with prompt and tool schema" +``` + +--- + +### Task 2: Add validation endpoint to app.py + +**Files:** +- Modify: `unredact/app.py` (add endpoint after `get_solve_results`) +- Test: `tests/test_app.py` + +**Step 1: Write the failing test** + +Add to `tests/test_app.py`. This test mocks the Anthropic API call to avoid real API usage: + +```python +@pytest.mark.anyio +async def test_validate_endpoint(monkeypatch): + """POST /api/solve/{id}/validate should return LLM-scored results.""" + from unredact.app import _solve_results + from unredact.pipeline import llm_validate + + fake_id = "validate_test" + _solve_results[fake_id] = [ + {"text": "Smith", "width_px": 50.0, "error_px": 0.1, "source": "names"}, + {"text": "house", "width_px": 50.0, "error_px": 0.2, "source": "words"}, + {"text": "running", "width_px": 50.0, "error_px": 0.3, "source": "words"}, + ] + + # Mock validate_candidates to avoid real API call + async def mock_validate(left, right, candidates): + return [95, 10, 5] + + monkeypatch.setattr(llm_validate, "validate_candidates", mock_validate) + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + resp = await client.post(f"/api/solve/{fake_id}/validate", json={ + "left_context": "Dear Mr.", + "right_context": ", we are writing", + }) + assert resp.status_code == 200 + data = resp.json() + assert len(data["results"]) == 3 + assert data["total"] == 3 + # Should be sorted by llm_score descending + assert data["results"][0]["text"] == "Smith" + assert data["results"][0]["llm_score"] == 95 + assert data["results"][1]["llm_score"] == 10 + assert data["results"][2]["llm_score"] == 5 + + _solve_results.pop(fake_id, None) + + +@pytest.mark.anyio +async def test_validate_unknown_solve_id(): + """POST /api/solve/{id}/validate with unknown id should 404.""" + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + resp = await client.post("/api/solve/nonexistent/validate", json={ + "left_context": "test", + "right_context": "test", + }) + assert resp.status_code == 404 +``` + +**Step 2: Run test to verify it fails** + +Run: `pytest tests/test_app.py::test_validate_endpoint -v` +Expected: FAIL — endpoint doesn't exist + +**Step 3: Add the endpoint** + +Add after the `get_solve_results` endpoint in `unredact/app.py` (around line 586): + +```python +class ValidateRequest(BaseModel): + left_context: str = "" + right_context: str = "" + + +@app.post("/api/solve/{solve_id}/validate") +async def validate_solve(solve_id: str, req: ValidateRequest): + if solve_id not in _solve_results: + return JSONResponse({"error": "solve not found"}, status_code=404) + + from unredact.pipeline.llm_validate import validate_candidates + + buf = _solve_results[solve_id] + candidates = [r["text"] for r in buf] + + try: + scores = await validate_candidates( + req.left_context, req.right_context, candidates, + ) + except Exception as e: + return JSONResponse({"error": str(e)}, status_code=500) + + # Merge scores and sort by score descending + scored = [] + for i, result in enumerate(buf): + scored.append({**result, "llm_score": scores[i] if i < len(scores) else 0}) + scored.sort(key=lambda x: x["llm_score"], reverse=True) + + return {"results": scored, "total": len(scored)} +``` + +**Step 4: Run test to verify it passes** + +Run: `pytest tests/test_app.py::test_validate_endpoint tests/test_app.py::test_validate_unknown_solve_id -v` +Expected: PASS + +**Step 5: Commit** + +```bash +git add unredact/app.py tests/test_app.py +git commit -m "feat: add POST /api/solve/{id}/validate endpoint" +``` + +--- + +### Task 3: Add Validate button and context panel to HTML + +**Files:** +- Modify: `unredact/static/index.html:149-157` (solve actions area) +- Modify: `unredact/static/dom.js` (add new element references) + +**Step 1: Add HTML elements** + +In `index.html`, after the `solve-accept` button (line 152) and before the status span, add: + +```html + +``` + +After the `solve-load-more` button (line 156), add the validate panel: + +```html +