feat: add word frequency filtering with vocabulary size control

Sort word lists by Google Books frequency (most common first).
Add vocabulary dropdown (Common/Standard/Extended/Full) to word
mode UI that controls how many words are searched.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

scripts/build_word_lists.py

@@ -5,10 +5,11 @@ Usage:
 
 Extracts nouns and adjectives from WordNet, generates plural forms using
 irregular mappings from Wiktionary/GitHub and standard English rules, then
-writes three files to unredact/data/:
-  - nouns.txt          (singular nouns, one per line)
-  - nouns_plural.txt   (corresponding plural for each noun)
-  - adjectives.txt     (adjectives, one per line)
+sorts all lists by word frequency (Google Books corpus) and writes three
+files to unredact/data/:
+  - nouns.txt          (singular nouns, sorted by frequency)
+  - nouns_plural.txt   (corresponding plural for each noun, same order)
+  - adjectives.txt     (adjectives, sorted by frequency)
 """
 
 import argparse
@@ -26,6 +27,9 @@ OUTPUT_DIR = Path(__file__).parent.parent / "unredact" / "data"
 IRREGULAR_PLURALS_URL = (
     "https://raw.githubusercontent.com/djstrong/nouns-with-plurals/master/noun.csv"
 )
+FREQUENCY_URL = (
+    "https://raw.githubusercontent.com/hackerb9/gwordlist/master/frequency-alpha-alldicts.txt"
+)
 
 # Regex: only lowercase alpha, no digits/hyphens/underscores/spaces
 VALID_WORD = re.compile(r"^[a-z]+$")
@@ -102,8 +106,43 @@ def generate_plurals(nouns: list[str], irregulars: dict[str, str]) -> list[str]:
     return plurals
 
 
+def download_frequency_ranks() -> dict[str, int]:
+    """Download word frequency rankings from Google Books corpus.
+
+    Returns: {word: rank} dict where rank 1 = most common.
+    """
+    print(f"Downloading frequency data from hackerb9/gwordlist...")
+    try:
+        response = urllib.request.urlopen(FREQUENCY_URL)
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"  WARNING: failed to download frequency data: {exc}")
+        print("  Words will be sorted alphabetically instead.")
+        return {}
+    text = response.read().decode("utf-8")
+
+    ranks: dict[str, int] = {}
+    for line in text.splitlines():
+        if line.startswith("#") or not line.strip():
+            continue
+        parts = line.split()
+        if len(parts) >= 2:
+            rank = int(parts[0])
+            word = parts[1].lower()
+            if word not in ranks:
+                ranks[word] = rank
+
+    print(f"  -> loaded {len(ranks)} word frequency rankings")
+    return ranks
+
+
+def sort_by_frequency(words: list[str], freq_ranks: dict[str, int]) -> list[str]:
+    """Sort words by frequency rank. Unranked words go to the end."""
+    max_rank = len(freq_ranks) + 1
+    return sorted(words, key=lambda w: freq_ranks.get(w, max_rank))
+
+
 def write_word_list(path: Path, words: list[str]) -> None:
-    """Write a sorted word list, one word per line."""
+    """Write a word list, one word per line."""
     path.write_text("\n".join(words) + "\n")
     print(f"Wrote {len(words)} words to {path}")
 
@@ -138,14 +177,32 @@ def main():
     # Download irregular plural mappings
     irregulars = download_irregular_plurals()
 
-    # Generate plurals
-    print("Generating plural forms...")
-    plurals = generate_plurals(nouns, irregulars)
+    # Download frequency data
+    freq_ranks = download_frequency_ranks()
+
+    # Sort by frequency (most common first)
+    if freq_ranks:
+        print("Sorting by frequency...")
+        nouns_sorted = sort_by_frequency(nouns, freq_ranks)
+        adjectives_sorted = sort_by_frequency(adjectives, freq_ranks)
+
+        # Build noun->plural mapping, then reorder plurals to match sorted nouns
+        noun_to_plural = dict(zip(nouns, generate_plurals(nouns, irregulars)))
+        plurals_sorted = [noun_to_plural[n] for n in nouns_sorted]
+
+        ranked_nouns = sum(1 for n in nouns_sorted if n in freq_ranks)
+        ranked_adj = sum(1 for a in adjectives_sorted if a in freq_ranks)
+        print(f"  -> {ranked_nouns}/{len(nouns_sorted)} nouns have frequency data")
+        print(f"  -> {ranked_adj}/{len(adjectives_sorted)} adjectives have frequency data")
+    else:
+        nouns_sorted = nouns
+        adjectives_sorted = adjectives
+        plurals_sorted = generate_plurals(nouns, irregulars)
 
     # Write output files
-    write_word_list(args.data_dir / "nouns.txt", nouns)
-    write_word_list(args.data_dir / "nouns_plural.txt", plurals)
-    write_word_list(args.data_dir / "adjectives.txt", adjectives)
+    write_word_list(args.data_dir / "nouns.txt", nouns_sorted)
+    write_word_list(args.data_dir / "nouns_plural.txt", plurals_sorted)
+    write_word_list(args.data_dir / "adjectives.txt", adjectives_sorted)
 
     print("Done.")
 

unredact/app.py

@@ -287,6 +287,7 @@ class SolveRequest(BaseModel):
     known_start: str = ""
     known_end: str = ""
     ensure_plural: bool = False
+    vocab_size: int = 0  # 0 = no limit; >0 = use top N most common words
 
 
 def _build_rust_solve_payload(
@@ -443,6 +444,7 @@ async def solve(req: SolveRequest):
                     known_start=req.known_start,
                     known_end=req.known_end,
                     ensure_plural=req.ensure_plural,
+                    vocab_size=req.vocab_size,
                 ):
                     if _active_solves.get(solve_id):
                         break

unredact/pipeline/dictionary.py

@@ -275,11 +275,15 @@ def solve_word_dictionary(
     known_start: str = "",
     known_end: str = "",
     ensure_plural: bool = False,
+    vocab_size: int = 0,
 ):
     """Search English nouns (single-word) and adj+noun phrases (two-word).
 
     Yields results as found for SSE streaming. Phase 1 searches single nouns,
     phase 2 searches two-word combinations using binary search.
+
+    Word lists are frequency-sorted (most common first). vocab_size > 0 slices
+    to the N most common words; 0 means no limit.
     """
     from unredact.pipeline.word_filter import (
         _get_nouns,
@@ -288,6 +292,11 @@ def solve_word_dictionary(
 
     nouns = _get_nouns()
     plurals = _get_nouns_plural()
+
+    if vocab_size > 0:
+        nouns = nouns[:vocab_size]
+        plurals = plurals[:vocab_size]
+
     word_list = plurals if ensure_plural else nouns
 
     ks_lower = known_start.lower()
@@ -324,6 +333,8 @@ def solve_word_dictionary(
     from unredact.pipeline.word_filter import _get_adjectives
 
     adjectives = _get_adjectives()
+    if vocab_size > 0:
+        adjectives = adjectives[:vocab_size]
 
     # Word1 pool: adjectives + nouns (deduplicated, sorted)
     word1_set = set(adjectives) | set(nouns)

unredact/static/dom.js

@@ -31,6 +31,8 @@ export const solveKnownEnd = /** @type {HTMLInputElement} */ (document.getElemen
 export const filterLabel = document.getElementById("filter-label");
 export const pluralLabel = document.getElementById("plural-label");
 export const solvePlural = /** @type {HTMLInputElement} */ (document.getElementById("solve-plural"));
+export const vocabLabel = document.getElementById("vocab-label");
+export const solveVocab = /** @type {HTMLSelectElement} */ (document.getElementById("solve-vocab"));
 export const solveStart = document.getElementById("solve-start");
 export const solveStop = document.getElementById("solve-stop");
 export const solveAccept = document.getElementById("solve-accept");

unredact/static/index.html

@@ -128,6 +128,15 @@
               <label id="plural-label" hidden>
                 <input type="checkbox" id="solve-plural"> Ensure plural
               </label>
+              <label id="vocab-label" hidden>
+                Vocabulary
+                <select id="solve-vocab">
+                  <option value="3300">Common</option>
+                  <option value="6400" selected>Standard</option>
+                  <option value="12000">Extended</option>
+                  <option value="0">Full</option>
+                </select>
+              </label>
               <label>
                 Known start
                 <input type="text" id="solve-known-start" placeholder="e.g. j" spellcheck="false" autocomplete="off">

unredact/static/popover.js

@@ -11,7 +11,7 @@ import {
   solveResults, solveStatus, solveStart, solveStop, solveAccept,
   solveTolerance, solveTolValue,
   solveKnownStart, solveKnownEnd,
-  solveMode, filterLabel, pluralLabel,
+  solveMode, filterLabel, pluralLabel, vocabLabel,
 } from './dom.js';
 import { renderCanvas } from './canvas.js';
 
@@ -41,7 +41,17 @@ export function openPopover(id) {
   solveStop.hidden = true;
   // For approved redactions, restore the solution as preview for re-editing
   if (r.status === "approved" && r.solution) {
-    r.preview = r.solution.text;
+    let restored = r.solution.text;
+    const ks = solveKnownStart.value;
+    const ke = solveKnownEnd.value;
+    if (ks && restored.toLowerCase().startsWith(ks.toLowerCase())) {
+      restored = restored.slice(ks.length);
+    }
+    if (ke && restored.toLowerCase().endsWith(ke.toLowerCase())) {
+      restored = restored.slice(0, -ke.length);
+    }
+    r.solveFullText = r.solution.text;
+    r.preview = restored;
     r.status = "analyzed";
     solveAccept.hidden = true;
   } else {
@@ -175,6 +185,7 @@ export function initPopover() {
     if (!r) return;
     const guess = redactionMarker.value.trim();
     r.preview = guess || null;
+    r.solveFullText = null;  // manual input replaces solver result
     redactionMarker.className = guess ? "redaction-marker preview" : "redaction-marker";
     solveAccept.hidden = !guess;
     renderCanvas();
@@ -186,6 +197,8 @@ export function initPopover() {
 
   solveMode.addEventListener("change", () => {
     filterLabel.hidden = solveMode.value !== "enumerate";
-    pluralLabel.hidden = solveMode.value !== "word";
+    const isWord = solveMode.value === "word";
+    pluralLabel.hidden = !isWord;
+    vocabLabel.hidden = !isWord;
   });
 }

unredact/static/solver.js

@@ -4,7 +4,7 @@
 import { state } from './state.js';
 import {
   solveCharset, solveTolerance, solveMode, solveFilter,
-  solveKnownStart, solveKnownEnd, solvePlural,
+  solveKnownStart, solveKnownEnd, solvePlural, solveVocab,
   solveResults, solveStatus, solveStart, solveStop,
   solveAccept, redactionMarker, escapeHtml,
 } from './dom.js';
@@ -52,6 +52,7 @@ export function startSolve() {
     known_start: solveKnownStart.value,
     known_end: solveKnownEnd.value,
     ensure_plural: solvePlural.checked,
+    vocab_size: parseInt(solveVocab.value) || 0,
   };
 
   const abortController = new AbortController();
@@ -145,8 +146,21 @@ function handleSolveEvent(data, redactionId) {
     div.addEventListener("click", () => {
       const r = state.redactions[redactionId];
       if (!r) return;
-      r.preview = data.text;
-      redactionMarker.value = data.text;
+
+      // Strip known start/end from preview — those are already visible
+      let previewText = data.text;
+      const ks = solveKnownStart.value;
+      const ke = solveKnownEnd.value;
+      if (ks && previewText.toLowerCase().startsWith(ks.toLowerCase())) {
+        previewText = previewText.slice(ks.length);
+      }
+      if (ke && previewText.toLowerCase().endsWith(ke.toLowerCase())) {
+        previewText = previewText.slice(0, -ke.length);
+      }
+
+      r.preview = previewText;
+      r.solveFullText = data.text;
+      redactionMarker.value = previewText;
       redactionMarker.className = "redaction-marker preview";
       renderCanvas();
       solveResults.querySelectorAll(".solve-result").forEach(el => el.classList.remove("active"));
@@ -208,12 +222,13 @@ export function acceptSolution() {
   r.status = "approved";
   const solFontName = state.fonts.find(f => f.id === (o.fontId || r.analysis.font.id))?.name || r.analysis.font.name;
   r.solution = {
-    text: r.preview,
+    text: r.solveFullText || r.preview,
     fontName: solFontName,
     fontSize: o.fontSize || r.analysis.font.size,
   };
   r.approvedText = leftText + r.preview + rightText;
   r.preview = null;
+  r.solveFullText = null;
 }
 
 /** Set up solver button listeners. Call once from main.js. */