From a0e2bf6ec19bcc420986dd50fc6bb2c5b24afd8f Mon Sep 17 00:00:00 2001 From: Alex-Gilbert Date: Mon, 23 Feb 2026 15:43:00 -0600 Subject: [PATCH] feat: disable two-word adjective+noun search by default The new Wiktionary dictionary already includes 31k multi-word nouns, making the combinatorial adjective+noun phase redundant. Add two_word parameter (default False) to solve_word_dictionary to opt in when needed. Co-Authored-By: Claude Opus 4.6 --- tests/test_word_solve.py | 23 ++++++++++++++++++----- unredact/pipeline/dictionary.py | 11 ++++++++--- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/test_word_solve.py b/tests/test_word_solve.py index 6c30770..4d3c7b6 100644 --- a/tests/test_word_solve.py +++ b/tests/test_word_solve.py @@ -94,7 +94,7 @@ class TestSolveWordDictionaryTwoWord: target = font.getlength("large house") results = list(solve_word_dictionary( font, target, tolerance=1.0, - casing="lowercase", known_start="large", + casing="lowercase", known_start="large", two_word=True, )) texts = [r.text for r in results] assert "large house" in texts @@ -104,7 +104,7 @@ class TestSolveWordDictionaryTwoWord: target = font.getlength("Large House") results = list(solve_word_dictionary( font, target, tolerance=1.0, - casing="capitalized", known_start="Large", + casing="capitalized", known_start="Large", two_word=True, )) texts = [r.text for r in results] assert "Large House" in texts @@ -115,6 +115,7 @@ class TestSolveWordDictionaryTwoWord: results = list(solve_word_dictionary( font, target, tolerance=1.0, casing="lowercase", ensure_plural=True, known_start="large", + two_word=True, )) texts = [r.text for r in results] assert "large houses" in texts @@ -124,7 +125,7 @@ class TestSolveWordDictionaryTwoWord: target = font.getlength("large house") results = list(solve_word_dictionary( font, target, tolerance=1.0, - casing="lowercase", known_start="la", + casing="lowercase", known_start="la", two_word=True, )) texts = [r.text for r in results] assert "large house" in texts @@ -135,18 +136,30 @@ class TestSolveWordDictionaryTwoWord: target = font.getlength("large house") results = list(solve_word_dictionary( font, target, tolerance=1.0, - casing="lowercase", known_end="se", + casing="lowercase", known_end="se", two_word=True, )) texts = [r.text for r in results] assert "large house" in texts assert all(t.lower().endswith("se") for t in texts) + def test_two_word_disabled_by_default(self): + """Without two_word=True, only dictionary nouns are searched.""" + font = _get_test_font() + target = font.getlength("large house") + results = list(solve_word_dictionary( + font, target, tolerance=1.0, + casing="lowercase", known_start="large", + )) + texts = [r.text for r in results] + # "large house" is not a dictionary noun, so it should NOT appear + assert "large house" not in texts + def test_no_duplicates_across_phases(self): font = _get_test_font() target = font.getlength("dog") results = list(solve_word_dictionary( font, target, tolerance=1.0, - casing="lowercase", + casing="lowercase", two_word=True, )) texts = [r.text for r in results] assert len(texts) == len(set(texts)) diff --git a/unredact/pipeline/dictionary.py b/unredact/pipeline/dictionary.py index 1783bfe..b9d263b 100644 --- a/unredact/pipeline/dictionary.py +++ b/unredact/pipeline/dictionary.py @@ -276,11 +276,13 @@ def solve_word_dictionary( known_end: str = "", ensure_plural: bool = False, vocab_size: int = 0, + two_word: bool = False, ): - """Search English nouns (single-word) and adj+noun phrases (two-word). + """Search English nouns and optionally adj+noun phrases. - Yields results as found for SSE streaming. Phase 1 searches single nouns, - phase 2 searches two-word combinations using binary search. + Yields results as found for SSE streaming. Phase 1 searches nouns + (including multi-word nouns already in the dictionary). Phase 2 + (disabled by default) searches adjective+noun combinations. Word lists are frequency-sorted (most common first). vocab_size > 0 slices to the N most common words; 0 means no limit. @@ -327,6 +329,9 @@ def solve_word_dictionary( if error <= tolerance: yield SolveResult(text=display, width=float(width), error=float(error)) + if not two_word: + return + # Phase 2: two-word search (word1 + " " + noun) import bisect