mirror of
https://github.com/PR0M3TH3AN/unredact.git
synced 2026-10-27 08:40:29 +00:00
feat: disable two-word adjective+noun search by default
The new Wiktionary dictionary already includes 31k multi-word nouns, making the combinatorial adjective+noun phase redundant. Add two_word parameter (default False) to solve_word_dictionary to opt in when needed. Co-Authored-By: Claude Opus 4.6 <[email protected]>
This commit is contained in:
1 parent
131c945641
commit
a0e2bf6ec1
2 files changed
+26
-8
No files matched your search
@@ -94,7 +94,7 @@ class TestSolveWordDictionaryTwoWord:
|
||||
target = font.getlength("large house")
|
||||
results = list(solve_word_dictionary(
|
||||
font, target, tolerance=1.0,
|
||||
casing="lowercase", known_start="large",
|
||||
casing="lowercase", known_start="large", two_word=True,
|
||||
))
|
||||
texts = [r.text for r in results]
|
||||
assert "large house" in texts
|
||||
@@ -104,7 +104,7 @@ class TestSolveWordDictionaryTwoWord:
|
||||
target = font.getlength("Large House")
|
||||
results = list(solve_word_dictionary(
|
||||
font, target, tolerance=1.0,
|
||||
casing="capitalized", known_start="Large",
|
||||
casing="capitalized", known_start="Large", two_word=True,
|
||||
))
|
||||
texts = [r.text for r in results]
|
||||
assert "Large House" in texts
|
||||
@@ -115,6 +115,7 @@ class TestSolveWordDictionaryTwoWord:
|
||||
results = list(solve_word_dictionary(
|
||||
font, target, tolerance=1.0,
|
||||
casing="lowercase", ensure_plural=True, known_start="large",
|
||||
two_word=True,
|
||||
))
|
||||
texts = [r.text for r in results]
|
||||
assert "large houses" in texts
|
||||
@@ -124,7 +125,7 @@ class TestSolveWordDictionaryTwoWord:
|
||||
target = font.getlength("large house")
|
||||
results = list(solve_word_dictionary(
|
||||
font, target, tolerance=1.0,
|
||||
casing="lowercase", known_start="la",
|
||||
casing="lowercase", known_start="la", two_word=True,
|
||||
))
|
||||
texts = [r.text for r in results]
|
||||
assert "large house" in texts
|
||||
@@ -135,18 +136,30 @@ class TestSolveWordDictionaryTwoWord:
|
||||
target = font.getlength("large house")
|
||||
results = list(solve_word_dictionary(
|
||||
font, target, tolerance=1.0,
|
||||
casing="lowercase", known_end="se",
|
||||
casing="lowercase", known_end="se", two_word=True,
|
||||
))
|
||||
texts = [r.text for r in results]
|
||||
assert "large house" in texts
|
||||
assert all(t.lower().endswith("se") for t in texts)
|
||||
|
||||
def test_two_word_disabled_by_default(self):
|
||||
"""Without two_word=True, only dictionary nouns are searched."""
|
||||
font = _get_test_font()
|
||||
target = font.getlength("large house")
|
||||
results = list(solve_word_dictionary(
|
||||
font, target, tolerance=1.0,
|
||||
casing="lowercase", known_start="large",
|
||||
))
|
||||
texts = [r.text for r in results]
|
||||
# "large house" is not a dictionary noun, so it should NOT appear
|
||||
assert "large house" not in texts
|
||||
|
||||
def test_no_duplicates_across_phases(self):
|
||||
font = _get_test_font()
|
||||
target = font.getlength("dog")
|
||||
results = list(solve_word_dictionary(
|
||||
font, target, tolerance=1.0,
|
||||
casing="lowercase",
|
||||
casing="lowercase", two_word=True,
|
||||
))
|
||||
texts = [r.text for r in results]
|
||||
assert len(texts) == len(set(texts))
|
||||
@@ -276,11 +276,13 @@ def solve_word_dictionary(
|
||||
known_end: str = "",
|
||||
ensure_plural: bool = False,
|
||||
vocab_size: int = 0,
|
||||
two_word: bool = False,
|
||||
):
|
||||
"""Search English nouns (single-word) and adj+noun phrases (two-word).
|
||||
"""Search English nouns and optionally adj+noun phrases.
|
||||
|
||||
Yields results as found for SSE streaming. Phase 1 searches single nouns,
|
||||
phase 2 searches two-word combinations using binary search.
|
||||
Yields results as found for SSE streaming. Phase 1 searches nouns
|
||||
(including multi-word nouns already in the dictionary). Phase 2
|
||||
(disabled by default) searches adjective+noun combinations.
|
||||
|
||||
Word lists are frequency-sorted (most common first). vocab_size > 0 slices
|
||||
to the N most common words; 0 means no limit.
|
||||
@@ -327,6 +329,9 @@ def solve_word_dictionary(
|
||||
if error <= tolerance:
|
||||
yield SolveResult(text=display, width=float(width), error=float(error))
|
||||
|
||||
if not two_word:
|
||||
return
|
||||
|
||||
# Phase 2: two-word search (word1 + " " + noun)
|
||||
import bisect
|
||||
|
||||
|
||||
Reference in new issue
Block a user