feat: add noun/adjective/plural word list loaders

Co-Authored-By: Claude Opus 4.6 <[email protected]>

tests/test_word_lists.py

@@ -0,0 +1,41 @@
+# tests/test_word_lists.py
+"""Tests for word list loading (nouns, adjectives, plurals)."""
+
+from unredact.pipeline.word_filter import (
+    _get_nouns,
+    _get_nouns_plural,
+    _get_adjectives,
+)
+
+
+class TestWordListLoaders:
+    def test_nouns_loads(self):
+        nouns = _get_nouns()
+        assert len(nouns) > 1000
+        assert isinstance(nouns, list)
+        assert "dog" in nouns
+
+    def test_nouns_plural_loads(self):
+        plurals = _get_nouns_plural()
+        assert len(plurals) > 1000
+        assert isinstance(plurals, list)
+        assert "dogs" in plurals
+
+    def test_adjectives_loads(self):
+        adjectives = _get_adjectives()
+        assert len(adjectives) > 1000
+        assert isinstance(adjectives, list)
+        assert "large" in adjectives
+
+    def test_nouns_and_plurals_same_length(self):
+        nouns = _get_nouns()
+        plurals = _get_nouns_plural()
+        assert len(nouns) == len(plurals)
+
+    def test_all_words_lowercase_alpha(self):
+        import re
+        pattern = re.compile(r"^[a-z]+$")
+        for word in _get_nouns()[:100]:
+            assert pattern.match(word), f"Non-alpha noun: {word}"
+        for word in _get_adjectives()[:100]:
+            assert pattern.match(word), f"Non-alpha adjective: {word}"

unredact/pipeline/word_filter.py

@@ -92,6 +92,39 @@ def _get_associate_variants() -> list[str]:
     return _associate_variants
 
 
+
+_nouns: list[str] | None = None
+_nouns_plural: list[str] | None = None
+_adjectives: list[str] | None = None
+
+
+def _load_list(filename: str) -> list[str]:
+    path = DATA_DIR / filename
+    if not path.exists():
+        return []
+    return [line.strip() for line in path.read_text().splitlines() if line.strip()]
+
+
+def _get_nouns() -> list[str]:
+    global _nouns
+    if _nouns is None:
+        _nouns = _load_list("nouns.txt")
+    return _nouns
+
+
+def _get_nouns_plural() -> list[str]:
+    global _nouns_plural
+    if _nouns_plural is None:
+        _nouns_plural = _load_list("nouns_plural.txt")
+    return _nouns_plural
+
+
+def _get_adjectives() -> list[str]:
+    global _adjectives
+    if _adjectives is None:
+        _adjectives = _load_list("adjectives.txt")
+    return _adjectives
+
 def _get_all_names() -> set[str]:
     return _get_first_names() | _get_last_names()