Add KB scale stress tests and launch mode docs

This commit is contained in:
thePR0M3TH3AN committed 2026-03-01 18:33:11 -05:00
1 parent 9d1160f281
commit 176d12d9ae
6 files changed
+372

No files matched your search

+12
View File
@@ -446,12 +446,24 @@ Once installed, launch the interactive TUI with:
seedpass
```
Force the new Textual TUI v2 explicitly:
```bash
seedpass tui2
```
To launch the legacy interactive TUI explicitly:
```bash
seedpass legacy
```
You can also force legacy from any command invocation with:
```bash
seedpass --legacy-tui
```
You can also run directly from the repository with:
```bash
+3
View File
@@ -38,12 +38,15 @@ This directory contains SeedPass project documentation.
- `ai_agent_tui_testing.md`
- `agent_test_format.md`
- `installer_testing.md`
- `kb_scale_validation.md`
- `ci-health-2026-02-19.md`
- `agent-handoffs/README.md`
## Discoverability Commands
- `seedpass --help`
- `seedpass`
- `seedpass tui2`
- `seedpass --legacy-tui`
- `seedpass legacy`
- `seedpass <group> --help`
+46
View File
@@ -0,0 +1,46 @@
# KB Scale Validation
Status: Added on `beta` (March 1, 2026).
This document defines stress-oriented validation for knowledge-base behavior
under large index counts.
## Coverage
1. Sort/tag/search over large indexes:
- `src/tests/test_kb_scale_stress.py::test_kb_sort_tag_search_scale`
2. High-degree graph link handling:
- `src/tests/test_kb_scale_stress.py::test_kb_graph_high_degree_links_scale`
3. Textual TUI v2 large-index interactions:
- `src/tests/test_tui_v2_kb_scale_stress.py::test_tui2_kb_large_index_navigation`
## Dataset Profiles
- Standard:
- `10,000` entries for sort/search/tag
- `1,000` graph edges from one source node
- `10,000` TUI rows
- Stress (`--stress`):
- `100,000` entries for sort/search/tag
- `5,000` graph edges from one source node
- `50,000` TUI rows
## Commands
Standard run:
```bash
.venv/bin/python -m pytest -q src/tests/test_kb_scale_stress.py src/tests/test_tui_v2_kb_scale_stress.py
```
Stress run:
```bash
.venv/bin/python -m pytest -q --stress src/tests/test_kb_scale_stress.py src/tests/test_tui_v2_kb_scale_stress.py
```
## Notes
- Stress scenarios are opt-in and require `--stress`.
- Time budgets are intentionally conservative to catch regressions while
reducing CI flakiness across environments.
+10
View File
@@ -290,3 +290,13 @@
- Refactored legacy launch path through a shared helper in `seedpass.cli` so root fallback, `--legacy-tui`, `tui2` fallback, and `legacy` command all use consistent behavior and fingerprint forwarding.
- Added CLI regression tests for `seedpass legacy` with and without `--fingerprint`.
- Updated discoverability docs in `README.md` and `docs/README.md`.
## 2026-03-01 KB scale stress coverage + launch docs
- Added `src/tests/test_kb_scale_stress.py` for large-index KB validation:
- sort/tag/search across `10k` entries (and `100k` with `--stress`)
- high-degree graph link validation (`1k` edges standard, `5k` edges stress)
- Added `src/tests/test_tui_v2_kb_scale_stress.py` for Textual large-index interaction validation:
- pagination/navigation/search/filter behavior at `10k` rows standard and `50k` rows under `--stress`
- Added docs `docs/kb_scale_validation.md` with exact commands for standard and `--stress` runs.
- Updated launch documentation to clearly show all interactive launch modes:
- default (`seedpass`), explicit v2 (`seedpass tui2`), and legacy (`seedpass legacy` / `seedpass --legacy-tui`).
+176
View File
@@ -0,0 +1,176 @@
from __future__ import annotations
from pathlib import Path
from tempfile import TemporaryDirectory
from time import perf_counter
import pytest
from helpers import TEST_PASSWORD, TEST_SEED, create_vault
from seedpass.core.backup import BackupManager
from seedpass.core.config_manager import ConfigManager
from seedpass.core.entry_management import EntryManager
from seedpass.core.entry_types import EntryType
def _entry_manager(tmp_path: Path) -> EntryManager:
vault, _ = create_vault(tmp_path, TEST_SEED, TEST_PASSWORD)
cfg_mgr = ConfigManager(vault, tmp_path)
backup_mgr = BackupManager(tmp_path, cfg_mgr)
return EntryManager(vault, backup_mgr)
def _seed_bulk_entries(em: EntryManager, rows: int) -> None:
data = em._load_index(force_reload=True)
entries: dict[str, dict] = {}
for idx in range(rows):
label = f"KB-ENTRY-{idx:06d}"
tags = [f"tag-{idx % 100:03d}", "kb", "ops"]
if idx % 5 == 0:
entries[str(idx)] = {
"type": EntryType.DOCUMENT.value,
"kind": EntryType.DOCUMENT.value,
"label": label,
"content": f"doc-content-{idx}",
"file_type": "md",
"tags": tags,
"notes": "",
"links": [],
"archived": False,
"modified_ts": idx,
}
else:
entries[str(idx)] = {
"type": EntryType.PASSWORD.value,
"kind": EntryType.PASSWORD.value,
"label": label,
"username": f"user-{idx}",
"url": f"https://example/{idx}",
"length": 16,
"tags": tags,
"notes": "",
"links": [],
"archived": False,
"modified_ts": idx,
}
data["entries"] = entries
em._save_index(data)
@pytest.mark.parametrize(
("rows", "max_seconds"),
[
(10000, 10.0),
pytest.param(100000, 60.0, marks=pytest.mark.stress),
],
)
def test_kb_sort_tag_search_scale(rows: int, max_seconds: float) -> None:
with TemporaryDirectory() as tmpdir:
em = _entry_manager(Path(tmpdir))
_seed_bulk_entries(em, rows)
start = perf_counter()
by_label = em.list_entries(sort_by="label", verbose=False)
by_updated = em.list_entries(sort_by="updated", verbose=False)
tag_hits = em.search_entries("tag-042")
label_hits = em.search_entries(f"KB-ENTRY-{rows - 1:06d}")
elapsed = perf_counter() - start
assert len(by_label) == rows
assert by_label[0][1] == "KB-ENTRY-000000"
assert by_label[-1][1] == f"KB-ENTRY-{rows - 1:06d}"
assert len(by_updated) == rows
assert by_updated[0][0] == rows - 1
assert by_updated[-1][0] == 0
expected_tag_hits = rows // 100 + (1 if rows % 100 > 42 else 0)
assert len(tag_hits) == expected_tag_hits
assert label_hits == [
(
rows - 1,
f"KB-ENTRY-{rows - 1:06d}",
f"user-{rows - 1}",
f"https://example/{rows - 1}",
False,
EntryType.PASSWORD,
)
] or label_hits == [
(
rows - 1,
f"KB-ENTRY-{rows - 1:06d}",
None,
None,
False,
EntryType.DOCUMENT,
)
]
assert (
elapsed <= max_seconds
), f"KB scale sort/search exceeded budget: {elapsed:.3f}s > {max_seconds:.3f}s"
@pytest.mark.parametrize(
("degree", "max_seconds"),
[
(1000, 6.0),
pytest.param(5000, 30.0, marks=pytest.mark.stress),
],
)
def test_kb_graph_high_degree_links_scale(degree: int, max_seconds: float) -> None:
with TemporaryDirectory() as tmpdir:
em = _entry_manager(Path(tmpdir))
data = em._load_index(force_reload=True)
entries: dict[str, dict] = {
"0": {
"type": EntryType.DOCUMENT.value,
"kind": EntryType.DOCUMENT.value,
"label": "KB Root",
"content": "root",
"file_type": "md",
"tags": ["kb", "graph"],
"links": [],
"archived": False,
"modified_ts": 0,
}
}
for i in range(1, degree + 1):
entries[str(i)] = {
"type": EntryType.KEY_VALUE.value,
"kind": EntryType.KEY_VALUE.value,
"label": f"Node-{i:05d}",
"key": f"k{i}",
"value": f"v{i}",
"tags": [f"group-{i % 10}"],
"links": [],
"archived": False,
"modified_ts": i,
}
entries["0"]["links"] = [
{
"target_id": i,
"relation": "depends_on" if i % 2 else "references",
"note": f"edge-{i}",
}
for i in range(1, degree + 1)
]
data["entries"] = entries
em._save_index(data)
start = perf_counter()
resolved = em.get_links(0)
relation_hits = em.search_entries("depends_on")
target_hits = em.search_entries(f"Node-{degree:05d}")
target_depends_on = degree if degree % 2 else degree - 1
remaining = em.remove_link(0, target_depends_on, relation="depends_on")
elapsed = perf_counter() - start
assert len(resolved) == degree
assert resolved[0]["target_label"].startswith("Node-")
assert resolved[0]["target_kind"] == EntryType.KEY_VALUE.value
assert relation_hits and relation_hits[0][0] == 0
assert target_hits and target_hits[0][0] == 0
assert len(remaining) == degree - 1
assert (
elapsed <= max_seconds
), f"KB graph high-degree operations exceeded budget: {elapsed:.3f}s > {max_seconds:.3f}s"
+125
View File
@@ -0,0 +1,125 @@
from __future__ import annotations
from time import perf_counter
from types import SimpleNamespace
import pytest
pytest.importorskip("textual")
from textual.widgets import Input, ListView
from seedpass.tui_v2.app import launch_tui2
class LargeKbService:
def __init__(self, rows: int) -> None:
self._entries = {}
for idx in range(1, rows + 1):
kind = "document" if idx % 5 == 0 else "password"
entry = {
"id": idx,
"kind": kind,
"label": f"KB-{idx:06d}",
"archived": False,
"tags": [f"tag-{idx % 100:03d}", "kb"],
}
if kind == "document":
entry["content"] = f"doc-{idx}"
entry["file_type"] = "md"
else:
entry["length"] = 16
entry["username"] = f"user-{idx}"
entry["url"] = f"https://example/{idx}"
self._entries[idx] = entry
def search_entries(self, query: str, kinds: list[str] | None = None):
q = (query or "").strip().lower()
rows = []
for idx in sorted(self._entries.keys()):
entry = self._entries[idx]
kind = str(entry.get("kind", "password"))
if kinds and kind not in kinds:
continue
label = str(entry.get("label", ""))
tags = [str(tag).lower() for tag in entry.get("tags", [])]
if q and q not in label.lower() and not any(q in t for t in tags):
continue
rows.append(
(
idx,
label,
entry.get("username"),
entry.get("url"),
bool(entry.get("archived", False)),
SimpleNamespace(value=kind),
)
)
return rows
def retrieve_entry(self, entry_id: int):
return dict(self._entries.get(int(entry_id), {}))
def get_links(self, entry_id: int):
_ = entry_id
return []
def generate_password(self, length: int, entry_id: int) -> str:
return f"pw-{entry_id}-{length}"
def _build_app(service: LargeKbService):
holder: dict[str, object] = {}
def _hook(app):
holder["app"] = app
launched = launch_tui2(entry_service_factory=lambda: service, app_hook=_hook)
assert launched is True
app = holder.get("app")
assert app is not None
return app
def _filters(app) -> str:
return str(app.query_one("#filters").render())
@pytest.mark.anyio
@pytest.mark.parametrize(
("rows", "max_seconds"),
[
(10000, 6.0),
pytest.param(50000, 18.0, marks=pytest.mark.stress),
],
)
async def test_tui2_kb_large_index_navigation(rows: int, max_seconds: float) -> None:
app = _build_app(LargeKbService(rows))
async with app.run_test() as pilot:
start = perf_counter()
await pilot.pause()
list_view = app.query_one("#entry-list", ListView)
assert len(list_view.children) == 200
assert "Page: 1/" in _filters(app)
app.action_next_page()
app.action_next_page()
await pilot.pause()
assert "Page: 3/" in _filters(app)
app.query_one("#search", Input).value = f"KB-{rows:06d}"
app._load_entries(query=f"KB-{rows:06d}", reset_page=True)
await pilot.pause()
assert len(list_view.children) == 1
app._run_palette_command("filter document")
await pilot.pause()
app._run_palette_command("search tag-040")
await pilot.pause()
assert len(list_view.children) >= 1
elapsed = perf_counter() - start
assert (
elapsed <= max_seconds
), f"TUI large-index interaction exceeded budget: {elapsed:.3f}s > {max_seconds:.3f}s"