mirror of
https://github.com/PR0M3TH3AN/SeedPass.git
synced 2026-10-27 16:02:05 +00:00
369 lines
11 KiB
Python
Executable File
369 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Deterministic AI-agent harness for SeedPass TUI v2.
|
|
|
|
This harness uses Textual's test pilot API to simulate user/agent behavior and
|
|
produces a machine-readable report for CI and local regressions.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import time
|
|
from dataclasses import dataclass, asdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
from seedpass.tui_v2.app import check_tui2_runtime, launch_tui2
|
|
|
|
|
|
@dataclass
|
|
class StepResult:
|
|
step_id: str
|
|
description: str
|
|
status: str
|
|
duration_sec: float
|
|
detail: str | None = None
|
|
|
|
|
|
class HarnessError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class HarnessEntryService:
|
|
def __init__(self) -> None:
|
|
self.entries = {
|
|
1: {
|
|
"id": 1,
|
|
"kind": "document",
|
|
"label": "Runbook",
|
|
"content": "alpha",
|
|
"file_type": "md",
|
|
"tags": ["ops"],
|
|
"archived": False,
|
|
},
|
|
2: {
|
|
"id": 2,
|
|
"kind": "password",
|
|
"label": "Console Login",
|
|
"archived": False,
|
|
},
|
|
}
|
|
self.links = {
|
|
1: [{"target": 2, "relation": "references", "note": "credential"}],
|
|
2: [],
|
|
}
|
|
|
|
def search_entries(self, query: str, kinds: list[str] | None = None):
|
|
q = (query or "").strip().lower()
|
|
out = []
|
|
for idx in sorted(self.entries.keys()):
|
|
entry = self.entries[idx]
|
|
kind = str(entry.get("kind", "password"))
|
|
if kinds and kind not in kinds:
|
|
continue
|
|
label = str(entry.get("label", ""))
|
|
if q and q not in label.lower():
|
|
continue
|
|
out.append(
|
|
(
|
|
idx,
|
|
label,
|
|
None,
|
|
None,
|
|
bool(entry.get("archived", False)),
|
|
SimpleNamespace(value=kind),
|
|
)
|
|
)
|
|
return out
|
|
|
|
def retrieve_entry(self, entry_id: int):
|
|
return dict(self.entries.get(int(entry_id), {}))
|
|
|
|
def modify_entry(self, entry_id: int, **kwargs):
|
|
entry = self.entries[int(entry_id)]
|
|
for key, value in kwargs.items():
|
|
if value is not None:
|
|
entry[key] = value
|
|
|
|
def archive_entry(self, entry_id: int):
|
|
self.entries[int(entry_id)]["archived"] = True
|
|
|
|
def restore_entry(self, entry_id: int):
|
|
self.entries[int(entry_id)]["archived"] = False
|
|
|
|
def get_links(self, entry_id: int):
|
|
return [dict(link) for link in self.links.get(int(entry_id), [])]
|
|
|
|
def add_link(
|
|
self,
|
|
entry_id: int,
|
|
target_id: int,
|
|
*,
|
|
relation: str = "related_to",
|
|
note: str = "",
|
|
):
|
|
links = self.links.setdefault(int(entry_id), [])
|
|
links.append({"target": int(target_id), "relation": relation, "note": note})
|
|
return [dict(link) for link in links]
|
|
|
|
def remove_link(
|
|
self, entry_id: int, target_id: int, *, relation: str | None = None
|
|
):
|
|
src = self.links.setdefault(int(entry_id), [])
|
|
kept = []
|
|
for link in src:
|
|
if int(link.get("target", -1)) != int(target_id):
|
|
kept.append(link)
|
|
continue
|
|
if relation is not None and str(link.get("relation")) != relation:
|
|
kept.append(link)
|
|
self.links[int(entry_id)] = kept
|
|
return [dict(link) for link in kept]
|
|
|
|
|
|
def _parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Run deterministic TUI v2 agent validation harness."
|
|
)
|
|
parser.add_argument(
|
|
"--scenario",
|
|
choices=("core", "extended"),
|
|
default="extended",
|
|
help="Validation scenario profile (default: extended).",
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
type=Path,
|
|
default=Path("artifacts/agent_tui2_test"),
|
|
help="Directory for report artifacts.",
|
|
)
|
|
parser.add_argument(
|
|
"--verbose",
|
|
action="store_true",
|
|
help="Print step progress.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def _status(app) -> str:
|
|
return str(app.query_one("#status").render())
|
|
|
|
|
|
def _assert_contains(haystack: str, needle: str, context: str) -> None:
|
|
if needle not in haystack:
|
|
raise HarnessError(f"{context}: expected {needle!r} in {haystack!r}")
|
|
|
|
|
|
async def _scenario_core(
|
|
app, service: HarnessEntryService, coverage: dict[str, bool], pilot
|
|
) -> None:
|
|
from textual.widgets import Input
|
|
|
|
await pilot.pause()
|
|
coverage["startup"] = True
|
|
|
|
app.query_one("#search", Input).value = "runbook"
|
|
app._load_entries(query="runbook", reset_page=True)
|
|
await pilot.pause()
|
|
coverage["search"] = True
|
|
|
|
app._run_palette_command("open 1")
|
|
await pilot.pause()
|
|
app.action_edit_document()
|
|
await pilot.pause()
|
|
app.query_one("#doc-edit-label", Input).value = "Runbook Updated"
|
|
app.query_one("#doc-edit-file-type", Input).value = "txt"
|
|
app.query_one("#doc-edit-tags", Input).value = "ops,core"
|
|
if len(app.query("#doc-edit-content")) > 0:
|
|
area = app.query_one("#doc-edit-content")
|
|
if hasattr(area, "load_text"):
|
|
area.load_text("alpha\nbeta")
|
|
else:
|
|
area.text = "alpha\nbeta"
|
|
else:
|
|
app.query_one("#doc-edit-content-single", Input).value = "alpha\nbeta"
|
|
app.action_save_document()
|
|
await pilot.pause()
|
|
_assert_contains(_status(app), "Saved document", "save-doc")
|
|
coverage["document_edit"] = True
|
|
|
|
app._run_palette_command("link-add 2 references auth")
|
|
await pilot.pause()
|
|
app.action_open_link_target()
|
|
await pilot.pause()
|
|
_assert_contains(_status(app), "Opened linked entry 2", "link-open")
|
|
coverage["graph_navigation"] = True
|
|
|
|
app._run_palette_command("open 1")
|
|
await pilot.pause()
|
|
app.action_toggle_archive()
|
|
await pilot.pause()
|
|
if "archived" not in _status(app):
|
|
raise HarnessError(f"archive: expected archived status, got {_status(app)!r}")
|
|
app._run_palette_command("open 1")
|
|
await pilot.pause()
|
|
app.action_toggle_archive()
|
|
await pilot.pause()
|
|
if service.entries[1].get("archived", False):
|
|
raise HarnessError(
|
|
"restore: expected entry 1 to be unarchived after second toggle"
|
|
)
|
|
coverage["archive_restore"] = True
|
|
|
|
|
|
async def _scenario_extended(
|
|
app, service: HarnessEntryService, coverage: dict[str, bool], pilot
|
|
) -> None:
|
|
await _scenario_core(app, service, coverage, pilot)
|
|
app.action_open_palette()
|
|
await pilot.pause()
|
|
app._run_palette_command("unknown-cmd")
|
|
await pilot.pause()
|
|
_assert_contains(_status(app), "Unknown command", "palette-unknown")
|
|
|
|
app._run_palette_command("page 1")
|
|
await pilot.pause()
|
|
coverage["palette_navigation"] = True
|
|
|
|
app.action_toggle_help()
|
|
await pilot.pause()
|
|
_assert_contains(_status(app), "Help opened", "help-open")
|
|
app.action_cancel_document_edit()
|
|
await pilot.pause()
|
|
_assert_contains(_status(app), "Help closed", "help-close")
|
|
coverage["help_overlay"] = True
|
|
|
|
|
|
async def _run_scenario(
|
|
app, service: HarnessEntryService, coverage: dict[str, bool], scenario: str
|
|
) -> None:
|
|
async with app.run_test() as pilot:
|
|
if scenario == "core":
|
|
await _scenario_core(app, service, coverage, pilot)
|
|
else:
|
|
await _scenario_extended(app, service, coverage, pilot)
|
|
|
|
|
|
def _run_step(
|
|
step_id: str,
|
|
description: str,
|
|
fn,
|
|
results: list[StepResult],
|
|
*,
|
|
verbose: bool = False,
|
|
):
|
|
start = time.monotonic()
|
|
try:
|
|
asyncio.run(fn())
|
|
except Exception as exc:
|
|
duration = time.monotonic() - start
|
|
results.append(
|
|
StepResult(
|
|
step_id=step_id,
|
|
description=description,
|
|
status="failed",
|
|
duration_sec=duration,
|
|
detail=str(exc),
|
|
)
|
|
)
|
|
raise
|
|
duration = time.monotonic() - start
|
|
results.append(
|
|
StepResult(
|
|
step_id=step_id,
|
|
description=description,
|
|
status="passed",
|
|
duration_sec=duration,
|
|
)
|
|
)
|
|
if verbose:
|
|
print(f"[PASS] {step_id} ({duration:.2f}s)")
|
|
|
|
|
|
def main() -> int:
|
|
args = _parse_args()
|
|
runtime = check_tui2_runtime()
|
|
if not runtime.get("textual_available", False):
|
|
print("Error: TUI v2 runtime unavailable. Install textual.")
|
|
return 2
|
|
|
|
artifact_dir = args.output_dir / datetime.now(timezone.utc).strftime(
|
|
"%Y%m%dT%H%M%SZ"
|
|
)
|
|
artifact_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
service = HarnessEntryService()
|
|
holder: dict[str, object] = {}
|
|
|
|
def _hook(app):
|
|
holder["app"] = app
|
|
|
|
launched = launch_tui2(entry_service_factory=lambda: service, app_hook=_hook)
|
|
if not launched:
|
|
print("Error: failed to launch TUI v2 app")
|
|
return 2
|
|
app = holder.get("app")
|
|
if app is None:
|
|
print("Error: app hook did not receive app")
|
|
return 2
|
|
|
|
coverage = {
|
|
"startup": False,
|
|
"search": False,
|
|
"document_edit": False,
|
|
"graph_navigation": False,
|
|
"archive_restore": False,
|
|
"palette_navigation": False,
|
|
"help_overlay": False,
|
|
}
|
|
results: list[StepResult] = []
|
|
start = time.monotonic()
|
|
status = "passed"
|
|
failure = ""
|
|
try:
|
|
if args.scenario == "core":
|
|
_run_step(
|
|
"core",
|
|
"Drive core TUI v2 user/agent workflow.",
|
|
lambda: _run_scenario(app, service, coverage, "core"),
|
|
results,
|
|
verbose=args.verbose,
|
|
)
|
|
else:
|
|
_run_step(
|
|
"extended",
|
|
"Drive extended TUI v2 user/agent workflow.",
|
|
lambda: _run_scenario(app, service, coverage, "extended"),
|
|
results,
|
|
verbose=args.verbose,
|
|
)
|
|
except Exception as exc:
|
|
status = "failed"
|
|
failure = str(exc)
|
|
|
|
report = {
|
|
"timestamp_utc": datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ"),
|
|
"scenario": args.scenario,
|
|
"status": status,
|
|
"failure": failure,
|
|
"duration_sec": round(time.monotonic() - start, 3),
|
|
"coverage_points": coverage,
|
|
"steps": [asdict(item) for item in results],
|
|
"activity_log": list(getattr(app, "_activity_log", [])),
|
|
}
|
|
report_file = artifact_dir / "report.json"
|
|
report_file.write_text(json.dumps(report, indent=2), encoding="utf-8")
|
|
print(f"AI agent TUI v2 report: {report_file}")
|
|
print(f"Status: {status}")
|
|
if failure:
|
|
print(f"Failure: {failure}")
|
|
return 0 if status == "passed" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|