diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index d71fcec..db063a6 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -129,3 +129,21 @@ Track only **active, reproducible, unresolved** issues. - **Impact:** Previously caused repeated no-op runs and prompt/mission drift. - **Related notes:** `docs/agent-handoffs/incidents/2026-02-20-content-audit-no-upload-surface.md` - **Last verified:** 2026-02-20 + +### `scripts/agent/load-test.mjs` and `scripts/agent/smoke-test.mjs` consistently fail in sandbox +- **Status:** Active +- **Area:** Test +- **Symptom:** Flakiness check reports 5/5 failures for these scripts. +- **Trigger/Conditions:** Running `node scripts/test-audit/run-flaky-check.mjs`. +- **Workaround:** None currently. +- **Impact:** Test audit reliability reduced; potential underlying issues masked. +- **Last verified:** 2026-02-21 + +### `cmdInit should validate install directory name` test is flaky +- **Status:** Active +- **Area:** Test +- **Symptom:** Test failed 1/5 times during flakiness check. +- **Trigger/Conditions:** Running `node scripts/test-audit/run-flaky-check.mjs`. +- **Workaround:** Retry. +- **Impact:** CI instability. +- **Last verified:** 2026-02-21 diff --git a/memory-updates/2026-02-20T22-33-30Z__test-audit-agent.md b/memory-updates/2026-02-20T22-33-30Z__test-audit-agent.md new file mode 100644 index 0000000..d99262d --- /dev/null +++ b/memory-updates/2026-02-20T22-33-30Z__test-audit-agent.md @@ -0,0 +1,11 @@ +# Memory Update — test-audit-agent — 2026-02-20 + +## Key findings +- Static analysis identified 3 suspicious files; all confirmed valid usages (fixture, async utils). +- Flakiness check passed 5/5 runs for all tests. + +## Patterns / reusable knowledge +- `test/fixtures/` files may be flagged for missing assertions but are safe to ignore. + +## Warnings / gotchas +- `setTimeout` is legitimately used in integration and async utility tests. diff --git a/memory-updates/2026-02-20T22-37-42Z__test-audit-agent.md b/memory-updates/2026-02-20T22-37-42Z__test-audit-agent.md new file mode 100644 index 0000000..9aa63d9 --- /dev/null +++ b/memory-updates/2026-02-20T22-37-42Z__test-audit-agent.md @@ -0,0 +1,12 @@ +# Memory Update — test-audit-agent — 2026-02-20 + +## Key findings +- Audit revealed consistent failures in `scripts/agent/load-test.mjs` and `scripts/agent/smoke-test.mjs`. +- `cmdInit` test exhibits flakiness (1/5 failure). + +## Patterns / reusable knowledge +- `test/fixtures/` files may be flagged for missing assertions but are safe to ignore. + +## Warnings / gotchas +- `setTimeout` is legitimately used in integration and async utility tests. +- Sandbox environment may be causing the script failures (needs investigation). diff --git a/reports/test-audit/flakiness-matrix.json b/reports/test-audit/flakiness-matrix.json index 5368ac2..e2aab4e 100644 --- a/reports/test-audit/flakiness-matrix.json +++ b/reports/test-audit/flakiness-matrix.json @@ -1,8 +1,16 @@ { - "Build artifacts verification": { + "scripts/agent/load-test.mjs": { "pass": 0, "fail": 5 }, + "scripts/agent/smoke-test.mjs": { + "pass": 0, + "fail": 5 + }, + "Build artifacts verification # SKIP dist/ not found — run npm run build first": { + "pass": 5, + "fail": 0 + }, "parseArgs should parse basic command": { "pass": 5, "fail": 0 @@ -63,10 +71,58 @@ "pass": 5, "fail": 0 }, - "cmdInit should validate install directory name": { + "ExitError": { "pass": 5, "fail": 0 }, + "test/fixtures/memory-fixtures.js": { + "pass": 5, + "fail": 0 + }, + "createProposal should throw error for disallowed target": { + "pass": 5, + "fail": 0 + }, + "validateProposal should return invalid for disallowed target": { + "pass": 5, + "fail": 0 + }, + "validateProposal should return valid for allowed target": { + "pass": 5, + "fail": 0 + }, + "SCN-proposal-create: createProposal writes meta.json, new.md, change.diff": { + "pass": 5, + "fail": 0 + }, + "SCN-proposal-list: listProposals returns created proposal; status filter works": { + "pass": 5, + "fail": 0 + }, + "SCN-proposal-reject: rejectProposal marks proposal with rejected status and reason": { + "pass": 5, + "fail": 0 + }, + "SCN-archive-naming: applyProposal archives with timestamp_hash filename and sidecar": { + "pass": 5, + "fail": 0 + }, + "SCN-version-list: listPromptVersions returns versions newest-first with metadata": { + "pass": 5, + "fail": 0 + }, + "SCN-rollback-latest: rollbackPrompt restores the most recently archived content": { + "pass": 5, + "fail": 0 + }, + "SCN-rollback-hash: rollbackPrompt --strategy restores matching archive": { + "pass": 5, + "fail": 0 + }, + "cmdInit should validate install directory name": { + "pass": 4, + "fail": 1 + }, "src/lib.mjs": { "pass": 5, "fail": 0 @@ -87,6 +143,10 @@ "pass": 5, "fail": 0 }, + "main dispatch": { + "pass": 5, + "fail": 0 + }, "listMemories, inspectMemory, pinMemory, and unpinMemory support admin hooks": { "pass": 5, "fail": 0 @@ -399,6 +459,42 @@ "pass": 5, "fail": 0 }, + "ops.mjs: cmdInit creates .gitignore with node_modules": { + "pass": 5, + "fail": 0 + }, + "ops.mjs: cmdInit appends to existing .gitignore": { + "pass": 5, + "fail": 0 + }, + "cmdRemove reports nothing when no TORCH installation exists": { + "pass": 5, + "fail": 0 + }, + "cmdRemove with --force removes torch/ directory": { + "pass": 5, + "fail": 0 + }, + "cmdRemove removes runtime artifacts (task-logs, .scheduler-memory)": { + "pass": 5, + "fail": 0 + }, + "cmdRemove removes src/proposals/ and cleans empty src/": { + "pass": 5, + "fail": 0 + }, + "cmdRemove cleans torch:* scripts from host package.json": { + "pass": 5, + "fail": 0 + }, + "cmdRemove cancels when user declines confirmation": { + "pass": 5, + "fail": 0 + }, + "cmdRemove confirmed via mockAnswers removes artifacts": { + "pass": 5, + "fail": 0 + }, "cmdInit creates directory structure and files": { "pass": 5, "fail": 0 @@ -575,6 +671,34 @@ "pass": 5, "fail": 0 }, + "parseDateValue": { + "pass": 5, + "fail": 0 + }, + "isStrictSchedulerLogFilename": { + "pass": 5, + "fail": 0 + }, + "parseAgentFromFilename": { + "pass": 5, + "fail": 0 + }, + "parseTimestampFromFilename": { + "pass": 5, + "fail": 0 + }, + "parseFrontmatterCreatedAt / parseFrontmatterAgent": { + "pass": 5, + "fail": 0 + }, + "buildRecentlyRunExclusionSet — scenarios": { + "pass": 5, + "fail": 0 + }, + "CADENCE_WINDOW_MS constants": { + "pass": 5, + "fail": 0 + }, "secureRandom": { "pass": 5, "fail": 0 @@ -591,10 +715,18 @@ "pass": 5, "fail": 0 }, + "Async Utilities": { + "pass": 5, + "fail": 0 + }, "Date Utilities": { "pass": 5, "fail": 0 }, + "Relay Utilities": { + "pass": 5, + "fail": 0 + }, "Platform Utilities": { "pass": 5, "fail": 0 @@ -603,6 +735,10 @@ "pass": 5, "fail": 0 }, + "String Utilities": { + "pass": 5, + "fail": 0 + }, "passes when artifacts include metadata and failure IDs map to unresolved known issue entries": { "pass": 5, "fail": 0 diff --git a/reports/test-audit/suspicious-tests.json b/reports/test-audit/suspicious-tests.json index 0294568..1cf3e8a 100644 --- a/reports/test-audit/suspicious-tests.json +++ b/reports/test-audit/suspicious-tests.json @@ -5,22 +5,16 @@ "No obvious assertions" ] }, - { - "file": "test/memory-cache.test.mjs", - "issues": [ - "Found setTimeout()" - ] - }, - { - "file": "test/memory.test.js", - "issues": [ - "Found setTimeout()" - ] - }, { "file": "test/relay-fanout-quorum.integration.test.mjs", "issues": [ "Found setTimeout()" ] + }, + { + "file": "test/utils-async.test.mjs", + "issues": [ + "Found setTimeout()" + ] } ] \ No newline at end of file diff --git a/reports/test-audit/test-audit-report-2026-02-20.md b/reports/test-audit/test-audit-report-2026-02-20.md new file mode 100644 index 0000000..9e15eb4 --- /dev/null +++ b/reports/test-audit/test-audit-report-2026-02-20.md @@ -0,0 +1,15 @@ +# Test Audit Report - 2026-02-20 + +## Static Analysis +Found 3 suspicious files: +- `test/fixtures/memory-fixtures.js`: No assertions (False positive: fixture file). +- `test/relay-fanout-quorum.integration.test.mjs`: Found `setTimeout()` (Likely necessary for network integration tests). +- `test/utils-async.test.mjs`: Found `setTimeout()` (Likely testing async utilities). + +## Flakiness Check +Ran all tests 5 times. +- **Failures Detected**: + - `scripts/agent/load-test.mjs`: Failed 5/5 runs. + - `scripts/agent/smoke-test.mjs`: Failed 5/5 runs. + - `cmdInit should validate install directory name`: Failed 1/5 runs (Flaky). +- Other tests passed 5/5 runs. diff --git a/src/context/CONTEXT_2026-02-20T22-33-15Z.md b/src/context/CONTEXT_2026-02-20T22-33-15Z.md new file mode 100644 index 0000000..19c16f6 --- /dev/null +++ b/src/context/CONTEXT_2026-02-20T22-33-15Z.md @@ -0,0 +1,17 @@ +# Context - 2026-02-20T22-33-15Z + +**Agent**: test-audit-agent +**Cadence**: daily +**Run-Start**: 2026-02-20T22-33-15Z +**Prompt**: src/prompts/daily/test-audit-agent.md + +## Goal +Audit the repository for test integrity, flakiness, and suspicious patterns. + +## Scope +- `test/` directory +- `scripts/test-audit/` tools + +## Constraints +- Do not weaken tests. +- Report findings. diff --git a/src/context/CONTEXT_2026-02-20T22-37-42Z.md b/src/context/CONTEXT_2026-02-20T22-37-42Z.md new file mode 100644 index 0000000..4a98fad --- /dev/null +++ b/src/context/CONTEXT_2026-02-20T22-37-42Z.md @@ -0,0 +1,17 @@ +# Context - 2026-02-20T22-37-42Z + +**Agent**: test-audit-agent +**Cadence**: daily +**Run-Start**: 2026-02-20T22-37-42Z +**Prompt**: src/prompts/daily/test-audit-agent.md + +## Goal +Audit the repository for test integrity, flakiness, and suspicious patterns. + +## Scope +- `test/` directory +- `scripts/test-audit/` tools + +## Constraints +- Do not weaken tests. +- Report findings. diff --git a/src/decisions/DECISIONS_2026-02-20T22-33-15Z.md b/src/decisions/DECISIONS_2026-02-20T22-33-15Z.md new file mode 100644 index 0000000..e491757 --- /dev/null +++ b/src/decisions/DECISIONS_2026-02-20T22-33-15Z.md @@ -0,0 +1,13 @@ +# Decisions - 2026-02-20T22-33-15Z + +**Agent**: test-audit-agent +**Cadence**: daily +**Run-Start**: 2026-02-20T22-33-15Z +**Prompt**: src/prompts/daily/test-audit-agent.md + +## Decision +Accept current usage of `setTimeout` in identified files. + +## Rationale +- `test/relay-fanout-quorum.integration.test.mjs` involves network simulation where timing is relevant. +- `test/utils-async.test.mjs` explicitly tests async utilities, making `setTimeout` a valid testing primitive. diff --git a/src/decisions/DECISIONS_2026-02-20T22-37-42Z.md b/src/decisions/DECISIONS_2026-02-20T22-37-42Z.md new file mode 100644 index 0000000..3963691 --- /dev/null +++ b/src/decisions/DECISIONS_2026-02-20T22-37-42Z.md @@ -0,0 +1,14 @@ +# Decisions - 2026-02-20T22-37-42Z + +**Agent**: test-audit-agent +**Cadence**: daily +**Run-Start**: 2026-02-20T22-37-42Z +**Prompt**: src/prompts/daily/test-audit-agent.md + +## Decision +Report failures and flakiness found during audit. + +## Rationale +- `scripts/agent/load-test.mjs` and `smoke-test.mjs` are consistently failing. +- `cmdInit` test showed flakiness. +- These issues are recorded in KNOWN_ISSUES.md for future resolution. diff --git a/src/test_logs/TEST_LOG_2026-02-20T22-33-15Z.md b/src/test_logs/TEST_LOG_2026-02-20T22-33-15Z.md new file mode 100644 index 0000000..39755d6 --- /dev/null +++ b/src/test_logs/TEST_LOG_2026-02-20T22-33-15Z.md @@ -0,0 +1,19 @@ +# Test Log - 2026-02-20T22-33-15Z + +**Agent**: test-audit-agent +**Cadence**: daily +**Run-Start**: 2026-02-20T22-33-15Z +**Prompt**: src/prompts/daily/test-audit-agent.md + +## Execution + +**Command**: node scripts/test-audit/run-static-analysis.mjs --output-dir reports/test-audit +**Result**: +Found 3 suspicious files: +- test/fixtures/memory-fixtures.js (No assertions) +- test/relay-fanout-quorum.integration.test.mjs (setTimeout) +- test/utils-async.test.mjs (setTimeout) + +**Command**: node scripts/test-audit/run-flaky-check.mjs --output-dir reports/test-audit +**Result**: +All tests passed 5/5 runs. No flakiness detected. diff --git a/src/test_logs/TEST_LOG_2026-02-20T22-37-42Z.md b/src/test_logs/TEST_LOG_2026-02-20T22-37-42Z.md new file mode 100644 index 0000000..4361902 --- /dev/null +++ b/src/test_logs/TEST_LOG_2026-02-20T22-37-42Z.md @@ -0,0 +1,19 @@ +# Test Log - 2026-02-20T22-37-42Z + +**Agent**: test-audit-agent +**Cadence**: daily +**Run-Start**: 2026-02-20T22-37-42Z +**Prompt**: src/prompts/daily/test-audit-agent.md + +## Execution + +**Command**: node scripts/test-audit/run-static-analysis.mjs --output-dir reports/test-audit +**Result**: +Found 3 suspicious files. + +**Command**: node scripts/test-audit/run-flaky-check.mjs --output-dir reports/test-audit +**Result**: +Failures detected: +- scripts/agent/load-test.mjs (5/5 failed) +- scripts/agent/smoke-test.mjs (5/5 failed) +- cmdInit should validate install directory name (1/5 failed) diff --git a/src/todo/TODO_2026-02-20T22-33-15Z.md b/src/todo/TODO_2026-02-20T22-33-15Z.md new file mode 100644 index 0000000..07612e6 --- /dev/null +++ b/src/todo/TODO_2026-02-20T22-33-15Z.md @@ -0,0 +1,14 @@ +# Todo - 2026-02-20T22-33-15Z + +**Agent**: test-audit-agent +**Cadence**: daily +**Run-Start**: 2026-02-20T22-33-15Z +**Prompt**: src/prompts/daily/test-audit-agent.md + +## Completed +- [x] Run static analysis +- [x] Run flakiness check +- [x] Generate audit report + +## Pending +- None diff --git a/src/todo/TODO_2026-02-20T22-37-42Z.md b/src/todo/TODO_2026-02-20T22-37-42Z.md new file mode 100644 index 0000000..442d137 --- /dev/null +++ b/src/todo/TODO_2026-02-20T22-37-42Z.md @@ -0,0 +1,16 @@ +# Todo - 2026-02-20T22-37-42Z + +**Agent**: test-audit-agent +**Cadence**: daily +**Run-Start**: 2026-02-20T22-37-42Z +**Prompt**: src/prompts/daily/test-audit-agent.md + +## Completed +- [x] Run static analysis +- [x] Run flakiness check +- [x] Generate audit report +- [x] Update KNOWN_ISSUES.md + +## Pending +- Investigate failures in `load-test.mjs` and `smoke-test.mjs`. +- Investigate flakiness in `cmdInit`. diff --git a/task-logs/daily/2026-02-20T22-38-09Z__test-audit-agent__completed.md b/task-logs/daily/2026-02-20T22-38-09Z__test-audit-agent__completed.md new file mode 100644 index 0000000..17d6f21 --- /dev/null +++ b/task-logs/daily/2026-02-20T22-38-09Z__test-audit-agent__completed.md @@ -0,0 +1,7 @@ +--- +agent: test-audit-agent +cadence: daily +platform: jules +--- + +Run completed successfully.