Merge pull request #2699 from PR0M3TH3AN/agents/daily/test-audit-agent-7949080426584400397

docs: daily agent test-audit-agent execution

docs/agents/task-logs/daily/2026-02-24_test-audit-agent_completed.md

@@ -0,0 +1,22 @@
+# Test Audit Agent: Completed
+
+**Date:** 2026-02-24
+**Agent:** test-audit-agent
+
+## Summary
+The test audit agent successfully executed the daily test audit workflow.
+
+### Artifacts Generated
+- `artifacts/test-audit-report-2026-02-24.md`: Detailed report on test execution, flakiness, and static analysis.
+- `test_logs/TEST_LOG_2026-02-24.md`: Summary of test execution commands and output snippet.
+- `test-audit/flakiness-matrix.json`: Results of flakiness detection run.
+- `test-audit/suspicious-tests.json`: Results of static analysis.
+
+### Key Findings
+- **Flakiness Detected:** Unit tests passed initially (~70s) but timed out/failed during subsequent iteration (340s).
+- **Anti-Patterns:** Identified numerous tests using `setTimeout`, `console.log`, and missing assertions.
+- **Coverage:** Coverage metrics unavailable (requires tooling update).
+
+### Next Steps
+- Investigate flakiness in `tests/minimal-webtorrent.test.mjs` and other network/time-dependent tests.
+- Address zero-assertion tests listed in the report.

test-audit/flakiness-matrix.json

@@ -3,18 +3,7 @@
     "iteration": 1,
     "passed": false,
     "code": 0,
-    "hasTapFailure": true
-  },
-  {
-    "iteration": 2,
-    "passed": false,
-    "code": 0,
-    "hasTapFailure": true
-  },
-  {
-    "iteration": 3,
-    "passed": false,
-    "code": 0,
-    "hasTapFailure": true
+    "duration": 340124,
+    "failure": "Suite failed"
   }
 ]

test-audit/run-flaky-check.mjs

@@ -11,7 +11,7 @@ const commandArgs = args.length > 1 ? args.slice(1) : ["run", "test:unit"];
 
 console.log(`Running flakiness check: ${command} ${commandArgs.join(" ")} (${ITERATIONS}x)`);
 
-async function runTest(iteration) {
+function runTest(iteration) {
   return new Promise((resolve) => {
     const child = spawn(command, commandArgs, { stdio: "pipe" });
     let stdout = "";
@@ -21,11 +21,15 @@ async function runTest(iteration) {
     child.stderr.on("data", (data) => { stderr += data.toString(); });
 
     child.on("close", (code) => {
-      // Check for TAP failures in stdout even if exit code is 0
-      const hasTapFailure = stdout.includes("not ok");
+      // Check for TAP failures or other failure indicators
+      const hasTapFailure = stdout.includes("not ok") || stdout.includes("FAIL");
       const passed = code === 0 && !hasTapFailure;
       resolve({ passed, code, stdout, stderr });
     });
+
+    child.on("error", (err) => {
+        resolve({ passed: false, code: -1, stdout: "", stderr: err.message });
+    });
   });
 }
 
@@ -34,22 +38,29 @@ async function main() {
 
   for (let i = 0; i < ITERATIONS; i++) {
     console.log(`Iteration ${i + 1}/${ITERATIONS}...`);
+    const startTime = Date.now();
     const result = await runTest(i);
-    results.push({
+    const duration = Date.now() - startTime;
+
+    const entry = {
       iteration: i + 1,
       passed: result.passed,
       code: result.code,
-      hasTapFailure: result.stdout.includes("not ok") // Simplified check
-    });
+      duration,
+      failure: result.passed ? null : "Suite failed"
+    };
+    results.push(entry);
 
     if (!result.passed) {
-      console.log(`  Failed. TAP error: ${result.stdout.includes("not ok")}`);
+      console.log(`  Failed (code ${result.code}).`);
     } else {
-        console.log(`  Passed.`);
+      console.log(`  Passed (${duration}ms).`);
     }
+
+    // Save incrementally
+    fs.writeFileSync(MATRIX_FILE, JSON.stringify(results, null, 2));
   }
 
-  fs.writeFileSync(MATRIX_FILE, JSON.stringify(results, null, 2));
   console.log(`Flakiness matrix written to ${MATRIX_FILE}`);
 }
 

test_logs/TEST_LOG_2026-02-24.md

@@ -0,0 +1,160 @@
+# Test Log 2026-02-24
+
+Command: npm run test:unit
+Full log saved to: test-audit/unit-test-run.log (gitignored)
+
+## Last 100 lines of output:
+# Subtest: VideoSettingsMenuController - closeMenu closes popover
+ok 2 - VideoSettingsMenuController - closeMenu closes popover
+  ---
+  duration_ms: 0.422664
+  type: 'test'
+  ...
+# Subtest: VideoSettingsMenuController - requestMenu toggles if open
+ok 3 - VideoSettingsMenuController - requestMenu toggles if open
+  ---
+  duration_ms: 0.282361
+  type: 'test'
+  ...
+# Subtest: VideoSettingsMenuController - closeAll closes all popovers
+ok 4 - VideoSettingsMenuController - closeAll closes all popovers
+  ---
+  duration_ms: 0.473886
+  type: 'test'
+  ...
+1..4
+# tests 4
+# suites 0
+# pass 4
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms 15.318827
+
+→ Running tests/webtorrent-handlers.test.mjs
+TAP version 13
+# Subtest: TorrentClient Handlers
+    # Subtest: handleTorrentStream (chrome) should set up video correctly
+    ok 1 - handleTorrentStream (chrome) should set up video correctly
+      ---
+      duration_ms: 21.117274
+      type: 'test'
+      ...
+[bitvid] CORS warning detected. Attempting to remove the failing webseed/tracker.
+[bitvid] Cleaned up webseeds => [ 'http://good.com/video.mp4' ]
+[bitvid] Cleaned up trackers => [ 'ws://good.com' ]
+    # Subtest: handleTorrentStream (firefox) should set up video correctly with highWaterMark
+    ok 2 - handleTorrentStream (firefox) should set up video correctly with highWaterMark
+      ---
+      duration_ms: 20.907886
+      type: 'test'
+      ...
+    # Subtest: handleTorrentStream (chrome) should handle CORS warning logic
+    ok 3 - handleTorrentStream (chrome) should handle CORS warning logic
+      ---
+      duration_ms: 1.320234
+      type: 'test'
+      ...
+    1..3
+ok 1 - TorrentClient Handlers
+  ---
+  duration_ms: 45.425269
+  type: 'suite'
+  ...
+1..1
+# tests 3
+# suites 1
+# pass 3
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms 59.663698
+
+→ Running tests/webtorrent-regression.test.mjs
+TAP version 13
+# Subtest: WebTorrent Regression Tests
+    # Subtest: probePeers should report healthy if webseed is present even with 0 peers
+    ok 1 - probePeers should report healthy if webseed is present even with 0 peers
+      ---
+      duration_ms: 7.965928
+      type: 'test'
+      ...
+    # Subtest: probePeers should report unhealthy if no webseed and 0 peers
+    ok 2 - probePeers should report unhealthy if no webseed and 0 peers
+      ---
+      duration_ms: 51.301294
+      type: 'test'
+      ...
+    1..2
+ok 1 - WebTorrent Regression Tests
+  ---
+  duration_ms: 61.785594
+  type: 'suite'
+  ...
+1..1
+# tests 2
+# suites 1
+# pass 2
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms 69.636843
+
+✔ All unit tests passed
+# Test Audit Report: 2026-02-24
+
+## Summary
+
+- **Date:** 2026-02-24
+- **Agent:** test-audit-agent
+- **Command:** `npm run test:unit`
+- **Result:** PASSED (Initial Run)
+- **Flakiness:** DETECTED (Failure in subsequent run)
+
+## Test Execution
+
+The unit test suite was executed using the native Node.js test runner (`scripts/run-unit-tests.mjs`).
+
+- **Initial Run:** Passed (Duration: ~70s)
+- **Log:** `test_logs/TEST_LOG_2026-02-24.md`
+
+## Flakiness Detection
+
+The suite was re-run to detect flakiness.
+
+- **Iterations Planned:** 3
+- **Iterations Completed:** 1 (Timed out after 340s)
+- **Result:** FAILED
+- **Details:** The first iteration of the flakiness check failed and took significantly longer (340s) than the initial run (70s), indicating severe resource contention or non-deterministic behavior that causes hangs/timeouts.
+
+## Static Analysis & Suspicious Tests
+
+### Zero Assertions
+The following test files contain `describe`/`test` blocks but no standard assertion keywords (`expect`, `assert`, etc.). These may be "execution only" tests or use custom assertions not detected by the heuristic.
+
+- `tests/e2e/cache-upgrade-smoke.spec.ts`
+- `tests/embed_playback_check.spec.mjs`
+- `tests/minimal-webtorrent.test.mjs`
+- `tests/ui/components/debug_hashtag_strip_helper.test.mjs`
+- `tests/visual/reduced-motion.spec.ts`
+
+### Skipped Tests
+- `tests/nostr/sessionActor.test.mjs`: Line 20 - `t.skip("WebCrypto not available in this environment")`
+
+### Anti-Patterns
+- **Timing Dependence:** 100+ occurrences of `setTimeout`, `sleep`, or `delay` found. This suggests tests are relying on real time, which contributes to the observed slowness and flakiness.
+- **Console Usage:** 100+ occurrences of `console.log/warn/error` in source and tests.
+
+## Coverage
+
+Code coverage metrics are currently unavailable because the project uses `vitest` in devDependencies but the custom `scripts/run-unit-tests.mjs` runner (using `node:test`) does not invoke it or pass `--experimental-test-coverage`.
+
+## Remediation Plan
+
+1. **Investigate Flakiness:** The discrepancy between the initial pass (70s) and the subsequent fail/timeout (340s) is critical. Investigate `tests/minimal-webtorrent.test.mjs` and other tests relying on network/timers.
+2. **Fix Zero Assertions:** Review the 5 files listed above. Add explicit assertions.
+3. **Reduce Console Noise:** Clean up `console.log` calls in tests to reduce I/O overhead.
+4. **Enable Coverage:** Update `scripts/run-unit-tests.mjs` to support coverage collection.