From f099e7be3a485275695bbc52cbccf557ee0b943d Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Sat, 21 Feb 2026 21:03:49 -0500 Subject: [PATCH] improved playright integration --- AGENTS.md | 1 + docs/testing/playwright-function-coverage.md | 88 +++ package.json | 2 + playwright.config.ts | 1 + .../generate-function-coverage-report.mjs | 519 ++++++++++++++++++ tests/e2e/cache-upgrade-smoke.spec.ts | 2 +- tests/e2e/helpers/bitvidTestFixture.ts | 11 + tests/e2e/helpers/instrumentedTest.ts | 15 + .../helpers/playwrightCoverageGlobalSetup.ts | 6 + .../playwrightCoverageInstrumentation.ts | 130 +++++ tests/e2e/nostr-runtime-coverage.spec.ts | 428 +++++++++++++++ tests/e2e/popover.spec.ts | 2 +- tests/e2e/repro_issue.spec.ts | 2 +- tests/e2e/toast-service.spec.ts | 2 +- tests/e2e/ui-modals.spec.ts | 2 +- tests/e2e/video-modal-share-bug.spec.ts | 2 +- tests/e2e/webseed-playback.spec.ts | 2 +- 17 files changed, 1208 insertions(+), 7 deletions(-) create mode 100644 docs/testing/playwright-function-coverage.md create mode 100644 scripts/playwright/generate-function-coverage-report.mjs create mode 100644 tests/e2e/helpers/instrumentedTest.ts create mode 100644 tests/e2e/helpers/playwrightCoverageGlobalSetup.ts create mode 100644 tests/e2e/helpers/playwrightCoverageInstrumentation.ts create mode 100644 tests/e2e/nostr-runtime-coverage.spec.ts diff --git a/AGENTS.md b/AGENTS.md index f75a0f61..63a60db7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -458,6 +458,7 @@ Use these for stable element targeting: | `tests/e2e/agent-testability.spec.ts` | Infrastructure validation tests | | `scripts/playwright/run-extension-persistent.mjs` | Dedicated persistent-context launcher for extension smoke tests | | `docs/testing/playwright-integration-recommendations-2026-02-22.md` | Playwright/extension integration recommendations and follow-ups | +| `docs/testing/playwright-function-coverage.md` | Playwright coverage integration, latest baseline metrics, and improvement plan | --- diff --git a/docs/testing/playwright-function-coverage.md b/docs/testing/playwright-function-coverage.md new file mode 100644 index 00000000..a1f02786 --- /dev/null +++ b/docs/testing/playwright-function-coverage.md @@ -0,0 +1,88 @@ +# Playwright Function Coverage + Console Diagnostics + +Use this workflow to measure how much of bitvid's runtime code is exercised by E2E Playwright tests and to export browser console/page errors for triage. + +## Run + +```bash +npm run test:e2e:coverage +``` + +This command: + +1. Builds `dist/` +2. Runs `tests/e2e` on Chromium with browser JS coverage enabled +3. Captures browser `console` and `pageerror` output for every test +4. Generates aggregate reports + +## Artifacts + +- `artifacts/playwright-coverage/raw/*.json` + - Per-test raw coverage + console/page-error logs +- `artifacts/playwright-coverage/function-coverage-summary.json` + - Machine-readable function coverage report +- `artifacts/playwright-coverage/function-coverage-summary.md` + - Human-readable summary with coverage percentage and low-coverage files +- `artifacts/playwright-coverage/console-log-summary.json` + - Full exported console/page-error logs +- `artifacts/playwright-coverage/console-issues.log` + - Grouped issue signals for fast triage + +## Notes + +- Coverage is captured from Chromium runtime execution (`page.coverage`), then combined with static function estimates for source files in `js/` and `torrent/`. +- Firefox coverage is not included in this report because Playwright JS coverage is Chromium-only. + +## Current Baseline (2026-02-22) + +From a full real run (`npm run test:e2e:coverage`) on 2026-02-22: + +- E2E result: `38 passed` +- Function coverage: `29.41%` (`2826/9609`) +- Loaded source files: `297/313` +- Console logs: `68` +- Console error-like entries: `6` +- Page errors: `0` + +Primary issue groups captured: + +1. `404 Not Found` resource errors (2 occurrences) +2. Forced decrypt-timeout warning in diagnostics test (expected by test, 1 occurrence) +3. CORS-blocked `.torrent` fetch in webseed playback test (1 occurrence) +4. `net::ERR_FAILED` in webseed playback test (1 occurrence) +5. WebSocket tracker handshake failure for `wss://tracker.btorrent.xyz/` (1 occurrence) + +Recent high-impact coverage improvements: + +- `js/nostr/commentEvents.js`: `90.91%` (`20/20`) +- `js/nostr/reactionEvents.js`: `78.57%` (`11/14`) +- `js/nostr/publishHelpers.js`: `26.47%` (`9/34`) + +## Integration Rules For Agents + +1. For `tests/e2e/*.spec.ts`, import from `tests/e2e/helpers/instrumentedTest.ts` so coverage/log capture stays enabled. +2. For harness-based tests, use `tests/e2e/helpers/bitvidTestFixture.ts` (it already includes auto-capture). +3. Keep coverage runs Chromium-only when collecting percentages (`page.coverage` is Chromium-specific). +4. Do not remove `playwright.config.ts` global setup (`tests/e2e/helpers/playwrightCoverageGlobalSetup.ts`), which resets raw coverage artifacts. +5. Use `npm run test:e2e:coverage:report` only after a prior coverage-enabled run; it reads existing raw artifacts. + +## How To Improve Coverage + +Priority order for fastest gains: + +1. Add E2E for high-function files currently at `0%` but loaded: + - Start with `js/nostr/commentEvents.js`, `js/nostr/reactionEvents.js`, `js/nostr/publishHelpers.js`, `js/nostr/sessionActor.js`. +2. Add deterministic DM coverage lane in E2E harness: + - Exercise NIP-04 and NIP-44 decrypt/encrypt flows through UI and harness hooks. +3. Add upload + publish deep-path tests: + - Validate URL-only, magnet-only, and dual-source publish paths with assertions on fallback behavior. +4. Add negative-path assertions that currently only surface in console: + - Explicitly assert/handle CORS/torrent fetch failures and tracker handshake failures. +5. Keep tests routed through test-harness APIs: + - Prefer `setTestRelays`, `setSignerDecryptBehavior`, and seeded relay events over live-network dependence. + +## Suggested Quality Gates + +1. Track `functionCoveragePct` trend in CI artifacts per PR. +2. Add a soft warning threshold first (for example `>=30%`), then ratchet upward as flaky paths are stabilized. +3. Fail PRs only after baseline is stable across multiple runs. diff --git a/package.json b/package.json index dac3b88e..335c5317 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,8 @@ "test:e2e:chromium": "npm run build && npx playwright test tests/e2e --project=e2e", "test:e2e:firefox": "npm run build && npx playwright test tests/e2e --project=e2e-firefox", "test:e2e:deterministic": "npm run build && npx playwright test tests/e2e --project=e2e --project=e2e-firefox", + "test:e2e:coverage": "npm run build && cross-env PLAYWRIGHT_COVERAGE=1 PLAYWRIGHT_CAPTURE_CONSOLE=1 npx playwright test tests/e2e --project=e2e && node scripts/playwright/generate-function-coverage-report.mjs", + "test:e2e:coverage:report": "node scripts/playwright/generate-function-coverage-report.mjs", "test:e2e:extension": "npm run build && node scripts/playwright/run-extension-persistent.mjs", "test:load": "node scripts/agent/load-test.mjs", "test:smoke": "node scripts/agent/smoke-test.mjs", diff --git a/playwright.config.ts b/playwright.config.ts index fadf1125..4eaba0fc 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -47,6 +47,7 @@ if (ENABLE_EXTENSION_E2E && EXTENSION_PATH) { export default defineConfig({ testDir: "./tests", + globalSetup: "./tests/e2e/helpers/playwrightCoverageGlobalSetup.ts", outputDir: "artifacts/test-results", timeout: 60_000, reporter: process.env.CI diff --git a/scripts/playwright/generate-function-coverage-report.mjs b/scripts/playwright/generate-function-coverage-report.mjs new file mode 100644 index 00000000..c867f272 --- /dev/null +++ b/scripts/playwright/generate-function-coverage-report.mjs @@ -0,0 +1,519 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +const ROOT_DIR = process.cwd(); +const RAW_DIR = path.join(ROOT_DIR, "artifacts", "playwright-coverage", "raw"); +const OUTPUT_DIR = path.join(ROOT_DIR, "artifacts", "playwright-coverage"); +const SOURCE_DIRS = ["js", "torrent"]; +const SOURCE_EXTENSIONS = new Set([".js"]); +const METHOD_KEYWORD_BLOCKLIST = new Set([ + "if", + "for", + "while", + "switch", + "catch", + "return", + "else", + "do", + "try", + "class", + "function", + "import", + "export", + "new", +]); + +function normalizeSlashes(value) { + return value.split(path.sep).join("/"); +} + +function toRelativeIfInRepo(absPath) { + const rel = path.relative(ROOT_DIR, absPath); + if (rel.startsWith("..")) return null; + return normalizeSlashes(rel); +} + +async function walkFiles(startDir) { + const entries = await fs.readdir(startDir, { withFileTypes: true }); + const files = []; + for (const entry of entries) { + const fullPath = path.join(startDir, entry.name); + if (entry.isDirectory()) { + files.push(...(await walkFiles(fullPath))); + continue; + } + files.push(fullPath); + } + return files; +} + +function maskStringsAndComments(source) { + const out = source.split(""); + let i = 0; + let state = "code"; + while (i < out.length) { + const ch = out[i]; + const next = out[i + 1]; + + if (state === "line-comment") { + if (ch === "\n") { + state = "code"; + } else { + out[i] = " "; + } + i += 1; + continue; + } + if (state === "block-comment") { + if (ch === "*" && next === "/") { + out[i] = " "; + out[i + 1] = " "; + i += 2; + state = "code"; + } else { + out[i] = ch === "\n" ? "\n" : " "; + i += 1; + } + continue; + } + if (state === "single-quote") { + if (ch === "\\" && i + 1 < out.length) { + out[i] = " "; + out[i + 1] = " "; + i += 2; + continue; + } + out[i] = ch === "\n" ? "\n" : " "; + if (ch === "'") state = "code"; + i += 1; + continue; + } + if (state === "double-quote") { + if (ch === "\\" && i + 1 < out.length) { + out[i] = " "; + out[i + 1] = " "; + i += 2; + continue; + } + out[i] = ch === "\n" ? "\n" : " "; + if (ch === "\"") state = "code"; + i += 1; + continue; + } + if (state === "template") { + if (ch === "\\" && i + 1 < out.length) { + out[i] = " "; + out[i + 1] = " "; + i += 2; + continue; + } + if (ch === "`") { + out[i] = " "; + i += 1; + state = "code"; + continue; + } + if (ch === "$" && next === "{") { + out[i] = " "; + out[i + 1] = " "; + i += 2; + let depth = 1; + while (i < out.length && depth > 0) { + const c = out[i]; + if (c === "{") depth += 1; + else if (c === "}") depth -= 1; + if (depth > 0) out[i] = c === "\n" ? "\n" : " "; + i += 1; + } + continue; + } + out[i] = ch === "\n" ? "\n" : " "; + i += 1; + continue; + } + + if (ch === "/" && next === "/") { + out[i] = " "; + out[i + 1] = " "; + i += 2; + state = "line-comment"; + continue; + } + if (ch === "/" && next === "*") { + out[i] = " "; + out[i + 1] = " "; + i += 2; + state = "block-comment"; + continue; + } + if (ch === "'") { + out[i] = " "; + state = "single-quote"; + i += 1; + continue; + } + if (ch === "\"") { + out[i] = " "; + state = "double-quote"; + i += 1; + continue; + } + if (ch === "`") { + out[i] = " "; + state = "template"; + i += 1; + continue; + } + i += 1; + } + return out.join(""); +} + +function estimateStaticFunctionCount(source) { + const masked = maskStringsAndComments(source); + const seenPositions = new Set(); + + const functionPattern = /\bfunction\b\s*[A-Za-z0-9_$]*\s*\(/g; + for (const match of masked.matchAll(functionPattern)) { + seenPositions.add(match.index); + } + + const arrowPattern = /(?:^|[=(:,\[])\s*(?:async\s*)?(?:\([^()]*\)|[A-Za-z_$][\w$]*)\s*=>/gm; + for (const match of masked.matchAll(arrowPattern)) { + seenPositions.add(match.index); + } + + const methodPattern = /(?:^|\n)\s*(?:async\s+)?(?:get\s+|set\s+)?([A-Za-z_$][\w$]*)\s*\([^()\n{};]*\)\s*\{/gm; + for (const match of masked.matchAll(methodPattern)) { + const methodName = match[1]; + if (METHOD_KEYWORD_BLOCKLIST.has(methodName)) continue; + seenPositions.add(match.index); + } + + return seenPositions.size; +} + +function isLikelyRootCoverageFunction(fn, sourceLength) { + if (!fn || !Array.isArray(fn.ranges) || fn.ranges.length === 0) return false; + const startOffset = Math.min(...fn.ranges.map((range) => range.startOffset)); + const endOffset = Math.max(...fn.ranges.map((range) => range.endOffset)); + const rootSpan = startOffset === 0 && endOffset >= sourceLength - 1; + const name = fn.functionName || ""; + const anonymous = name === "" || name === "(anonymous)"; + return rootSpan && anonymous; +} + +function normalizeCoverageUrlToFile(url) { + if (!url) return null; + try { + const parsed = new URL(url); + const pathname = decodeURIComponent(parsed.pathname || ""); + const rel = pathname.replace(/^\/+/, ""); + if (!rel) return null; + return normalizeSlashes(rel); + } catch { + if (url.startsWith("/")) return normalizeSlashes(url.replace(/^\/+/, "")); + return null; + } +} + +function normalizeMessage(text) { + return String(text) + .replace(/\b\d+\b/g, "") + .replace(/\b0x[a-f0-9]+\b/gi, "0x") + .replace(/https?:\/\/\S+/g, "") + .trim(); +} + +async function main() { + await fs.mkdir(OUTPUT_DIR, { recursive: true }); + + let rawFiles = []; + try { + rawFiles = (await walkFiles(RAW_DIR)).filter((file) => file.endsWith(".json")); + } catch { + rawFiles = []; + } + + if (rawFiles.length === 0) { + const message = `No raw Playwright coverage artifacts found at ${normalizeSlashes(path.relative(ROOT_DIR, RAW_DIR))}. Run with PLAYWRIGHT_COVERAGE=1 first.`; + console.error(message); + process.exit(1); + } + + const perFileObserved = new Map(); + const allConsoleLogs = []; + const allPageErrors = []; + + for (const rawPath of rawFiles) { + const raw = JSON.parse(await fs.readFile(rawPath, "utf8")); + const rawRelPath = toRelativeIfInRepo(rawPath) ?? rawPath; + + if (Array.isArray(raw.logs)) { + for (const log of raw.logs) { + allConsoleLogs.push({ + test: raw.titlePath || [raw.title || rawRelPath], + project: raw.project || "unknown", + type: log.type || "log", + text: log.text || "", + location: log.location || null, + }); + } + } + + if (Array.isArray(raw.pageErrors)) { + for (const pageError of raw.pageErrors) { + allPageErrors.push({ + test: raw.titlePath || [raw.title || rawRelPath], + project: raw.project || "unknown", + message: pageError.message || String(pageError), + stack: pageError.stack || "", + }); + } + } + + if (!Array.isArray(raw.coverage)) continue; + + for (const entry of raw.coverage) { + const relFile = normalizeCoverageUrlToFile(entry.url); + if (!relFile) continue; + const ext = path.extname(relFile); + if (!SOURCE_EXTENSIONS.has(ext)) continue; + if (!SOURCE_DIRS.some((prefix) => relFile.startsWith(`${prefix}/`))) continue; + + const absFile = path.join(ROOT_DIR, relFile); + try { + await fs.access(absFile); + } catch { + continue; + } + + const existing = perFileObserved.get(relFile) || { + functionMap: new Map(), + }; + + for (const fn of entry.functions || []) { + if (isLikelyRootCoverageFunction(fn, entry.text?.length ?? 0)) continue; + const startOffset = Math.min(...fn.ranges.map((range) => range.startOffset)); + const endOffset = Math.max(...fn.ranges.map((range) => range.endOffset)); + const key = `${startOffset}:${endOffset}:${fn.functionName || ""}`; + const covered = fn.ranges.some((range) => range.count > 0); + const previous = existing.functionMap.get(key) || false; + existing.functionMap.set(key, previous || covered); + } + + perFileObserved.set(relFile, existing); + } + } + + const sourceFiles = []; + for (const sourceDir of SOURCE_DIRS) { + const absDir = path.join(ROOT_DIR, sourceDir); + let files = []; + try { + files = await walkFiles(absDir); + } catch { + files = []; + } + for (const absPath of files) { + const ext = path.extname(absPath); + if (!SOURCE_EXTENSIONS.has(ext)) continue; + const rel = toRelativeIfInRepo(absPath); + if (!rel) continue; + sourceFiles.push(rel); + } + } + + sourceFiles.sort(); + + const fileSummaries = []; + let overallCoveredFunctions = 0; + let overallEstimatedFunctions = 0; + let loadedFileCount = 0; + + for (const relFile of sourceFiles) { + const absPath = path.join(ROOT_DIR, relFile); + const source = await fs.readFile(absPath, "utf8"); + const estimatedFunctions = estimateStaticFunctionCount(source); + const observed = perFileObserved.get(relFile); + const observedTotal = observed ? observed.functionMap.size : 0; + const observedCovered = observed + ? Array.from(observed.functionMap.values()).filter(Boolean).length + : 0; + const effectiveTotal = Math.max(estimatedFunctions, observedTotal); + const effectiveCovered = Math.min(observedCovered, effectiveTotal); + const pct = effectiveTotal > 0 ? (effectiveCovered / effectiveTotal) * 100 : 100; + + if (observed) loadedFileCount += 1; + overallEstimatedFunctions += effectiveTotal; + overallCoveredFunctions += effectiveCovered; + + fileSummaries.push({ + file: relFile, + loadedByPlaywright: Boolean(observed), + estimatedFunctions, + observedFunctions: observedTotal, + coveredFunctions: effectiveCovered, + functionCoveragePct: Number(pct.toFixed(2)), + }); + } + + fileSummaries.sort((a, b) => a.functionCoveragePct - b.functionCoveragePct); + + const overallCoveragePct = + overallEstimatedFunctions > 0 + ? Number(((overallCoveredFunctions / overallEstimatedFunctions) * 100).toFixed(2)) + : 0; + + const consoleErrorLike = allConsoleLogs.filter((entry) => + ["error", "warning", "assert"].includes(String(entry.type).toLowerCase()), + ); + + const groupedIssues = new Map(); + for (const entry of consoleErrorLike) { + const key = `console:${entry.type}:${normalizeMessage(entry.text)}`; + const existing = groupedIssues.get(key) || { + kind: "console", + level: entry.type, + message: normalizeMessage(entry.text), + count: 0, + samples: [], + }; + existing.count += 1; + if (existing.samples.length < 5) { + existing.samples.push({ + test: entry.test.join(" > "), + rawMessage: entry.text, + }); + } + groupedIssues.set(key, existing); + } + + for (const entry of allPageErrors) { + const key = `pageerror:${normalizeMessage(entry.message)}`; + const existing = groupedIssues.get(key) || { + kind: "pageerror", + level: "error", + message: normalizeMessage(entry.message), + count: 0, + samples: [], + }; + existing.count += 1; + if (existing.samples.length < 5) { + existing.samples.push({ + test: entry.test.join(" > "), + rawMessage: entry.message, + }); + } + groupedIssues.set(key, existing); + } + + const topIssues = Array.from(groupedIssues.values()) + .sort((a, b) => b.count - a.count) + .slice(0, 50); + + const summary = { + generatedAt: new Date().toISOString(), + scope: { + sourceDirs: SOURCE_DIRS, + extensionFilter: Array.from(SOURCE_EXTENSIONS), + }, + coverage: { + totalSourceFiles: sourceFiles.length, + loadedSourceFiles: loadedFileCount, + estimatedFunctionCount: overallEstimatedFunctions, + coveredFunctionCount: overallCoveredFunctions, + functionCoveragePct: overallCoveragePct, + }, + testArtifacts: { + rawArtifactCount: rawFiles.length, + consoleLogCount: allConsoleLogs.length, + consoleErrorLikeCount: consoleErrorLike.length, + pageErrorCount: allPageErrors.length, + }, + topIssues, + files: fileSummaries, + }; + + const lowCoverageFiles = fileSummaries + .filter((item) => item.estimatedFunctions >= 5) + .slice(0, 25); + + const summaryMd = [ + "# Playwright Function Coverage Summary", + "", + `Generated: ${summary.generatedAt}`, + "", + `Overall function coverage: **${overallCoveragePct}%**`, + "", + `Covered functions: ${overallCoveredFunctions}/${overallEstimatedFunctions}`, + "", + `Source files loaded during Playwright run: ${loadedFileCount}/${sourceFiles.length}`, + "", + "## Console/Page Error Signals", + "", + `Console logs captured: ${allConsoleLogs.length}`, + "", + `Console error-like entries: ${consoleErrorLike.length}`, + "", + `Page errors: ${allPageErrors.length}`, + "", + "## Highest-Risk Coverage Gaps", + "", + "| File | Coverage % | Covered/Total | Loaded |", + "| --- | ---: | ---: | :---: |", + ...lowCoverageFiles.map( + (item) => + `| ${item.file} | ${item.functionCoveragePct}% | ${item.coveredFunctions}/${item.estimatedFunctions} | ${item.loadedByPlaywright ? "yes" : "no"} |`, + ), + "", + ].join("\n"); + + const issueLogLines = [ + `Generated: ${summary.generatedAt}`, + `Console logs: ${allConsoleLogs.length}`, + `Console error-like: ${consoleErrorLike.length}`, + `Page errors: ${allPageErrors.length}`, + "", + ...topIssues.map((issue, index) => { + const samples = issue.samples + .map((sample) => ` - ${sample.test}: ${sample.rawMessage}`) + .join("\n"); + return `${index + 1}. [${issue.kind}/${issue.level}] x${issue.count}\n ${issue.message}\n${samples}`; + }), + ].join("\n"); + + await fs.writeFile( + path.join(OUTPUT_DIR, "function-coverage-summary.json"), + JSON.stringify(summary, null, 2), + "utf8", + ); + await fs.writeFile(path.join(OUTPUT_DIR, "function-coverage-summary.md"), summaryMd, "utf8"); + await fs.writeFile(path.join(OUTPUT_DIR, "console-issues.log"), issueLogLines, "utf8"); + await fs.writeFile( + path.join(OUTPUT_DIR, "console-log-summary.json"), + JSON.stringify( + { + generatedAt: summary.generatedAt, + consoleLogs: allConsoleLogs, + pageErrors: allPageErrors, + topIssues, + }, + null, + 2, + ), + "utf8", + ); + + console.log(`Playwright function coverage: ${overallCoveragePct}% (${overallCoveredFunctions}/${overallEstimatedFunctions})`); + console.log(`Loaded source files: ${loadedFileCount}/${sourceFiles.length}`); + console.log(`Console/page issues: ${topIssues.length} grouped signals`); + console.log(`Summary JSON: artifacts/playwright-coverage/function-coverage-summary.json`); + console.log(`Summary MD: artifacts/playwright-coverage/function-coverage-summary.md`); + console.log(`Issue log: artifacts/playwright-coverage/console-issues.log`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/tests/e2e/cache-upgrade-smoke.spec.ts b/tests/e2e/cache-upgrade-smoke.spec.ts index 0162bd74..f2846d31 100644 --- a/tests/e2e/cache-upgrade-smoke.spec.ts +++ b/tests/e2e/cache-upgrade-smoke.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect, test } from "./helpers/instrumentedTest"; test("reload after deploy upgrade picks latest hashed bundle without manual cache clear", async ({ page, diff --git a/tests/e2e/helpers/bitvidTestFixture.ts b/tests/e2e/helpers/bitvidTestFixture.ts index bfbaeee7..7d959dc9 100644 --- a/tests/e2e/helpers/bitvidTestFixture.ts +++ b/tests/e2e/helpers/bitvidTestFixture.ts @@ -20,6 +20,7 @@ import { test as base, expect, type Page } from "@playwright/test"; import { startRelay } from "../../../scripts/agent/simple-relay.mjs"; import { finalizeEvent, getPublicKey } from "nostr-tools"; import { WebSocket } from "ws"; +import { attachCoverageAndConsoleCapture } from "./playwrightCoverageInstrumentation"; // Polyfill WebSocket for node environment if (!global.WebSocket) { @@ -143,6 +144,7 @@ async function loginWithTestKey(page: Page) { // ------------------------------------------------------------------ type BitvidFixtures = { + _coverageAndConsoleCapture: void; relay: Awaited>; seedEvent: (video: TestVideoEvent) => Promise; seedRawEvent: (event: any) => Promise; @@ -175,6 +177,15 @@ type BitvidFixtures = { // ------------------------------------------------------------------ export const test = base.extend({ + _coverageAndConsoleCapture: [ + async ({ page, browserName }, use, testInfo) => { + const stopCapture = await attachCoverageAndConsoleCapture(page, testInfo, browserName); + await use(); + await stopCapture(); + }, + { auto: true }, + ], + relay: [ async ({}, use, testInfo) => { // Avoid port collisions when running in parallel workers diff --git a/tests/e2e/helpers/instrumentedTest.ts b/tests/e2e/helpers/instrumentedTest.ts new file mode 100644 index 00000000..29b7ad59 --- /dev/null +++ b/tests/e2e/helpers/instrumentedTest.ts @@ -0,0 +1,15 @@ +import { test as base, expect } from "@playwright/test"; +import { attachCoverageAndConsoleCapture } from "./playwrightCoverageInstrumentation"; + +export const test = base.extend({ + _coverageAndConsoleCapture: [ + async ({ page, browserName }, use, testInfo) => { + const stopCapture = await attachCoverageAndConsoleCapture(page, testInfo, browserName); + await use(); + await stopCapture(); + }, + { auto: true }, + ], +}); + +export { expect }; diff --git a/tests/e2e/helpers/playwrightCoverageGlobalSetup.ts b/tests/e2e/helpers/playwrightCoverageGlobalSetup.ts new file mode 100644 index 00000000..a44b14fd --- /dev/null +++ b/tests/e2e/helpers/playwrightCoverageGlobalSetup.ts @@ -0,0 +1,6 @@ +import type { FullConfig } from "@playwright/test"; +import { prepareCoverageArtifactsDir } from "./playwrightCoverageInstrumentation"; + +export default async function globalSetup(_config: FullConfig): Promise { + await prepareCoverageArtifactsDir(); +} diff --git a/tests/e2e/helpers/playwrightCoverageInstrumentation.ts b/tests/e2e/helpers/playwrightCoverageInstrumentation.ts new file mode 100644 index 00000000..57531751 --- /dev/null +++ b/tests/e2e/helpers/playwrightCoverageInstrumentation.ts @@ -0,0 +1,130 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { Page, TestInfo } from "@playwright/test"; + +type ConsoleEntry = { + type: string; + text: string; + location?: { + url?: string; + lineNumber?: number; + columnNumber?: number; + }; +}; + +type PageErrorEntry = { + message: string; + stack?: string; +}; + +type CoverageEntry = { + url: string; + text: string; + functions: Array<{ + functionName: string; + isBlockCoverage: boolean; + ranges: Array<{ startOffset: number; endOffset: number; count: number }>; + }>; +}; + +function sanitizeSegment(value: string): string { + return value + .replace(/[^a-zA-Z0-9._-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^-|-$/g, "") + .slice(0, 120); +} + +function getRawArtifactDir(): string { + return path.join(process.cwd(), "artifacts", "playwright-coverage", "raw"); +} + +export async function prepareCoverageArtifactsDir(): Promise { + if (process.env.PLAYWRIGHT_COVERAGE !== "1") return; + const dir = getRawArtifactDir(); + await fs.rm(dir, { recursive: true, force: true }); + await fs.mkdir(dir, { recursive: true }); +} + +export async function attachCoverageAndConsoleCapture( + page: Page, + testInfo: TestInfo, + browserName: string, +): Promise<() => Promise> { + const captureCoverage = process.env.PLAYWRIGHT_COVERAGE === "1"; + const captureConsole = process.env.PLAYWRIGHT_CAPTURE_CONSOLE === "1" || captureCoverage; + const rawDir = getRawArtifactDir(); + + const consoleEntries: ConsoleEntry[] = []; + const pageErrors: PageErrorEntry[] = []; + let coverageStarted = false; + + const onConsole = (message: any) => { + consoleEntries.push({ + type: message.type(), + text: message.text(), + location: message.location(), + }); + }; + + const onPageError = (error: Error) => { + pageErrors.push({ + message: error?.message || String(error), + stack: error?.stack, + }); + }; + + if (captureConsole) { + page.on("console", onConsole); + page.on("pageerror", onPageError); + } + + if (captureCoverage && browserName === "chromium") { + await page.coverage.startJSCoverage({ + resetOnNavigation: false, + reportAnonymousScripts: false, + }); + coverageStarted = true; + } + + return async () => { + const timestamp = new Date().toISOString(); + const testId = sanitizeSegment( + `${testInfo.project.name}-${testInfo.titlePath.join(" ")}-retry-${testInfo.retry}`, + ); + + page.off("console", onConsole); + page.off("pageerror", onPageError); + + let coverage: CoverageEntry[] = []; + if (coverageStarted) { + coverage = (await page.coverage.stopJSCoverage()) as CoverageEntry[]; + } + + if (!captureCoverage && !captureConsole) return; + + await fs.mkdir(rawDir, { recursive: true }); + await fs.writeFile( + path.join(rawDir, `${testId}.json`), + JSON.stringify( + { + timestamp, + project: testInfo.project.name, + title: testInfo.title, + titlePath: testInfo.titlePath, + file: testInfo.file, + retry: testInfo.retry, + status: testInfo.status, + expectedStatus: testInfo.expectedStatus, + browserName, + logs: consoleEntries, + pageErrors, + coverage, + }, + null, + 2, + ), + "utf8", + ); + }; +} diff --git a/tests/e2e/nostr-runtime-coverage.spec.ts b/tests/e2e/nostr-runtime-coverage.spec.ts new file mode 100644 index 00000000..9c544bfe --- /dev/null +++ b/tests/e2e/nostr-runtime-coverage.spec.ts @@ -0,0 +1,428 @@ +import { expect, test } from "./helpers/instrumentedTest"; + +const HEX_A = "a".repeat(64); +const HEX_B = "b".repeat(64); +const HEX_C = "c".repeat(64); +const HEX_D = "d".repeat(64); + +test.describe("nostr runtime coverage (deterministic)", () => { + test.beforeEach(async ({ page }) => { + await page.goto("/docs/kitchen-sink.html", { waitUntil: "networkidle" }); + }); + + test("exercises publish helper signing + rebroadcast guard paths", async ({ + page, + }) => { + const result = await page.evaluate(async ({ actorA, actorB }) => { + const { + deriveRebroadcastScope, + deriveRebroadcastBucketIndex, + rememberRebroadcastAttempt, + hasRecentRebroadcastAttempt, + getRebroadcastCooldownState, + signAndPublishEvent, + } = await import("/js/nostr/publishHelpers.js"); + + const scope = deriveRebroadcastScope(actorA, "f".repeat(64)); + const bucket = deriveRebroadcastBucketIndex(1_700_000_000); + rememberRebroadcastAttempt(scope, bucket); + const hasRecent = hasRecentRebroadcastAttempt(scope, bucket); + const cooldown = getRebroadcastCooldownState(scope); + + const publishCalls = []; + const client = { + pubkey: actorA, + writeRelays: ["wss://relay.one", "wss://relay.two"], + relays: ["wss://relay.one"], + sessionActor: null, + ensureSessionActor: async () => { + client.sessionActor = { + pubkey: actorB, + privateKey: "1".repeat(64), + source: "nip46", + }; + return actorB; + }, + pool: { + publish(relays, event) { + publishCalls.push({ relays, event }); + return { + on(eventName, handler) { + if (eventName === "ok") { + Promise.resolve().then(() => handler()); + } + }, + }; + }, + }, + }; + + const activeEvent = { + kind: 1, + pubkey: actorA, + created_at: 123, + tags: [], + content: "active-signer", + }; + + const activeResult = await signAndPublishEvent({ + client, + event: activeEvent, + options: { context: "test-active", logName: "active" }, + resolveActiveSigner: () => ({ + signEvent: async (event) => ({ + ...event, + id: "id-active", + sig: "sig-active", + }), + }), + shouldRequestExtensionPermissions: () => false, + signEventWithPrivateKey: () => { + throw new Error("unexpected-session-fallback"); + }, + }); + + let fallbackSignCalls = 0; + const fallbackEvent = { + kind: 1, + pubkey: actorB, + created_at: 456, + tags: [], + content: "session-signer", + }; + + const fallbackResult = await signAndPublishEvent({ + client, + event: fallbackEvent, + options: { context: "test-fallback", logName: "fallback" }, + resolveActiveSigner: () => null, + shouldRequestExtensionPermissions: () => false, + signEventWithPrivateKey: (event) => { + fallbackSignCalls += 1; + return { + ...event, + id: "id-session", + sig: "sig-session", + }; + }, + }); + + return { + hasRecent, + cooldownRemainingPositive: Number(cooldown?.remainingMs || 0) > 0, + activeAcceptedCount: activeResult.summary.accepted.length, + activeSignerPubkey: activeResult.signerPubkey, + fallbackAcceptedCount: fallbackResult.summary.accepted.length, + fallbackSignerPubkey: fallbackResult.signerPubkey, + fallbackSignCalls, + publishCallCount: publishCalls.length, + }; + }, { actorA: HEX_A, actorB: HEX_B }); + + expect(result.hasRecent).toBe(true); + expect(result.cooldownRemainingPositive).toBe(true); + expect(result.activeAcceptedCount).toBeGreaterThan(0); + expect(result.activeSignerPubkey).toBe(HEX_A); + expect(result.fallbackAcceptedCount).toBeGreaterThan(0); + expect(result.fallbackSignerPubkey).toBe(HEX_B); + expect(result.fallbackSignCalls).toBe(1); + expect(result.publishCallCount).toBeGreaterThanOrEqual(2); + }); + + test("exercises comment publish/list/subscribe and session-actor publish guard", async ({ + page, + }) => { + const result = await page.evaluate(async ({ actor }) => { + const { + COMMENT_EVENT_KIND, + publishComment, + listVideoComments, + subscribeVideoComments, + } = await import("/js/nostr/commentEvents.js"); + + const relay = "wss://relay.comments"; + const videoEventId = "e".repeat(64); + const listCalls = []; + const publishedEvents = []; + let subEventHandler = null; + let unsubCalled = false; + + const client = { + pubkey: actor, + relays: [relay], + ensureExtensionPermissions: async () => ({ ok: true }), + pool: { + publish(relays, event) { + publishedEvents.push({ relays, event }); + return { + on(eventName, handler) { + if (eventName === "ok") { + Promise.resolve().then(() => handler()); + } + }, + }; + }, + async list(relays, filters) { + listCalls.push({ relays, filters }); + const match = publishedEvents[0]?.event || null; + const invalid = { + id: "invalid-comment-id", + kind: COMMENT_EVENT_KIND, + created_at: 1, + tags: [["e", "not-the-video"]], + content: "invalid", + pubkey: actor, + sig: "s", + }; + return [match, invalid].filter(Boolean); + }, + sub(relays, filters) { + listCalls.push({ relays, filters, type: "sub" }); + return { + on(eventName, handler) { + if (eventName === "event") { + subEventHandler = handler; + } + }, + unsub() { + unsubCalled = true; + }, + }; + }, + }, + }; + + const signer = { + signEvent: async (event) => ({ + ...event, + id: "comment-event-id", + sig: "comment-event-sig", + }), + }; + + const publishResult = await publishComment( + client, + { videoEventPointer: ["e", videoEventId, relay] }, + { + content: { text: "coverage comment" }, + additionalTags: [["client", "playwright"]], + }, + { + resolveActiveSigner: () => signer, + shouldRequestExtensionPermissions: () => false, + DEFAULT_NIP07_PERMISSION_METHODS: [], + }, + ); + + const listedFirst = await listVideoComments( + client, + { videoEventPointer: ["e", videoEventId, relay] }, + { forceRefresh: true, relays: [relay] }, + ); + + const listCallCountAfterFirst = listCalls.length; + const listedSecond = await listVideoComments( + client, + { videoEventPointer: ["e", videoEventId, relay] }, + { relays: [relay] }, + ); + const listCallCountAfterSecond = listCalls.length; + + const seenIds = []; + const unsubscribe = subscribeVideoComments( + client, + { videoEventPointer: ["e", videoEventId, relay] }, + { + relays: [relay], + onEvent: (event) => { + seenIds.push(event.id); + }, + }, + ); + + await Promise.resolve(); + if (typeof subEventHandler === "function") { + subEventHandler(publishResult.event); + subEventHandler({ + id: "mismatch", + kind: COMMENT_EVENT_KIND, + created_at: 2, + tags: [["e", "not-the-video"]], + pubkey: actor, + content: "skip", + sig: "s", + }); + } + unsubscribe(); + + const blockedClient = { + pubkey: actor, + sessionActor: { pubkey: actor, source: "nip46" }, + relays: [relay], + pool: client.pool, + }; + const blockedPublishResult = await publishComment( + blockedClient, + { videoEventPointer: ["e", videoEventId, relay] }, + { content: "blocked" }, + { + resolveActiveSigner: () => signer, + shouldRequestExtensionPermissions: () => false, + DEFAULT_NIP07_PERMISSION_METHODS: [], + }, + ); + + return { + publishOk: publishResult.ok, + publishKind: publishResult.event?.kind, + listedFirstCount: listedFirst.length, + listedSecondCount: listedSecond.length, + usedCacheOnSecondList: + listCallCountAfterSecond === listCallCountAfterFirst, + seenIds, + unsubCalled, + blockedError: blockedPublishResult.error, + }; + }, { actor: HEX_C }); + + expect(result.publishOk).toBe(true); + expect(result.publishKind).toBeGreaterThan(0); + expect(result.listedFirstCount).toBe(1); + expect(result.listedSecondCount).toBe(1); + expect(result.usedCacheOnSecondList).toBe(true); + expect(result.seenIds).toEqual(["comment-event-id"]); + expect(result.unsubCalled).toBe(true); + expect(result.blockedError).toBe("session-actor-publish-blocked"); + }); + + test("exercises reaction publish/list for active signer and session fallback", async ({ + page, + }) => { + const result = await page.evaluate(async ({ actorA, actorB }) => { + const { publishVideoReaction, listVideoReactions } = await import( + "/js/nostr/reactionEvents.js" + ); + + const relay = "wss://relay.reactions"; + const videoEventId = "9".repeat(64); + const listCalls = []; + const publishedEvents = []; + let fallbackSignCalls = 0; + + const client = { + pubkey: actorA, + relays: [relay], + sessionActor: null, + ensureExtensionPermissions: async () => ({ ok: true }), + ensureSessionActor: async (forceNew = false) => { + if (!client.sessionActor || forceNew) { + client.sessionActor = { + pubkey: forceNew ? actorB : actorA, + privateKey: "2".repeat(64), + source: "nip46", + }; + } + return client.sessionActor.pubkey; + }, + pool: { + publish(relays, event) { + publishedEvents.push({ relays, event }); + return { + on(eventName, handler) { + if (eventName === "ok") { + Promise.resolve().then(() => handler()); + } + }, + }; + }, + async list(relays, filters) { + listCalls.push({ relays, filters }); + const primary = publishedEvents[0]?.event || null; + const invalid = { + id: "invalid-reaction", + kind: 7, + created_at: 2, + tags: [["e", "deadbeef"]], + pubkey: actorA, + content: "+", + sig: "s", + }; + return [primary, invalid].filter(Boolean); + }, + }, + }; + + const activeResult = await publishVideoReaction( + client, + ["e", videoEventId, relay], + { content: "+" }, + { + resolveActiveSigner: () => ({ + signEvent: async (event) => ({ + ...event, + id: "reaction-active-id", + sig: "reaction-active-sig", + }), + }), + shouldRequestExtensionPermissions: () => false, + signEventWithPrivateKey: () => { + throw new Error("unexpected-session-fallback"); + }, + DEFAULT_NIP07_PERMISSION_METHODS: [], + }, + ); + + const reactionsFirst = await listVideoReactions( + client, + { id: videoEventId }, + { forceRefresh: true, relays: [relay] }, + ); + const listCountAfterFirst = listCalls.length; + const reactionsSecond = await listVideoReactions( + client, + { id: videoEventId }, + { relays: [relay] }, + ); + const listCountAfterSecond = listCalls.length; + + client.pubkey = actorA; + client.sessionActor = { pubkey: actorB, privateKey: "3".repeat(64), source: "nip46" }; + const fallbackResult = await publishVideoReaction( + client, + ["e", videoEventId, relay], + { content: "🔥" }, + { + resolveActiveSigner: () => null, + shouldRequestExtensionPermissions: () => false, + signEventWithPrivateKey: (event) => { + fallbackSignCalls += 1; + return { + ...event, + id: "reaction-session-id", + sig: "reaction-session-sig", + }; + }, + DEFAULT_NIP07_PERMISSION_METHODS: [], + }, + ); + + return { + activeOk: activeResult.ok, + activeAccepted: activeResult.acceptedRelays.length, + reactionsFirstCount: reactionsFirst.length, + reactionsSecondCount: reactionsSecond.length, + usedCacheOnSecondList: listCountAfterSecond === listCountAfterFirst, + fallbackOk: fallbackResult.ok, + fallbackSignCalls, + }; + }, { actorA: HEX_A, actorB: HEX_D }); + + expect(result.activeOk).toBe(true); + expect(result.activeAccepted).toBeGreaterThan(0); + expect(result.reactionsFirstCount).toBe(1); + expect(result.reactionsSecondCount).toBe(1); + expect(result.usedCacheOnSecondList).toBe(true); + expect(result.fallbackOk).toBe(true); + expect(result.fallbackSignCalls).toBe(1); + }); +}); diff --git a/tests/e2e/popover.spec.ts b/tests/e2e/popover.spec.ts index 37482c5d..5ba129d3 100644 --- a/tests/e2e/popover.spec.ts +++ b/tests/e2e/popover.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect, test } from "./helpers/instrumentedTest"; import { getPanelMetrics, getPanelWithTriggerMetrics, diff --git a/tests/e2e/repro_issue.spec.ts b/tests/e2e/repro_issue.spec.ts index 92026bec..966c3a4f 100644 --- a/tests/e2e/repro_issue.spec.ts +++ b/tests/e2e/repro_issue.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from './helpers/instrumentedTest'; import { startRelay } from '../../scripts/agent/simple-relay.mjs'; import { WebSocket } from 'ws'; import { finalizeEvent, getPublicKey, nip19 } from 'nostr-tools'; diff --git a/tests/e2e/toast-service.spec.ts b/tests/e2e/toast-service.spec.ts index 34dfecf0..532ea605 100644 --- a/tests/e2e/toast-service.spec.ts +++ b/tests/e2e/toast-service.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect, test } from "./helpers/instrumentedTest"; import { applyReducedMotion, failOnConsoleErrors } from "./helpers/uiTestUtils"; test.describe("toast service in full DOM", () => { diff --git a/tests/e2e/ui-modals.spec.ts b/tests/e2e/ui-modals.spec.ts index e66b650a..3439a309 100644 --- a/tests/e2e/ui-modals.spec.ts +++ b/tests/e2e/ui-modals.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect, test } from "./helpers/instrumentedTest"; import { applyReducedMotion, ensureTestAssets, diff --git a/tests/e2e/video-modal-share-bug.spec.ts b/tests/e2e/video-modal-share-bug.spec.ts index 47446903..b738f8ce 100644 --- a/tests/e2e/video-modal-share-bug.spec.ts +++ b/tests/e2e/video-modal-share-bug.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect, test } from "./helpers/instrumentedTest"; import { applyReducedMotion, failOnConsoleErrors } from "./helpers/uiTestUtils"; test.describe("video modal share button", () => { diff --git a/tests/e2e/webseed-playback.spec.ts b/tests/e2e/webseed-playback.spec.ts index 0f80c6a7..ea36c595 100644 --- a/tests/e2e/webseed-playback.spec.ts +++ b/tests/e2e/webseed-playback.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "./helpers/instrumentedTest"; test.describe("Webseed Stream Playback", () => { const VIDEO_URL =