mirror of
https://github.com/PR0M3TH3AN/bitvid.git
synced 2026-10-30 09:22:24 +00:00
Merge branch 'unstable' into agents/daily/test-audit-agent-2026-02-23-4813203485121112615
This commit is contained in:
64 files changed
+1728
-291
No files matched your search
@@ -1,98 +1,108 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
|
||||
const rootDir = process.cwd();
|
||||
const reportFile = `test-audit-report-${new Date().toISOString().split("T")[0]}.md`;
|
||||
const logFile = "test-audit/coverage-run.log";
|
||||
const flakinessFile = "test-audit/flakiness-matrix.json";
|
||||
const suspiciousFile = "test-audit/suspicious-tests.json";
|
||||
const coverageFile = "coverage/coverage-summary.json";
|
||||
const FLAKINESS_FILE = "test-audit/flakiness-matrix.json";
|
||||
const SUSPICIOUS_FILE = "test-audit/suspicious-tests.json";
|
||||
const LOG_FILE = "test-audit/coverage-run.log";
|
||||
|
||||
async function readJson(file) {
|
||||
// Dynamic date
|
||||
const date = new Date().toISOString().split("T")[0];
|
||||
const REPORT_FILE = `test-audit-report-${date}.md`;
|
||||
|
||||
function main() {
|
||||
const flakiness = JSON.parse(fs.readFileSync(FLAKINESS_FILE, "utf8"));
|
||||
const suspicious = JSON.parse(fs.readFileSync(SUSPICIOUS_FILE, "utf8"));
|
||||
let log = "";
|
||||
try {
|
||||
const content = await fs.readFile(file, "utf8");
|
||||
return JSON.parse(content);
|
||||
log = fs.readFileSync(LOG_FILE, "utf8");
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const flakiness = await readJson(flakinessFile);
|
||||
const suspicious = await readJson(suspiciousFile);
|
||||
const coverage = await readJson(coverageFile);
|
||||
const logContent = await fs.readFile(logFile, "utf8");
|
||||
|
||||
let report = `# Test Audit Report - ${new Date().toISOString().split("T")[0]}\n\n`;
|
||||
|
||||
// Summary
|
||||
report += "## Summary\n";
|
||||
report += "- **Runner**: Custom (`scripts/run-unit-tests.mjs`)\n";
|
||||
report += "- **Command**: `npx c8 npm run test:unit`\n";
|
||||
|
||||
// Failures
|
||||
if (logContent.includes("failed with exit code")) {
|
||||
report += "- **Status**: FAIL (Failures detected)\n";
|
||||
} else {
|
||||
report += "- **Status**: PASS\n";
|
||||
log = "";
|
||||
}
|
||||
|
||||
// Coverage Stats
|
||||
if (coverage && coverage.total) {
|
||||
report += `- **Coverage**: ${coverage.total.lines.pct}% Lines, ${coverage.total.statements.pct}% Statements, ${coverage.total.functions.pct}% Functions, ${coverage.total.branches.pct}% Branches\n`;
|
||||
} else {
|
||||
report += "- **Coverage**: N/A\n";
|
||||
}
|
||||
const failingTests = flakiness.filter(r => !r.passed || r.hasTapFailure);
|
||||
const passed = failingTests.length === 0;
|
||||
|
||||
// Flakiness
|
||||
report += "\n## Flakiness Check\n";
|
||||
if (flakiness && flakiness.error) {
|
||||
report += `- **Error**: ${flakiness.error}\n`;
|
||||
} else if (flakiness && flakiness.runs) {
|
||||
// Process flakiness
|
||||
report += `- **Runs**: ${flakiness.runs.length}\n`;
|
||||
// Add details
|
||||
} else {
|
||||
report += "- **Status**: Skipped or failed to produce results.\n";
|
||||
}
|
||||
// Filter suspicious zero-assertion to actual test files
|
||||
const zeroAssertionTests = suspicious.zeroAssertions ? suspicious.zeroAssertions.filter(f => f.match(/\.(test|spec)\.(mjs|js|ts)$/)) : [];
|
||||
|
||||
// Suspicious Tests
|
||||
report += "\n## Suspicious Tests\n";
|
||||
if (suspicious) {
|
||||
let count = 0;
|
||||
for (const [file, issues] of Object.entries(suspicious)) {
|
||||
count++;
|
||||
report += `- **${file}**:\n`;
|
||||
issues.forEach(i => report += ` - ${i}\n`);
|
||||
}
|
||||
if (count === 0) report += "None found.\n";
|
||||
} else {
|
||||
report += "Analysis not run or failed.\n";
|
||||
}
|
||||
|
||||
// Coverage Gaps (Critical Files)
|
||||
report += "\n## Coverage Gaps (< 70%)\n";
|
||||
if (coverage) {
|
||||
let gaps = 0;
|
||||
for (const [file, stats] of Object.entries(coverage)) {
|
||||
if (file === "total") continue;
|
||||
const relativePath = path.relative(rootDir, file);
|
||||
if (stats.lines.pct < 70) {
|
||||
gaps++;
|
||||
report += `- **${relativePath}**: ${stats.lines.pct}% (Lines)\n`;
|
||||
// Extract failures from log dynamically
|
||||
let failureDetails = "";
|
||||
if (!passed) {
|
||||
const lines = log.split('\n');
|
||||
const failures = lines.filter(l => l.includes('not ok') || l.includes('FAIL'));
|
||||
if (failures.length > 0) {
|
||||
failureDetails = failures.slice(0, 10).map(l => `- \`${l.trim()}\``).join('\n');
|
||||
if (failures.length > 10) failureDetails += `\n- ... and ${failures.length - 10} more.`;
|
||||
} else {
|
||||
failureDetails = "Check logs for details (exit code non-zero but no explicit 'not ok' found).";
|
||||
}
|
||||
}
|
||||
if (gaps === 0) report += "No files under 70%.\n";
|
||||
}
|
||||
|
||||
// Recommendations
|
||||
report += "\n## Recommendations\n";
|
||||
report += "1. Fix suspicious tests (especially timeouts and console usage).\n";
|
||||
report += "2. Improve coverage for files < 70%.\n";
|
||||
report += "3. Investigate potential flakiness (if checked).\n";
|
||||
let content = `# Test Audit Report - ${date}
|
||||
|
||||
await fs.writeFile(reportFile, report);
|
||||
console.log(`Report generated: ${reportFile}`);
|
||||
## Summary
|
||||
- **Status**: ${passed ? "PASS" : "FAIL"}
|
||||
- **Test Command**: (Check \`test_logs/\`)
|
||||
- **Coverage**: Skipped (missing \`c8\` / \`@vitest/coverage-v8\`)
|
||||
|
||||
## Flakiness & Failures
|
||||
`;
|
||||
|
||||
if (passed) {
|
||||
content += "- No failures or flakiness detected.\n";
|
||||
} else {
|
||||
content += `- **Failures**: ${failingTests.length}/${flakiness.length} runs failed.\n`;
|
||||
content += `\n### Failure Details\n${failureDetails}\n`;
|
||||
content += "\n### Run Matrix\n";
|
||||
content += "| Iteration | Passed | TAP Failure |\n|---|---|---|\n";
|
||||
flakiness.forEach(r => {
|
||||
content += `| ${r.iteration} | ${r.passed ? "✅" : "❌"} | ${r.hasTapFailure ? "Yes" : "No"} |\n`;
|
||||
});
|
||||
}
|
||||
|
||||
content += `
|
||||
## Suspicious Tests (Static Analysis)
|
||||
|
||||
### Zero Assertions (Heuristic)
|
||||
These test files contain \`test(\` or \`describe(\` but no obvious assertion keywords.
|
||||
`;
|
||||
|
||||
if (zeroAssertionTests.length === 0) {
|
||||
content += "- None found.\n";
|
||||
} else {
|
||||
zeroAssertionTests.forEach(f => content += `- ${f}\n`);
|
||||
}
|
||||
|
||||
content += `
|
||||
### Skipped or Focused Tests (\`.skip\`, \`.only\`)
|
||||
`;
|
||||
if (suspicious.skippedOrFocused.length === 0) {
|
||||
content += "- None found.\n";
|
||||
} else {
|
||||
suspicious.skippedOrFocused.forEach(f => content += `- ${f.trim()}\n`);
|
||||
}
|
||||
|
||||
content += `
|
||||
### Timing Dependencies (\`setTimeout\`, \`sleep\`)
|
||||
Found ${suspicious.timing.length} occurrences. Examples:
|
||||
`;
|
||||
suspicious.timing.slice(0, 10).forEach(f => content += `- \`${f.trim()}\`\n`);
|
||||
if (suspicious.timing.length > 10) content += `- ... and ${suspicious.timing.length - 10} more.\n`;
|
||||
|
||||
content += `
|
||||
## Coverage Gaps
|
||||
- **Critical**: Coverage metrics could not be generated.
|
||||
- **Action**: Install \`c8\` or configure \`vitest\` coverage.
|
||||
|
||||
## Recommendations
|
||||
1. **Fix Failures**: Investigate any failures listed above.
|
||||
2. **Remove Skips**: Address skipped tests.
|
||||
3. **Reduce Timing Flakes**: Refactor tests using real \`setTimeout\` to use mock timers or deterministic waits.
|
||||
4. **Tooling**: Add \`c8\` to \`devDependencies\` to enable coverage reporting for Node.js native runner.
|
||||
|
||||
`;
|
||||
|
||||
fs.writeFileSync(REPORT_FILE, content);
|
||||
console.log(`Report generated: ${REPORT_FILE}`);
|
||||
}
|
||||
|
||||
run().catch(console.error);
|
||||
main();
|
||||
@@ -1,97 +1,67 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
|
||||
const RUNS = 3;
|
||||
const RESULTS_FILE = "test-audit/flakiness-matrix.json";
|
||||
const ITERATIONS = 3;
|
||||
const MATRIX_FILE = "test-audit/flakiness-matrix.json";
|
||||
|
||||
async function runTests(runIndex) {
|
||||
console.log(`Starting Run ${runIndex + 1}/${RUNS}...`);
|
||||
// Parse command line arguments or use default
|
||||
const args = process.argv.slice(2);
|
||||
const command = args.length > 0 ? args[0] : "npm";
|
||||
const commandArgs = args.length > 1 ? args.slice(1) : ["run", "test:unit"];
|
||||
|
||||
console.log(`Running flakiness check: ${command} ${commandArgs.join(" ")} (${ITERATIONS}x)`);
|
||||
|
||||
function runTest(iteration) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("npm", ["run", "test:unit"], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
shell: true
|
||||
});
|
||||
const child = spawn(command, commandArgs, { stdio: "pipe" });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
let output = "";
|
||||
child.stdout.on("data", (d) => output += d.toString());
|
||||
child.stderr.on("data", (d) => output += d.toString());
|
||||
child.stdout.on("data", (data) => { stdout += data.toString(); });
|
||||
child.stderr.on("data", (data) => { stderr += data.toString(); });
|
||||
|
||||
child.on("close", (code) => {
|
||||
resolve({ code, output });
|
||||
// Check for TAP failures or other failure indicators
|
||||
const hasTapFailure = stdout.includes("not ok") || stdout.includes("FAIL");
|
||||
const passed = code === 0 && !hasTapFailure;
|
||||
resolve({ passed, code, stdout, stderr });
|
||||
});
|
||||
|
||||
child.on("error", (err) => {
|
||||
resolve({ passed: false, code: -1, stdout: "", stderr: err.message });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function parseFailedFiles(output) {
|
||||
const failedFiles = [];
|
||||
const lines = output.split("\n");
|
||||
for (const line of lines) {
|
||||
// Look for "failed with exit code" or similar from run-unit-tests.mjs
|
||||
// The script logs: "Error: <relativePath> failed with exit code <code>"
|
||||
// or "Error: <relativePath> timed out..."
|
||||
const match = line.match(/Error: (.+) (failed with exit code|timed out)/);
|
||||
if (match) {
|
||||
failedFiles.push(match[1]);
|
||||
}
|
||||
}
|
||||
return failedFiles;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const matrix = {
|
||||
runs: [],
|
||||
flakyFiles: []
|
||||
};
|
||||
const results = [];
|
||||
|
||||
const fileStatus = {}; // { filename: { pass: 0, fail: 0 } }
|
||||
for (let i = 0; i < ITERATIONS; i++) {
|
||||
console.log(`Iteration ${i + 1}/${ITERATIONS}...`);
|
||||
const startTime = Date.now();
|
||||
const result = await runTest(i);
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
for (let i = 0; i < RUNS; i++) {
|
||||
const { code, output } = await runTests(i);
|
||||
const failedFiles = parseFailedFiles(output);
|
||||
const passed = code === 0;
|
||||
const entry = {
|
||||
iteration: i + 1,
|
||||
passed: result.passed,
|
||||
code: result.code,
|
||||
duration,
|
||||
failure: result.passed ? null : "Suite failed"
|
||||
};
|
||||
results.push(entry);
|
||||
|
||||
matrix.runs.push({
|
||||
run: i + 1,
|
||||
passed,
|
||||
failedFiles
|
||||
});
|
||||
|
||||
// We assume all files ran if the suite finished, or at least the ones that didn't fail passed.
|
||||
// But since run-unit-tests.mjs aborts on first failure?
|
||||
// Wait, let's check run-unit-tests.mjs logic.
|
||||
// It loops through `selectedTests` and awaits each spawn.
|
||||
// If a test fails, `finalize` is called with error.
|
||||
// `child.on("close", ...)` calls finalize.
|
||||
// The `run()` function in `run-unit-tests.mjs`:
|
||||
// `try { await run(); } catch ...`
|
||||
// Inside `run()`, it loops. If `spawn` promise rejects, the loop breaks?
|
||||
// `await new Promise(...)`. If reject is called, `run()` throws and exits?
|
||||
// Yes: `try { await run() } catch (e) { ... process.exit(1) }`.
|
||||
// So if one file fails, the whole suite stops.
|
||||
// This makes flakiness detection harder because subsequent files aren't run.
|
||||
// However, if the suite passes, all files passed.
|
||||
// If it fails, one file failed.
|
||||
|
||||
// We can just track which file failed if any.
|
||||
if (failedFiles.length > 0) {
|
||||
for (const file of failedFiles) {
|
||||
if (!fileStatus[file]) fileStatus[file] = { failures: 0 };
|
||||
fileStatus[file].failures++;
|
||||
}
|
||||
if (!result.passed) {
|
||||
console.log(` Failed (code ${result.code}).`);
|
||||
} else {
|
||||
console.log(` Passed (${duration}ms).`);
|
||||
}
|
||||
|
||||
// Save incrementally
|
||||
fs.writeFileSync(MATRIX_FILE, JSON.stringify(results, null, 2));
|
||||
}
|
||||
|
||||
// Identify flaky files: files that failed in some runs but not all?
|
||||
// Or just list files that failed at all.
|
||||
// If a file fails every time, it's broken, not flaky (unless we fixed it and it broke again).
|
||||
// If it fails sometimes, it's flaky.
|
||||
// Since we stop on first failure, we can't know if subsequent files would have failed.
|
||||
|
||||
// We'll just record the matrix.
|
||||
|
||||
await fs.writeFile(RESULTS_FILE, JSON.stringify(matrix, null, 2));
|
||||
console.log(`Flakiness check complete. Saved to ${RESULTS_FILE}`);
|
||||
console.log(`Flakiness matrix written to ${MATRIX_FILE}`);
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
main();
|
||||
@@ -1,89 +1,72 @@
|
||||
import fs from "node:fs/promises";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const rootDir = process.cwd();
|
||||
const testsDir = path.join(rootDir, "tests");
|
||||
const resultsFile = path.join(rootDir, "test-audit/suspicious-tests.json");
|
||||
const OUTPUT_FILE = "test-audit/suspicious-tests.json";
|
||||
|
||||
const checks = {
|
||||
zeroAssertions: {
|
||||
regex: /(expect\(|assert\.|t\.is|chai\.expect|should\.)/,
|
||||
message: "No obvious assertions found",
|
||||
inverted: true // fails if NOT found
|
||||
},
|
||||
focusedOrSkipped: {
|
||||
regex: /\.(only|skip)\(/,
|
||||
message: "Test is focused (.only) or skipped (.skip)"
|
||||
},
|
||||
sleeps: {
|
||||
regex: /(setTimeout\(|sleep\(|await delay\(|new Promise\(r => setTimeout)/,
|
||||
message: "Uses setTimeout/sleep (brittle)"
|
||||
},
|
||||
network: {
|
||||
regex: /(fetch\(|axios\.|new WebSocket|WebSocket\()/,
|
||||
message: "Direct network usage found"
|
||||
},
|
||||
mocking: {
|
||||
regex: /(jest\.mock|sinon\.stub|proxyquire|rewire|vi\.mock|vi\.spy)/,
|
||||
message: "Heavy mocking detected"
|
||||
},
|
||||
console: {
|
||||
regex: /console\.(log|warn|error)/,
|
||||
message: "Console usage inside test"
|
||||
}
|
||||
};
|
||||
|
||||
async function collectTests(dir, testFiles = []) {
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.name.startsWith(".")) continue;
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === "visual") continue; // Skip visual tests if needed?
|
||||
await collectTests(fullPath, testFiles);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.isFile() && (entry.name.endsWith(".test.mjs") || entry.name.endsWith(".test.js"))) {
|
||||
testFiles.push(fullPath);
|
||||
}
|
||||
}
|
||||
return testFiles;
|
||||
}
|
||||
|
||||
async function analyzeFile(filepath) {
|
||||
const content = await fs.readFile(filepath, "utf8");
|
||||
const issues = [];
|
||||
|
||||
for (const [key, check] of Object.entries(checks)) {
|
||||
const match = check.regex.test(content);
|
||||
if (check.inverted) {
|
||||
if (!match) issues.push(check.message);
|
||||
function walkDir(dir, callback) {
|
||||
if (!fs.existsSync(dir)) return;
|
||||
const files = fs.readdirSync(dir);
|
||||
for (const f of files) {
|
||||
const fullPath = path.join(dir, f);
|
||||
const stat = fs.statSync(fullPath);
|
||||
if (stat.isDirectory()) {
|
||||
if (f !== "node_modules" && f !== "artifacts" && f !== "test-audit") {
|
||||
walkDir(fullPath, callback);
|
||||
}
|
||||
} else {
|
||||
if (match) issues.push(check.message);
|
||||
callback(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const testFiles = await collectTests(testsDir);
|
||||
const suspiciousTests = {};
|
||||
function runAnalysis() {
|
||||
const skippedOrFocused = [];
|
||||
const timing = [];
|
||||
const consoleUsage = [];
|
||||
const zeroAssertions = [];
|
||||
|
||||
for (const file of testFiles) {
|
||||
const issues = await analyzeFile(file);
|
||||
if (issues.length > 0) {
|
||||
const relativePath = path.relative(rootDir, file);
|
||||
suspiciousTests[relativePath] = issues;
|
||||
const files = [];
|
||||
walkDir("js", (f) => files.push(f));
|
||||
walkDir("tests", (f) => files.push(f));
|
||||
|
||||
for (const file of files) {
|
||||
if (!file.match(/\.(js|mjs|ts)$/)) continue;
|
||||
|
||||
const content = fs.readFileSync(file, "utf8");
|
||||
const lines = content.split("\n");
|
||||
|
||||
// Zero assertions check (heuristic) - only for test files
|
||||
if (file.match(/\.(test|spec)\.(mjs|js|ts)$/)) {
|
||||
if (content.match(/describe\(|test\(|it\(/)) {
|
||||
if (!content.match(/expect\(|assert\.|t\.is|chai\.expect|should\.|ok\(/)) {
|
||||
zeroAssertions.push(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lines.forEach((line, index) => {
|
||||
const lineNum = index + 1;
|
||||
if (line.match(/\.(only|skip)\(/)) {
|
||||
skippedOrFocused.push(`${file}:${lineNum}:${line.trim()}`);
|
||||
}
|
||||
if (line.match(/setTimeout\(|sleep\(|await delay\(|new Promise\(r => setTimeout/)) {
|
||||
timing.push(`${file}:${lineNum}:${line.trim()}`);
|
||||
}
|
||||
if (line.match(/console\.(log|warn|error)/)) {
|
||||
consoleUsage.push(`${file}:${lineNum}:${line.trim()}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await fs.writeFile(resultsFile, JSON.stringify(suspiciousTests, null, 2));
|
||||
console.log(`Static analysis complete. Found issues in ${Object.keys(suspiciousTests).length} files.`);
|
||||
const report = {
|
||||
skippedOrFocused,
|
||||
timing,
|
||||
consoleUsage,
|
||||
zeroAssertions
|
||||
};
|
||||
|
||||
fs.writeFileSync(OUTPUT_FILE, JSON.stringify(report, null, 2));
|
||||
console.log(`Static analysis report written to ${OUTPUT_FILE}`);
|
||||
}
|
||||
|
||||
run().catch(console.error);
|
||||
runAnalysis();
|
||||
Reference in new issue
Block a user