import fs from 'node:fs'; import path from 'node:path'; // Configuration const ARTIFACTS_DIR = 'artifacts'; const REPORTS_DIR = 'ai/reports'; const TEST_LOG = 'test_output.log'; // Regex patterns for PII sanitization const PII_PATTERNS = [ { name: 'IP', regex: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g, replacement: '[IP]' }, { name: 'EMAIL', regex: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, replacement: '[EMAIL]' }, // Hex keys (64 chars) - common in Nostr (private/public keys, event IDs, signatures) // We want to mask potential keys but maybe keep event IDs if we are sure they are IDs? // For safety, we mask all 64-char hex strings that look like keys/signatures. // However, stack traces might contain IDs that are useful for debugging. // The prompt says "Strip IPs, keys, emails... Keep only stack traces and counts." // It's better to be safe than sorry with keys. // We will mask 64-char hex strings. { name: 'HEX_KEY', regex: /\b[a-fA-F0-9]{64}\b/g, replacement: '[HEX_KEY]' }, // Nostr Bech32 keys (nsec, npub, nprofile, etc.) { name: 'NSEC', regex: /\bnsec1[a-z0-9]{50,}\b/g, replacement: '[NSEC_KEY]' }, { name: 'NPUB', regex: /\bnpub1[a-z0-9]{50,}\b/g, replacement: '[NPUB_KEY]' }, ]; function sanitize(text) { if (!text) return text; let sanitized = text; for (const pattern of PII_PATTERNS) { sanitized = sanitized.replace(pattern.regex, pattern.replacement); } return sanitized; } // ----------------------------------------------------------------------------- // Collectors // ----------------------------------------------------------------------------- function collectUnitTests() { const errors = []; if (!fs.existsSync(TEST_LOG)) { console.warn(`Unit test log not found at ${TEST_LOG}`); return errors; } const content = fs.readFileSync(TEST_LOG, 'utf8'); const lines = content.split('\n'); let currentError = null; let insideErrorBlock = false; // TAP Parsing Logic (simplified) // We look for "not ok - " // Then capture indentation blocks as details/stack for (let i = 0; i < lines.length; i++) { const line = lines[i]; // Start of a failure if (line.match(/^not ok \d+ - /)) { if (currentError) { errors.push(currentError); } const title = line.replace(/^not ok \d+ - /, '').trim(); currentError = { source: 'unit-test', title: sanitize(title), details: [], severity: 'High' }; insideErrorBlock = true; continue; } // End of failure block (next test or summary) if (line.match(/^(ok \d+|# tests \d+|1\.\.\d+)/) || (insideErrorBlock && !line.startsWith(' '))) { if (currentError && insideErrorBlock) { // The indentation check is a bit flaky with TAP, // usually YAML blocks are indented. // But "ok" or "#" definitely ends it. // If line doesn't start with space and it's not a TAP directive, // it might be console output which we might want to capture? // For now, let's assume TAP structure: // not ok 1 - test // --- // error: ... // ... // ... } } if (currentError && insideErrorBlock) { // Check if we exited the indented block if (line.trim() !== '' && !line.startsWith(' ') && !line.startsWith('\t')) { insideErrorBlock = false; errors.push(currentError); currentError = null; } else { currentError.details.push(sanitize(line)); } } } if (currentError) { errors.push(currentError); } // Post-process to extract stack trace for fingerprinting errors.forEach(e => { const fullText = e.details.join('\n'); // Simple fingerprint: file path + error message // Try to extract "stack: |-" block const stackMatch = fullText.match(/stack: \|-([\s\S]+?)(?:\n\s*\w+:|$)/); if (stackMatch) { e.stack = stackMatch[1].trim(); } else { // Fallback to title e.stack = e.title; } // Fingerprint: remove line numbers from stack // Also prepend title to ensure different tests with similar stacks (common in runners) are distinct e.fingerprint = e.title + '::' + e.stack.replace(/:\d+:\d+/g, ':x:x').replace(/\d+/g, 'N'); }); return errors; } function collectSmokeTests() { const errors = []; if (!fs.existsSync(ARTIFACTS_DIR)) return errors; const files = fs.readdirSync(ARTIFACTS_DIR).filter(f => f.startsWith('smoke-report-') && f.endsWith('.json')); for (const file of files) { try { const content = JSON.parse(fs.readFileSync(path.join(ARTIFACTS_DIR, file), 'utf8')); if (content.stats && content.stats.failures > 0) { const failedSteps = content.stats.details.filter(d => !d.success); for (const step of failedSteps) { errors.push({ source: 'smoke-test', title: sanitize(`Smoke Test Failed: ${step.name}`), details: [sanitize(step.error)], stack: sanitize(step.error), fingerprint: sanitize(step.name), // Group by step name severity: 'Critical' }); } } } catch (err) { console.warn(`Failed to parse smoke report ${file}:`, err); } } return errors; } // ----------------------------------------------------------------------------- // Aggregation // ----------------------------------------------------------------------------- function aggregate(errors) { const grouped = {}; for (const err of errors) { const key = err.fingerprint || err.title; if (!grouped[key]) { grouped[key] = { fingerprint: key, title: err.title, count: 0, sources: new Set(), severity: err.severity, exampleStack: err.stack }; } grouped[key].count++; grouped[key].sources.add(err.source); // Escalating severity if we see critical if (err.severity === 'Critical') grouped[key].severity = 'Critical'; } return Object.values(grouped).sort((a, b) => { // Sort by Severity (Critical > High > Medium) then Count (Desc) const severityScore = { 'Critical': 3, 'High': 2, 'Medium': 1 }; const scoreA = severityScore[a.severity] || 0; const scoreB = severityScore[b.severity] || 0; if (scoreA !== scoreB) return scoreB - scoreA; return b.count - a.count; }); } // ----------------------------------------------------------------------------- // Reporting // ----------------------------------------------------------------------------- function generateReport(aggregates) { const dateStr = new Date().toISOString().slice(0, 10).replace(/-/g, ''); const filename = `telemetry-${dateStr}.md`; const filepath = path.join(REPORTS_DIR, filename); if (!fs.existsSync(REPORTS_DIR)) { fs.mkdirSync(REPORTS_DIR, { recursive: true }); } let md = `# Telemetry Report - ${new Date().toISOString().slice(0,10)}\n\n`; md += `**Total Unique Issues:** ${aggregates.length}\n`; md += `**Generated:** ${new Date().toISOString()}\n\n`; md += `## Top 10 Priority Issues\n\n`; md += `| Priority | Count | Issue | Sources |\n`; md += `| :--- | :---: | :--- | :--- |\n`; const top10 = aggregates.slice(0, 10); for (const item of top10) { const titleShort = item.title.length > 80 ? item.title.substring(0, 77) + '...' : item.title; md += `| **${item.severity}** | ${item.count} | ${titleShort} | ${Array.from(item.sources).join(', ')} |\n`; } md += `\n## Detailed Breakdown\n\n`; for (const item of top10) { md += `### [${item.severity}] ${item.title}\n`; md += `- **Occurrences:** ${item.count}\n`; md += `- **Sources:** ${Array.from(item.sources).join(', ')}\n`; md += `- **Suggested Owner:** TBD (Automated Triage)\n`; md += `\n**Stack Trace / Details:**\n`; md += "```\n"; md += item.exampleStack || "No stack trace available"; md += "\n```\n\n"; } // Privacy note md += `\n---\n*Privacy Notice: PII (IPs, emails, keys) has been sanitized from this report.*\n`; fs.writeFileSync(filepath, md); console.log(`Report generated: ${filepath}`); return filepath; } function generateJSONArtifact(aggregates) { const filepath = path.join(ARTIFACTS_DIR, 'error-aggregates.json'); // Convert Set to Array for JSON const data = aggregates.map(a => ({ ...a, sources: Array.from(a.sources) })); fs.writeFileSync(filepath, JSON.stringify(data, null, 2)); console.log(`Artifact generated: ${filepath}`); } // ----------------------------------------------------------------------------- // Main // ----------------------------------------------------------------------------- async function main() { console.log('Starting Telemetry Aggregation...'); // 1. Collect const unitTestErrors = collectUnitTests(); const smokeTestErrors = collectSmokeTests(); console.log(`Collected ${unitTestErrors.length} unit test errors.`); console.log(`Collected ${smokeTestErrors.length} smoke test errors.`); const allErrors = [...unitTestErrors, ...smokeTestErrors]; // 2. Aggregate const aggregates = aggregate(allErrors); console.log(`Aggregated into ${aggregates.length} unique issues.`); // 3. Report generateJSONArtifact(aggregates); generateReport(aggregates); console.log('Telemetry Aggregation Complete.'); } main();