From 30e0d9e287a375374b2a83b707fa86ae01dcfd67 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sun, 18 Jan 2026 06:06:37 +0000 Subject: [PATCH] feat(tools): add privacy-preserving telemetry aggregator - Added `scripts/telemetry-aggregator.mjs` to collect, sanitize, and aggregate errors from unit and smoke tests. - Added `telemetry:aggregate` script to `package.json`. - Implemented PII sanitization for IPs, emails, and hex keys. - Configured aggregation to prioritize critical/high severity issues. --- package.json | 3 +- scripts/telemetry-aggregator.mjs | 280 +++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 scripts/telemetry-aggregator.mjs diff --git a/package.json b/package.json index cb95b6ff..15daa499 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "check:css": "npm run build:css", "test:visual": "npm run build && npx playwright test", "test:visual:update": "npm run build && cross-env UPDATE_VISUAL_BASELINES=1 npx playwright test", - "test:unit": "node scripts/run-unit-tests.mjs" + "test:unit": "node scripts/run-unit-tests.mjs", + "telemetry:aggregate": "node scripts/telemetry-aggregator.mjs" }, "devDependencies": { "@playwright/test": "^1.57.0", diff --git a/scripts/telemetry-aggregator.mjs b/scripts/telemetry-aggregator.mjs new file mode 100644 index 00000000..dce666ad --- /dev/null +++ b/scripts/telemetry-aggregator.mjs @@ -0,0 +1,280 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +// Configuration +const ARTIFACTS_DIR = 'artifacts'; +const REPORTS_DIR = 'ai/reports'; +const TEST_LOG = 'test_output.log'; + +// Regex patterns for PII sanitization +const PII_PATTERNS = [ + { name: 'IP', regex: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g, replacement: '[IP]' }, + { name: 'EMAIL', regex: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, replacement: '[EMAIL]' }, + // Hex keys (64 chars) - common in Nostr (private/public keys, event IDs, signatures) + // We want to mask potential keys but maybe keep event IDs if we are sure they are IDs? + // For safety, we mask all 64-char hex strings that look like keys/signatures. + // However, stack traces might contain IDs that are useful for debugging. + // The prompt says "Strip IPs, keys, emails... Keep only stack traces and counts." + // It's better to be safe than sorry with keys. + // We will mask 64-char hex strings. + { name: 'HEX_KEY', regex: /\b[a-fA-F0-9]{64}\b/g, replacement: '[HEX_KEY]' }, + // Nostr Bech32 keys (nsec, npub, nprofile, etc.) + { name: 'NSEC', regex: /\bnsec1[a-z0-9]{50,}\b/g, replacement: '[NSEC_KEY]' }, + { name: 'NPUB', regex: /\bnpub1[a-z0-9]{50,}\b/g, replacement: '[NPUB_KEY]' }, +]; + +function sanitize(text) { + if (!text) return text; + let sanitized = text; + for (const pattern of PII_PATTERNS) { + sanitized = sanitized.replace(pattern.regex, pattern.replacement); + } + return sanitized; +} + +// ----------------------------------------------------------------------------- +// Collectors +// ----------------------------------------------------------------------------- + +function collectUnitTests() { + const errors = []; + if (!fs.existsSync(TEST_LOG)) { + console.warn(`Unit test log not found at ${TEST_LOG}`); + return errors; + } + + const content = fs.readFileSync(TEST_LOG, 'utf8'); + const lines = content.split('\n'); + let currentError = null; + let insideErrorBlock = false; + + // TAP Parsing Logic (simplified) + // We look for "not ok - " + // Then capture indentation blocks as details/stack + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Start of a failure + if (line.match(/^not ok \d+ - /)) { + if (currentError) { + errors.push(currentError); + } + const title = line.replace(/^not ok \d+ - /, '').trim(); + currentError = { + source: 'unit-test', + title: sanitize(title), + details: [], + severity: 'High' + }; + insideErrorBlock = true; + continue; + } + + // End of failure block (next test or summary) + if (line.match(/^(ok \d+|# tests \d+|1\.\.\d+)/) || (insideErrorBlock && !line.startsWith(' '))) { + if (currentError && insideErrorBlock) { + // The indentation check is a bit flaky with TAP, + // usually YAML blocks are indented. + // But "ok" or "#" definitely ends it. + + // If line doesn't start with space and it's not a TAP directive, + // it might be console output which we might want to capture? + // For now, let's assume TAP structure: + // not ok 1 - test + // --- + // error: ... + // ... + // ... + } + } + + if (currentError && insideErrorBlock) { + // Check if we exited the indented block + if (line.trim() !== '' && !line.startsWith(' ') && !line.startsWith('\t')) { + insideErrorBlock = false; + errors.push(currentError); + currentError = null; + } else { + currentError.details.push(sanitize(line)); + } + } + } + + if (currentError) { + errors.push(currentError); + } + + // Post-process to extract stack trace for fingerprinting + errors.forEach(e => { + const fullText = e.details.join('\n'); + // Simple fingerprint: file path + error message + // Try to extract "stack: |-" block + const stackMatch = fullText.match(/stack: \|-([\s\S]+?)(?:\n\s*\w+:|$)/); + if (stackMatch) { + e.stack = stackMatch[1].trim(); + } else { + // Fallback to title + e.stack = e.title; + } + + // Fingerprint: remove line numbers from stack + // Also prepend title to ensure different tests with similar stacks (common in runners) are distinct + e.fingerprint = e.title + '::' + e.stack.replace(/:\d+:\d+/g, ':x:x').replace(/\d+/g, 'N'); + }); + + return errors; +} + +function collectSmokeTests() { + const errors = []; + if (!fs.existsSync(ARTIFACTS_DIR)) return errors; + + const files = fs.readdirSync(ARTIFACTS_DIR).filter(f => f.startsWith('smoke-report-') && f.endsWith('.json')); + + for (const file of files) { + try { + const content = JSON.parse(fs.readFileSync(path.join(ARTIFACTS_DIR, file), 'utf8')); + if (content.stats && content.stats.failures > 0) { + const failedSteps = content.stats.details.filter(d => !d.success); + for (const step of failedSteps) { + errors.push({ + source: 'smoke-test', + title: sanitize(`Smoke Test Failed: ${step.name}`), + details: [sanitize(step.error)], + stack: sanitize(step.error), + fingerprint: sanitize(step.name), // Group by step name + severity: 'Critical' + }); + } + } + } catch (err) { + console.warn(`Failed to parse smoke report ${file}:`, err); + } + } + return errors; +} + +// ----------------------------------------------------------------------------- +// Aggregation +// ----------------------------------------------------------------------------- + +function aggregate(errors) { + const grouped = {}; + + for (const err of errors) { + const key = err.fingerprint || err.title; + if (!grouped[key]) { + grouped[key] = { + fingerprint: key, + title: err.title, + count: 0, + sources: new Set(), + severity: err.severity, + exampleStack: err.stack + }; + } + grouped[key].count++; + grouped[key].sources.add(err.source); + // Escalating severity if we see critical + if (err.severity === 'Critical') grouped[key].severity = 'Critical'; + } + + return Object.values(grouped).sort((a, b) => { + // Sort by Severity (Critical > High > Medium) then Count (Desc) + const severityScore = { 'Critical': 3, 'High': 2, 'Medium': 1 }; + const scoreA = severityScore[a.severity] || 0; + const scoreB = severityScore[b.severity] || 0; + + if (scoreA !== scoreB) return scoreB - scoreA; + return b.count - a.count; + }); +} + +// ----------------------------------------------------------------------------- +// Reporting +// ----------------------------------------------------------------------------- + +function generateReport(aggregates) { + const dateStr = new Date().toISOString().slice(0, 10).replace(/-/g, ''); + const filename = `telemetry-${dateStr}.md`; + const filepath = path.join(REPORTS_DIR, filename); + + if (!fs.existsSync(REPORTS_DIR)) { + fs.mkdirSync(REPORTS_DIR, { recursive: true }); + } + + let md = `# Telemetry Report - ${new Date().toISOString().slice(0,10)}\n\n`; + md += `**Total Unique Issues:** ${aggregates.length}\n`; + md += `**Generated:** ${new Date().toISOString()}\n\n`; + + md += `## Top 10 Priority Issues\n\n`; + md += `| Priority | Count | Issue | Sources |\n`; + md += `| :--- | :---: | :--- | :--- |\n`; + + const top10 = aggregates.slice(0, 10); + + for (const item of top10) { + const titleShort = item.title.length > 80 ? item.title.substring(0, 77) + '...' : item.title; + md += `| **${item.severity}** | ${item.count} | ${titleShort} | ${Array.from(item.sources).join(', ')} |\n`; + } + + md += `\n## Detailed Breakdown\n\n`; + + for (const item of top10) { + md += `### [${item.severity}] ${item.title}\n`; + md += `- **Occurrences:** ${item.count}\n`; + md += `- **Sources:** ${Array.from(item.sources).join(', ')}\n`; + md += `- **Suggested Owner:** TBD (Automated Triage)\n`; + md += `\n**Stack Trace / Details:**\n`; + md += "```\n"; + md += item.exampleStack || "No stack trace available"; + md += "\n```\n\n"; + } + + // Privacy note + md += `\n---\n*Privacy Notice: PII (IPs, emails, keys) has been sanitized from this report.*\n`; + + fs.writeFileSync(filepath, md); + console.log(`Report generated: ${filepath}`); + return filepath; +} + +function generateJSONArtifact(aggregates) { + const filepath = path.join(ARTIFACTS_DIR, 'error-aggregates.json'); + // Convert Set to Array for JSON + const data = aggregates.map(a => ({ + ...a, + sources: Array.from(a.sources) + })); + fs.writeFileSync(filepath, JSON.stringify(data, null, 2)); + console.log(`Artifact generated: ${filepath}`); +} + +// ----------------------------------------------------------------------------- +// Main +// ----------------------------------------------------------------------------- + +async function main() { + console.log('Starting Telemetry Aggregation...'); + + // 1. Collect + const unitTestErrors = collectUnitTests(); + const smokeTestErrors = collectSmokeTests(); + console.log(`Collected ${unitTestErrors.length} unit test errors.`); + console.log(`Collected ${smokeTestErrors.length} smoke test errors.`); + + const allErrors = [...unitTestErrors, ...smokeTestErrors]; + + // 2. Aggregate + const aggregates = aggregate(allErrors); + console.log(`Aggregated into ${aggregates.length} unique issues.`); + + // 3. Report + generateJSONArtifact(aggregates); + generateReport(aggregates); + + console.log('Telemetry Aggregation Complete.'); +} + +main();