From c9201c777e14448acdc38c91b4ebecb0d39055c4 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sat, 24 Jan 2026 06:29:10 +0000 Subject: [PATCH] Add privacy-preserving telemetry aggregation script Moves and enhances the telemetry aggregator script to `scripts/agent/telemetry-aggregator.mjs`. Updates `package.json` to point to the new location. Implements robust parsing for smoke test summaries and fuzz reports. Adds PII sanitization (IPs, emails, keys, paths). Enforces opt-in via `ENABLE_TELEMETRY` environment variable. Adds `ai/reports/` to `.gitignore`. --- .gitignore | 1 + package.json | 2 +- scripts/{ => agent}/telemetry-aggregator.mjs | 107 +++++++++++++++---- 3 files changed, 86 insertions(+), 24 deletions(-) rename scripts/{ => agent}/telemetry-aggregator.mjs (78%) diff --git a/.gitignore b/.gitignore index 2f2cee23..23aad3df 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ test-results/ tests/visual/__snapshots__/ artifacts/ css/tailwind.generated.css +ai/reports/ diff --git a/package.json b/package.json index 484cbe69..83d2d6be 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "test:unit:shard1": "cross-env UNIT_TEST_SHARD=1/3 node scripts/run-unit-tests.mjs", "test:unit:shard2": "cross-env UNIT_TEST_SHARD=2/3 node scripts/run-unit-tests.mjs", "test:unit:shard3": "cross-env UNIT_TEST_SHARD=3/3 node scripts/run-unit-tests.mjs", - "telemetry:aggregate": "node scripts/telemetry-aggregator.mjs" + "telemetry:aggregate": "node scripts/agent/telemetry-aggregator.mjs" }, "devDependencies": { "@playwright/test": "^1.57.0", diff --git a/scripts/telemetry-aggregator.mjs b/scripts/agent/telemetry-aggregator.mjs similarity index 78% rename from scripts/telemetry-aggregator.mjs rename to scripts/agent/telemetry-aggregator.mjs index 179d6c93..8146c4b9 100644 --- a/scripts/telemetry-aggregator.mjs +++ b/scripts/agent/telemetry-aggregator.mjs @@ -94,9 +94,6 @@ function collectUnitTests() { currentError = null; } else { // Capture detail lines - // If line is empty or indented, it's likely part of the error - // If it's unindented but not a TAP command, it might still be part of it (console logs) - // We'll be greedy here. currentError.details.push(sanitize(line)); } } @@ -175,28 +172,85 @@ function collectSmokeTests() { const errors = []; if (!fs.existsSync(ARTIFACTS_DIR)) return errors; - const files = fs.readdirSync(ARTIFACTS_DIR).filter(f => f.startsWith('smoke-report-') && f.endsWith('.json')); + // Look for smoke-summary-*.json + const files = fs.readdirSync(ARTIFACTS_DIR).filter(f => f.startsWith('smoke-summary-') && f.endsWith('.json')); for (const file of files) { try { const content = JSON.parse(fs.readFileSync(path.join(ARTIFACTS_DIR, file), 'utf8')); - if (content.stats && content.stats.failures > 0) { - const failedSteps = content.stats.details.filter(d => !d.success); - for (const step of failedSteps) { - const sanitizedError = sanitize(step.error); - errors.push({ - source: 'smoke-test', - title: sanitize(`Smoke Test Failed: ${step.name}`), - details: [sanitizedError], - stack: sanitizedError, - fingerprint: getFingerprint(sanitizedError + step.name), - severity: 'Critical', - owner: suggestOwner(sanitizedError) || 'QA Team' - }); - } + + // Expected format: { timestamp, status, error, logs } + if (content.status === 'failure') { + const errorMsg = content.error || 'Unknown Smoke Test Failure'; + const sanitizedError = sanitize(errorMsg); + + // Combine relevant logs if available for context + const logs = (content.logs || []) + .map(l => sanitize(l)) + .join('\n') + .substring(0, 2000); // Limit size + + errors.push({ + source: 'smoke-test', + title: `Smoke Test Failed: ${sanitizedError.split('\n')[0]}`, + details: content.logs ? content.logs.map(sanitize) : [], + stack: sanitizedError + (logs ? '\n\nContext Logs:\n' + logs : ''), + fingerprint: getFingerprint(sanitizedError), + severity: 'Critical', + owner: suggestOwner(sanitizedError) || 'QA Team' + }); } } catch (err) { - console.warn(`Failed to parse smoke report ${file}:`, err.message); + console.warn(`Failed to parse smoke summary ${file}:`, err.message); + } + } + return errors; +} + +function collectFuzzReports() { + const errors = []; + if (!fs.existsSync(ARTIFACTS_DIR)) return errors; + + // Look for fuzz-report-*.json + const files = fs.readdirSync(ARTIFACTS_DIR).filter(f => f.startsWith('fuzz-report-') && f.endsWith('.json')); + + for (const file of files) { + try { + const content = JSON.parse(fs.readFileSync(path.join(ARTIFACTS_DIR, file), 'utf8')); + + // Support both 'crashes' (fuzz-shared.mjs) and 'issues' (fuzz-lib.mjs) + const crashes = content.crashes || content.issues || []; + + for (const crash of crashes) { + // Crash structure might vary slightly + // fuzz-shared: { error: { message, stack }, reproducer } + // fuzz-lib: { message, stack, hash, input } + + let message = crash.message; + let stack = crash.stack; + + if (!message && crash.error) { + message = crash.error.message; + stack = crash.error.stack; + } + + if (!message) continue; + + const sanitizedMessage = sanitize(message); + const sanitizedStack = sanitize(stack || message); + + errors.push({ + source: `fuzz-test:${content.target || 'unknown'}`, + title: `Fuzz Crash: ${sanitizedMessage}`, + details: [sanitizedStack], + stack: sanitizedStack, + fingerprint: getFingerprint(sanitizedStack), + severity: 'High', + owner: suggestOwner(sanitizedStack) + }); + } + } catch (err) { + console.warn(`Failed to parse fuzz report ${file}:`, err.message); } } return errors; @@ -206,10 +260,13 @@ function collectAgentLogs() { const errors = []; if (!fs.existsSync(ARTIFACTS_DIR)) return errors; - // Look for any .log files in artifacts that aren't the main ones we already checked (if any) + // Look for any .log files in artifacts const files = fs.readdirSync(ARTIFACTS_DIR).filter(f => f.endsWith('.log')); for (const file of files) { + // Avoid reparsing if we have specific logic (though generic is fine for now) + if (file === 'test_output.log') continue; + errors.push(...collectGenericLogs(path.join(ARTIFACTS_DIR, file), `agent-log:${file}`)); } return errors; @@ -239,8 +296,6 @@ function aggregate(errors) { grouped[key].sources.add(err.source); // Escalating severity if we see critical if (err.severity === 'Critical') grouped[key].severity = 'Critical'; - - // Update title if new one is longer/better (optional, keeping first for now) } return Object.values(grouped).sort((a, b) => { @@ -326,7 +381,8 @@ function generateJSONArtifact(aggregates) { // ----------------------------------------------------------------------------- async function main() { - if (process.env.ENABLE_TELEMETRY !== 'true') { + const enabled = process.env.ENABLE_TELEMETRY === 'true' || process.env.ENABLE_TELEMETRY === '1'; + if (!enabled) { console.log('Telemetry is opt-in. Set ENABLE_TELEMETRY=true to run.'); process.exit(0); } @@ -354,6 +410,11 @@ async function main() { console.log(`Collected ${smokeTestErrors.length} smoke test errors.`); errors.push(...smokeTestErrors); + // Fuzz Reports + const fuzzErrors = collectFuzzReports(); + console.log(`Collected ${fuzzErrors.length} fuzz reports.`); + errors.push(...fuzzErrors); + // Agent Logs const agentErrors = collectAgentLogs(); console.log(`Collected ${agentErrors.length} agent log errors.`);