From da64e308af94c6669056d115a5a4017832ffc63a Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 19:26:49 +0000 Subject: [PATCH] feat(agent): add load testing script and report --- scripts/agent/load-test.mjs | 540 ++++++++++++++++++------------------ 1 file changed, 276 insertions(+), 264 deletions(-) diff --git a/scripts/agent/load-test.mjs b/scripts/agent/load-test.mjs index abacb81c..f0d026c6 100644 --- a/scripts/agent/load-test.mjs +++ b/scripts/agent/load-test.mjs @@ -1,306 +1,318 @@ import WebSocket from 'ws'; -import { finalizeEvent, generateSecretKey, getPublicKey } from 'nostr-tools'; -import { writeFile, mkdir } from 'fs/promises'; +import { generateSecretKey, getPublicKey, finalizeEvent } from 'nostr-tools'; +import { spawn } from 'child_process'; +import fs from 'fs'; import path from 'path'; -import { performance } from 'perf_hooks'; +import { fileURLToPath } from 'url'; -// --- Configuration --- -const ARGS = process.argv.slice(2); -const getConfig = (key, defaultVal) => { - const idx = ARGS.indexOf(`--${key}`); - return idx !== -1 ? ARGS[idx + 1] : defaultVal; -}; - -const hasFlag = (flag) => ARGS.includes(`--${flag}`); +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const PROJECT_ROOT = path.resolve(__dirname, '../..'); +// Configuration const CONFIG = { - clients: parseInt(getConfig('clients', '100'), 10), - duration: parseInt(getConfig('duration', '60'), 10), // seconds - relay: getConfig('relay', 'ws://localhost:8008'), - rate: parseInt(getConfig('rate', '50'), 10), // target events per second (global) - unsafe: hasFlag('unsafe'), - outputDir: 'artifacts', + relayUrl: 'ws://localhost:8008', + numClients: 1000, + durationSeconds: 60, + eventsPerSecond: 100, + mix: { + viewEvent: 0.8, + videoPost: 0.2 + }, + unsafe: false }; -// --- Safety Check --- +// Parse args +const args = process.argv.slice(2); +for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--clients') CONFIG.numClients = parseInt(args[++i], 10); + else if (arg === '--duration') CONFIG.durationSeconds = parseInt(args[++i], 10); + else if (arg === '--rate') CONFIG.eventsPerSecond = parseInt(args[++i], 10); + else if (arg === '--relay') CONFIG.relayUrl = args[++i]; + else if (arg === '--unsafe') CONFIG.unsafe = true; +} + +console.log('Load Test Configuration:', CONFIG); + +// Guardrail function isSafeRelay(url) { if (CONFIG.unsafe) return true; try { const u = new URL(url); - if (u.hostname === 'localhost' || u.hostname === '127.0.0.1' || u.hostname === '[::1]') return true; - // Check for private IPs (10.x, 192.168.x, 172.16-31.x) could be added here, - // but for now localhost is the strict guardrail. - return false; - } catch (e) { + return u.hostname === 'localhost' || u.hostname === '127.0.0.1' || u.hostname === '[::1]'; + } catch { return false; } } -if (!isSafeRelay(CONFIG.relay)) { - console.error(`\n[FATAL] Guardrail triggered: Attempting to load test a non-local relay (${CONFIG.relay}) without --unsafe flag.`); - console.error("Please target a local relay or use --unsafe if you are sure you own the infrastructure.\n"); +if (!isSafeRelay(CONFIG.relayUrl)) { + console.error('ERROR: Guardrail active. Cannot run against public relays without --unsafe flag.'); process.exit(1); } -// --- Helpers --- -const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +// Artifacts setup +const ARTIFACTS_DIR = path.join(PROJECT_ROOT, 'artifacts'); +if (!fs.existsSync(ARTIFACTS_DIR)) { + fs.mkdirSync(ARTIFACTS_DIR, { recursive: true }); +} -const NOTE_TYPES = { - VIDEO_POST: 30078, - VIEW_EVENT: 30079 +// Helpers +function buildViewEvent(pubkey) { + return { + kind: 30079, + created_at: Math.floor(Date.now() / 1000), + tags: [ + ['d', 'watch-history'], + ['t', 'view'], + ['session', 'true'] + ], + content: JSON.stringify({ timestamp: Date.now(), random: Math.random() }), + pubkey + }; +} + +function buildVideoPostEvent(pubkey) { + return { + kind: 30078, + created_at: Math.floor(Date.now() / 1000), + tags: [ + ['d', Math.random().toString(36).substring(7)], + ['t', 'video'] + ], + content: JSON.stringify({ + version: 1, + title: `Load Test Video ${Math.random().toString(36)}`, + videoRootId: Math.random().toString(36), + mode: 's', + duration: 120 + }), + pubkey + }; +} + +// State +const metrics = { + sent: 0, + ok: 0, + failed: 0, + latencies: [], + signingTimes: [], + startTime: Date.now(), + startCpu: process.cpuUsage(), + errors: [] }; -// Simplified Event Builders based on js/nostrEventSchemas.js -function buildVideoPostEvent(signer, contentData) { - const created_at = Math.floor(Date.now() / 1000); - const event = { - kind: NOTE_TYPES.VIDEO_POST, - created_at, - tags: [ - ['t', 'video'], - ['d', `video-${created_at}-${Math.random().toString(36).slice(2)}`] - ], - content: JSON.stringify(contentData), - pubkey: getPublicKey(signer.privateKey), - }; - return finalizeEvent(event, signer.privateKey); -} +// Main execution +async function main() { + let relayProcess = null; + const isLocalRelay = CONFIG.relayUrl.includes('localhost') || CONFIG.relayUrl.includes('127.0.0.1'); -function buildViewEvent(signer, videoId) { - const created_at = Math.floor(Date.now() / 1000); - const event = { - kind: NOTE_TYPES.VIEW_EVENT, - created_at, - tags: [ - ['t', 'view'], - ['e', videoId], - ['d', `view-${created_at}-${Math.random().toString(36).slice(2)}`] // Dedupe tag - ], - content: '', - pubkey: getPublicKey(signer.privateKey), - }; - return finalizeEvent(event, signer.privateKey); -} - -// --- Client Class --- -class LoadClient { - constructor(relayUrl, id) { - this.id = id; - this.relayUrl = relayUrl; - this.privateKey = generateSecretKey(); - this.pubkey = getPublicKey(this.privateKey); - this.ws = null; - this.connected = false; - this.pending = new Map(); // eventId -> startTime + if (isLocalRelay) { + try { + await checkRelay(CONFIG.relayUrl); + console.log('Relay is already running.'); + } catch (e) { + console.log('Relay not running, spawning simple-relay.mjs...'); + const relayScript = path.join(PROJECT_ROOT, 'scripts/agent/simple-relay.mjs'); + if (!fs.existsSync(relayScript)) { + console.error(`Simple relay script not found at ${relayScript}`); + process.exit(1); + } + relayProcess = spawn('node', [relayScript], { + stdio: 'inherit', + detached: false + }); + await new Promise(resolve => setTimeout(resolve, 2000)); } + } - connect() { - return new Promise((resolve, reject) => { - this.ws = new WebSocket(this.relayUrl); - this.ws.on('open', () => { - this.connected = true; - resolve(); - }); - this.ws.on('error', (err) => { - // console.error(`Client ${this.id} error:`, err.message); - reject(err); - }); - this.ws.on('message', (data) => this.handleMessage(data)); - this.ws.on('close', () => { - this.connected = false; - }); + console.log(`Simulating ${CONFIG.numClients} clients...`); + + const MAX_CONNS = Math.min(CONFIG.numClients, 100); + const connections = []; + + for (let i = 0; i < MAX_CONNS; i++) { + const ws = new WebSocket(CONFIG.relayUrl); + ws.on('error', (err) => { + metrics.errors.push(err.message); + }); + connections.push(ws); + } + + await Promise.all(connections.map(ws => new Promise(resolve => { + if (ws.readyState === WebSocket.OPEN) resolve(); + else ws.on('open', resolve); + setTimeout(resolve, 1000); + }))); + + console.log(`${connections.length} connections established.`); + + const keypairs = []; + for (let i = 0; i < CONFIG.numClients; i++) { + const sk = generateSecretKey(); + keypairs.push({ + sk, + pk: getPublicKey(sk) + }); + } + + console.log('Starting load generation...'); + const intervalMs = 1000 / CONFIG.eventsPerSecond; + const endAt = Date.now() + (CONFIG.durationSeconds * 1000); + + const pendingLatencies = new Map(); + + connections.forEach(ws => { + ws.removeAllListeners('message'); + ws.on('message', (data) => { + try { + const msg = JSON.parse(data); + if (msg[0] === 'OK') { + const eventId = msg[1]; + if (pendingLatencies.has(eventId)) { + const start = pendingLatencies.get(eventId); + const latency = Date.now() - start; + metrics.latencies.push(latency); + pendingLatencies.delete(eventId); + } + metrics.ok++; + } else if (msg[0] === 'NOTICE') { + metrics.errors.push(msg[1]); + } + } catch (e) {} + }); + }); + + const loop = async () => { + while (Date.now() < endAt) { + const loopStart = Date.now(); + + const client = keypairs[Math.floor(Math.random() * keypairs.length)]; + const ws = connections[Math.floor(Math.random() * connections.length)]; + + if (ws.readyState === WebSocket.OPEN) { + let eventTemplate; + if (Math.random() < CONFIG.mix.viewEvent) { + eventTemplate = buildViewEvent(client.pk); + } else { + eventTemplate = buildVideoPostEvent(client.pk); + } + + try { + const signStart = process.hrtime(); + const event = finalizeEvent(eventTemplate, client.sk); + const signEnd = process.hrtime(signStart); + const signMs = (signEnd[0] * 1000) + (signEnd[1] / 1e6); + metrics.signingTimes.push(signMs); + + const msg = JSON.stringify(['EVENT', event]); + + if (Math.random() < 0.05) { + pendingLatencies.set(event.id, Date.now()); + } + + ws.send(msg); + metrics.sent++; + } catch (err) { + metrics.failed++; + metrics.errors.push(err.message); + } + } + + const elapsed = Date.now() - loopStart; + const targetDelay = 1000 / CONFIG.eventsPerSecond; + if (elapsed < targetDelay) { + await new Promise(r => setTimeout(r, targetDelay - elapsed)); + } + } + finish(relayProcess); + }; + + loop(); +} + +function checkRelay(url) { + return new Promise((resolve, reject) => { + const ws = new WebSocket(url); + ws.on('open', () => { + ws.close(); + resolve(); }); - } - - handleMessage(data) { - try { - const msg = JSON.parse(data); - if (msg[0] === 'OK') { - const eventId = msg[1]; - const success = msg[2]; - const message = msg[3]; - - if (this.pending.has(eventId)) { - const startTime = this.pending.get(eventId); - const latency = Date.now() - startTime; - this.pending.delete(eventId); - stats.recordAck(latency, success, message); - } - } - } catch (e) { - console.error('Error parsing message:', e); - } - } - - publish(event) { - if (!this.connected) return; - this.pending.set(event.id, Date.now()); - this.ws.send(JSON.stringify(['EVENT', event])); - stats.recordSent(); - } - - close() { - if (this.ws) this.ws.close(); - } + ws.on('error', (err) => { + reject(err); + }); + }); } -// --- Metrics --- -const stats = { - startTime: Date.now(), - sent: 0, - acked: 0, - failed: 0, - latencies: [], - signingTimes: [], - errors: [], - startCpu: process.cpuUsage(), +function finish(relayProcess) { + console.log('Test finished. Generating report...'); - recordSent() { - this.sent++; - }, + const cpuUsage = process.cpuUsage(metrics.startCpu); + const memUsage = process.memoryUsage(); - recordAck(latency, success, message) { - if (success) { - this.acked++; - this.latencies.push(latency); - } else { - this.failed++; - this.errors.push(message); - } - }, + const avgLatency = metrics.latencies.reduce((a, b) => a + b, 0) / (metrics.latencies.length || 1); + const sortedLatencies = metrics.latencies.sort((a, b) => a - b); + const p95 = sortedLatencies[Math.floor(sortedLatencies.length * 0.95)] || 0; + const maxLatency = sortedLatencies[sortedLatencies.length - 1] || 0; - recordSigningTime(ms) { - this.signingTimes.push(ms); - }, + const avgSignTime = metrics.signingTimes.reduce((a, b) => a + b, 0) / (metrics.signingTimes.length || 1); + const sortedSignTimes = metrics.signingTimes.sort((a, b) => a - b); + const p95SignTime = sortedSignTimes[Math.floor(sortedSignTimes.length * 0.95)] || 0; - getSummary() { - this.latencies.sort((a, b) => a - b); - this.signingTimes.sort((a, b) => a - b); + const durationSec = (Date.now() - metrics.startTime) / 1000; + const throughput = metrics.ok / durationSec; - const calcStats = (arr) => { - const p50 = arr[Math.floor(arr.length * 0.5)] || 0; - const p95 = arr[Math.floor(arr.length * 0.95)] || 0; - const p99 = arr[Math.floor(arr.length * 0.99)] || 0; - const avg = arr.reduce((a, b) => a + b, 0) / (arr.length || 1); - return { avg: avg.toFixed(2), p50: p50.toFixed(2), p95: p95.toFixed(2), p99: p99.toFixed(2) }; - }; - - const durationSec = (Date.now() - this.startTime) / 1000; - const latencyStats = calcStats(this.latencies); - const signingStats = calcStats(this.signingTimes); - - const cpuUsage = process.cpuUsage(this.startCpu); - const memUsage = process.memoryUsage(); - - return { - config: CONFIG, - durationSec: durationSec.toFixed(2), - events: { - sent: this.sent, - acked: this.acked, - failed: this.failed, + const report = { + timestamp: new Date().toISOString(), + config: CONFIG, + metrics: { + durationSeconds: durationSec, + totalSent: metrics.sent, + totalOk: metrics.ok, + totalFailed: metrics.failed, + throughputEventsPerSec: throughput, + latencyMs: { + avg: avgLatency, + p95: p95, + max: maxLatency }, - throughput: { - sentPerSec: (this.sent / durationSec).toFixed(2), - ackedPerSec: (this.acked / durationSec).toFixed(2), + crypto: { + avgSigningTimeMs: avgSignTime, + p95SigningTimeMs: p95SignTime }, - latencyMs: latencyStats, - signingTimeMs: signingStats, resources: { - cpuUserSec: (cpuUsage.user / 1e6).toFixed(2), - cpuSystemSec: (cpuUsage.system / 1e6).toFixed(2), - memoryRssMb: (memUsage.rss / 1024 / 1024).toFixed(2), - memoryHeapMb: (memUsage.heapUsed / 1024 / 1024).toFixed(2) + cpuUser: cpuUsage.user, + cpuSystem: cpuUsage.system, + memoryRss: memUsage.rss, + memoryHeapTotal: memUsage.heapTotal, + memoryHeapUsed: memUsage.heapUsed }, - errors: this.errors.slice(0, 10), // Sample top 10 - hotFunctions: [ - { name: "finalizeEvent (signing)", avgMs: signingStats.avg } - ], - bottlenecks: signingStats.avg > 100 ? ["Cryptographic operations (signing)"] : [] - }; - } -}; - -// --- Main --- -async function run() { - console.log(`Starting load test with ${CONFIG.clients} clients against ${CONFIG.relay}`); - console.log(`Target rate: ${CONFIG.rate} events/sec`); - console.log(`Duration: ${CONFIG.duration} seconds`); - - const clients = []; - for (let i = 0; i < CONFIG.clients; i++) { - clients.push(new LoadClient(CONFIG.relay, i)); - } - - console.log('Connecting clients...'); - await Promise.all(clients.map(c => c.connect().catch(e => null))); - const connectedCount = clients.filter(c => c.connected).length; - console.log(`Connected ${connectedCount}/${CONFIG.clients} clients.`); - - if (connectedCount === 0) { - console.error("No clients connected. Aborting."); - process.exit(1); - } - - const endTime = Date.now() + (CONFIG.duration * 1000); - const intervalMs = 1000 / CONFIG.rate; // Delay between events to match rate - - console.log('Starting load generation...'); - - // Simple loop to maintain rate - let running = true; - const loop = async () => { - while (running && Date.now() < endTime) { - const startLoop = Date.now(); - - // Pick random client - const client = clients[Math.floor(Math.random() * clients.length)]; - - // 10% Video Post, 90% View Event - const isVideo = Math.random() < 0.1; - let event; - - const signStart = performance.now(); - if (isVideo) { - const content = { - title: `Load Test Video ${Date.now()}`, - description: "Load test content", - videoRootId: `root-${Date.now()}`, - version: 3 - }; - event = buildVideoPostEvent(client, content); - } else { - event = buildViewEvent(client, "some-video-id"); - } - const signEnd = performance.now(); - stats.recordSigningTime(signEnd - signStart); - - client.publish(event); - - const elapsed = Date.now() - startLoop; - const wait = Math.max(0, intervalMs - elapsed); - await delay(wait); - } + errorSample: metrics.errors.slice(0, 10) + }, + bottlenecks: [ + avgLatency > 1000 ? "High Average Latency (>1s)" : null, + avgSignTime > 10 ? "Cryptographic Bottleneck (Signing > 10ms)" : null, + metrics.failed > 0 ? "Failed Events Detected" : null + ].filter(Boolean), + remediation: "If latency is high, consider scaling the relay. If signing is slow, review crypto library or CPU resources." }; - await loop(); - running = false; + const dateStr = new Date().toISOString().split('T')[0].replace(/-/g, ''); + const reportPath = path.join(ARTIFACTS_DIR, `load-report-${dateStr}.json`); + fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); - console.log('Load generation finished. Waiting for pending ACKs (5s)...'); - await delay(5000); + console.log(`Report saved to ${reportPath}`); + console.log(JSON.stringify(report.metrics, null, 2)); - clients.forEach(c => c.close()); - - const summary = stats.getSummary(); - console.log('--- Load Test Report ---'); - console.log(JSON.stringify(summary, null, 2)); - - await mkdir(CONFIG.outputDir, { recursive: true }); - // Use fixed date format YYYYMMDD for PR requirements - const dateStr = new Date().toISOString().slice(0, 10).replace(/-/g, ''); - const filename = path.join(CONFIG.outputDir, `load-report-${dateStr}.json`); - await writeFile(filename, JSON.stringify(summary, null, 2)); - console.log(`Report saved to ${filename}`); + if (relayProcess) { + console.log('Stopping local relay...'); + relayProcess.kill(); + } + process.exit(0); } -run().catch(console.error); +main().catch(err => { + console.error(err); + process.exit(1); +});