mirror of
https://github.com/PR0M3TH3AN/SeedPass.git
synced 2026-11-06 21:02:04 +00:00
update
This commit is contained in:
thePR0M3TH3AN committed
1 parent
3477e74df4
commit
b93b1db31b
608 files changed
+62504
No files matched your search
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env node
|
||||
import { runRelayHealthCheck } from '../../src/relay-health.mjs';
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
cadence: 'daily',
|
||||
timeoutMs: 6000,
|
||||
allRelaysDownMinutes: 10,
|
||||
minSuccessRate: 0.7,
|
||||
windowMinutes: 60,
|
||||
};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const value = argv[i];
|
||||
if (!value.startsWith('--') && !args.cadence) {
|
||||
args.cadence = value;
|
||||
continue;
|
||||
}
|
||||
if (value === '--cadence') {
|
||||
args.cadence = argv[i + 1] || args.cadence;
|
||||
i += 1;
|
||||
} else if (value === '--timeout-ms') {
|
||||
args.timeoutMs = Number.parseInt(argv[i + 1], 10) || args.timeoutMs;
|
||||
i += 1;
|
||||
} else if (value === '--all-relays-down-minutes') {
|
||||
args.allRelaysDownMinutes = Number.parseInt(argv[i + 1], 10) || args.allRelaysDownMinutes;
|
||||
i += 1;
|
||||
} else if (value === '--min-success-rate') {
|
||||
const parsed = Number.parseFloat(argv[i + 1]);
|
||||
args.minSuccessRate = Number.isFinite(parsed) ? parsed : args.minSuccessRate;
|
||||
i += 1;
|
||||
} else if (value === '--window-minutes') {
|
||||
args.windowMinutes = Number.parseInt(argv[i + 1], 10) || args.windowMinutes;
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.cadence !== 'daily' && args.cadence !== 'weekly') {
|
||||
console.error('Usage: node scripts/agent/check-relay-health.mjs --cadence <daily|weekly>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const result = await runRelayHealthCheck(args);
|
||||
if (!result.ok) {
|
||||
result.failureCategory = 'all relays unhealthy';
|
||||
}
|
||||
console.log(JSON.stringify(result));
|
||||
process.exit(result.ok ? 0 : 2);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
const message = error instanceof Error ? error.message : String(error ?? 'unknown error');
|
||||
console.log(JSON.stringify({ ok: false, failureCategory: 'health check failed', error: message }));
|
||||
process.exit(2);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { chromium } from '@playwright/test';
|
||||
import fs from 'fs';
|
||||
|
||||
const LOG_FILE = 'artifacts/debug_frontend.log';
|
||||
|
||||
(async () => {
|
||||
const browser = await chromium.launch();
|
||||
const page = await browser.newPage();
|
||||
const errors = [];
|
||||
|
||||
const logStream = fs.createWriteStream(LOG_FILE, { flags: 'w' });
|
||||
const log = (msg) => {
|
||||
console.log(msg);
|
||||
logStream.write(msg + '\n');
|
||||
};
|
||||
|
||||
log(`[${new Date().toISOString()}] Starting debug session...`);
|
||||
|
||||
page.on('console', msg => {
|
||||
const text = msg.text();
|
||||
const type = msg.type();
|
||||
const output = `[console:${type}] ${text}`;
|
||||
|
||||
if (type === 'error') {
|
||||
log(output);
|
||||
errors.push(output);
|
||||
} else {
|
||||
// Log non-errors only to file to reduce noise, or log everything?
|
||||
// For debugging, everything is useful.
|
||||
log(output);
|
||||
}
|
||||
});
|
||||
|
||||
page.on('pageerror', err => {
|
||||
const output = `[pageerror] ${err.message}\n${err.stack}`;
|
||||
log(output);
|
||||
errors.push(output);
|
||||
});
|
||||
|
||||
page.on('requestfailed', request => {
|
||||
const failure = request.failure();
|
||||
const errorText = failure ? failure.errorText : 'Unknown error';
|
||||
const output = `[requestfailed] ${request.url()} - ${errorText}`;
|
||||
log(output);
|
||||
errors.push(output);
|
||||
});
|
||||
|
||||
try {
|
||||
log(`Navigating to http://localhost:4173/dashboard/...`);
|
||||
await page.goto('http://localhost:4173/dashboard/', { waitUntil: 'networkidle', timeout: 10000 });
|
||||
log(`Navigation complete.`);
|
||||
// Give it a bit more time for any delayed errors
|
||||
await page.waitForTimeout(2000);
|
||||
} catch (e) {
|
||||
log(`[script:error] Navigation failed or timeout: ${e.message}`);
|
||||
errors.push(`Navigation failed: ${e.message}`);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
// Don't close stream immediately as logs might still be flushing? No, synchronous write.
|
||||
await new Promise(resolve => logStream.end(resolve));
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.error(`\nFound ${errors.length} errors.`);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log(`\nNo errors found.`);
|
||||
process.exit(0);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,49 @@
|
||||
import { chromium } from '@playwright/test';
|
||||
|
||||
(async () => {
|
||||
const browser = await chromium.launch();
|
||||
const page = await browser.newPage();
|
||||
|
||||
let errorCount = 0;
|
||||
|
||||
page.on('console', msg => {
|
||||
const type = msg.type();
|
||||
if (type === 'error' || type === 'warning') {
|
||||
console.log(`CONSOLE [${type.toUpperCase()}]: ${msg.text()}`);
|
||||
if (type === 'error') errorCount++;
|
||||
}
|
||||
});
|
||||
|
||||
page.on('pageerror', exception => {
|
||||
console.log(`PAGE ERROR: ${exception}`);
|
||||
errorCount++;
|
||||
});
|
||||
|
||||
page.on('requestfailed', request => {
|
||||
// Check if failure() is not null before accessing errorText
|
||||
const failure = request.failure();
|
||||
const errorText = failure ? failure.errorText : 'Unknown error';
|
||||
console.log(`REQUEST FAILED: ${request.url()} - ${errorText}`);
|
||||
errorCount++;
|
||||
});
|
||||
|
||||
try {
|
||||
console.log('Navigating to http://localhost:4173/dashboard/ ...');
|
||||
await page.goto('http://localhost:4173/dashboard/');
|
||||
console.log('Page loaded. Waiting 5s for any async errors...');
|
||||
await page.waitForTimeout(5000);
|
||||
} catch (e) {
|
||||
console.error('Navigation failed:', e);
|
||||
errorCount++;
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
if (errorCount > 0) {
|
||||
console.log(`\nFound ${errorCount} errors.`);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log('\nNo errors found.');
|
||||
process.exit(0);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,140 @@
|
||||
import { parseLockEvent } from '../../src/lib.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
// Seeded random number generator (simple LCG)
|
||||
function mulberry32(a) {
|
||||
return function () {
|
||||
let t = (a += 0x6d2b79f5);
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
const SEED = process.env.SEED ? parseInt(process.env.SEED, 10) : Date.now();
|
||||
const ITERATIONS = 10000;
|
||||
const rand = mulberry32(SEED);
|
||||
|
||||
console.log(`Starting fuzz run. SEED=${SEED}, ITERATIONS=${ITERATIONS}`);
|
||||
|
||||
const REPRO_DIR = 'examples/reproducers/fuzz-lock-event-' + new Date().toISOString().slice(0, 10).replace(/-/g, '');
|
||||
if (!fs.existsSync(REPRO_DIR)) {
|
||||
fs.mkdirSync(REPRO_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
let failures = 0;
|
||||
|
||||
function randomString(length) {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()_+-=[]{}|;:,.<>?';
|
||||
let result = '';
|
||||
for (let i = 0; i < length; i++) {
|
||||
result += chars.charAt(Math.floor(rand() * chars.length));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function randomJson() {
|
||||
const types = ['object', 'array', 'string', 'number', 'boolean', 'null'];
|
||||
const type = types[Math.floor(rand() * types.length)];
|
||||
|
||||
if (type === 'object') {
|
||||
const obj = {};
|
||||
const keys = Math.floor(rand() * 5);
|
||||
for (let i = 0; i < keys; i++) {
|
||||
obj[randomString(5)] = randomJson();
|
||||
}
|
||||
return obj;
|
||||
} else if (type === 'array') {
|
||||
const arr = [];
|
||||
const len = Math.floor(rand() * 5);
|
||||
for (let i = 0; i < len; i++) {
|
||||
arr.push(randomJson());
|
||||
}
|
||||
return arr;
|
||||
} else if (type === 'string') {
|
||||
return randomString(10);
|
||||
} else if (type === 'number') {
|
||||
return rand() * 1000;
|
||||
} else if (type === 'boolean') {
|
||||
return rand() > 0.5;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function generateEvent() {
|
||||
const malformed = rand() > 0.8;
|
||||
const content = malformed ? randomString(100) : JSON.stringify(randomJson());
|
||||
|
||||
const tags = [];
|
||||
const numTags = Math.floor(rand() * 5);
|
||||
for (let i = 0; i < numTags; i++) {
|
||||
const tagLen = Math.floor(rand() * 3) + 1;
|
||||
const tag = [];
|
||||
for (let j = 0; j < tagLen; j++) {
|
||||
tag.push(randomString(10));
|
||||
}
|
||||
tags.push(tag);
|
||||
}
|
||||
|
||||
// Sometimes add specific tags
|
||||
if (rand() > 0.5) {
|
||||
tags.push(['d', randomString(20)]);
|
||||
}
|
||||
if (rand() > 0.5) {
|
||||
tags.push(['expiration', String(Math.floor(Date.now() / 1000) + 1000)]);
|
||||
}
|
||||
|
||||
return {
|
||||
id: randomString(64), // Mock ID
|
||||
pubkey: randomString(64), // Mock Pubkey
|
||||
created_at: Math.floor(Date.now() / 1000),
|
||||
tags: tags,
|
||||
content: content
|
||||
};
|
||||
}
|
||||
|
||||
for (let i = 0; i < ITERATIONS; i++) {
|
||||
const event = generateEvent();
|
||||
|
||||
try {
|
||||
parseLockEvent(event);
|
||||
} catch (err) {
|
||||
failures++;
|
||||
console.error(`Failure #${failures} at iteration ${i}: ${err.message}`);
|
||||
|
||||
const reproFile = path.join(REPRO_DIR, `repro-${i}.json`);
|
||||
fs.writeFileSync(reproFile, JSON.stringify(event, null, 2));
|
||||
|
||||
const reproScript = path.join(REPRO_DIR, `run-repro-${i}.mjs`);
|
||||
const scriptContent = `
|
||||
import { parseLockEvent } from '../../../src/lib.mjs';
|
||||
import fs from 'node:fs';
|
||||
|
||||
const event = JSON.parse(fs.readFileSync('repro-${i}.json', 'utf8'));
|
||||
console.log('Running repro case...');
|
||||
try {
|
||||
parseLockEvent(event);
|
||||
console.log('Success (no crash)');
|
||||
} catch (err) {
|
||||
console.error('Crash reproduced:', err);
|
||||
process.exit(1);
|
||||
}
|
||||
`;
|
||||
fs.writeFileSync(reproScript, scriptContent);
|
||||
}
|
||||
}
|
||||
|
||||
if (failures === 0) {
|
||||
console.log('Fuzzing completed successfully. No failures found.');
|
||||
// Clean up empty directory if no failures
|
||||
try {
|
||||
fs.rmdirSync(REPRO_DIR);
|
||||
} catch {
|
||||
// Ignore error if directory is not empty or missing
|
||||
}
|
||||
} else {
|
||||
console.log(`Fuzzing completed with ${failures} failures.`);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,640 @@
|
||||
import WebSocket from 'ws';
|
||||
import { Relay, useWebSocketImplementation } from 'nostr-tools/relay';
|
||||
import { finalizeEvent, generateSecretKey } from 'nostr-tools/pure';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { KIND_APP_DATA, DEFAULT_RELAYS } from '../../src/constants.mjs';
|
||||
import { ensureDir } from '../../src/utils.mjs';
|
||||
|
||||
useWebSocketImplementation(WebSocket);
|
||||
|
||||
const KIND_VIEW_EVENT = 1;
|
||||
const KNOWN_PUBLIC_RELAY_HOSTS = new Set(DEFAULT_RELAYS.map((relay) => new URL(relay).hostname));
|
||||
const RELAY_SAFETY_ACK_VALUE = 'LOAD_TEST_INFRA_CONFIRMED';
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const raw = argv[i];
|
||||
if (!raw.startsWith('--')) continue;
|
||||
const token = raw.slice(2);
|
||||
if (!token) continue;
|
||||
const eqIndex = token.indexOf('=');
|
||||
if (eqIndex >= 0) {
|
||||
args[token.slice(0, eqIndex)] = token.slice(eqIndex + 1);
|
||||
continue;
|
||||
}
|
||||
const next = argv[i + 1];
|
||||
if (next && !next.startsWith('--')) {
|
||||
args[token] = next;
|
||||
i += 1;
|
||||
} else {
|
||||
args[token] = '1';
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function getConfigValue(args, key, envKey, fallback) {
|
||||
if (args[key] !== undefined) return args[key];
|
||||
if (process.env[envKey] !== undefined) return process.env[envKey];
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function parseIntStrict(value, name) {
|
||||
if (!/^-?\d+$/.test(String(value))) {
|
||||
throw new Error(`${name} must be an integer (got "${value}")`);
|
||||
}
|
||||
return Number.parseInt(value, 10);
|
||||
}
|
||||
|
||||
function parseFloatStrict(value, name) {
|
||||
if (!/^[-+]?\d+(\.\d+)?$/.test(String(value))) {
|
||||
throw new Error(`${name} must be a number (got "${value}")`);
|
||||
}
|
||||
return Number.parseFloat(value);
|
||||
}
|
||||
|
||||
function parseBoolean(value, fallback = false) {
|
||||
if (value === undefined || value === null || value === '') return fallback;
|
||||
const normalized = String(value).trim().toLowerCase();
|
||||
if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
|
||||
if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function assertRange(value, name, min, max) {
|
||||
if (value < min || value > max) {
|
||||
throw new Error(`${name} out of range (${min}..${max}): ${value}`);
|
||||
}
|
||||
}
|
||||
|
||||
function isPrivateIpv4(hostname) {
|
||||
const match = hostname.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
||||
if (!match) return false;
|
||||
const parts = match.slice(1).map((part) => Number.parseInt(part, 10));
|
||||
if (parts.some((part) => Number.isNaN(part) || part < 0 || part > 255)) return false;
|
||||
if (parts[0] === 10) return true;
|
||||
if (parts[0] === 127) return true;
|
||||
if (parts[0] === 192 && parts[1] === 168) return true;
|
||||
if (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) return true;
|
||||
if (parts[0] === 169 && parts[1] === 254) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function isLocalOrPrivateHost(hostname) {
|
||||
const lower = hostname.toLowerCase();
|
||||
return lower === 'localhost'
|
||||
|| lower === '::1'
|
||||
|| lower.endsWith('.local')
|
||||
|| isPrivateIpv4(lower);
|
||||
}
|
||||
|
||||
function classifyError(error) {
|
||||
const text = String(error?.message || error || '').toLowerCase();
|
||||
if (text.includes('timeout')) return 'timeout';
|
||||
if (text.includes('reject') || text.includes('invalid signature') || text.includes('validation')) return 'reject';
|
||||
if (text.includes('closed') || text.includes('disconnect') || text.includes('econnreset')) return 'disconnect';
|
||||
if (text.includes('refused') || text.includes('enotfound') || text.includes('etimedout')) return 'network';
|
||||
return 'publish_error';
|
||||
}
|
||||
|
||||
function xmur3(seed) {
|
||||
let h = 1779033703 ^ seed.length;
|
||||
for (let i = 0; i < seed.length; i += 1) {
|
||||
h = Math.imul(h ^ seed.charCodeAt(i), 3432918353);
|
||||
h = (h << 13) | (h >>> 19);
|
||||
}
|
||||
return () => {
|
||||
h = Math.imul(h ^ (h >>> 16), 2246822507);
|
||||
h = Math.imul(h ^ (h >>> 13), 3266489909);
|
||||
h ^= h >>> 16;
|
||||
return h >>> 0;
|
||||
};
|
||||
}
|
||||
|
||||
function mulberry32(seedInt) {
|
||||
let seed = seedInt >>> 0;
|
||||
return () => {
|
||||
seed += 0x6d2b79f5;
|
||||
let t = seed;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
function percentile(samples, p) {
|
||||
if (samples.length === 0) return 0;
|
||||
const sorted = [...samples].sort((a, b) => a - b);
|
||||
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
|
||||
return sorted[index];
|
||||
}
|
||||
|
||||
function summarizePercentiles(samples) {
|
||||
if (samples.length === 0) {
|
||||
return {
|
||||
count: 0,
|
||||
minMs: 0,
|
||||
avgMs: 0,
|
||||
p50Ms: 0,
|
||||
p90Ms: 0,
|
||||
p95Ms: 0,
|
||||
p99Ms: 0,
|
||||
maxMs: 0,
|
||||
};
|
||||
}
|
||||
const minMs = samples.reduce((min, value) => Math.min(min, value), Number.POSITIVE_INFINITY);
|
||||
const maxMs = samples.reduce((max, value) => Math.max(max, value), 0);
|
||||
const sumMs = samples.reduce((sum, value) => sum + value, 0);
|
||||
return {
|
||||
count: samples.length,
|
||||
minMs,
|
||||
avgMs: sumMs / samples.length,
|
||||
p50Ms: percentile(samples, 50),
|
||||
p90Ms: percentile(samples, 90),
|
||||
p95Ms: percentile(samples, 95),
|
||||
p99Ms: percentile(samples, 99),
|
||||
maxMs,
|
||||
};
|
||||
}
|
||||
|
||||
function withTimeout(promise, timeoutMs) {
|
||||
let timeoutHandle = null;
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise((_, reject) => {
|
||||
timeoutHandle = setTimeout(() => reject(new Error(`publish timeout after ${timeoutMs}ms`)), timeoutMs);
|
||||
}),
|
||||
]).finally(() => {
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle);
|
||||
});
|
||||
}
|
||||
|
||||
function buildViewEvent(secretKey, nowMs) {
|
||||
return finalizeEvent({
|
||||
kind: KIND_VIEW_EVENT,
|
||||
created_at: Math.floor(nowMs / 1000),
|
||||
tags: [
|
||||
['t', 'torch-load-test'],
|
||||
['t', 'view-event'],
|
||||
],
|
||||
content: JSON.stringify({
|
||||
type: 'view_event',
|
||||
source: 'load-test-harness',
|
||||
viewed_at: new Date(nowMs).toISOString(),
|
||||
}),
|
||||
}, secretKey);
|
||||
}
|
||||
|
||||
function buildMultipartMetadataEvent(secretKey, nowMs, rng, sequence, partsPerEvent) {
|
||||
const videoId = `video-${Math.floor(rng() * 1000000).toString(16)}-${sequence}`;
|
||||
const part = 1 + Math.floor(rng() * partsPerEvent);
|
||||
const content = {
|
||||
type: 'video_metadata_multipart',
|
||||
source: 'load-test-harness',
|
||||
schema_basis: 'nostr-app-data-kind-30078',
|
||||
video_id: videoId,
|
||||
part,
|
||||
total_parts: partsPerEvent,
|
||||
payload: {
|
||||
title: `Load test clip ${sequence}`,
|
||||
duration_sec: 30 + Math.floor(rng() * 600),
|
||||
codecs: ['h264', 'aac'],
|
||||
chunk_manifest: Array.from({ length: partsPerEvent }, (_, idx) => ({
|
||||
part: idx + 1,
|
||||
bytes: 500 + Math.floor(rng() * 2500),
|
||||
})),
|
||||
},
|
||||
};
|
||||
return finalizeEvent({
|
||||
kind: KIND_APP_DATA,
|
||||
created_at: Math.floor(nowMs / 1000),
|
||||
tags: [
|
||||
['t', 'torch-load-test'],
|
||||
['t', 'video-metadata-multipart'],
|
||||
['d', `${videoId}-${part}`],
|
||||
],
|
||||
content: JSON.stringify(content),
|
||||
}, secretKey);
|
||||
}
|
||||
|
||||
function summarizeRelay(relayUrl) {
|
||||
const parsed = new URL(relayUrl);
|
||||
return `${parsed.protocol}//${parsed.hostname}${parsed.port ? `:${parsed.port}` : ''}`;
|
||||
}
|
||||
|
||||
function redactRelayForReport(relayUrl) {
|
||||
const parsed = new URL(relayUrl);
|
||||
return {
|
||||
protocol: parsed.protocol,
|
||||
hostname: parsed.hostname,
|
||||
port: parsed.port || '(default)',
|
||||
};
|
||||
}
|
||||
|
||||
function buildRemediation(report) {
|
||||
const items = [];
|
||||
const { results } = report;
|
||||
if (results.backpressureDrops > 0) {
|
||||
items.push('Reduce RATE_EPS or increase MAX_INFLIGHT only after confirming relay capacity; backpressure drops indicate local pipeline saturation.');
|
||||
}
|
||||
if ((results.errorBreakdown.timeout || 0) > 0) {
|
||||
items.push('Investigate relay write acknowledgements and tune PUBLISH_TIMEOUT_MS; timeout errors were observed during publish acknowledgements.');
|
||||
}
|
||||
if ((results.errorBreakdown.disconnect || 0) > 0 || (results.errorBreakdown.network || 0) > 0) {
|
||||
items.push('Check relay network stability (TCP resets, DNS, proxy paths) before raising client count.');
|
||||
}
|
||||
if (results.latency.p99Ms > 500) {
|
||||
items.push('Profile relay-side event acceptance path; p99 publish latency exceeded 500ms.');
|
||||
}
|
||||
if (results.eventLoopLag.p95Ms > 50) {
|
||||
items.push('Reduce local scheduling pressure (lower RATE_EPS or CLIENTS) to keep event loop lag under 50ms.');
|
||||
}
|
||||
if (results.resource.maxRssMb > 512) {
|
||||
items.push('Cap MAX_CONNECTIONS or lower CLIENTS to reduce harness memory pressure (RSS exceeded 512MB).');
|
||||
}
|
||||
if (items.length === 0) {
|
||||
items.push('No significant bottlenecks observed in this run; increase load gradually and repeat to find saturation point.');
|
||||
}
|
||||
return items.slice(0, 10);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const relayUrlRaw = getConfigValue(args, 'relay-url', 'RELAY_URL', '');
|
||||
const relayUrl = String(relayUrlRaw || '').trim();
|
||||
if (!relayUrl) throw new Error('RELAY_URL (or --relay-url) is required.');
|
||||
|
||||
const parsedRelay = new URL(relayUrl);
|
||||
if (!['ws:', 'wss:'].includes(parsedRelay.protocol)) {
|
||||
throw new Error(`Unsupported relay protocol ${parsedRelay.protocol}. Use ws:// or wss://.`);
|
||||
}
|
||||
|
||||
const allowDedicated = parseBoolean(getConfigValue(args, 'allow-dedicated-test-relay', 'ALLOW_DEDICATED_TEST_RELAY', '0'), false);
|
||||
const relaySafetyAck = String(getConfigValue(args, 'relay-safety-ack', 'RELAY_SAFETY_ACK', '') || '').trim();
|
||||
const host = parsedRelay.hostname.toLowerCase();
|
||||
|
||||
if (KNOWN_PUBLIC_RELAY_HOSTS.has(host)) {
|
||||
throw new Error(`Refusing to target known public relay host: ${host}`);
|
||||
}
|
||||
|
||||
if (!isLocalOrPrivateHost(host)) {
|
||||
if (!allowDedicated || relaySafetyAck !== RELAY_SAFETY_ACK_VALUE) {
|
||||
throw new Error(
|
||||
`Refusing non-local relay host "${host}". Set ALLOW_DEDICATED_TEST_RELAY=1 and RELAY_SAFETY_ACK=${RELAY_SAFETY_ACK_VALUE} for dedicated test infrastructure.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const clients = parseIntStrict(getConfigValue(args, 'clients', 'CLIENTS', '1000'), 'CLIENTS');
|
||||
const durationSec = parseIntStrict(getConfigValue(args, 'duration-sec', 'DURATION_SEC', '600'), 'DURATION_SEC');
|
||||
const rateEps = parseFloatStrict(getConfigValue(args, 'rate-eps', 'RATE_EPS', '20'), 'RATE_EPS');
|
||||
const mix = parseFloatStrict(getConfigValue(args, 'mix', 'MIX', '0.2'), 'MIX');
|
||||
const seed = String(getConfigValue(args, 'seed', 'SEED', '') || '').trim();
|
||||
const dryRun = parseBoolean(getConfigValue(args, 'dry-run', 'DRY_RUN', '1'), true);
|
||||
const connectConcurrency = parseIntStrict(getConfigValue(args, 'connect-concurrency', 'CONNECT_CONCURRENCY', '25'), 'CONNECT_CONCURRENCY');
|
||||
const maxConnections = parseIntStrict(getConfigValue(args, 'max-connections', 'MAX_CONNECTIONS', String(Math.min(clients, 250))), 'MAX_CONNECTIONS');
|
||||
const maxInflight = parseIntStrict(getConfigValue(args, 'max-inflight', 'MAX_INFLIGHT', '300'), 'MAX_INFLIGHT');
|
||||
const publishTimeoutMs = parseIntStrict(getConfigValue(args, 'publish-timeout-ms', 'PUBLISH_TIMEOUT_MS', '15000'), 'PUBLISH_TIMEOUT_MS');
|
||||
const resourceSampleMs = parseIntStrict(getConfigValue(args, 'resource-sample-ms', 'RESOURCE_SAMPLE_MS', '5000'), 'RESOURCE_SAMPLE_MS');
|
||||
const eventLoopSampleMs = parseIntStrict(getConfigValue(args, 'event-loop-sample-ms', 'EVENT_LOOP_SAMPLE_MS', '200'), 'EVENT_LOOP_SAMPLE_MS');
|
||||
const metadataParts = parseIntStrict(getConfigValue(args, 'metadata-parts', 'METADATA_PARTS', '4'), 'METADATA_PARTS');
|
||||
|
||||
assertRange(clients, 'CLIENTS', 1, 100000);
|
||||
assertRange(durationSec, 'DURATION_SEC', 1, 86400);
|
||||
assertRange(rateEps, 'RATE_EPS', 0.1, 100000);
|
||||
assertRange(mix, 'MIX', 0, 1);
|
||||
assertRange(connectConcurrency, 'CONNECT_CONCURRENCY', 1, 1000);
|
||||
assertRange(maxConnections, 'MAX_CONNECTIONS', 1, clients);
|
||||
assertRange(maxInflight, 'MAX_INFLIGHT', 1, 100000);
|
||||
assertRange(publishTimeoutMs, 'PUBLISH_TIMEOUT_MS', 100, 120000);
|
||||
assertRange(resourceSampleMs, 'RESOURCE_SAMPLE_MS', 500, 120000);
|
||||
assertRange(eventLoopSampleMs, 'EVENT_LOOP_SAMPLE_MS', 20, 2000);
|
||||
assertRange(metadataParts, 'METADATA_PARTS', 2, 20);
|
||||
|
||||
const rng = seed ? mulberry32(xmur3(seed)()) : Math.random;
|
||||
const reportDir = path.resolve(process.cwd(), 'reports', 'load-test');
|
||||
ensureDir(reportDir);
|
||||
|
||||
const relaySummary = summarizeRelay(relayUrl);
|
||||
console.log('Load test configuration');
|
||||
console.log(` relay: ${relaySummary}`);
|
||||
console.log(` clients: ${clients}`);
|
||||
console.log(` durationSec: ${durationSec}`);
|
||||
console.log(` rateEps: ${rateEps}`);
|
||||
console.log(` mix(metadata): ${mix}`);
|
||||
console.log(` seed: ${seed || '(random)'}`);
|
||||
console.log(` dryRun: ${dryRun}`);
|
||||
console.log(` maxConnections: ${maxConnections}`);
|
||||
console.log(` connectConcurrency: ${connectConcurrency}`);
|
||||
console.log(` maxInflight: ${maxInflight}`);
|
||||
|
||||
if (dryRun) {
|
||||
console.log('DRY_RUN enabled: no relay connections or publish calls will be made.');
|
||||
}
|
||||
|
||||
const clientKeys = Array.from({ length: clients }, () => generateSecretKey());
|
||||
|
||||
const stats = {
|
||||
startedAtIso: new Date().toISOString(),
|
||||
startedAtMs: Date.now(),
|
||||
attempted: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
backpressureDrops: 0,
|
||||
inFlightPeak: 0,
|
||||
latenciesMs: [],
|
||||
errorBreakdown: {},
|
||||
stackSamples: {},
|
||||
throughputBySecond: {},
|
||||
resourceSnapshots: [],
|
||||
eventLoopLagMs: [],
|
||||
mode: dryRun ? 'dry-run' : 'live',
|
||||
};
|
||||
|
||||
const cpuStart = process.cpuUsage();
|
||||
let previousCpu = process.cpuUsage();
|
||||
let previousCpuWallMs = Date.now();
|
||||
|
||||
const eventLoopTimer = setInterval(() => {
|
||||
const expected = Date.now() + eventLoopSampleMs;
|
||||
setTimeout(() => {
|
||||
const lag = Math.max(0, Date.now() - expected);
|
||||
stats.eventLoopLagMs.push(lag);
|
||||
}, eventLoopSampleMs);
|
||||
}, eventLoopSampleMs);
|
||||
|
||||
const resourceTimer = setInterval(() => {
|
||||
const nowMs = Date.now();
|
||||
const mem = process.memoryUsage();
|
||||
const cpuDiff = process.cpuUsage(previousCpu);
|
||||
const wallMs = Math.max(1, nowMs - previousCpuWallMs);
|
||||
const cpuMs = (cpuDiff.user + cpuDiff.system) / 1000;
|
||||
const cpuPercent = Number(((cpuMs / wallMs) * 100).toFixed(2));
|
||||
previousCpu = process.cpuUsage();
|
||||
previousCpuWallMs = nowMs;
|
||||
stats.resourceSnapshots.push({
|
||||
tSec: Math.floor((nowMs - stats.startedAtMs) / 1000),
|
||||
rssMb: Number((mem.rss / (1024 * 1024)).toFixed(2)),
|
||||
heapUsedMb: Number((mem.heapUsed / (1024 * 1024)).toFixed(2)),
|
||||
heapTotalMb: Number((mem.heapTotal / (1024 * 1024)).toFixed(2)),
|
||||
cpuPercent,
|
||||
});
|
||||
}, resourceSampleMs);
|
||||
|
||||
const recordAttempt = () => {
|
||||
const sec = Math.floor((Date.now() - stats.startedAtMs) / 1000);
|
||||
const bucket = stats.throughputBySecond[sec] || { attempted: 0, succeeded: 0, failed: 0, backpressureDrops: 0 };
|
||||
bucket.attempted += 1;
|
||||
stats.throughputBySecond[sec] = bucket;
|
||||
};
|
||||
|
||||
const recordResult = (kind) => {
|
||||
const sec = Math.floor((Date.now() - stats.startedAtMs) / 1000);
|
||||
const bucket = stats.throughputBySecond[sec] || { attempted: 0, succeeded: 0, failed: 0, backpressureDrops: 0 };
|
||||
bucket[kind] += 1;
|
||||
stats.throughputBySecond[sec] = bucket;
|
||||
};
|
||||
|
||||
const relays = [];
|
||||
if (!dryRun) {
|
||||
let nextConnectionIndex = 0;
|
||||
const connectionErrors = [];
|
||||
const workers = Array.from({ length: Math.min(connectConcurrency, maxConnections) }, () => (async () => {
|
||||
while (nextConnectionIndex < maxConnections) {
|
||||
const current = nextConnectionIndex;
|
||||
nextConnectionIndex += 1;
|
||||
try {
|
||||
const relay = await Relay.connect(relayUrl);
|
||||
relays[current] = relay;
|
||||
} catch (error) {
|
||||
connectionErrors.push(String(error?.message || error));
|
||||
}
|
||||
}
|
||||
})());
|
||||
|
||||
await Promise.all(workers);
|
||||
if (!relays.length) {
|
||||
throw new Error(`Failed to establish any relay connections. errors=${connectionErrors.slice(0, 3).join(' | ') || '(none)'}`);
|
||||
}
|
||||
if (connectionErrors.length) {
|
||||
stats.errorBreakdown.connection = connectionErrors.length;
|
||||
stats.stackSamples.connection = connectionErrors.slice(0, 3);
|
||||
}
|
||||
console.log(`Established ${relays.length}/${maxConnections} relay connections.`);
|
||||
}
|
||||
|
||||
const publishContext = {
|
||||
inFlight: 0,
|
||||
tasks: new Set(),
|
||||
metadataSeq: 0,
|
||||
};
|
||||
|
||||
async function publishOne(clientIndex, isMetadata) {
|
||||
const nowMs = Date.now();
|
||||
const secretKey = clientKeys[clientIndex];
|
||||
const event = isMetadata
|
||||
? buildMultipartMetadataEvent(secretKey, nowMs, rng, publishContext.metadataSeq += 1, metadataParts)
|
||||
: buildViewEvent(secretKey, nowMs);
|
||||
|
||||
const started = performance.now();
|
||||
try {
|
||||
if (dryRun) {
|
||||
const syntheticLatency = 5 + Math.floor(rng() * 20);
|
||||
await sleep(syntheticLatency);
|
||||
} else {
|
||||
const relay = relays[clientIndex % relays.length];
|
||||
await withTimeout(relay.publish(event), publishTimeoutMs);
|
||||
}
|
||||
const latencyMs = Number((performance.now() - started).toFixed(3));
|
||||
stats.succeeded += 1;
|
||||
stats.latenciesMs.push(latencyMs);
|
||||
recordResult('succeeded');
|
||||
} catch (error) {
|
||||
stats.failed += 1;
|
||||
const category = classifyError(error);
|
||||
stats.errorBreakdown[category] = (stats.errorBreakdown[category] || 0) + 1;
|
||||
const stack = String(error?.stack || error?.message || error);
|
||||
const trimmed = stack.split('\n').slice(0, 3).join('\n');
|
||||
if (!stats.stackSamples[category]) stats.stackSamples[category] = [];
|
||||
if (stats.stackSamples[category].length < 3) {
|
||||
stats.stackSamples[category].push(trimmed);
|
||||
}
|
||||
recordResult('failed');
|
||||
}
|
||||
}
|
||||
|
||||
const durationMs = durationSec * 1000;
|
||||
const endAt = stats.startedAtMs + durationMs;
|
||||
const paceMs = 1000 / rateEps;
|
||||
let nextTick = Date.now();
|
||||
|
||||
while (Date.now() < endAt) {
|
||||
if (Date.now() < nextTick) {
|
||||
await sleep(Math.max(0, nextTick - Date.now()));
|
||||
continue;
|
||||
}
|
||||
nextTick += paceMs;
|
||||
|
||||
stats.attempted += 1;
|
||||
recordAttempt();
|
||||
if (publishContext.inFlight >= maxInflight) {
|
||||
stats.backpressureDrops += 1;
|
||||
stats.failed += 1;
|
||||
stats.errorBreakdown.backpressure = (stats.errorBreakdown.backpressure || 0) + 1;
|
||||
recordResult('failed');
|
||||
recordResult('backpressureDrops');
|
||||
continue;
|
||||
}
|
||||
|
||||
const clientIndex = Math.floor(rng() * clients);
|
||||
const isMetadata = rng() < mix;
|
||||
publishContext.inFlight += 1;
|
||||
stats.inFlightPeak = Math.max(stats.inFlightPeak, publishContext.inFlight);
|
||||
const task = publishOne(clientIndex, isMetadata).finally(() => {
|
||||
publishContext.inFlight -= 1;
|
||||
publishContext.tasks.delete(task);
|
||||
});
|
||||
publishContext.tasks.add(task);
|
||||
}
|
||||
|
||||
await Promise.allSettled([...publishContext.tasks]);
|
||||
|
||||
clearInterval(eventLoopTimer);
|
||||
clearInterval(resourceTimer);
|
||||
for (const relay of relays) relay.close();
|
||||
|
||||
const completedAtMs = Date.now();
|
||||
const completedAtIso = new Date(completedAtMs).toISOString();
|
||||
const cpuTotal = process.cpuUsage(cpuStart);
|
||||
const runDurationSec = Math.max(0.001, (completedAtMs - stats.startedAtMs) / 1000);
|
||||
const latency = summarizePercentiles(stats.latenciesMs);
|
||||
const eventLoopLag = summarizePercentiles(stats.eventLoopLagMs);
|
||||
const maxRssMb = stats.resourceSnapshots.reduce((max, item) => Math.max(max, item.rssMb), 0);
|
||||
const maxHeapMb = stats.resourceSnapshots.reduce((max, item) => Math.max(max, item.heapUsedMb), 0);
|
||||
const avgCpuPercent = stats.resourceSnapshots.length
|
||||
? stats.resourceSnapshots.reduce((sum, item) => sum + item.cpuPercent, 0) / stats.resourceSnapshots.length
|
||||
: 0;
|
||||
|
||||
const throughputSeries = Object.entries(stats.throughputBySecond)
|
||||
.map(([second, values]) => ({ second: Number(second), ...values }))
|
||||
.sort((a, b) => a.second - b.second);
|
||||
|
||||
const report = {
|
||||
generatedAt: completedAtIso,
|
||||
config: {
|
||||
relay: redactRelayForReport(relayUrl),
|
||||
relaySummary,
|
||||
dryRun,
|
||||
clients,
|
||||
durationSec,
|
||||
rateEps,
|
||||
mixMetadata: mix,
|
||||
seed: seed || null,
|
||||
connectConcurrency,
|
||||
maxConnections,
|
||||
maxInflight,
|
||||
publishTimeoutMs,
|
||||
metadataParts,
|
||||
},
|
||||
results: {
|
||||
attempted: stats.attempted,
|
||||
succeeded: stats.succeeded,
|
||||
failed: stats.failed,
|
||||
successRate: Number((stats.succeeded / Math.max(1, stats.attempted)).toFixed(4)),
|
||||
throughputEps: Number((stats.succeeded / runDurationSec).toFixed(3)),
|
||||
backpressureDrops: stats.backpressureDrops,
|
||||
inFlightPeak: stats.inFlightPeak,
|
||||
latency,
|
||||
eventLoopLag: {
|
||||
avgMs: Number(eventLoopLag.avgMs.toFixed(3)),
|
||||
p95Ms: Number(eventLoopLag.p95Ms.toFixed(3)),
|
||||
},
|
||||
errorBreakdown: stats.errorBreakdown,
|
||||
topStackTraces: stats.stackSamples,
|
||||
resource: {
|
||||
samples: stats.resourceSnapshots,
|
||||
sampleCount: stats.resourceSnapshots.length,
|
||||
maxRssMb,
|
||||
maxHeapUsedMb: maxHeapMb,
|
||||
avgCpuPercent: Number(avgCpuPercent.toFixed(2)),
|
||||
cpuUserMs: Number((cpuTotal.user / 1000).toFixed(3)),
|
||||
cpuSystemMs: Number((cpuTotal.system / 1000).toFixed(3)),
|
||||
},
|
||||
throughputSeries,
|
||||
hotFunctions: 'not measured',
|
||||
startedAtIso: stats.startedAtIso,
|
||||
completedAtIso,
|
||||
runDurationSec: Number(runDurationSec.toFixed(3)),
|
||||
},
|
||||
};
|
||||
|
||||
report.remediation = buildRemediation(report);
|
||||
|
||||
const dateStr = new Date().toISOString().slice(0, 10);
|
||||
const jsonPath = path.join(reportDir, `load-report-${dateStr}.json`);
|
||||
const markdownPath = path.join(reportDir, `load-test-report-${dateStr}.md`);
|
||||
fs.writeFileSync(jsonPath, JSON.stringify(report, null, 2));
|
||||
|
||||
const markdown = [
|
||||
`# Load Test Report - ${dateStr}`,
|
||||
'',
|
||||
'## Safety',
|
||||
'- Does not target public relays.',
|
||||
'- Requires explicit `RELAY_URL`.',
|
||||
`- Relay target: ${relaySummary}`,
|
||||
`- Mode: ${dryRun ? 'dry-run (no network sends)' : 'live publish test'}`,
|
||||
'',
|
||||
'## Configuration',
|
||||
`- Clients: ${clients}`,
|
||||
`- Duration: ${durationSec}s`,
|
||||
`- Rate: ${rateEps} events/sec`,
|
||||
`- Metadata mix: ${mix}`,
|
||||
`- Max connections: ${maxConnections}`,
|
||||
`- Max in-flight publishes: ${maxInflight}`,
|
||||
`- Seed: ${seed || '(random)'}`,
|
||||
'',
|
||||
'## Summary',
|
||||
`- Attempted: ${report.results.attempted}`,
|
||||
`- Succeeded: ${report.results.succeeded}`,
|
||||
`- Failed: ${report.results.failed}`,
|
||||
`- Throughput: ${report.results.throughputEps} events/sec`,
|
||||
`- Success rate: ${(report.results.successRate * 100).toFixed(2)}%`,
|
||||
'',
|
||||
'## Latency',
|
||||
`- p50: ${report.results.latency.p50Ms.toFixed(3)} ms`,
|
||||
`- p90: ${report.results.latency.p90Ms.toFixed(3)} ms`,
|
||||
`- p95: ${report.results.latency.p95Ms.toFixed(3)} ms`,
|
||||
`- p99: ${report.results.latency.p99Ms.toFixed(3)} ms`,
|
||||
'',
|
||||
'## Event Loop Lag',
|
||||
`- Avg: ${report.results.eventLoopLag.avgMs} ms`,
|
||||
`- p95: ${report.results.eventLoopLag.p95Ms} ms`,
|
||||
'',
|
||||
'## Error Taxonomy',
|
||||
`- ${Object.entries(report.results.errorBreakdown).map(([key, value]) => `${key}: ${value}`).join(', ') || 'none'}`,
|
||||
'',
|
||||
'## Hot Functions',
|
||||
'- not measured',
|
||||
'',
|
||||
'## Prioritized Remediation',
|
||||
...report.remediation.map((item, index) => `${index + 1}. ${item}`),
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
fs.writeFileSync(markdownPath, markdown);
|
||||
|
||||
console.log(`JSON report written: ${jsonPath}`);
|
||||
console.log(`Markdown report written: ${markdownPath}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`Load test harness failed: ${error?.stack || error}`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT_DIR = path.resolve(__dirname, '../../');
|
||||
const DAILY_DIR = path.join(ROOT_DIR, 'src/prompts/daily');
|
||||
const WEEKLY_DIR = path.join(ROOT_DIR, 'src/prompts/weekly');
|
||||
const REPORT_DIR = path.join(ROOT_DIR, 'reports/prompt-safety');
|
||||
|
||||
if (!fs.existsSync(REPORT_DIR)) {
|
||||
fs.mkdirSync(REPORT_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
const SAFETY_KEYWORDS = [
|
||||
'FAILURE MODES',
|
||||
'EXIT CRITERIA',
|
||||
'SKIP',
|
||||
'NO-OP',
|
||||
'STOP IF',
|
||||
'DO NOTHING',
|
||||
'NO ACTION'
|
||||
];
|
||||
|
||||
const FORCEFUL_KEYWORDS = [
|
||||
'MUST',
|
||||
'ALWAYS',
|
||||
'FORCE',
|
||||
'REQUIRED'
|
||||
];
|
||||
|
||||
function analyzeFile(filepath) {
|
||||
const content = fs.readFileSync(filepath, 'utf8');
|
||||
const upperContent = content.toUpperCase();
|
||||
const filename = path.basename(filepath);
|
||||
|
||||
const hasSafetySection = SAFETY_KEYWORDS.some(k => upperContent.includes(k));
|
||||
|
||||
// Simple heuristic for forceful language without mitigation
|
||||
// This is a naive check and might have false positives
|
||||
let forcefulCount = 0;
|
||||
FORCEFUL_KEYWORDS.forEach(k => {
|
||||
const regex = new RegExp(`\\b${k}\\b`, 'g');
|
||||
const matches = (upperContent.match(regex) || []).length;
|
||||
forcefulCount += matches;
|
||||
});
|
||||
|
||||
return {
|
||||
filename,
|
||||
hasSafetySection,
|
||||
forcefulCount,
|
||||
status: hasSafetySection ? 'SAFE' : 'NEEDS_REVIEW'
|
||||
};
|
||||
}
|
||||
|
||||
function runAudit() {
|
||||
const files = [
|
||||
...fs.readdirSync(DAILY_DIR).map(f => path.join(DAILY_DIR, f)),
|
||||
...fs.readdirSync(WEEKLY_DIR).map(f => path.join(WEEKLY_DIR, f))
|
||||
].filter(f => f.endsWith('.md'));
|
||||
|
||||
const results = files.map(analyzeFile);
|
||||
|
||||
const safeCount = results.filter(r => r.status === 'SAFE').length;
|
||||
const needsReviewCount = results.filter(r => r.status === 'NEEDS_REVIEW').length;
|
||||
const totalCount = results.length;
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/:/g, '-').split('.')[0];
|
||||
const reportFile = path.join(REPORT_DIR, `audit-${timestamp}.md`);
|
||||
|
||||
let reportContent = `# Prompt Safety Audit Report - ${timestamp}\n\n`;
|
||||
reportContent += `**Total Prompts:** ${totalCount}\n`;
|
||||
reportContent += `**Safe:** ${safeCount}\n`;
|
||||
reportContent += `**Needs Review:** ${needsReviewCount}\n\n`;
|
||||
|
||||
reportContent += `## Findings\n\n`;
|
||||
reportContent += `| Filename | Status | Has Safety Section | Forceful Keywords (Approx) |\n`;
|
||||
reportContent += `|---|---|---|---|\n`;
|
||||
|
||||
results.forEach(r => {
|
||||
reportContent += `| ${r.filename} | ${r.status} | ${r.hasSafetySection ? 'Yes' : 'No'} | ${r.forcefulCount} |\n`;
|
||||
});
|
||||
|
||||
reportContent += `\n## Recommendations\n`;
|
||||
reportContent += `- Review prompts marked 'NEEDS_REVIEW'.\n`;
|
||||
reportContent += `- Ensure all prompts have a 'FAILURE MODES' or 'EXIT CRITERIA' section.\n`;
|
||||
|
||||
fs.writeFileSync(reportFile, reportContent);
|
||||
console.log(`Report written to ${reportFile}`);
|
||||
|
||||
// Memory Update
|
||||
let memoryContent = `Goal: Audit prompt safety.\n`;
|
||||
memoryContent += `Result: Audited ${totalCount} prompts. ${safeCount} safe, ${needsReviewCount} need review.\n`;
|
||||
if (needsReviewCount > 0) {
|
||||
memoryContent += `Flagged prompts: ${results.filter(r => r.status === 'NEEDS_REVIEW').map(r => r.filename).join(', ')}\n`;
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(ROOT_DIR, 'memory-update.md'), memoryContent);
|
||||
console.log('Memory update written to memory-update.md');
|
||||
}
|
||||
|
||||
runAudit();
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
|
||||
const REQUIRED_ENV = ['SCHEDULER_PROMPT_PATH', 'SCHEDULER_AGENT', 'SCHEDULER_CADENCE'];
|
||||
|
||||
function fail(message, code = 1) {
|
||||
console.error(`[scheduler] ${message}`);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
function resolveRunner(promptMarkdown) {
|
||||
const customRunner = process.env.SCHEDULER_AGENT_RUNNER_COMMAND?.trim();
|
||||
if (customRunner) {
|
||||
return { command: 'bash', args: ['-lc', customRunner] };
|
||||
}
|
||||
|
||||
const platform = (process.env.AGENT_PLATFORM || 'codex').trim().toLowerCase();
|
||||
if (platform === 'codex') {
|
||||
return { command: 'codex', args: ['exec', promptMarkdown] };
|
||||
}
|
||||
|
||||
if (platform === 'claude') {
|
||||
return { command: 'claude', args: ['-p', promptMarkdown] };
|
||||
}
|
||||
|
||||
if (platform === 'linux') {
|
||||
return { command: 'echo', args: [`[scheduler] Platform 'linux' detected. simulated execution for prompt.`] };
|
||||
}
|
||||
|
||||
fail(`Unsupported AGENT_PLATFORM="${platform}". Set SCHEDULER_AGENT_RUNNER_COMMAND to override.`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
for (const key of REQUIRED_ENV) {
|
||||
if (!process.env[key] || !String(process.env[key]).trim()) {
|
||||
fail(`Missing required environment variable ${key}.`);
|
||||
}
|
||||
}
|
||||
|
||||
const promptPath = path.resolve(process.cwd(), process.env.SCHEDULER_PROMPT_PATH);
|
||||
let promptMarkdown;
|
||||
|
||||
try {
|
||||
promptMarkdown = await fs.readFile(promptPath, 'utf8');
|
||||
} catch {
|
||||
fail(`Unable to read scheduler prompt at ${promptPath}.`);
|
||||
}
|
||||
|
||||
if (!promptMarkdown.trim()) {
|
||||
fail(`Scheduler prompt at ${promptPath} is empty.`);
|
||||
}
|
||||
|
||||
const cadence = process.env.SCHEDULER_CADENCE;
|
||||
const memoriesPath = path.resolve(process.cwd(), `.scheduler-memory/latest/${cadence}/memories.md`);
|
||||
try {
|
||||
const memories = await fs.readFile(memoriesPath, 'utf8');
|
||||
if (memories && memories.trim()) {
|
||||
promptMarkdown += `\n\n# Retrieved Memories\n\n${memories.trim()}\n`;
|
||||
console.error(`[scheduler] Injected memories from ${memoriesPath}`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error(`[scheduler] Warning: Failed to read memories from ${memoriesPath}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const runner = resolveRunner(promptMarkdown);
|
||||
|
||||
// Compute a stable, per-run memory output file path so agents know where to write learnings.
|
||||
// store.mjs reads this env var; if absent it falls back to the root memory-update.md placeholder.
|
||||
const agentName = process.env.SCHEDULER_AGENT || 'unknown-agent';
|
||||
const runTs = new Date().toISOString().replace(/:/g, '-').replace(/\.\d+Z$/, 'Z');
|
||||
const memoryFile = `memory-updates/${runTs}__${agentName}.md`;
|
||||
|
||||
const child = spawn(runner.command, runner.args, {
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
SCHEDULER_PROMPT_PATH: promptPath,
|
||||
SCHEDULER_PROMPT_MARKDOWN: promptMarkdown,
|
||||
SCHEDULER_MEMORY_FILE: memoryFile,
|
||||
},
|
||||
stdio: 'inherit',
|
||||
});
|
||||
|
||||
child.on('error', (error) => {
|
||||
if (error.code === 'ENOENT') {
|
||||
fail(`Runner command not found: ${runner.command}`, 127);
|
||||
}
|
||||
fail(`Runner execution failed: ${error.message}`);
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
process.exit(code ?? 1);
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,173 @@
|
||||
import {
|
||||
runCommand,
|
||||
parseJsonFromOutput,
|
||||
parseJsonEventsFromOutput,
|
||||
excerptText,
|
||||
formatDurationMs,
|
||||
getRunDateKey,
|
||||
sleep,
|
||||
} from './scheduler-utils.mjs';
|
||||
|
||||
const LOCK_INCIDENT_LINK = 'docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md';
|
||||
|
||||
export function classifyLockBackendError(outputText) {
|
||||
const text = String(outputText || '').toLowerCase();
|
||||
if (!text.trim()) return 'unknown_backend_error';
|
||||
|
||||
const errorCategoryMatch = text.match(/error_category=([a-z0-9_]+)/);
|
||||
if (errorCategoryMatch?.[1]) {
|
||||
return errorCategoryMatch[1];
|
||||
}
|
||||
|
||||
if ((text.includes('relay') || text.includes('query')) && text.includes('timeout')) {
|
||||
return 'relay_query_timeout';
|
||||
}
|
||||
|
||||
if (text.includes('publish failed to all relays') || text.includes('failed to publish to any relay')) {
|
||||
return 'relay_publish_quorum_failure';
|
||||
}
|
||||
|
||||
if (
|
||||
text.includes('connection refused')
|
||||
|| text.includes('econnrefused')
|
||||
|| text.includes('getaddrinfo')
|
||||
|| text.includes('enotfound')
|
||||
|| text.includes('eai_again')
|
||||
|| (text.includes('websocket') && text.includes('dns'))
|
||||
) {
|
||||
return 'websocket_connection_refused_or_dns';
|
||||
}
|
||||
|
||||
if (
|
||||
text.includes('invalid url')
|
||||
|| text.includes('malformed')
|
||||
|| text.includes('unsupported protocol')
|
||||
|| text.includes('must start with ws')
|
||||
|| text.includes('invalid relay')
|
||||
) {
|
||||
return 'malformed_relay_url_config';
|
||||
}
|
||||
|
||||
return 'unknown_backend_error';
|
||||
}
|
||||
|
||||
export function buildLockBackendRemediation({ cadence, retryWindowMs, maxDeferrals, incidentSignalId = null }) {
|
||||
const steps = [
|
||||
'Likely backend/relay connectivity issue during lock acquisition.',
|
||||
`Retry window: ${formatDurationMs(retryWindowMs)} (max deferrals: ${maxDeferrals}).`,
|
||||
`Retry command: npm run scheduler:${cadence}`,
|
||||
`Run health check: npm run lock:health -- --cadence ${cadence}`,
|
||||
`Review incident runbook: ${LOCK_INCIDENT_LINK}`,
|
||||
];
|
||||
if (incidentSignalId) {
|
||||
steps.push(`Incident signal: ${incidentSignalId}`);
|
||||
}
|
||||
return `Recommended auto-remediation: ${steps.join(' ')}`;
|
||||
}
|
||||
|
||||
export async function runLockHealthPreflight({ cadence, platform }) {
|
||||
const preflight = await runCommand('npm', ['run', 'lock:health', '--', '--cadence', cadence], {
|
||||
env: { AGENT_PLATFORM: platform },
|
||||
});
|
||||
const payload = parseJsonFromOutput(`${preflight.stdout}\n${preflight.stderr}`) || {};
|
||||
const relayList = Array.isArray(payload.relays)
|
||||
? payload.relays.filter((relay) => typeof relay === 'string' && relay.trim()).map((relay) => relay.trim())
|
||||
: [];
|
||||
const combinedOutput = `${preflight.stderr}\n${preflight.stdout}`;
|
||||
const failureCategory = payload.failureCategory || classifyLockBackendError(combinedOutput);
|
||||
|
||||
return {
|
||||
code: preflight.code,
|
||||
payload,
|
||||
relayList,
|
||||
failureCategory,
|
||||
stderrExcerpt: excerptText(preflight.stderr),
|
||||
stdoutExcerpt: excerptText(preflight.stdout),
|
||||
};
|
||||
}
|
||||
|
||||
export async function acquireLockWithRetry({ selectedAgent, cadence, platform, model, lockRetry, idempotencyKey }) {
|
||||
const lockCommandArgs = ['run', 'lock:lock', '--', '--agent', selectedAgent, '--cadence', cadence];
|
||||
if (model) {
|
||||
lockCommandArgs.push('--model', model);
|
||||
}
|
||||
const backoffScheduleMs = [];
|
||||
let attempts = 0;
|
||||
const correlationId = idempotencyKey || `${cadence}:${selectedAgent}:${getRunDateKey()}`;
|
||||
|
||||
while (true) {
|
||||
attempts += 1;
|
||||
const result = await runCommand('npm', lockCommandArgs, {
|
||||
env: {
|
||||
AGENT_PLATFORM: platform,
|
||||
...(model ? { AGENT_MODEL: model } : {}),
|
||||
...(idempotencyKey ? { SCHEDULER_LOCK_IDEMPOTENCY_KEY: idempotencyKey } : {}),
|
||||
SCHEDULER_LOCK_CORRELATION_ID: correlationId,
|
||||
SCHEDULER_LOCK_ATTEMPT_ID: String(attempts),
|
||||
},
|
||||
});
|
||||
|
||||
if (result.code !== 2) {
|
||||
return { result, attempts, backoffScheduleMs, correlationId };
|
||||
}
|
||||
|
||||
if (attempts > lockRetry.maxRetries) {
|
||||
return {
|
||||
result,
|
||||
attempts,
|
||||
backoffScheduleMs,
|
||||
correlationId,
|
||||
finalBackendCategory: classifyLockBackendError(`${result.stderr}\n${result.stdout}`),
|
||||
};
|
||||
}
|
||||
|
||||
const exponentialBase = lockRetry.backoffMs * (2 ** (attempts - 1));
|
||||
const jitter = lockRetry.jitterMs > 0
|
||||
? Math.floor(Math.random() * (lockRetry.jitterMs + 1))
|
||||
: 0;
|
||||
const delayMs = exponentialBase + jitter;
|
||||
backoffScheduleMs.push(delayMs);
|
||||
|
||||
console.log(JSON.stringify({
|
||||
event: 'scheduler.lock.retry',
|
||||
attempt: attempts,
|
||||
max_retries: lockRetry.maxRetries,
|
||||
delay_ms: delayMs,
|
||||
correlation_id: correlationId,
|
||||
selected_agent: selectedAgent,
|
||||
cadence,
|
||||
}));
|
||||
|
||||
if (delayMs > 0) {
|
||||
await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function summarizeLockFailureReasons(outputText) {
|
||||
const events = parseJsonEventsFromOutput(outputText);
|
||||
const quorumFailure = events.findLast((entry) => entry.event === 'lock_publish_quorum_failed');
|
||||
if (quorumFailure && quorumFailure.reasonDistribution && typeof quorumFailure.reasonDistribution === 'object') {
|
||||
return {
|
||||
reasonDistribution: quorumFailure.reasonDistribution,
|
||||
attemptId: quorumFailure.attemptId || null,
|
||||
correlationId: quorumFailure.correlationId || null,
|
||||
totalElapsedMs: quorumFailure.totalElapsedMs ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
const distribution = {};
|
||||
for (const entry of events) {
|
||||
if (entry.event !== 'lock_publish_failure') continue;
|
||||
const reason = typeof entry.reason === 'string' && entry.reason.trim()
|
||||
? entry.reason.trim()
|
||||
: 'unknown';
|
||||
distribution[reason] = (distribution[reason] || 0) + 1;
|
||||
}
|
||||
return {
|
||||
reasonDistribution: distribution,
|
||||
attemptId: null,
|
||||
correlationId: null,
|
||||
totalElapsedMs: null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
|
||||
const ALLOWED_ENV_KEYS = new Set([
|
||||
'PATH', 'Path', // Windows compatibility
|
||||
'HOME', 'USER', 'SHELL', 'TERM', 'LANG', 'LC_ALL', 'LC_CTYPE', 'TZ',
|
||||
'NODE_ENV', 'NODE_OPTIONS', 'NODE_PATH',
|
||||
'APPDATA', 'LOCALAPPDATA', 'PROGRAMDATA', 'SystemRoot', 'windir', 'ComSpec', 'PATHEXT',
|
||||
'TMPDIR', 'TEMP', 'TMP',
|
||||
'EDITOR', 'VISUAL',
|
||||
// CI variables
|
||||
'CI', 'GITHUB_ACTIONS', 'GITHUB_REF', 'GITHUB_HEAD_REF', 'GITHUB_BASE_REF', 'GITHUB_EVENT_NAME', 'GITHUB_SHA',
|
||||
]);
|
||||
|
||||
export function getSafeEnv() {
|
||||
const safeEnv = {};
|
||||
for (const key in process.env) {
|
||||
if (ALLOWED_ENV_KEYS.has(key) ||
|
||||
key.startsWith('npm_') ||
|
||||
key.startsWith('NOSTR_') ||
|
||||
key.startsWith('TORCH_') ||
|
||||
key.startsWith('SCHEDULER_') ||
|
||||
key.startsWith('AGENT_') ||
|
||||
key.startsWith('JULES_') ||
|
||||
key.startsWith('CODEX_') ||
|
||||
key.startsWith('CLAUDE_') ||
|
||||
key.startsWith('ANTHROPIC_') ||
|
||||
key.startsWith('GOOSE_')) {
|
||||
safeEnv[key] = process.env[key];
|
||||
}
|
||||
}
|
||||
return safeEnv;
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawns a child process with a sanitized environment by default.
|
||||
* @param {string} command
|
||||
* @param {string[]} args
|
||||
* @param {object} options
|
||||
* @param {object} [options.env] - Additional environment variables to merge.
|
||||
* @param {boolean} [options.inheritProcessEnv=false] - If true, use full process.env. If false (default), use sanitized env.
|
||||
*/
|
||||
export async function runCommand(command, args = [], options = {}) {
|
||||
const baseEnv = options.inheritProcessEnv ? process.env : getSafeEnv();
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd: process.cwd(),
|
||||
env: { ...baseEnv, ...(options.env || {}) },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
const OUTPUT_LIMIT = 20000; // 20KB
|
||||
let stdoutWritten = 0;
|
||||
let stderrWritten = 0;
|
||||
let stdoutTruncated = false;
|
||||
let stderrTruncated = false;
|
||||
|
||||
child.stdout.on('data', (chunk) => {
|
||||
const text = chunk.toString();
|
||||
stdout += text;
|
||||
|
||||
if (stdoutWritten < OUTPUT_LIMIT) {
|
||||
const remaining = OUTPUT_LIMIT - stdoutWritten;
|
||||
if (text.length <= remaining) {
|
||||
process.stdout.write(text);
|
||||
stdoutWritten += text.length;
|
||||
} else {
|
||||
process.stdout.write(text.slice(0, remaining));
|
||||
stdoutWritten += remaining;
|
||||
}
|
||||
} else if (!stdoutTruncated) {
|
||||
process.stdout.write('\n...[stdout truncated]...\n');
|
||||
stdoutTruncated = true;
|
||||
}
|
||||
});
|
||||
|
||||
child.stderr.on('data', (chunk) => {
|
||||
const text = chunk.toString();
|
||||
stderr += text;
|
||||
|
||||
if (stderrWritten < OUTPUT_LIMIT) {
|
||||
const remaining = OUTPUT_LIMIT - stderrWritten;
|
||||
if (text.length <= remaining) {
|
||||
process.stderr.write(text);
|
||||
stderrWritten += text.length;
|
||||
} else {
|
||||
process.stderr.write(text.slice(0, remaining));
|
||||
stderrWritten += remaining;
|
||||
}
|
||||
} else if (!stderrTruncated) {
|
||||
process.stderr.write('\n...[stderr truncated]...\n');
|
||||
stderrTruncated = true;
|
||||
}
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
resolve({ code: code ?? 1, stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function parseJsonFromOutput(text) {
|
||||
const trimmed = String(text || '').trim();
|
||||
if (!trimmed) return null;
|
||||
const lines = trimmed.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
||||
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
||||
const line = lines[i];
|
||||
if (!line.startsWith('{') && !line.startsWith('[')) continue;
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
// Keep scanning from the end.
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseJsonEventsFromOutput(text) {
|
||||
const events = [];
|
||||
for (const line of String(text || '').split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed.startsWith('{')) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||||
events.push(parsed);
|
||||
}
|
||||
} catch {
|
||||
// Ignore non-JSON lines.
|
||||
}
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
export async function readJson(filePath, fallback) {
|
||||
try {
|
||||
const raw = await fs.readFile(filePath, 'utf8');
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeStringList(value, fallback = []) {
|
||||
if (!Array.isArray(value)) return fallback;
|
||||
return value.filter((item) => typeof item === 'string' && item.trim()).map((item) => item.trim());
|
||||
}
|
||||
|
||||
export function parseNonNegativeInt(value, fallback) {
|
||||
const parsed = Number.parseInt(String(value ?? ''), 10);
|
||||
if (Number.isFinite(parsed) && parsed >= 0) {
|
||||
return parsed;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function parseBooleanFlag(value, fallback = false) {
|
||||
if (value === undefined || value === null || value === '') return fallback;
|
||||
const normalized = String(value).trim().toLowerCase();
|
||||
if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
|
||||
if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function redactSensitive(text) {
|
||||
if (!text) return '';
|
||||
return String(text)
|
||||
.replace(/\b(BEARER\s+)[A-Za-z0-9._~+/=-]+/gi, '$1[REDACTED]')
|
||||
.replace(/\b(token|api[_-]?key|secret(?:[_-]?key)?|password|passwd|authorization)\s*[:=]\s*[^\s,;]+/gi, '$1=[REDACTED]')
|
||||
.replace(/\b(sk|pk|ghp|xoxb|xoxp)_[A-Za-z0-9_-]+\b/g, '[REDACTED]');
|
||||
}
|
||||
|
||||
export function excerptText(text, maxChars = 600) {
|
||||
const clean = redactSensitive(String(text || '').trim());
|
||||
if (!clean) return '';
|
||||
if (clean.length <= maxChars) return clean;
|
||||
return `${clean.slice(0, maxChars)}…`;
|
||||
}
|
||||
|
||||
export function formatDurationMs(ms) {
|
||||
if (!Number.isFinite(ms) || ms < 0) return 'unknown';
|
||||
const minutes = Math.round(ms / 60000);
|
||||
return `${minutes} minute(s)`;
|
||||
}
|
||||
|
||||
export function getRunDateKey(now = new Date()) {
|
||||
return now.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
export function toYamlScalar(value) {
|
||||
const str = String(value ?? '');
|
||||
return `'${str.replace(/'/g, "''")}'`;
|
||||
}
|
||||
|
||||
export function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Time-window guard helpers
|
||||
// Exported so they can be unit-tested independently of run-scheduler-cycle.mjs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Window sizes for the per-cadence duplicate-run guard. */
|
||||
export const CADENCE_WINDOW_MS = {
|
||||
daily: 24 * 60 * 60 * 1000, // 24 hours
|
||||
weekly: 7 * 24 * 60 * 60 * 1000, // 7 days
|
||||
};
|
||||
|
||||
/** Parse a date string to milliseconds, or null if invalid. */
|
||||
export function parseDateValue(value) {
|
||||
if (!value || typeof value !== 'string') return null;
|
||||
const ms = Date.parse(value);
|
||||
return Number.isFinite(ms) ? ms : null;
|
||||
}
|
||||
|
||||
/** Returns true when a filename matches the canonical scheduler log pattern. */
|
||||
export function isStrictSchedulerLogFilename(filename) {
|
||||
return /^(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z)__[^_]+__(completed|failed)\.md$/.test(String(filename));
|
||||
}
|
||||
|
||||
/** Extract agent name from a canonical log filename, or null. */
|
||||
export function parseAgentFromFilename(filename) {
|
||||
const match = String(filename).match(/^.+__([^_]+?)__(completed|failed)\.md$/);
|
||||
return match?.[1] || null;
|
||||
}
|
||||
|
||||
/** Extract the ISO timestamp embedded in a canonical log filename as ms, or null. */
|
||||
export function parseTimestampFromFilename(filename) {
|
||||
const match = String(filename).match(/^(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z)__[^_]+__(completed|failed)\.md$/);
|
||||
if (!match?.[1]) return null;
|
||||
const iso = match[1].replace(/T(\d{2})-(\d{2})-(\d{2})Z$/, 'T$1:$2:$3Z');
|
||||
return parseDateValue(iso);
|
||||
}
|
||||
|
||||
/** Read a single YAML frontmatter key from a markdown string, or null. */
|
||||
export function parseFrontmatterValue(markdown, key) {
|
||||
const lines = String(markdown).split(/\r?\n/);
|
||||
if (lines[0]?.trim() !== '---') return null;
|
||||
const keyPattern = new RegExp(`^${key}\\s*:\\s*(.+)$`, 'i');
|
||||
for (let i = 1; i < lines.length; i += 1) {
|
||||
const line = lines[i];
|
||||
if (line.trim() === '---') break;
|
||||
const match = line.match(keyPattern);
|
||||
if (match?.[1]) {
|
||||
return match[1].trim().replace(/^['"]|['"]$/g, '');
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Read `created_at` from YAML frontmatter. */
|
||||
export function parseFrontmatterCreatedAt(markdown) {
|
||||
return parseFrontmatterValue(markdown, 'created_at');
|
||||
}
|
||||
|
||||
/** Read `agent` from YAML frontmatter. */
|
||||
export function parseFrontmatterAgent(markdown) {
|
||||
return parseFrontmatterValue(markdown, 'agent');
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans task-log files for a cadence and returns a Set of agent names that
|
||||
* completed or failed within the cadence's time window (24 h daily, 7 d weekly).
|
||||
*
|
||||
* This is the local-filesystem companion to the Nostr relay lock check. It
|
||||
* catches the cross-midnight edge case where the relay's date-scoped lock has
|
||||
* rolled over but the agent ran less than a full window ago.
|
||||
*
|
||||
* Only `completed` and `failed` logs are considered. `deferred` logs are
|
||||
* intentionally skipped so the scheduler retries lock-backend failures.
|
||||
*
|
||||
* Non-fatal: if the log directory cannot be read, an empty Set is returned.
|
||||
*
|
||||
* @param {string} logDir - Absolute or relative path to task-logs/<cadence>/.
|
||||
* @param {'daily'|'weekly'} cadence
|
||||
* @param {number} [now] - Current epoch ms (injectable for tests).
|
||||
* @returns {Promise<Set<string>>} Agent names that ran within the window.
|
||||
*/
|
||||
export async function buildRecentlyRunExclusionSet(logDir, cadence, now = Date.now()) {
|
||||
const windowMs = CADENCE_WINDOW_MS[cadence] ?? CADENCE_WINDOW_MS.daily;
|
||||
const cutoff = now - windowMs;
|
||||
const recentlyRun = new Set();
|
||||
|
||||
let entries;
|
||||
try {
|
||||
await fs.mkdir(logDir, { recursive: true });
|
||||
entries = await fs.readdir(logDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return recentlyRun;
|
||||
}
|
||||
|
||||
const logFiles = entries
|
||||
.filter((e) => e.isFile() && isStrictSchedulerLogFilename(e.name))
|
||||
.map((e) => e.name);
|
||||
|
||||
await Promise.all(logFiles.map(async (filename) => {
|
||||
const agentName = parseAgentFromFilename(filename);
|
||||
if (!agentName) return;
|
||||
|
||||
// Prefer frontmatter created_at (more precise) over the filename timestamp.
|
||||
let effectiveMs = parseTimestampFromFilename(filename);
|
||||
try {
|
||||
const content = await fs.readFile(path.join(logDir, filename), 'utf8');
|
||||
const frontmatterMs = parseDateValue(parseFrontmatterCreatedAt(content));
|
||||
if (frontmatterMs !== null) effectiveMs = frontmatterMs;
|
||||
} catch {
|
||||
// Read failure: fall back to filename timestamp.
|
||||
}
|
||||
|
||||
if (effectiveMs !== null && effectiveMs >= cutoff) {
|
||||
recentlyRun.add(agentName);
|
||||
}
|
||||
}));
|
||||
|
||||
return recentlyRun;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { WebSocketServer } from 'ws';
|
||||
import http from 'http';
|
||||
|
||||
const server = http.createServer();
|
||||
const wss = new WebSocketServer({ server });
|
||||
|
||||
wss.on('connection', (ws) => {
|
||||
ws.on('message', (message) => {
|
||||
try {
|
||||
const data = JSON.parse(message);
|
||||
const type = data[0];
|
||||
if (type === 'REQ') {
|
||||
const subId = data[1];
|
||||
ws.send(JSON.stringify(['EOSE', subId]));
|
||||
} else if (type === 'EVENT') {
|
||||
const event = data[1];
|
||||
ws.send(JSON.stringify(['OK', event.id, true, '']));
|
||||
}
|
||||
} catch (_err) {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
console.log(`ws://127.0.0.1:${address.port}`);
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import { Relay, useWebSocketImplementation } from 'nostr-tools/relay';
|
||||
import { generateSecretKey, finalizeEvent, getPublicKey } from 'nostr-tools/pure';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
import WebSocket from 'ws';
|
||||
|
||||
useWebSocketImplementation(WebSocket);
|
||||
|
||||
// --- Configuration ---
|
||||
const ARGS = process.argv.slice(2);
|
||||
const getArg = (key) => {
|
||||
const idx = ARGS.findIndex(a => a.startsWith(`--${key}=`));
|
||||
if (idx !== -1) return ARGS[idx].split('=')[1];
|
||||
const flagIdx = ARGS.indexOf(`--${key}`);
|
||||
if (flagIdx !== -1 && ARGS[flagIdx + 1] && !ARGS[flagIdx + 1].startsWith('--')) return ARGS[flagIdx + 1];
|
||||
return null;
|
||||
};
|
||||
const RELAYS = (getArg('relays') || process.env.RELAY_URLS || '').split(',').filter(Boolean);
|
||||
const SERVE_CMD = getArg('serve') || 'npx';
|
||||
const OUT_DIR = getArg('out') || 'artifacts';
|
||||
|
||||
// --- Logging ---
|
||||
const LOG_FILE = path.join(OUT_DIR, `smoke-${new Date().toISOString().replace(/[:.]/g, '-')}.log`);
|
||||
const JSON_FILE = path.join(OUT_DIR, `smoke-${new Date().toISOString().split('T')[0]}.json`);
|
||||
|
||||
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
|
||||
function log(msg) {
|
||||
const ts = new Date().toISOString();
|
||||
const line = `[${ts}] ${msg}`;
|
||||
console.log(line);
|
||||
fs.appendFileSync(LOG_FILE, line + '\n');
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
async function waitForServer(url, timeoutMs = 10000) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (res.ok) return true;
|
||||
} catch (_e) {
|
||||
// ignore
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// --- Main ---
|
||||
async function main() {
|
||||
log('Starting Smoke Test Agent...');
|
||||
log(`Config: RELAYS=${RELAYS.join(',')}, SERVE=${SERVE_CMD}, OUT=${OUT_DIR}`);
|
||||
|
||||
if (RELAYS.length === 0) {
|
||||
log('ERROR: No relays specified. Use --relays or RELAY_URLS.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const results = {
|
||||
timestamp: new Date().toISOString(),
|
||||
steps: [],
|
||||
success: false
|
||||
};
|
||||
|
||||
let serverProcess = null;
|
||||
|
||||
try {
|
||||
// 1. Start Server
|
||||
if (SERVE_CMD !== 'none') {
|
||||
log('Step 1: Starting local server...');
|
||||
const port = 8000;
|
||||
if (SERVE_CMD === 'npx') {
|
||||
serverProcess = spawn('npx', ['serve', 'dist', '-p', String(port)], { stdio: 'ignore', shell: true });
|
||||
} else if (SERVE_CMD === 'python') {
|
||||
serverProcess = spawn('python3', ['-m', 'http.server', String(port), '-d', 'dist'], { stdio: 'ignore', shell: true });
|
||||
}
|
||||
|
||||
const serverUrl = `http://localhost:${port}`;
|
||||
log(`Waiting for server at ${serverUrl}...`);
|
||||
const ready = await waitForServer(serverUrl);
|
||||
if (!ready) {
|
||||
throw new Error('Server failed to start within timeout');
|
||||
}
|
||||
log('Server is ready.');
|
||||
results.steps.push({ name: 'start-server', status: 'ok', details: `Started ${SERVE_CMD} on port ${port}` });
|
||||
} else {
|
||||
log('Step 1: Skipping server start (none selected).');
|
||||
results.steps.push({ name: 'start-server', status: 'skipped' });
|
||||
}
|
||||
|
||||
// 2. Connect to Relay
|
||||
log(`Step 2: Connecting to relay ${RELAYS[0]}...`);
|
||||
const relay = await Relay.connect(RELAYS[0]);
|
||||
log(`Connected to ${RELAYS[0]}`);
|
||||
results.steps.push({ name: 'connect-relay', status: 'ok', details: RELAYS[0] });
|
||||
|
||||
// 3. Generate Identity
|
||||
log('Step 3: Generating ephemeral identity...');
|
||||
const sk = generateSecretKey();
|
||||
const pk = getPublicKey(sk);
|
||||
log(`Pubkey: ${pk}`);
|
||||
results.steps.push({ name: 'generate-identity', status: 'ok', details: pk });
|
||||
|
||||
// 4. Publish Event
|
||||
log('Step 4: Publishing smoke test event...');
|
||||
const eventTemplate = {
|
||||
kind: 1,
|
||||
created_at: Math.floor(Date.now() / 1000),
|
||||
tags: [['t', 'smoke-test']],
|
||||
content: `Smoke test run at ${new Date().toISOString()}`,
|
||||
};
|
||||
const event = finalizeEvent(eventTemplate, sk);
|
||||
await relay.publish(event);
|
||||
log(`Published event: ${event.id}`);
|
||||
results.steps.push({ name: 'publish-event', status: 'ok', details: event.id });
|
||||
|
||||
// 5. Verify Event
|
||||
log('Step 5: Verifying event read-back...');
|
||||
const sub = relay.subscribe([
|
||||
{ ids: [event.id] },
|
||||
], {
|
||||
onevent(evt) {
|
||||
log(`Received event: ${evt.id}`);
|
||||
if (evt.id === event.id) {
|
||||
log('Verification SUCCESS: Event ID matches.');
|
||||
results.steps.push({ name: 'verify-event', status: 'ok', details: 'Event retrieved and matched' });
|
||||
sub.close();
|
||||
}
|
||||
},
|
||||
oneose() {
|
||||
sub.close();
|
||||
}
|
||||
});
|
||||
|
||||
// Wait for event
|
||||
let attempts = 0;
|
||||
let verified = false;
|
||||
while (attempts < 10) {
|
||||
await sleep(1000);
|
||||
const found = results.steps.find(s => s.name === 'verify-event');
|
||||
if (found) {
|
||||
verified = true;
|
||||
break;
|
||||
}
|
||||
attempts++;
|
||||
}
|
||||
|
||||
if (!verified) {
|
||||
throw new Error('Verification failed: Event not received within timeout.');
|
||||
}
|
||||
|
||||
results.success = true;
|
||||
relay.close();
|
||||
|
||||
} catch (err) {
|
||||
log(`ERROR: ${err.message}`);
|
||||
results.error = err.message;
|
||||
results.success = false;
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
if (serverProcess) {
|
||||
log('Stopping server...');
|
||||
serverProcess.kill();
|
||||
}
|
||||
fs.writeFileSync(JSON_FILE, JSON.stringify(results, null, 2));
|
||||
log(`Artifacts written to ${OUT_DIR}`);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
const VALID_CADENCES = ['daily', 'weekly'];
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
sinceDays: 14,
|
||||
logRoot: path.resolve(process.cwd(), 'task-logs'),
|
||||
outDir: path.resolve(process.cwd(), 'artifacts', 'lock-reliability'),
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const token = argv[i];
|
||||
if (token === '--since-days') {
|
||||
options.sinceDays = Number.parseInt(argv[i + 1] || '14', 10);
|
||||
i += 1;
|
||||
} else if (token === '--log-root') {
|
||||
options.logRoot = path.resolve(process.cwd(), argv[i + 1] || 'task-logs');
|
||||
i += 1;
|
||||
} else if (token === '--out-dir') {
|
||||
options.outDir = path.resolve(process.cwd(), argv[i + 1] || path.join('artifacts', 'lock-reliability'));
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (!Number.isFinite(options.sinceDays) || options.sinceDays < 0) {
|
||||
throw new Error(`Invalid --since-days value: ${options.sinceDays}`);
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function parseFrontmatter(markdown) {
|
||||
const text = String(markdown || '');
|
||||
if (!text.startsWith('---\n')) return {};
|
||||
const end = text.indexOf('\n---', 4);
|
||||
if (end === -1) return {};
|
||||
const frontmatter = text.slice(4, end).split(/\r?\n/);
|
||||
const result = {};
|
||||
for (const line of frontmatter) {
|
||||
const match = line.match(/^([a-zA-Z0-9_]+):\s*(.+)\s*$/);
|
||||
if (!match) continue;
|
||||
const [, key, rawValue] = match;
|
||||
result[key] = rawValue.replace(/^'/, '').replace(/'$/, '').replace(/''/g, "'");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function inc(map, key) {
|
||||
map.set(key, (map.get(key) || 0) + 1);
|
||||
}
|
||||
|
||||
function collectRelayEndpoints({ frontmatter, body }) {
|
||||
const relays = new Set();
|
||||
const relayList = frontmatter.relay_list || '';
|
||||
relayList.split(',').map((v) => v.trim()).filter(Boolean).forEach((relay) => relays.add(relay));
|
||||
|
||||
const allText = `${body}\n${Object.values(frontmatter).join('\n')}`;
|
||||
const urlMatches = allText.match(/wss?:\/\/[^\s'`),]+/g) || [];
|
||||
for (const url of urlMatches) {
|
||||
relays.add(url.trim());
|
||||
}
|
||||
|
||||
if (!relays.size) relays.add('unknown');
|
||||
return [...relays];
|
||||
}
|
||||
|
||||
function getBackendCategory({ frontmatter, body }) {
|
||||
if (frontmatter.backend_category) return frontmatter.backend_category;
|
||||
if (frontmatter.preflight_failure_category) return `preflight:${frontmatter.preflight_failure_category}`;
|
||||
|
||||
const reasonMatch = body.match(/- reason:\s*([^\n]+)/i) || body.match(/reason:\s*([^\n]+)/i);
|
||||
const reason = reasonMatch?.[1]?.trim() || '';
|
||||
|
||||
if (/lock backend/i.test(reason)) return 'lock backend error (uncategorized)';
|
||||
if (/completion publish failed/i.test(reason)) return 'completion publish failed';
|
||||
if (/preflight/i.test(reason)) return 'preflight failed (uncategorized)';
|
||||
return null;
|
||||
}
|
||||
|
||||
function toSortedObject(counterMap) {
|
||||
return Object.fromEntries([...counterMap.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])));
|
||||
}
|
||||
|
||||
function asMarkdown(report) {
|
||||
const lines = [];
|
||||
lines.push('# Lock Reliability Summary');
|
||||
lines.push('');
|
||||
lines.push(`- Generated at: ${report.generatedAt}`);
|
||||
lines.push(`- Window: last ${report.sinceDays} day(s)`);
|
||||
lines.push(`- Logs scanned: ${report.logsScanned}`);
|
||||
lines.push(`- Runs considered: ${report.runsConsidered}`);
|
||||
lines.push('');
|
||||
|
||||
lines.push('## Totals');
|
||||
lines.push('');
|
||||
lines.push(`- completed: ${report.statusCounts.completed || 0}`);
|
||||
lines.push(`- failed: ${report.statusCounts.failed || 0}`);
|
||||
lines.push('');
|
||||
|
||||
const sections = [
|
||||
['By platform', report.byPlatform],
|
||||
['By cadence', report.byCadence],
|
||||
['By backend error category', report.byBackendCategory],
|
||||
['By relay endpoint', report.byRelayEndpoint],
|
||||
];
|
||||
|
||||
for (const [title, obj] of sections) {
|
||||
lines.push(`## ${title}`);
|
||||
lines.push('');
|
||||
const entries = Object.entries(obj);
|
||||
if (!entries.length) {
|
||||
lines.push('- (none)');
|
||||
} else {
|
||||
for (const [key, value] of entries) {
|
||||
lines.push(`- ${key}: ${value}`);
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
if (report.recurringBackendErrors.length) {
|
||||
lines.push('## Recurring backend errors (count >= 2)');
|
||||
lines.push('');
|
||||
for (const entry of report.recurringBackendErrors) {
|
||||
lines.push(`- ${entry.category}: ${entry.count}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function readLogFile(filePath, cadence, cutoffMs) {
|
||||
const stat = await fs.stat(filePath);
|
||||
if (!stat.isFile() || stat.mtimeMs < cutoffMs) return null;
|
||||
|
||||
const basename = path.basename(filePath);
|
||||
const match = basename.match(/^.+__([^_]+?)__(completed|failed)\.md$/);
|
||||
if (!match) return null;
|
||||
|
||||
const body = await fs.readFile(filePath, 'utf8');
|
||||
const frontmatter = parseFrontmatter(body);
|
||||
const status = frontmatter.status || match[2];
|
||||
const platform = frontmatter.platform || 'unknown';
|
||||
const backendCategory = status === 'failed'
|
||||
? getBackendCategory({ frontmatter, body })
|
||||
: 'n/a';
|
||||
const relayEndpoints = collectRelayEndpoints({ frontmatter, body });
|
||||
|
||||
return {
|
||||
filePath,
|
||||
cadence: frontmatter.cadence || cadence,
|
||||
status,
|
||||
platform,
|
||||
backendCategory,
|
||||
relayEndpoints,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const now = Date.now();
|
||||
const cutoffMs = now - (options.sinceDays * 24 * 60 * 60 * 1000);
|
||||
|
||||
const byPlatform = new Map();
|
||||
const byCadence = new Map();
|
||||
const byBackendCategory = new Map();
|
||||
const byRelayEndpoint = new Map();
|
||||
const statusCounts = new Map();
|
||||
const records = [];
|
||||
let logsScanned = 0;
|
||||
|
||||
for (const cadence of VALID_CADENCES) {
|
||||
const cadenceDir = path.join(options.logRoot, cadence);
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(cadenceDir);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const promises = entries.map(async (entry) => {
|
||||
const fullPath = path.join(cadenceDir, entry);
|
||||
const record = await readLogFile(fullPath, cadence, cutoffMs);
|
||||
return record;
|
||||
});
|
||||
|
||||
const results = await Promise.all(promises);
|
||||
logsScanned += entries.length;
|
||||
|
||||
for (const record of results) {
|
||||
if (!record) continue;
|
||||
|
||||
records.push(record);
|
||||
inc(statusCounts, record.status);
|
||||
inc(byPlatform, record.platform);
|
||||
inc(byCadence, record.cadence);
|
||||
if (record.status === 'failed' && record.backendCategory) {
|
||||
inc(byBackendCategory, record.backendCategory);
|
||||
}
|
||||
for (const relay of record.relayEndpoints) {
|
||||
inc(byRelayEndpoint, relay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const recurringBackendErrors = [...byBackendCategory.entries()]
|
||||
.filter(([, count]) => count >= 2)
|
||||
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
||||
.map(([category, count]) => ({ category, count }));
|
||||
|
||||
const report = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
sinceDays: options.sinceDays,
|
||||
logRoot: options.logRoot,
|
||||
logsScanned,
|
||||
runsConsidered: records.length,
|
||||
statusCounts: toSortedObject(statusCounts),
|
||||
byPlatform: toSortedObject(byPlatform),
|
||||
byCadence: toSortedObject(byCadence),
|
||||
byBackendCategory: toSortedObject(byBackendCategory),
|
||||
byRelayEndpoint: toSortedObject(byRelayEndpoint),
|
||||
recurringBackendErrors,
|
||||
};
|
||||
|
||||
await fs.mkdir(options.outDir, { recursive: true });
|
||||
const markdown = asMarkdown(report);
|
||||
const markdownPath = path.join(options.outDir, 'lock-reliability-summary.md');
|
||||
const jsonPath = path.join(options.outDir, 'lock-reliability-summary.json');
|
||||
|
||||
await fs.writeFile(markdownPath, markdown, 'utf8');
|
||||
await fs.writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
markdown: path.relative(process.cwd(), markdownPath),
|
||||
json: path.relative(process.cwd(), jsonPath),
|
||||
recurringBackendErrors,
|
||||
}));
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error?.stack || error?.message || String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,374 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
const REQUIRED_ARTIFACTS = [
|
||||
{
|
||||
dir: 'src/context',
|
||||
expectedPrefix: 'CONTEXT_',
|
||||
structureCheck: (content) =>
|
||||
/\bgoal\b/i.test(content) && /\bscope\b/i.test(content) && /\bconstraints\b/i.test(content),
|
||||
structureHint: 'must include goal/scope/constraints',
|
||||
},
|
||||
{
|
||||
dir: 'src/todo',
|
||||
expectedPrefix: 'TODO_',
|
||||
structureCheck: (content) =>
|
||||
/\bpending(?:\s+tasks?)?\b[\s\S]{0,400}^\s*[-*]\s+/im.test(content)
|
||||
|| /\bcompleted(?:\s+tasks?)?\b[\s\S]{0,400}^\s*[-*]\s+/im.test(content),
|
||||
structureHint: 'must include at least one pending or completed item',
|
||||
},
|
||||
{
|
||||
dir: 'src/decisions',
|
||||
expectedPrefix: 'DECISIONS_',
|
||||
structureCheck: (content) => /\bdecision\b/i.test(content) && /\brationale\b/i.test(content),
|
||||
structureHint: 'must include decision + rationale',
|
||||
},
|
||||
{
|
||||
dir: 'src/test_logs',
|
||||
expectedPrefix: 'TEST_LOG_',
|
||||
structureCheck: (content) => {
|
||||
const hasCommand = /(^|\n)\s*[-*]?\s*\**command\**\s*:/im.test(content);
|
||||
const hasResult = /(^|\n)\s*[-*]?\s*\**result\**\s*:/im.test(content);
|
||||
return hasCommand && hasResult;
|
||||
},
|
||||
structureHint: 'must include command/result pairs',
|
||||
},
|
||||
];
|
||||
|
||||
const FAILURE_KEYWORD_RE = /\b(fail|failed|failure|error)\b/i;
|
||||
const STATUS_RESOLVED_RE = /\b(resolved|closed|fixed)\b/i;
|
||||
const IDENTIFIER_INLINE_PATTERN = String.raw`\[([A-Za-z0-9._-]{3,})\]`;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
since: null,
|
||||
session: null,
|
||||
checkFailureNotes: false,
|
||||
agent: null,
|
||||
cadence: null,
|
||||
promptPath: null,
|
||||
runStart: null,
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const value = argv[i];
|
||||
if (value === '--since') {
|
||||
args.since = argv[i + 1] || null;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === '--session') {
|
||||
args.session = argv[i + 1] || null;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === '--check-failure-notes') {
|
||||
args.checkFailureNotes = true;
|
||||
continue;
|
||||
}
|
||||
if (value === '--agent') {
|
||||
args.agent = argv[i + 1] || null;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === '--cadence') {
|
||||
args.cadence = argv[i + 1] || null;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === '--prompt-path') {
|
||||
args.promptPath = argv[i + 1] || null;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (value === '--run-start') {
|
||||
args.runStart = argv[i + 1] || null;
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function toMs(iso) {
|
||||
if (!iso) return null;
|
||||
const value = Date.parse(iso);
|
||||
return Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function isNotNeededNote(content) {
|
||||
return /\bnot needed\b/i.test(content);
|
||||
}
|
||||
|
||||
async function listRecentFiles(dirPath, sinceMs) {
|
||||
try {
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
const files = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
const filePath = path.join(dirPath, entry.name);
|
||||
const stat = await fs.stat(filePath);
|
||||
if (sinceMs != null && stat.mtimeMs < sinceMs) continue;
|
||||
files.push({ name: entry.name, filePath, mtimeMs: stat.mtimeMs });
|
||||
}
|
||||
return files.sort((a, b) => a.mtimeMs - b.mtimeMs);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function hasExplicitNotNeeded(files) {
|
||||
for (const file of files) {
|
||||
const content = await fs.readFile(file.filePath, 'utf8').catch(() => '');
|
||||
if (isNotNeededNote(content)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function getMatchingArtifacts(files, { expectedPrefix, session }) {
|
||||
if (session) {
|
||||
return files.filter((file) => file.name.includes(session));
|
||||
}
|
||||
return files.filter((file) => file.name.startsWith(expectedPrefix));
|
||||
}
|
||||
|
||||
async function hasValidArtifactStructure(files, artifact) {
|
||||
for (const file of files) {
|
||||
const content = await fs.readFile(file.filePath, 'utf8').catch(() => '');
|
||||
if (artifact.structureCheck(content)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function readMetadataValue(content, keys) {
|
||||
for (const key of keys) {
|
||||
const re = new RegExp(`^\\s*(?:[-*]\\s*)?\\**${key}\\**\\s*:\\s*(.+)$`, 'im');
|
||||
const match = content.match(re);
|
||||
if (match && match[1]) {
|
||||
return match[1].trim();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractIdentifiers(content) {
|
||||
const ids = new Set();
|
||||
const lines = content.split(/\r?\n/);
|
||||
for (const line of lines) {
|
||||
// Match "failure id" as a distinct word phrase (ignores "failure identifiers" or "failure IDs")
|
||||
if (/(?:issue|incident|failure)[-_\s]*id\b/i.test(line)) {
|
||||
const match = line.match(/([A-Za-z0-9._-]{3,})\s*$/);
|
||||
if (match) ids.add(match[1]);
|
||||
}
|
||||
for (const inline of line.matchAll(new RegExp(IDENTIFIER_INLINE_PATTERN, 'g'))) {
|
||||
ids.add(inline[1]);
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
function collectUnresolvedKnownIssueIds(content) {
|
||||
const ids = new Set();
|
||||
const sections = content.split(/^###\s+/m).slice(1);
|
||||
for (const sectionRaw of sections) {
|
||||
const section = sectionRaw.trim();
|
||||
const statusMatch = section.match(/\*\*Status:\*\*\s*(.+)$/im);
|
||||
const status = (statusMatch?.[1] || '').trim();
|
||||
if (status && STATUS_RESOLVED_RE.test(status)) {
|
||||
continue;
|
||||
}
|
||||
for (const id of extractIdentifiers(section)) {
|
||||
ids.add(id);
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
async function validateArtifactMetadata(files, { agent, cadence, promptPath, runStart }) {
|
||||
const failures = [];
|
||||
const promptPathResolved = promptPath ? path.resolve(process.cwd(), promptPath) : null;
|
||||
const promptRelative = promptPathResolved ? path.relative(process.cwd(), promptPathResolved) : null;
|
||||
const promptBase = promptPathResolved ? path.basename(promptPathResolved) : null;
|
||||
|
||||
for (const file of files) {
|
||||
const content = await fs.readFile(file.filePath, 'utf8').catch(() => '');
|
||||
const fileErrors = [];
|
||||
|
||||
const agentValue = readMetadataValue(content, ['agent']);
|
||||
const cadenceValue = readMetadataValue(content, ['cadence']);
|
||||
const sessionValue = readMetadataValue(content, ['session']);
|
||||
const runStartValue = readMetadataValue(content, ['run-start', 'run_start', 'run start']);
|
||||
|
||||
if (!agentValue) fileErrors.push('missing `agent:` metadata');
|
||||
if (!cadenceValue) fileErrors.push('missing `cadence:` metadata');
|
||||
if (!sessionValue && !runStartValue) fileErrors.push('missing `session:` or `run-start:` metadata');
|
||||
|
||||
if (agent && agentValue && agentValue !== agent) {
|
||||
fileErrors.push(`agent metadata "${agentValue}" does not match active SCHEDULER_AGENT "${agent}"`);
|
||||
}
|
||||
if (cadence && cadenceValue && cadenceValue !== cadence) {
|
||||
fileErrors.push(`cadence metadata "${cadenceValue}" does not match active cadence "${cadence}"`);
|
||||
}
|
||||
if (runStart && runStartValue && runStartValue !== runStart) {
|
||||
fileErrors.push(`run-start metadata "${runStartValue}" does not match active run-start "${runStart}"`);
|
||||
}
|
||||
|
||||
if (agent && !content.includes(agent)) {
|
||||
fileErrors.push(`content does not reference active SCHEDULER_AGENT "${agent}"`);
|
||||
}
|
||||
|
||||
if (promptPathResolved) {
|
||||
const referencesPrompt = [promptPathResolved, promptRelative, promptBase].filter(Boolean)
|
||||
.some((candidate) => content.includes(candidate));
|
||||
if (!referencesPrompt) {
|
||||
fileErrors.push(`content does not reference active prompt file (${promptRelative || promptPathResolved})`);
|
||||
}
|
||||
}
|
||||
|
||||
if (fileErrors.length) {
|
||||
failures.push(`${path.relative(process.cwd(), file.filePath)}: ${fileErrors.join('; ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function checkFailureTracking({ sinceMs }) {
|
||||
const recentTestLogs = await listRecentFiles(path.resolve(process.cwd(), 'src/test_logs'), sinceMs);
|
||||
const failureIds = new Set();
|
||||
const logsWithFailureNoIds = [];
|
||||
|
||||
for (const file of recentTestLogs) {
|
||||
const content = await fs.readFile(file.filePath, 'utf8').catch(() => '');
|
||||
// Filter out passing TAP lines, comments, and YAML blocks to avoid false positives on test descriptions
|
||||
const relevantContent = content.split(/\r?\n/)
|
||||
.filter((line) => {
|
||||
const trimmed = line.trim();
|
||||
return !/^ok\s+\d+/.test(trimmed) &&
|
||||
!trimmed.startsWith('#') &&
|
||||
!trimmed.startsWith('>') &&
|
||||
!/^(failureType|code|name|stack|operator|expected|actual):/.test(trimmed);
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
if (FAILURE_KEYWORD_RE.test(relevantContent)) {
|
||||
const ids = extractIdentifiers(content); // Extract from full content to find IDs anywhere
|
||||
if (!ids.size) {
|
||||
logsWithFailureNoIds.push(path.relative(process.cwd(), file.filePath));
|
||||
}
|
||||
for (const id of ids) failureIds.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
if (!failureIds.size && !logsWithFailureNoIds.length) {
|
||||
return { ok: true, message: null };
|
||||
}
|
||||
|
||||
if (logsWithFailureNoIds.length) {
|
||||
return {
|
||||
ok: false,
|
||||
message: `Failure-related output detected in test logs without identifiers. Add Issue/Incident identifiers to: ${logsWithFailureNoIds.join(', ')}`,
|
||||
};
|
||||
}
|
||||
|
||||
const knownIssuesPath = path.resolve(process.cwd(), 'KNOWN_ISSUES.md');
|
||||
const knownIssuesContent = await fs.readFile(knownIssuesPath, 'utf8').catch(() => '');
|
||||
const unresolvedKnownIssueIds = collectUnresolvedKnownIssueIds(knownIssuesContent);
|
||||
const incidents = await listRecentFiles(path.resolve(process.cwd(), 'docs/agent-handoffs/incidents'), sinceMs);
|
||||
|
||||
const incidentIds = new Set();
|
||||
for (const incident of incidents) {
|
||||
const incidentContent = await fs.readFile(incident.filePath, 'utf8').catch(() => '');
|
||||
for (const id of extractIdentifiers(incidentContent)) incidentIds.add(id);
|
||||
}
|
||||
|
||||
const unmatched = [...failureIds].filter((id) => !unresolvedKnownIssueIds.has(id) && !incidentIds.has(id));
|
||||
|
||||
if (!unmatched.length) {
|
||||
return { ok: true, message: null };
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
message: `Failure identifiers found in src/test_logs are not cross-linked to unresolved KNOWN_ISSUES.md entries or new incident notes: ${unmatched.join(', ')}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const sinceMs = toMs(args.since);
|
||||
const activeAgent = args.agent || process.env.SCHEDULER_AGENT || null;
|
||||
const activeCadence = args.cadence || process.env.SCHEDULER_CADENCE || null;
|
||||
const activePromptPath = args.promptPath || process.env.SCHEDULER_PROMPT_PATH || null;
|
||||
const activeRunStart = args.runStart || null;
|
||||
|
||||
if (args.since && sinceMs == null) {
|
||||
console.error(`Invalid --since value: ${args.since}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
if (!activeAgent || !activePromptPath) {
|
||||
console.error('Missing scheduler context: provide --agent and --prompt-path (or set SCHEDULER_AGENT/SCHEDULER_PROMPT_PATH).');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const missing = [];
|
||||
|
||||
for (const artifact of REQUIRED_ARTIFACTS) {
|
||||
const dirPath = path.resolve(process.cwd(), artifact.dir);
|
||||
const files = await listRecentFiles(dirPath, sinceMs);
|
||||
const matchingArtifacts = getMatchingArtifacts(files, {
|
||||
expectedPrefix: artifact.expectedPrefix,
|
||||
session: args.session,
|
||||
});
|
||||
|
||||
if (!matchingArtifacts.length) {
|
||||
const hasNotNeeded = await hasExplicitNotNeeded(files);
|
||||
if (hasNotNeeded) continue;
|
||||
|
||||
const expectedName = args.session
|
||||
? `${artifact.expectedPrefix}${args.session}.md`
|
||||
: `${artifact.expectedPrefix}<timestamp>.md`;
|
||||
missing.push(`${artifact.dir}/${expectedName}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const hasValidStructure = await hasValidArtifactStructure(matchingArtifacts, artifact);
|
||||
if (!hasValidStructure) {
|
||||
missing.push(`${artifact.dir}: ${artifact.structureHint}`);
|
||||
}
|
||||
|
||||
const metadataFailures = await validateArtifactMetadata(matchingArtifacts, {
|
||||
agent: activeAgent,
|
||||
cadence: activeCadence,
|
||||
promptPath: activePromptPath,
|
||||
runStart: activeRunStart,
|
||||
});
|
||||
missing.push(...metadataFailures);
|
||||
}
|
||||
|
||||
if (args.checkFailureNotes) {
|
||||
const failureTracking = await checkFailureTracking({ sinceMs });
|
||||
if (!failureTracking.ok && failureTracking.message) {
|
||||
missing.push(failureTracking.message);
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length) {
|
||||
console.error('Missing required run artifacts:');
|
||||
for (const item of missing) {
|
||||
console.error(`- ${item}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('Run artifacts verified.');
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in new issue
Block a user