diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index f83ca50..e4a9c64 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -30,7 +30,8 @@ Track only **active, reproducible, unresolved** issues. - **Area:** Runtime - **Symptom:** Multiple scheduler runs fail with `Lock backend error` before prompt handoff/validation. - **Trigger/Conditions:** Running scheduler cycles when relay connectivity is unstable or lock backend operations time out; observed in both `daily` and `weekly` task logs. -- **Workaround:** Retry with explicit platform and inspect reliability report output (`npm run report:lock-reliability`), then validate relay health (`node scripts/agent/check-relay-health.mjs --cadence `) before rerunning scheduler. -- **Impact:** Scheduled agent execution is skipped for affected cycles until lock acquisition succeeds. -- **Last verified:** 2026-02-14 +- **Workaround:** Run relay preflight (`npm run lock:health -- --cadence `) and inspect `task-logs/relay-health/.jsonl` for trend/alert data before rerunning scheduler. If scheduler logs `All relays unhealthy preflight`, treat it as an incident signal, defer lock attempts, and escalate relay/network checks (DNS/TLS/connectivity). +- **Impact:** Scheduled agent execution may be deferred early when all relays are unhealthy, preventing noisy lock retries until relay health recovers. +- **Related notes:** `docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md` +- **Last verified:** 2026-02-15 diff --git a/README.md b/README.md index 8a753ee..2f9ce91 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ If you are developing `torch-lock` itself: - `npm run lock:check:daily` - `npm run lock:check:weekly` - `npm run lock:list` +- `npm run lock:health -- --cadence daily` (relay websocket + publish/read probe; writes history to `task-logs/relay-health/`) - `npm run dashboard:serve` - `npm test` (run unit tests) - `npm run lint` (run linter) diff --git a/docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md b/docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md new file mode 100644 index 0000000..1559af0 --- /dev/null +++ b/docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md @@ -0,0 +1,26 @@ +# Relay health preflight job before scheduler lock acquisition + +## Context +- Scheduler runs were intermittently failing during lock acquisition when relay connectivity degraded. +- Existing preflight checks only validated lock query behavior and did not preserve health history for trend analysis. + +## Observation +- A dedicated relay probe that checks websocket reachability plus publish/read behavior per relay provides a clearer readiness signal before lock acquisition. +- Persisting each probe result enables windowed success-rate alerts and detection of prolonged all-relay outages. + +## Action taken +- Added `torch-lock health` / `npm run lock:health` to run relay websocket + publish/read probes per relay. +- Integrated scheduler preflight to call `lock:health` immediately before lock acquisition. +- Updated scheduler behavior to defer early (status `_deferred`) with reason `All relays unhealthy preflight` and incident metadata when no relay is healthy. +- Stored probe history in `task-logs/relay-health/.jsonl` and evaluated alert thresholds: + - all relays down for more than N minutes, + - success rate below X% over a rolling time window. + +## Validation performed +- Ran targeted unit tests for relay health threshold evaluation. +- Ran scheduler flow parity checks to ensure preflight remains before lock acquisition in required ordering. + +## Recommendation for next agents +- Keep `lock:health` enabled for scheduler preflight in automation. +- Review `task-logs/relay-health/.jsonl` during incidents to distinguish transient single-relay flakiness from full relay outages. +- Tune threshold defaults (`--all-relays-down-minutes`, `--min-success-rate`, `--window-minutes`) per environment reliability goals. diff --git a/package.json b/package.json index 1d3583b..07d720a 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "src/dashboard.mjs", "src/roster.mjs", "src/torch-config.mjs", + "src/relay-health.mjs", "src/prompts/", "src/docs/", "scripts/check-file-size.mjs", @@ -32,6 +33,7 @@ "lock:check:weekly": "node bin/torch-lock.mjs check --cadence weekly", "lock:list": "node bin/torch-lock.mjs list", "lock:lock": "node bin/torch-lock.mjs lock", + "lock:health": "node bin/torch-lock.mjs health", "dashboard:serve": "node bin/torch-lock.mjs dashboard", "build": "node build.mjs", "lint": "eslint .", diff --git a/scripts/agent/check-relay-health.mjs b/scripts/agent/check-relay-health.mjs index b132760..3e29f28 100644 --- a/scripts/agent/check-relay-health.mjs +++ b/scripts/agent/check-relay-health.mjs @@ -1,10 +1,14 @@ #!/usr/bin/env node -import { getRelays, getNamespace, getQueryTimeoutMs } from '../../src/torch-config.mjs'; -import { queryLocks } from '../../src/lock-ops.mjs'; -import { todayDateStr } from '../../src/utils.mjs'; +import { runRelayHealthCheck } from '../../src/relay-health.mjs'; function parseArgs(argv) { - const args = { cadence: 'daily' }; + const args = { + cadence: 'daily', + timeoutMs: 6000, + allRelaysDownMinutes: 10, + minSuccessRate: 0.7, + windowMinutes: 60, + }; for (let i = 0; i < argv.length; i += 1) { const value = argv[i]; if (!value.startsWith('--') && !args.cadence) { @@ -14,93 +18,41 @@ function parseArgs(argv) { if (value === '--cadence') { args.cadence = argv[i + 1] || args.cadence; i += 1; + } else if (value === '--timeout-ms') { + args.timeoutMs = Number.parseInt(argv[i + 1], 10) || args.timeoutMs; + i += 1; + } else if (value === '--all-relays-down-minutes') { + args.allRelaysDownMinutes = Number.parseInt(argv[i + 1], 10) || args.allRelaysDownMinutes; + i += 1; + } else if (value === '--min-success-rate') { + const parsed = Number.parseFloat(argv[i + 1]); + args.minSuccessRate = Number.isFinite(parsed) ? parsed : args.minSuccessRate; + i += 1; + } else if (value === '--window-minutes') { + args.windowMinutes = Number.parseInt(argv[i + 1], 10) || args.windowMinutes; + i += 1; } } return args; } -function classifyRelayFailure(outputText) { - const text = String(outputText || '').toLowerCase(); - if (!text.trim()) return 'unknown backend error'; - - if ((text.includes('relay') || text.includes('query')) && text.includes('timeout')) { - return 'relay query timeout'; - } - - if ( - text.includes('connection refused') - || text.includes('econnrefused') - || text.includes('getaddrinfo') - || text.includes('enotfound') - || text.includes('eai_again') - || (text.includes('websocket') && text.includes('dns')) - ) { - return 'websocket connection refused/dns'; - } - - if ( - text.includes('invalid url') - || text.includes('malformed') - || text.includes('unsupported protocol') - || text.includes('must start with ws') - || text.includes('invalid relay') - ) { - return 'malformed relay url/config'; - } - - return 'unknown backend error'; -} - async function main() { - const { cadence } = parseArgs(process.argv.slice(2)); - if (cadence !== 'daily' && cadence !== 'weekly') { + const args = parseArgs(process.argv.slice(2)); + if (args.cadence !== 'daily' && args.cadence !== 'weekly') { console.error('Usage: node scripts/agent/check-relay-health.mjs --cadence '); process.exit(1); } - const relays = getRelays(); - const namespace = getNamespace(); - const timeoutMs = getQueryTimeoutMs(); - const dateStr = todayDateStr(); - - try { - const locks = await queryLocks(relays, cadence, dateStr, namespace); - console.log(JSON.stringify({ - ok: true, - cadence, - namespace, - timeoutMs, - relays, - lockCount: locks.length, - })); - process.exit(0); - } catch (error) { - const message = error instanceof Error ? error.message : String(error ?? 'unknown error'); - const failureCategory = classifyRelayFailure(message); - console.log(JSON.stringify({ - ok: false, - cadence, - namespace, - timeoutMs, - relays, - failureCategory, - error: message, - })); - process.exit(2); + const result = await runRelayHealthCheck(args); + if (!result.ok) { + result.failureCategory = 'all relays unhealthy'; } + console.log(JSON.stringify(result)); + process.exit(result.ok ? 0 : 2); } main().catch((error) => { const message = error instanceof Error ? error.message : String(error ?? 'unknown error'); - const relays = getRelays(); - const namespace = getNamespace(); - const failureCategory = classifyRelayFailure(message); - console.log(JSON.stringify({ - ok: false, - namespace, - relays, - failureCategory, - error: message, - })); + console.log(JSON.stringify({ ok: false, failureCategory: 'health check failed', error: message })); process.exit(2); }); diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index 6bfe19b..147fed3 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -417,7 +417,7 @@ function createIdempotencyKey({ cadence, selectedAgent, runDate }) { } async function runLockHealthPreflight({ cadence, platform }) { - const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], { + const preflight = await runCommand('npm', ['run', 'lock:health', '--', '--cadence', cadence], { env: { AGENT_PLATFORM: platform }, }); const payload = parseJsonFromOutput(`${preflight.stdout}\n${preflight.stderr}`) || {}; @@ -593,22 +593,32 @@ async function main() { if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) { const preflight = await runLockHealthPreflight({ cadence, platform }); if (preflight.code !== 0) { + const allRelaysUnhealthy = Boolean(preflight.payload?.summary?.allRelaysUnhealthy); + const incidentSignal = preflight.payload?.incidentSignal || null; await writeLog({ cadence, agent: 'scheduler', - status: 'failed', + status: allRelaysUnhealthy ? 'deferred' : 'failed', platform, - reason: 'Lock backend unavailable preflight', - detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`, + reason: allRelaysUnhealthy + ? 'All relays unhealthy preflight' + : 'Lock backend unavailable preflight', + detail: allRelaysUnhealthy + ? `Deferred run before lock acquisition: ${incidentSignal?.reason || 'all relays unhealthy'}.` + : `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`, metadata: { failure_class: 'backend_unavailable', preflight_failure_category: preflight.failureCategory, relay_list: preflight.relayList.join(', ') || '(none)', preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)', preflight_stdout_excerpt: preflight.stdoutExcerpt || '(empty)', + incident_signal_id: incidentSignal?.id || null, + incident_signal_severity: incidentSignal?.severity || null, + preflight_alerts: JSON.stringify(preflight.payload?.alerts || []), + relay_health_history_path: preflight.payload?.historyPath || null, }, }); - process.exit(preflight.code || 1); + process.exit(allRelaysUnhealthy ? 0 : (preflight.code || 1)); } } diff --git a/src/docs/TORCH.md b/src/docs/TORCH.md index 67813f2..c7b563a 100644 --- a/src/docs/TORCH.md +++ b/src/docs/TORCH.md @@ -13,6 +13,9 @@ AGENT_PLATFORM=codex node bin/torch-lock.mjs lock --agent docs-agent --cadence d # List all active locks node bin/torch-lock.mjs list + +# Probe relay health before scheduler lock acquisition +node bin/torch-lock.mjs health --cadence daily ``` ## Defaults @@ -77,7 +80,7 @@ Operational note: scheduler handoff commands are treated as required execution s Scheduler failure classes in task logs: -- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals. +- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals; includes relay health alert metadata and incident signal IDs when all relays are unhealthy. - `prompt_validation_error` — prompt/handoff, memory evidence, artifact verification, and validation command failures. diff --git a/src/lib.mjs b/src/lib.mjs index b671af9..f9e8ade 100644 --- a/src/lib.mjs +++ b/src/lib.mjs @@ -32,6 +32,7 @@ import { } from './services/memory/index.js'; import { ExitError } from './errors.mjs'; import { todayDateStr, nowUnix, getIsoWeekStr } from './utils.mjs'; +import { runRelayHealthCheck } from './relay-health.mjs'; import fs from 'node:fs/promises'; import path from 'node:path'; @@ -527,6 +528,10 @@ function parseArgs(argv) { offset: null, retentionMs: null, windowMs: null, + timeoutMs: null, + allRelaysDownMinutes: null, + minSuccessRate: null, + windowMinutes: null, }; let i = 0; @@ -589,6 +594,22 @@ function parseArgs(argv) { args.windowMs = parseInt(argv[++i], 10); } else if (arg.startsWith('--window-ms=')) { args.windowMs = parseInt(arg.split('=')[1], 10); + } else if (arg === '--timeout-ms') { + args.timeoutMs = parseInt(argv[++i], 10); + } else if (arg.startsWith('--timeout-ms=')) { + args.timeoutMs = parseInt(arg.split('=')[1], 10); + } else if (arg === '--all-relays-down-minutes') { + args.allRelaysDownMinutes = parseInt(argv[++i], 10); + } else if (arg.startsWith('--all-relays-down-minutes=')) { + args.allRelaysDownMinutes = parseInt(arg.split('=')[1], 10); + } else if (arg === '--min-success-rate') { + args.minSuccessRate = parseFloat(argv[++i]); + } else if (arg.startsWith('--min-success-rate=')) { + args.minSuccessRate = parseFloat(arg.split('=')[1]); + } else if (arg === '--window-minutes') { + args.windowMinutes = parseInt(argv[++i], 10); + } else if (arg.startsWith('--window-minutes=')) { + args.windowMinutes = parseInt(arg.split('=')[1], 10); } } @@ -603,6 +624,7 @@ Commands: lock --agent --cadence Claim a lock complete --agent --cadence Mark task as completed (permanent) list [--cadence ] Print active lock table + health --cadence Probe relay websocket + publish/read health dashboard [--port ] Serve the dashboard (default: ${DEFAULT_DASHBOARD_PORT}) init [--force] Initialize torch/ directory in current project update [--force] Update torch/ configuration (backups, merges) @@ -697,6 +719,26 @@ export async function main(argv) { break; } + case 'health': { + if (!args.cadence || !VALID_CADENCES.has(args.cadence)) { + console.error(`ERROR: --cadence <${[...VALID_CADENCES].join('|')}> is required for health`); + throw new ExitError(1, 'Missing cadence'); + } + const result = await runRelayHealthCheck({ + cadence: args.cadence, + ...(Number.isFinite(args.timeoutMs) ? { timeoutMs: args.timeoutMs } : {}), + ...(Number.isFinite(args.allRelaysDownMinutes) ? { allRelaysDownMinutes: args.allRelaysDownMinutes } : {}), + ...(Number.isFinite(args.minSuccessRate) ? { minSuccessRate: args.minSuccessRate } : {}), + ...(Number.isFinite(args.windowMinutes) ? { windowMinutes: args.windowMinutes } : {}), + }); + if (!result.ok) { + result.failureCategory = 'all relays unhealthy'; + } + console.log(JSON.stringify(result, null, 2)); + if (!result.ok) throw new ExitError(2, 'Relay health check failed'); + break; + } + case 'dashboard': { await cmdDashboard(args.port); break; diff --git a/src/prompts/scheduler-flow.md b/src/prompts/scheduler-flow.md index bb19c3e..e64b503 100644 --- a/src/prompts/scheduler-flow.md +++ b/src/prompts/scheduler-flow.md @@ -81,16 +81,17 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co 5. When lock health preflight is enabled (`scheduler.lockHealthPreflight: true` or env `SCHEDULER_LOCK_HEALTH_PREFLIGHT=1`), verify relay/query health before selecting an agent or calling `lock:lock`: ```bash - node scripts/agent/check-relay-health.mjs --cadence + npm run lock:health -- --cadence ``` - Escape hatch: set `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1` to skip this check for local/offline workflows. - - If preflight exits non-zero, write `_failed.md` with reason `Lock backend unavailable preflight` and include: + - If preflight exits non-zero because every relay is unhealthy, write `_deferred.md` with reason `All relays unhealthy preflight`, include `incident_signal_id`, and stop before lock acquisition. + - For other non-zero preflight failures, write `_failed.md` with reason `Lock backend unavailable preflight` and include: - `relay_list` - `preflight_failure_category` - `preflight_stderr_excerpt` - `preflight_stdout_excerpt` - - Stop the run immediately when preflight fails. + - Always include any preflight alert payload (`preflight_alerts`) in the scheduler metadata for operational triage. 6. Find latest cadence log file, derive the previous agent, then choose the next roster agent not in exclusion set: diff --git a/src/relay-health.mjs b/src/relay-health.mjs new file mode 100644 index 0000000..e8b2fac --- /dev/null +++ b/src/relay-health.mjs @@ -0,0 +1,281 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import WebSocket from 'ws'; +import { finalizeEvent, generateSecretKey, getPublicKey } from 'nostr-tools/pure'; +import { getNamespace, getRelays } from './torch-config.mjs'; + +const PROBE_KIND = 27235; + +function withTimeout(promise, timeoutMs, message) { + let timeoutId; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => reject(new Error(message)), timeoutMs); + }); + return Promise.race([promise, timeoutPromise]).finally(() => clearTimeout(timeoutId)); +} + +function nowIso() { + return new Date().toISOString(); +} + +export function summarizeHistory(historyEntries, { nowMs = Date.now(), windowMinutes = 60 } = {}) { + const windowStartMs = nowMs - (windowMinutes * 60_000); + const recent = historyEntries.filter((entry) => Date.parse(entry.timestamp || '') >= windowStartMs); + const relayProbeCount = recent.reduce((sum, entry) => sum + (entry.summary?.totalRelays || 0), 0); + const successCount = recent.reduce((sum, entry) => sum + (entry.summary?.healthyRelays || 0), 0); + const successRate = relayProbeCount > 0 ? successCount / relayProbeCount : 1; + + let lastHealthyAtMs = null; + for (let i = historyEntries.length - 1; i >= 0; i -= 1) { + if ((historyEntries[i].summary?.healthyRelays || 0) > 0) { + lastHealthyAtMs = Date.parse(historyEntries[i].timestamp || ''); + break; + } + } + + const allDownDurationMinutes = lastHealthyAtMs === null + ? null + : Math.max(0, Math.floor((nowMs - lastHealthyAtMs) / 60_000)); + + return { + windowMinutes, + sampleCount: recent.length, + successRate, + allDownDurationMinutes, + }; +} + +export function evaluateAlertThresholds(currentResult, historyEntries, thresholds, { nowMs = Date.now() } = {}) { + const stats = summarizeHistory(historyEntries, { nowMs, windowMinutes: thresholds.windowMinutes }); + const alerts = []; + + if (currentResult.summary.allRelaysUnhealthy && stats.allDownDurationMinutes !== null && stats.allDownDurationMinutes >= thresholds.allRelaysDownMinutes) { + alerts.push({ + type: 'all_relays_down_duration', + thresholdMinutes: thresholds.allRelaysDownMinutes, + actualMinutes: stats.allDownDurationMinutes, + severity: 'critical', + }); + } + + if (stats.sampleCount > 0 && stats.successRate < thresholds.minSuccessRate) { + alerts.push({ + type: 'relay_success_rate_below_threshold', + threshold: thresholds.minSuccessRate, + actual: Number(stats.successRate.toFixed(4)), + windowMinutes: thresholds.windowMinutes, + severity: 'warning', + }); + } + + return { alerts, stats }; +} + +async function probeWebSocketReachability(relayUrl, timeoutMs) { + const startedAt = Date.now(); + return withTimeout(new Promise((resolve, reject) => { + const ws = new WebSocket(relayUrl); + let settled = false; + + const finish = (fn) => (value) => { + if (settled) return; + settled = true; + try { + ws.close(); + } catch { + // ignore + } + fn(value); + }; + + ws.once('open', finish(() => resolve({ ok: true, latencyMs: Date.now() - startedAt }))); + ws.once('error', finish((err) => reject(err))); + }), timeoutMs, `WebSocket reachability timeout after ${timeoutMs}ms`); +} + +async function probePublishRead(relayUrl, namespace, timeoutMs) { + const ws = new WebSocket(relayUrl); + const sk = generateSecretKey(); + const pk = getPublicKey(sk); + const createdAt = Math.floor(Date.now() / 1000); + const nonce = `${Date.now()}-${Math.random().toString(16).slice(2, 10)}`; + const probeTag = `${namespace}-health-probe-${nonce}`; + const event = finalizeEvent({ + kind: PROBE_KIND, + created_at: createdAt, + tags: [ + ['t', `${namespace}-relay-health`], + ['d', probeTag], + ], + content: JSON.stringify({ type: 'relay_health_probe', nonce, namespace, pubkey: pk }), + }, sk); + + const subId = `probe-${nonce}`; + + return withTimeout(new Promise((resolve, reject) => { + let okAck = false; + let readSeen = false; + let settled = false; + + const settle = (fn) => (value) => { + if (settled) return; + settled = true; + try { + ws.close(); + } catch { + // ignore + } + fn(value); + }; + + ws.once('open', () => { + ws.send(JSON.stringify(['EVENT', event])); + ws.send(JSON.stringify(['REQ', subId, { ids: [event.id], limit: 1 }])); + }); + + ws.on('message', (raw) => { + let message; + try { + message = JSON.parse(raw.toString()); + } catch { + return; + } + if (!Array.isArray(message) || message.length < 2) return; + const type = message[0]; + + if (type === 'OK' && message[1] === event.id) { + okAck = Boolean(message[2]); + if (!okAck) { + settle(reject)(new Error(`Relay rejected health probe event: ${String(message[3] || 'unknown reason')}`)); + return; + } + } + + if (type === 'EVENT' && message[1] === subId && message[2]?.id === event.id) { + readSeen = true; + } + + if (okAck && readSeen) { + ws.send(JSON.stringify(['CLOSE', subId])); + settle(resolve)({ ok: true, eventId: event.id }); + } + }); + + ws.once('error', settle(reject)); + }), timeoutMs, `Publish/read probe timeout after ${timeoutMs}ms`); +} + +async function readHistory(historyPath) { + try { + const raw = await fs.readFile(historyPath, 'utf8'); + return raw + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + try { + return JSON.parse(line); + } catch { + return null; + } + }) + .filter((entry) => entry && typeof entry === 'object'); + } catch (error) { + if (error && error.code === 'ENOENT') return []; + throw error; + } +} + +async function appendHistory(historyPath, entry) { + await fs.mkdir(path.dirname(historyPath), { recursive: true }); + await fs.appendFile(historyPath, `${JSON.stringify(entry)}\n`, 'utf8'); +} + +export async function runRelayHealthCheck(options = {}) { + const { + cadence = 'daily', + relays = getRelays(), + namespace = getNamespace(), + timeoutMs = 6000, + historyPath = path.resolve(process.cwd(), 'task-logs', 'relay-health', `${cadence}.jsonl`), + allRelaysDownMinutes = 10, + minSuccessRate = 0.7, + windowMinutes = 60, + } = options; + + const relayResults = []; + for (const relay of relays) { + const relayResult = { + relay, + websocketReachable: false, + publishReadOk: false, + healthy: false, + checks: {}, + }; + + try { + const wsProbe = await probeWebSocketReachability(relay, timeoutMs); + relayResult.websocketReachable = true; + relayResult.checks.websocket = wsProbe; + } catch (error) { + relayResult.checks.websocket = { ok: false, error: error instanceof Error ? error.message : String(error) }; + } + + if (relayResult.websocketReachable) { + try { + const prProbe = await probePublishRead(relay, namespace, timeoutMs); + relayResult.publishReadOk = true; + relayResult.checks.publishRead = prProbe; + } catch (error) { + relayResult.checks.publishRead = { ok: false, error: error instanceof Error ? error.message : String(error) }; + } + } + + relayResult.healthy = relayResult.websocketReachable && relayResult.publishReadOk; + relayResults.push(relayResult); + } + + const summary = { + totalRelays: relayResults.length, + healthyRelays: relayResults.filter((relay) => relay.healthy).length, + }; + summary.unhealthyRelays = summary.totalRelays - summary.healthyRelays; + summary.allRelaysUnhealthy = summary.healthyRelays === 0; + + const result = { + ok: !summary.allRelaysUnhealthy, + cadence, + namespace, + timestamp: nowIso(), + relays, + relayResults, + summary, + incidentSignal: null, + alerts: [], + alertStats: null, + historyPath, + }; + + const historyEntries = await readHistory(historyPath); + const prospectiveHistory = [...historyEntries, result]; + const thresholdEval = evaluateAlertThresholds(result, prospectiveHistory, { + allRelaysDownMinutes, + minSuccessRate, + windowMinutes, + }); + result.alerts = thresholdEval.alerts; + result.alertStats = thresholdEval.stats; + + if (summary.allRelaysUnhealthy) { + const incidentId = `relay-health-${cadence}-${Date.now()}`; + result.incidentSignal = { + id: incidentId, + reason: 'All relays unhealthy before lock acquisition', + severity: 'critical', + suggestedAction: 'Defer scheduler run and investigate relay availability before retrying lock acquisition.', + }; + } + + await appendHistory(historyPath, result); + return result; +} diff --git a/test/relay-health-thresholds.test.mjs b/test/relay-health-thresholds.test.mjs new file mode 100644 index 0000000..fe5e9be --- /dev/null +++ b/test/relay-health-thresholds.test.mjs @@ -0,0 +1,50 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { evaluateAlertThresholds } from '../src/relay-health.mjs'; + +function entry(timestamp, healthyRelays, totalRelays = 3) { + return { + timestamp, + summary: { + healthyRelays, + totalRelays, + allRelaysUnhealthy: healthyRelays === 0, + }, + }; +} + +test('raises all-relays-down duration alert after threshold', () => { + const now = Date.parse('2026-02-15T12:00:00Z'); + const history = [ + entry('2026-02-15T11:30:00Z', 1), + entry('2026-02-15T11:45:00Z', 0), + entry('2026-02-15T11:55:00Z', 0), + entry('2026-02-15T12:00:00Z', 0), + ]; + + const current = history[history.length - 1]; + const result = evaluateAlertThresholds(current, history, { + allRelaysDownMinutes: 10, + minSuccessRate: 0.1, + windowMinutes: 60, + }, { nowMs: now }); + + assert.ok(result.alerts.some((alert) => alert.type === 'all_relays_down_duration')); +}); + +test('raises success-rate alert when rolling health falls below threshold', () => { + const history = [ + entry('2026-02-15T11:50:00Z', 3), + entry('2026-02-15T11:55:00Z', 0), + entry('2026-02-15T12:00:00Z', 0), + ]; + const current = history[history.length - 1]; + + const result = evaluateAlertThresholds(current, history, { + allRelaysDownMinutes: 999, + minSuccessRate: 0.5, + windowMinutes: 30, + }, { nowMs: Date.parse('2026-02-15T12:00:00Z') }); + + assert.ok(result.alerts.some((alert) => alert.type === 'relay_success_rate_below_threshold')); +});